df = pd.read_csv(file_name,
sep=',',
skiprows=[0, 1, 2, 3, 4, 5, 7],
encoding='latin1',
na_values=['null', 'NAN'])
# indicate the date format
df['TIMESTAMP'] = pd.to_datetime(df['TIMESTAMP'], dayfirst=False)
# set the date as the index
df = df.set_index([('TIMESTAMP')])
# delete consequive data
df['E_Avg'][((df.E_Avg - df.E_Avg.shift(1)) == 0.0) |
((df.E_Avg - df.E_Avg.shift(-1)) == 0.0)] = np.nan
# count the number of observations per day
CC_min_i = df.resample('D').count()
# apply the L1 filters
df = Master.L1_filter(site, df, extra_filters=False, sheet_name='cael')
# recount the number of observations per day
CC_min_f = df.resample('D').count()
# obtain the number of observations filtered
CC_min = CC_min_i - CC_min_f
# define where the filtered data report is going to be stored
CC_min_route = os.path.join(L1, 'stat',
date + '-' + site + '_STAT_L1.csv')
# save the filtered data report
CC_min.to_csv(CC_min_route, na_rep='NaN', line_terminator='\r\n')
# add the header to the filtered data report
Master.add_header(site, CC_min_route, area='Filtered Data')
# calculation of the hourly standard deviation
std60 = Master.df_mean(df, site, sheet_name='cael')
std60.columns = ['E_SD']
# save the L1 data (minute & hour files)
file_name = os.path.join(L0, date + '-' + site + '_L0.dat')
print(file_name)
# read the data
df = pd.read_csv(file_name, sep=',',skiprows=[0,1,2,3,4,5,7],encoding='latin1',na_values=['null','NAN'])
# indicate the date format
df['TIMESTAMP'] = pd.to_datetime(df['TIMESTAMP'], dayfirst = False)
# set the date as the index
df = df.set_index([('TIMESTAMP')])
# delete RECORD
del df['RECORD']
# correction for the column's name if needed (rain & visibility)
df, Visibility_name = Master.columns_correction(df)
# count the number of observations per day
CC_min_i = df.resample('D').count()
# apply the L1 filters
df = Master.L1_filter(site, df, extra_filters = True)
# recount the number of observations per day
CC_min_f = df.resample('D').count()
# obtain the number of observations filtered
CC_min = CC_min_i - CC_min_f
# define where the filtered data report is going to be stored
CC_min_route = os.path.join(L1, 'stat', date + '-' + site + '_STAT_L1.csv');
# save the filtered data report
CC_min.to_csv(CC_min_route, na_rep='NaN', line_terminator = '\r\n')
# add the header to the filtered data report
Master.add_header(site, CC_min_route, area = 'Filtered Data')
# calculation of the hourly averages
min60 = Master.df_mean(df, site)
# save the L1 data (minute & hour files)
Master.save_month(os.path.join(L1, 'Minuto'), site, df, version = 'minuto_L1', MDR = False)
Master.save_month(os.path.join(L1, 'Hora'), site, min60, version = 'hora_L1', MDR = False)
