def read_file(): # this function is usefull for hydrographs from 1n file file_folder = '../data/SLICED_171020141500_130420150600/hydrographs/' file_name = 'Farge-ALL_10min.all' wtype = "waterlevel" usecols = [1, 2, 3, 4, 5, 6, 7] colnames = ['GW1', 'GW2', 'GW3', 'GW4', 'GW5', 'GW6', 'W1'] wtype = "waterlevel" path = os.path.dirname(sys.argv[0]) fname = os.path.abspath(os.path.join(path, file_folder, file_name) ) print fname y2 = read_waterlevel_values(fname, usecols=[7], skiprows=2, delimiter=';') for col, name in zip(usecols, colnames): print name, col fig_name = os.path.abspath(os.path.join(path, file_folder, 'distribution_hydrograph_'+name+'__-3._5.0.pdf') ) #y = read_waterlevel_values(fname, usecols=[col], skiprows=2, delimiter=';') y = process2pandas.read_hydrographs_into_pandas(fname, datetime_indexes=True, log=False, delimiter=';', usecols=[0, col], skiprows=1) #print y #print 'mean, std:', plot_pandas.calculate_mean_std(y) if not col == 7: data_river = y2 pass else: data_river = None data_river = None plt_title = '{0}: Measured waterlevel'.format(name) with sns.axes_style("whitegrid"): plot_pandas.plot_statistical_analysis(y, data2=data_river, save=False, figurename=fig_name, plot_title=plt_title, ylims=[-3., 5.0], ylabel1="m AMSL", xlabel1="number of data points", ylabel2="Normal PDF", xlabel2="m AMSL", ylabel3="Normal CDF", xlabel3="m AMSL", papersize='A4', axeslabel_fontsize=18., title_fontsize=20., axesvalues_fontsize=18., annotation_fontsize=18., legend_fontsize=18.)
figure_name = 'out/figure.pdf' ampl_fnames = ['GW_1_amplitude.all', 'GW_2_amplitude.all', 'GW_3_amplitude.all', 'GW_4_amplitude.all', 'GW_5_amplitude.all', 'GW_6_amplitude.all', 'W_1_amplitude.all'] # --------------------------------------- # END user inputs END # --------------------------------------- path = os.path.dirname(sys.argv[0]) fname = os.path.abspath(os.path.join(path, file_folder, file_name) ) fign = os.path.abspath(os.path.join(path, figure_name)) # read extremum-data for all wells into pd.dataframe and save it to list... AMPLITUDES = list() for ampl_fn in ampl_fnames: a_fn = os.path.abspath(os.path.join(path, amplitudes_folder, ampl_fn)) ht, lt, amp = process2pandas.read_amplitudes_into_pandas(a_fn) AMPLITUDES.append([ht, lt]) # read hydrograph-data into pd.dataframe data = process2pandas.read_hydrographs_into_pandas(fname, datetime_indexes=True) # now plot figure... # --------------------- #AMPLITUDES = None #fign = None with sns.axes_style("whitegrid"): plot(data, saveName=fign, extrem=AMPLITUDES, axeslabel_fontsize=18., title_fontsize=20., axesvalues_fontsize=18., annotation_fontsize=18., legend_fontsize=18.)
def method_2_from_cycle_std(savemode=None): ''' savemod = None - Default. No output will be saved 'excel' or 'xls' - output will be saved into EXCEL spreedsheet 'csv' - output will be saved into CSV text file Functions caclulates tidal effcicency for each t.cycle Ei in two ways: Ei = std(Hydrograph_gw_i)/std(Hydrograph_w_i) Ei = amplitude(Hydrograph_gw_i)/amplitude(Hydrograph_w_i) and them saves all values into output file ''' # --------------------------------------- # user inputs # --------------------------------------- file_folder = '../data/SLICED_171020141500_130420150600/hydrographs/' amplitudes_folder = '../data/SLICED_171020141500_130420150600/amplitude/' file_name = 'Farge-ALL_10min.all' path_out = 'out/' fname_out = 'output_tidal_efficiency' ampl_fnames = ['GW_1_amplitude.all', 'GW_2_amplitude.all', 'GW_3_amplitude.all', 'GW_4_amplitude.all', 'GW_5_amplitude.all', 'GW_6_amplitude.all'] river_ampl_fname = 'W_1_amplitude.all' # --------------------------------------- # END user inputs END # --------------------------------------- path = os.path.dirname(sys.argv[0]) fname = os.path.abspath(os.path.join(path, file_folder, file_name) ) # checking how to save output if savemode in ['xls', 'xlsx', 'excel', 'msexcel', 'Excel', 'XLS', 'XLSX']: outputfname = os.path.abspath(os.path.join(path, path_out, fname_out+'.xls')) writer = pd.ExcelWriter(outputfname) elif savemode in ['CSV', 'csv']: writer = None else: writer = None # read hydrographs into pandas dataframe data = process2pandas.read_hydrographs_into_pandas(fname, log=False) print '-'*50 print 'Standart deviation all hydrographs' for n in ['GW_1', 'GW_2', 'GW_3', 'GW_4', 'GW_5', 'GW_6', 'W_1', ]: print '\t', n, data[n].std() print '-'*50 # read amplitude&high/low-tide into pandas dataframe river_hightide, river_lowtide, river_amp = process2pandas.read_amplitudes_into_pandas(os.path.join(path, amplitudes_folder, river_ampl_fname)) # ------------------------------------------------------------------------------- # (1) WORKING WITH River data............................ # ------------------------------------------------------------------------------- W_1 = pd.DataFrame() # creating new dataframe W_1['Datetime High Tide'] = river_hightide['datetime'] W_1['Datetime Low Tide'] = river_lowtide['datetime'] W_1['High Tide [m AMSL]'] = river_hightide['y'] W_1['Low Tide [m AMSL]'] = river_lowtide['y'] W_1['Amplitude_W [m]'] = river_amp W_1['STD_W_i'] = 0 W_1['H_gw1_at_W_low[m AMSL]'] = 0 # waterlevel at GW wells at the moment of River peaks W_1['H_gw2_at_W_low[m AMSL]'] = 0 # waterlevel at GW wells at the moment of River peaks W_1['H_gw3_at_W_low[m AMSL]'] = 0 # waterlevel at GW wells at the moment of River peaks W_1['H_gw4_at_W_low[m AMSL]'] = 0 # waterlevel at GW wells at the moment of River peaks W_1['H_gw5_at_W_low[m AMSL]'] = 0 # waterlevel at GW wells at the moment of River peaks W_1['H_gw6_at_W_low[m AMSL]'] = 0 # waterlevel at GW wells at the moment of River peaks W_1['H_gw1_at_W_high[m AMSL]'] = 0 # waterlevel at GW wells at the moment of River peaks W_1['H_gw2_at_W_high[m AMSL]'] = 0 # waterlevel at GW wells at the moment of River peaks W_1['H_gw3_at_W_high[m AMSL]'] = 0 # waterlevel at GW wells at the moment of River peaks W_1['H_gw4_at_W_high[m AMSL]'] = 0 # waterlevel at GW wells at the moment of River peaks W_1['H_gw5_at_W_high[m AMSL]'] = 0 # waterlevel at GW wells at the moment of River peaks W_1['H_gw6_at_W_high[m AMSL]'] = 0 # waterlevel at GW wells at the moment of River peaks # now loop over tidal-cycles, picking values at corresponding H/L-tides in wells nCycle = 0 for t_ht_w, t_lt_w in zip(river_hightide["datetime"], river_lowtide["datetime"]): i_ht_w = data[data['datetime'] == t_ht_w].index[0] # should return index of an element <t_ht_gw> in series <data> i_start_cycle_w = i_ht_w - 18 # here we go 180min before highpeak i_end_cycle_w = i_start_cycle_w + 73 # here we go 730min after beggining of cycle std_w = data.ix[i_start_cycle_w:i_end_cycle_w, "W_1"].std() # calculate standart deviation of current tidal-cycle for river W_1.ix[nCycle, 'STD_W_i'] = std_w # save value W_1.ix[nCycle, 'H_gw1_at_W_high[m AMSL]'] = data.ix[i_ht_w, 'GW_1'] W_1.ix[nCycle, 'H_gw2_at_W_high[m AMSL]'] = data.ix[i_ht_w, 'GW_2'] W_1.ix[nCycle, 'H_gw3_at_W_high[m AMSL]'] = data.ix[i_ht_w, 'GW_3'] W_1.ix[nCycle, 'H_gw4_at_W_high[m AMSL]'] = data.ix[i_ht_w, 'GW_4'] W_1.ix[nCycle, 'H_gw5_at_W_high[m AMSL]'] = data.ix[i_ht_w, 'GW_5'] W_1.ix[nCycle, 'H_gw6_at_W_high[m AMSL]'] = data.ix[i_ht_w, 'GW_6'] i_lt_w = data[data['datetime'] == t_lt_w].index[0] # should return index of an element <t_lt_gw> in series <data> W_1.ix[nCycle, 'H_gw1_at_W_low[m AMSL]'] = data.ix[i_lt_w, 'GW_1'] W_1.ix[nCycle, 'H_gw2_at_W_low[m AMSL]'] = data.ix[i_lt_w, 'GW_2'] W_1.ix[nCycle, 'H_gw3_at_W_low[m AMSL]'] = data.ix[i_lt_w, 'GW_3'] W_1.ix[nCycle, 'H_gw4_at_W_low[m AMSL]'] = data.ix[i_lt_w, 'GW_4'] W_1.ix[nCycle, 'H_gw5_at_W_low[m AMSL]'] = data.ix[i_lt_w, 'GW_5'] W_1.ix[nCycle, 'H_gw6_at_W_low[m AMSL]'] = data.ix[i_lt_w, 'GW_6'] nCycle += 1 # calculate overheads (ovepressure?) for each well W_1['Overhead_gw1_at_W_low[m]'] = W_1['H_gw1_at_W_low[m AMSL]']-W_1['Low Tide [m AMSL]'] W_1['Overhead_gw2_at_W_low[m]'] = W_1['H_gw2_at_W_low[m AMSL]']-W_1['Low Tide [m AMSL]'] W_1['Overhead_gw3_at_W_low[m]'] = W_1['H_gw3_at_W_low[m AMSL]']-W_1['Low Tide [m AMSL]'] W_1['Overhead_gw4_at_W_low[m]'] = W_1['H_gw4_at_W_low[m AMSL]']-W_1['Low Tide [m AMSL]'] W_1['Overhead_gw5_at_W_low[m]'] = W_1['H_gw5_at_W_low[m AMSL]']-W_1['Low Tide [m AMSL]'] W_1['Overhead_gw6_at_W_low[m]'] = W_1['H_gw6_at_W_low[m AMSL]']-W_1['Low Tide [m AMSL]'] W_1['Overhead_gw1_at_W_high[m]'] = W_1['H_gw1_at_W_high[m AMSL]']-W_1['High Tide [m AMSL]'] W_1['Overhead_gw2_at_W_high[m]'] = W_1['H_gw2_at_W_high[m AMSL]']-W_1['High Tide [m AMSL]'] W_1['Overhead_gw3_at_W_high[m]'] = W_1['H_gw3_at_W_high[m AMSL]']-W_1['High Tide [m AMSL]'] W_1['Overhead_gw4_at_W_high[m]'] = W_1['H_gw4_at_W_high[m AMSL]']-W_1['High Tide [m AMSL]'] W_1['Overhead_gw5_at_W_high[m]'] = W_1['H_gw5_at_W_high[m AMSL]']-W_1['High Tide [m AMSL]'] W_1['Overhead_gw6_at_W_high[m]'] = W_1['H_gw6_at_W_high[m AMSL]']-W_1['High Tide [m AMSL]'] # SAVING...... W_1['STD_W_i'] = W_1['STD_W_i'].map(lambda x: '%.3f' % x) if savemode in ['xls', 'xlsx', 'excel', 'msexcel', 'Excel', 'XLS', 'XLSX']: W_1.to_excel(writer, 'RIVER', na_rep='---', index=False) elif savemode in ['CSV', 'csv']: outputfname = os.path.abspath(os.path.join(path, path_out, 'output_'+'river_data'+'.csv')) datetime_fmt = '%d.%m.%Y %H:%M' with open(outputfname, 'w') as csv: header = '\n'.join( [unicode(line, 'utf8') for line in [ 'This File is created by script "calculate_tidal_efficiencies.py', 'In order to convert it to EXCEL use parameter savemode="xlsx"', 'It is possible to save all files to different excel-sheets', '-'*100, '', # Ends up being the header name for the index. ] ] ) for line in header: csv.write(line) hightide.to_csv(csv, sep=';', na_rep='---', index=False, encoding='utf-8', date_format=datetime_fmt, float_format='%.3f') csv.close() print "File created:", outputfname # ------------------------------------------------------------------------------- # (2) WORKING WITH GW data............................ # ------------------------------------------------------------------------------- # Loop over files with amplitude data for gw well... for ampl_fn in ampl_fnames: a_fn = os.path.abspath(os.path.join(path, amplitudes_folder, ampl_fn)) hightide, lowtide, amp = process2pandas.read_amplitudes_into_pandas(a_fn) # read amplitude&high/low-tide into pandas dataframe #add columns.... hightide['Amplitude_GW [m]'] = amp hightide['Amplitude_W [m]'] = river_amp hightide['STD_GW_i'] = 0. hightide['STD_W_i'] = 0. hightide['E_i (amplitude ratio)'] = hightide['Amplitude_GW [m]']/hightide['Amplitude_W [m]'] hightide['E_i (std ratio)'] = 0. # now loop trough all tidal-cycles (more precisely - hightide datetimes) for nCycle, t_ht_gw in enumerate(hightide["datetime"]): # nCycle - number of current tidal-cycle # find indexes in hydrograph series for gw... i_ht_gw = data[data['datetime'] == t_ht_gw].index[0] # should return index of an element <t_ht_gw> in series <data> i_start_cycle_gw = i_ht_gw - 18 # (same values as in River section above) here we go 180min before highpeak i_end_cycle_gw = i_start_cycle_gw + 73 # (same values as in River section above) here we go 730min after beggining of cycle # find indexes in hydrograph series for w... t_ht_w = river_hightide.ix[nCycle, "datetime"] i_ht_w = data[data['datetime'] == t_ht_w].index[0] # should return index of an element <t_ht_w> in series <data> i_start_cycle_w = i_ht_w - 18 # (same values as in River section above) here we go 180min before highpeak i_end_cycle_w = i_start_cycle_w + 73 # (same values as in River section above) here we go 730min after beggining of cycle # calculate STD for current tidal-cycle std_w = data.ix[i_start_cycle_w:i_end_cycle_w, "W_1"].std() std_gw = data.ix[i_start_cycle_gw:i_end_cycle_gw, ampl_fn[:4]].std() E_i = std_gw/std_w #print ampl_fn[:4], '\te_cycle:', E_i, std_w, std_gw hightide.ix[nCycle, 'STD_GW_i'] = std_gw # save to dataframe hightide.ix[nCycle, 'STD_W_i'] = std_w # save to dataframe hightide.ix[nCycle, 'E_i (std ratio)'] = E_i # save to dataframe #-------------------------------------------------------------------------------- # at this moment everything is already caclculated and stays in memory.... #-------------------------------------------------------------------------------- # Now create nice file.... col_names = list(hightide.columns.values) col_names[0] = "Datetime High Tide" col_names[1] = "High Tide [m AMSL]" hightide.columns = col_names hightide.insert(1, 'Datetime Low Tide', lowtide["datetime"]) hightide.insert(3, 'Low Tide [m AMSL]', lowtide["y"]) E1 = hightide['E_i (std ratio)'].mean() E2 = hightide['E_i (amplitude ratio)'].mean() print ampl_fn print 'mean_E(std), mean_E(ampl)', E1, E2 # change precision to 3 digits hightide['STD_GW_i'] = hightide['STD_GW_i'].map(lambda x: '%.3f' % x) hightide['STD_W_i'] = hightide['STD_W_i'].map(lambda x: '%.3f' % x) hightide['E_i (std ratio)'] = hightide['E_i (std ratio)'].map(lambda x: '%.3f' % x) hightide['E_i (amplitude ratio)'] = hightide['E_i (amplitude ratio)'].map(lambda x: '%.3f' % x) # Write to file if savemode in ['xls', 'xlsx', 'excel', 'msexcel', 'Excel', 'XLS', 'XLSX']: hightide.to_excel(writer, ampl_fn[:4], na_rep='---', index=False) elif savemode in ['CSV', 'csv']: outputfname = os.path.abspath(os.path.join(path, path_out, 'output_'+ampl_fn[:-4]+'.csv')) datetime_fmt = '%d.%m.%Y %H:%M' with open(outputfname, 'w') as csv: header = '\n'.join( [unicode(line, 'utf8') for line in [ 'This File is created by script "calculate_tidal_efficiencies.py', 'In order to convert it to EXCEL use parameter savemode="xlsx"', 'It is possible to save all files to different excel-sheets', '-'*100, '', # Ends up being the header name for the index. ] ] ) for line in header: csv.write(line) hightide.to_csv(csv, sep=';', na_rep='---', index=False, encoding='utf-8', date_format=datetime_fmt, float_format='%.3f') csv.close() print "File created:", outputfname # close XLS writer if exist if savemode and writer: writer.save() print "File created:", outputfname