def read_file():
    # this function is usefull for hydrographs from 1n file
    file_folder = '../data/SLICED_171020141500_130420150600/hydrographs/'
    file_name = 'Farge-ALL_10min.all'
    wtype = "waterlevel"
    usecols = [1, 2, 3, 4, 5, 6, 7]
    colnames = ['GW1', 'GW2', 'GW3', 'GW4', 'GW5', 'GW6', 'W1']
    wtype = "waterlevel"

    path = os.path.dirname(sys.argv[0])
    fname = os.path.abspath(os.path.join(path, file_folder, file_name) )
    print fname

    

    y2 = read_waterlevel_values(fname, usecols=[7], skiprows=2, delimiter=';')
    for col, name in zip(usecols, colnames):
        print name, col
        fig_name = os.path.abspath(os.path.join(path, file_folder, 'distribution_hydrograph_'+name+'__-3._5.0.pdf') )
        

        #y = read_waterlevel_values(fname, usecols=[col], skiprows=2, delimiter=';')
        y = process2pandas.read_hydrographs_into_pandas(fname, datetime_indexes=True, log=False, delimiter=';', usecols=[0, col], skiprows=1)
        #print y
        #print 'mean, std:', plot_pandas.calculate_mean_std(y)
        
        

        if not col == 7:
            data_river = y2
            pass
        else:
            data_river = None

        data_river = None
        

        plt_title = '{0}: Measured waterlevel'.format(name)
        with sns.axes_style("whitegrid"):
            plot_pandas.plot_statistical_analysis(y, data2=data_river, save=False, figurename=fig_name, plot_title=plt_title, ylims=[-3., 5.0],
                    ylabel1="m AMSL",                                     xlabel1="number of data points",
                    ylabel2="Normal PDF", xlabel2="m AMSL",
                    ylabel3="Normal CDF",      xlabel3="m AMSL",
                    papersize='A4',
                    axeslabel_fontsize=18., title_fontsize=20., axesvalues_fontsize=18., annotation_fontsize=18., legend_fontsize=18.)
    figure_name = 'out/figure.pdf'
    ampl_fnames = ['GW_1_amplitude.all', 'GW_2_amplitude.all', 'GW_3_amplitude.all', 'GW_4_amplitude.all', 'GW_5_amplitude.all', 'GW_6_amplitude.all', 'W_1_amplitude.all']
    
    # ---------------------------------------
    # END user inputs END
    # ---------------------------------------

    path = os.path.dirname(sys.argv[0])
    fname = os.path.abspath(os.path.join(path, file_folder, file_name) )
    fign = os.path.abspath(os.path.join(path, figure_name))

    # read extremum-data for all wells into pd.dataframe and save it to list...
    AMPLITUDES = list()
    for ampl_fn in ampl_fnames:
        a_fn = os.path.abspath(os.path.join(path, amplitudes_folder, ampl_fn))
        ht, lt, amp = process2pandas.read_amplitudes_into_pandas(a_fn)
        AMPLITUDES.append([ht, lt])

    
    # read hydrograph-data into pd.dataframe
    data = process2pandas.read_hydrographs_into_pandas(fname, datetime_indexes=True)
    
    # now plot figure...
    # ---------------------
    #AMPLITUDES = None
    #fign = None

    with sns.axes_style("whitegrid"):
        plot(data, saveName=fign, extrem=AMPLITUDES,
            axeslabel_fontsize=18., title_fontsize=20., axesvalues_fontsize=18., annotation_fontsize=18., legend_fontsize=18.)
def method_2_from_cycle_std(savemode=None):
    '''
        savemod =
                None              - Default. No output will be saved
                'excel' or 'xls'  - output will be saved into EXCEL spreedsheet
                'csv'             - output will be saved into CSV text file



        Functions caclulates tidal effcicency  for each t.cycle Ei in two ways:
                Ei = std(Hydrograph_gw_i)/std(Hydrograph_w_i)
                Ei = amplitude(Hydrograph_gw_i)/amplitude(Hydrograph_w_i)
        and them saves all values into output file
    '''
    # ---------------------------------------
    # user inputs
    # ---------------------------------------
    
    file_folder       = '../data/SLICED_171020141500_130420150600/hydrographs/'
    amplitudes_folder = '../data/SLICED_171020141500_130420150600/amplitude/'
    file_name         = 'Farge-ALL_10min.all'
    path_out          = 'out/'
    fname_out         = 'output_tidal_efficiency'
    
    ampl_fnames = ['GW_1_amplitude.all', 'GW_2_amplitude.all', 'GW_3_amplitude.all', 'GW_4_amplitude.all', 'GW_5_amplitude.all', 'GW_6_amplitude.all']
    river_ampl_fname  = 'W_1_amplitude.all'
    # ---------------------------------------
    # END user inputs END
    # ---------------------------------------


    path = os.path.dirname(sys.argv[0])
    fname = os.path.abspath(os.path.join(path, file_folder, file_name) )


    # checking how to save output
    if savemode in ['xls', 'xlsx', 'excel', 'msexcel', 'Excel', 'XLS', 'XLSX']:
            outputfname = os.path.abspath(os.path.join(path, path_out, fname_out+'.xls'))
            writer = pd.ExcelWriter(outputfname)
    elif savemode in ['CSV', 'csv']:
        writer = None
    else:
        writer = None


    # read hydrographs into pandas dataframe
    data = process2pandas.read_hydrographs_into_pandas(fname, log=False)
    
    print '-'*50
    print 'Standart deviation all hydrographs'
    for n in ['GW_1', 'GW_2', 'GW_3', 'GW_4', 'GW_5', 'GW_6', 'W_1', ]:
        print '\t', n, data[n].std()
    print '-'*50
    
    # read amplitude&high/low-tide into pandas dataframe
    river_hightide, river_lowtide, river_amp = process2pandas.read_amplitudes_into_pandas(os.path.join(path, amplitudes_folder, river_ampl_fname))
    
    # -------------------------------------------------------------------------------
    # (1) WORKING WITH River data............................
    # -------------------------------------------------------------------------------
    W_1 = pd.DataFrame()  # creating new dataframe
    
    W_1['Datetime High Tide'] = river_hightide['datetime']
    W_1['Datetime Low Tide']  = river_lowtide['datetime']
    W_1['High Tide [m AMSL]'] = river_hightide['y']
    W_1['Low Tide [m AMSL]']  = river_lowtide['y']
    W_1['Amplitude_W [m]']    = river_amp
    W_1['STD_W_i']            = 0

    W_1['H_gw1_at_W_low[m AMSL]'] = 0  # waterlevel at GW wells at the moment of River peaks
    W_1['H_gw2_at_W_low[m AMSL]'] = 0  # waterlevel at GW wells at the moment of River peaks
    W_1['H_gw3_at_W_low[m AMSL]'] = 0  # waterlevel at GW wells at the moment of River peaks
    W_1['H_gw4_at_W_low[m AMSL]'] = 0  # waterlevel at GW wells at the moment of River peaks
    W_1['H_gw5_at_W_low[m AMSL]'] = 0  # waterlevel at GW wells at the moment of River peaks
    W_1['H_gw6_at_W_low[m AMSL]'] = 0  # waterlevel at GW wells at the moment of River peaks
    
    W_1['H_gw1_at_W_high[m AMSL]'] = 0  # waterlevel at GW wells at the moment of River peaks
    W_1['H_gw2_at_W_high[m AMSL]'] = 0  # waterlevel at GW wells at the moment of River peaks
    W_1['H_gw3_at_W_high[m AMSL]'] = 0  # waterlevel at GW wells at the moment of River peaks
    W_1['H_gw4_at_W_high[m AMSL]'] = 0  # waterlevel at GW wells at the moment of River peaks
    W_1['H_gw5_at_W_high[m AMSL]'] = 0  # waterlevel at GW wells at the moment of River peaks
    W_1['H_gw6_at_W_high[m AMSL]'] = 0  # waterlevel at GW wells at the moment of River peaks

    
    # now loop over tidal-cycles, picking values at corresponding H/L-tides in wells
    nCycle = 0
    for t_ht_w, t_lt_w in zip(river_hightide["datetime"], river_lowtide["datetime"]):
        
        i_ht_w = data[data['datetime'] == t_ht_w].index[0]  # should return index of an element <t_ht_gw> in series <data>
        
        i_start_cycle_w = i_ht_w - 18           # here we go 180min before highpeak
        i_end_cycle_w   = i_start_cycle_w + 73  # here we go 730min after beggining of cycle
        
        std_w = data.ix[i_start_cycle_w:i_end_cycle_w, "W_1"].std()  # calculate standart deviation of current tidal-cycle for river
        W_1.ix[nCycle, 'STD_W_i'] = std_w  # save value

        W_1.ix[nCycle, 'H_gw1_at_W_high[m AMSL]'] = data.ix[i_ht_w, 'GW_1']
        W_1.ix[nCycle, 'H_gw2_at_W_high[m AMSL]'] = data.ix[i_ht_w, 'GW_2']
        W_1.ix[nCycle, 'H_gw3_at_W_high[m AMSL]'] = data.ix[i_ht_w, 'GW_3']
        W_1.ix[nCycle, 'H_gw4_at_W_high[m AMSL]'] = data.ix[i_ht_w, 'GW_4']
        W_1.ix[nCycle, 'H_gw5_at_W_high[m AMSL]'] = data.ix[i_ht_w, 'GW_5']
        W_1.ix[nCycle, 'H_gw6_at_W_high[m AMSL]'] = data.ix[i_ht_w, 'GW_6']

        i_lt_w = data[data['datetime'] == t_lt_w].index[0]  # should return index of an element <t_lt_gw> in series <data>
        W_1.ix[nCycle, 'H_gw1_at_W_low[m AMSL]'] = data.ix[i_lt_w, 'GW_1']
        W_1.ix[nCycle, 'H_gw2_at_W_low[m AMSL]'] = data.ix[i_lt_w, 'GW_2']
        W_1.ix[nCycle, 'H_gw3_at_W_low[m AMSL]'] = data.ix[i_lt_w, 'GW_3']
        W_1.ix[nCycle, 'H_gw4_at_W_low[m AMSL]'] = data.ix[i_lt_w, 'GW_4']
        W_1.ix[nCycle, 'H_gw5_at_W_low[m AMSL]'] = data.ix[i_lt_w, 'GW_5']
        W_1.ix[nCycle, 'H_gw6_at_W_low[m AMSL]'] = data.ix[i_lt_w, 'GW_6']
        
        nCycle += 1


    # calculate overheads (ovepressure?) for each well
    W_1['Overhead_gw1_at_W_low[m]'] = W_1['H_gw1_at_W_low[m AMSL]']-W_1['Low Tide [m AMSL]']
    W_1['Overhead_gw2_at_W_low[m]'] = W_1['H_gw2_at_W_low[m AMSL]']-W_1['Low Tide [m AMSL]']
    W_1['Overhead_gw3_at_W_low[m]'] = W_1['H_gw3_at_W_low[m AMSL]']-W_1['Low Tide [m AMSL]']
    W_1['Overhead_gw4_at_W_low[m]'] = W_1['H_gw4_at_W_low[m AMSL]']-W_1['Low Tide [m AMSL]']
    W_1['Overhead_gw5_at_W_low[m]'] = W_1['H_gw5_at_W_low[m AMSL]']-W_1['Low Tide [m AMSL]']
    W_1['Overhead_gw6_at_W_low[m]'] = W_1['H_gw6_at_W_low[m AMSL]']-W_1['Low Tide [m AMSL]']

    W_1['Overhead_gw1_at_W_high[m]'] = W_1['H_gw1_at_W_high[m AMSL]']-W_1['High Tide [m AMSL]']
    W_1['Overhead_gw2_at_W_high[m]'] = W_1['H_gw2_at_W_high[m AMSL]']-W_1['High Tide [m AMSL]']
    W_1['Overhead_gw3_at_W_high[m]'] = W_1['H_gw3_at_W_high[m AMSL]']-W_1['High Tide [m AMSL]']
    W_1['Overhead_gw4_at_W_high[m]'] = W_1['H_gw4_at_W_high[m AMSL]']-W_1['High Tide [m AMSL]']
    W_1['Overhead_gw5_at_W_high[m]'] = W_1['H_gw5_at_W_high[m AMSL]']-W_1['High Tide [m AMSL]']
    W_1['Overhead_gw6_at_W_high[m]'] = W_1['H_gw6_at_W_high[m AMSL]']-W_1['High Tide [m AMSL]']


    # SAVING......
    W_1['STD_W_i'] = W_1['STD_W_i'].map(lambda x: '%.3f' % x)
    if savemode in ['xls', 'xlsx', 'excel', 'msexcel', 'Excel', 'XLS', 'XLSX']:
        W_1.to_excel(writer, 'RIVER', na_rep='---', index=False)
    elif savemode in ['CSV', 'csv']:
            outputfname = os.path.abspath(os.path.join(path, path_out, 'output_'+'river_data'+'.csv'))
            datetime_fmt = '%d.%m.%Y %H:%M'
 
            with open(outputfname, 'w') as csv:
                header = '\n'.join(
                    [unicode(line, 'utf8') for line in 
                        [ 'This File is created by script "calculate_tidal_efficiencies.py',
                        'In order to convert it to EXCEL use parameter savemode="xlsx"',
                        'It is possible to save all files to different excel-sheets',
                        '-'*100, '',  # Ends up being the header name for the index.
                        ]
                    ]
                )
                for line in header:
                    csv.write(line)
                hightide.to_csv(csv, sep=';', na_rep='---', index=False, encoding='utf-8', date_format=datetime_fmt, float_format='%.3f')
                csv.close()
            print "File created:", outputfname
    


    # -------------------------------------------------------------------------------
    # (2) WORKING WITH GW data............................
    # -------------------------------------------------------------------------------

    # Loop over files with amplitude data for gw well...
    for ampl_fn in ampl_fnames:
        a_fn = os.path.abspath(os.path.join(path, amplitudes_folder, ampl_fn))
        hightide, lowtide, amp = process2pandas.read_amplitudes_into_pandas(a_fn)  # read amplitude&high/low-tide into pandas dataframe
        
        #add columns....
        hightide['Amplitude_GW [m]'] = amp
        hightide['Amplitude_W [m]']  = river_amp
        hightide['STD_GW_i']         = 0.
        hightide['STD_W_i']          = 0.
        hightide['E_i (amplitude ratio)'] = hightide['Amplitude_GW [m]']/hightide['Amplitude_W [m]']
        hightide['E_i (std ratio)']  = 0.
        
        # now loop trough all tidal-cycles (more precisely - hightide datetimes)
        for nCycle, t_ht_gw in enumerate(hightide["datetime"]):  # nCycle - number of current tidal-cycle
            # find indexes in hydrograph series for gw...
            i_ht_gw = data[data['datetime'] == t_ht_gw].index[0]  # should return index of an element <t_ht_gw> in series <data>
            i_start_cycle_gw = i_ht_gw - 18           # (same values as in River section above) here we go 180min before highpeak
            i_end_cycle_gw   = i_start_cycle_gw + 73  # (same values as in River section above) here we go 730min after beggining of cycle
            # find indexes in hydrograph series for w...
            t_ht_w = river_hightide.ix[nCycle, "datetime"]
            i_ht_w = data[data['datetime'] == t_ht_w].index[0]  # should return index of an element <t_ht_w> in series <data>
            i_start_cycle_w = i_ht_w - 18           # (same values as in River section above) here we go 180min before highpeak
            i_end_cycle_w   = i_start_cycle_w + 73  # (same values as in River section above) here we go 730min after beggining of cycle

            # calculate STD for current tidal-cycle
            std_w  = data.ix[i_start_cycle_w:i_end_cycle_w, "W_1"].std()
            std_gw = data.ix[i_start_cycle_gw:i_end_cycle_gw, ampl_fn[:4]].std()
            E_i = std_gw/std_w
            
            #print ampl_fn[:4], '\te_cycle:', E_i, std_w, std_gw
            hightide.ix[nCycle, 'STD_GW_i'] = std_gw  # save to dataframe
            hightide.ix[nCycle, 'STD_W_i']  = std_w   # save to dataframe
            hightide.ix[nCycle, 'E_i (std ratio)'] = E_i  # save to dataframe

        #--------------------------------------------------------------------------------
        # at this moment everything is already caclculated and stays in memory....
        #--------------------------------------------------------------------------------

        # Now create nice file....
        col_names    = list(hightide.columns.values)
        col_names[0] = "Datetime High Tide"
        col_names[1] = "High Tide [m AMSL]"
        hightide.columns = col_names
        hightide.insert(1, 'Datetime Low Tide', lowtide["datetime"])
        hightide.insert(3, 'Low Tide [m AMSL]', lowtide["y"])
        E1 = hightide['E_i (std ratio)'].mean()
        E2 = hightide['E_i (amplitude ratio)'].mean()
        
        print ampl_fn
        print 'mean_E(std), mean_E(ampl)', E1, E2

        # change precision to 3 digits
        hightide['STD_GW_i'] = hightide['STD_GW_i'].map(lambda x: '%.3f' % x)
        hightide['STD_W_i']  = hightide['STD_W_i'].map(lambda x: '%.3f' % x)
        hightide['E_i (std ratio)'] = hightide['E_i (std ratio)'].map(lambda x: '%.3f' % x)
        hightide['E_i (amplitude ratio)'] = hightide['E_i (amplitude ratio)'].map(lambda x: '%.3f' % x)



        # Write to file
        if savemode in ['xls', 'xlsx', 'excel', 'msexcel', 'Excel', 'XLS', 'XLSX']:
            hightide.to_excel(writer, ampl_fn[:4], na_rep='---', index=False)
        
        elif savemode in ['CSV', 'csv']:
            outputfname = os.path.abspath(os.path.join(path, path_out, 'output_'+ampl_fn[:-4]+'.csv'))
            datetime_fmt = '%d.%m.%Y %H:%M'
 
            with open(outputfname, 'w') as csv:
                header = '\n'.join(
                    [unicode(line, 'utf8') for line in
                        [ 'This File is created by script "calculate_tidal_efficiencies.py',
                        'In order to convert it to EXCEL use parameter savemode="xlsx"',
                        'It is possible to save all files to different excel-sheets',
                        '-'*100, '',  # Ends up being the header name for the index.
                        ]
                    ]
                )
                for line in header:
                    csv.write(line)
                hightide.to_csv(csv, sep=';', na_rep='---', index=False, encoding='utf-8', date_format=datetime_fmt, float_format='%.3f')
                csv.close()
            print "File created:", outputfname

    # close XLS writer if exist
    if savemode and writer:
        writer.save()
        print "File created:", outputfname