def daily_pivot_table(var_code, agg_function, abbreviation): df = get_table_for_variable_code(var_code) try: dfp = pivot_dv_df(df) except ValueError: df = remove_duplicates(df) dfp = pivot_dv_df(df) dfd = dfp.resample('D') aggrd = dfd.agg(agg_function) rnmed = rename_cols(aggrd, abbreviation) return rnmed
def tide_when_rain_max(rn_mx_time_df): td_df = get_table_for_variable_code('six_min_tide') try: td_df = pivot_dv_df(td_df) except: td_df = remove_duplicates(td_df) td_df = pivot_dv_df(td_df) td_df = td_df.resample('15T').mean() rn_mx_time_rnd = cln_n_rnd_times(rn_mx_time_df) l = [] for c in rn_mx_time_rnd.columns: times = rn_mx_time_rnd.loc[:, c] tides = td_df.loc[times].resample('D').max() rain_var = c.split('_')[0] rain_site = c.split('-')[-1] new_cols = ['{}-{}_td-{}'.format(rain_var, rain_site, col) for col in tides.columns] tides.columns = new_cols l.append(tides) new_df = pd.concat(l, axis=1) new_df.sort_index(inplace=True) return new_df
# coding: utf-8 import numpy as np from hr_db_scripts.main_db_script import get_table_for_variable_code, get_db_table_as_df, hr_db_filename import datetime import sqlite3 df = get_table_for_variable_code('rainfall') dfp = df.pivot(columns='SiteID', values='Value') dfp_med = dfp.median(1) dfp_max = dfp.max(1) dfp_std = dfp.std(1) dfp_nz = dfp[dfp_max > 0] dfp_nzi = dfp_nz.index dfp_nz_med = dfp_med.loc[dfp_nzi] dfp_nz_std = dfp_std.loc[dfp_nzi] dfp_nz_filt = dfp_nz.apply(lambda x: np.where( (abs(x - dfp_nz_med) > dfp_nz_std * 2) & (x > 1.5), np.nan, x)) dfp_nz_filt.reset_index(inplace=True) dfp_filt_mlt = dfp_nz_filt.melt(id_vars='Datetime') df.reset_index(inplace=True) dfp_filt_mlt.set_index(['Datetime', 'SiteID'], inplace=True) df.set_index(['Datetime', 'SiteID'], inplace=True) dfp_filt_i = dfp_filt_mlt.index df_unfilt = df.loc[dfp_filt_i, :] changed_ind = df_unfilt[ df_unfilt['Value'].isnull() != dfp_filt_mlt['value'].isnull()].index changed_df = df.loc[changed_ind, :]