def most_recent_ens_timeseries(start_stop=(dt.datetime(2015,12,16,0), dt.datetime(2016,1,19,0)), pointcode=71699, shift_steno_one=False): """ star_stop can be a tupple with 2 date tim objects. The first is the first time step in the time series, the second is the last. """ plt.close('all') ylabels = ['[$\degree $C]', '[m/s]', '[%]', '[W/m$^2$]'] suffix = ''.join(['_geo', str(pointcode), '_', ens.timestamp_str(start_stop[0]), \ '_to_', ens.timestamp_str(start_stop[1]), '.npy']) timesteps = ens.gen_hourly_timesteps(start_stop[0], start_stop[1]) Steno_data = np.load('Q:/Weatherdata/Steno_weatherstation/Steno_hourly_2015120111_to_2016011800.npz') Steno_Tvhs = Steno_data['Tout_vWind_hum_sunRad'] Steno_timesteps = Steno_data['timesteps'] for v, ylab in zip(weathervars, ylabels): plt.figure(figsize=(15,20)) plt.grid(True) plt.subplot(2,1,1) ens_data = np.load('time_series/' + v + suffix) BBSYD_measured = sq.fetch_BrabrandSydWeather(v, start_stop[0], start_stop[1]) Steno_measured = Steno_Tvhs[:,weathervars.index(v)] if shift_steno_one: Steno_measured = np.roll(Steno_measured, -1) if v =='Tout': ens_data = ens.Kelvin_to_Celcius(ens_data) elif v=='hum': ens_data = ens.frac_to_percent(ens_data) # convert to percentage plt.plot_date(timesteps, ens_data, '-') plt.plot_date(timesteps, BBSYD_measured, 'k-', lw=2, label='Measured: Brabrand Syd') plt.plot_date(Steno_timesteps, Steno_measured, 'r-', lw=2, label='Measured: Steno Museum') plt.ylabel(ylab) plt.grid(True) plt.xlim(start_stop) plt.title(v) plt.legend() plt.subplot(2,1,2) plt.plot_date(timesteps, ens.ensemble_std(ens_data), '-', label='Ensemble std') plt.plot_date(timesteps, ens.ensemble_abs_spread(ens_data), '-', label='Max ensemble spread') plt.ylabel(ylab) plt.legend() plt.grid(True) plt.tight_layout() figfilename = v + '_most_recent_ens_timeseries.pdf' plt.savefig('figures/' + figfilename)
def check_for_timeshift(): """ This function chec if there is a time shift between data from the Brabrand Syd weather station and the Steno Museum one. It appears that Steno data is one hour fast.. """ plt.close('all') start_stop = (dt.datetime(2015, 12, 16, 0), dt.datetime(2016, 1, 16, 0)) timesteps = ens.gen_hourly_timesteps(start_stop[0], start_stop[1]) Steno_data = np.load( 'Q:/Weatherdata/Steno_weatherstation/Steno_hourly_2015120111_to_2016011800.npz' ) Steno_Tvhs = Steno_data['Tout_vWind_hum_sunRad'] Steno_timesteps = Steno_data['timesteps'] start_index = np.where(Steno_timesteps == start_stop[0])[0] end_index = np.where(Steno_timesteps == start_stop[1])[0] + 1 Steno_Tvhs_short = Steno_Tvhs[start_index:end_index, :] Steno_timesteps_new = Steno_timesteps[start_index:end_index] assert (all(Steno_timesteps_new == timesteps)) for v in weathervars: plt.figure() for offset in range(-2, 3, 1): plt.subplot(5, 1, offset + 3) BBSYD_measured = sq.fetch_BrabrandSydWeather( v, start_stop[0], start_stop[1]) Steno_measured = Steno_Tvhs_short[:, weathervars.index(v)] Steno_with_offset = np.roll(Steno_measured, offset) MAPE = np.mean(np.abs((Steno_with_offset - BBSYD_measured))) plt.title('offset %i, MAE = %2.4f ' % (offset, MAPE)) plt.plot_date(timesteps, BBSYD_measured, 'k') plt.plot_date(timesteps, Steno_with_offset, 'r') plt.tight_layout() plt.suptitle(v)
def check_for_timeshift(): """ This function chec if there is a time shift between data from the Brabrand Syd weather station and the Steno Museum one. It appears that Steno data is one hour fast.. """ plt.close('all') start_stop=(dt.datetime(2015,12,16,0), dt.datetime(2016,1,16,0)) timesteps = ens.gen_hourly_timesteps(start_stop[0], start_stop[1]) Steno_data = np.load('Q:/Weatherdata/Steno_weatherstation/Steno_hourly_2015120111_to_2016011800.npz') Steno_Tvhs = Steno_data['Tout_vWind_hum_sunRad'] Steno_timesteps = Steno_data['timesteps'] start_index = np.where(Steno_timesteps==start_stop[0])[0] end_index = np.where(Steno_timesteps==start_stop[1])[0] + 1 Steno_Tvhs_short = Steno_Tvhs[start_index:end_index, :] Steno_timesteps_new = Steno_timesteps[start_index:end_index] assert(all(Steno_timesteps_new==timesteps)) for v in weathervars: plt.figure() for offset in range(-2,3,1): plt.subplot(5,1,offset+3) BBSYD_measured = sq.fetch_BrabrandSydWeather(v, start_stop[0], start_stop[1]) Steno_measured = Steno_Tvhs_short[:, weathervars.index(v)] Steno_with_offset = np.roll(Steno_measured, offset) MAPE = np.mean(np.abs((Steno_with_offset-BBSYD_measured))) plt.title('offset %i, MAE = %2.4f '%(offset,MAPE)) plt.plot_date(timesteps, BBSYD_measured, 'k') plt.plot_date(timesteps, Steno_with_offset, 'r') plt.tight_layout() plt.suptitle(v)
if res.resid.mean()>5: print "WARNING: Bias in model: " + res.resid.mean() return True, None ts_start = dt.datetime(2015, 10, 17, 1) ts_end = dt.datetime(2016,1,16,0) timesteps = gen_hourly_timesteps(ts_start, ts_end) df = pd.DataFrame() df['prod'] = sq.fetch_production(ts_start, ts_end) df['prod24h_before'] = sq.fetch_production(ts_start + dt.timedelta(days=-1), \ ts_end + dt.timedelta(days=-1)) for v in ['Tout', 'vWind', 'sunRad', 'hum']: df[v] = sq.fetch_BrabrandSydWeather(v, ts_start, ts_end) df[v + '24h_before'] = sq.fetch_BrabrandSydWeather(v, ts_start + dt.timedelta(days=-1), \ ts_end + dt.timedelta(days=-1)) df[v + '24hdiff'] = df[v] - df[v + '24h_before'] cols = ['Tout24hdiff', 'vWind24hdiff', 'prod24h_before', 'sunRad24hdiff', 'hum24hdiff'] good_fit = False while not good_fit: X = df[cols] res = mlin_regression(df['prod'], X, add_const=False) print res.summary() good_fit, problem_var = check_fit(res) try: cols.remove(problem_var) except: print "Final cols were: " + str(cols)
def most_recent_ens_timeseries(start_stop=(dt.datetime(2015, 12, 16, 0), dt.datetime(2016, 1, 19, 0)), pointcode=71699, shift_steno_one=False): """ star_stop can be a tupple with 2 date tim objects. The first is the first time step in the time series, the second is the last. """ plt.close('all') ylabels = ['[$\degree $C]', '[m/s]', '[%]', '[W/m$^2$]'] suffix = ''.join(['_geo', str(pointcode), '_', ens.timestamp_str(start_stop[0]), \ '_to_', ens.timestamp_str(start_stop[1]), '.npy']) timesteps = ens.gen_hourly_timesteps(start_stop[0], start_stop[1]) Steno_data = np.load( 'Q:/Weatherdata/Steno_weatherstation/Steno_hourly_2015120111_to_2016011800.npz' ) Steno_Tvhs = Steno_data['Tout_vWind_hum_sunRad'] Steno_timesteps = Steno_data['timesteps'] for v, ylab in zip(weathervars, ylabels): plt.figure(figsize=(15, 20)) plt.grid(True) plt.subplot(2, 1, 1) ens_data = np.load('time_series/' + v + suffix) BBSYD_measured = sq.fetch_BrabrandSydWeather(v, start_stop[0], start_stop[1]) Steno_measured = Steno_Tvhs[:, weathervars.index(v)] if shift_steno_one: Steno_measured = np.roll(Steno_measured, -1) if v == 'Tout': ens_data = ens.Kelvin_to_Celcius(ens_data) elif v == 'hum': ens_data = ens.frac_to_percent(ens_data) # convert to percentage plt.plot_date(timesteps, ens_data, '-') plt.plot_date(timesteps, BBSYD_measured, 'k-', lw=2, label='Measured: Brabrand Syd') plt.plot_date(Steno_timesteps, Steno_measured, 'r-', lw=2, label='Measured: Steno Museum') plt.ylabel(ylab) plt.grid(True) plt.xlim(start_stop) plt.title(v) plt.legend() plt.subplot(2, 1, 2) plt.plot_date(timesteps, ens.ensemble_std(ens_data), '-', label='Ensemble std') plt.plot_date(timesteps, ens.ensemble_abs_spread(ens_data), '-', label='Max ensemble spread') plt.ylabel(ylab) plt.legend() plt.grid(True) plt.tight_layout() figfilename = v + '_most_recent_ens_timeseries.pdf' plt.savefig('figures/' + figfilename)
def main(argv): plt.close('all') try: station = argv[0] no_sigma = argv[1] if not station in PI_T_sup_dict.keys(): print "Use rundhoej, holme or hoerning and a float for the uncertainty bound" return except: print "No station provided. Defaults to holme, no_sigma=2" station = 'holme' no_sigma = 2 print station, no_sigma # old tsstart dt.datetime(2014,12,17,1) ts1 = ens.gen_hourly_timesteps(dt.datetime(2015, 3, 1, 1), dt.datetime(2016, 1, 15, 0)) ts2 = ens.gen_hourly_timesteps(dt.datetime(2016, 1, 19, 1), dt.datetime(2016, 3, 1, 0)) all_ts = ts1 + ts2 df = pd.DataFrame(index=all_ts) if station == 'holme': PI_Q1 = PI_Q_dict[station] PI_Q2 = PI_Q_dict2[station] df['Q1']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_Q1, ts1[0], ts1[-1]),\ sq.fetch_hourly_vals_from_PIno(PI_Q1, ts2[0], ts2[-1])]) df['Q2']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_Q2, ts1[0], ts1[-1]),\ sq.fetch_hourly_vals_from_PIno(PI_Q2, ts2[0], ts2[-1])]) df['Q'] = df['Q1'] + df['Q2'] else: PI_Q = PI_Q_dict[station] df['Q']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_Q, ts1[0], ts1[-1]),\ sq.fetch_hourly_vals_from_PIno(PI_Q, ts2[0], ts2[-1])]) PI_T_sup = PI_T_sup_dict[station] df['T_sup']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_T_sup, ts1[0], \ ts1[-1]),sq.fetch_hourly_vals_from_PIno(PI_T_sup, ts2[0], ts2[-1])]) PI_T_ret = PI_T_ret_dict[station] df['T_ret']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_T_ret, ts1[0], \ ts1[-1]),sq.fetch_hourly_vals_from_PIno(PI_T_ret, ts2[0], ts2[-1])]) df['ts'] = all_ts df['cons'] = specific_heat_water * density_water * df['Q'] * (df['T_sup'] - df['T_ret']) Tout1 = sq.fetch_BrabrandSydWeather('Tout', ts1[0], ts1[-1]) Tout2 = sq.fetch_BrabrandSydWeather('Tout', ts2[0], ts2[-1]) Tout = np.concatenate([Tout1, Tout2]) Tout_low_pass = [ Tout[range(i - 23, i + 1)].mean() for i in range(len(Tout)) ] df['Toutsmooth'] = Tout_low_pass Tsup_vs_Tout(df, station) #%% fitting and testing consumption prediction fit_data, vali_data, test_data, all_data = load_cons_model_dfs(df) fit_y = fit_data['cons'] columns = ['cons24hbefore', 'Tout24hdiff', 'vWind24hdiff', 'sunRad24hdiff'] X = fit_data[columns] res = mlin_regression(fit_y, X, add_const=False) fiterr = res.fittedvalues - fit_y print "Errors fit period: ", rmse(fiterr), mae(fiterr), mape(fiterr, fit_y) vali_pred = linear_map(vali_data, res.params, columns) valierr = vali_pred - vali_data['cons'] print "Errors validation period: ", rmse(valierr), mae(valierr), mape( valierr, vali_data['cons']) test_pred = linear_map(test_data, res.params, columns) testerr = test_pred - test_data['cons'] print "Errors test period: ", rmse(testerr), mae(testerr), mape( testerr, test_data['cons']) plt.figure() ens_dfs = load_cons_model_ens_dfs(df) ens_preds = np.empty((len(ens_dfs[0]), len(ens_dfs))) for edf, i in zip(ens_dfs, range(len(ens_dfs))): ens_pred = linear_map(edf, res.params, columns) ens_preds[:, i] = ens_pred plt.plot_date(all_data.index, ens_pred, 'grey', lw=0.5) ens_preds = pd.DataFrame(ens_preds, index=all_data.index) plt.plot_date(all_data.index, all_data['cons'], 'k-', lw=2) plt.plot_date(all_data.index, np.concatenate([res.fittedvalues, vali_pred, test_pred]), 'r-', lw=2) plt.title(station + ' forecasts of consumption') nonfit_errors = pd.concat([valierr, testerr]) all_pred = np.concatenate([res.fittedvalues, vali_pred, test_pred]) all_pred = pd.Series(all_pred, index=all_data.index) print res.summary() #%% TminofTout_fun = get_TminofTout_func(df, station, frac_below=0.005) sim_input = df.ix[all_data.index] sim_input['T_ret1hbefore'] = np.roll(sim_input['T_ret'], 1) sim_input['cons_pred'] = all_pred sc2_errormargin = pd.Series(no_sigma * np.ones(len(sim_input)) * nonfit_errors.std(), index=sim_input.index) nonfit_ts_start = vali_data.index[0] nonfit_ts_end = test_data.index[-1] quantile_sc2 = 1. - percent_above_forecasterrormargin(\ sc2_errormargin.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end,'cons']) sc3_model_uncert = model_based_uncertainty_alaGorm(\ ens_preds.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons'], no_sigma, quantile_sc2) sc3_errormargin = pd.Series(no_sigma * ens_preds.std(axis=1) + sc3_model_uncert, index=sim_input.index) sig_m = model_based_sigma_alaChi2( ens_preds.loc[nonfit_ts_start:nonfit_ts_end], sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons']) sig_t = np.sqrt(ens_preds.std(axis=1)**2 + sig_m**2) sc35scale = total_uncertainty_scale_alaChi2(\ ens_preds.loc[nonfit_ts_start:nonfit_ts_end],\ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'],\ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons'],\ quantile_sc2) print sig_m #sc35_errormargin = pd.Series(no_sigma*np.sqrt(ens_preds.std(axis=1)**2+sig_m**2), index=sim_input.index) sc35_errormargin = pd.Series(sc35scale * sig_t, index=sim_input.index) use_sc35 = False if use_sc35: sc3_errormargin = sc35_errormargin sim_results_sc2 = simulate_operation(sim_input, sc2_errormargin, TminofTout_fun, station) sim_results_sc3 = simulate_operation(sim_input, sc3_errormargin, TminofTout_fun, station) #%% synthetic consumption, controlled variable model uncertainty model_stds = [ 0.5 * sim_input['cons'].std(), 0.1 * sim_input['cons'].std(), 0.05 * sim_input['cons'].std() ] # sim_input['cons'].std()*np.linspace(0,1,10) sc2_synth_results = [] sc3_synth_results = [] model_uncerts = [] for model_std in model_stds: synth_cons = gen_synthetic_cons(ens_preds, sim_input['cons_pred'], model_std) sim_input_synth = sim_input.copy(deep=True) sim_input_synth['cons'] = synth_cons synth_resid = sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'] - sim_input_synth.loc[ nonfit_ts_start:nonfit_ts_end, 'cons'] sc2_errormargin_synth = pd.Series( no_sigma * np.ones(len(sim_input_synth)) * synth_resid.std(), index=sim_input_synth.index) quantile_sc2_synth = 1. - percent_above_forecasterrormargin(\ sc2_errormargin_synth.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,'cons']) print "Sc2 q: ", quantile_sc2_synth sc3_model_uncert_synth = model_based_uncertainty_alaGorm(\ ens_preds.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons'], no_sigma, quantile_sc2_synth) model_uncerts.append(sc3_model_uncert_synth) sc3_errormargin_synth = pd.Series(no_sigma * ens_preds.std(axis=1) + sc3_model_uncert_synth, index=sim_input_synth.index) sim_results_sc2_synth = simulate_operation(sim_input_synth, sc2_errormargin_synth, TminofTout_fun, station) sim_results_sc3_synth = simulate_operation(sim_input_synth, sc3_errormargin_synth, TminofTout_fun, station) sc2_synth_results.append(sim_results_sc2_synth) sc3_synth_results.append(sim_results_sc3_synth) mean_Tsupdiff = [] mean_heatlossreduced = [] for sc2_res, sc3_res in zip(sc2_synth_results, sc3_synth_results): mean_Tsupdiff.append(np.mean(sc2_res['T_sup'] - sc3_res['T_sup'])) mean_heatlossreduced.append( np.mean(100 * (1 - (sc3_res['T_sup'] - T_grnd) / (sc2_res['T_sup'] - T_grnd)))) plt.figure() plt.plot(model_uncerts, mean_Tsupdiff, 'k.') plt.title('Mean temp reduction vs model uncert.') print "Perc above errormargin, sc2: ", percent_above_forecasterrormargin(\ sc2_errormargin.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end,'cons']) print "Perc above errormargin, sc3: ", percent_above_forecasterrormargin(sc3_errormargin.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end,'cons']) print "mean errormargin, sc2: ", sc2_errormargin.mean() print "mean errormargin, sc3: ", sc3_errormargin.mean() print "rms errormargin, sc2: ", rmse(sc2_errormargin) print "rms errormargin, sc3: ", rmse(sc3_errormargin) print "Synth Perc above errormargin, sc2: ", percent_above_forecasterrormargin(\ sc2_errormargin_synth.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,'cons']) print "Synth Perc above errormargin, sc3: ", percent_above_forecasterrormargin(sc3_errormargin_synth.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,'cons']) print "Synth mean errormargin, sc2: ", sc2_errormargin_synth.mean() print "Synth mean errormargin, sc3: ", sc3_errormargin_synth.mean() print "Synth rms errormargin, sc2: ", rmse(sc2_errormargin_synth) print "Synth rms errormargin, sc3: ", rmse(sc3_errormargin_synth) #% error margins: fig_error_margins(sc2_errormargin, sc3_errormargin, sim_input, sc3_model_uncert, station, no_sigma) fig_error_margins(sc2_errormargin_synth, sc3_errormargin_synth, sim_input_synth, sc3_model_uncert_synth, station, no_sigma) sns.jointplot(np.abs(nonfit_errors), ens_preds.loc[nonfit_ts_start:nonfit_ts_end].std(axis=1)) sns.jointplot(np.abs(synth_resid), ens_preds.loc[nonfit_ts_start:nonfit_ts_end].std(axis=1)) #% T Q scatter plots fig, axes = plt.subplots(3, 1, figsize=(10, 16), sharex=True, sharey=True) axes[0].scatter(sim_input['T_sup'], sim_input['Q'], c=sim_input['cons']) axes[0].set_title(station + ': ' + 'Scenario 1') axes[1].scatter(sim_results_sc2['T_sup'], sim_results_sc2['Q'], c=sim_results_sc2['cons']) axes[1].set_title(station + ': Scenario 2: ' + str(no_sigma) + r'$\sigma$') axes[2].scatter(sim_results_sc3['T_sup'], sim_results_sc3['Q'], c=sim_results_sc3['cons']) axes[2].set_title(station + ': Scenario 3: ' + str(no_sigma) + r'$\sigma$') axes[1].set_ylabel(u'Water flow rate [m%s/h]' % uni_tothethird, size=8) axes[2].set_xlabel(u'Supply temperature [%sC]' % uni_degree, size=8) fig.tight_layout() fig.savefig(figpath + 'TQscatter_%2.2f' % (no_sigma) + 'sigma_' + station + '.pdf') # T_sup time series fig fig, axes = plt.subplots(3, 1, figsize=(15, 15), sharex=True) axes[0].plot_date(sim_input.index, sim_input['T_sup'], 'k-', label='Scenario 1') axes[0].plot_date(sim_input.index, sim_results_sc2['T_sup'], 'r-', lw=3, label='Scenario 2') axes[0].plot_date(sim_input.index, sim_results_sc2['T_sup'], 'g-', label='Scenario 3') axes[0].set_title(station + ', ' + str(no_sigma) + r'$\sigma$' + ': Supply temperature') axes[0].set_ylabel(u'Supply temperature [%sC]' % uni_degree, size=8) axes[0].legend() axes[1].plot_date(sim_input.index, sim_input['Q'], 'k-', label='Scenario 1') axes[1].plot_date(sim_input.index, sim_results_sc2['Q'], 'r-', label='Scenario 2') axes[1].plot_date(sim_input.index, sim_results_sc2['Q_ref'], 'b-', lw=1, label=r'$Q_{ref}$' + 'Scenario 2') axes[1].set_ylabel(u'Water flow rate [m%s/h]' % uni_tothethird, size=8) axes[1].legend() axes[2].plot_date(sim_input.index, sim_input['Q'], 'k-', label='Scenario 1') axes[2].plot_date(sim_input.index, sim_results_sc3['Q'], 'g-', label='Scenario 3') axes[2].plot_date(sim_input.index, sim_results_sc3['Q_ref'], 'b-', lw=1, label=r'$Q_{ref}$' + 'Scenario 3') axes[2].set_ylabel(u'Water flow rate [m%s/h]' % uni_tothethird, size=8) axes[2].legend() fig.savefig(figpath + 'TQtimeseries_%2.2f' % (no_sigma) + 'sigma_' + station + '.pdf') # Differencen in supply temperature between the scenarios fig_heat_loss(sim_input, sim_results_sc2, sim_results_sc3, station, no_sigma) fig_heat_loss(sim_input_synth, sim_results_sc2_synth, sim_results_sc3_synth, station, no_sigma, save=False) return #%% The below section only runs if we view Tmin as a function of Q (the old way) # note: SOME OF THIS USES CONSTANT TRET!! TminofQ = False if TminofQ: # outlierdetection X = df[['T_sup', 'Q']] outlier_detection = False if outlier_detection: detect_outliers(X, station) else: inlierpred = np.ones(len(df), dtype=bool) fig, ax1 = plt.subplots() ax2 = ax1.twinx() cond_df = df ax1.plot_date(np.array(cond_df['ts']), np.array(cond_df['Q']), 'b') ax2.plot_date(np.array(cond_df['ts']), np.array(cond_df['T_sup']), 'r-') plt.figure() plt.plot_date(df['ts'], df['cons'], 'g-') plt.title(station) plt.figure() plt.scatter(df['T_sup'], df['Q'], c=df['cons'], alpha=0.25) plt.colorbar() plt.title(station) outliers = df[np.logical_not(inlierpred)] plt.plot(np.array(outliers['T_sup']), np.array(outliers['Q']), 'ko') #%% #plot_Tmin_Q_quantiles(df, inlierpred) Q = np.linspace(df[inlierpred]['Q'].min(), df[inlierpred]['Q'].max(), 500) qs = [0.001, 0.005, 0.01, 0.02275, 0.05, 0.1] for q in qs: T_min_func, Q_quantiles = get_Tmin_func(df[inlierpred], T_min_q=q, N_Q_q=21) plt.plot(T_min_func(Q), Q, label=str(q), lw=2) plt.legend() for Q_qua in Q_quantiles: plt.axhline(y=Q_qua) #%% P vs Q (T=Tmin(Q)) T_min_func, Q_quantiles = get_Tmin_func(df, T_min_q=0.02275, N_Q_q=21) plt.figure() plt.plot(Q, T_min_func(Q), 'r', label='Tmin') P = specific_heat_water * density_water * Q * (T_min_func(Q) - T_ret) plt.plot(Q, P, 'b', label='Cons') plt.xlabel('Q') plt.legend() plt.figure() simP = df['cons'] res = [ op_model(cons, T_min_func, Q_max=Q_max_dict[station], T_ret=T_ret) for cons in simP ] simT, simQ = zip(*res) plt.scatter(df['T_sup'], df['Q'], c='k', alpha=0.1) plt.scatter(simT, simQ, c=simP) plt.colorbar()
def main(argv): plt.close('all') try: station = argv[0] no_sigma = argv[1] if not station in PI_T_sup_dict.keys(): print "Use rundhoej, holme or hoerning and a float for the uncertainty bound" return except: print "No station provided. Defaults to holme, no_sigma=2" station = 'holme' no_sigma=2 print station, no_sigma # old tsstart dt.datetime(2014,12,17,1) ts1 = ens.gen_hourly_timesteps(dt.datetime(2015,3,1,1), dt.datetime(2016,1,15,0)) ts2 = ens.gen_hourly_timesteps(dt.datetime(2016,1,19,1), dt.datetime(2016,3,1,0)) all_ts = ts1 + ts2 df = pd.DataFrame(index=all_ts) if station == 'holme': PI_Q1 = PI_Q_dict[station] PI_Q2 = PI_Q_dict2[station] df['Q1']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_Q1, ts1[0], ts1[-1]),\ sq.fetch_hourly_vals_from_PIno(PI_Q1, ts2[0], ts2[-1])]) df['Q2']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_Q2, ts1[0], ts1[-1]),\ sq.fetch_hourly_vals_from_PIno(PI_Q2, ts2[0], ts2[-1])]) df['Q'] = df['Q1']+df['Q2'] else: PI_Q = PI_Q_dict[station] df['Q']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_Q, ts1[0], ts1[-1]),\ sq.fetch_hourly_vals_from_PIno(PI_Q, ts2[0], ts2[-1])]) PI_T_sup = PI_T_sup_dict[station] df['T_sup']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_T_sup, ts1[0], \ ts1[-1]),sq.fetch_hourly_vals_from_PIno(PI_T_sup, ts2[0], ts2[-1])]) PI_T_ret = PI_T_ret_dict[station] df['T_ret']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_T_ret, ts1[0], \ ts1[-1]),sq.fetch_hourly_vals_from_PIno(PI_T_ret, ts2[0], ts2[-1])]) df['ts'] = all_ts df['cons'] = specific_heat_water*density_water*df['Q']*(df['T_sup']-df['T_ret']) Tout1 = sq.fetch_BrabrandSydWeather('Tout', ts1[0], ts1[-1]) Tout2 = sq.fetch_BrabrandSydWeather('Tout', ts2[0], ts2[-1]) Tout = np.concatenate([Tout1, Tout2]) Tout_low_pass = [Tout[range(i-23,i+1)].mean() for i in range(len(Tout))] df['Toutsmooth'] = Tout_low_pass Tsup_vs_Tout(df, station) #%% fitting and testing consumption prediction fit_data, vali_data, test_data, all_data = load_cons_model_dfs(df) fit_y = fit_data['cons'] columns = ['cons24hbefore', 'Tout24hdiff', 'vWind24hdiff', 'sunRad24hdiff'] X = fit_data[columns] res = mlin_regression(fit_y,X, add_const=False) fiterr = res.fittedvalues - fit_y print "Errors fit period: ", rmse(fiterr), mae(fiterr), mape(fiterr, fit_y) vali_pred = linear_map(vali_data, res.params, columns) valierr = vali_pred - vali_data['cons'] print "Errors validation period: ", rmse(valierr), mae(valierr), mape(valierr, vali_data['cons']) test_pred = linear_map(test_data, res.params, columns) testerr = test_pred - test_data['cons'] print "Errors test period: ", rmse(testerr), mae(testerr), mape(testerr, test_data['cons']) plt.figure() ens_dfs = load_cons_model_ens_dfs(df) ens_preds = np.empty((len(ens_dfs[0]), len(ens_dfs))) for edf, i in zip(ens_dfs, range(len(ens_dfs))): ens_pred = linear_map(edf, res.params, columns) ens_preds[:,i] = ens_pred plt.plot_date(all_data.index, ens_pred, 'grey', lw=0.5) ens_preds = pd.DataFrame(ens_preds, index=all_data.index) plt.plot_date(all_data.index, all_data['cons'], 'k-', lw=2) plt.plot_date(all_data.index, np.concatenate([res.fittedvalues, vali_pred, test_pred]), 'r-', lw=2) plt.title(station + ' forecasts of consumption') nonfit_errors = pd.concat([valierr, testerr]) all_pred = np.concatenate([res.fittedvalues, vali_pred, test_pred]) all_pred = pd.Series(all_pred, index=all_data.index) print res.summary() #%% TminofTout_fun = get_TminofTout_func(df, station, frac_below = 0.005) sim_input = df.ix[all_data.index] sim_input['T_ret1hbefore'] = np.roll(sim_input['T_ret'], 1) sim_input['cons_pred'] = all_pred sc2_errormargin = pd.Series(no_sigma*np.ones(len(sim_input))*nonfit_errors.std(), index=sim_input.index) nonfit_ts_start = vali_data.index[0] nonfit_ts_end = test_data.index[-1] quantile_sc2 = 1. - percent_above_forecasterrormargin(\ sc2_errormargin.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end,'cons']) sc3_model_uncert = model_based_uncertainty_alaGorm(\ ens_preds.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons'], no_sigma, quantile_sc2) sc3_errormargin = pd.Series(no_sigma*ens_preds.std(axis=1) + sc3_model_uncert, index=sim_input.index) sig_m = model_based_sigma_alaChi2(ens_preds.loc[nonfit_ts_start:nonfit_ts_end], sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons']) sig_t = np.sqrt(ens_preds.std(axis=1)**2+sig_m**2) sc35scale = total_uncertainty_scale_alaChi2(\ ens_preds.loc[nonfit_ts_start:nonfit_ts_end],\ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'],\ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons'],\ quantile_sc2) print sig_m #sc35_errormargin = pd.Series(no_sigma*np.sqrt(ens_preds.std(axis=1)**2+sig_m**2), index=sim_input.index) sc35_errormargin = pd.Series(sc35scale*sig_t, index=sim_input.index) use_sc35 = False if use_sc35: sc3_errormargin = sc35_errormargin sim_results_sc2 = simulate_operation(sim_input, sc2_errormargin, TminofTout_fun, station) sim_results_sc3 = simulate_operation(sim_input, sc3_errormargin, TminofTout_fun, station) #%% synthetic consumption, controlled variable model uncertainty model_stds = [0.5*sim_input['cons'].std(), 0.1*sim_input['cons'].std(), 0.05*sim_input['cons'].std()]# sim_input['cons'].std()*np.linspace(0,1,10) sc2_synth_results = [] sc3_synth_results = [] model_uncerts = [] for model_std in model_stds: synth_cons = gen_synthetic_cons(ens_preds, sim_input['cons_pred'], model_std) sim_input_synth = sim_input.copy(deep=True) sim_input_synth['cons'] = synth_cons synth_resid = sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'] - sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons'] sc2_errormargin_synth = pd.Series(no_sigma*np.ones(len(sim_input_synth))*synth_resid.std(), index=sim_input_synth.index) quantile_sc2_synth = 1. - percent_above_forecasterrormargin(\ sc2_errormargin_synth.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,'cons']) print "Sc2 q: ", quantile_sc2_synth sc3_model_uncert_synth = model_based_uncertainty_alaGorm(\ ens_preds.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons'], no_sigma, quantile_sc2_synth) model_uncerts.append(sc3_model_uncert_synth) sc3_errormargin_synth = pd.Series(no_sigma*ens_preds.std(axis=1) + sc3_model_uncert_synth, index=sim_input_synth.index) sim_results_sc2_synth = simulate_operation(sim_input_synth, sc2_errormargin_synth, TminofTout_fun, station) sim_results_sc3_synth = simulate_operation(sim_input_synth, sc3_errormargin_synth, TminofTout_fun, station) sc2_synth_results.append(sim_results_sc2_synth) sc3_synth_results.append(sim_results_sc3_synth) mean_Tsupdiff = [] mean_heatlossreduced = [] for sc2_res, sc3_res in zip(sc2_synth_results, sc3_synth_results): mean_Tsupdiff.append(np.mean(sc2_res['T_sup'] - sc3_res['T_sup'])) mean_heatlossreduced.append(np.mean(100*(1-(sc3_res['T_sup']-T_grnd)/(sc2_res['T_sup'] - T_grnd)))) plt.figure() plt.plot(model_uncerts, mean_Tsupdiff, 'k.') plt.title('Mean temp reduction vs model uncert.') print "Perc above errormargin, sc2: ", percent_above_forecasterrormargin(\ sc2_errormargin.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end,'cons']) print "Perc above errormargin, sc3: ", percent_above_forecasterrormargin(sc3_errormargin.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input.loc[nonfit_ts_start:nonfit_ts_end,'cons']) print "mean errormargin, sc2: ", sc2_errormargin.mean() print "mean errormargin, sc3: ", sc3_errormargin.mean() print "rms errormargin, sc2: ", rmse(sc2_errormargin) print "rms errormargin, sc3: ", rmse(sc3_errormargin) print "Synth Perc above errormargin, sc2: ", percent_above_forecasterrormargin(\ sc2_errormargin_synth.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,'cons']) print "Synth Perc above errormargin, sc3: ", percent_above_forecasterrormargin(sc3_errormargin_synth.loc[nonfit_ts_start:nonfit_ts_end], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \ sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,'cons']) print "Synth mean errormargin, sc2: ", sc2_errormargin_synth.mean() print "Synth mean errormargin, sc3: ", sc3_errormargin_synth.mean() print "Synth rms errormargin, sc2: ", rmse(sc2_errormargin_synth) print "Synth rms errormargin, sc3: ", rmse(sc3_errormargin_synth) #% error margins: fig_error_margins(sc2_errormargin, sc3_errormargin, sim_input, sc3_model_uncert, station, no_sigma) fig_error_margins(sc2_errormargin_synth, sc3_errormargin_synth, sim_input_synth, sc3_model_uncert_synth, station, no_sigma) sns.jointplot(np.abs(nonfit_errors), ens_preds.loc[nonfit_ts_start:nonfit_ts_end].std(axis=1)) sns.jointplot(np.abs(synth_resid), ens_preds.loc[nonfit_ts_start:nonfit_ts_end].std(axis=1)) #% T Q scatter plots fig, axes = plt.subplots(3,1, figsize=(10,16), sharex=True, sharey=True) axes[0].scatter(sim_input['T_sup'], sim_input['Q'], c=sim_input['cons']) axes[0].set_title(station + ': ' + 'Scenario 1') axes[1].scatter(sim_results_sc2['T_sup'], sim_results_sc2['Q'], c=sim_results_sc2['cons']) axes[1].set_title(station + ': Scenario 2: ' + str(no_sigma) + r'$\sigma$' ) axes[2].scatter(sim_results_sc3['T_sup'], sim_results_sc3['Q'], c=sim_results_sc3['cons']) axes[2].set_title(station + ': Scenario 3: ' + str(no_sigma) + r'$\sigma$') axes[1].set_ylabel(u'Water flow rate [m%s/h]'%uni_tothethird, size=8) axes[2].set_xlabel(u'Supply temperature [%sC]'%uni_degree, size=8) fig.tight_layout() fig.savefig(figpath + 'TQscatter_%2.2f'%(no_sigma) + 'sigma_' + station + '.pdf') # T_sup time series fig fig, axes = plt.subplots(3,1, figsize=(15,15), sharex=True) axes[0].plot_date(sim_input.index, sim_input['T_sup'], 'k-', label='Scenario 1') axes[0].plot_date(sim_input.index, sim_results_sc2['T_sup'], 'r-', lw=3, label='Scenario 2') axes[0].plot_date(sim_input.index, sim_results_sc2['T_sup'], 'g-', label='Scenario 3') axes[0].set_title(station + ', ' + str(no_sigma) + r'$\sigma$' + ': Supply temperature') axes[0].set_ylabel(u'Supply temperature [%sC]'%uni_degree, size=8) axes[0].legend() axes[1].plot_date(sim_input.index, sim_input['Q'], 'k-', label='Scenario 1' ) axes[1].plot_date(sim_input.index, sim_results_sc2['Q'], 'r-', label='Scenario 2') axes[1].plot_date(sim_input.index, sim_results_sc2['Q_ref'], 'b-', lw=1, label=r'$Q_{ref}$' + 'Scenario 2') axes[1].set_ylabel(u'Water flow rate [m%s/h]'%uni_tothethird, size=8) axes[1].legend() axes[2].plot_date(sim_input.index, sim_input['Q'], 'k-', label='Scenario 1' ) axes[2].plot_date(sim_input.index, sim_results_sc3['Q'], 'g-', label='Scenario 3') axes[2].plot_date(sim_input.index, sim_results_sc3['Q_ref'], 'b-', lw=1, label=r'$Q_{ref}$' + 'Scenario 3') axes[2].set_ylabel(u'Water flow rate [m%s/h]'%uni_tothethird, size=8) axes[2].legend() fig.savefig(figpath + 'TQtimeseries_%2.2f'%(no_sigma) + 'sigma_' + station + '.pdf') # Differencen in supply temperature between the scenarios fig_heat_loss(sim_input, sim_results_sc2, sim_results_sc3, station, no_sigma) fig_heat_loss(sim_input_synth, sim_results_sc2_synth, sim_results_sc3_synth, station, no_sigma, save=False) return #%% The below section only runs if we view Tmin as a function of Q (the old way) # note: SOME OF THIS USES CONSTANT TRET!! TminofQ = False if TminofQ: # outlierdetection X = df[['T_sup','Q']] outlier_detection = False if outlier_detection: detect_outliers(X, station) else: inlierpred = np.ones(len(df), dtype=bool) fig, ax1 = plt.subplots() ax2 = ax1.twinx() cond_df = df ax1.plot_date(np.array(cond_df['ts']), np.array(cond_df['Q']), 'b') ax2.plot_date(np.array(cond_df['ts']), np.array(cond_df['T_sup']), 'r-') plt.figure() plt.plot_date(df['ts'], df['cons'], 'g-') plt.title(station) plt.figure() plt.scatter(df['T_sup'], df['Q'], c=df['cons'], alpha=0.25) plt.colorbar() plt.title(station) outliers = df[np.logical_not(inlierpred)] plt.plot(np.array(outliers['T_sup']), np.array(outliers['Q']), 'ko') #%% #plot_Tmin_Q_quantiles(df, inlierpred) Q = np.linspace(df[inlierpred]['Q'].min(), df[inlierpred]['Q'].max(), 500) qs = [0.001, 0.005, 0.01, 0.02275, 0.05, 0.1] for q in qs: T_min_func, Q_quantiles = get_Tmin_func(df[inlierpred],T_min_q=q, N_Q_q=21) plt.plot(T_min_func(Q), Q, label=str(q), lw=2) plt.legend() for Q_qua in Q_quantiles: plt.axhline(y=Q_qua) #%% P vs Q (T=Tmin(Q)) T_min_func, Q_quantiles = get_Tmin_func(df, T_min_q=0.02275, N_Q_q=21) plt.figure() plt.plot(Q, T_min_func(Q), 'r', label='Tmin') P = specific_heat_water*density_water*Q*(T_min_func(Q)-T_ret) plt.plot(Q, P, 'b', label='Cons') plt.xlabel('Q') plt.legend() plt.figure() simP = df['cons'] res = [op_model(cons, T_min_func, Q_max=Q_max_dict[station], T_ret=T_ret) for cons in simP] simT, simQ = zip(*res) plt.scatter(df['T_sup'], df['Q'], c='k', alpha=0.1) plt.scatter(simT,simQ,c=simP) plt.colorbar()