def most_recent_ens_timeseries(start_stop=(dt.datetime(2015,12,16,0), dt.datetime(2016,1,19,0)), pointcode=71699, shift_steno_one=False):
""" star_stop can be a tupple with 2 date tim objects. The first
is the first time step in the time series, the second is the last.
"""
plt.close('all')
ylabels = ['[$\degree $C]', '[m/s]', '[%]', '[W/m$^2$]']
suffix = ''.join(['_geo', str(pointcode), '_', ens.timestamp_str(start_stop[0]), \
'_to_', ens.timestamp_str(start_stop[1]), '.npy'])
timesteps = ens.gen_hourly_timesteps(start_stop[0], start_stop[1])
Steno_data = np.load('Q:/Weatherdata/Steno_weatherstation/Steno_hourly_2015120111_to_2016011800.npz')
Steno_Tvhs = Steno_data['Tout_vWind_hum_sunRad']
Steno_timesteps = Steno_data['timesteps']
for v, ylab in zip(weathervars, ylabels):
plt.figure(figsize=(15,20))
plt.grid(True)
plt.subplot(2,1,1)
ens_data = np.load('time_series/' + v + suffix)
BBSYD_measured = sq.fetch_BrabrandSydWeather(v, start_stop[0], start_stop[1])
Steno_measured = Steno_Tvhs[:,weathervars.index(v)]
if shift_steno_one:
Steno_measured = np.roll(Steno_measured, -1)
if v =='Tout':
ens_data = ens.Kelvin_to_Celcius(ens_data)
elif v=='hum':
ens_data = ens.frac_to_percent(ens_data) # convert to percentage
plt.plot_date(timesteps, ens_data, '-')
plt.plot_date(timesteps, BBSYD_measured, 'k-', lw=2, label='Measured: Brabrand Syd')
plt.plot_date(Steno_timesteps, Steno_measured, 'r-', lw=2, label='Measured: Steno Museum')
plt.ylabel(ylab)
plt.grid(True)
plt.xlim(start_stop)
plt.title(v)
plt.legend()
plt.subplot(2,1,2)
plt.plot_date(timesteps, ens.ensemble_std(ens_data), '-', label='Ensemble std')
plt.plot_date(timesteps, ens.ensemble_abs_spread(ens_data), '-', label='Max ensemble spread')
plt.ylabel(ylab)
plt.legend()
plt.grid(True)
plt.tight_layout()
figfilename = v + '_most_recent_ens_timeseries.pdf'
plt.savefig('figures/' + figfilename)
def check_for_timeshift():
""" This function chec if there is a time shift between data from
the Brabrand Syd weather station and the Steno Museum one.
It appears that Steno data is one hour fast..
"""
plt.close('all')
start_stop = (dt.datetime(2015, 12, 16, 0), dt.datetime(2016, 1, 16, 0))
timesteps = ens.gen_hourly_timesteps(start_stop[0], start_stop[1])
Steno_data = np.load(
'Q:/Weatherdata/Steno_weatherstation/Steno_hourly_2015120111_to_2016011800.npz'
)
Steno_Tvhs = Steno_data['Tout_vWind_hum_sunRad']
Steno_timesteps = Steno_data['timesteps']
start_index = np.where(Steno_timesteps == start_stop[0])[0]
end_index = np.where(Steno_timesteps == start_stop[1])[0] + 1
Steno_Tvhs_short = Steno_Tvhs[start_index:end_index, :]
Steno_timesteps_new = Steno_timesteps[start_index:end_index]
assert (all(Steno_timesteps_new == timesteps))
for v in weathervars:
plt.figure()
for offset in range(-2, 3, 1):
plt.subplot(5, 1, offset + 3)
BBSYD_measured = sq.fetch_BrabrandSydWeather(
v, start_stop[0], start_stop[1])
Steno_measured = Steno_Tvhs_short[:, weathervars.index(v)]
Steno_with_offset = np.roll(Steno_measured, offset)
MAPE = np.mean(np.abs((Steno_with_offset - BBSYD_measured)))
plt.title('offset %i, MAE = %2.4f ' % (offset, MAPE))
plt.plot_date(timesteps, BBSYD_measured, 'k')
plt.plot_date(timesteps, Steno_with_offset, 'r')
plt.tight_layout()
plt.suptitle(v)
def check_for_timeshift():
""" This function chec if there is a time shift between data from
the Brabrand Syd weather station and the Steno Museum one.
It appears that Steno data is one hour fast..
"""
plt.close('all')
start_stop=(dt.datetime(2015,12,16,0), dt.datetime(2016,1,16,0))
timesteps = ens.gen_hourly_timesteps(start_stop[0], start_stop[1])
Steno_data = np.load('Q:/Weatherdata/Steno_weatherstation/Steno_hourly_2015120111_to_2016011800.npz')
Steno_Tvhs = Steno_data['Tout_vWind_hum_sunRad']
Steno_timesteps = Steno_data['timesteps']
start_index = np.where(Steno_timesteps==start_stop[0])[0]
end_index = np.where(Steno_timesteps==start_stop[1])[0] + 1
Steno_Tvhs_short = Steno_Tvhs[start_index:end_index, :]
Steno_timesteps_new = Steno_timesteps[start_index:end_index]
assert(all(Steno_timesteps_new==timesteps))
for v in weathervars:
plt.figure()
for offset in range(-2,3,1):
plt.subplot(5,1,offset+3)
BBSYD_measured = sq.fetch_BrabrandSydWeather(v, start_stop[0], start_stop[1])
Steno_measured = Steno_Tvhs_short[:, weathervars.index(v)]
Steno_with_offset = np.roll(Steno_measured, offset)
MAPE = np.mean(np.abs((Steno_with_offset-BBSYD_measured)))
plt.title('offset %i, MAE = %2.4f '%(offset,MAPE))
plt.plot_date(timesteps, BBSYD_measured, 'k')
plt.plot_date(timesteps, Steno_with_offset, 'r')
plt.tight_layout()
plt.suptitle(v)
if res.resid.mean()>5:
print "WARNING: Bias in model: " + res.resid.mean()
return True, None
ts_start = dt.datetime(2015, 10, 17, 1)
ts_end = dt.datetime(2016,1,16,0)
timesteps = gen_hourly_timesteps(ts_start, ts_end)
df = pd.DataFrame()
df['prod'] = sq.fetch_production(ts_start, ts_end)
df['prod24h_before'] = sq.fetch_production(ts_start + dt.timedelta(days=-1), \
ts_end + dt.timedelta(days=-1))
for v in ['Tout', 'vWind', 'sunRad', 'hum']:
df[v] = sq.fetch_BrabrandSydWeather(v, ts_start, ts_end)
df[v + '24h_before'] = sq.fetch_BrabrandSydWeather(v, ts_start + dt.timedelta(days=-1), \
ts_end + dt.timedelta(days=-1))
df[v + '24hdiff'] = df[v] - df[v + '24h_before']
cols = ['Tout24hdiff', 'vWind24hdiff', 'prod24h_before', 'sunRad24hdiff', 'hum24hdiff']
good_fit = False
while not good_fit:
X = df[cols]
res = mlin_regression(df['prod'], X, add_const=False)
print res.summary()
good_fit, problem_var = check_fit(res)
try:
cols.remove(problem_var)
except:
print "Final cols were: " + str(cols)
def most_recent_ens_timeseries(start_stop=(dt.datetime(2015, 12, 16, 0),
dt.datetime(2016, 1, 19, 0)),
pointcode=71699,
shift_steno_one=False):
""" star_stop can be a tupple with 2 date tim objects. The first
is the first time step in the time series, the second is the last.
"""
plt.close('all')
ylabels = ['[$\degree $C]', '[m/s]', '[%]', '[W/m$^2$]']
suffix = ''.join(['_geo', str(pointcode), '_', ens.timestamp_str(start_stop[0]), \
'_to_', ens.timestamp_str(start_stop[1]), '.npy'])
timesteps = ens.gen_hourly_timesteps(start_stop[0], start_stop[1])
Steno_data = np.load(
'Q:/Weatherdata/Steno_weatherstation/Steno_hourly_2015120111_to_2016011800.npz'
)
Steno_Tvhs = Steno_data['Tout_vWind_hum_sunRad']
Steno_timesteps = Steno_data['timesteps']
for v, ylab in zip(weathervars, ylabels):
plt.figure(figsize=(15, 20))
plt.grid(True)
plt.subplot(2, 1, 1)
ens_data = np.load('time_series/' + v + suffix)
BBSYD_measured = sq.fetch_BrabrandSydWeather(v, start_stop[0],
start_stop[1])
Steno_measured = Steno_Tvhs[:, weathervars.index(v)]
if shift_steno_one:
Steno_measured = np.roll(Steno_measured, -1)
if v == 'Tout':
ens_data = ens.Kelvin_to_Celcius(ens_data)
elif v == 'hum':
ens_data = ens.frac_to_percent(ens_data) # convert to percentage
plt.plot_date(timesteps, ens_data, '-')
plt.plot_date(timesteps,
BBSYD_measured,
'k-',
lw=2,
label='Measured: Brabrand Syd')
plt.plot_date(Steno_timesteps,
Steno_measured,
'r-',
lw=2,
label='Measured: Steno Museum')
plt.ylabel(ylab)
plt.grid(True)
plt.xlim(start_stop)
plt.title(v)
plt.legend()
plt.subplot(2, 1, 2)
plt.plot_date(timesteps,
ens.ensemble_std(ens_data),
'-',
label='Ensemble std')
plt.plot_date(timesteps,
ens.ensemble_abs_spread(ens_data),
'-',
label='Max ensemble spread')
plt.ylabel(ylab)
plt.legend()
plt.grid(True)
plt.tight_layout()
figfilename = v + '_most_recent_ens_timeseries.pdf'
plt.savefig('figures/' + figfilename)
def main(argv):
plt.close('all')
try:
station = argv[0]
no_sigma = argv[1]
if not station in PI_T_sup_dict.keys():
print "Use rundhoej, holme or hoerning and a float for the uncertainty bound"
return
except:
print "No station provided. Defaults to holme, no_sigma=2"
station = 'holme'
no_sigma = 2
print station, no_sigma
# old tsstart dt.datetime(2014,12,17,1)
ts1 = ens.gen_hourly_timesteps(dt.datetime(2015, 3, 1, 1),
dt.datetime(2016, 1, 15, 0))
ts2 = ens.gen_hourly_timesteps(dt.datetime(2016, 1, 19, 1),
dt.datetime(2016, 3, 1, 0))
all_ts = ts1 + ts2
df = pd.DataFrame(index=all_ts)
if station == 'holme':
PI_Q1 = PI_Q_dict[station]
PI_Q2 = PI_Q_dict2[station]
df['Q1']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_Q1, ts1[0], ts1[-1]),\
sq.fetch_hourly_vals_from_PIno(PI_Q1, ts2[0], ts2[-1])])
df['Q2']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_Q2, ts1[0], ts1[-1]),\
sq.fetch_hourly_vals_from_PIno(PI_Q2, ts2[0], ts2[-1])])
df['Q'] = df['Q1'] + df['Q2']
else:
PI_Q = PI_Q_dict[station]
df['Q']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_Q, ts1[0], ts1[-1]),\
sq.fetch_hourly_vals_from_PIno(PI_Q, ts2[0], ts2[-1])])
PI_T_sup = PI_T_sup_dict[station]
df['T_sup']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_T_sup, ts1[0], \
ts1[-1]),sq.fetch_hourly_vals_from_PIno(PI_T_sup, ts2[0], ts2[-1])])
PI_T_ret = PI_T_ret_dict[station]
df['T_ret']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_T_ret, ts1[0], \
ts1[-1]),sq.fetch_hourly_vals_from_PIno(PI_T_ret, ts2[0], ts2[-1])])
df['ts'] = all_ts
df['cons'] = specific_heat_water * density_water * df['Q'] * (df['T_sup'] -
df['T_ret'])
Tout1 = sq.fetch_BrabrandSydWeather('Tout', ts1[0], ts1[-1])
Tout2 = sq.fetch_BrabrandSydWeather('Tout', ts2[0], ts2[-1])
Tout = np.concatenate([Tout1, Tout2])
Tout_low_pass = [
Tout[range(i - 23, i + 1)].mean() for i in range(len(Tout))
]
df['Toutsmooth'] = Tout_low_pass
Tsup_vs_Tout(df, station)
#%% fitting and testing consumption prediction
fit_data, vali_data, test_data, all_data = load_cons_model_dfs(df)
fit_y = fit_data['cons']
columns = ['cons24hbefore', 'Tout24hdiff', 'vWind24hdiff', 'sunRad24hdiff']
X = fit_data[columns]
res = mlin_regression(fit_y, X, add_const=False)
fiterr = res.fittedvalues - fit_y
print "Errors fit period: ", rmse(fiterr), mae(fiterr), mape(fiterr, fit_y)
vali_pred = linear_map(vali_data, res.params, columns)
valierr = vali_pred - vali_data['cons']
print "Errors validation period: ", rmse(valierr), mae(valierr), mape(
valierr, vali_data['cons'])
test_pred = linear_map(test_data, res.params, columns)
testerr = test_pred - test_data['cons']
print "Errors test period: ", rmse(testerr), mae(testerr), mape(
testerr, test_data['cons'])
plt.figure()
ens_dfs = load_cons_model_ens_dfs(df)
ens_preds = np.empty((len(ens_dfs[0]), len(ens_dfs)))
for edf, i in zip(ens_dfs, range(len(ens_dfs))):
ens_pred = linear_map(edf, res.params, columns)
ens_preds[:, i] = ens_pred
plt.plot_date(all_data.index, ens_pred, 'grey', lw=0.5)
ens_preds = pd.DataFrame(ens_preds, index=all_data.index)
plt.plot_date(all_data.index, all_data['cons'], 'k-', lw=2)
plt.plot_date(all_data.index,
np.concatenate([res.fittedvalues, vali_pred, test_pred]),
'r-',
lw=2)
plt.title(station + ' forecasts of consumption')
nonfit_errors = pd.concat([valierr, testerr])
all_pred = np.concatenate([res.fittedvalues, vali_pred, test_pred])
all_pred = pd.Series(all_pred, index=all_data.index)
print res.summary()
#%%
TminofTout_fun = get_TminofTout_func(df, station, frac_below=0.005)
sim_input = df.ix[all_data.index]
sim_input['T_ret1hbefore'] = np.roll(sim_input['T_ret'], 1)
sim_input['cons_pred'] = all_pred
sc2_errormargin = pd.Series(no_sigma * np.ones(len(sim_input)) *
nonfit_errors.std(),
index=sim_input.index)
nonfit_ts_start = vali_data.index[0]
nonfit_ts_end = test_data.index[-1]
quantile_sc2 = 1. - percent_above_forecasterrormargin(\
sc2_errormargin.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end,'cons'])
sc3_model_uncert = model_based_uncertainty_alaGorm(\
ens_preds.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons'], no_sigma, quantile_sc2)
sc3_errormargin = pd.Series(no_sigma * ens_preds.std(axis=1) +
sc3_model_uncert,
index=sim_input.index)
sig_m = model_based_sigma_alaChi2(
ens_preds.loc[nonfit_ts_start:nonfit_ts_end],
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'],
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons'])
sig_t = np.sqrt(ens_preds.std(axis=1)**2 + sig_m**2)
sc35scale = total_uncertainty_scale_alaChi2(\
ens_preds.loc[nonfit_ts_start:nonfit_ts_end],\
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'],\
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons'],\
quantile_sc2)
print sig_m
#sc35_errormargin = pd.Series(no_sigma*np.sqrt(ens_preds.std(axis=1)**2+sig_m**2), index=sim_input.index)
sc35_errormargin = pd.Series(sc35scale * sig_t, index=sim_input.index)
use_sc35 = False
if use_sc35:
sc3_errormargin = sc35_errormargin
sim_results_sc2 = simulate_operation(sim_input, sc2_errormargin,
TminofTout_fun, station)
sim_results_sc3 = simulate_operation(sim_input, sc3_errormargin,
TminofTout_fun, station)
#%% synthetic consumption, controlled variable model uncertainty
model_stds = [
0.5 * sim_input['cons'].std(), 0.1 * sim_input['cons'].std(),
0.05 * sim_input['cons'].std()
] # sim_input['cons'].std()*np.linspace(0,1,10)
sc2_synth_results = []
sc3_synth_results = []
model_uncerts = []
for model_std in model_stds:
synth_cons = gen_synthetic_cons(ens_preds, sim_input['cons_pred'],
model_std)
sim_input_synth = sim_input.copy(deep=True)
sim_input_synth['cons'] = synth_cons
synth_resid = sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,
'cons_pred'] - sim_input_synth.loc[
nonfit_ts_start:nonfit_ts_end,
'cons']
sc2_errormargin_synth = pd.Series(
no_sigma * np.ones(len(sim_input_synth)) * synth_resid.std(),
index=sim_input_synth.index)
quantile_sc2_synth = 1. - percent_above_forecasterrormargin(\
sc2_errormargin_synth.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,'cons'])
print "Sc2 q: ", quantile_sc2_synth
sc3_model_uncert_synth = model_based_uncertainty_alaGorm(\
ens_preds.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons'], no_sigma, quantile_sc2_synth)
model_uncerts.append(sc3_model_uncert_synth)
sc3_errormargin_synth = pd.Series(no_sigma * ens_preds.std(axis=1) +
sc3_model_uncert_synth,
index=sim_input_synth.index)
sim_results_sc2_synth = simulate_operation(sim_input_synth,
sc2_errormargin_synth,
TminofTout_fun, station)
sim_results_sc3_synth = simulate_operation(sim_input_synth,
sc3_errormargin_synth,
TminofTout_fun, station)
sc2_synth_results.append(sim_results_sc2_synth)
sc3_synth_results.append(sim_results_sc3_synth)
mean_Tsupdiff = []
mean_heatlossreduced = []
for sc2_res, sc3_res in zip(sc2_synth_results, sc3_synth_results):
mean_Tsupdiff.append(np.mean(sc2_res['T_sup'] - sc3_res['T_sup']))
mean_heatlossreduced.append(
np.mean(100 * (1 - (sc3_res['T_sup'] - T_grnd) /
(sc2_res['T_sup'] - T_grnd))))
plt.figure()
plt.plot(model_uncerts, mean_Tsupdiff, 'k.')
plt.title('Mean temp reduction vs model uncert.')
print "Perc above errormargin, sc2: ", percent_above_forecasterrormargin(\
sc2_errormargin.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end,'cons'])
print "Perc above errormargin, sc3: ", percent_above_forecasterrormargin(sc3_errormargin.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end,'cons'])
print "mean errormargin, sc2: ", sc2_errormargin.mean()
print "mean errormargin, sc3: ", sc3_errormargin.mean()
print "rms errormargin, sc2: ", rmse(sc2_errormargin)
print "rms errormargin, sc3: ", rmse(sc3_errormargin)
print "Synth Perc above errormargin, sc2: ", percent_above_forecasterrormargin(\
sc2_errormargin_synth.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,'cons'])
print "Synth Perc above errormargin, sc3: ", percent_above_forecasterrormargin(sc3_errormargin_synth.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,'cons'])
print "Synth mean errormargin, sc2: ", sc2_errormargin_synth.mean()
print "Synth mean errormargin, sc3: ", sc3_errormargin_synth.mean()
print "Synth rms errormargin, sc2: ", rmse(sc2_errormargin_synth)
print "Synth rms errormargin, sc3: ", rmse(sc3_errormargin_synth)
#% error margins:
fig_error_margins(sc2_errormargin, sc3_errormargin, sim_input,
sc3_model_uncert, station, no_sigma)
fig_error_margins(sc2_errormargin_synth, sc3_errormargin_synth,
sim_input_synth, sc3_model_uncert_synth, station,
no_sigma)
sns.jointplot(np.abs(nonfit_errors),
ens_preds.loc[nonfit_ts_start:nonfit_ts_end].std(axis=1))
sns.jointplot(np.abs(synth_resid),
ens_preds.loc[nonfit_ts_start:nonfit_ts_end].std(axis=1))
#% T Q scatter plots
fig, axes = plt.subplots(3, 1, figsize=(10, 16), sharex=True, sharey=True)
axes[0].scatter(sim_input['T_sup'], sim_input['Q'], c=sim_input['cons'])
axes[0].set_title(station + ': ' + 'Scenario 1')
axes[1].scatter(sim_results_sc2['T_sup'],
sim_results_sc2['Q'],
c=sim_results_sc2['cons'])
axes[1].set_title(station + ': Scenario 2: ' + str(no_sigma) + r'$\sigma$')
axes[2].scatter(sim_results_sc3['T_sup'],
sim_results_sc3['Q'],
c=sim_results_sc3['cons'])
axes[2].set_title(station + ': Scenario 3: ' + str(no_sigma) + r'$\sigma$')
axes[1].set_ylabel(u'Water flow rate [m%s/h]' % uni_tothethird, size=8)
axes[2].set_xlabel(u'Supply temperature [%sC]' % uni_degree, size=8)
fig.tight_layout()
fig.savefig(figpath + 'TQscatter_%2.2f' % (no_sigma) + 'sigma_' + station +
'.pdf')
# T_sup time series fig
fig, axes = plt.subplots(3, 1, figsize=(15, 15), sharex=True)
axes[0].plot_date(sim_input.index,
sim_input['T_sup'],
'k-',
label='Scenario 1')
axes[0].plot_date(sim_input.index,
sim_results_sc2['T_sup'],
'r-',
lw=3,
label='Scenario 2')
axes[0].plot_date(sim_input.index,
sim_results_sc2['T_sup'],
'g-',
label='Scenario 3')
axes[0].set_title(station + ', ' + str(no_sigma) + r'$\sigma$' +
': Supply temperature')
axes[0].set_ylabel(u'Supply temperature [%sC]' % uni_degree, size=8)
axes[0].legend()
axes[1].plot_date(sim_input.index,
sim_input['Q'],
'k-',
label='Scenario 1')
axes[1].plot_date(sim_input.index,
sim_results_sc2['Q'],
'r-',
label='Scenario 2')
axes[1].plot_date(sim_input.index,
sim_results_sc2['Q_ref'],
'b-',
lw=1,
label=r'$Q_{ref}$' + 'Scenario 2')
axes[1].set_ylabel(u'Water flow rate [m%s/h]' % uni_tothethird, size=8)
axes[1].legend()
axes[2].plot_date(sim_input.index,
sim_input['Q'],
'k-',
label='Scenario 1')
axes[2].plot_date(sim_input.index,
sim_results_sc3['Q'],
'g-',
label='Scenario 3')
axes[2].plot_date(sim_input.index,
sim_results_sc3['Q_ref'],
'b-',
lw=1,
label=r'$Q_{ref}$' + 'Scenario 3')
axes[2].set_ylabel(u'Water flow rate [m%s/h]' % uni_tothethird, size=8)
axes[2].legend()
fig.savefig(figpath + 'TQtimeseries_%2.2f' % (no_sigma) + 'sigma_' +
station + '.pdf')
# Differencen in supply temperature between the scenarios
fig_heat_loss(sim_input, sim_results_sc2, sim_results_sc3, station,
no_sigma)
fig_heat_loss(sim_input_synth,
sim_results_sc2_synth,
sim_results_sc3_synth,
station,
no_sigma,
save=False)
return
#%% The below section only runs if we view Tmin as a function of Q (the old way)
# note: SOME OF THIS USES CONSTANT TRET!!
TminofQ = False
if TminofQ:
# outlierdetection
X = df[['T_sup', 'Q']]
outlier_detection = False
if outlier_detection:
detect_outliers(X, station)
else:
inlierpred = np.ones(len(df), dtype=bool)
fig, ax1 = plt.subplots()
ax2 = ax1.twinx()
cond_df = df
ax1.plot_date(np.array(cond_df['ts']), np.array(cond_df['Q']), 'b')
ax2.plot_date(np.array(cond_df['ts']), np.array(cond_df['T_sup']),
'r-')
plt.figure()
plt.plot_date(df['ts'], df['cons'], 'g-')
plt.title(station)
plt.figure()
plt.scatter(df['T_sup'], df['Q'], c=df['cons'], alpha=0.25)
plt.colorbar()
plt.title(station)
outliers = df[np.logical_not(inlierpred)]
plt.plot(np.array(outliers['T_sup']), np.array(outliers['Q']), 'ko')
#%%
#plot_Tmin_Q_quantiles(df, inlierpred)
Q = np.linspace(df[inlierpred]['Q'].min(), df[inlierpred]['Q'].max(),
500)
qs = [0.001, 0.005, 0.01, 0.02275, 0.05, 0.1]
for q in qs:
T_min_func, Q_quantiles = get_Tmin_func(df[inlierpred],
T_min_q=q,
N_Q_q=21)
plt.plot(T_min_func(Q), Q, label=str(q), lw=2)
plt.legend()
for Q_qua in Q_quantiles:
plt.axhline(y=Q_qua)
#%% P vs Q (T=Tmin(Q))
T_min_func, Q_quantiles = get_Tmin_func(df, T_min_q=0.02275, N_Q_q=21)
plt.figure()
plt.plot(Q, T_min_func(Q), 'r', label='Tmin')
P = specific_heat_water * density_water * Q * (T_min_func(Q) - T_ret)
plt.plot(Q, P, 'b', label='Cons')
plt.xlabel('Q')
plt.legend()
plt.figure()
simP = df['cons']
res = [
op_model(cons, T_min_func, Q_max=Q_max_dict[station], T_ret=T_ret)
for cons in simP
]
simT, simQ = zip(*res)
plt.scatter(df['T_sup'], df['Q'], c='k', alpha=0.1)
plt.scatter(simT, simQ, c=simP)
plt.colorbar()
def main(argv):
plt.close('all')
try:
station = argv[0]
no_sigma = argv[1]
if not station in PI_T_sup_dict.keys():
print "Use rundhoej, holme or hoerning and a float for the uncertainty bound"
return
except:
print "No station provided. Defaults to holme, no_sigma=2"
station = 'holme'
no_sigma=2
print station, no_sigma
# old tsstart dt.datetime(2014,12,17,1)
ts1 = ens.gen_hourly_timesteps(dt.datetime(2015,3,1,1), dt.datetime(2016,1,15,0))
ts2 = ens.gen_hourly_timesteps(dt.datetime(2016,1,19,1), dt.datetime(2016,3,1,0))
all_ts = ts1 + ts2
df = pd.DataFrame(index=all_ts)
if station == 'holme':
PI_Q1 = PI_Q_dict[station]
PI_Q2 = PI_Q_dict2[station]
df['Q1']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_Q1, ts1[0], ts1[-1]),\
sq.fetch_hourly_vals_from_PIno(PI_Q1, ts2[0], ts2[-1])])
df['Q2']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_Q2, ts1[0], ts1[-1]),\
sq.fetch_hourly_vals_from_PIno(PI_Q2, ts2[0], ts2[-1])])
df['Q'] = df['Q1']+df['Q2']
else:
PI_Q = PI_Q_dict[station]
df['Q']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_Q, ts1[0], ts1[-1]),\
sq.fetch_hourly_vals_from_PIno(PI_Q, ts2[0], ts2[-1])])
PI_T_sup = PI_T_sup_dict[station]
df['T_sup']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_T_sup, ts1[0], \
ts1[-1]),sq.fetch_hourly_vals_from_PIno(PI_T_sup, ts2[0], ts2[-1])])
PI_T_ret = PI_T_ret_dict[station]
df['T_ret']=np.concatenate([sq.fetch_hourly_vals_from_PIno(PI_T_ret, ts1[0], \
ts1[-1]),sq.fetch_hourly_vals_from_PIno(PI_T_ret, ts2[0], ts2[-1])])
df['ts'] = all_ts
df['cons'] = specific_heat_water*density_water*df['Q']*(df['T_sup']-df['T_ret'])
Tout1 = sq.fetch_BrabrandSydWeather('Tout', ts1[0], ts1[-1])
Tout2 = sq.fetch_BrabrandSydWeather('Tout', ts2[0], ts2[-1])
Tout = np.concatenate([Tout1, Tout2])
Tout_low_pass = [Tout[range(i-23,i+1)].mean() for i in range(len(Tout))]
df['Toutsmooth'] = Tout_low_pass
Tsup_vs_Tout(df, station)
#%% fitting and testing consumption prediction
fit_data, vali_data, test_data, all_data = load_cons_model_dfs(df)
fit_y = fit_data['cons']
columns = ['cons24hbefore', 'Tout24hdiff', 'vWind24hdiff', 'sunRad24hdiff']
X = fit_data[columns]
res = mlin_regression(fit_y,X, add_const=False)
fiterr = res.fittedvalues - fit_y
print "Errors fit period: ", rmse(fiterr), mae(fiterr), mape(fiterr, fit_y)
vali_pred = linear_map(vali_data, res.params, columns)
valierr = vali_pred - vali_data['cons']
print "Errors validation period: ", rmse(valierr), mae(valierr), mape(valierr, vali_data['cons'])
test_pred = linear_map(test_data, res.params, columns)
testerr = test_pred - test_data['cons']
print "Errors test period: ", rmse(testerr), mae(testerr), mape(testerr, test_data['cons'])
plt.figure()
ens_dfs = load_cons_model_ens_dfs(df)
ens_preds = np.empty((len(ens_dfs[0]), len(ens_dfs)))
for edf, i in zip(ens_dfs, range(len(ens_dfs))):
ens_pred = linear_map(edf, res.params, columns)
ens_preds[:,i] = ens_pred
plt.plot_date(all_data.index, ens_pred, 'grey', lw=0.5)
ens_preds = pd.DataFrame(ens_preds, index=all_data.index)
plt.plot_date(all_data.index, all_data['cons'], 'k-', lw=2)
plt.plot_date(all_data.index, np.concatenate([res.fittedvalues, vali_pred, test_pred]), 'r-', lw=2)
plt.title(station + ' forecasts of consumption')
nonfit_errors = pd.concat([valierr, testerr])
all_pred = np.concatenate([res.fittedvalues, vali_pred, test_pred])
all_pred = pd.Series(all_pred, index=all_data.index)
print res.summary()
#%%
TminofTout_fun = get_TminofTout_func(df, station, frac_below = 0.005)
sim_input = df.ix[all_data.index]
sim_input['T_ret1hbefore'] = np.roll(sim_input['T_ret'], 1)
sim_input['cons_pred'] = all_pred
sc2_errormargin = pd.Series(no_sigma*np.ones(len(sim_input))*nonfit_errors.std(), index=sim_input.index)
nonfit_ts_start = vali_data.index[0]
nonfit_ts_end = test_data.index[-1]
quantile_sc2 = 1. - percent_above_forecasterrormargin(\
sc2_errormargin.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end,'cons'])
sc3_model_uncert = model_based_uncertainty_alaGorm(\
ens_preds.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons'], no_sigma, quantile_sc2)
sc3_errormargin = pd.Series(no_sigma*ens_preds.std(axis=1) + sc3_model_uncert, index=sim_input.index)
sig_m = model_based_sigma_alaChi2(ens_preds.loc[nonfit_ts_start:nonfit_ts_end], sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons'])
sig_t = np.sqrt(ens_preds.std(axis=1)**2+sig_m**2)
sc35scale = total_uncertainty_scale_alaChi2(\
ens_preds.loc[nonfit_ts_start:nonfit_ts_end],\
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'],\
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons'],\
quantile_sc2)
print sig_m
#sc35_errormargin = pd.Series(no_sigma*np.sqrt(ens_preds.std(axis=1)**2+sig_m**2), index=sim_input.index)
sc35_errormargin = pd.Series(sc35scale*sig_t, index=sim_input.index)
use_sc35 = False
if use_sc35:
sc3_errormargin = sc35_errormargin
sim_results_sc2 = simulate_operation(sim_input, sc2_errormargin, TminofTout_fun, station)
sim_results_sc3 = simulate_operation(sim_input, sc3_errormargin, TminofTout_fun, station)
#%% synthetic consumption, controlled variable model uncertainty
model_stds = [0.5*sim_input['cons'].std(), 0.1*sim_input['cons'].std(), 0.05*sim_input['cons'].std()]# sim_input['cons'].std()*np.linspace(0,1,10)
sc2_synth_results = []
sc3_synth_results = []
model_uncerts = []
for model_std in model_stds:
synth_cons = gen_synthetic_cons(ens_preds, sim_input['cons_pred'], model_std)
sim_input_synth = sim_input.copy(deep=True)
sim_input_synth['cons'] = synth_cons
synth_resid = sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'] - sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons']
sc2_errormargin_synth = pd.Series(no_sigma*np.ones(len(sim_input_synth))*synth_resid.std(), index=sim_input_synth.index)
quantile_sc2_synth = 1. - percent_above_forecasterrormargin(\
sc2_errormargin_synth.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,'cons'])
print "Sc2 q: ", quantile_sc2_synth
sc3_model_uncert_synth = model_based_uncertainty_alaGorm(\
ens_preds.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons'], no_sigma, quantile_sc2_synth)
model_uncerts.append(sc3_model_uncert_synth)
sc3_errormargin_synth = pd.Series(no_sigma*ens_preds.std(axis=1) + sc3_model_uncert_synth, index=sim_input_synth.index)
sim_results_sc2_synth = simulate_operation(sim_input_synth, sc2_errormargin_synth, TminofTout_fun, station)
sim_results_sc3_synth = simulate_operation(sim_input_synth, sc3_errormargin_synth, TminofTout_fun, station)
sc2_synth_results.append(sim_results_sc2_synth)
sc3_synth_results.append(sim_results_sc3_synth)
mean_Tsupdiff = []
mean_heatlossreduced = []
for sc2_res, sc3_res in zip(sc2_synth_results, sc3_synth_results):
mean_Tsupdiff.append(np.mean(sc2_res['T_sup'] - sc3_res['T_sup']))
mean_heatlossreduced.append(np.mean(100*(1-(sc3_res['T_sup']-T_grnd)/(sc2_res['T_sup'] - T_grnd))))
plt.figure()
plt.plot(model_uncerts, mean_Tsupdiff, 'k.')
plt.title('Mean temp reduction vs model uncert.')
print "Perc above errormargin, sc2: ", percent_above_forecasterrormargin(\
sc2_errormargin.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end,'cons'])
print "Perc above errormargin, sc3: ", percent_above_forecasterrormargin(sc3_errormargin.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input.loc[nonfit_ts_start:nonfit_ts_end,'cons'])
print "mean errormargin, sc2: ", sc2_errormargin.mean()
print "mean errormargin, sc3: ", sc3_errormargin.mean()
print "rms errormargin, sc2: ", rmse(sc2_errormargin)
print "rms errormargin, sc3: ", rmse(sc3_errormargin)
print "Synth Perc above errormargin, sc2: ", percent_above_forecasterrormargin(\
sc2_errormargin_synth.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,'cons'])
print "Synth Perc above errormargin, sc3: ", percent_above_forecasterrormargin(sc3_errormargin_synth.loc[nonfit_ts_start:nonfit_ts_end], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end, 'cons_pred'], \
sim_input_synth.loc[nonfit_ts_start:nonfit_ts_end,'cons'])
print "Synth mean errormargin, sc2: ", sc2_errormargin_synth.mean()
print "Synth mean errormargin, sc3: ", sc3_errormargin_synth.mean()
print "Synth rms errormargin, sc2: ", rmse(sc2_errormargin_synth)
print "Synth rms errormargin, sc3: ", rmse(sc3_errormargin_synth)
#% error margins:
fig_error_margins(sc2_errormargin, sc3_errormargin, sim_input, sc3_model_uncert, station, no_sigma)
fig_error_margins(sc2_errormargin_synth, sc3_errormargin_synth, sim_input_synth, sc3_model_uncert_synth, station, no_sigma)
sns.jointplot(np.abs(nonfit_errors), ens_preds.loc[nonfit_ts_start:nonfit_ts_end].std(axis=1))
sns.jointplot(np.abs(synth_resid), ens_preds.loc[nonfit_ts_start:nonfit_ts_end].std(axis=1))
#% T Q scatter plots
fig, axes = plt.subplots(3,1, figsize=(10,16), sharex=True, sharey=True)
axes[0].scatter(sim_input['T_sup'], sim_input['Q'], c=sim_input['cons'])
axes[0].set_title(station + ': ' + 'Scenario 1')
axes[1].scatter(sim_results_sc2['T_sup'], sim_results_sc2['Q'], c=sim_results_sc2['cons'])
axes[1].set_title(station + ': Scenario 2: ' + str(no_sigma) + r'$\sigma$' )
axes[2].scatter(sim_results_sc3['T_sup'], sim_results_sc3['Q'], c=sim_results_sc3['cons'])
axes[2].set_title(station + ': Scenario 3: ' + str(no_sigma) + r'$\sigma$')
axes[1].set_ylabel(u'Water flow rate [m%s/h]'%uni_tothethird, size=8)
axes[2].set_xlabel(u'Supply temperature [%sC]'%uni_degree, size=8)
fig.tight_layout()
fig.savefig(figpath + 'TQscatter_%2.2f'%(no_sigma) + 'sigma_' + station + '.pdf')
# T_sup time series fig
fig, axes = plt.subplots(3,1, figsize=(15,15), sharex=True)
axes[0].plot_date(sim_input.index, sim_input['T_sup'], 'k-', label='Scenario 1')
axes[0].plot_date(sim_input.index, sim_results_sc2['T_sup'], 'r-', lw=3, label='Scenario 2')
axes[0].plot_date(sim_input.index, sim_results_sc2['T_sup'], 'g-', label='Scenario 3')
axes[0].set_title(station + ', ' + str(no_sigma) + r'$\sigma$' + ': Supply temperature')
axes[0].set_ylabel(u'Supply temperature [%sC]'%uni_degree, size=8)
axes[0].legend()
axes[1].plot_date(sim_input.index, sim_input['Q'], 'k-', label='Scenario 1' )
axes[1].plot_date(sim_input.index, sim_results_sc2['Q'], 'r-', label='Scenario 2')
axes[1].plot_date(sim_input.index, sim_results_sc2['Q_ref'], 'b-', lw=1, label=r'$Q_{ref}$' + 'Scenario 2')
axes[1].set_ylabel(u'Water flow rate [m%s/h]'%uni_tothethird, size=8)
axes[1].legend()
axes[2].plot_date(sim_input.index, sim_input['Q'], 'k-', label='Scenario 1' )
axes[2].plot_date(sim_input.index, sim_results_sc3['Q'], 'g-', label='Scenario 3')
axes[2].plot_date(sim_input.index, sim_results_sc3['Q_ref'], 'b-', lw=1, label=r'$Q_{ref}$' + 'Scenario 3')
axes[2].set_ylabel(u'Water flow rate [m%s/h]'%uni_tothethird, size=8)
axes[2].legend()
fig.savefig(figpath + 'TQtimeseries_%2.2f'%(no_sigma) + 'sigma_' + station + '.pdf')
# Differencen in supply temperature between the scenarios
fig_heat_loss(sim_input, sim_results_sc2, sim_results_sc3, station, no_sigma)
fig_heat_loss(sim_input_synth, sim_results_sc2_synth, sim_results_sc3_synth, station, no_sigma, save=False)
return
#%% The below section only runs if we view Tmin as a function of Q (the old way)
# note: SOME OF THIS USES CONSTANT TRET!!
TminofQ = False
if TminofQ:
# outlierdetection
X = df[['T_sup','Q']]
outlier_detection = False
if outlier_detection:
detect_outliers(X, station)
else:
inlierpred = np.ones(len(df), dtype=bool)
fig, ax1 = plt.subplots()
ax2 = ax1.twinx()
cond_df = df
ax1.plot_date(np.array(cond_df['ts']), np.array(cond_df['Q']), 'b')
ax2.plot_date(np.array(cond_df['ts']), np.array(cond_df['T_sup']), 'r-')
plt.figure()
plt.plot_date(df['ts'], df['cons'], 'g-')
plt.title(station)
plt.figure()
plt.scatter(df['T_sup'], df['Q'], c=df['cons'], alpha=0.25)
plt.colorbar()
plt.title(station)
outliers = df[np.logical_not(inlierpred)]
plt.plot(np.array(outliers['T_sup']), np.array(outliers['Q']), 'ko')
#%%
#plot_Tmin_Q_quantiles(df, inlierpred)
Q = np.linspace(df[inlierpred]['Q'].min(), df[inlierpred]['Q'].max(), 500)
qs = [0.001, 0.005, 0.01, 0.02275, 0.05, 0.1]
for q in qs:
T_min_func, Q_quantiles = get_Tmin_func(df[inlierpred],T_min_q=q, N_Q_q=21)
plt.plot(T_min_func(Q), Q, label=str(q), lw=2)
plt.legend()
for Q_qua in Q_quantiles:
plt.axhline(y=Q_qua)
#%% P vs Q (T=Tmin(Q))
T_min_func, Q_quantiles = get_Tmin_func(df, T_min_q=0.02275, N_Q_q=21)
plt.figure()
plt.plot(Q, T_min_func(Q), 'r', label='Tmin')
P = specific_heat_water*density_water*Q*(T_min_func(Q)-T_ret)
plt.plot(Q, P, 'b', label='Cons')
plt.xlabel('Q')
plt.legend()
plt.figure()
simP = df['cons']
res = [op_model(cons, T_min_func, Q_max=Q_max_dict[station], T_ret=T_ret) for cons in simP]
simT, simQ = zip(*res)
plt.scatter(df['T_sup'], df['Q'], c='k', alpha=0.1)
plt.scatter(simT,simQ,c=simP)
plt.colorbar()
