def production_model(): # figure 3
plt.close('all')
cols = ['prod24h_before', 'Tout24hdiff', 'vWind24hdiff', 'sunRad24hdiff']
ts1 = ens.gen_hourly_timesteps(dt.datetime(2015, 12, 17, 1),
dt.datetime(2016, 1, 15, 0))
ts2 = ens.gen_hourly_timesteps(dt.datetime(2016, 1, 20, 1),
dt.datetime(2016, 2, 5, 0))
#load the data
fit_data = ens.repack_ens_mean_as_df()
fit_data['prod24h_before'] = sq.fetch_production(
ts1[0] + dt.timedelta(days=-1), ts1[-1] + dt.timedelta(days=-1))
fit_data['Tout24hdiff'] = fit_data['Tout'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['Tout']).mean(axis=1)
fit_data['vWind24hdiff'] = fit_data['vWind'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['vWind']).mean(axis=1)
fit_data['sunRad24hdiff'] = fit_data['sunRad'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['sunRad']).mean(axis=1)
vali_data = ens.repack_ens_mean_as_df(ts2[0], ts2[-1])
vali_data['prod24h_before'] = sq.fetch_production(
ts2[0] + dt.timedelta(days=-1), ts2[-1] + dt.timedelta(days=-1))
vali_data['Tout24hdiff'] = vali_data['Tout'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['Tout']).mean(axis=1)
vali_data['vWind24hdiff'] = vali_data['vWind'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['vWind']).mean(axis=1)
vali_data['sunRad24hdiff'] = vali_data['sunRad'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['sunRad']).mean(axis=1)
# correct error in production:
new_val = (vali_data['prod'][116] + vali_data['prod'][116]) / 2
vali_data['prod'][116] = new_val
vali_data['prod'][117] = new_val
vali_data['prod24h_before'][116 + 24] = new_val
vali_data['prod24h_before'][117 + 24] = new_val
# do the fit
X = fit_data[cols]
y = fit_data['prod']
res = mlin_regression(y, X, add_const=False)
fig, [ax1, ax2] = plt.subplots(2,
1,
sharex=True,
figsize=(dcolwidth, 0.57 * dcolwidth),
gridspec_kw={'height_ratios': [4, 1]})
# load ensemble data
ens_data1 = ens.load_ens_timeseries_as_df(ts_start=ts1[0], ts_end=ts1[-1],\
weathervars=['Tout', 'vWind', 'sunRad'])
ens_data1['prod24h_before'] = fit_data['prod24h_before']
ens_data1_24h_before = ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['Tout', 'vWind', 'sunRad'])
ens_data2 = ens.load_ens_timeseries_as_df(ts_start=ts2[0], ts_end=ts2[-1],\
weathervars=['Tout', 'vWind', 'sunRad'])
ens_data2['prod24h_before'] = vali_data['prod24h_before']
ens_data2_24h_before = ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['Tout', 'vWind', 'sunRad'])
for i in range(25):
for v in ['Tout', 'vWind', 'sunRad']:
key_raw = v + str(i)
key_diff = v + '24hdiff' + str(i)
ens_data1[
key_diff] = ens_data1[key_raw] - ens_data1_24h_before[key_raw]
ens_data2[
key_diff] = ens_data2[key_raw] - ens_data2_24h_before[key_raw]
all_ens_data = pd.concat([ens_data1, ens_data2])
all_ts = ts1 + ts2
#
#
# calculate production for each ensemble member
ens_prods = np.zeros((len(all_ts), 25))
for i in range(25):
ens_cols = ['Tout24hdiff' + str(i), 'vWind24hdiff' + str(i),\
'sunRad24hdiff' + str(i), 'prod24h_before']
ens_params = pd.Series({
'Tout24hdiff' + str(i):
res.params['Tout24hdiff'],
'vWind24hdiff' + str(i):
res.params['vWind24hdiff'],
'sunRad24hdiff' + str(i):
res.params['sunRad24hdiff'],
'prod24h_before':
res.params['prod24h_before']
})
ens_prods[:, i] = linear_map(all_ens_data, ens_params, ens_cols)
# calculate combined confint
ens_std = ens_prods.std(axis=1)
vali_resid = linear_map(vali_data, res.params, cols) - vali_data['prod']
vali_resid_corrig = vali_resid - np.sign(
vali_resid) * 1.9599 * ens_std[len(ts1):]
#mean_conf_int_spread = (vali_resid_corrig.quantile(0.95) - vali_resid_corrig.quantile(0.05))/2 # this conf_int is not used anymore
fit_resid = res.resid
fit_resid_corrig = fit_resid - np.sign(
fit_resid) * 1.9599 * ens_std[0:len(ts1)]
conf_int_spread_lower = -fit_resid_corrig.quantile(0.025)
conf_int_spread_higher = fit_resid_corrig.quantile(0.975)
combined_conf_ints = conf_int_spread_lower + conf_int_spread_higher + 2 * 1.9599 * ens_std
all_prod_model = np.concatenate(
[res.fittedvalues,
linear_map(vali_data, res.params, cols)])
combined_ub95 = all_prod_model + conf_int_spread_higher + 1.9599 * ens_std
combined_lb95 = all_prod_model - (conf_int_spread_lower + 1.9599 * ens_std)
# plot confint
ax1.fill_between(all_ts[len(ts1):],
combined_lb95[len(ts1):],
combined_ub95[len(ts1):],
label='95% prediction intervals')
ax1.fill_between(all_ts[len(ts1):],
all_prod_model[len(ts1):] - 1.9599 * ens_std[len(ts1):],
all_prod_model[len(ts1):] + 1.9599 * ens_std[len(ts1):],
facecolor='grey',
label='Weather ensemble 95% conf. int.')
# plot ensempble models
ax1.plot_date(all_ts[len(ts1):], ens_prods[len(ts1):], '-', lw=0.5)
ax1.plot_date(ts2,
vali_data['prod'],
'k-',
lw=2,
label='Historical production')
ax1.plot_date(ts2,
linear_map(vali_data, res.params, cols),
'-',
c=red,
lw=2,
label='Production model')
ax1.set_ylabel('Production [MW]', size=8)
ax1.tick_params(axis='both', which='major', labelsize=8)
ax1.xaxis.set_major_formatter(DateFormatter('%b %d'))
ax1.legend(loc=1, prop={'size': 8})
ax1.set_ylim([300, 1100])
N = conf_int_spread_higher + 1.9599 * ens_std[len(ts1):].max()
ax2.fill_between(ts2,
-(1.9599 * ens_std[len(ts1):] + conf_int_spread_lower) /
N,
-1.9599 * ens_std[len(ts1):] / N,
alpha=0.5)
ax2.fill_between(ts2,
-1.9599 * ens_std[len(ts1):] / N,
np.zeros(len(ts2)),
facecolor='grey',
alpha=0.5)
ax2.fill_between(ts2, 1.9599 * ens_std[len(ts1):] / N, facecolor='grey')
ax2.fill_between(ts2, 1.9599 * ens_std[len(ts1):] / N,
(conf_int_spread_higher + 1.9599 * ens_std[len(ts1):]) /
N)
ax2.set_ylabel('Prediction intervals \n[normalized]', size=8)
ax2.tick_params(axis='y', which='major', labelsize=8)
ax2.set_xlim(dt.datetime(2016, 1, 20, 0), dt.datetime(2016, 2, 5, 0))
fig.tight_layout()
print "Min_normalized pos conf bound. ", np.min(1.9599 *
ens_std[len(ts1):] / N +
conf_int_spread_higher / N)
print "MAE = " + str(mae(vali_resid))
print "MAPE = " + str(mape(vali_resid, vali_data['prod']))
print "RMSE = " + str(rmse(vali_resid))
print "ME = " + str(np.mean(vali_resid))
print "MAE (fit) = " + str(mae(res.resid))
print "MAPE (fit) = " + str(mape(res.resid, fit_data['prod']))
print "RMSE (fit)= " + str(rmse(res.resid))
print "ME (fit)= " + str(np.mean(res.resid))
print "Width of const blue bands (MW)", conf_int_spread_lower, conf_int_spread_higher
plt.savefig('Q:/Projekter/Ens Article 1/figures/production_model.pdf',
dpi=400)
EO3_fc1 = sq.fetch_EO3_midnight_forecast(ts1[0], ts1[-1])
EO3_fc2 = sq.fetch_EO3_midnight_forecast(ts2[0], ts2[-1])
EO3_err = EO3_fc2 - vali_data['prod']
EO3_err_fit = EO3_fc1 - fit_data['prod']
print "MAE (EO3) = " + str(mae(EO3_err))
print "MAPE (EO3) = " + str(mape(EO3_err, vali_data['prod']))
print "RMSE (EO3)= " + str(rmse(EO3_err))
print "ME (EO3)= " + str(np.mean(EO3_err))
print "MAE (EO3_fit) = " + str(mae(EO3_err_fit))
print "MAPE (EO3_fit) = " + str(mape(EO3_err_fit, fit_data['prod']))
print "RMSE (EO3_fit)= " + str(rmse(EO3_err_fit))
print "ME (EO3_fit)= " + str(np.mean(EO3_err_fit))
print np.min(combined_conf_ints[len(ts1):] / combined_conf_ints.max())
np.savez('combined_conf_int',
combined_conf_int=(conf_int_spread_higher + 1.9599 * ens_std),
timesteps=all_ts)
print "Corr coeff: vali ", np.corrcoef(
vali_data['prod'], linear_map(vali_data, res.params, cols))[0, 1]
print "Corr coeff: vali EO3 ", np.corrcoef(vali_data['prod'], EO3_fc2)[0,
1]
print "Corr coeff: fit ", np.corrcoef(fit_data['prod'],
res.fittedvalues)[0, 1]
print "Corr coeff: fit EO3 ", np.corrcoef(fit_data['prod'], EO3_fc1)[0, 1]
print "% of actual production in vali period above upper", float(
len(
np.where(vali_data['prod'] >
(conf_int_spread_higher + 1.9599 * ens_std[len(ts1):] +
linear_map(vali_data, res.params, cols)))[0])) / len(ts2)
print "plus minus: ", 0.5 / len(ts2)
print "% of actual production in vali period below lower", float(
len(
np.where(vali_data['prod'] <
(linear_map(vali_data, res.params, cols) -
(conf_int_spread_lower + 1.9599 * ens_std[len(ts1):])))
[0])) / len(ts2)
print "plus minus: ", 0.5 / len(ts2)
return res, fit_data
def second_ens_prod_fig():
""" This plot is based on a production model taking into account:
the production 24 hours before as well as the change in
temparature, windspeed and solar radiotion from 24 hours ago to now.
"""
plt.close('all')
cols = ['prod24h_before', 'Tout24hdiff', 'vWind24hdiff', 'sunRad24hdiff']
ts1 = ens.gen_hourly_timesteps(dt.datetime(2015, 12, 17, 1),
dt.datetime(2016, 1, 15, 0))
ts2 = ens.gen_hourly_timesteps(dt.datetime(2016, 1, 20, 1),
dt.datetime(2016, 2, 5, 0))
#load the data
fit_data = ens.repack_ens_mean_as_df()
fit_data['prod24h_before'] = sq.fetch_production(
ts1[0] + dt.timedelta(days=-1), ts1[-1] + dt.timedelta(days=-1))
fit_data['Tout24hdiff'] = fit_data['Tout'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['Tout']).mean(axis=1)
fit_data['vWind24hdiff'] = fit_data['vWind'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['vWind']).mean(axis=1)
fit_data['sunRad24hdiff'] = fit_data['sunRad'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['sunRad']).mean(axis=1)
vali_data = ens.repack_ens_mean_as_df(ts2[0], ts2[-1])
vali_data['prod24h_before'] = sq.fetch_production(
ts2[0] + dt.timedelta(days=-1), ts2[-1] + dt.timedelta(days=-1))
vali_data['Tout24hdiff'] = vali_data['Tout'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['Tout']).mean(axis=1)
vali_data['vWind24hdiff'] = vali_data['vWind'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['vWind']).mean(axis=1)
vali_data['sunRad24hdiff'] = vali_data['sunRad'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['sunRad']).mean(axis=1)
# correct error in production:
new_val = (vali_data['prod'][116] + vali_data['prod'][116]) / 2
vali_data['prod'][116] = new_val
vali_data['prod'][117] = new_val
vali_data['prod24h_before'][116 + 24] = new_val
vali_data['prod24h_before'][117 + 24] = new_val
# do the fit
X = fit_data[cols]
y = fit_data['prod']
res = mlin_regression(y, X, add_const=False)
fig, [ax1, ax2] = plt.subplots(2, 1, figsize=(40, 20))
# load ensemble data
ens_data1 = ens.load_ens_timeseries_as_df(ts_start=ts1[0], ts_end=ts1[-1],\
weathervars=['Tout', 'vWind', 'sunRad'])
ens_data1['prod24h_before'] = fit_data['prod24h_before']
ens_data1_24h_before = ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['Tout', 'vWind', 'sunRad'])
ens_data2 = ens.load_ens_timeseries_as_df(ts_start=ts2[0], ts_end=ts2[-1],\
weathervars=['Tout', 'vWind', 'sunRad'])
ens_data2['prod24h_before'] = vali_data['prod24h_before']
ens_data2_24h_before = ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['Tout', 'vWind', 'sunRad'])
for i in range(25):
for v in ['Tout', 'vWind', 'sunRad']:
key_raw = v + str(i)
key_diff = v + '24hdiff' + str(i)
ens_data1[
key_diff] = ens_data1[key_raw] - ens_data1_24h_before[key_raw]
ens_data2[
key_diff] = ens_data2[key_raw] - ens_data2_24h_before[key_raw]
all_ens_data = pd.concat([ens_data1, ens_data2])
all_ts = ts1 + ts2
#
#
# calculate production for each ensemble member
ens_prods = np.zeros((len(all_ts), 25))
for i in range(25):
ens_cols = ['Tout24hdiff' + str(i), 'vWind24hdiff' + str(i),\
'sunRad24hdiff' + str(i), 'prod24h_before']
ens_params = pd.Series({
'Tout24hdiff' + str(i):
res.params['Tout24hdiff'],
'vWind24hdiff' + str(i):
res.params['vWind24hdiff'],
'sunRad24hdiff' + str(i):
res.params['sunRad24hdiff'],
'prod24h_before':
res.params['prod24h_before']
})
ens_prods[:, i] = linear_map(all_ens_data, ens_params, ens_cols)
# calculate combined confint
ens_std = ens_prods.std(axis=1)
vali_resid = linear_map(vali_data, res.params, cols) - vali_data['prod']
vali_resid_corrig = vali_resid - np.sign(
vali_resid) * 1.9599 * ens_std[len(ts1):]
mean_conf_int_spread = (vali_resid_corrig.quantile(0.95) -
vali_resid_corrig.quantile(0.05)) / 2
combined_conf_int = mean_conf_int_spread + 1.9599 * ens_std
all_prod_model = np.concatenate(
[res.fittedvalues,
linear_map(vali_data, res.params, cols)])
combined_ub95 = all_prod_model + combined_conf_int
combined_lb95 = all_prod_model - combined_conf_int
# plot confint
ax1.fill_between(all_ts,
combined_lb95,
combined_ub95,
label='Combined 95% conf. int.')
ax1.fill_between(all_ts,
all_prod_model - 1.9599 * ens_std,
all_prod_model + 1.9599 * ens_std,
facecolor='grey',
label='Ensemble 95% conf. int.')
# plot ensempble models
ax1.plot_date(all_ts, ens_prods, '-', lw=0.5)
ax1.plot_date(ts1, y, 'k-', lw=2, label='Actual production')
ax1.plot_date(ts1,
res.fittedvalues,
'r-',
lw=2,
label='Model on ensemble mean')
ax1.plot_date(ts2, vali_data['prod'], 'k-', lw=2, label='')
ax1.plot_date(ts2, linear_map(vali_data, res.params, cols), 'r-', lw=2)
ax1.set_ylabel('[MW]')
ax1.legend(loc=2)
ax1.set_ylim([0, 1100])
ax2.plot_date(ts1, res.resid, '-', label='Residual, fitted data')
ax2.plot_date(ts2, vali_resid, '-', label='Residual, validation data')
ax2.set_ylabel('[MW]')
ax2.legend(loc=2)
ax2.set_ylim([-550, 550])
print "MAE = " + str(mae(vali_resid))
print "MAPE = " + str(mape(vali_resid, vali_data['prod']))
print "RMSE = " + str(rmse(vali_resid))
print "ME = " + str(np.mean(vali_resid))
print "MAE (fit) = " + str(mae(res.resid))
print "MAPE (fit) = " + str(mape(res.resid, fit_data['prod']))
print "RMSE (fit)= " + str(rmse(res.resid))
print "ME (fit)= " + str(np.mean(res.resid))
plt.savefig('figures/ens_prod_models_v2.pdf', dpi=600)
plt.figure()
plt.plot_date(all_ts, ens_std)
plt.ylabel('Std. of ensemble production models [MW]')
plt.savefig('figures/std_ens_prod_models.pdf', dpi=600)
#
vali_ens_std = ens_std[len(ts1):]
sns.jointplot(x=pd.Series(vali_ens_std), y=np.abs(vali_resid))
sns.jointplot(x=vali_data['prod'],
y=pd.Series(linear_map(vali_data, res.params, cols)))
EO3_fc1 = sq.fetch_EO3_midnight_forecast(ts1[0], ts1[-1])
EO3_fc2 = sq.fetch_EO3_midnight_forecast(ts2[0], ts2[-1])
plt.figure()
plt.plot_date(ts1, fit_data['prod'], 'k-', label='Actual production')
plt.plot_date(ts2, vali_data['prod'], 'k-')
plt.plot_date(ts1, EO3_fc1, 'r-', label='EO3 forecast')
plt.plot_date(ts2, EO3_fc2, 'r-')
EO3_err = EO3_fc2 - vali_data['prod']
EO3_err_fit = EO3_fc1 - fit_data['prod']
print "MAE (EO3) = " + str(mae(EO3_err))
print "MAPE (EO3) = " + str(mape(EO3_err, vali_data['prod']))
print "RMSE (EO3)= " + str(rmse(EO3_err))
print "ME (EO3)= " + str(np.mean(EO3_err))
print "MAE (EO3_fit) = " + str(mae(EO3_err_fit))
print "MAPE (EO3_fit) = " + str(mape(EO3_err_fit, fit_data['prod']))
print "RMSE (EO3_fit)= " + str(rmse(EO3_err_fit))
print "ME (EO3_fit)= " + str(np.mean(EO3_err_fit))
sns.jointplot(x=pd.Series(vali_ens_std), y=np.abs(EO3_err))
plt.figure(figsize=(20, 10))
plt.subplot(2, 1, 1)
plt.plot_date(all_ts, combined_conf_int / combined_conf_int.max(), '-')
plt.ylabel('Model + ensemble uncertainty \n [normalized]')
plt.ylim(0, 1)
plt.subplot(2, 1, 2)
plt.plot_date(all_ts,
(1 - 0.2 * combined_conf_int / combined_conf_int.max()),
'-',
label='Dynamic setpoint')
plt.plot_date(all_ts,
0.8 * np.ones(len(all_ts)),
'--',
label='Static setpoint')
plt.ylabel(
'Setpoint for pump massflow \n temperature [fraction of max pump cap]')
plt.legend()
plt.ylim(.7, 1)
plt.savefig('figures/setpoint.pdf')
return vali_data, fit_data, res, ens_std, vali_resid
def production_model(): # figure 3
plt.close('all')
cols = ['prod24h_before', 'Tout24hdiff', 'vWind24hdiff', 'sunRad24hdiff']
ts1 = ens.gen_hourly_timesteps(dt.datetime(2015,12,17,1), dt.datetime(2016,1,15,0))
ts2 = ens.gen_hourly_timesteps(dt.datetime(2016,1,20,1), dt.datetime(2016,2,5,0))
#load the data
fit_data = ens.repack_ens_mean_as_df()
fit_data['prod24h_before'] = sq.fetch_production(ts1[0]+dt.timedelta(days=-1), ts1[-1]+dt.timedelta(days=-1))
fit_data['Tout24hdiff'] = fit_data['Tout'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['Tout']).mean(axis=1)
fit_data['vWind24hdiff'] = fit_data['vWind'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['vWind']).mean(axis=1)
fit_data['sunRad24hdiff'] = fit_data['sunRad'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['sunRad']).mean(axis=1)
vali_data = ens.repack_ens_mean_as_df(ts2[0], ts2[-1])
vali_data['prod24h_before'] = sq.fetch_production(ts2[0]+dt.timedelta(days=-1), ts2[-1]+dt.timedelta(days=-1))
vali_data['Tout24hdiff'] = vali_data['Tout'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['Tout']).mean(axis=1)
vali_data['vWind24hdiff'] = vali_data['vWind'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['vWind']).mean(axis=1)
vali_data['sunRad24hdiff'] = vali_data['sunRad'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['sunRad']).mean(axis=1)
# correct error in production:
new_val = (vali_data['prod'][116] +vali_data['prod'][116])/2
vali_data['prod'][116] = new_val
vali_data['prod'][117] = new_val
vali_data['prod24h_before'][116+24] = new_val
vali_data['prod24h_before'][117+24] = new_val
# do the fit
X = fit_data[cols]
y = fit_data['prod']
res = mlin_regression(y, X, add_const=False)
fig, [ax1, ax2] = plt.subplots(2, 1, sharex=True, figsize=(dcolwidth, 0.57*dcolwidth), gridspec_kw={'height_ratios':[4,1]})
# load ensemble data
ens_data1 = ens.load_ens_timeseries_as_df(ts_start=ts1[0], ts_end=ts1[-1],\
weathervars=['Tout', 'vWind', 'sunRad'])
ens_data1['prod24h_before'] = fit_data['prod24h_before']
ens_data1_24h_before = ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['Tout', 'vWind', 'sunRad'])
ens_data2 = ens.load_ens_timeseries_as_df(ts_start=ts2[0], ts_end=ts2[-1],\
weathervars=['Tout', 'vWind', 'sunRad'])
ens_data2['prod24h_before'] = vali_data['prod24h_before']
ens_data2_24h_before = ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['Tout', 'vWind', 'sunRad'])
for i in range(25):
for v in ['Tout', 'vWind', 'sunRad']:
key_raw = v + str(i)
key_diff = v + '24hdiff' + str(i)
ens_data1[key_diff] = ens_data1[key_raw] - ens_data1_24h_before[key_raw]
ens_data2[key_diff] = ens_data2[key_raw] - ens_data2_24h_before[key_raw]
all_ens_data = pd.concat([ens_data1, ens_data2])
all_ts = ts1 + ts2
#
#
# calculate production for each ensemble member
ens_prods = np.zeros((len(all_ts), 25))
for i in range(25):
ens_cols = ['Tout24hdiff' + str(i), 'vWind24hdiff' + str(i),\
'sunRad24hdiff' + str(i), 'prod24h_before']
ens_params = pd.Series({'Tout24hdiff' + str(i):res.params['Tout24hdiff'],
'vWind24hdiff' + str(i):res.params['vWind24hdiff'],
'sunRad24hdiff' + str(i):res.params['sunRad24hdiff'],
'prod24h_before':res.params['prod24h_before']})
ens_prods[:,i] = linear_map(all_ens_data, ens_params, ens_cols)
# calculate combined confint
ens_std = ens_prods.std(axis=1)
vali_resid = linear_map(vali_data, res.params, cols) - vali_data['prod']
vali_resid_corrig = vali_resid - np.sign(vali_resid)*1.9599*ens_std[len(ts1):]
#mean_conf_int_spread = (vali_resid_corrig.quantile(0.95) - vali_resid_corrig.quantile(0.05))/2 # this conf_int is not used anymore
fit_resid = res.resid
fit_resid_corrig = fit_resid - np.sign(fit_resid)*1.9599*ens_std[0:len(ts1)]
conf_int_spread_lower = - fit_resid_corrig.quantile(0.025)
conf_int_spread_higher = fit_resid_corrig.quantile(0.975)
combined_conf_ints = conf_int_spread_lower + conf_int_spread_higher + 2*1.9599*ens_std
all_prod_model = np.concatenate([res.fittedvalues, linear_map(vali_data, res.params, cols)])
combined_ub95 = all_prod_model + conf_int_spread_higher + 1.9599*ens_std
combined_lb95 = all_prod_model - (conf_int_spread_lower + 1.9599*ens_std)
# plot confint
ax1.fill_between(all_ts[len(ts1):], combined_lb95[len(ts1):], combined_ub95[len(ts1):], label='95% prediction intervals')
ax1.fill_between(all_ts[len(ts1):], all_prod_model[len(ts1):] - 1.9599*ens_std[len(ts1):], all_prod_model[len(ts1):] + 1.9599*ens_std[len(ts1):], facecolor='grey', label='Weather ensemble 95% conf. int.')
# plot ensempble models
ax1.plot_date(all_ts[len(ts1):], ens_prods[len(ts1):], '-', lw=0.5)
ax1.plot_date(ts2, vali_data['prod'], 'k-', lw=2, label='Historical production')
ax1.plot_date(ts2, linear_map(vali_data, res.params, cols), '-', c=red, lw=2, label='Production model')
ax1.set_ylabel('Production [MW]', size=8)
ax1.tick_params(axis='both', which='major', labelsize=8)
ax1.xaxis.set_major_formatter(DateFormatter('%b %d') )
ax1.legend(loc=1, prop={'size':8})
ax1.set_ylim([300,1100])
N = conf_int_spread_higher + 1.9599*ens_std[len(ts1):].max()
ax2.fill_between(ts2, -(1.9599*ens_std[len(ts1):]+conf_int_spread_lower)/N, -1.9599*ens_std[len(ts1):]/N, alpha=0.5)
ax2.fill_between(ts2, -1.9599*ens_std[len(ts1):]/N, np.zeros(len(ts2)), facecolor='grey',alpha=0.5)
ax2.fill_between(ts2, 1.9599*ens_std[len(ts1):]/N, facecolor='grey')
ax2.fill_between(ts2, 1.9599*ens_std[len(ts1):]/N, (conf_int_spread_higher+1.9599*ens_std[len(ts1):])/N)
ax2.set_ylabel('Prediction intervals \n[normalized]', size=8)
ax2.tick_params(axis='y', which='major', labelsize=8)
ax2.set_xlim(dt.datetime(2016,1,20,0), dt.datetime(2016,2,5,0))
fig.tight_layout()
print "Min_normalized pos conf bound. ", np.min(1.9599*ens_std[len(ts1):]/N+conf_int_spread_higher/N)
print "MAE = " + str(mae(vali_resid))
print "MAPE = " + str(mape(vali_resid, vali_data['prod']))
print "RMSE = " + str(rmse(vali_resid))
print "ME = " + str(np.mean(vali_resid))
print "MAE (fit) = " + str(mae(res.resid))
print "MAPE (fit) = " + str(mape(res.resid, fit_data['prod']))
print "RMSE (fit)= " + str(rmse(res.resid))
print "ME (fit)= " + str(np.mean(res.resid))
print "Width of const blue bands (MW)", conf_int_spread_lower, conf_int_spread_higher
plt.savefig('Q:/Projekter/Ens Article 1/figures/production_model.pdf', dpi=400)
EO3_fc1 = sq.fetch_EO3_midnight_forecast(ts1[0], ts1[-1])
EO3_fc2 = sq.fetch_EO3_midnight_forecast(ts2[0], ts2[-1])
EO3_err = EO3_fc2-vali_data['prod']
EO3_err_fit = EO3_fc1-fit_data['prod']
print "MAE (EO3) = " + str(mae(EO3_err))
print "MAPE (EO3) = " + str(mape(EO3_err, vali_data['prod']))
print "RMSE (EO3)= " + str(rmse(EO3_err))
print "ME (EO3)= " + str(np.mean(EO3_err))
print "MAE (EO3_fit) = " + str(mae(EO3_err_fit))
print "MAPE (EO3_fit) = " + str(mape(EO3_err_fit, fit_data['prod']))
print "RMSE (EO3_fit)= " + str(rmse(EO3_err_fit))
print "ME (EO3_fit)= " + str(np.mean(EO3_err_fit))
print np.min(combined_conf_ints[len(ts1):]/combined_conf_ints.max())
np.savez('combined_conf_int', combined_conf_int=(conf_int_spread_higher+1.9599*ens_std), timesteps=all_ts)
print "Corr coeff: vali ", np.corrcoef(vali_data['prod'],linear_map(vali_data, res.params, cols))[0,1]
print "Corr coeff: vali EO3 ", np.corrcoef(vali_data['prod'], EO3_fc2)[0,1]
print "Corr coeff: fit ", np.corrcoef(fit_data['prod'],res.fittedvalues)[0,1]
print "Corr coeff: fit EO3 ", np.corrcoef(fit_data['prod'], EO3_fc1)[0,1]
print "% of actual production in vali period above upper", float(len(np.where(vali_data['prod']>(conf_int_spread_higher+1.9599*ens_std[len(ts1):]+linear_map(vali_data, res.params, cols)))[0]))/len(ts2)
print "plus minus: ", 0.5/len(ts2)
print "% of actual production in vali period below lower", float(len(np.where(vali_data['prod']<(linear_map(vali_data, res.params, cols)-(conf_int_spread_lower+1.9599*ens_std[len(ts1):])))[0]))/len(ts2)
print "plus minus: ", 0.5/len(ts2)
return res, fit_data
def second_ens_prod_fig():
""" This plot is based on a production model taking into account:
the production 24 hours before as well as the change in
temparature, windspeed and solar radiotion from 24 hours ago to now.
"""
plt.close('all')
cols = ['prod24h_before', 'Tout24hdiff', 'vWind24hdiff', 'sunRad24hdiff']
ts1 = ens.gen_hourly_timesteps(dt.datetime(2015,12,17,1), dt.datetime(2016,1,15,0))
ts2 = ens.gen_hourly_timesteps(dt.datetime(2016,1,20,1), dt.datetime(2016,2,5,0))
#load the data
fit_data = ens.repack_ens_mean_as_df()
fit_data['prod24h_before'] = sq.fetch_production(ts1[0]+dt.timedelta(days=-1), ts1[-1]+dt.timedelta(days=-1))
fit_data['Tout24hdiff'] = fit_data['Tout'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['Tout']).mean(axis=1)
fit_data['vWind24hdiff'] = fit_data['vWind'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['vWind']).mean(axis=1)
fit_data['sunRad24hdiff'] = fit_data['sunRad'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['sunRad']).mean(axis=1)
vali_data = ens.repack_ens_mean_as_df(ts2[0], ts2[-1])
vali_data['prod24h_before'] = sq.fetch_production(ts2[0]+dt.timedelta(days=-1), ts2[-1]+dt.timedelta(days=-1))
vali_data['Tout24hdiff'] = vali_data['Tout'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['Tout']).mean(axis=1)
vali_data['vWind24hdiff'] = vali_data['vWind'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['vWind']).mean(axis=1)
vali_data['sunRad24hdiff'] = vali_data['sunRad'] \
- ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['sunRad']).mean(axis=1)
# correct error in production:
new_val = (vali_data['prod'][116] +vali_data['prod'][116])/2
vali_data['prod'][116] = new_val
vali_data['prod'][117] = new_val
vali_data['prod24h_before'][116+24] = new_val
vali_data['prod24h_before'][117+24] = new_val
# do the fit
X = fit_data[cols]
y = fit_data['prod']
res = mlin_regression(y, X, add_const=False)
fig, [ax1, ax2] = plt.subplots(2,1, figsize=(40,20))
# load ensemble data
ens_data1 = ens.load_ens_timeseries_as_df(ts_start=ts1[0], ts_end=ts1[-1],\
weathervars=['Tout', 'vWind', 'sunRad'])
ens_data1['prod24h_before'] = fit_data['prod24h_before']
ens_data1_24h_before = ens.load_ens_timeseries_as_df(\
ts_start=ts1[0]+dt.timedelta(days=-1),\
ts_end=ts1[-1]+dt.timedelta(days=-1), \
weathervars=['Tout', 'vWind', 'sunRad'])
ens_data2 = ens.load_ens_timeseries_as_df(ts_start=ts2[0], ts_end=ts2[-1],\
weathervars=['Tout', 'vWind', 'sunRad'])
ens_data2['prod24h_before'] = vali_data['prod24h_before']
ens_data2_24h_before = ens.load_ens_timeseries_as_df(\
ts_start=ts2[0]+dt.timedelta(days=-1),\
ts_end=ts2[-1]+dt.timedelta(days=-1), \
weathervars=['Tout', 'vWind', 'sunRad'])
for i in range(25):
for v in ['Tout', 'vWind', 'sunRad']:
key_raw = v + str(i)
key_diff = v + '24hdiff' + str(i)
ens_data1[key_diff] = ens_data1[key_raw] - ens_data1_24h_before[key_raw]
ens_data2[key_diff] = ens_data2[key_raw] - ens_data2_24h_before[key_raw]
all_ens_data = pd.concat([ens_data1, ens_data2])
all_ts = ts1 + ts2
#
#
# calculate production for each ensemble member
ens_prods = np.zeros((len(all_ts), 25))
for i in range(25):
ens_cols = ['Tout24hdiff' + str(i), 'vWind24hdiff' + str(i),\
'sunRad24hdiff' + str(i), 'prod24h_before']
ens_params = pd.Series({'Tout24hdiff' + str(i):res.params['Tout24hdiff'],
'vWind24hdiff' + str(i):res.params['vWind24hdiff'],
'sunRad24hdiff' + str(i):res.params['sunRad24hdiff'],
'prod24h_before':res.params['prod24h_before']})
ens_prods[:,i] = linear_map(all_ens_data, ens_params, ens_cols)
# calculate combined confint
ens_std = ens_prods.std(axis=1)
vali_resid = linear_map(vali_data, res.params, cols) - vali_data['prod']
vali_resid_corrig = vali_resid - np.sign(vali_resid)*1.9599*ens_std[len(ts1):]
mean_conf_int_spread = (vali_resid_corrig.quantile(0.95) - vali_resid_corrig.quantile(0.05))/2
combined_conf_int = mean_conf_int_spread + 1.9599*ens_std
all_prod_model = np.concatenate([res.fittedvalues, linear_map(vali_data, res.params, cols)])
combined_ub95 = all_prod_model + combined_conf_int
combined_lb95 = all_prod_model - combined_conf_int
# plot confint
ax1.fill_between(all_ts, combined_lb95, combined_ub95, label='Combined 95% conf. int.')
ax1.fill_between(all_ts, all_prod_model - 1.9599*ens_std, all_prod_model + 1.9599*ens_std, facecolor='grey', label='Ensemble 95% conf. int.')
# plot ensempble models
ax1.plot_date(all_ts, ens_prods, '-', lw=0.5)
ax1.plot_date(ts1, y, 'k-', lw=2, label='Actual production')
ax1.plot_date(ts1, res.fittedvalues,'r-', lw=2, label='Model on ensemble mean')
ax1.plot_date(ts2, vali_data['prod'], 'k-', lw=2, label='')
ax1.plot_date(ts2, linear_map(vali_data, res.params, cols), 'r-', lw=2)
ax1.set_ylabel('[MW]')
ax1.legend(loc=2)
ax1.set_ylim([0,1100])
ax2.plot_date(ts1, res.resid, '-', label='Residual, fitted data')
ax2.plot_date(ts2, vali_resid, '-', label='Residual, validation data')
ax2.set_ylabel('[MW]')
ax2.legend(loc=2)
ax2.set_ylim([-550, 550])
print "MAE = " + str(mae(vali_resid))
print "MAPE = " + str(mape(vali_resid, vali_data['prod']))
print "RMSE = " + str(rmse(vali_resid))
print "ME = " + str(np.mean(vali_resid))
print "MAE (fit) = " + str(mae(res.resid))
print "MAPE (fit) = " + str(mape(res.resid, fit_data['prod']))
print "RMSE (fit)= " + str(rmse(res.resid))
print "ME (fit)= " + str(np.mean(res.resid))
plt.savefig('figures/ens_prod_models_v2.pdf', dpi=600)
plt.figure()
plt.plot_date(all_ts, ens_std)
plt.ylabel('Std. of ensemble production models [MW]')
plt.savefig('figures/std_ens_prod_models.pdf', dpi=600)
#
vali_ens_std = ens_std[len(ts1):]
sns.jointplot(x=pd.Series(vali_ens_std), y=np.abs(vali_resid))
sns.jointplot(x=vali_data['prod'], y=pd.Series(linear_map(vali_data, res.params, cols)))
EO3_fc1 = sq.fetch_EO3_midnight_forecast(ts1[0], ts1[-1])
EO3_fc2 = sq.fetch_EO3_midnight_forecast(ts2[0], ts2[-1])
plt.figure()
plt.plot_date(ts1, fit_data['prod'], 'k-', label='Actual production')
plt.plot_date(ts2, vali_data['prod'], 'k-')
plt.plot_date(ts1, EO3_fc1, 'r-', label='EO3 forecast')
plt.plot_date(ts2, EO3_fc2, 'r-')
EO3_err = EO3_fc2-vali_data['prod']
EO3_err_fit = EO3_fc1-fit_data['prod']
print "MAE (EO3) = " + str(mae(EO3_err))
print "MAPE (EO3) = " + str(mape(EO3_err, vali_data['prod']))
print "RMSE (EO3)= " + str(rmse(EO3_err))
print "ME (EO3)= " + str(np.mean(EO3_err))
print "MAE (EO3_fit) = " + str(mae(EO3_err_fit))
print "MAPE (EO3_fit) = " + str(mape(EO3_err_fit, fit_data['prod']))
print "RMSE (EO3_fit)= " + str(rmse(EO3_err_fit))
print "ME (EO3_fit)= " + str(np.mean(EO3_err_fit))
sns.jointplot(x=pd.Series(vali_ens_std), y=np.abs(EO3_err))
plt.figure(figsize=(20,10))
plt.subplot(2,1,1)
plt.plot_date(all_ts, combined_conf_int/combined_conf_int.max(), '-')
plt.ylabel('Model + ensemble uncertainty \n [normalized]')
plt.ylim(0,1)
plt.subplot(2,1,2)
plt.plot_date(all_ts, (1-0.2*combined_conf_int/combined_conf_int.max()), '-', label='Dynamic setpoint')
plt.plot_date(all_ts, 0.8*np.ones(len(all_ts)), '--', label='Static setpoint')
plt.ylabel('Setpoint for pump massflow \n temperature [fraction of max pump cap]')
plt.legend()
plt.ylim(.7,1)
plt.savefig('figures/setpoint.pdf')
return vali_data, fit_data, res, ens_std, vali_resid
fit_y = fit_data['prod']
results = []
for columns in all_combs:
X = fit_data[columns]
res = mlin_regression(fit_y,X, add_const=False)
results.append(res)
vali_preds = []
for cols in all_combs:
vali_pred = linear_map(vali_data, res.params, cols)
vali_preds.append(vali_pred)
rmses = [rmse(vp-vali_data['prod']) for vp in vali_preds]
aics = [r.aic for r in results]
for c,r,a in zip(all_combs, rmses, aics):
print c,r,a
right_columns = ['prod24h_before', 'Tout24hdiff', 'vWind24hdiff', 'sunRad24hdiff']
test_pred = linear_map(test_data, results[all_combs.index(right_columns)].params, right_columns)
print "Test RMSE", rmse(test_pred-test_data['prod'])
print "Test MAE", mae(test_pred-test_data['prod'])
print "Test MAPE", mape(test_pred-test_data['prod'], test_data['prod'])
EO3_fc_test = sq.fetch_EO3_midnight_forecast(test_ts[0], test_ts[-1])
EO3_err = EO3_fc_test-test_data['prod']
print "MAE (EO3) = " + str(mae(EO3_err))
print "MAPE (EO3) = " + str(mape(EO3_err, test_data['prod']))
print "RMSE (EO3)= " + str(rmse(EO3_err))
print "ME (EO3)= " + str(np.mean(EO3_err))
res = mlin_regression(fit_y, X, add_const=False)
results.append(res)
vali_preds = []
for cols in all_combs:
vali_pred = linear_map(vali_data, res.params, cols)
vali_preds.append(vali_pred)
rmses = [rmse(vp - vali_data['prod']) for vp in vali_preds]
aics = [r.aic for r in results]
for c, r, a in zip(all_combs, rmses, aics):
print c, r, a
right_columns = [
'prod24h_before', 'Tout24hdiff', 'vWind24hdiff', 'sunRad24hdiff'
]
test_pred = linear_map(test_data,
results[all_combs.index(right_columns)].params,
right_columns)
print "Test RMSE", rmse(test_pred - test_data['prod'])
print "Test MAE", mae(test_pred - test_data['prod'])
print "Test MAPE", mape(test_pred - test_data['prod'], test_data['prod'])
EO3_fc_test = sq.fetch_EO3_midnight_forecast(test_ts[0], test_ts[-1])
EO3_err = EO3_fc_test - test_data['prod']
print "MAE (EO3) = " + str(mae(EO3_err))
print "MAPE (EO3) = " + str(mape(EO3_err, test_data['prod']))
print "RMSE (EO3)= " + str(rmse(EO3_err))
print "ME (EO3)= " + str(np.mean(EO3_err))
