rps_mean = np.zeros(len(lead_times))
rps_mean_clim = np.zeros(len(lead_times))
for j in range(len(lead_times)):
rps_mean[j] = (((prob[:, :, j] - oni_cls_prob)**2)).sum(axis=0).mean()
rps_mean_clim[j] = (((prob_clim[:, :, j] -
oni_cls_prob)**2)).sum(axis=0).mean()
rpss = 1 - rps_mean / rps_mean_clim
print(rpss)
#%% =============================================================================
# All season correlation skill
# =============================================================================
# scores on the full time series
r, p = evaluation_correlation(f'{model_name}_prob', variable_name=f'median')
# score in different decades
r_dec, p_dec = evaluation_decadal_correlation(f'{model_name}_prob',
variable_name=f'median')
# plot correlation skills
ax = plt.figure(figsize=(6.5, 3.5)).gca()
for j in range(n_decades - 1):
plt.plot(lead_times,
r_dec[:, j],
c=decade_color[j],
label=f"Deep Ens. ({decade_name[j]})")
plt.plot(lead_times, r, label="Deep Ens. (1963-2017)", c='k', lw=2)
from ninolearn.learn.evaluation import evaluation_srmse, evaluation_decadal_srmse, evaluation_seasonal_srmse
from ninolearn.plot.evaluation import plot_seasonal_skill
from mlr import pipeline, mlr, pipeline_noise
#
model_name = 'mlr_review'
cross_hindcast(mlr, pipeline, f'{model_name}')
#%% =============================================================================
# All season correlation skill
# =============================================================================
plt.close("all")
# scores on the full time series
r, p = evaluation_correlation(f'{model_name}', variable_name='prediction')
# score in different decades
r_dec, p_dec = evaluation_decadal_correlation(f'{model_name}', variable_name='prediction')
# plot correlation skills
ax = plt.figure(figsize=(6.5,3.5)).gca()
for j in range(n_decades-1):
plt.plot(lead_times, r_dec[:,j], c=decade_color[j], label=f"Deep Ens. ({decade_name[j]})")
plt.plot(lead_times, r, label="Deep Ens. (1962-2017)", c='k', lw=2)
plt.ylim(0,1)
plt.xlim(0.,lead_times[-1])
plt.xlabel('Lead Time [Months]')
plt.ylabel('r')
fig, axs = plt.subplots(6, figsize=(7,7))
lead = [0, 3, 6, 9, 12,15]
for i in range(6):
plot_timeseries_spread(lead[i], axs[i])
if i!=5:
axs[i].set_xticks([])
plt.tight_layout()
#%% =============================================================================
# All season correlation skill
# =============================================================================
# scores on the full time series
r, p = evaluation_correlation(f'{model_name}', variable_name=f'quantile0.5')
# score in different decades
r_dec, p_dec = evaluation_decadal_correlation(f'{model_name}',variable_name=f'quantile0.5')
# plot correlation skills
ax = plt.figure(figsize=(5.,3.)).gca()
for j in range(n_decades-1):
plt.plot(lead_times, r_dec[:,j], c=decade_color[j], label=f"{decade_name[j]}")
plt.plot(lead_times, r, label="1963-2017", c='k', lw=2)
plt.ylim(0,1)
plt.xlim(0.,lead_times[-1])
plt.xlabel('Lead Time [Months]')
plt.ylabel('ACC')
from ninolearn.learn.fit import cross_hindcast, n_decades, lead_times, decade_color, decade_name
from ninolearn.learn.evaluation import evaluation_correlation, evaluation_decadal_correlation, evaluation_seasonal_correlation
from ninolearn.learn.evaluation import evaluation_srmse, evaluation_decadal_srmse, evaluation_seasonal_srmse
from ninolearn.plot.evaluation import plot_seasonal_skill
from mlr import pipeline, mlr
#
cross_hindcast(mlr, pipeline, 'mlr')
#%% =============================================================================
# All season correlation skill
# =============================================================================
plt.close("all")
# scores on the full time series
r, p = evaluation_correlation('mlr', variable_name='prediction')
# score in different decades
r_dec, p_dec = evaluation_decadal_correlation('mlr',
variable_name='prediction')
# plot correlation skills
ax = plt.figure(figsize=(6.5, 3.5)).gca()
for j in range(n_decades - 1):
plt.plot(lead_times,
r_dec[:, j],
c=decade_color[j],
label=f"Deep Ens. ({decade_name[j]})")
plt.plot(lead_times, r, label="Deep Ens. (1962-2017)", c='k', lw=2)
#cross_hindcast(DEM, pipeline_small, f'{model_name}')
start = '1963-01'
end = '2017-12'
reader = data_reader(startdate=start, enddate=end)
oni = reader.read_csv('oni')
data = xr.open_dataset(join(processeddir, f'{model_name}_forecasts.nc'))
# Make a std estimate for comparison purposes
data = data.assign({
'std_estimate':
(('target_season', 'lead'), np.zeros((len(oni), len(lead_times))))
})
r, p = evaluation_correlation(f'{model_name}')
std_seasonal = oni.groupby(oni.index.month).std()
std_estimate = np.zeros((len(r), len(std_seasonal)))
for lead in range(len(lead_times)):
for month in range(len(std_seasonal)):
std_estimate[lead,
month] = std_seasonal.values[month] * (1 -
r[lead]**2)**0.5
data['std_estimate'][month::12, lead] = std_estimate[lead, month]
data.to_netcdf(
join(processeddir, f'{model_name}_forecasts_with_std_estimated.nc'))
#%% =============================================================================
gdnn_model = 'gdnn_ex_pca'
qrnn_model = 'qnn_ex_pca_tanh'
file_gdnn = join(processeddir, f'{gdnn_model}_forecasts_with_std_estimated.nc')
file_qrnn = join(processeddir, f'{qrnn_model}_forecasts_with_std_estimated.nc')
data_gdnn = xr.open_dataset(file_gdnn)
data_qrnn = xr.open_dataset(file_qrnn)
#%% =============================================================================
# All season correlation skill Figure 2
# =============================================================================
fig, ax = plt.subplots(1, 2, figsize=(8.5, 3.5))
r_gdnn, _ = evaluation_correlation(f'{gdnn_model}', variable_name=f'mean')
r_dec_gdnn, _ = evaluation_decadal_correlation(f'{gdnn_model}',
variable_name=f'mean')
r_qrnn, _ = evaluation_correlation(f'{qrnn_model}',
variable_name=f'quantile0.5')
r_dec_qrnn, _ = evaluation_decadal_correlation(f'{qrnn_model}',
variable_name=f'quantile0.5')
for j in range(n_decades - 1):
ax[0].plot(lead_times,
r_dec_gdnn[:, j],
c=decade_color[j],
label=f"{decade_name[j]}")
ax[1].plot(lead_times,
r_dec_qrnn[:, j],
plt.plot(lead_times, nll, label="Deep Ens. (1963-2017)", c='k', lw=2)
plt.ylim(-0.6, 0.6)
plt.xlim(0., lead_times[-1])
plt.xlabel('Lead Time [Months]')
plt.ylabel('NLL')
plt.grid()
plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
plt.tight_layout()
#%% =============================================================================
# All season correlation skill
# =============================================================================
# scores on the full time series
r, p = evaluation_correlation('dem')
# score in different decades
r_dec, p_dec = evaluation_decadal_correlation('dem')
# plot correlation skills
ax = plt.figure(figsize=(6.5, 3.5)).gca()
for j in range(n_decades - 1):
plt.plot(lead_times,
r_dec[:, j],
c=decade_color[j],
label=f"Deep Ens. ({decade_name[j]})")
plt.plot(lead_times, r, label="Deep Ens. (1963-2017)", c='k', lw=2)
plt.ylim(0, 1)
lw=1)
plt.ylim(-0.7, 0.3)
plt.xlim(0., lead_times[-1])
plt.xlabel('Lead Time [Months]')
plt.ylabel('NLL')
plt.grid()
plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
plt.tight_layout()
#%% =============================================================================
# All season correlation skill
# =============================================================================
# scores on the full time series
r, p = evaluation_correlation(f'{model_name}')
# score in different decades
r_dec, p_dec = evaluation_decadal_correlation(f'{model_name}')
# plot correlation skills
ax = plt.figure(figsize=(5, 3)).gca()
for j in range(n_decades - 1):
plt.plot(lead_times,
r_dec[:, j],
c=decade_color[j],
label=f"{decade_name[j]}")
plt.plot(lead_times, r, label="1963-2017", c='k', lw=2)
plt.ylim(0, 1)