def test_pm_metric_skipna(PM_da_initialized_3d, PM_da_control_3d, metric): """Test skipna in compute_perfect_model.""" PM_da_initialized_3d = PM_da_initialized_3d.copy() # manipulating data PM_da_initialized_3d.values[1:3, 1:4, 1:4, 4:6, 4:6] = np.nan base = compute_perfect_model( PM_da_initialized_3d, PM_da_control_3d, metric=metric, skipna=False, dim='init', comparison='m2e', ).mean('member') skipping = compute_perfect_model( PM_da_initialized_3d, PM_da_control_3d, metric=metric, skipna=True, dim='init', comparison='m2e', ).mean('member') assert ((base - skipping) != 0.0).any() assert base.isel(lead=2, x=5, y=5).isnull() assert not skipping.isel(lead=2, x=5, y=5).isnull()
def test_pm_metric_weights_m2x(PM_da_initialized_3d, PM_da_control_3d, comparison, metric): """Test init weights in compute_perfect_model.""" # distribute weights on initializations dim = 'init' base = compute_perfect_model( PM_da_initialized_3d, PM_da_control_3d, dim=dim, metric=metric, comparison=comparison, ) weights = xr.DataArray(np.arange(1, 1 + PM_da_initialized_3d[dim].size), dims=dim) weights = xr.DataArray( np.arange( 1, 1 + PM_da_initialized_3d[dim].size * PM_da_initialized_3d['member'].size, ), dims='init', ) weighted = compute_perfect_model( PM_da_initialized_3d, PM_da_control_3d, dim=dim, comparison=comparison, metric=metric, weights=weights, ) print((base / weighted).mean(['x', 'y'])) # test for difference assert (xs.smape(base, weighted, ['x', 'y']) > 0.01).any()
def test_compute_perfect_model_lead0_lead1( PM_da_initialized_1d, PM_da_initialized_1d_lead0, PM_da_control_1d, comparison, metric, dim, ): """ Checks that metric results are identical for a lead 0 and lead 1 setup. """ res1 = compute_perfect_model( PM_da_initialized_1d, PM_da_control_1d, comparison=comparison, metric=metric, dim=dim, ) res2 = compute_perfect_model( PM_da_initialized_1d_lead0, PM_da_control_1d, comparison=comparison, metric=metric, dim=dim, ) assert (res1.values == res2.values).all()
def test_compute_pm_probabilistic_metric_non_probabilistic_comparison_fails( PM_ds_initialized_1d, PM_ds_control_1d, metric, comparison): with pytest.raises(ValueError, match=f"Probabilistic metric `{metric}` requires"): compute_perfect_model( PM_ds_initialized_1d, PM_ds_control_1d, comparison=comparison, metric=metric, )
def test_compute_pm_probabilistic_metric_non_probabilistic_comparison_fails( pm_da_ds1d, pm_da_control1d, metric, comparison): with pytest.raises(ValueError) as excinfo: compute_perfect_model(pm_da_ds1d, pm_da_control1d, comparison=comparison, metric=metric) assert ( f'Probabilistic metric {metric} cannot work with comparison {comparison}' in str(excinfo.value))
def test_compute_perfect_model_metric_keyerrors(pm_da_ds1d, pm_da_control1d, metric): """ Checks that wrong metric names get caught. """ with pytest.raises(KeyError) as excinfo: compute_perfect_model(pm_da_ds1d, pm_da_control1d, comparison='e2c', metric=metric) assert 'Specify metric from' in str(excinfo.value)
def test_compute_pm_probabilistic_metric_non_probabilistic_comparison_fails( PM_da_initialized_1d, PM_da_control_1d, metric, comparison): with pytest.raises(ValueError) as excinfo: compute_perfect_model( PM_da_initialized_1d, PM_da_control_1d, comparison=comparison, metric=metric, ) assert f"Probabilistic metric `{metric}` requires comparison" in str( excinfo.value)
def test_compute_perfect_model_metric_keyerrors( PM_da_initialized_1d, PM_da_control_1d, metric ): """ Checks that wrong metric names get caught. """ with pytest.raises(KeyError) as excinfo: compute_perfect_model( PM_da_initialized_1d, PM_da_control_1d, comparison="e2c", metric=metric, ) assert "Specify metric from" in str(excinfo.value)
def test_compute_perfect_model_comparison_keyerrors( PM_da_initialized_1d, PM_da_control_1d, comparison ): """ Checks that wrong comparison names get caught. """ with pytest.raises(KeyError) as excinfo: compute_perfect_model( PM_da_initialized_1d, PM_da_control_1d, comparison=comparison, metric="mse", ) assert "Specify comparison from" in str(excinfo.value)
def test_compute_perfect_model_comparison_keyerrors(pm_da_ds1d, pm_da_control1d, comparison): """ Checks that wrong comparison names get caught. """ with pytest.raises(KeyError) as excinfo: compute_perfect_model(pm_da_ds1d, pm_da_control1d, comparison=comparison, metric='mse') assert 'Specify comparison from' in str(excinfo.value)
def test_compute_pm_probabilistic_metric_not_dim_member_warn( pm_da_ds1d, pm_da_control1d, metric, dim): with pytest.warns(UserWarning) as record: compute_perfect_model(pm_da_ds1d, pm_da_control1d, comparison='m2c', metric=metric, dim=dim) expected = (f'Probabilistic metric {metric} requires to be ' f'computed over dimension `dim="member"`. ' f'Set automatically.') assert record[0].message.args[0] == expected
def test_compute_perfect_model_lead0_lead1(pm_da_ds1d, pm_da_ds1d_lead0, pm_da_control1d, comparison, metric): """ Checks that metric results are identical for a lead 0 and lead 1 setup. """ res1 = compute_perfect_model(pm_da_ds1d, pm_da_control1d, comparison=comparison, metric=metric) res2 = compute_perfect_model(pm_da_ds1d_lead0, pm_da_control1d, comparison=comparison, metric=metric) assert (res1.values == res2.values).all()
def test_new_comparison_passed_to_compute(PM_da_initialized_1d, PM_da_control_1d, metric): actual = compute_perfect_model( PM_da_initialized_1d, PM_da_control_1d, comparison=my_m2me_comparison, metric=metric, ) expected = compute_perfect_model(PM_da_initialized_1d, PM_da_control_1d, comparison="m2e", metric="mse") assert (actual - expected).mean() != 0
def test_compute_perfect_model_ds_not_nan(PM_ds_ds, PM_ds_control, metric, comparison): actual = compute_perfect_model(PM_ds_ds, PM_ds_control, metric=metric, comparison=comparison).isnull().any() assert actual == False
def test_compute_after_smooth_goddard_2013(pm_da_ds3d, pm_da_control3d): """Test compute_perfect_model works after smoothings.""" pm_da_control3d = smooth_goddard_2013(pm_da_control3d) pm_da_ds3d = smooth_goddard_2013(pm_da_ds3d) actual = compute_perfect_model(pm_da_ds3d, pm_da_control3d) north_atlantic = actual.sel(lat=slice(40, 50), lon=slice(-30, -20)) assert not north_atlantic.isnull().any()
def test_compute_perfect_model_da1d_not_nan(PM_da_initialized_1d, PM_da_control_1d, comparison, metric, dim): """ Checks that there are no NaNs on perfect model metrics of 1D time series. """ if metric == "contingency": metric_kwargs = { "forecast_category_edges": category_edges, "observation_category_edges": category_edges, "score": "accuracy", } else: metric_kwargs = {} # acc on dim member only is ill defined if dim == "member" and metric in [ "pearson_r", "spearman_r", "pearson_r_p_value", "spearman_r_p_value", "msess_murphy", "bias_slope", "conditional_bias", ]: dim = ["init", "member"] actual = compute_perfect_model(PM_da_initialized_1d, PM_da_control_1d, comparison=comparison, metric=metric, dim=dim, **metric_kwargs) if metric == "contingency": assert not actual.isnull().all() else: assert not actual.isnull().any()
def test_compute_perfect_model_da1d_not_nan_probabilistic( pm_da_ds1d, pm_da_control1d, metric, comparison): """ Checks that there are no NaNs on perfect model probabilistic metrics of 1D time series. """ if 'threshold' in metric: threshold = 10.5 else: threshold = None if metric == 'brier_score': def func(x): return x > 0 else: func = None actual = compute_perfect_model( pm_da_ds1d, pm_da_control1d, comparison=comparison, metric=metric, threshold=threshold, gaussian=True, func=func, dim='member', ) actual = actual.isnull().any() assert not actual
def test_seasonal_resolution_perfect_model(monthly_initialized, monthly_obs): """Tests that seasonal resolution perfect model predictions work.""" seasonal_pm = ( monthly_initialized.rolling(lead=3, center=True).mean().dropna(dim="lead") ) seasonal_pm = seasonal_pm.isel(lead=slice(0, None, 3)) seasonal_obs = monthly_obs.rolling(time=3, center=True).mean().dropna(dim="time") assert compute_perfect_model(seasonal_pm, seasonal_obs).all()
def test_compute_perfect_model_stack_dims_True_and_False_quite_close( pm_da_ds1d, pm_da_control1d): """Test whether dim=['init','member'] for stack_dims=False and dim='member' for stack_dims=True give similar results.""" stack_dims_true = compute_perfect_model( pm_da_ds1d, pm_da_control1d, comparison='m2c', metric='rmse', dim=['init', 'member'], ) stack_dims_false = compute_perfect_model(pm_da_ds1d, pm_da_control1d, comparison='m2c', metric='rmse', dim='member').mean(['init']) # no more than 10% difference assert_allclose(stack_dims_true, stack_dims_false, rtol=0.1, atol=0.03)
def test_compute_after_smooth_goddard_2013(PM_da_initialized_3d_full, PM_da_control_3d_full): """Test compute_perfect_model works after smoothings.""" PM_da_control_3d_full = smooth_goddard_2013(PM_da_control_3d_full) PM_da_initialized_3d_full = smooth_goddard_2013(PM_da_initialized_3d_full) actual = compute_perfect_model(PM_da_initialized_3d_full, PM_da_control_3d_full) north_atlantic = actual.sel(lat=slice(40, 50), lon=slice(-30, -20)) assert not north_atlantic.isnull().any()
def time_compute_perfect_model(self, metric, comparison): """Take time for `compute_perfect_model`.""" dim = 'member' if metric in PROBABILISTIC_METRICS else None ensure_loaded( compute_perfect_model(self.ds, self.control, metric=metric, comparison=comparison, dim=dim))
def test_custom_metric_passed_to_compute(PM_da_initialized_1d, PM_da_control_1d, comparison): """Test custom metric in compute_perfect_model.""" actual = compute_perfect_model( PM_da_initialized_1d, PM_da_control_1d, comparison=comparison, metric=my_mse, ) expected = compute_perfect_model( PM_da_initialized_1d, PM_da_control_1d, comparison=comparison, metric='mse', ) assert_allclose(actual, expected)
def test_compute_pm_probabilistic_metric_not_dim_member_warn( PM_da_initialized_1d, PM_da_control_1d, metric, dim ): with pytest.warns(UserWarning) as record: compute_perfect_model( PM_da_initialized_1d, PM_da_control_1d, comparison='m2c', metric=metric, dim=dim, ) expected = ( f'Probabilistic metric {metric} requires to be ' f'computed over dimension `dim="member"`. ' f'Set automatically.' ) # get second warning here assert record[1].message.args[0] == expected
def peakmem_compute_perfect_model(self, metric, comparison): """Take memory peak for `compute_perfect_model`.""" dim = "member" if metric in PROBABILISTIC_METRICS else None ensure_loaded( compute_perfect_model(self.ds, self.control, metric=metric, comparison=comparison, dim=dim))
def test_compute_perfect_model_da1d_not_nan(pm_da_ds1d, pm_da_control1d, comparison, metric): """ Checks that there are no NaNs on perfect model metrics of 1D time series. """ actual = (compute_perfect_model(pm_da_ds1d, pm_da_control1d, comparison=comparison, metric=metric).isnull().any()) assert not actual
def test_compute_perfect_model_dim_over_member(pm_da_ds1d, pm_da_control1d, comparison): """Test deterministic metric calc skill over member dim.""" actual = compute_perfect_model(pm_da_ds1d, pm_da_control1d, comparison=comparison, metric='rmse', dim='member') assert 'init' in actual.dims assert not actual.isnull().any()
def test_compute_perfect_model_different_dims_quite_close( PM_da_initialized_1d, PM_da_control_1d): """Test whether dim=['init','member'] and dim='member' results.""" stack_dims_true = compute_perfect_model( PM_da_initialized_1d, PM_da_control_1d, comparison='m2c', metric='rmse', dim=['init', 'member'], ) stack_dims_false = compute_perfect_model( PM_da_initialized_1d, PM_da_control_1d, comparison='m2c', metric='rmse', dim='member', ).mean(['init']) # no more than 10% difference assert_allclose(stack_dims_true, stack_dims_false, rtol=0.1, atol=0.03)
def test_compute_perfect_model_ds_not_nan(PM_ds_ds3d, PM_ds_control3d, metric, comparison): """ Checks that there are no NaNs on perfect model comparison for Dataset. """ actual = compute_perfect_model(PM_ds_ds3d, PM_ds_control3d, metric=metric, comparison=comparison).isnull().any() for var in actual.data_vars: assert not actual[var]
def test_pm_crpss_orientation(pm_da_ds1d, pm_da_control1d): """ Checks that CRPSS in PM as skill score > 0. """ actual = compute_perfect_model(pm_da_ds1d, pm_da_control1d, comparison='m2m', metric='crpss', dim='member') if 'init' in actual.coords: actual = actual.mean('init') assert not (actual.isel(lead=[0, 1]) < 0).any()
def Sef2018_Fig1_Different_PH_Definitions(ds, control, unit='PgC/yr', sig=95, bootstrap=1000): # from esmtools.prediction import predictability_horizon from PMMPIESM.plot import _set_integer_xaxis rsig = (100 - sig)/100 _control = control _ds = ds ss = compute_perfect_model( _ds, _control, metric='rmse', comparison='m2e') ss['lead'] = np.arange(1, ss.lead.size + 1) # ss.name = 'every' ss_boot = bootstrap_perfect_model(_ds, _control, metric='rmse', comparison='m2e', sig=sig, bootstrap=bootstrap) ss_p = ss_boot.sel(kind='uninit', results='p') ss_ci_high = ss_boot.sel(kind='uninit', results='low_ci') ph_Spring_2019 = predictability_horizon( ss.where(ss_p < rsig)).values b_m2e, ph_Sef_2018, c_m2e = fit_ph_int(ss.to_series()) print('ph_Sef_2018', ph_Sef_2018) print('ph_Spring_2019', int(ph_Spring_2019)) fig, ax = plt.subplots(figsize=(10, 4)) std = _control.std('time').values every_color = 'mediumorchid' ss.name = 'skill' ss.to_dataframe().plot(ax=ax, label='skill', color='k', marker='o') t_fit = np.arange(0, _ds.lead.size) ax.plot(t_fit[1:], func(t_fit, b_m2e, ph_Sef_2018, c_m2e)[1:], linewidth=3, color=every_color, label='Sef 2018 breakpoint fit') ax.axvline(x=ph_Sef_2018, linestyle='-.', color=every_color, label='PH Sef 2018') ax.axhline(y=std, ls='--', c='k', alpha=.3, label='std control') ax.axhline(y=ss_ci_high.mean('lead'), ls=':', c='royalblue', label='Bootstrapped high CI') ax.axvline(x=ph_Spring_2019, ls='-.', c='royalblue', label='PH Spring 2019') ax.set_xlabel('Lead Time [time]') ax.set_ylabel('RMSE [' + unit + ']') ax.set_ylim([0, ss.max() * 1.1]) ax.set_xlim([0, 10]) _set_integer_xaxis(ax) ax.legend(frameon=False, ncol=2) ax.set_xticks(range(1, 11)) ax.set_title( ' Global oceanic CO$_2$ flux: Differences in definitions of Predictability Horizon') if savefig: plt.tight_layout() plt.savefig('FigureSI_Differences_PH_definition')