def test_hindcast_metric_weights(hind_da_initialized_3d, reconstruction_da_3d, comparison, metric): """Test time=lead weights in compute_hindcast.""" dim = 'init' base = compute_hindcast( hind_da_initialized_3d, reconstruction_da_3d, dim=dim, metric=metric, comparison=comparison, ) weights = xr.DataArray( np.arange( 1, 1 + hind_da_initialized_3d[dim].size - hind_da_initialized_3d.lead.size, ), dims='time', ) weighted = compute_hindcast( hind_da_initialized_3d, reconstruction_da_3d, dim=dim, comparison=comparison, metric=metric, weights=weights, ) # test for difference assert ((base / weighted).mean(['nlon', 'nlat']) != 1).any()
def test_hindcast_metric_weights_x2r(hind_da_initialized_3d, reconstruction_da_3d, comparison, metric): """Test init weights in compute_hindcast.""" dim = 'init' base = compute_hindcast( hind_da_initialized_3d, reconstruction_da_3d, dim=dim, metric=metric, comparison=comparison, ) weights = xr.DataArray(np.arange(1, 1 + hind_da_initialized_3d[dim].size), dims=dim) weights = xr.DataArray( np.arange( 1, 1 + hind_da_initialized_3d[dim].size * hind_da_initialized_3d['member'].size, ), dims='init', ) weighted = compute_hindcast( hind_da_initialized_3d, reconstruction_da_3d, dim=dim, comparison=comparison, metric=metric, weights=weights, ) print((base / weighted).mean(['nlon', 'nlat'])) # test for difference assert (xs.smape(base, weighted, ['nlat', 'nlon']) > 0.01).any()
def test_compute_hindcast_less_e2r(initialized_da, reconstruction_da): """Test raise KeyError for LESS e2r, because needs member.""" with pytest.raises(KeyError) as excinfo: compute_hindcast( initialized_da, reconstruction_da, metric='less', comparison='e2r' ) assert 'LESS requires member dimension' in str(excinfo.value)
def test_compute_hindcast_metric_keyerrors(initialized_ds, reconstruction_ds, metric): """ Checks that wrong metric names get caught. """ with pytest.raises(KeyError) as excinfo: compute_hindcast( initialized_ds, reconstruction_ds, comparison='e2r', metric=metric ) assert 'Specify metric from' in str(excinfo.value)
def test_same_verifs_raises_error_when_not_possible( hind_ds_initialized_1d_cftime, reconstruction_ds_1d_cftime): """Tests that appropriate error is raised when a common set of verification dates cannot be found with the supplied initializations.""" hind = hind_ds_initialized_1d_cftime.isel(lead=slice(0, 3), init=[1, 3, 5, 7, 9]) with pytest.raises(CoordinateError): compute_hindcast(hind, reconstruction_ds_1d_cftime, alignment="same_verifs")
def test_compute_hindcast_comparison_keyerrors(initialized_ds, reconstruction_ds, comparison): """ Checks that wrong comparison names get caught. """ with pytest.raises(KeyError) as excinfo: compute_hindcast(initialized_ds, reconstruction_ds, comparison=comparison, metric='mse') assert 'Specify comparison from' in str(excinfo.value)
def test_compute_hindcast_probabilistic_metric_e2o_fails( hind_da_initialized_1d, observations_da_1d, metric): metric = METRIC_ALIASES.get(metric, metric) with pytest.raises(ValueError) as excinfo: compute_hindcast( hind_da_initialized_1d, observations_da_1d, comparison="e2o", metric=metric, dim="member", ) assert f"Probabilistic metric `{metric}` requires" in str(excinfo.value)
def test_log_compute_hindcast(hind_ds_initialized_1d_cftime, reconstruction_ds_1d_cftime, caplog): """Tests that logging works for compute_hindcast.""" LOG_STRINGS = ["lead", "inits", "verifs"] with caplog.at_level(logging.INFO): compute_hindcast(hind_ds_initialized_1d_cftime, reconstruction_ds_1d_cftime) for i, record in enumerate(caplog.record_tuples): # Skip header information. if i >= 2: print(record) assert all(x in record[2] for x in LOG_STRINGS)
def test_compute_hindcast_metric_keyerrors(hind_ds_initialized_1d, reconstruction_ds_1d, metric): """ Checks that wrong metric names get caught. """ with pytest.raises(KeyError) as excinfo: compute_hindcast( hind_ds_initialized_1d, reconstruction_ds_1d, comparison="e2o", metric=metric, ) assert "Specify metric from" in str(excinfo.value)
def test_compute_hindcast_probabilistic_metric_e2r_fails( initialized_da, observations_da, metric): metric = METRIC_ALIASES.get(metric, metric) with pytest.raises(ValueError) as excinfo: compute_hindcast( initialized_da, observations_da, comparison='e2r', metric=metric, dim='member', ) assert f'Probabilistic metric `{metric}` requires comparison `m2r`.' in str( excinfo.value)
def test_compute_hindcast_probabilistic_metric_not_dim_member_warn( initialized_da, observations_da, metric, dim): metric = METRIC_ALIASES.get(metric, metric) with pytest.warns(UserWarning) as record: compute_hindcast(initialized_da, observations_da, comparison='m2r', metric=metric, dim=dim) expected = (f'Probabilistic metric {metric} requires to be ' f'computed over dimension `dim="member"`. ' f'Set automatically.') assert record[0].message.args[0] == expected
def test_compute_hindcast_lead0_lead1( initialized_ds, initialized_ds_lead0, reconstruction_ds, metric, comparison ): """ Checks that compute hindcast returns the same results with a lead-0 and lead-1 framework. """ res1 = compute_hindcast( initialized_ds, reconstruction_ds, metric=metric, comparison=comparison ) res2 = compute_hindcast( initialized_ds_lead0, reconstruction_ds, metric=metric, comparison=comparison ) assert (res1.SST.values == res2.SST.values).all()
def test_same_verifs_verification_dates(hind_ds_initialized_1d_cftime, reconstruction_ds_1d_cftime, caplog): """Tests that verifs are identical at all leads for `same_verifs` alignment.""" with caplog.at_level(logging.INFO): compute_hindcast( hind_ds_initialized_1d_cftime, reconstruction_ds_1d_cftime, alignment="same_verifs", ) for i, record in enumerate(caplog.record_tuples): if i >= 2: print(record) assert "verifs: 1964-01-01 00:00:00-2017-01-01 00:00:00" in record[ 2]
def test_yearly_resolution_hindcast(monthly_initialized, monthly_obs): """Tests that yearly resolution hindcast predictions work.""" yearly_hindcast = (monthly_initialized.resample(init='YS').mean().isel( lead=slice(None, 2))) yearly_obs = monthly_obs.resample(time='YS').mean() yearly_hindcast.lead.attrs['units'] = 'years' assert compute_hindcast(yearly_hindcast, yearly_obs).all()
def test_compute_hindcast( hind_ds_initialized_1d, reconstruction_ds_1d, metric, comparison ): """ Checks that compute hindcast works without breaking. """ if metric == "contingency": metric_kwargs = { "forecast_category_edges": category_edges, "observation_category_edges": category_edges, "score": "accuracy", } else: metric_kwargs = {} res = ( compute_hindcast( hind_ds_initialized_1d, reconstruction_ds_1d, metric=metric, comparison=comparison, **metric_kwargs ) .isnull() .any() ) for var in res.data_vars: assert not res[var]
def test_compute_hindcast_CESM_3D_keep_coords( hind_da_initialized_3d, reconstruction_da_3d ): """Test that no coords are lost in compute_hindcast with the CESM sample data.""" s = compute_hindcast(hind_da_initialized_3d, reconstruction_da_3d) for c in hind_da_initialized_3d.drop("init").coords: assert c in s.coords
def test_compute_hindcast_less_m2r(initialized_da, reconstruction_da): """Test LESS m2r runs through.""" actual = (compute_hindcast(initialized_da, reconstruction_da, metric='less', comparison='m2r').isnull().any()) assert not actual
def test_compute_hindcast_probabilistic(initialized_da, observations_da, metric, comparison): """ Checks that compute hindcast works without breaking. """ if 'threshold' in metric: threshold = 0.5 # initialized_da.mean() else: threshold = None if metric == 'brier_score': def func(x): return x > 0.5 else: func = None res = compute_hindcast( initialized_da, observations_da, metric=metric, comparison=comparison, threshold=threshold, func=func, dim='member', ) # mean init because skill has still coords for init lead if 'init' in res.coords: res = res.mean('init') res = res.isnull().any() assert not res
def test_eff_spearman_p_greater_or_equal_to_normal_p_hind_da_initialized_1d( hind_da_initialized_1d, reconstruction_da_1d, comparison): """Tests that the Pearson effective p value (more conservative) is greater than or equal to the standard p value.""" normal_p = compute_hindcast( hind_da_initialized_1d, reconstruction_da_1d, metric="spearman_r_p_value", comparison=comparison, ) eff_p = compute_hindcast( hind_da_initialized_1d, reconstruction_da_1d, metric="spearman_r_eff_p_value", comparison=comparison, ) assert (normal_p <= eff_p).all()
def test_seasonal_resolution_hindcast(monthly_initialized, monthly_obs): """Tests that seasonal resolution hindcast predictions work.""" seasonal_hindcast = ( monthly_initialized.rolling(lead=3, center=True).mean().dropna(dim="lead") ) seasonal_hindcast = seasonal_hindcast.isel(lead=slice(0, None, 3)) seasonal_obs = monthly_obs.rolling(time=3, center=True).mean().dropna(dim="time") seasonal_hindcast.lead.attrs["units"] = "seasons" assert compute_hindcast(seasonal_hindcast, seasonal_obs).all()
def test_compute_hindcast_less_m2o(hind_da_initialized_1d, reconstruction_da_1d): """Test LESS m2o runs through""" actual = (compute_hindcast( hind_da_initialized_1d, reconstruction_da_1d, metric='less', comparison='m2o', ).isnull().any()) assert not actual
def test_maximize_alignment_verifs(hind_ds_initialized_1d_cftime, reconstruction_ds_1d_cftime, caplog): """Tests that appropriate verifs are selected for `maximize` alignment.""" with caplog.at_level(logging.INFO): compute_hindcast( hind_ds_initialized_1d_cftime, reconstruction_ds_1d_cftime, alignment="maximize", ) # Add dummy values for the first two lines since they are just metadata. for i, record in zip( np.concatenate( ([0, 0], hind_ds_initialized_1d_cftime.lead.values)), caplog.record_tuples, ): if i >= 1: print(record) assert (f"verifs: {1955+i}-01-01 00:00:00-2017-01-01 00:00:00" in record[2])
def test_compute_hindcast(initialized_ds, reconstruction_ds, metric, comparison): """ Checks that compute reference works without breaking. """ res = (compute_hindcast(initialized_ds, reconstruction_ds, metric=metric, comparison=comparison).isnull().any()) for var in res.data_vars: assert not res[var]
def time_compute_hindcast(self, metric, comparison): """Take time for `compute_hindcast`.""" dim = "member" if metric in PROBABILISTIC_METRICS else "init" ensure_loaded( compute_hindcast( self.hind, self.observations, metric=metric, comparison=comparison, dim=dim, ))
def peakmem_compute_hindcast(self, metric, comparison): """Take memory peak for `compute_hindcast`.""" dim = 'member' if metric in PROBABILISTIC_METRICS else 'init' ensure_loaded( compute_hindcast( self.hind, self.observations, metric=metric, comparison=comparison, dim=dim, ))
def test_compute_hindcast_probabilistic_metric_not_dim_member_warn( hind_da_initialized_1d, observations_da_1d, metric, dim ): metric = METRIC_ALIASES.get(metric, metric) with pytest.warns(UserWarning) as record: compute_hindcast( hind_da_initialized_1d, observations_da_1d, comparison='m2o', metric=metric, dim=dim, ) expected = ( f'Probabilistic metric {metric} requires to be ' f'computed over dimension `dim="member"`. ' f'Set automatically.' ) # Set this to the third message since the first two are about converting the integer # time to annual `cftime`. assert record[0].message.args[0] == expected
def test_eff_sample_size_smaller_than_n_hind_da_initialized_1d( hind_da_initialized_1d, reconstruction_da_1d, comparison): """Tests that effective sample size is less than or equal to the actual sample size of the data.""" N = hind_da_initialized_1d.mean("member").count("init") eff_N = compute_hindcast( hind_da_initialized_1d, reconstruction_da_1d, metric="eff_n", ) assert (eff_N <= N).all()
def test_compute_hindcast_lead0_lead1(hind_ds_initialized_1d, hind_ds_initialized_1d_lead0, reconstruction_ds_1d): """ Checks that compute hindcast returns the same results with a lead-0 and lead-1 framework. """ res1 = compute_hindcast( hind_ds_initialized_1d, reconstruction_ds_1d, metric='rmse', comparison='e2o', ) res2 = compute_hindcast( hind_ds_initialized_1d_lead0, reconstruction_ds_1d, metric='rmse', comparison='e2o', ) assert (res1.SST.values == res2.SST.values).all()
def test_same_verifs_initializations(hind_ds_initialized_1d_cftime, reconstruction_ds_1d_cftime, caplog): """Tests that appropriate verifs are being used at each lead for `same_inits` alignment.""" with caplog.at_level(logging.INFO): FIRST_INIT, LAST_INIT = 1964, 2017 compute_hindcast( hind_ds_initialized_1d_cftime, reconstruction_ds_1d_cftime, alignment="same_verifs", ) nleads = hind_ds_initialized_1d_cftime["lead"].size for i, record in zip( np.arange(nleads + 2), caplog.record_tuples, ): if i >= 2: print(record) assert ( f"inits: {FIRST_INIT-i}-01-01 00:00:00-{LAST_INIT-i}-01-01 00:00:00" in record[2])
def test_hindcast_crpss_orientation(initialized_da, observations_da): """ Checks that CRPSS hindcast as skill score > 0. """ actual = compute_hindcast(initialized_da, observations_da, comparison='m2r', metric='crpss', dim='member') if 'init' in actual.coords: actual = actual.mean('init') assert not (actual.isel(lead=[0, 1]) < 0).any()