def test_hindcast_metric_weights(hind_da_initialized_3d, reconstruction_da_3d,
comparison, metric):
"""Test time=lead weights in compute_hindcast."""
dim = 'init'
base = compute_hindcast(
hind_da_initialized_3d,
reconstruction_da_3d,
dim=dim,
metric=metric,
comparison=comparison,
)
weights = xr.DataArray(
np.arange(
1,
1 + hind_da_initialized_3d[dim].size -
hind_da_initialized_3d.lead.size,
),
dims='time',
)
weighted = compute_hindcast(
hind_da_initialized_3d,
reconstruction_da_3d,
dim=dim,
comparison=comparison,
metric=metric,
weights=weights,
)
# test for difference
assert ((base / weighted).mean(['nlon', 'nlat']) != 1).any()
def test_hindcast_metric_weights_x2r(hind_da_initialized_3d,
reconstruction_da_3d, comparison, metric):
"""Test init weights in compute_hindcast."""
dim = 'init'
base = compute_hindcast(
hind_da_initialized_3d,
reconstruction_da_3d,
dim=dim,
metric=metric,
comparison=comparison,
)
weights = xr.DataArray(np.arange(1, 1 + hind_da_initialized_3d[dim].size),
dims=dim)
weights = xr.DataArray(
np.arange(
1,
1 + hind_da_initialized_3d[dim].size *
hind_da_initialized_3d['member'].size,
),
dims='init',
)
weighted = compute_hindcast(
hind_da_initialized_3d,
reconstruction_da_3d,
dim=dim,
comparison=comparison,
metric=metric,
weights=weights,
)
print((base / weighted).mean(['nlon', 'nlat']))
# test for difference
assert (xs.smape(base, weighted, ['nlat', 'nlon']) > 0.01).any()
def test_compute_hindcast_less_e2r(initialized_da, reconstruction_da):
"""Test raise KeyError for LESS e2r, because needs member."""
with pytest.raises(KeyError) as excinfo:
compute_hindcast(
initialized_da, reconstruction_da, metric='less', comparison='e2r'
)
assert 'LESS requires member dimension' in str(excinfo.value)
def test_compute_hindcast_metric_keyerrors(initialized_ds, reconstruction_ds, metric):
"""
Checks that wrong metric names get caught.
"""
with pytest.raises(KeyError) as excinfo:
compute_hindcast(
initialized_ds, reconstruction_ds, comparison='e2r', metric=metric
)
assert 'Specify metric from' in str(excinfo.value)
def test_same_verifs_raises_error_when_not_possible(
hind_ds_initialized_1d_cftime, reconstruction_ds_1d_cftime):
"""Tests that appropriate error is raised when a common set of verification dates
cannot be found with the supplied initializations."""
hind = hind_ds_initialized_1d_cftime.isel(lead=slice(0, 3),
init=[1, 3, 5, 7, 9])
with pytest.raises(CoordinateError):
compute_hindcast(hind,
reconstruction_ds_1d_cftime,
alignment="same_verifs")
def test_compute_hindcast_comparison_keyerrors(initialized_ds,
reconstruction_ds, comparison):
"""
Checks that wrong comparison names get caught.
"""
with pytest.raises(KeyError) as excinfo:
compute_hindcast(initialized_ds,
reconstruction_ds,
comparison=comparison,
metric='mse')
assert 'Specify comparison from' in str(excinfo.value)
def test_compute_hindcast_probabilistic_metric_e2o_fails(
hind_da_initialized_1d, observations_da_1d, metric):
metric = METRIC_ALIASES.get(metric, metric)
with pytest.raises(ValueError) as excinfo:
compute_hindcast(
hind_da_initialized_1d,
observations_da_1d,
comparison="e2o",
metric=metric,
dim="member",
)
assert f"Probabilistic metric `{metric}` requires" in str(excinfo.value)
def test_log_compute_hindcast(hind_ds_initialized_1d_cftime,
reconstruction_ds_1d_cftime, caplog):
"""Tests that logging works for compute_hindcast."""
LOG_STRINGS = ["lead", "inits", "verifs"]
with caplog.at_level(logging.INFO):
compute_hindcast(hind_ds_initialized_1d_cftime,
reconstruction_ds_1d_cftime)
for i, record in enumerate(caplog.record_tuples):
# Skip header information.
if i >= 2:
print(record)
assert all(x in record[2] for x in LOG_STRINGS)
def test_compute_hindcast_metric_keyerrors(hind_ds_initialized_1d,
reconstruction_ds_1d, metric):
"""
Checks that wrong metric names get caught.
"""
with pytest.raises(KeyError) as excinfo:
compute_hindcast(
hind_ds_initialized_1d,
reconstruction_ds_1d,
comparison="e2o",
metric=metric,
)
assert "Specify metric from" in str(excinfo.value)
def test_compute_hindcast_probabilistic_metric_e2r_fails(
initialized_da, observations_da, metric):
metric = METRIC_ALIASES.get(metric, metric)
with pytest.raises(ValueError) as excinfo:
compute_hindcast(
initialized_da,
observations_da,
comparison='e2r',
metric=metric,
dim='member',
)
assert f'Probabilistic metric `{metric}` requires comparison `m2r`.' in str(
excinfo.value)
def test_compute_hindcast_probabilistic_metric_not_dim_member_warn(
initialized_da, observations_da, metric, dim):
metric = METRIC_ALIASES.get(metric, metric)
with pytest.warns(UserWarning) as record:
compute_hindcast(initialized_da,
observations_da,
comparison='m2r',
metric=metric,
dim=dim)
expected = (f'Probabilistic metric {metric} requires to be '
f'computed over dimension `dim="member"`. '
f'Set automatically.')
assert record[0].message.args[0] == expected
def test_compute_hindcast_lead0_lead1(
initialized_ds, initialized_ds_lead0, reconstruction_ds, metric, comparison
):
"""
Checks that compute hindcast returns the same results with a lead-0 and lead-1
framework.
"""
res1 = compute_hindcast(
initialized_ds, reconstruction_ds, metric=metric, comparison=comparison
)
res2 = compute_hindcast(
initialized_ds_lead0, reconstruction_ds, metric=metric, comparison=comparison
)
assert (res1.SST.values == res2.SST.values).all()
def test_same_verifs_verification_dates(hind_ds_initialized_1d_cftime,
reconstruction_ds_1d_cftime, caplog):
"""Tests that verifs are identical at all leads for `same_verifs` alignment."""
with caplog.at_level(logging.INFO):
compute_hindcast(
hind_ds_initialized_1d_cftime,
reconstruction_ds_1d_cftime,
alignment="same_verifs",
)
for i, record in enumerate(caplog.record_tuples):
if i >= 2:
print(record)
assert "verifs: 1964-01-01 00:00:00-2017-01-01 00:00:00" in record[
2]
def test_yearly_resolution_hindcast(monthly_initialized, monthly_obs):
"""Tests that yearly resolution hindcast predictions work."""
yearly_hindcast = (monthly_initialized.resample(init='YS').mean().isel(
lead=slice(None, 2)))
yearly_obs = monthly_obs.resample(time='YS').mean()
yearly_hindcast.lead.attrs['units'] = 'years'
assert compute_hindcast(yearly_hindcast, yearly_obs).all()
def test_compute_hindcast(
hind_ds_initialized_1d, reconstruction_ds_1d, metric, comparison
):
"""
Checks that compute hindcast works without breaking.
"""
if metric == "contingency":
metric_kwargs = {
"forecast_category_edges": category_edges,
"observation_category_edges": category_edges,
"score": "accuracy",
}
else:
metric_kwargs = {}
res = (
compute_hindcast(
hind_ds_initialized_1d,
reconstruction_ds_1d,
metric=metric,
comparison=comparison,
**metric_kwargs
)
.isnull()
.any()
)
for var in res.data_vars:
assert not res[var]
def test_compute_hindcast_CESM_3D_keep_coords(
hind_da_initialized_3d, reconstruction_da_3d
):
"""Test that no coords are lost in compute_hindcast with the CESM sample data."""
s = compute_hindcast(hind_da_initialized_3d, reconstruction_da_3d)
for c in hind_da_initialized_3d.drop("init").coords:
assert c in s.coords
def test_compute_hindcast_less_m2r(initialized_da, reconstruction_da):
"""Test LESS m2r runs through."""
actual = (compute_hindcast(initialized_da,
reconstruction_da,
metric='less',
comparison='m2r').isnull().any())
assert not actual
def test_compute_hindcast_probabilistic(initialized_da, observations_da,
metric, comparison):
"""
Checks that compute hindcast works without breaking.
"""
if 'threshold' in metric:
threshold = 0.5 # initialized_da.mean()
else:
threshold = None
if metric == 'brier_score':
def func(x):
return x > 0.5
else:
func = None
res = compute_hindcast(
initialized_da,
observations_da,
metric=metric,
comparison=comparison,
threshold=threshold,
func=func,
dim='member',
)
# mean init because skill has still coords for init lead
if 'init' in res.coords:
res = res.mean('init')
res = res.isnull().any()
assert not res
def test_eff_spearman_p_greater_or_equal_to_normal_p_hind_da_initialized_1d(
hind_da_initialized_1d, reconstruction_da_1d, comparison):
"""Tests that the Pearson effective p value (more conservative) is greater than or
equal to the standard p value."""
normal_p = compute_hindcast(
hind_da_initialized_1d,
reconstruction_da_1d,
metric="spearman_r_p_value",
comparison=comparison,
)
eff_p = compute_hindcast(
hind_da_initialized_1d,
reconstruction_da_1d,
metric="spearman_r_eff_p_value",
comparison=comparison,
)
assert (normal_p <= eff_p).all()
def test_seasonal_resolution_hindcast(monthly_initialized, monthly_obs):
"""Tests that seasonal resolution hindcast predictions work."""
seasonal_hindcast = (
monthly_initialized.rolling(lead=3, center=True).mean().dropna(dim="lead")
)
seasonal_hindcast = seasonal_hindcast.isel(lead=slice(0, None, 3))
seasonal_obs = monthly_obs.rolling(time=3, center=True).mean().dropna(dim="time")
seasonal_hindcast.lead.attrs["units"] = "seasons"
assert compute_hindcast(seasonal_hindcast, seasonal_obs).all()
def test_compute_hindcast_less_m2o(hind_da_initialized_1d,
reconstruction_da_1d):
"""Test LESS m2o runs through"""
actual = (compute_hindcast(
hind_da_initialized_1d,
reconstruction_da_1d,
metric='less',
comparison='m2o',
).isnull().any())
assert not actual
def test_maximize_alignment_verifs(hind_ds_initialized_1d_cftime,
reconstruction_ds_1d_cftime, caplog):
"""Tests that appropriate verifs are selected for `maximize` alignment."""
with caplog.at_level(logging.INFO):
compute_hindcast(
hind_ds_initialized_1d_cftime,
reconstruction_ds_1d_cftime,
alignment="maximize",
)
# Add dummy values for the first two lines since they are just metadata.
for i, record in zip(
np.concatenate(
([0, 0], hind_ds_initialized_1d_cftime.lead.values)),
caplog.record_tuples,
):
if i >= 1:
print(record)
assert (f"verifs: {1955+i}-01-01 00:00:00-2017-01-01 00:00:00"
in record[2])
def test_compute_hindcast(initialized_ds, reconstruction_ds, metric,
comparison):
"""
Checks that compute reference works without breaking.
"""
res = (compute_hindcast(initialized_ds,
reconstruction_ds,
metric=metric,
comparison=comparison).isnull().any())
for var in res.data_vars:
assert not res[var]
def time_compute_hindcast(self, metric, comparison):
"""Take time for `compute_hindcast`."""
dim = "member" if metric in PROBABILISTIC_METRICS else "init"
ensure_loaded(
compute_hindcast(
self.hind,
self.observations,
metric=metric,
comparison=comparison,
dim=dim,
))
def peakmem_compute_hindcast(self, metric, comparison):
"""Take memory peak for `compute_hindcast`."""
dim = 'member' if metric in PROBABILISTIC_METRICS else 'init'
ensure_loaded(
compute_hindcast(
self.hind,
self.observations,
metric=metric,
comparison=comparison,
dim=dim,
))
def test_compute_hindcast_probabilistic_metric_not_dim_member_warn(
hind_da_initialized_1d, observations_da_1d, metric, dim
):
metric = METRIC_ALIASES.get(metric, metric)
with pytest.warns(UserWarning) as record:
compute_hindcast(
hind_da_initialized_1d,
observations_da_1d,
comparison='m2o',
metric=metric,
dim=dim,
)
expected = (
f'Probabilistic metric {metric} requires to be '
f'computed over dimension `dim="member"`. '
f'Set automatically.'
)
# Set this to the third message since the first two are about converting the integer
# time to annual `cftime`.
assert record[0].message.args[0] == expected
def test_eff_sample_size_smaller_than_n_hind_da_initialized_1d(
hind_da_initialized_1d, reconstruction_da_1d, comparison):
"""Tests that effective sample size is less than or equal to the actual sample size
of the data."""
N = hind_da_initialized_1d.mean("member").count("init")
eff_N = compute_hindcast(
hind_da_initialized_1d,
reconstruction_da_1d,
metric="eff_n",
)
assert (eff_N <= N).all()
def test_compute_hindcast_lead0_lead1(hind_ds_initialized_1d,
hind_ds_initialized_1d_lead0,
reconstruction_ds_1d):
"""
Checks that compute hindcast returns the same results with a lead-0 and lead-1
framework.
"""
res1 = compute_hindcast(
hind_ds_initialized_1d,
reconstruction_ds_1d,
metric='rmse',
comparison='e2o',
)
res2 = compute_hindcast(
hind_ds_initialized_1d_lead0,
reconstruction_ds_1d,
metric='rmse',
comparison='e2o',
)
assert (res1.SST.values == res2.SST.values).all()
def test_same_verifs_initializations(hind_ds_initialized_1d_cftime,
reconstruction_ds_1d_cftime, caplog):
"""Tests that appropriate verifs are being used at each lead for `same_inits`
alignment."""
with caplog.at_level(logging.INFO):
FIRST_INIT, LAST_INIT = 1964, 2017
compute_hindcast(
hind_ds_initialized_1d_cftime,
reconstruction_ds_1d_cftime,
alignment="same_verifs",
)
nleads = hind_ds_initialized_1d_cftime["lead"].size
for i, record in zip(
np.arange(nleads + 2),
caplog.record_tuples,
):
if i >= 2:
print(record)
assert (
f"inits: {FIRST_INIT-i}-01-01 00:00:00-{LAST_INIT-i}-01-01 00:00:00"
in record[2])
def test_hindcast_crpss_orientation(initialized_da, observations_da):
"""
Checks that CRPSS hindcast as skill score > 0.
"""
actual = compute_hindcast(initialized_da,
observations_da,
comparison='m2r',
metric='crpss',
dim='member')
if 'init' in actual.coords:
actual = actual.mean('init')
assert not (actual.isel(lead=[0, 1]) < 0).any()
