def test_bootstrap_hindcast_alignment(
hind_da_initialized_1d, hist_da_uninitialized_1d, observations_da_1d, alignment,
):
"""Test bootstrap_hindcast for all alginments when resampling member."""
bootstrap_hindcast(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
iterations=ITERATIONS,
comparison='e2o',
metric='mse',
resample_dim='member',
alignment=alignment,
)
def test_bootstrap_hindcast_resample_dim(
hind_da_initialized_1d, hist_da_uninitialized_1d, observations_da_1d, resample_dim,
):
"""Test bootstrap_hindcast when resampling member or init and alignment
same_inits."""
bootstrap_hindcast(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
iterations=ITERATIONS,
comparison='e2o',
metric='mse',
resample_dim=resample_dim,
alignment='same_inits',
)
def test_bootstrap_hindcast_lazy(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
chunk,
comparison,
):
"""Test bootstrap_hindcast works lazily."""
if chunk:
hind_da_initialized_1d = hind_da_initialized_1d.chunk({"lead": 2})
hist_da_uninitialized_1d = hist_da_uninitialized_1d.chunk({"time": -1})
observations_da_1d = observations_da_1d.chunk({"time": -1})
else:
hind_da_initialized_1d = hind_da_initialized_1d.compute()
hist_da_uninitialized_1d = hist_da_uninitialized_1d.compute()
observations_da_1d = observations_da_1d.compute()
s = bootstrap_hindcast(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
iterations=ITERATIONS,
comparison=comparison,
metric="mse",
)
assert dask.is_dask_collection(s) == chunk
def test_bootstrap_hindcast_raises_error(hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d):
"""Test that error is raised when user tries to resample over init and align over
same_verifs."""
with pytest.raises(KeywordError):
bootstrap_hindcast(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
iterations=ITERATIONS,
comparison="e2o",
metric="mse",
resample_dim="init",
alignment="same_verifs",
)
def peakmem_bootstrap_hindcast(self, metric, comparison):
"""Take memory peak for `bootstrap_hindcast`."""
dim = 'member' if metric in PROBABILISTIC_METRICS else 'init'
ensure_loaded(
bootstrap_hindcast(
self.hind,
self.uninit,
self.observations,
metric=metric,
comparison=comparison,
iterations=self.iterations,
dim=dim,
))
def time_bootstrap_hindcast(self, metric, comparison):
"""Take time for `bootstrap_hindcast`."""
dim = "member" if metric in PROBABILISTIC_METRICS else "init"
ensure_loaded(
bootstrap_hindcast(
self.hind,
self.uninit,
self.observations,
metric=metric,
comparison=comparison,
iterations=self.iterations,
dim=dim,
))
def test_bootstrap_hindcast_keeps_lead_units(hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d):
"""Test that lead units is kept in compute."""
sig = 95
units = 'years'
hind_da_initialized_1d['lead'].attrs['units'] = units
actual = bootstrap_hindcast(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
metric='mse',
iterations=ITERATIONS,
comparison='e2o',
sig=sig,
dim='init',
)
assert actual.lead.attrs['units'] == units
def test_bootstrap_hindcast_keeps_lead_units(hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d):
"""Test that lead units is kept in compute."""
sig = 95
units = "years"
hind_da_initialized_1d["lead"].attrs["units"] = units
actual = bootstrap_hindcast(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
metric="mse",
iterations=ITERATIONS,
comparison="e2o",
sig=sig,
dim="init",
)
assert actual.lead.attrs["units"] == units
def test_bootstrap_hindcast_da1d_not_nan(initialized_da, uninitialized_da,
reconstruction_da):
"""
Checks that there are no NaNs on bootstrap hindcast of 1D da.
"""
actual = bootstrap_hindcast(
initialized_da,
uninitialized_da,
reconstruction_da,
metric='rmse',
comparison='e2r',
sig=50,
bootstrap=2,
)
actual_init_skill = actual.sel(kind='init', results='skill').isnull().any()
assert not actual_init_skill
actual_uninit_p = actual.sel(kind='uninit', results='p').isnull().any()
assert not actual_uninit_p
def test_bootstrap_hindcast_da1d_not_nan(hind_da_initialized_1d,
hist_da_uninitialized_1d,
reconstruction_da_1d):
"""
Checks that there are no NaNs on bootstrap hindcast of 1D da.
"""
actual = bootstrap_hindcast(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
reconstruction_da_1d,
metric='rmse',
comparison='e2o',
sig=50,
iterations=ITERATIONS,
)
actual_init_skill = actual.sel(kind='init', results='skill').isnull().any()
assert not actual_init_skill
actual_uninit_p = actual.sel(kind='uninit', results='p').isnull().any()
assert not actual_uninit_p
def test_bootstrap_hindcast_da1d_not_nan_probabilistic(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
metric,
comparison,
):
"""
Checks that there are no NaNs on hindcast probabilistic metrics of 1D
time series.
"""
if 'threshold' in metric:
threshold = 10.5
else:
threshold = None
if metric == 'brier_score':
def func(x):
return x > 0
else:
func = None
actual = bootstrap_hindcast(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
comparison=comparison,
metric=metric,
threshold=threshold,
gaussian=True,
func=func,
iterations=ITERATIONS,
dim='member',
resample_dim='member',
)
for kind in ['init', 'uninit']:
actualk = actual.sel(kind=kind, results='skill')
if 'init' in actualk.coords:
actualk = actualk.mean('init')
actualk = actualk.isnull().any()
assert not actualk
def test_bootstrap_hindcast_dim(initialized_da, uninitialized_da,
observations_da):
"""Test whether bootstrap_hindcast calcs skill over member dim and
returns init dim."""
actual = bootstrap_hindcast(
initialized_da,
uninitialized_da,
observations_da,
metric='rmse',
dim='member',
comparison='m2r',
bootstrap=3,
)
assert 'init' in actual.dims
for kind in ['init', 'uninit']:
actualk = actual.sel(kind=kind, results='skill')
if 'init' in actualk.coords:
actualk = actualk.mean('init')
actualk = actualk.isnull().any()
assert not actualk
def test_bootstrap_hindcast_dim(hind_da_initialized_1d,
hist_da_uninitialized_1d, observations_da_1d):
"""Test whether bootstrap_hindcast calcs skill over member dim and
returns init dim."""
actual = bootstrap_hindcast(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
metric="rmse",
dim="member",
comparison="m2o",
iterations=ITERATIONS,
resample_dim="member",
)
assert "init" in actual.dims
for kind in ["initialized", "uninitialized"]:
actualk = actual.sel(kind=kind, results="skill")
if "init" in actualk.coords:
actualk = actualk.mean("init")
actualk = actualk.isnull().any()
assert not actualk
def test_bootstrap_hindcast_dim(hind_da_initialized_1d,
hist_da_uninitialized_1d, observations_da_1d):
"""Test whether bootstrap_hindcast calcs skill over member dim and
returns init dim."""
actual = bootstrap_hindcast(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
metric='rmse',
dim='member',
comparison='m2o',
iterations=ITERATIONS,
resample_dim='member',
)
assert 'init' in actual.dims
for kind in ['init', 'uninit']:
actualk = actual.sel(kind=kind, results='skill')
if 'init' in actualk.coords:
actualk = actualk.mean('init')
actualk = actualk.isnull().any()
assert not actualk
def test_bootstrap_hindcast_da1d_not_nan(
hind_da_initialized_1d, hist_da_uninitialized_1d, reconstruction_da_1d
):
"""
Checks that there are no NaNs on bootstrap hindcast of 1D da.
"""
actual = bootstrap_hindcast(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
reconstruction_da_1d,
metric="rmse",
comparison="e2o",
sig=50,
iterations=ITERATIONS,
)
actual_init_skill = (
actual.sel(skill="initialized", results="verify skill").isnull().any()
)
assert not actual_init_skill
actual_uninit_p = actual.sel(skill="uninitialized", results="p").isnull().any()
assert not actual_uninit_p
def test_bootstrap_hindcast_da1d_not_nan_probabilistic(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
metric,
comparison,
):
"""
Checks that there are no NaNs on hindcast probabilistic metrics of 1D
time series.
"""
metric_kwargs = {
"comparison": comparison,
"metric": metric,
"dim": "member",
"iterations": ITERATIONS,
"alignment": "same_verif",
}
if "threshold" in metric:
metric_kwargs["threshold"] = 0.5
if metric == "brier_score":
def func(x):
return x > 0
metric_kwargs["logical"] = func
actual = bootstrap_hindcast(
hind_da_initialized_1d,
hist_da_uninitialized_1d,
observations_da_1d,
resample_dim="member",
)
for kind in ["initialized", "uninitialized"]:
actualk = actual.sel(kind=kind, results="skill")
if "init" in actualk.coords:
actualk = actualk.mean("init")
actualk = actualk.isnull().any()
assert not actualk
