def test_climpred_warnings(hindcast_recon_1d_dm, option_bool):
with climpred.set_options(warn_for_failed_PredictionEnsemble_xr_call=True):
with climpred.set_options(climpred_warnings=option_bool):
with pytest.warns(UserWarning if option_bool else None) as record:
hindcast_recon_1d_dm.sel(lead=[1, 2])
if not option_bool:
assert len(record) == 0, print(record[0])
def test_seasonality_remove_bias(hindcast_recon_1d_dm, cross_validate):
"""Test the climpred.set_option(seasonality) changes bias reduction."""
hindcast = hindcast_recon_1d_dm
hindcast._datasets["initialized"] = (hindcast.get_initialized().resample(
init="1MS").interpolate("linear"))
alignment = "maximize"
kw = {
"metric": "mse",
"comparison": "e2o",
"dim": "init",
"alignment": alignment,
"reference": None,
}
with climpred.set_options(seasonality="dayofyear"):
dayofyear_seasonality = hindcast.remove_bias(
alignment=alignment, cross_validate=cross_validate)
with climpred.set_options(seasonality="weekofyear"):
weekofyear_seasonality = hindcast.remove_bias(
alignment=alignment, cross_validate=cross_validate)
assert not dayofyear_seasonality.get_initialized().to_array().isnull().all(
)
assert not weekofyear_seasonality.get_initialized().to_array().isnull(
).all()
assert not weekofyear_seasonality.get_initialized().equals(
dayofyear_seasonality.get_initialized())
assert not weekofyear_seasonality.verify(**kw).equals(
dayofyear_seasonality.verify(**kw))
def test_seasonality_remove_bias(hindcast_recon_1d_dm, cross_validate):
"""Test the climpred.set_option(seasonality) changes bias reduction. Currently fails for cross_validate bias reduction."""
hindcast = hindcast_recon_1d_dm
hindcast._datasets["initialized"] = (
hindcast.get_initialized().resample(init="1MS").interpolate("linear")
)
print(hindcast.get_initialized().init.to_index())
print(hindcast.get_observations().time.to_index())
alignment = "maximize"
kw = {
"metric": "mse",
"comparison": "e2o",
"dim": "init",
"alignment": alignment,
"reference": None,
}
print(hindcast.get_initialized().init.dt.dayofyear.values)
with climpred.set_options(seasonality="dayofyear"):
dayofyear_seasonality = hindcast.remove_bias(
alignment=alignment, cross_validate=cross_validate
)
with climpred.set_options(seasonality="weekofyear"):
weekofyear_seasonality = hindcast.remove_bias(
alignment=alignment, cross_validate=cross_validate
)
assert not weekofyear_seasonality.get_initialized().identical(
dayofyear_seasonality.get_initialized()
)
assert not weekofyear_seasonality.verify(**kw).identical(
dayofyear_seasonality.verify(**kw)
)
def test_remove_bias_difference_seasonality(hindcast_recon_1d_mm, how):
"""Test HindcastEnsemble.remove_bias yields different results for seasonality."""
verify_kwargs = dict(metric="rmse",
dim="init",
comparison="e2o",
alignment="same_inits",
skipna=True)
hindcast = hindcast_recon_1d_mm.isel(lead=range(3))
v = "SST"
bias_reduced_skill = []
seasonalities = GROUPBY_SEASONALITIES
for seasonality in seasonalities:
with set_options(seasonality=seasonality):
hindcast_rb = hindcast.remove_bias(
how=how, alignment=verify_kwargs["alignment"], cv=False)
bias_reduced_skill.append(hindcast_rb.verify(**verify_kwargs)[v])
bias_reduced_skill = xr.concat(
bias_reduced_skill,
"seasonality").assign_coords(seasonality=seasonalities)
# check not identical
for s in seasonalities:
assert bias_reduced_skill.sel(seasonality=s).notnull().all()
for s2 in seasonalities:
if s != s2:
assert (bias_reduced_skill.sel(
seasonality=[s, s2]).diff("seasonality").notnull().any())
def test_seasonality_climatology(hindcast_recon_1d_dm):
"""Test the climpred.set_option(seasonality) changes climatology."""
hindcast = hindcast_recon_1d_dm
alignment = "maximize"
kw = {
"metric": "mse",
"comparison": "e2o",
"dim": "init",
"alignment": alignment,
"reference": "climatology",
}
with climpred.set_options(seasonality="dayofyear"):
dayofyear_seasonality = hindcast.verify(**kw).sel(skill="climatology")
with climpred.set_options(seasonality="month"):
month_seasonality = hindcast.verify(**kw).sel(skill="climatology")
assert not month_seasonality.identical(dayofyear_seasonality)
def test_remove_bias_unfair_artificial_skill_over_fair(hindcast_NMME_Nino34,
how, seasonality,
alignment):
"""Show how method unfair better skill than fair."""
verify_kwargs = dict(metric="rmse",
comparison="e2o",
dim="init",
alignment=alignment,
skipna=True)
with set_options(seasonality=seasonality):
he = (hindcast_NMME_Nino34.sel(lead=[4, 5]).sel(
model="GEM-NEMO").dropna(
"member", how="all").sel(init=slice("2000", "2009")))
v = "sst"
print("\n unfair \n")
he_unfair = he.remove_bias(
how=how,
alignment=alignment,
train_test_split="unfair",
)
unfair_skill = he_unfair.verify(**verify_kwargs)
print("\n unfair-cv \n")
he_unfair_cv = he.remove_bias(
how=how,
alignment=alignment,
train_test_split="unfair-cv",
cv="LOO",
)
unfair_cv_skill = he_unfair_cv.verify(**verify_kwargs)
print("\n fair \n")
kw = (dict(train_time=slice("2000", "2004")) if alignment
== "same_verifs" else dict(train_init=slice("2000", "2004")))
he_fair = he.remove_bias(
how=how,
alignment=alignment,
train_test_split="fair",
**kw,
)
fair_skill = he_fair.verify(**verify_kwargs)
assert not unfair_skill[v].isnull().all()
assert not fair_skill[v].isnull().all()
# allow 1 or 20% margin (worse skill)
f = 1.2 if how in ["modified_quantile", "basic_quantile"] else 1.01
assert (fair_skill * f > unfair_skill)[v].all(), print(
fair_skill[v], unfair_skill[v])
print("checking unfair-cv")
assert not unfair_cv_skill[v].isnull().all()
assert (fair_skill * f > unfair_cv_skill)[v].all(), print(
fair_skill[v], unfair_cv_skill[v])
def test_monthly_leads_remove_bias_LOO(hindcast_NMME_Nino34, how, seasonality,
alignment):
"""Get different HindcastEnsemble depending on CV or not."""
with set_options(seasonality=seasonality):
he = (hindcast_NMME_Nino34.isel(lead=[0, 1]).isel(
model=2, drop=True).sel(init=slice("2005", "2006")))
assert not he.remove_bias(
how=how, alignment=alignment, cv=False,
train_test_split="unfair").equals(
he.remove_bias(how=how,
alignment=alignment,
cv="LOO",
train_test_split="unfair-cv"))
def test_option_resample_iterations_func(hindcast_recon_1d_ym):
"""Singleton dimension makes resample_iterations_idx fail."""
with climpred.set_options(
resample_iterations_func="resample_iterations_idx"):
with pytest.raises(ValueError):
hindcast_recon_1d_ym.expand_dims("lon").bootstrap(
metric="mse",
comparison="e2o",
dim="init",
alignment="maximize",
iterations=2,
resample_dim="member",
)
with climpred.set_options(resample_iterations_func="resample_iterations"):
hindcast_recon_1d_ym.expand_dims("lon").bootstrap(
metric="mse",
comparison="e2o",
dim="init",
alignment="maximize",
iterations=2,
resample_dim="member",
)
def test_remove_bias_compare_scaling_and_mean(hindcast_recon_1d_mm):
"""Compare Scaling and additive_mean to be similar."""
he = hindcast_recon_1d_mm.isel(lead=[0, 1])
hind_scaling = he.remove_bias(
how="Scaling",
kind="+",
alignment="same_inits",
train_test_split="unfair",
group="time.dayofyear",
)
with set_options(seasonality="dayofyear"):
hind_mean = hindcast_recon_1d_mm.remove_bias(
how="additive_mean",
alignment="same_inits",
train_test_split="unfair",
)
assert ((hind_scaling - hind_mean).get_initialized().mean(
["member", "init"]) < 0.02).SST.all()
import numpy as np
import xarray as xr
from dask.distributed import Client
from climpred import HindcastEnsemble, PerfectModelEnsemble, set_options
from climpred.metrics import PROBABILISTIC_METRICS
from climpred.tutorial import load_dataset
from . import _skip_slow, ensure_loaded, parameterized, randn, requires_dask
# only take subselection of all possible metrics
METRICS = ["mse", "crps"]
REFERENCES = ["uninitialized", "climatology", "persistence"]
ITERATIONS = 8
set_options(climpred_warnings=False)
warnings.filterwarnings("ignore",
message="Index.ravel returning ndarray is deprecated")
class Compute:
"""
Benchmark time and peak memory of `PredictionEnsemble.verify` and
`PredictionEnsemble.bootstrap`.
"""
# https://asv.readthedocs.io/en/stable/benchmarks.html
timeout = 300.0
repeat = 1
number = 5
def test_remove_bias(hindcast_recon_1d_mm, alignment, how, seasonality, cv):
"""Test remove mean bias, ensure than skill doesnt degrade and keeps attrs."""
def check_hindcast_coords_maintained_except_init(hindcast,
hindcast_bias_removed):
# init only slighty cut due to alignment
for c in hindcast.coords:
if c in ["init", "valid_time"]:
assert hindcast.coords[c].size >= hindcast_bias_removed.coords[
c].size
else:
assert hindcast.coords[c].size == hindcast_bias_removed.coords[
c].size
with set_options(seasonality=seasonality):
metric = "rmse"
dim = "init"
comparison = "e2o"
hindcast = hindcast_recon_1d_mm.isel(lead=range(2))
hindcast._datasets["initialized"].attrs["test"] = "test"
hindcast._datasets["initialized"]["SST"].attrs["units"] = "test_unit"
verify_kwargs = dict(
metric=metric,
alignment=alignment,
dim=dim,
comparison=comparison,
keep_attrs=True,
)
# add how bias
if "additive" in how:
with xr.set_options(keep_attrs=True):
hindcast._datasets["observations"] = (
hindcast._datasets["observations"] + 0.1)
elif "multiplicative" in how:
with xr.set_options(keep_attrs=True):
hindcast._datasets["observations"] = (
hindcast._datasets["observations"] * 1.1)
biased_skill = hindcast.verify(**verify_kwargs)
hindcast_bias_removed = hindcast.remove_bias(how=how,
alignment=alignment,
cv=False)
check_hindcast_coords_maintained_except_init(hindcast,
hindcast_bias_removed)
bias_removed_skill = hindcast_bias_removed.verify(**verify_kwargs)
seasonality = OPTIONS["seasonality"]
if cv:
hindcast_bias_removed_properly = hindcast.remove_bias(
how=how, cv="LOO", alignment=alignment)
check_hindcast_coords_maintained_except_init(
hindcast, hindcast_bias_removed_properly)
bias_removed_skill_properly = hindcast_bias_removed_properly.verify(
**verify_kwargs)
# checks
assert seasonality not in bias_removed_skill_properly.coords
assert (biased_skill > bias_removed_skill_properly).all()
assert (bias_removed_skill_properly >= bias_removed_skill).all()
assert (biased_skill > bias_removed_skill).all()
assert seasonality not in bias_removed_skill.coords
# keeps data_vars attrs
for v in hindcast_bias_removed.get_initialized().data_vars:
if cv:
assert (hindcast_bias_removed_properly.get_initialized()
[v].attrs == hindcast.get_initialized()[v].attrs)
assert (hindcast_bias_removed.get_initialized()[v].attrs ==
hindcast.get_initialized()[v].attrs)
# keeps dataset attrs
if cv:
assert (hindcast_bias_removed_properly.get_initialized().attrs ==
hindcast.get_initialized().attrs)
assert (hindcast_bias_removed.get_initialized().attrs ==
hindcast.get_initialized().attrs)
# keep lead attrs
assert (hindcast_bias_removed.get_initialized().lead.attrs ==
hindcast.get_initialized().lead.attrs)