def test_mae_test_alpha_hwci(a_1d, a_1d_worse_less_corr, b_1d):
"""Test mae_test with larger alpha leads to smaller hwci."""
(
actual_large_alpha_significantly_different,
actual_large_alpha_diff,
actual_large_alpha_alpha,
) = mae_test(a_1d, a_1d_worse_less_corr, b_1d, alpha=0.1)
(
actual_small_alpha_significantly_different,
actual_small_alpha_diff,
actual_small_alpha_alpha,
) = mae_test(a_1d, a_1d_worse_less_corr, b_1d, alpha=0.01)
assert actual_large_alpha_alpha < actual_small_alpha_alpha
def test_mae_test_inputs(a_1d, a_1d_worse_less_corr, b_1d, input, chunk,
alpha):
"""Test mae_test with xr inputs and chunked."""
if "Dataset" in input:
name = "var"
a_1d = a_1d.to_dataset(name=name)
a_1d_worse_less_corr = a_1d_worse_less_corr.to_dataset(name=name)
b_1d = b_1d.to_dataset(name=name)
if input == "multidim Dataset":
a_1d["var2"] = a_1d["var"] * 2
a_1d_worse_less_corr["var2"] = a_1d_worse_less_corr["var"] * 2
b_1d["var2"] = b_1d["var"] * 2
elif input == "mixed":
name = "var"
a_1d = a_1d.to_dataset(name=name)
if chunk:
a_1d = a_1d.chunk()
a_1d_worse_less_corr = a_1d_worse_less_corr.chunk()
b_1d = b_1d.chunk()
actual_significantly_different, actual_diff, actual_hwci = mae_test(
a_1d, a_1d_worse_less_corr, b_1d, alpha=alpha)
# check dask collection preserved
for actual in [actual_significantly_different, actual_diff, actual_hwci]:
assert is_dask_collection(
actual) if chunk else not is_dask_collection(actual)
def test_mae_test_better_significant(a_1d, a_1d_worse_less_corr, b_1d, alpha):
"""Test mae_test favors better forecast."""
a_1d_worse_less_corr = a_1d.copy()
# make a_worse worse every second timestep
step = 3
a_1d_worse_less_corr[::step] = a_1d_worse_less_corr[::step] + OFFSET * 3
actual_significantly_different, actual_diff, actual_hwci = mae_test(
a_1d, a_1d_worse_less_corr, b_1d, alpha=alpha)
assert actual_significantly_different
def test_mae_test_dim(a, b, dim):
"""Test mae_test for different dim on ."""
a_worse = a.copy()
# make a_worse worse every second timestep
a_worse[::2, :, :] = a_worse[::2, :, :] + OFFSET
actual_significantly_different, actual_diff, actual_hwci = mae_test(
a, a_worse, b, dim=dim)
# difference larger than half width ci
assert (actual_diff > actual_hwci).mean() > 0.5
for d in dim:
assert d not in actual_diff.dims, print(d, "found, but shouldnt")
def test_mae_test_climpred(a_1d, b_1d):
"""Test mae_test as climpred would use it with observations=None."""
a_1d_worse_less_corr = a_1d.copy()
# make a_worse worse every second timestep
a_1d_worse_less_corr[::2] = a_1d_worse_less_corr[::2] + OFFSET
# calc skill before as in climpred
dim = []
time_dim = "time"
mae_f1o = mae(a_1d, b_1d, dim=dim)
mae_f2o = mae(a_1d_worse_less_corr, b_1d, dim=dim)
actual_significantly_different, actual_diff, actual_hwci = mae_test(
mae_f1o,
mae_f2o,
observations=None,
dim=dim,
time_dim=time_dim,
)
expected_significantly_different, expected_diff, expected_hwci = mae_test(
a_1d, a_1d_worse_less_corr, b_1d, dim=dim, time_dim=time_dim)
assert_equal(actual_significantly_different,
expected_significantly_different)
assert_equal(actual_diff, expected_diff)
assert_equal(actual_hwci, expected_hwci)
def test_mae_test_alpha(a_1d, a_1d_worse_less_corr, b_1d, alpha):
"""Test mae_test alpha error messages."""
with pytest.raises(ValueError) as e:
mae_test(a_1d, a_1d_worse_less_corr, b_1d, alpha=alpha)
assert "`alpha` must be between 0 and 1 or `return_p`" in str(e.value)