def test_reliability(o, f_prob, dim, obj):
"""Test that reliability object can be generated"""
if "ds" in obj:
name = "var"
o = o.to_dataset(name=name)
f_prob = f_prob.to_dataset(name=name)
if "chunked" in obj:
o = o.chunk()
f_prob = f_prob.chunk()
if dim == []:
with pytest.raises(ValueError):
reliability(o > 0.5, (f_prob > 0.5).mean("member"), dim)
else:
reliability(o > 0.5, (f_prob > 0.5).mean("member"), dim=dim)
def test_reliability_api_and_inputs(o, f_prob, dim, chunk_bool, input_type):
"""Test that reliability keeps chunking and input types."""
o, f_prob = modify_inputs(o, f_prob, input_type, chunk_bool)
if dim == []:
with pytest.raises(ValueError):
reliability(o > 0.5, (f_prob > 0.5).mean("member"), dim)
else:
actual = reliability(o > 0.5, (f_prob > 0.5).mean("member"), dim=dim)
# test that returns chunks
assert_chunk(actual, chunk_bool)
# test that attributes are kept
# assert_keep_attrs(actual, o, keep_attrs) # TODO: implement
# test that input types equal output types
assign_type_input_output(actual, o)
def test_reliability_perfect_values(o):
"""Test values for perfect forecast"""
f_prob = xr.concat(10 * [o], dim="member")
actual = reliability(o > 0.5, (f_prob > 0.5).mean("member"))
expected_true_samples = (o > 0.5).sum()
expected_false_samples = (o <= 0.5).sum()
assert np.allclose(actual[0], 0)
assert np.allclose(actual[-1], 1)
assert np.allclose(actual["samples"][0], expected_false_samples)
assert np.allclose(actual["samples"][-1], expected_true_samples)
assert np.allclose(actual["samples"].sum(), o.size)
def test_reliability_accessor(o, f_prob, threshold, outer_bool):
actual = reliability(o > threshold, (f_prob > threshold).mean("member"))
ds = xr.Dataset()
ds["o"] = o > threshold
ds["f_prob"] = (f_prob > threshold).mean("member")
if outer_bool:
ds = ds.drop_vars("f_prob")
expected = ds.xs.reliability("o", (f_prob > threshold).mean("member"))
else:
expected = ds.xs.reliability("o", "f_prob")
assert_allclose(actual, expected)
def test_reliability_values(o, f_prob):
"""Test 1D reliability values against sklearn calibration_curve"""
for lon in f_prob.lon:
for lat in f_prob.lat:
o_1d = o.sel(lon=lon, lat=lat) > 0.5
f_1d = (f_prob.sel(lon=lon, lat=lat) > 0.5).mean("member")
actual = reliability(o_1d, f_1d)
expected, _ = calibration_curve(o_1d,
f_1d,
normalize=False,
n_bins=5,
strategy="uniform")
npt.assert_allclose(actual.where(actual.notnull(), drop=True),
expected)
npt.assert_allclose(actual["samples"].sum(), o_1d.size)
def test_reliability_values(o, f_prob):
"""Test 1D reliability values against sklearn calibration_curve"""
for lon in f_prob.lon:
for lat in f_prob.lat:
o_1d = o.sel(lon=lon, lat=lat) > 0.5
f_1d = (f_prob.sel(lon=lon, lat=lat) > 0.5).mean("member")
# scipy bins are only left-edge inclusive and 1e-8 is added to the last bin, whereas
# xhistogram the rightmost edge of xhistogram bins is included - mimic scipy behaviour
actual = reliability(
o_1d, f_1d, probability_bin_edges=np.linspace(0, 1 + 1e-8, 6)
)
expected, _ = calibration_curve(
o_1d, f_1d, normalize=False, n_bins=5, strategy="uniform"
)
npt.assert_allclose(actual.where(actual.notnull(), drop=True), expected)
npt.assert_allclose(actual["samples"].sum(), o_1d.size)
def test_reliability_dask(o_dask, f_prob_dask):
"""Test that reliability returns dask array if provided dask array"""
actual = reliability(o_dask > 0.5, (f_prob_dask > 0.5).mean("member"))
assert actual.chunks is not None