def test_brier_score_vs_fair_brier_score(o, f_prob, dim):
"""Test that brier_score scores lower for limited ensemble than bias brier_score."""
fbs = brier_score((o > 0.5), (f_prob > 0.5), dim=dim, fair=True)
bs = brier_score((o > 0.5), (f_prob > 0.5).mean("member"),
dim=dim,
fair=False)
assert (fbs <= bs).all(), print("fairBS", fbs, "\nBS", bs)
def test_brier_score_forecast_member_dim(o, f_prob):
"""Check that brier_score allows forecasts with member dim and binary/boolean
data."""
f_prob > 0.5
o = o > 0.5
assert_identical(brier_score(o, f_prob),
brier_score(o, f_prob.mean("member")))
def test_2_category_rps_equals_brier_score(o, f_prob):
"""Test that RPS for two categories equals the Brier Score."""
category_edges = np.array([0.0, 0.5, 1.0])
assert_allclose(
rps(o, f_prob, category_edges=category_edges, dim=None),
brier_score(o > 0.5, (f_prob > 0.5).mean("member"), dim=None),
)
def test_brier_score_dim(o, f_prob, dim, fair_bool):
"""Check that brier_score reduces only dim."""
f_prob > 0.5
if not fair_bool:
f_prob = f_prob.mean("member")
o = o > 0.5
actual = brier_score(o, f_prob, dim=dim, fair=fair_bool)
assert_only_dim_reduced(dim, actual, o)
def test_2_category_rps_equals_brier_score(o, f_prob, fair_bool):
"""Test that RPS for two categories equals the Brier Score."""
category_edges = np.array([0.0, 0.5, 1.0])
assert_allclose(
rps(o, f_prob, category_edges=category_edges, dim=None,
fair=fair_bool).drop(
["forecasts_category_edge", "observations_category_edge"]),
brier_score(o > 0.5, (f_prob > 0.5), dim=None, fair=fair_bool),
)
def test_brier_score(o, f_prob, keep_attrs):
actual = brier_score(
(o > 0.5).assign_attrs(**o.attrs),
(f_prob > 0.5).mean("member"),
keep_attrs=keep_attrs,
)
assert actual.chunks is None or actual.chunks == ()
if keep_attrs:
assert actual.attrs == o.attrs
else:
assert actual.attrs == {}
def test_brier_score_accessor(o, f_prob, threshold, outer_bool):
actual = brier_score(o > threshold, (f_prob > threshold).mean("member"))
ds = xr.Dataset()
ds["o"] = o > threshold
ds["f_prob"] = (f_prob > threshold).mean("member")
if outer_bool:
ds = ds.drop_vars("f_prob")
expected = ds.xs.brier_score("o", (f_prob > threshold).mean("member"))
else:
expected = ds.xs.brier_score("o", "f_prob")
assert_allclose(actual, expected)
def test_brier_score_float_forecast_or_observations(o, f_prob):
"""Test that forecast and observations can be float."""
o = o > 0.5
f_prob = (f_prob > 0.5).mean("member")
brier_score(o, f_prob)
brier_score(o, 0.5)
brier_score(1, f_prob)
def test_brier_score_accessor(o, f, threshold, dask_bool, outer_bool):
if dask_bool:
o = o.chunk()
f = f.chunk()
actual = brier_score(o > threshold, (f > threshold).mean('member'))
ds = xr.Dataset()
ds['o'] = o > threshold
ds['f'] = (f > threshold).mean('member')
if outer_bool:
ds = ds.drop_vars('f')
expected = ds.xs.brier_score('o', (f > threshold).mean('member'))
else:
expected = ds.xs.brier_score('o', 'f')
assert_allclose(actual, expected)
def test_brier_score_api_and_inputs(o, f_prob, keep_attrs, fair_bool,
chunk_bool, input_type):
"""Test that brier_score keeps attributes, chunking, input types."""
o, f_prob = modify_inputs(o, f_prob, input_type, chunk_bool)
f_prob > 0.5
if not fair_bool:
f_prob = f_prob.mean("member")
o = (o > 0.5).assign_attrs(**o.attrs)
actual = brier_score(
o,
f_prob,
keep_attrs=keep_attrs,
fair=fair_bool,
)
# test that returns chunks
assert_chunk(actual, chunk_bool)
# test that attributes are kept
assert_keep_attrs(actual, o, keep_attrs)
# test that input types equal output types
assign_type_input_output(actual, o)
def test_brier_score_dask(o_dask, f_prob_dask, keep_attrs):
actual = brier_score(
(o_dask > 0.5).assign_attrs(**o_dask.attrs),
(f_prob_dask > 0.5).mean("member"),
keep_attrs=keep_attrs,
)
assert actual.chunks is not None
expected = properscoring.brier_score((o_dask > 0.5),
(f_prob_dask > 0.5).mean("member"))
expected = xr.DataArray(expected, coords=o_dask.coords).mean()
# test for numerical identity of brier_score and properscoring brier_score
assert_allclose(actual, expected)
# test that xskillscore brier_score returns chunks
assert actual.chunks is not None
# show that properscoring brier_score returns no chunks
assert expected.chunks is None
if keep_attrs:
assert actual.attrs == o_dask.attrs
else:
assert actual.attrs == {}
def test_brier_score_dim(o, f_prob, dim):
actual = brier_score((o > 0.5), (f_prob > 0.5).mean("member"), dim=dim)
assert_only_dim_reduced(dim, actual, o)
