def test_brier_score_vs_fair_brier_score(o, f_prob, dim): """Test that brier_score scores lower for limited ensemble than bias brier_score.""" fbs = brier_score((o > 0.5), (f_prob > 0.5), dim=dim, fair=True) bs = brier_score((o > 0.5), (f_prob > 0.5).mean("member"), dim=dim, fair=False) assert (fbs <= bs).all(), print("fairBS", fbs, "\nBS", bs)
def test_brier_score_forecast_member_dim(o, f_prob): """Check that brier_score allows forecasts with member dim and binary/boolean data.""" f_prob > 0.5 o = o > 0.5 assert_identical(brier_score(o, f_prob), brier_score(o, f_prob.mean("member")))
def test_2_category_rps_equals_brier_score(o, f_prob): """Test that RPS for two categories equals the Brier Score.""" category_edges = np.array([0.0, 0.5, 1.0]) assert_allclose( rps(o, f_prob, category_edges=category_edges, dim=None), brier_score(o > 0.5, (f_prob > 0.5).mean("member"), dim=None), )
def test_brier_score_dim(o, f_prob, dim, fair_bool): """Check that brier_score reduces only dim.""" f_prob > 0.5 if not fair_bool: f_prob = f_prob.mean("member") o = o > 0.5 actual = brier_score(o, f_prob, dim=dim, fair=fair_bool) assert_only_dim_reduced(dim, actual, o)
def test_2_category_rps_equals_brier_score(o, f_prob, fair_bool): """Test that RPS for two categories equals the Brier Score.""" category_edges = np.array([0.0, 0.5, 1.0]) assert_allclose( rps(o, f_prob, category_edges=category_edges, dim=None, fair=fair_bool).drop( ["forecasts_category_edge", "observations_category_edge"]), brier_score(o > 0.5, (f_prob > 0.5), dim=None, fair=fair_bool), )
def test_brier_score(o, f_prob, keep_attrs): actual = brier_score( (o > 0.5).assign_attrs(**o.attrs), (f_prob > 0.5).mean("member"), keep_attrs=keep_attrs, ) assert actual.chunks is None or actual.chunks == () if keep_attrs: assert actual.attrs == o.attrs else: assert actual.attrs == {}
def test_brier_score_accessor(o, f_prob, threshold, outer_bool): actual = brier_score(o > threshold, (f_prob > threshold).mean("member")) ds = xr.Dataset() ds["o"] = o > threshold ds["f_prob"] = (f_prob > threshold).mean("member") if outer_bool: ds = ds.drop_vars("f_prob") expected = ds.xs.brier_score("o", (f_prob > threshold).mean("member")) else: expected = ds.xs.brier_score("o", "f_prob") assert_allclose(actual, expected)
def test_brier_score_float_forecast_or_observations(o, f_prob): """Test that forecast and observations can be float.""" o = o > 0.5 f_prob = (f_prob > 0.5).mean("member") brier_score(o, f_prob) brier_score(o, 0.5) brier_score(1, f_prob)
def test_brier_score_accessor(o, f, threshold, dask_bool, outer_bool): if dask_bool: o = o.chunk() f = f.chunk() actual = brier_score(o > threshold, (f > threshold).mean('member')) ds = xr.Dataset() ds['o'] = o > threshold ds['f'] = (f > threshold).mean('member') if outer_bool: ds = ds.drop_vars('f') expected = ds.xs.brier_score('o', (f > threshold).mean('member')) else: expected = ds.xs.brier_score('o', 'f') assert_allclose(actual, expected)
def test_brier_score_api_and_inputs(o, f_prob, keep_attrs, fair_bool, chunk_bool, input_type): """Test that brier_score keeps attributes, chunking, input types.""" o, f_prob = modify_inputs(o, f_prob, input_type, chunk_bool) f_prob > 0.5 if not fair_bool: f_prob = f_prob.mean("member") o = (o > 0.5).assign_attrs(**o.attrs) actual = brier_score( o, f_prob, keep_attrs=keep_attrs, fair=fair_bool, ) # test that returns chunks assert_chunk(actual, chunk_bool) # test that attributes are kept assert_keep_attrs(actual, o, keep_attrs) # test that input types equal output types assign_type_input_output(actual, o)
def test_brier_score_dask(o_dask, f_prob_dask, keep_attrs): actual = brier_score( (o_dask > 0.5).assign_attrs(**o_dask.attrs), (f_prob_dask > 0.5).mean("member"), keep_attrs=keep_attrs, ) assert actual.chunks is not None expected = properscoring.brier_score((o_dask > 0.5), (f_prob_dask > 0.5).mean("member")) expected = xr.DataArray(expected, coords=o_dask.coords).mean() # test for numerical identity of brier_score and properscoring brier_score assert_allclose(actual, expected) # test that xskillscore brier_score returns chunks assert actual.chunks is not None # show that properscoring brier_score returns no chunks assert expected.chunks is None if keep_attrs: assert actual.attrs == o_dask.attrs else: assert actual.attrs == {}
def test_brier_score_dim(o, f_prob, dim): actual = brier_score((o > 0.5), (f_prob > 0.5).mean("member"), dim=dim) assert_only_dim_reduced(dim, actual, o)