def test_roc_returns( observation, forecast, symmetric_edges, dim, return_results, input, chunk, drop_intermediate_bool, ): """testing keywords and inputs pass""" if "Dataset" in input: name = "var" forecast = forecast.to_dataset(name=name) observation = observation.to_dataset(name=name) if chunk: forecast = forecast.chunk() observation = observation.chunk() roc( observation, forecast, symmetric_edges, dim=dim, drop_intermediate=drop_intermediate_bool, return_results=return_results, )
def test_roc_bin_edges_drop_intermediate(forecast_1d_long, observation_1d_long): """Test that drop_intermediate reduces probability_bins in xs.roc .""" fp = forecast_1d_long.clip(0, 1) # prob ob = observation_1d_long > 0 # binary # xs txs_fpr, txs_tpr, txs_area = roc( ob, fp, "continuous", drop_intermediate=True, return_results="all_as_tuple" ) fxs_fpr, fxs_tpr, fxs_area = roc( ob, fp, "continuous", drop_intermediate=False, return_results="all_as_tuple" ) # same area np.testing.assert_allclose(fxs_area, txs_area) # same or less probability_bins assert len(fxs_fpr) >= len(txs_fpr) assert len(fxs_tpr) >= len(txs_tpr)
def test_roc_auc_score_perfect_forecast(forecast_1d_long, symmetric_edges): """Test that ROC AUC equals 1 for perfect forecast.""" area = roc( forecast_1d_long, forecast_1d_long, symmetric_edges, drop_intermediate=False, dim="time", return_results="area", ) assert area == 1.0
def test_roc_auc_score_random_forecast(forecast_1d_long, observation_1d_long, symmetric_edges): """Test that ROC AUC around 0.5 for random forecast.""" area = roc( observation_1d_long, forecast_1d_long, symmetric_edges, dim="time", return_results="area", ) assert area < 0.65 assert area > 0.35
def test_roc_bin_edges_symmetric_asc_or_desc(observation_1d_long, forecast_1d_long, symmetric_edges): """Test that roc bin_edges works increasing or decreasing order.""" fpr, tpr, area = roc( observation_1d_long, forecast_1d_long, symmetric_edges, drop_intermediate=False, return_results="all_as_tuple", ) fpr2, tpr2, area2 = roc( observation_1d_long, forecast_1d_long, symmetric_edges[::-1], drop_intermediate=False, return_results="all_as_tuple", ) assert_identical(fpr, fpr2.sortby(fpr.probability_bin)) assert_identical(tpr, tpr2.sortby(tpr.probability_bin)) assert_identical(area, area2)
def test_roc_keeps_probability_bin_as_coord(observation_1d_long, forecast_1d_long, symmetric_edges): """Test that roc keeps probability_bin as coords.""" fpr, tpr, area = roc( observation_1d_long, forecast_1d_long, symmetric_edges, drop_intermediate=False, return_results="all_as_tuple", ) assert (tpr.probability_bin == symmetric_edges).all() assert (fpr.probability_bin == symmetric_edges).all()
def test_roc_auc_score_out_of_range_edges(forecast_1d_long, observation_1d_long, symmetric_edges, drop_intermediate_bool): """Test that ROC AUC equals 0.5 for out of range edges.""" area = roc( observation_1d_long, forecast_1d_long, symmetric_edges + 100, drop_intermediate=drop_intermediate_bool, dim="time", return_results="area", ) assert float(area) == 0.5
def test_roc_accessor(o, f_prob, outer_bool): bin_edges = np.linspace(0, 1, 6) actual = roc(o, f_prob, bin_edges=bin_edges) ds = xr.Dataset() ds["o"] = o ds["f_prob"] = f_prob if outer_bool: ds = ds.drop_vars("f_prob") expected = ds.xs.roc("o", f_prob, bin_edges=bin_edges) else: expected = ds.xs.roc("o", "f_prob", bin_edges=bin_edges) assert_allclose(actual, expected)
def test_roc_auc_score_out_of_range_observation( forecast_1d_long, observation_1d_long, symmetric_edges, drop_intermediate_bool ): """Test that ROC AUC equals 0.0 for out of range observation.""" area = roc( xr.ones_like(observation_1d_long) + 100, forecast_1d_long, symmetric_edges, drop_intermediate=drop_intermediate_bool, dim="time", return_results="area", ) np.testing.assert_almost_equal(area, 0.0, decimal=2)
def test_roc_auc_score_out_of_range_forecast( forecast_1d_long, observation_1d_long, symmetric_edges, drop_intermediate_bool ): """Test that ROC AUC equals 0.0 for out of range forecast.""" area = roc( observation_1d_long, xr.ones_like(forecast_1d_long) + 100, symmetric_edges, drop_intermediate=drop_intermediate_bool, dim="time", return_results="area", ) assert float(area) in [0.0, 0.5] # expect 0.0 but sometimes evaluates at 0.5 in CI
def test_roc_auc_score_constant_forecast(forecast_1d_long, observation_1d_long, symmetric_edges, drop_intermediate_bool): """Test that ROC AUC equals 0.5 for constant forecast.""" xs_area = roc( observation_1d_long > 0, forecast_1d_long * 0, symmetric_edges, drop_intermediate=drop_intermediate_bool, dim="time", return_results="area", ) sk_area = roc_auc_score(observation_1d_long > 0, forecast_1d_long * 0) np.testing.assert_allclose(xs_area, sk_area)
def test_roc_bin_edges_continuous_against_sklearn( forecast_1d_long, observation_1d_long, drop_intermediate_bool ): """Test xs.roc against sklearn.metrics.roc_curve/auc_score.""" fp = forecast_1d_long.clip(0, 1) # prob ob = observation_1d_long > 0 # binary # sklearn sk_fpr, sk_tpr, _ = roc_curve(ob, fp, drop_intermediate=drop_intermediate_bool) sk_area = roc_auc_score(ob, fp) # xs xs_fpr, xs_tpr, xs_area = roc( ob, fp, "continuous", drop_intermediate=drop_intermediate_bool, return_results="all_as_tuple", ) np.testing.assert_allclose(xs_area, sk_area) if not drop_intermediate_bool: # drops sometimes one too much or too little assert (xs_fpr == sk_fpr).all() assert (xs_tpr == sk_tpr).all()