def test_roc_curve_hard(setup):
# roc_curve for hard decisions
y_true, pred, probas_pred = make_prediction(binary=True)
# always predict one
trivial_pred = np.ones(y_true.shape)
fpr, tpr, thresholds = roc_curve(y_true, trivial_pred)
roc_auc = auc(fpr, tpr).fetch()
np.testing.assert_array_almost_equal(roc_auc, 0.50, decimal=2)
assert fpr.shape == tpr.shape
assert fpr.shape == thresholds.shape
# always predict zero
trivial_pred = np.zeros(y_true.shape)
fpr, tpr, thresholds = roc_curve(y_true, trivial_pred)
roc_auc = auc(fpr, tpr).fetch()
np.testing.assert_array_almost_equal(roc_auc, 0.50, decimal=2)
assert fpr.shape == tpr.shape
assert fpr.shape == thresholds.shape
# hard decisions
fpr, tpr, thresholds = roc_curve(y_true, pred)
roc_auc = auc(fpr, tpr).fetch()
np.testing.assert_array_almost_equal(roc_auc, 0.78, decimal=2)
assert fpr.shape == tpr.shape
assert fpr.shape == thresholds.shape
def test_roc_curve_confidence(setup):
# roc_curve for confidence scores
y_true, _, probas_pred = make_prediction(binary=True)
fpr, tpr, thresholds = roc_curve(y_true, probas_pred - 0.5)
roc_auc = auc(fpr, tpr).fetch()
np.testing.assert_array_almost_equal(roc_auc, 0.90, decimal=2)
assert fpr.shape == tpr.shape
assert fpr.shape == thresholds.shape
def test_roc_curve(setup):
for drop in [True, False]:
# Test Area under Receiver Operating Characteristic (ROC) curve
y_true, _, probas_pred = make_prediction(binary=True)
expected_auc = _auc(y_true, probas_pred)
fpr, tpr, thresholds = roc_curve(y_true, probas_pred,
drop_intermediate=drop).execute().fetch()
roc_auc = auc(fpr, tpr).to_numpy()
np.testing.assert_array_almost_equal(roc_auc, expected_auc, decimal=2)
np.testing.assert_almost_equal(roc_auc, roc_auc_score(y_true, probas_pred))
assert fpr.shape == tpr.shape
assert fpr.shape == thresholds.shape
def test_roc_returns_consistency(setup):
# Test whether the returned threshold matches up with tpr
# make small toy dataset
y_true, _, probas_pred = make_prediction(binary=True)
fpr, tpr, thresholds = roc_curve(y_true, probas_pred).fetch()
# use the given thresholds to determine the tpr
tpr_correct = []
for t in thresholds:
tp = np.sum((probas_pred >= t) & y_true)
p = np.sum(y_true)
tpr_correct.append(1.0 * tp / p)
# compare tpr and tpr_correct to see if the thresholds' order was correct
np.testing.assert_array_almost_equal(tpr, tpr_correct, decimal=2)
assert fpr.shape == tpr.shape
assert fpr.shape == thresholds.shape
def test_roc_curve_multi(setup):
# roc_curve not applicable for multi-class problems
y_true, _, probas_pred = make_prediction(binary=False)
with pytest.raises(ValueError):
roc_curve(y_true, probas_pred)
def testRocCurveMulti(self):
# roc_curve not applicable for multi-class problems
y_true, _, probas_pred = make_prediction(binary=False)
with self.assertRaises(ValueError):
roc_curve(y_true, probas_pred)
