def test_roc_curve_hard(setup): # roc_curve for hard decisions y_true, pred, probas_pred = make_prediction(binary=True) # always predict one trivial_pred = np.ones(y_true.shape) fpr, tpr, thresholds = roc_curve(y_true, trivial_pred) roc_auc = auc(fpr, tpr).fetch() np.testing.assert_array_almost_equal(roc_auc, 0.50, decimal=2) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape # always predict zero trivial_pred = np.zeros(y_true.shape) fpr, tpr, thresholds = roc_curve(y_true, trivial_pred) roc_auc = auc(fpr, tpr).fetch() np.testing.assert_array_almost_equal(roc_auc, 0.50, decimal=2) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape # hard decisions fpr, tpr, thresholds = roc_curve(y_true, pred) roc_auc = auc(fpr, tpr).fetch() np.testing.assert_array_almost_equal(roc_auc, 0.78, decimal=2) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape
def test_roc_curve_confidence(setup): # roc_curve for confidence scores y_true, _, probas_pred = make_prediction(binary=True) fpr, tpr, thresholds = roc_curve(y_true, probas_pred - 0.5) roc_auc = auc(fpr, tpr).fetch() np.testing.assert_array_almost_equal(roc_auc, 0.90, decimal=2) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape
def test_roc_curve(setup): for drop in [True, False]: # Test Area under Receiver Operating Characteristic (ROC) curve y_true, _, probas_pred = make_prediction(binary=True) expected_auc = _auc(y_true, probas_pred) fpr, tpr, thresholds = roc_curve(y_true, probas_pred, drop_intermediate=drop).execute().fetch() roc_auc = auc(fpr, tpr).to_numpy() np.testing.assert_array_almost_equal(roc_auc, expected_auc, decimal=2) np.testing.assert_almost_equal(roc_auc, roc_auc_score(y_true, probas_pred)) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape
def test_roc_returns_consistency(setup): # Test whether the returned threshold matches up with tpr # make small toy dataset y_true, _, probas_pred = make_prediction(binary=True) fpr, tpr, thresholds = roc_curve(y_true, probas_pred).fetch() # use the given thresholds to determine the tpr tpr_correct = [] for t in thresholds: tp = np.sum((probas_pred >= t) & y_true) p = np.sum(y_true) tpr_correct.append(1.0 * tp / p) # compare tpr and tpr_correct to see if the thresholds' order was correct np.testing.assert_array_almost_equal(tpr, tpr_correct, decimal=2) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape
def test_roc_curve_multi(setup): # roc_curve not applicable for multi-class problems y_true, _, probas_pred = make_prediction(binary=False) with pytest.raises(ValueError): roc_curve(y_true, probas_pred)
def testRocCurveMulti(self): # roc_curve not applicable for multi-class problems y_true, _, probas_pred = make_prediction(binary=False) with self.assertRaises(ValueError): roc_curve(y_true, probas_pred)