def test_precision_recall_display_string_labels(pyplot): # regression test #15738 cancer = load_breast_cancer() X, y = cancer.data, cancer.target_names[cancer.target] lr = make_pipeline(StandardScaler(), LogisticRegression()) lr.fit(X, y) for klass in cancer.target_names: assert klass in lr.classes_ display = PrecisionRecallDisplay.from_estimator(lr, X, y) y_pred = lr.predict_proba(X)[:, 1] avg_prec = average_precision_score(y, y_pred, pos_label=lr.classes_[1]) assert display.average_precision == pytest.approx(avg_prec) assert display.estimator_name == lr.__class__.__name__ err_msg = r"y_true takes value in {'benign', 'malignant'}" with pytest.raises(ValueError, match=err_msg): PrecisionRecallDisplay.from_predictions(y, y_pred) display = PrecisionRecallDisplay.from_predictions(y, y_pred, pos_label=lr.classes_[1]) assert display.average_precision == pytest.approx(avg_prec)
def test_precision_recall_display_name(pyplot, constructor_name, default_label): """Check the behaviour of the name parameters""" X, y = make_classification(n_classes=2, n_samples=100, random_state=0) pos_label = 1 classifier = LogisticRegression().fit(X, y) classifier.fit(X, y) y_pred = classifier.predict_proba(X)[:, pos_label] # safe guard for the binary if/else construction assert constructor_name in ("from_estimator", "from_predictions") if constructor_name == "from_estimator": display = PrecisionRecallDisplay.from_estimator(classifier, X, y) else: display = PrecisionRecallDisplay.from_predictions(y, y_pred, pos_label=pos_label) average_precision = average_precision_score(y, y_pred, pos_label=pos_label) # check that the default name is used assert display.line_.get_label() == default_label.format(average_precision) # check that the name can be set display.plot(name="MySpecialEstimator") assert (display.line_.get_label() == f"MySpecialEstimator (AP = {average_precision:.2f})")
def test_precision_recall_display_validation(pyplot): """Check that we raise the proper error when validating parameters.""" X, y = make_classification(n_samples=100, n_informative=5, n_classes=5, random_state=0) with pytest.raises(NotFittedError): PrecisionRecallDisplay.from_estimator(SVC(), X, y) regressor = SVR().fit(X, y) y_pred_regressor = regressor.predict(X) classifier = SVC(probability=True).fit(X, y) y_pred_classifier = classifier.predict_proba(X)[:, -1] err_msg = "PrecisionRecallDisplay.from_estimator only supports classifiers" with pytest.raises(ValueError, match=err_msg): PrecisionRecallDisplay.from_estimator(regressor, X, y) err_msg = "Expected 'estimator' to be a binary classifier, but got SVC" with pytest.raises(ValueError, match=err_msg): PrecisionRecallDisplay.from_estimator(classifier, X, y) err_msg = "{} format is not supported" with pytest.raises(ValueError, match=err_msg.format("continuous")): # Force `y_true` to be seen as a regression problem PrecisionRecallDisplay.from_predictions(y + 0.5, y_pred_classifier, pos_label=1) with pytest.raises(ValueError, match=err_msg.format("multiclass")): PrecisionRecallDisplay.from_predictions(y, y_pred_regressor, pos_label=1) err_msg = "Found input variables with inconsistent numbers of samples" with pytest.raises(ValueError, match=err_msg): PrecisionRecallDisplay.from_predictions(y, y_pred_classifier[::2]) X, y = make_classification(n_classes=2, n_samples=50, random_state=0) y += 10 classifier.fit(X, y) y_pred_classifier = classifier.predict_proba(X)[:, -1] err_msg = r"y_true takes value in {10, 11} and pos_label is not specified" with pytest.raises(ValueError, match=err_msg): PrecisionRecallDisplay.from_predictions(y, y_pred_classifier)
def test_precision_recall_display_plotting(pyplot, constructor_name, response_method): """Check the overall plotting rendering.""" X, y = make_classification(n_classes=2, n_samples=50, random_state=0) pos_label = 1 classifier = LogisticRegression().fit(X, y) classifier.fit(X, y) y_pred = getattr(classifier, response_method)(X) y_pred = y_pred if y_pred.ndim == 1 else y_pred[:, pos_label] # safe guard for the binary if/else construction assert constructor_name in ("from_estimator", "from_predictions") if constructor_name == "from_estimator": display = PrecisionRecallDisplay.from_estimator( classifier, X, y, response_method=response_method) else: display = PrecisionRecallDisplay.from_predictions(y, y_pred, pos_label=pos_label) precision, recall, _ = precision_recall_curve(y, y_pred, pos_label=pos_label) average_precision = average_precision_score(y, y_pred, pos_label=pos_label) np.testing.assert_allclose(display.precision, precision) np.testing.assert_allclose(display.recall, recall) assert display.average_precision == pytest.approx(average_precision) import matplotlib as mpl assert isinstance(display.line_, mpl.lines.Line2D) assert isinstance(display.ax_, mpl.axes.Axes) assert isinstance(display.figure_, mpl.figure.Figure) assert display.ax_.get_xlabel() == "Recall (Positive label: 1)" assert display.ax_.get_ylabel() == "Precision (Positive label: 1)" # plotting passing some new parameters display.plot(alpha=0.8, name="MySpecialEstimator") expected_label = f"MySpecialEstimator (AP = {average_precision:0.2f})" assert display.line_.get_label() == expected_label assert display.line_.get_alpha() == pytest.approx(0.8)
from sklearn.metrics import PrecisionRecallDisplay display = PrecisionRecallDisplay.from_estimator(classifier, X_test, y_test, name="LinearSVC") _ = display.ax_.set_title("2-class Precision-Recall curve") # %% # If we already got the estimated probabilities or scores for # our model, then we can use # :func:`~sklearn.metrics.PrecisionRecallDisplay.from_predictions`. y_score = classifier.decision_function(X_test) display = PrecisionRecallDisplay.from_predictions(y_test, y_score, name="LinearSVC") _ = display.ax_.set_title("2-class Precision-Recall curve") # %% # In multi-label settings # ----------------------- # # The precision-recall curve does not support the multilabel setting. However, # one can decide how to handle this case. We show such an example below. # # Create multi-label data, fit, and predict # ......................................... # # We create a multi-label dataset, to illustrate the precision-recall in # multi-label settings.
def test_plot_precision_recall_pos_label(pyplot, constructor_name, response_method): # check that we can provide the positive label and display the proper # statistics X, y = load_breast_cancer(return_X_y=True) # create an highly imbalanced version of the breast cancer dataset idx_positive = np.flatnonzero(y == 1) idx_negative = np.flatnonzero(y == 0) idx_selected = np.hstack([idx_negative, idx_positive[:25]]) X, y = X[idx_selected], y[idx_selected] X, y = shuffle(X, y, random_state=42) # only use 2 features to make the problem even harder X = X[:, :2] y = np.array(["cancer" if c == 1 else "not cancer" for c in y], dtype=object) X_train, X_test, y_train, y_test = train_test_split( X, y, stratify=y, random_state=0, ) classifier = LogisticRegression() classifier.fit(X_train, y_train) # sanity check to be sure the positive class is classes_[0] and that we # are betrayed by the class imbalance assert classifier.classes_.tolist() == ["cancer", "not cancer"] y_pred = getattr(classifier, response_method)(X_test) # we select the correcponding probability columns or reverse the decision # function otherwise y_pred_cancer = -1 * y_pred if y_pred.ndim == 1 else y_pred[:, 0] y_pred_not_cancer = y_pred if y_pred.ndim == 1 else y_pred[:, 1] if constructor_name == "from_estimator": display = PrecisionRecallDisplay.from_estimator( classifier, X_test, y_test, pos_label="cancer", response_method=response_method, ) else: display = PrecisionRecallDisplay.from_predictions( y_test, y_pred_cancer, pos_label="cancer", ) # we should obtain the statistics of the "cancer" class avg_prec_limit = 0.65 assert display.average_precision < avg_prec_limit assert -np.trapz(display.precision, display.recall) < avg_prec_limit # otherwise we should obtain the statistics of the "not cancer" class if constructor_name == "from_estimator": display = PrecisionRecallDisplay.from_estimator( classifier, X_test, y_test, response_method=response_method, pos_label="not cancer", ) else: display = PrecisionRecallDisplay.from_predictions( y_test, y_pred_not_cancer, pos_label="not cancer", ) avg_prec_limit = 0.95 assert display.average_precision > avg_prec_limit assert -np.trapz(display.precision, display.recall) > avg_prec_limit