class PrecisionRecallCurve(CurveFabric): def __init__(self, col_score, col_target, name=None, **kwargs): super().__init__(col_score, col_target, name=name) self.precision = None self.recall = None self.average_precision = None def fit(self, df): self.precision, self.recall, _ = precision_recall_curve( df[self.col_target], df[self.col_score]) self.average_precision = average_precision_score( df[self.col_target], df[self.col_score]) return self def plot(self, ax=None, title=None, **kwargs): if ax is None: fig, ax = plt.subplots() self.ax = ax self.viz = PrecisionRecallDisplay( precision=self.precision, recall=self.recall, average_precision=self.average_precision, estimator_name=self.name) if title: ax.set_title(title, fontsize=14, fontweight='bold') self.viz.plot(ax=ax, name=self.name, **kwargs) return self
def plotPrecisionRecallCurve(self, titleAdd: str = None): from sklearn.metrics import PrecisionRecallDisplay # only supported by newer versions of sklearn if not self._probabilitiesAvailable: raise Exception("Precision-recall curve requires probabilities") if not self.isBinary: raise Exception( "Precision-recall curve is not applicable to non-binary classification" ) probabilities = self.y_predictedClassProbabilities[ self.binaryPositiveLabel] precision, recall, thresholds = precision_recall_curve( y_true=self.y_true, probas_pred=probabilities, pos_label=self.binaryPositiveLabel) disp = PrecisionRecallDisplay(precision, recall) disp.plot() ax: plt.Axes = disp.ax_ ax.set_xlabel("recall") ax.set_ylabel("precision") title = "Precision-Recall Curve" if titleAdd is not None: title += "\n" + titleAdd ax.set_title(title) ax.xaxis.set_major_locator(plticker.MultipleLocator(base=0.1)) ax.yaxis.set_major_locator(plticker.MultipleLocator(base=0.1)) return disp.figure_
def evaluate(self, x_test, y_test): y_pred = self.model.predict(x_test) y_pred = [1 * (x[0] >= 0.5) for x in y_pred] print('MLP performance on test for', self.feature_name) print('Accuracy:', accuracy_score(y_test, y_pred), 'Precision:', precision_score(y_test, y_pred), 'Recall:', recall_score(y_test, y_pred)) # Confusion matrix cm = confusion_matrix(y_test, y_pred) cm_display = ConfusionMatrixDisplay(cm) # Precision recall precision, recall, _ = precision_recall_curve(y_test, y_pred) pr_display = PrecisionRecallDisplay(precision=precision, recall=recall) # Roc fpr, tpr, _ = roc_curve(y_test, y_pred) roc_display = RocCurveDisplay(fpr=fpr, tpr=tpr) # Figure figure: Figure = plt.figure(1, figsize=(15, 6)) figure.suptitle('MLP on {}'.format(self.feature_name), fontsize=20) (ax1, ax2, ax3) = figure.subplots(1, 3) ax1.set_title('Confusion matrix') cm_display.plot(ax=ax1) ax2.set_title('Precision recall') pr_display.plot(ax=ax2) ax3.set_title('Roc curve') roc_display.plot(ax=ax3) file_name = '{}-mlp.png'.format(self.feature_name) figure.savefig( os.path.join(get_folder_path_from_root('images'), file_name)) plt.show()
def test_default_labels(pyplot, average_precision, estimator_name, expected_label): prec = np.array([1, 0.5, 0]) recall = np.array([0, 0.5, 1]) disp = PrecisionRecallDisplay( prec, recall, average_precision=average_precision, estimator_name=estimator_name ) disp.plot() assert disp.line_.get_label() == expected_label
def plot_ap(y_true, y_pred_proba): # AP curve aps = average_precision_score(y_true, y_pred_proba) precision, recall, _ = precision_recall_curve(y_true, y_pred_proba) disp = PrecisionRecallDisplay(precision=precision, recall=recall, average_precision=aps, estimator_name=None) disp.plot() return disp.figure_
def gold_scores(alignment): pairs = read(alignment.l2_fn.name) for score in alignment.scores: gold_df = score['df'].copy() gold_df[gold_df.columns] = 0 for x, y in pairs: gold_df.loc[x, y] = 1 true_values = gold_df.to_numpy().flatten() pred_values = score['df'].to_numpy().flatten() print('-------') print(score['id']) print('spearman') rho, p_value = stats.spearmanr(pred_values.argsort(), true_values.argsort()) print(rho) print(p_value) print('kendall tau') tau, p_value = stats.kendalltau(pred_values.argsort(), true_values.argsort()) print(tau) print(p_value) print('-------') precision, recall, _ = metrics.precision_recall_curve( true_values, pred_values) disp = PrecisionRecallDisplay(precision, recall, 0, score['id']) disp.plot() a = score['id'] plt.savefig(f'plots/{a}.png') plt.close() # roc_auc = metrics.auc(precision, recall) # plt.title('Precision-recall curve') # plt.plot(precision, recall, 'b', label = 'AUC = %0.2f' % roc_auc) # plt.legend(loc = 'lower right') # plt.plot([0, 1], [0, 1],'r--') # plt.xlim([0, 1]) # plt.ylim([0, 1]) # plt.ylabel('Recall') # plt.xlabel('Precision') # a = score['id'] # plt.savefig(f'{a}.png') # plt.close() print('boa') # print(read('salsa'))
def test_default_labels(pyplot, average_precision, estimator_name, expected_label): """Check the default labels used in the display.""" precision = np.array([1, 0.5, 0]) recall = np.array([0, 0.5, 1]) display = PrecisionRecallDisplay( precision, recall, average_precision=average_precision, estimator_name=estimator_name, ) display.plot() assert display.line_.get_label() == expected_label
def plot(self, ax=None, figsize=(10, 5)): if ax is None: fig, ax = plt.subplots(1, 1, figsize=figsize) ax.set_title("Precision Recall Curve") possible_colors = GeneralUtils.shuffled_colors() for class_index, label in enumerate(self.labels): precision = self._recall_precision_curve[label]['precision'] recall = self._recall_precision_curve[label]['recall'] average_precision = self._average_precision[label] viz = PrecisionRecallDisplay(precision=precision, recall=recall, average_precision=average_precision, estimator_name='Classifier') viz.plot(ax=ax, name=label, color=possible_colors[class_index])
def precision_recall_curve(self, fig_name): precision, recall, thresholds = precision_recall_curve( self.label, self.pred) max_f1 = 0 max_f2 = 0 max_threshold = 0 for p, r, tr in zip(precision, recall, thresholds): f1 = self.f1_score(p, r) f2 = self.f2_score(p, r) if f1 >= max_f1: max_f1 = f1 max_threshold = tr if f2 >= max_f2: max_f2 = f2 viz = PrecisionRecallDisplay(precision=precision, recall=recall) viz.plot() if os.path.isdir(self.output_dir): fig_path = os.path.join(self.output_dir, fig_name) plt.savefig(fig_path) plt.close() detail = self.f1_details(max_threshold) return round(max_f1, 3), round(max_f2, 3), detail, max_threshold
f"{img_path}{img_name}"].to_numpy()[0][-1]) y_preds.append(0) #If there was an object and no bounding box was found then it's false negative if test_images.loc[test_images['PATH'] == f"{img_path}{img_name}", ['class']].iloc[0].to_numpy()[0] == 1: false_negatives += 1 all += 1 else: true_negatives += 1 all += 1 print(f"False Negatives {false_negatives} \n" f"True Negatives {true_negatives} \n" f"True Positives {true_positives} \n" f"False Positives {false_positives}") #cv2.imshow('img', img) #cv2.waitKey(0) cv2.imwrite('./results_imgs/{}.png'.format(idx), img) #Calculate mAP mAP = average_precision_score(y_true, y_preds) print(f"Model's mAP: {mAP}") precision, recall, thresholds = precision_recall_curve(y_true, y_preds) disp = PrecisionRecallDisplay(precision=precision, recall=recall) disp.plot() plt.show()
fpr, tpr, _ = roc_curve(y_test, y_score, pos_label=clf.classes_[1]) roc_display = RocCurveDisplay(fpr=fpr, tpr=tpr).plot() # %% # Create :class:`PrecisionRecallDisplay` ############################################################################## # Similarly, the precision recall curve can be plotted using `y_score` from # the prevision sections. from sklearn.metrics import precision_recall_curve from sklearn.metrics import PrecisionRecallDisplay prec, recall, _ = precision_recall_curve(y_test, y_score, pos_label=clf.classes_[1]) pr_display = PrecisionRecallDisplay(precision=prec, recall=recall).plot() # %% # Combining the display objects into a single plot ############################################################################## # The display objects store the computed values that were passed as arguments. # This allows for the visualizations to be easliy combined using matplotlib's # API. In the following example, we place the displays next to each other in a # row. # sphinx_gallery_thumbnail_number = 4 import matplotlib.pyplot as plt fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8)) roc_display.plot(ax=ax1) pr_display.plot(ax=ax2) plt.show()
# A "micro-average": quantifying score on all classes jointly precision["micro"], recall["micro"], _ = precision_recall_curve( Y_test.ravel(), y_score.ravel()) average_precision["micro"] = average_precision_score(Y_test, y_score, average="micro") # %% # Plot the micro-averaged Precision-Recall curve # .............................................. display = PrecisionRecallDisplay( recall=recall["micro"], precision=precision["micro"], average_precision=average_precision["micro"], ) display.plot() _ = display.ax_.set_title("Micro-averaged over all classes") # %% # Plot Precision-Recall curve for each class and iso-f1 curves # ............................................................ import matplotlib.pyplot as plt from itertools import cycle # setup plot details colors = cycle(["navy", "turquoise", "darkorange", "cornflowerblue", "teal"]) _, ax = plt.subplots(figsize=(7, 8)) f_scores = np.linspace(0.2, 0.8, num=4) lines, labels = [], []
if __name__ == '__main__': x_train, x_test, y_train, y_test = get_dataset() model = LogisticRegression(multi_class='ovr') model.fit(x_train, y_train) b_y = label_binarize(y_test, classes=[0, 1, 2]) y_scores = model.predict_proba(x_test) print(y_scores) for i in range(len(np.unique(y_test))): precision, recall, _ = p_r_curve(b_y[:, i], y_scores[:, i]) ap = compute_ap(recall, precision) plt.plot(recall, precision, drawstyle="steps-post", label=f'Precision-recall for class {i} (AP = {ap})') plt.xlabel("Recall") plt.ylabel("Precision") plt.title("Precision-Recall curve by ours") plt.legend(loc="lower left") # 通过sklear方法进行绘制 _, ax = plt.subplots() for i in range(len(np.unique(y_test))): precision, recall, _ = p_r_curve(b_y[:, i], y_scores[:, i]) ap = compute_ap(recall, precision) display = PrecisionRecallDisplay(recall=recall, precision=precision, average_precision=ap) display.plot(ax=ax, name=f"Precision-recall for class {i}") ax.set_title("Precision-Recall curve by sklearn") plt.show()
def create_visualization(model, X_test, Y_test, title="Model Metrics"): """Compute the metrics and creates the figures used for model visualization. Parameters ---------- model: The trained model to get the metrics from X_test: Data used to evaluate the model Y_test: Data labels for the evaluation data title: Title of the figure on which visualizations will be drawn """ try: Y_probabilities = model.predict_proba(X_test) except AttributeError: # Must be an SVM w/o probability=True Y_probabilities = model.decision_function(X_test) figure, axes = plt.subplots(ncols=3, figsize=(16, 5)) figure.suptitle(title) # Plot confusion matrix labels = unique_labels(Y_test) plot_confusion_matrix(model, X_test, Y_test, ax=axes[0], normalize="true", labels=labels) # Plot precision-recall and ROC curve for each class for index, class_label in enumerate(labels): # Plot precision-recall curve precision, recall, _ = precision_recall_curve(Y_test, Y_probabilities[:, index], pos_label=class_label) name = f"class {int(class_label)}" viz = PrecisionRecallDisplay(precision=precision, recall=recall, estimator_name=name) viz.plot(ax=axes[1], name=name) # Plot ROC curve fpr, tpr, _ = roc_curve(Y_test, Y_probabilities[:, index], pos_label=class_label) roc_auc = auc(fpr, tpr) viz = RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=roc_auc, estimator_name=name) viz.plot(ax=axes[2], name=name) precisions, recalls, fscores, supports = precision_recall_fscore_support( Y_test, model.predict(X_test)) for index, (precision, recall, fscore, support) in enumerate( zip(precisions, recalls, fscores, supports)): print( f"class {index} - precision: {precision:0.4f}, recall: {recall:0.4f}", f"fscore: {fscore:0.4f}, support: {support}", )