class PrecisionRecallCurve(CurveFabric):
def __init__(self, col_score, col_target, name=None, **kwargs):
super().__init__(col_score, col_target, name=name)
self.precision = None
self.recall = None
self.average_precision = None
def fit(self, df):
self.precision, self.recall, _ = precision_recall_curve(
df[self.col_target], df[self.col_score])
self.average_precision = average_precision_score(
df[self.col_target], df[self.col_score])
return self
def plot(self, ax=None, title=None, **kwargs):
if ax is None:
fig, ax = plt.subplots()
self.ax = ax
self.viz = PrecisionRecallDisplay(
precision=self.precision,
recall=self.recall,
average_precision=self.average_precision,
estimator_name=self.name)
if title:
ax.set_title(title, fontsize=14, fontweight='bold')
self.viz.plot(ax=ax, name=self.name, **kwargs)
return self
def plotPrecisionRecallCurve(self, titleAdd: str = None):
from sklearn.metrics import PrecisionRecallDisplay # only supported by newer versions of sklearn
if not self._probabilitiesAvailable:
raise Exception("Precision-recall curve requires probabilities")
if not self.isBinary:
raise Exception(
"Precision-recall curve is not applicable to non-binary classification"
)
probabilities = self.y_predictedClassProbabilities[
self.binaryPositiveLabel]
precision, recall, thresholds = precision_recall_curve(
y_true=self.y_true,
probas_pred=probabilities,
pos_label=self.binaryPositiveLabel)
disp = PrecisionRecallDisplay(precision, recall)
disp.plot()
ax: plt.Axes = disp.ax_
ax.set_xlabel("recall")
ax.set_ylabel("precision")
title = "Precision-Recall Curve"
if titleAdd is not None:
title += "\n" + titleAdd
ax.set_title(title)
ax.xaxis.set_major_locator(plticker.MultipleLocator(base=0.1))
ax.yaxis.set_major_locator(plticker.MultipleLocator(base=0.1))
return disp.figure_
def evaluate(self, x_test, y_test):
y_pred = self.model.predict(x_test)
y_pred = [1 * (x[0] >= 0.5) for x in y_pred]
print('MLP performance on test for', self.feature_name)
print('Accuracy:', accuracy_score(y_test, y_pred), 'Precision:',
precision_score(y_test, y_pred), 'Recall:',
recall_score(y_test, y_pred))
# Confusion matrix
cm = confusion_matrix(y_test, y_pred)
cm_display = ConfusionMatrixDisplay(cm)
# Precision recall
precision, recall, _ = precision_recall_curve(y_test, y_pred)
pr_display = PrecisionRecallDisplay(precision=precision, recall=recall)
# Roc
fpr, tpr, _ = roc_curve(y_test, y_pred)
roc_display = RocCurveDisplay(fpr=fpr, tpr=tpr)
# Figure
figure: Figure = plt.figure(1, figsize=(15, 6))
figure.suptitle('MLP on {}'.format(self.feature_name), fontsize=20)
(ax1, ax2, ax3) = figure.subplots(1, 3)
ax1.set_title('Confusion matrix')
cm_display.plot(ax=ax1)
ax2.set_title('Precision recall')
pr_display.plot(ax=ax2)
ax3.set_title('Roc curve')
roc_display.plot(ax=ax3)
file_name = '{}-mlp.png'.format(self.feature_name)
figure.savefig(
os.path.join(get_folder_path_from_root('images'), file_name))
plt.show()
def test_default_labels(pyplot, average_precision, estimator_name, expected_label):
prec = np.array([1, 0.5, 0])
recall = np.array([0, 0.5, 1])
disp = PrecisionRecallDisplay(
prec, recall, average_precision=average_precision, estimator_name=estimator_name
)
disp.plot()
assert disp.line_.get_label() == expected_label
def plot_ap(y_true, y_pred_proba):
# AP curve
aps = average_precision_score(y_true, y_pred_proba)
precision, recall, _ = precision_recall_curve(y_true, y_pred_proba)
disp = PrecisionRecallDisplay(precision=precision,
recall=recall,
average_precision=aps,
estimator_name=None)
disp.plot()
return disp.figure_
def gold_scores(alignment):
pairs = read(alignment.l2_fn.name)
for score in alignment.scores:
gold_df = score['df'].copy()
gold_df[gold_df.columns] = 0
for x, y in pairs:
gold_df.loc[x, y] = 1
true_values = gold_df.to_numpy().flatten()
pred_values = score['df'].to_numpy().flatten()
print('-------')
print(score['id'])
print('spearman')
rho, p_value = stats.spearmanr(pred_values.argsort(),
true_values.argsort())
print(rho)
print(p_value)
print('kendall tau')
tau, p_value = stats.kendalltau(pred_values.argsort(),
true_values.argsort())
print(tau)
print(p_value)
print('-------')
precision, recall, _ = metrics.precision_recall_curve(
true_values, pred_values)
disp = PrecisionRecallDisplay(precision, recall, 0, score['id'])
disp.plot()
a = score['id']
plt.savefig(f'plots/{a}.png')
plt.close()
# roc_auc = metrics.auc(precision, recall)
# plt.title('Precision-recall curve')
# plt.plot(precision, recall, 'b', label = 'AUC = %0.2f' % roc_auc)
# plt.legend(loc = 'lower right')
# plt.plot([0, 1], [0, 1],'r--')
# plt.xlim([0, 1])
# plt.ylim([0, 1])
# plt.ylabel('Recall')
# plt.xlabel('Precision')
# a = score['id']
# plt.savefig(f'{a}.png')
# plt.close()
print('boa')
# print(read('salsa'))
def test_default_labels(pyplot, average_precision, estimator_name,
expected_label):
"""Check the default labels used in the display."""
precision = np.array([1, 0.5, 0])
recall = np.array([0, 0.5, 1])
display = PrecisionRecallDisplay(
precision,
recall,
average_precision=average_precision,
estimator_name=estimator_name,
)
display.plot()
assert display.line_.get_label() == expected_label
def plot(self, ax=None, figsize=(10, 5)):
if ax is None:
fig, ax = plt.subplots(1, 1, figsize=figsize)
ax.set_title("Precision Recall Curve")
possible_colors = GeneralUtils.shuffled_colors()
for class_index, label in enumerate(self.labels):
precision = self._recall_precision_curve[label]['precision']
recall = self._recall_precision_curve[label]['recall']
average_precision = self._average_precision[label]
viz = PrecisionRecallDisplay(precision=precision,
recall=recall,
average_precision=average_precision,
estimator_name='Classifier')
viz.plot(ax=ax, name=label, color=possible_colors[class_index])
def precision_recall_curve(self, fig_name):
precision, recall, thresholds = precision_recall_curve(
self.label, self.pred)
max_f1 = 0
max_f2 = 0
max_threshold = 0
for p, r, tr in zip(precision, recall, thresholds):
f1 = self.f1_score(p, r)
f2 = self.f2_score(p, r)
if f1 >= max_f1:
max_f1 = f1
max_threshold = tr
if f2 >= max_f2:
max_f2 = f2
viz = PrecisionRecallDisplay(precision=precision, recall=recall)
viz.plot()
if os.path.isdir(self.output_dir):
fig_path = os.path.join(self.output_dir, fig_name)
plt.savefig(fig_path)
plt.close()
detail = self.f1_details(max_threshold)
return round(max_f1, 3), round(max_f2, 3), detail, max_threshold
f"{img_path}{img_name}"].to_numpy()[0][-1])
y_preds.append(0)
#If there was an object and no bounding box was found then it's false negative
if test_images.loc[test_images['PATH'] == f"{img_path}{img_name}",
['class']].iloc[0].to_numpy()[0] == 1:
false_negatives += 1
all += 1
else:
true_negatives += 1
all += 1
print(f"False Negatives {false_negatives} \n"
f"True Negatives {true_negatives} \n"
f"True Positives {true_positives} \n"
f"False Positives {false_positives}")
#cv2.imshow('img', img)
#cv2.waitKey(0)
cv2.imwrite('./results_imgs/{}.png'.format(idx), img)
#Calculate mAP
mAP = average_precision_score(y_true, y_preds)
print(f"Model's mAP: {mAP}")
precision, recall, thresholds = precision_recall_curve(y_true, y_preds)
disp = PrecisionRecallDisplay(precision=precision, recall=recall)
disp.plot()
plt.show()
fpr, tpr, _ = roc_curve(y_test, y_score, pos_label=clf.classes_[1]) roc_display = RocCurveDisplay(fpr=fpr, tpr=tpr).plot() # %% # Create :class:`PrecisionRecallDisplay` ############################################################################## # Similarly, the precision recall curve can be plotted using `y_score` from # the prevision sections. from sklearn.metrics import precision_recall_curve from sklearn.metrics import PrecisionRecallDisplay prec, recall, _ = precision_recall_curve(y_test, y_score, pos_label=clf.classes_[1]) pr_display = PrecisionRecallDisplay(precision=prec, recall=recall).plot() # %% # Combining the display objects into a single plot ############################################################################## # The display objects store the computed values that were passed as arguments. # This allows for the visualizations to be easliy combined using matplotlib's # API. In the following example, we place the displays next to each other in a # row. # sphinx_gallery_thumbnail_number = 4 import matplotlib.pyplot as plt fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8)) roc_display.plot(ax=ax1) pr_display.plot(ax=ax2) plt.show()
# A "micro-average": quantifying score on all classes jointly
precision["micro"], recall["micro"], _ = precision_recall_curve(
Y_test.ravel(), y_score.ravel())
average_precision["micro"] = average_precision_score(Y_test,
y_score,
average="micro")
# %%
# Plot the micro-averaged Precision-Recall curve
# ..............................................
display = PrecisionRecallDisplay(
recall=recall["micro"],
precision=precision["micro"],
average_precision=average_precision["micro"],
)
display.plot()
_ = display.ax_.set_title("Micro-averaged over all classes")
# %%
# Plot Precision-Recall curve for each class and iso-f1 curves
# ............................................................
import matplotlib.pyplot as plt
from itertools import cycle
# setup plot details
colors = cycle(["navy", "turquoise", "darkorange", "cornflowerblue", "teal"])
_, ax = plt.subplots(figsize=(7, 8))
f_scores = np.linspace(0.2, 0.8, num=4)
lines, labels = [], []
if __name__ == '__main__':
x_train, x_test, y_train, y_test = get_dataset()
model = LogisticRegression(multi_class='ovr')
model.fit(x_train, y_train)
b_y = label_binarize(y_test, classes=[0, 1, 2])
y_scores = model.predict_proba(x_test)
print(y_scores)
for i in range(len(np.unique(y_test))):
precision, recall, _ = p_r_curve(b_y[:, i], y_scores[:, i])
ap = compute_ap(recall, precision)
plt.plot(recall,
precision,
drawstyle="steps-post",
label=f'Precision-recall for class {i} (AP = {ap})')
plt.xlabel("Recall")
plt.ylabel("Precision")
plt.title("Precision-Recall curve by ours")
plt.legend(loc="lower left")
# 通过sklear方法进行绘制
_, ax = plt.subplots()
for i in range(len(np.unique(y_test))):
precision, recall, _ = p_r_curve(b_y[:, i], y_scores[:, i])
ap = compute_ap(recall, precision)
display = PrecisionRecallDisplay(recall=recall,
precision=precision,
average_precision=ap)
display.plot(ax=ax, name=f"Precision-recall for class {i}")
ax.set_title("Precision-Recall curve by sklearn")
plt.show()
def create_visualization(model, X_test, Y_test, title="Model Metrics"):
"""Compute the metrics and creates the figures used for model visualization.
Parameters
----------
model: The trained model to get the metrics from
X_test: Data used to evaluate the model
Y_test: Data labels for the evaluation data
title: Title of the figure on which visualizations will be drawn
"""
try:
Y_probabilities = model.predict_proba(X_test)
except AttributeError:
# Must be an SVM w/o probability=True
Y_probabilities = model.decision_function(X_test)
figure, axes = plt.subplots(ncols=3, figsize=(16, 5))
figure.suptitle(title)
# Plot confusion matrix
labels = unique_labels(Y_test)
plot_confusion_matrix(model,
X_test,
Y_test,
ax=axes[0],
normalize="true",
labels=labels)
# Plot precision-recall and ROC curve for each class
for index, class_label in enumerate(labels):
# Plot precision-recall curve
precision, recall, _ = precision_recall_curve(Y_test,
Y_probabilities[:,
index],
pos_label=class_label)
name = f"class {int(class_label)}"
viz = PrecisionRecallDisplay(precision=precision,
recall=recall,
estimator_name=name)
viz.plot(ax=axes[1], name=name)
# Plot ROC curve
fpr, tpr, _ = roc_curve(Y_test,
Y_probabilities[:, index],
pos_label=class_label)
roc_auc = auc(fpr, tpr)
viz = RocCurveDisplay(fpr=fpr,
tpr=tpr,
roc_auc=roc_auc,
estimator_name=name)
viz.plot(ax=axes[2], name=name)
precisions, recalls, fscores, supports = precision_recall_fscore_support(
Y_test, model.predict(X_test))
for index, (precision, recall, fscore, support) in enumerate(
zip(precisions, recalls, fscores, supports)):
print(
f"class {index} - precision: {precision:0.4f}, recall: {recall:0.4f}",
f"fscore: {fscore:0.4f}, support: {support}",
)