def evaluate(self, x_test, y_test): y_pred = self.model.predict(x_test) y_pred = [1 * (x[0] >= 0.5) for x in y_pred] print('MLP performance on test for', self.feature_name) print('Accuracy:', accuracy_score(y_test, y_pred), 'Precision:', precision_score(y_test, y_pred), 'Recall:', recall_score(y_test, y_pred)) # Confusion matrix cm = confusion_matrix(y_test, y_pred) cm_display = ConfusionMatrixDisplay(cm) # Precision recall precision, recall, _ = precision_recall_curve(y_test, y_pred) pr_display = PrecisionRecallDisplay(precision=precision, recall=recall) # Roc fpr, tpr, _ = roc_curve(y_test, y_pred) roc_display = RocCurveDisplay(fpr=fpr, tpr=tpr) # Figure figure: Figure = plt.figure(1, figsize=(15, 6)) figure.suptitle('MLP on {}'.format(self.feature_name), fontsize=20) (ax1, ax2, ax3) = figure.subplots(1, 3) ax1.set_title('Confusion matrix') cm_display.plot(ax=ax1) ax2.set_title('Precision recall') pr_display.plot(ax=ax2) ax3.set_title('Roc curve') roc_display.plot(ax=ax3) file_name = '{}-mlp.png'.format(self.feature_name) figure.savefig( os.path.join(get_folder_path_from_root('images'), file_name)) plt.show()
class PrecisionRecallCurve(CurveFabric): def __init__(self, col_score, col_target, name=None, **kwargs): super().__init__(col_score, col_target, name=name) self.precision = None self.recall = None self.average_precision = None def fit(self, df): self.precision, self.recall, _ = precision_recall_curve( df[self.col_target], df[self.col_score]) self.average_precision = average_precision_score( df[self.col_target], df[self.col_score]) return self def plot(self, ax=None, title=None, **kwargs): if ax is None: fig, ax = plt.subplots() self.ax = ax self.viz = PrecisionRecallDisplay( precision=self.precision, recall=self.recall, average_precision=self.average_precision, estimator_name=self.name) if title: ax.set_title(title, fontsize=14, fontweight='bold') self.viz.plot(ax=ax, name=self.name, **kwargs) return self
def test_precision_recall_display_string_labels(pyplot): # regression test #15738 cancer = load_breast_cancer() X, y = cancer.data, cancer.target_names[cancer.target] lr = make_pipeline(StandardScaler(), LogisticRegression()) lr.fit(X, y) for klass in cancer.target_names: assert klass in lr.classes_ display = PrecisionRecallDisplay.from_estimator(lr, X, y) y_pred = lr.predict_proba(X)[:, 1] avg_prec = average_precision_score(y, y_pred, pos_label=lr.classes_[1]) assert display.average_precision == pytest.approx(avg_prec) assert display.estimator_name == lr.__class__.__name__ err_msg = r"y_true takes value in {'benign', 'malignant'}" with pytest.raises(ValueError, match=err_msg): PrecisionRecallDisplay.from_predictions(y, y_pred) display = PrecisionRecallDisplay.from_predictions(y, y_pred, pos_label=lr.classes_[1]) assert display.average_precision == pytest.approx(avg_prec)
def test_precision_recall_display_name(pyplot, constructor_name, default_label): """Check the behaviour of the name parameters""" X, y = make_classification(n_classes=2, n_samples=100, random_state=0) pos_label = 1 classifier = LogisticRegression().fit(X, y) classifier.fit(X, y) y_pred = classifier.predict_proba(X)[:, pos_label] # safe guard for the binary if/else construction assert constructor_name in ("from_estimator", "from_predictions") if constructor_name == "from_estimator": display = PrecisionRecallDisplay.from_estimator(classifier, X, y) else: display = PrecisionRecallDisplay.from_predictions(y, y_pred, pos_label=pos_label) average_precision = average_precision_score(y, y_pred, pos_label=pos_label) # check that the default name is used assert display.line_.get_label() == default_label.format(average_precision) # check that the name can be set display.plot(name="MySpecialEstimator") assert (display.line_.get_label() == f"MySpecialEstimator (AP = {average_precision:.2f})")
def test_precision_recall_display_pipeline(pyplot, clf): X, y = make_classification(n_classes=2, n_samples=50, random_state=0) with pytest.raises(NotFittedError): PrecisionRecallDisplay.from_estimator(clf, X, y) clf.fit(X, y) display = PrecisionRecallDisplay.from_estimator(clf, X, y) assert display.estimator_name == clf.__class__.__name__
def plotPrecisionRecallCurve(self, titleAdd: str = None): from sklearn.metrics import PrecisionRecallDisplay # only supported by newer versions of sklearn if not self._probabilitiesAvailable: raise Exception("Precision-recall curve requires probabilities") if not self.isBinary: raise Exception( "Precision-recall curve is not applicable to non-binary classification" ) probabilities = self.y_predictedClassProbabilities[ self.binaryPositiveLabel] precision, recall, thresholds = precision_recall_curve( y_true=self.y_true, probas_pred=probabilities, pos_label=self.binaryPositiveLabel) disp = PrecisionRecallDisplay(precision, recall) disp.plot() ax: plt.Axes = disp.ax_ ax.set_xlabel("recall") ax.set_ylabel("precision") title = "Precision-Recall Curve" if titleAdd is not None: title += "\n" + titleAdd ax.set_title(title) ax.xaxis.set_major_locator(plticker.MultipleLocator(base=0.1)) ax.yaxis.set_major_locator(plticker.MultipleLocator(base=0.1)) return disp.figure_
def test_default_labels(pyplot, average_precision, estimator_name, expected_label): prec = np.array([1, 0.5, 0]) recall = np.array([0, 0.5, 1]) disp = PrecisionRecallDisplay( prec, recall, average_precision=average_precision, estimator_name=estimator_name ) disp.plot() assert disp.line_.get_label() == expected_label
def plot_ap(y_true, y_pred_proba): # AP curve aps = average_precision_score(y_true, y_pred_proba) precision, recall, _ = precision_recall_curve(y_true, y_pred_proba) disp = PrecisionRecallDisplay(precision=precision, recall=recall, average_precision=aps, estimator_name=None) disp.plot() return disp.figure_
def gold_scores(alignment): pairs = read(alignment.l2_fn.name) for score in alignment.scores: gold_df = score['df'].copy() gold_df[gold_df.columns] = 0 for x, y in pairs: gold_df.loc[x, y] = 1 true_values = gold_df.to_numpy().flatten() pred_values = score['df'].to_numpy().flatten() print('-------') print(score['id']) print('spearman') rho, p_value = stats.spearmanr(pred_values.argsort(), true_values.argsort()) print(rho) print(p_value) print('kendall tau') tau, p_value = stats.kendalltau(pred_values.argsort(), true_values.argsort()) print(tau) print(p_value) print('-------') precision, recall, _ = metrics.precision_recall_curve( true_values, pred_values) disp = PrecisionRecallDisplay(precision, recall, 0, score['id']) disp.plot() a = score['id'] plt.savefig(f'plots/{a}.png') plt.close() # roc_auc = metrics.auc(precision, recall) # plt.title('Precision-recall curve') # plt.plot(precision, recall, 'b', label = 'AUC = %0.2f' % roc_auc) # plt.legend(loc = 'lower right') # plt.plot([0, 1], [0, 1],'r--') # plt.xlim([0, 1]) # plt.ylim([0, 1]) # plt.ylabel('Recall') # plt.xlabel('Precision') # a = score['id'] # plt.savefig(f'{a}.png') # plt.close() print('boa') # print(read('salsa'))
def test_default_labels(pyplot, average_precision, estimator_name, expected_label): """Check the default labels used in the display.""" precision = np.array([1, 0.5, 0]) recall = np.array([0, 0.5, 1]) display = PrecisionRecallDisplay( precision, recall, average_precision=average_precision, estimator_name=estimator_name, ) display.plot() assert display.line_.get_label() == expected_label
def plot_auc(y_true, y_score, ax, color, label=''): lw = 2 roc_auc = roc_auc_score(y_true=y_true, y_score=y_score) pr_auc = average_precision_score(y_true=y_true, y_score=y_score) fpr, tpr, thresholds = roc_curve(y_true=y_true, y_score=y_score, pos_label=1) prec, recall, _ = precision_recall_curve(y_true=y_true, probas_pred=y_score, pos_label=1) ax[0].plot(fpr, tpr, lw=lw, label='{label} ROC curve (area = {roc_auc:0.4f})'.format(label=label, roc_auc=roc_auc), color=color) ax[0].legend() pr_label = '{label} PR AUC curve (area = {pr_auc:0.4f})' PrecisionRecallDisplay(precision=prec, recall=recall).plot(ax=ax[1], color=color, label=pr_label.format( label=label, pr_auc=pr_auc))
def plot(self, data_original_test): """"Plot ROC-AOC Curves of both original and synthetic in single figure""" X_test, y_test = self._split_xy(data_original_test) fig, ax = plt.subplots(1, 2, figsize=(14, 6)) sns.despine() # roc curve RocCurveDisplay.from_estimator(self.stats_original_, X_test, y_test, name=self.labels[0], color=COLOR_PALETTE[0], ax=ax[0]) RocCurveDisplay.from_estimator(self.stats_synthetic_, X_test, y_test, name=self.labels[1], color=COLOR_PALETTE[1], ax=ax[0]) ax[0].plot([0, 1], [0, 1], linestyle="--", lw=1, color="black", alpha=0.7) ax[0].set_title('ROC Curve') # pr curve PrecisionRecallDisplay.from_estimator(self.stats_original_, X_test, y_test, name=self.labels[0], color=COLOR_PALETTE[0], ax=ax[1]) PrecisionRecallDisplay.from_estimator(self.stats_synthetic_, X_test, y_test, name=self.labels[1], color=COLOR_PALETTE[1], ax=ax[1]) no_skill = len(y_test[y_test == 1]) / len(y_test) ax[1].plot([0, 1], [no_skill, no_skill], lw=1, linestyle='--', color='black', alpha=0.7) ax[1].set_title('Precision-Recall Curve')
def test_precision_recall_display_plotting(pyplot, constructor_name, response_method): """Check the overall plotting rendering.""" X, y = make_classification(n_classes=2, n_samples=50, random_state=0) pos_label = 1 classifier = LogisticRegression().fit(X, y) classifier.fit(X, y) y_pred = getattr(classifier, response_method)(X) y_pred = y_pred if y_pred.ndim == 1 else y_pred[:, pos_label] # safe guard for the binary if/else construction assert constructor_name in ("from_estimator", "from_predictions") if constructor_name == "from_estimator": display = PrecisionRecallDisplay.from_estimator( classifier, X, y, response_method=response_method) else: display = PrecisionRecallDisplay.from_predictions(y, y_pred, pos_label=pos_label) precision, recall, _ = precision_recall_curve(y, y_pred, pos_label=pos_label) average_precision = average_precision_score(y, y_pred, pos_label=pos_label) np.testing.assert_allclose(display.precision, precision) np.testing.assert_allclose(display.recall, recall) assert display.average_precision == pytest.approx(average_precision) import matplotlib as mpl assert isinstance(display.line_, mpl.lines.Line2D) assert isinstance(display.ax_, mpl.axes.Axes) assert isinstance(display.figure_, mpl.figure.Figure) assert display.ax_.get_xlabel() == "Recall (Positive label: 1)" assert display.ax_.get_ylabel() == "Precision (Positive label: 1)" # plotting passing some new parameters display.plot(alpha=0.8, name="MySpecialEstimator") expected_label = f"MySpecialEstimator (AP = {average_precision:0.2f})" assert display.line_.get_label() == expected_label assert display.line_.get_alpha() == pytest.approx(0.8)
def plot(self, ax=None, figsize=(10, 5)): if ax is None: fig, ax = plt.subplots(1, 1, figsize=figsize) ax.set_title("Precision Recall Curve") possible_colors = GeneralUtils.shuffled_colors() for class_index, label in enumerate(self.labels): precision = self._recall_precision_curve[label]['precision'] recall = self._recall_precision_curve[label]['recall'] average_precision = self._average_precision[label] viz = PrecisionRecallDisplay(precision=precision, recall=recall, average_precision=average_precision, estimator_name='Classifier') viz.plot(ax=ax, name=label, color=possible_colors[class_index])
def plot(self, ax=None, title=None, **kwargs): if ax is None: fig, ax = plt.subplots() self.ax = ax self.viz = PrecisionRecallDisplay( precision=self.precision, recall=self.recall, average_precision=self.average_precision, estimator_name=self.name) if title: ax.set_title(title, fontsize=14, fontweight='bold') self.viz.plot(ax=ax, name=self.name, **kwargs) return self
def precision_recall_curve(self, fig_name): precision, recall, thresholds = precision_recall_curve( self.label, self.pred) max_f1 = 0 max_f2 = 0 max_threshold = 0 for p, r, tr in zip(precision, recall, thresholds): f1 = self.f1_score(p, r) f2 = self.f2_score(p, r) if f1 >= max_f1: max_f1 = f1 max_threshold = tr if f2 >= max_f2: max_f2 = f2 viz = PrecisionRecallDisplay(precision=precision, recall=recall) viz.plot() if os.path.isdir(self.output_dir): fig_path = os.path.join(self.output_dir, fig_name) plt.savefig(fig_path) plt.close() detail = self.f1_details(max_threshold) return round(max_f1, 3), round(max_f2, 3), detail, max_threshold
def save_pr_curve(filename, y_true, y_score): """ filename:出力ファイル名 y_true:正解(2クラス 0 or 1) y_score:予測(2クラス クラス1の予測確率(0.0-1.0)) """ prec, recall, thresholds = precision_recall_curve(y_true, y_score) auc_score = auc(recall, prec) pr_display = PrecisionRecallDisplay( precision=prec, recall=recall).plot(name="PR curve (area = %0.3f)" % auc_score) plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.savefig(filename) plt.clf() return auc_score
def plot_precision_recall_curves(self, level: shared.enums.EvaluationLevel, dataset_type: shared.enums.DatasetType, font_size: int = 10): """ Plots precision-recall curves for each class in self._predictor.get_classes(). """ #set matplotlib font size globally plt.rcParams['font.size'] = font_size objs = self.__get_objects_according_to_evaluation_level( level=level, dataset_type=dataset_type) classes = self._predictor.get_classes() for Class in tqdm(classes): y_preds_raw = [] # list of the predicted percentages y_true = [] # list of True and False for obj in objs: # if the raw predictions contain NaN values, this is mostly because the wsi/case did not contain any tile # and therefore during prediction calculation a division by 0 resulted in NaN values # This is fixed in the latest version of the patient_manager. It now checks for tilesummaries, that do not # contain any top tile if (numpy.isnan(list(obj.predictions_raw.values())).any()): continue y_preds_raw.append(obj.predictions_raw[Class]) y_true.append((Class in obj.get_labels())) #print(f'{Class}') #print(f'y_true: {y_true}') #print(f'y_preds_raw: {y_preds_raw}') #print('') #print('------------------------------------------------------------------------') #print('') precision, recall, thresholds = precision_recall_curve(y_true, y_preds_raw, pos_label=1) average_precision = average_precision_score(y_true, y_preds_raw) pr_display = PrecisionRecallDisplay( precision=precision, recall=recall, average_precision=average_precision, estimator_name=Class).plot() #set matplotlib font size back to default plt.rcParams['font.size'] = 10
def plot_pr_curve(stats, outfile): print("Plotting Stat Curves") for c in ['prc']: # plt.figure() for s in stats: if s[c] is not None: PrecisionRecallDisplay( precision=s[c][0], recall=s[c][1], average_precision=s['ap']).plot(label=s['name'] + '_' + c) break plt.title( outfile.split('/')[-1] + '_' + c + ' || AP : ' + str(np.round(s['ap'], 3))) plt.legend(loc=3 if c == 'prc' else 4) plt.ylabel('precision' if c == 'prc' else '1-spec.') plt.xlabel('recall') plt.xticks(np.arange(0, 1.1, step=0.1)) plt.yticks(np.arange(0, 1.1, step=0.1)) plt.savefig(outfile + 'x_' + c + '.png') plt.close()
def eval_cnns(model, val_data): model.eval() val_features = val_data.features_input.to(device) val_times = val_data.times_input.to(device) val_words_labels = val_data.words_labels_input.to(device) val_preds = model1(val_features, val_times) utts = val_words_labels.shape[0] words = val_words_labels.shape[1] word_sums = torch.zeros(utts, words) for idx in range(len(val_times)): timetable = val_times[idx] for jdx in range(len(timetable)): span = timetable[jdx] if span[1] != 0. and span[0] != span[1]: start = int(span[0]) + 1 end = int(span[1]) + 1 word_sums[idx, jdx] = sum(val_preds[idx][start:end]) act_sig = nn.Sigmoid() val_preds = act_sig(word_sums).cpu().detach().numpy().flatten() val_preds_bin = np.where(val_preds > 0.5, 1, 0) val_labels = np.array(val_words_labels.squeeze(1).cpu().detach().numpy(), dtype=int).flatten() with open("eval/cnn_model.report", "w") as reportfile: reportfile.write( classification_report(val_labels, val_preds_bin, digits=4)) prec, rec, _ = precision_recall_curve(val_labels, val_preds) prec_rec_graph = PrecisionRecallDisplay(prec, rec, average_precision=0.5, estimator_name="CNN").plot() plt.savefig("eval/cnn_model_prec_rec_graph.png")
import numpy as np y = np.array([0, 1, 1, 0, 1, 0, 1, 0, 0, 1]) y_score = np.array([0.7, 0.8, 0.3, 0.4, 0.9, 0.6, 0.99, 0.1, 0.2, 0.5]) from sklearn.metrics import precision_recall_curve, PrecisionRecallDisplay, auc my_precision, my_recall, _ = precision_recall_curve(y_true=y, probas_pred=y_score, pos_label=1) PrecisionRecallDisplay(precision=my_precision, recall=my_recall).plot() import matplotlib.pyplot as plt plt.savefig('10-p-pr.pdf')
f"{img_path}{img_name}"].to_numpy()[0][-1]) y_preds.append(0) #If there was an object and no bounding box was found then it's false negative if test_images.loc[test_images['PATH'] == f"{img_path}{img_name}", ['class']].iloc[0].to_numpy()[0] == 1: false_negatives += 1 all += 1 else: true_negatives += 1 all += 1 print(f"False Negatives {false_negatives} \n" f"True Negatives {true_negatives} \n" f"True Positives {true_positives} \n" f"False Positives {false_positives}") #cv2.imshow('img', img) #cv2.waitKey(0) cv2.imwrite('./results_imgs/{}.png'.format(idx), img) #Calculate mAP mAP = average_precision_score(y_true, y_preds) print(f"Model's mAP: {mAP}") precision, recall, thresholds = precision_recall_curve(y_true, y_preds) disp = PrecisionRecallDisplay(precision=precision, recall=recall) disp.plot() plt.show()
def test_plot_precision_recall_pos_label(pyplot, constructor_name, response_method): # check that we can provide the positive label and display the proper # statistics X, y = load_breast_cancer(return_X_y=True) # create an highly imbalanced version of the breast cancer dataset idx_positive = np.flatnonzero(y == 1) idx_negative = np.flatnonzero(y == 0) idx_selected = np.hstack([idx_negative, idx_positive[:25]]) X, y = X[idx_selected], y[idx_selected] X, y = shuffle(X, y, random_state=42) # only use 2 features to make the problem even harder X = X[:, :2] y = np.array(["cancer" if c == 1 else "not cancer" for c in y], dtype=object) X_train, X_test, y_train, y_test = train_test_split( X, y, stratify=y, random_state=0, ) classifier = LogisticRegression() classifier.fit(X_train, y_train) # sanity check to be sure the positive class is classes_[0] and that we # are betrayed by the class imbalance assert classifier.classes_.tolist() == ["cancer", "not cancer"] y_pred = getattr(classifier, response_method)(X_test) # we select the correcponding probability columns or reverse the decision # function otherwise y_pred_cancer = -1 * y_pred if y_pred.ndim == 1 else y_pred[:, 0] y_pred_not_cancer = y_pred if y_pred.ndim == 1 else y_pred[:, 1] if constructor_name == "from_estimator": display = PrecisionRecallDisplay.from_estimator( classifier, X_test, y_test, pos_label="cancer", response_method=response_method, ) else: display = PrecisionRecallDisplay.from_predictions( y_test, y_pred_cancer, pos_label="cancer", ) # we should obtain the statistics of the "cancer" class avg_prec_limit = 0.65 assert display.average_precision < avg_prec_limit assert -np.trapz(display.precision, display.recall) < avg_prec_limit # otherwise we should obtain the statistics of the "not cancer" class if constructor_name == "from_estimator": display = PrecisionRecallDisplay.from_estimator( classifier, X_test, y_test, response_method=response_method, pos_label="not cancer", ) else: display = PrecisionRecallDisplay.from_predictions( y_test, y_pred_not_cancer, pos_label="not cancer", ) avg_prec_limit = 0.95 assert display.average_precision > avg_prec_limit assert -np.trapz(display.precision, display.recall) > avg_prec_limit
y_score = clf.decision_function(X_test) fpr, tpr, _ = roc_curve(y_test, y_score, pos_label=clf.classes_[1]) roc_display = RocCurveDisplay(fpr=fpr, tpr=tpr).plot() # %% # Create :class:`PrecisionRecallDisplay` ############################################################################## # Similarly, the precision recall curve can be plotted using `y_score` from # the prevision sections. from sklearn.metrics import precision_recall_curve from sklearn.metrics import PrecisionRecallDisplay prec, recall, _ = precision_recall_curve(y_test, y_score, pos_label=clf.classes_[1]) pr_display = PrecisionRecallDisplay(precision=prec, recall=recall).plot() # %% # Combining the display objects into a single plot ############################################################################## # The display objects store the computed values that were passed as arguments. # This allows for the visualizations to be easliy combined using matplotlib's # API. In the following example, we place the displays next to each other in a # row. # sphinx_gallery_thumbnail_number = 4 import matplotlib.pyplot as plt fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8)) roc_display.plot(ax=ax1)
# Plot the Precision-Recall curve # ............................... # # To plot the precision-recall curve, you should use # :class:`~sklearn.metrics.PrecisionRecallDisplay`. Indeed, there is two # methods available depending if you already computed the predictions of the # classifier or not. # # Let's first plot the precision-recall curve without the classifier # predictions. We use # :func:`~sklearn.metrics.PrecisionRecallDisplay.from_estimator` that # computes the predictions for us before plotting the curve. from sklearn.metrics import PrecisionRecallDisplay display = PrecisionRecallDisplay.from_estimator(classifier, X_test, y_test, name="LinearSVC") _ = display.ax_.set_title("2-class Precision-Recall curve") # %% # If we already got the estimated probabilities or scores for # our model, then we can use # :func:`~sklearn.metrics.PrecisionRecallDisplay.from_predictions`. y_score = classifier.decision_function(X_test) display = PrecisionRecallDisplay.from_predictions(y_test, y_score, name="LinearSVC") _ = display.ax_.set_title("2-class Precision-Recall curve") # %%
if __name__ == '__main__': x_train, x_test, y_train, y_test = get_dataset() model = LogisticRegression(multi_class='ovr') model.fit(x_train, y_train) b_y = label_binarize(y_test, classes=[0, 1, 2]) y_scores = model.predict_proba(x_test) print(y_scores) for i in range(len(np.unique(y_test))): precision, recall, _ = p_r_curve(b_y[:, i], y_scores[:, i]) ap = compute_ap(recall, precision) plt.plot(recall, precision, drawstyle="steps-post", label=f'Precision-recall for class {i} (AP = {ap})') plt.xlabel("Recall") plt.ylabel("Precision") plt.title("Precision-Recall curve by ours") plt.legend(loc="lower left") # 通过sklear方法进行绘制 _, ax = plt.subplots() for i in range(len(np.unique(y_test))): precision, recall, _ = p_r_curve(b_y[:, i], y_scores[:, i]) ap = compute_ap(recall, precision) display = PrecisionRecallDisplay(recall=recall, precision=precision, average_precision=ap) display.plot(ax=ax, name=f"Precision-recall for class {i}") ax.set_title("Precision-Recall curve by sklearn") plt.show()
def LSTM_trainer(input_dim, hidden_dim, layer_dim, output_dim, seq_dim, error, lr, optimizer, num_epochs, train_loader, test_loader): model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim) if error == 'BCELogit': error = nn.BCEWithLogitsLoss() else: error = nn.BCEWithLogitsLoss() if optimizer == 'Adadelta': optimizer = torch.optim.Adadelta(model.parameters(), lr=lr) else: optimizer = torch.optim.Adadelta(model.parameters(), lr=lr) loss_list_test = [] loss_list_train = [] iteration_list = [] accuracy_list = [] count = 0 for epoch in range(num_epochs): for i, (images, labels) in enumerate(train_loader): # Load images as a torch tensor with gradient accumulation abilities images = images.view(-1, seq_dim, input_dim).requires_grad_() # Clear gradients w.r.t. parameters optimizer.zero_grad() # Forward pass to get output/logits # outputs.size 100, 10 outputs = model(images) # Calculate Loss: softmax --> cross entropy loss labels = labels.unsqueeze(1) loss_train = error(outputs, labels.float()) # Getting gradients loss_train.backward() # Updating parameters optimizer.step() count += 1 if count % 500 == 0: # Calculate Accuracy predictions_list = [] labels_list = [] outputs_list = [] correct = 0 total = 0 size = 0 loss_test = 0 for images, labels in test_loader: images = images.view(-1, seq_dim, input_dim) # Forward pass only to get logits/output outputs = model(images) labels = labels.unsqueeze(1) loss_test = error(outputs, labels.float()) loss_test += loss_test.data.item()*labels.shape[0] size += labels.shape[0] predictions = torch.round(torch.sigmoid(outputs)) predictions_list.append(predictions.detach().numpy()) labels_list.append(labels.detach().numpy()) outputs_list.append(outputs.detach().numpy()) loss_list_test.append(loss_test/size) outputs_list = np.vstack(outputs_list) predictions_list = np.vstack(predictions_list) labels_list = np.concatenate(labels_list) prec, recall, thresholds = precision_recall_curve(labels_list, outputs_list) pr_display = PrecisionRecallDisplay(precision=prec, recall=recall, average_precision = 0, estimator_name = 'LSTM' ).plot() fpr, tpr, thresholds = roc_curve(labels_list, outputs_list) plt.plot(fpr, tpr, label = roc_auc_score(labels_list, outputs_list)) target_spec = 0.90 spec = [1-i for i in fpr] idx_spec = min(enumerate(spec), key=lambda x: abs(x[1]-target_spec)) #print('Specificity:' , spec[idx_spec[0]-2], 'Sensitivity:' , tpr[idx_spec[0]-2]) print('Specificity:' , spec[idx_spec[0]-1], 'Sensitivity:' , tpr[idx_spec[0]-1]) print('Specificity:' , spec[idx_spec[0]], 'Sensitivity:' , tpr[idx_spec[0]]) print('Specificity:' , spec[idx_spec[0]+1], 'Sensitivity:' , tpr[idx_spec[0]+1]) #print('Specificity:' , spec[idx_spec[0]+2], 'Sensitivity:' , tpr[idx_spec[0]+2]) print(classification_report(labels_list, predictions_list)) print('Matthews_corrcoef' , matthews_corrcoef(labels_list, predictions_list)) print('roc_auc_score' , roc_auc_score(labels_list, predictions_list)) loss_list_train.append(loss_train.data.item()) iteration_list.append(count) # Print Loss print('Iteration: {}. Loss_train: {}. Loss_test {}.'.format(count, loss_train.data.item(), loss_test/size))
pos_label = clf.classes_[1] fpr, tpr, _ = roc_curve(y_test, y_score, pos_label=pos_label) AUC = auc(fpr, tpr) roc_display = RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=AUC, estimator_name='demo').plot() #%% Create PR from sklearn.metrics import precision_recall_curve from sklearn.metrics import average_precision_score from sklearn.metrics import PrecisionRecallDisplay pos_label = clf.classes_[1] prec, recall, _ = precision_recall_curve(y_test, y_score, pos_label=pos_label) # alternative AUCpr (~AP), with a different computing method AP = average_precision_score(y_test, y_score, pos_label=pos_label) pr_display = PrecisionRecallDisplay(precision=prec, recall=recall, average_precision=AP, estimator_name='demo').plot() AUCpr = auc(x=recall, y=prec) #%% Combining the display objects (ROC and PR) into a single plot import matplotlib.pyplot as plt fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8)) roc_display.plot(ax=ax1) pr_display.plot(ax=ax2) plt.show()
def precision_recall(self, y_true, y_pred, average_precision): precisions, recalls, _ = precision_recall_curve(y_true, y_pred, pos_label=5) PrecisionRecallDisplay(precision = precisions, recall = recalls, average_precision = average_precision, estimator_name = "AP").plot() plt.title('Metrique Precision & Recall') plt.savefig('static/images/outputs/metriquePR.png', dpi=100) plt.clf()
tpr=tpr, roc_auc=AUC, estimator_name='logestic regression').plot() ''' Here we apply percesion and recall curve to see how much both variables changes when we change the threshold ''' from sklearn.metrics import precision_recall_curve from sklearn.metrics import PrecisionRecallDisplay from sklearn.metrics import average_precision_score average_precision = average_precision_score(y_test, y_score, pos_label='1') prec, recall, _ = precision_recall_curve(y_test, y_score, pos_label=pipeline1.classes_[1]) pr_display = PrecisionRecallDisplay( precision=prec, recall=recall, average_precision=average_precision, estimator_name='logestic regression').plot() #%% ''' Here we would like to take advantage of the classification_report module in sklearn ''' from sklearn.metrics import classification_report class_matrix = classification_report(y_test, y_pred) #%%