def plot_confusion_matrix(labels: np.ndarray, predictions: np.ndarray, class_label_names: Optional[Dict[Union[str, int], Union[str, int]]] = None, normalize: Optional[str] = None, title_fontsize: Optional[int] = 12, x_label_fontsize: Optional[int] = 12, y_label_fontsize: Optional[int] = 12, heatmap_color: Optional[str] = 'Greens') -> None: """Plot confusion matrix for a multiclass classification model. Args: labels: An array of true labels containing multiclass labels. predictions: An array of predictions containing multiclass labels. class_label_names: Dictionary of multiclass labels and corresponding target names. The type of both class lable and target names can be either 'int' or 'str'. E.g. {0: 'low_value', 1: 'mid_value', 2: 'high_value'}. normalize: A parameter controlling whether to normalize the counts in the matrix. title_fontsize: Font size of the figure title. x_label_fontsize: Font size of the x axis labels. y_label_fontsize: Font size of the y axis labels. heatmap_color: Color of the heatmap plot. Returns: Heatmap of confusion matrix. """ utils.assert_label_and_prediction_length_match(labels, predictions) if class_label_names is None: class_labels = list(set(labels)) target_names = ['%s' % l for l in class_labels] else: class_labels = list(class_label_names.keys()) target_names = list(class_label_names.values()) plot = ConfusionMatrixDisplay.from_predictions(y_true=labels, y_pred=predictions, labels=np.unique(labels), display_labels=target_names, normalize=normalize, include_values=True, cmap=heatmap_color) plot.ax_.set_title('Confusion matrix', fontsize=title_fontsize) plot.ax_.set_xlabel('Predicted label', fontsize=x_label_fontsize) plot.ax_.set_ylabel('Actual label', fontsize=y_label_fontsize) plt.show()
def display_confusion_matrix(y, y_pred, cm_filename): from sklearn.metrics import confusion_matrix from sklearn.metrics import ConfusionMatrixDisplay import matplotlib.pyplot as plt cmat = confusion_matrix(y.cpu().numpy(), y_pred.cpu().numpy()) for c in range(cmat.shape[0]): cmat[c, c] = 0.0 _, ax = plt.subplots(1, 1, figsize=(16, 16)) ConfusionMatrixDisplay(cmat).plot(ax=ax) if cm_filename is None: plt.show() else: plt.savefig(cm_filename) plt.close('all')
def evaluate_model(test_labels, params, scores, predictions): ''' Preparing information to show. :param test_labels: labels of test set :param params: parameters :param scores: scores of images per each class :param predictions: predicted labels :return: mean_accuracy, disp_confusion_matrix, indices_of_the_largest_error_images ''' print_statement('Evaluation of the model:') display_labels = get_ordered_list(test_labels) mean_accuracy = sklearn.metrics.accuracy_score(test_labels, predictions) confusion_matrix = sklearn.metrics.confusion_matrix(test_labels, predictions) disp_confusion_matrix = ConfusionMatrixDisplay(confusion_matrix, display_labels).plot() indices_of_the_largest_error_images = get_indices_of_the_largest_error_images(scores, test_labels, params) return mean_accuracy, disp_confusion_matrix, indices_of_the_largest_error_images
def plot_confusion_matrix(fp_type, preds, ax): """Plots confusion matrices for visualising predictions given as argument :param fp_type : specifies which of Handcrafted or Learned was used to generate predictions :param preds : predictions to visualise :param ax : Axes object to plot to """ cm = confusion_matrix(ground_truth, preds, labels) cm_plot = ConfusionMatrixDisplay(cm, labels) plot_cm_custom(cm_plot, ax=ax, values_format='d') ttl = ax.title ttl.set_text(fp_type) ttl.set_fontweight('bold') ttl.set_position([0.5, -0.3])
def plot_confusion_matrix(y_true, y_predicted, labels=[], ax=None, output=None): if ax is None: _, ax = plt.subplots(1, 1, figsize=(8, 8)) cm = confusion_matrix(y_true, y_predicted) cm = 100 * cm / cm.sum(axis=1)[:, np.newaxis] # ax.xaxis.tick_top() ConfusionMatrixDisplay( confusion_matrix=cm, display_labels=labels, ).plot(include_values=True, cmap=plt.cm.Blues, values_format='0.0f', ax=ax, xticks_rotation='vertical') if output is not None: plt.savefig(output)
def label_show_confusion_matrix(y_test, y_pred): from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix from matplotlib import pyplot as plt labels = ['Clean', 'Leak'] # print("Report:", classification_report(y_test, y_pred)) fig = plt.figure() ax = fig.add_subplot(111) ConfusionMatrixDisplay(confusion_matrix(y_pred, y_test, labels=[0, 1]), display_labels=labels).plot(values_format=".0f", ax=ax, cmap=plt.cm.Blues) plt.xlabel('Predicted') plt.ylabel('Actual') plt.show()
def predict(self, img_path, model_path, visualize=False): model = Classifier() ckpt = torch.load(model_path) model.load_state_dict(ckpt) model = model.to(self.device) model.eval() images = [f for f in os.listdir(img_path) if f.endswith('.png')] success = 0 y_true, y_pred = [], [] for imgname in images: image = cv2.imread(os.path.join(img_path, imgname)) img = image.copy() / 255.0 img = np.transpose(img, (2, 0, 1)) img = np.expand_dims(img, axis=0) img = torch.from_numpy(img).to(self.device).float() pred_label = model(img).cpu().detach().numpy() if ("Lateral" in imgname) and (pred_label < 0.5): success += 1 elif ("AP" in imgname) and (pred_label > 0.5): success += 1 true_label = "Lateral" if "Lateral" in imgname else "AP" pred_label = "Lateral" if pred_label < 0.5 else "AP" y_true.append(true_label) y_pred.append(pred_label) if visualize: cv2.putText(image, "pred: {}".format(pred_label), (20, 20), cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 255), 2) cv2.putText(image, "true: {}".format(true_label), (20, 40), cv2.FONT_HERSHEY_PLAIN, 1, (0, 255, 0), 2) cv2.imshow('predicted', image) cv2.waitKey(0) print("[INFO] accuracy is : {:.2f} %".format(100 * success / len(images))) cm = confusion_matrix(y_true, y_pred) print( classification_report(y_true, y_pred, target_names=["AP", "Lateral"])) #print("[INFO] confusion matrix:\n {}".format(cm)) disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=("AP", "Lateral")).plot() plt.show()
def plot_cfs_matrix(y_true,y_pred,labels): # if include_noise: # cm = confusion_matrix(y_true,y_pred,labels=["Trocar 1","Trocar 2","Trocar 3","Trocar 4","Incorrect Data"],normalize='true') # disp = ConfusionMatrixDisplay(confusion_matrix=cm, # display_labels=["Trocar 1","Trocar 2","Trocar 3","Trocar 4","Incorrect Data"]).plot() # else: # cm = confusion_matrix(y_true,y_pred,labels=["Trocar 1","Trocar 2","Trocar 3","Trocar 4"],normalize='true') # disp = ConfusionMatrixDisplay(confusion_matrix=cm, # display_labels=["Trocar 1","Trocar 2","Trocar 3","Trocar 4"]).plot() cm = confusion_matrix(y_true,y_pred,labels=labels,normalize='true') disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels).plot() # NOTE: Fill all variables here with default values of the plot_confusion_matrix plt.show()
def evaluate(): print("evaluating model!" ) # replace this with code to evaluate what must be evaluated df = pd.read_csv("predictions.csv", encoding='iso-8859-1') accuracy = accuracy_score(df['actual'], df['predictions']) * 100 accuracy = round(accuracy, 2) print("Accuracy score on Test data for SVC is: ", accuracy, '%') file = open('accuracy.txt', 'w') file.write(str(accuracy)) file.close() conf_mat = confusion_matrix(df['actual'], df['predictions'], labels=["male", "female"]) ConfusionMatrixDisplay(confusion_matrix=conf_mat, display_labels=["male", "female"]).plot() plt.title('Confusion-matrix') plt.savefig('Confusion-matrix.png')
def evaluate(model: nn.Module, test_loader: torch.utils.data.DataLoader, loss_fn=None, writer: SummaryWriter = None, epoch: int = -1): with torch.no_grad(): model.eval() epoch_losses = [] epoch_accuracy = [] conf_matrix = np.zeros((2, 2), dtype='int32') for step, (x, x_len, y) in enumerate(test_loader): x, y = x.cuda(), y.cuda() y_pred = model(x, x_len) y_argmax = torch.argmax(y_pred, 1) accuracy = y_argmax.eq(y).sum().item() / y.shape[0] epoch_accuracy.append(accuracy) conf_matrix += confusion_matrix(y.cpu().numpy(), y_argmax.cpu().numpy()) if loss_fn: loss_val = loss_fn(y_pred, y) epoch_losses.append(loss_val.item()) print(' Test batch {} of {}'.format(step + 1, len(test_loader)), file=sys.stderr) print( f'Test loss: {np.mean(epoch_losses):.4f}, accuracy: {np.mean(epoch_accuracy):.4f}' ) if writer: writer.add_scalar('Loss/test', np.mean(epoch_losses), global_step=epoch) writer.add_scalar('Accuracy/test', np.mean(epoch_accuracy), global_step=epoch) conf_plot = ConfusionMatrixDisplay( confusion_matrix=conf_matrix, display_labels=['ham', 'spam']).plot(cmap='Blues', values_format='d') writer.add_figure('ConfusionMatrix', conf_plot.figure_, global_step=epoch, close=True) return np.mean(epoch_accuracy)
def evaluation(X_Actual, Y_actual, classifier, classifier_name): Y_predicted = classifier.predict(X_Actual) cm = confusion_matrix(Y_actual, Y_predicted) ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=classifier.classes_).plot() plt.title(classifier_name) #5 fold cross validation x = cross_val_score(classifier, X_Actual, Y_actual, cv=5) print("cross validation score:", x, "\n") Yscores = cross_val_predict(classifier, X_Actual, Y_actual, cv=5,\ method="predict_proba") Yscores = Yscores[:, -1] #ROC curve fpr, tpr, thresholds = roc_curve(Y_actual, Yscores) #Metrics tn, fp, fn, tp = cm.ravel() sensitivity = tp / (tp + fn) fpr_cm = fp / (fp + tn) specificity = tn / (tn + fp) G = math.sqrt(sensitivity * specificity) LP = sensitivity / (1 - specificity) LR = (1 - sensitivity) / specificity #DP=(math.sqrt(3)/math.pi)*(math.log(sensitivity/(1-sensitivity))+math.log(specificity/(1-specificity))) gamma = sensitivity - (1 - sensitivity) BA = (1 / 2) * (sensitivity + specificity) WBA = (3 / 4) * (sensitivity) + (1 / 4) * specificity index_metrics = ["false alarm","hit rate","precision","accuracy","recall", \ "AUC","Kolmogorov–Smirnov","G-mean","Positive Likelihood Ratio",\ "Negative Likelihood Ratio", "Youden Index",\ "Balanced Accuracy", "Weighted Balance Accuracy"] d = {classifier_name: [fpr_cm, sensitivity, precision_score(Y_actual, Y_predicted)\ ,accuracy_score(Y_actual, Y_predicted),\ recall_score(Y_actual, Y_predicted),\ roc_auc_score(Y_actual, Yscores),\ stats.ks_2samp(Y_actual, Yscores)[0],G,LP,\ LR,gamma,BA,WBA]} df_metrics = pd.DataFrame(d, index_metrics) return fpr, tpr, classifier_name, df_metrics
def test_confusion_matrix_standard_format(pyplot): cm = np.array([[10000000, 0], [123456, 12345678]]) plotted_text = ConfusionMatrixDisplay(cm, [False, True]).plot().text_ # Values should be shown as whole numbers 'd', # except the first number which should be shown as 1e+07 (longer length) # and the last number will be showns as 1.2e+07 (longer length) test = [t.get_text() for t in plotted_text.ravel()] assert test == ['1e+07', '0', '123456', '1.2e+07'] cm = np.array([[0.1, 10], [100, 0.525]]) plotted_text = ConfusionMatrixDisplay(cm, [False, True]).plot().text_ # Values should now formatted as '.2g', since there's a float in # Values are have two dec places max, (e.g 100 becomes 1e+02) test = [t.get_text() for t in plotted_text.ravel()] assert test == ['0.1', '10', '1e+02', '0.53']
def show_confusion_matrix(confusion, index_range=(0, 11), kappas=None): """Diplay all confusion matrix within range :param confusion: the list of confusion matrices :type confusion: List: np.ndarray :param index_range: the range of indices within the confusion param to display, defaults to (0, 10) :type index_range: tuple(int), optional """ fig, axes = plt.subplots(nrows=len(confusion), figsize=(7, len(confusion) * 10)) ax = axes.ravel() for i in range(index_range[0], index_range[1]): if kappas is not None: #print('Kappa: ', kappas[i]) kappa = round(kappas[i], 2) if i == 0: # special case labels cmd = ConfusionMatrixDisplay( confusion[i], display_labels=[i for i in range(0, 10)]) cmd.plot(ax=ax[i]) if not kappas is None: cmd.ax_.set_title(f'All classes\nKappa: {kappa}', fontsize=20) # type: ignore else: cmd.ax_.set_title('All classes', fontsize=20) else: cmd = ConfusionMatrixDisplay(confusion[i], display_labels=['True', 'False']) cmd.plot(ax=ax[i]) label = label_def.get(i - 1, i) if not kappas is None: cmd.ax_.set_title(f'{label}\n Kappa: {kappa}', fontsize=20) # type: ignore else: cmd.ax_.set_title(f'{label}', fontsize=20) plt.subplots_adjust(wspace=0.40, hspace=0.1) #axes.flat[-1].set_visible(False) plt.show()
def run(args): # Cancer types cancer_types = pd.read_csv(filepath / 'data/combined_cancer_types', sep='\t', names=['CELL', 'CTYPE']) # Concat preds from all runs runs_dirs = [Path(p) for p in glob(str(trn_path / 'run_*'))] prd_te_all = agg_preds_from_cls_runs(runs_dirs, phase='_te.csv') prd_te_all = update_prd_table(prd_te_all, cancer_types=cancer_types) # Save aggregated master table prd_te_all.to_csv(trn_path / 'prd_te_all.csv', index=False) # Confusion Matrix conf = confusion_matrix(prd_te_all.y_true_cls, prd_te_all.y_pred_cls, normalize=None) conf_plot = ConfusionMatrixDisplay(conf, display_labels=['NoResp', 'Resp']) conf_plot.plot(include_values=True, cmap=plt.cm.Blues, ax=None, xticks_rotation=None, values_format='d') plt.show() plt.savefig(trn_path / 'conf_mat.png', dpi=100) # Confusion Matrix (normalized) conf = confusion_matrix(prd_te_all.y_true_cls, prd_te_all.y_pred_cls, normalize='all') conf_plot = ConfusionMatrixDisplay(conf, display_labels=['NoResp', 'Resp']) conf_plot.plot(include_values=True, cmap=plt.cm.Blues, ax=None, xticks_rotation=None, values_format='.2f') plt.savefig(trn_path / 'conf_mat_norm.png', dpi=100)
def test(self, img, lbl, classNames=None, f1=False, savefig=True, save_path=""): if '/' not in save_path and '\\' not in save_path: save_path += '/' prediction = self.model.predict(img, batch_size=self.batch) print(lbl.argmax(axis=1), prediction.argmax(axis=1)) if f1: print( classification_report(lbl.argmax(axis=1), prediction.argmax(axis=1), target_names=classNames)) if savefig: cm = confusion_matrix(lbl.argmax(axis=1), prediction.argmax(axis=1)) cm_display = ConfusionMatrixDisplay(cm).plot() cm_display.figure_.savefig( save_path + 'Confusion Matrix {}.png'.format(self.name))
def test_confusion_matrix_display_validation(pyplot): """Check that we raise the proper error when validating parameters.""" X, y = make_classification( n_samples=100, n_informative=5, n_classes=5, random_state=0 ) regressor = SVR().fit(X, y) y_pred_regressor = regressor.predict(X) y_pred_classifier = SVC().fit(X, y).predict(X) err_msg = "ConfusionMatrixDisplay.from_estimator only supports classifiers" with pytest.raises(ValueError, match=err_msg): ConfusionMatrixDisplay.from_estimator(regressor, X, y) err_msg = "Mix type of y not allowed, got types" with pytest.raises(ValueError, match=err_msg): # Force `y_true` to be seen as a regression problem ConfusionMatrixDisplay.from_predictions(y + 0.5, y_pred_classifier) with pytest.raises(ValueError, match=err_msg): ConfusionMatrixDisplay.from_predictions(y, y_pred_regressor) err_msg = "Found input variables with inconsistent numbers of samples" with pytest.raises(ValueError, match=err_msg): ConfusionMatrixDisplay.from_predictions(y, y_pred_classifier[::2])
def evaluation(y_pred, classifier, classifier_name): cm = confusion_matrix(Ytrain, y_pred) ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=classifier.classes_).plot() #5 fold Yscores = cross_val_predict(classifier, Xtrain_norm, Ytrain, cv=None, method="predict") fpr, tpr, thresholds = roc_curve(Ytrain, Yscores) plt.figure(figsize=(8, 6)) plot_roc_curve(fpr, tpr) plt.show() roc_auc_score(Ytrain, Yscores) y_pred = classifier.predict(Xtest_norm) print(classifier_name) print("precision:", precision_score(Ytest, y_pred)) print("accuracy:", accuracy_score(Ytest, y_pred)) print("recall:", recall_score(Ytest, y_pred)) print("AUC:", roc_auc_score(Ytest, y_pred), "\n")
def plot_results(prediction, labels, output_dir, rec_device, n_classes): acc = round(accuracy_score(prediction, labels) * 100, 2) cm = confusion_matrix(prediction, labels, normalize='true') fig, ax = plt.subplots(figsize=(15, 15)) plt.rcParams.update({'font.size': 18}) if n_classes == 10: cmd = ConfusionMatrixDisplay(cm, display_labels=list(LABELS_10.keys())) else: cmd = ConfusionMatrixDisplay(cm, display_labels=list(LABELS_3.keys())) cmd.plot(cmap='Blues', xticks_rotation=60, values_format='.2f', ax=ax) plt.title(f'Acoustic Scene Clasiffication {rec_device}- Confusion Matrix - Accuracy = {acc}%', fontsize=18, fontweight='bold') plt.xlabel('Predicted label', fontsize=18, fontweight='bold') plt.ylabel('True label', fontsize=18, fontweight='bold') plt.setp(ax.get_xticklabels(), fontsize=18) plt.setp(ax.get_yticklabels(), fontsize=18) plt.tight_layout() plt.savefig(osp.join(output_dir, f'confusion_matrix_{rec_device}.jpg'))
def analyze(prob_pred, y_true): y_pred = [np.argmax(prob) for prob in prob_pred] accuracy = accuracy_score(y_true, y_pred) global_precision = precision_score(y_true, y_pred, average = "micro") global_recall = recall_score(y_true, y_pred, average = "micro") global_f1 = f1_score(y_true, y_pred, average = "micro") weighted_roc_auc_ovr = roc_auc_score(y_true, prob_pred, multi_class="ovr", average="weighted") macro_roc_auc_ovr = roc_auc_score(y_true, prob_pred, multi_class="ovr", average="macro") weighted_roc_auc_ovo = roc_auc_score(y_true, prob_pred, multi_class="ovo", average="weighted") macro_roc_auc_ovo = roc_auc_score(y_true, prob_pred, multi_class="ovo", average="macro") print(classification_report(y_true, y_pred)) print("global recall: {}".format(global_recall)) print("global precision: {}".format(global_precision)) print("global f1: {}".format(global_f1)) print("One-vs-One ROC AUC scores:\n{:.6f} (macro),\n{:.6f} (weighted by prevalence)" .format(macro_roc_auc_ovo, weighted_roc_auc_ovo)) print("One-vs-Rest ROC AUC scores:\n{:.6f} (macro),\n{:.6f} (weighted by prevalence)" .format(macro_roc_auc_ovr, weighted_roc_auc_ovr)) conf_mx = confusion_matrix(y_true, y_pred) ConfusionMatrixDisplay(conf_mx).plot() plt.show() show_command = input("See the wrong answers? (y/n)") i = 0 while i < len(y_pred) and show_command == 'y': if y_pred[i] != y_true[i]: print('predicted:', y_pred[i], 'target:', y_true[i]) io.imshow(np.squeeze(X_test[i])) io.show() show_command = input("See the wrong answers? (y/n)") i+=1 return conf_mx
def test_confusion_matrix_contrast(pyplot): # make sure text color is appropriate depending on background cm = np.eye(2) / 2 disp = ConfusionMatrixDisplay(cm, display_labels=[0, 1]) disp.plot(cmap=pyplot.cm.gray) # diagonal text is black assert_allclose(disp.text_[0, 0].get_color(), [0.0, 0.0, 0.0, 1.0]) assert_allclose(disp.text_[1, 1].get_color(), [0.0, 0.0, 0.0, 1.0]) # oof-diagonal text is white assert_allclose(disp.text_[0, 1].get_color(), [1.0, 1.0, 1.0, 1.0]) assert_allclose(disp.text_[1, 0].get_color(), [1.0, 1.0, 1.0, 1.0]) disp.plot(cmap=pyplot.cm.gray_r) # diagonal text is white assert_allclose(disp.text_[0, 1].get_color(), [0.0, 0.0, 0.0, 1.0]) assert_allclose(disp.text_[1, 0].get_color(), [0.0, 0.0, 0.0, 1.0]) # oof-diagonal text is black assert_allclose(disp.text_[0, 0].get_color(), [1.0, 1.0, 1.0, 1.0]) assert_allclose(disp.text_[1, 1].get_color(), [1.0, 1.0, 1.0, 1.0])
# create an instance of the SVM classifier with a Linear kernel model = svm.SVC(kernel='linear') # train the model using the training set model.fit(X_train, y_train) # predict the classes in the test set y_pred = model.predict(X_test) # D. Evaluate the model # accuracy print("Accuracy: %.3f%%" % (metrics.accuracy_score(y_test, y_pred) * 100)) # precision print("Precision: %.3f " % metrics.precision_score(y_test, y_pred)) # recall print("Recall: %.3f" % metrics.recall_score(y_test, y_pred)) # F1 (F-Measure) print("F1: %.3f" % metrics.f1_score(y_test, y_pred)) # compute Confusion Matrix cm = confusion_matrix(y_test, y_pred, labels=model.classes_) # pretty print Confusion Matrix as a heatmap disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=dataset.target_names) disp.plot() plt.show()
def task1(): model_path = os.path.join("project_a_supp", "models", "MNIST1D.h5") model = keras.models.load_model(model_path) mnist1d = make_dataset() x_test = np.expand_dims(mnist1d["x_test"], axis=-1) y_test = mnist1d["y_test"] model.evaluate(x_test, y_test) num_classes = 10 num_true_positives = 0 num_correct_per_class = np.zeros(num_classes, dtype=np.int64) num_total_per_class = np.zeros(num_classes, dtype=np.int64) y_predicted_scores = [] num_wrong = 0 plt.figure(figsize=(20, 8)) should_plot_failures = False for i in range(len(x_test)): digit_input = x_test[i:i + 1] digit_label = y_test[i:i + 1] digit_prediction = model(digit_input).numpy() y_predicted_scores.append(digit_prediction) digit_prediction = np.argmax(digit_prediction) if digit_prediction == digit_label: if not should_plot_failures and (num_true_positives < 10): plt.subplot(2, 5, 1 + num_true_positives) plt.plot(np.squeeze(digit_input), "r") plt.axis("off") plt.title( f"label: {digit_label.item()} predicted: {digit_prediction}" ) num_true_positives += 1 num_correct_per_class[digit_label] += 1 else: if should_plot_failures and (num_wrong < 10): plt.subplot(2, 5, 1 + num_wrong) plt.plot(np.squeeze(digit_input), "r") plt.axis("off") plt.title( f"label: {digit_label.item()} predicted: {digit_prediction}" ) num_wrong += 1 num_total_per_class[digit_label] += 1 if should_plot_failures: plt.savefig(os.path.join("report", "images", "mnist1d-failures.png"), dpi=256) else: plt.savefig(os.path.join("report", "images", "mnist1d-successes.png"), dpi=256) plt.clf() y_predicted_scores = np.concatenate(y_predicted_scores, axis=0) print(f"Accuracy: {num_true_positives/len(x_test)}") classwise_accuracy = num_correct_per_class / num_total_per_class print(f"Class-wise ccuracy: {classwise_accuracy}") plt.bar(x=range(num_classes), height=classwise_accuracy) plt.title("MNIST-1D Class-wise Accuracy") plt.xticks(range(10)) plt.xlabel("Digit") plt.ylabel("Accuracy") plt.savefig(os.path.join("report", "images", "mnist1d-class-accuracy.png"), dpi=256) plt.clf() y_test_onehot = np.zeros((len(x_test), num_classes), dtype=np.int64) for y_idx, y_label in enumerate(y_test): y_test_onehot[y_idx, y_label] = 1 for i in range(num_classes): false_pos_rate, true_pos_rate, _ = roc_curve(y_test_onehot[:, i], y_predicted_scores[:, i]) digit_auc = auc(false_pos_rate, true_pos_rate) plt.plot(false_pos_rate, true_pos_rate, label=f"{i} (AUC = {digit_auc:.4f})") plt.xlabel("False Positive Rate") plt.ylabel("True Positive Rate") plt.title("MNIST-1D ROC-AUC Curves") plt.legend(loc="lower right") plt.savefig(os.path.join("report", "images", "mnist1d-roc-curve.png"), dpi=256) plt.clf() y_predicted_digits = np.argmax(y_predicted_scores, axis=1) digits_confusion_mtx = confusion_matrix(y_predicted_digits, y_test, normalize="true") plt.title("MNIST-1D Confusion Matrix") disp = ConfusionMatrixDisplay(digits_confusion_mtx) disp.plot() # plt.savefig(os.path.join("report", "images", "mnist1d-confusion-matrix"), dpi=256) plt.show() for i in range(num_classes): precision, recall, thresholds = precision_recall_curve( y_test_onehot[:, i], y_predicted_scores[:, i]) avg_precision = average_precision_score(y_test_onehot[:, i], y_predicted_scores[:, i]) f1 = f1_score(y_test_onehot[:, i], np.round(y_predicted_scores[:, i])) plt.plot(recall, precision, label=f"{i} (AP = {avg_precision:.3f}, F1 = {f1:.2f})") plt.xlabel("Recall") plt.ylabel("Precision") plt.title("MNIST-1D Precision-Recall Curves") plt.legend(loc="lower left") plt.savefig(os.path.join("report", "images", "mnist1d-precision-recall.png"), dpi=256) plt.clf()
x_test = joblib.load('data/x_test_w.joblib') y_test = joblib.load('data/y_test_w.joblib') print('train shape=', np.shape(X_train)) print('val shape=', np.shape(x_val)) ## Random forest model was selected having maximum accuracy and F1 score cvmodel = joblib.load('3-rf1.joblib') ## since we have less no. of samples, training this model again with entire training and validation data xtr = pd.concat([X_train, x_val]) ytr = pd.concat([y_train, y_val]) print('combined train shape=', np.shape(xtr)) cvmodel.fit(xtr, ytr) joblib.dump(cvmodel, 'white-final-trained-model.joblib') loaded = joblib.load('white-final-trained-model.joblib') pred = loaded.predict(x_test) print('test acc= ', accuracy_score(pred, y_test)) print('balanced test acc= ', balanced_accuracy_score(pred, y_test)) cm = confusion_matrix(y_test, pred) display = ConfusionMatrixDisplay(cm).plot() plt.figure(1) plt.title('Test white wine') classNames = ['poor', 'average', 'excellent'] tick_marks = np.arange(len(classNames)) plt.xticks(tick_marks, classNames) plt.yticks(tick_marks, classNames) plt.show(display)
print('Accuracy:', accuracy_score(y_test, y_pred)) print('Classification Report:') print(classification_report(y_test, y_pred)) le = LabelEncoder() y_test = le.fit_transform(y_test) y_pred = le.fit_transform(y_pred) print('ROC_AUC Score', roc_auc_score(y_test, y_pred)) # CONFUSION MATRIX # Plot non-normalized confusion matrix from sklearn.metrics import ConfusionMatrixDisplay cm = confusion_matrix(y_test, y_pred) cmd = ConfusionMatrixDisplay(cm, display_labels=['B', 'M']) cmd.plot(cmap='Blues') cmd.ax_.set(xlabel='Predicted', ylabel='True') plt.figure(figsize=(5, 5)) plt.show() tn, fp, fn, tp = cm.ravel() print('Coefficients Confusion DT - best parameters') print('tn', tn, 'fp', fp, 'fn', fn, 'tp', tp) # OUTPUT TO INPUT CAPSTONE PROJECT # OUTPUT THE MODEL import pickle with open('clf.DT', 'wb') as f: pickle.dump(gs, f)
y_9 = [] y_47 = [] for i in values: i = np.asarray(i) x_9.append(i[0][0]) y_9.append(i[1][0]) x_47.append(i[0][1]) y_47.append(i[1][1]) cm_9 = confusion_matrix(y_9, x_9) cm_47 = confusion_matrix(y_47, x_47) print('CM_9: \n', cm_9) print('CM_47: \n', cm_47) disp_9 = ConfusionMatrixDisplay(confusion_matrix=cm_9, display_labels=range(1, 10)) disp_47 = ConfusionMatrixDisplay(confusion_matrix=cm_47, display_labels=range(1, 48)) accuracy_9 = accuracy_score(y_9, x_9) accuracy_47 = accuracy_score(y_47, x_47) fig, ax = plt.subplots() disp = disp_9.plot(include_values=True, cmap=plt.cm.Blues, ax=ax, xticks_rotation='horizontal') disp.ax_.set_title('Accuracy_9: {:.2f}'.format(accuracy_9)) fig.savefig('./saved/cm/' + str(uid) + '_9.png', quality=50) if opt.display == 1: plt.show()
def nicegrid( imgs, all_labels, # all labels for dataset all_predictions, # all predicitons for dataset isfirstinstage, # bool True if first frame of a stage epochs_trained, training_parameters, log_path): # how many epochs did the model train for # get a classification report is_multi = 'multi' in training_parameters['model_name'] if is_multi: # class_id = [0, 1, 2, 3, 4] # class_labels = ['G0', 'G1', 'S', 'G2', 'M'] class_id = [0, 1, 2, 3] class_labels = ['G0/1', 'S', 'G2', 'M'] else: class_id = [0, 1] class_labels = ['not S', 'S'] crep_all = classification_report(all_labels, all_predictions, labels=class_id, target_names=class_labels, output_dict=True) crep_first = classification_report(all_labels[isfirstinstage], all_predictions[isfirstinstage], labels=class_id, target_names=class_labels, output_dict=True) # import pdb # pdb.set_trace() # bin_pred_type = { # second entry is prediction # (0, 0): {'str': 'not S', 'c': 'g'}, # (1, 1): {'str': 'S', 'c': 'g'}, # (0, 1): {'str': 'S', 'c': 'r'}, # (1, 0): {'str': 'not S', 'c': 'r'}, # } # set up figure specs fig = plt.figure(figsize=(8.3, 11.7)) gs = gridspec.GridSpec(nrows=3, ncols=2, left=0.04, right=0.92, bottom=0.01, top=0.98) # axis with text based info tax = fig.add_subplot(gs[0, 0]) tax.set_visible(False) tbox = tax.get_position() # grect = [gbox.x0, gbox.y0, gbox.width, gbox.height] was_scheduler = training_parameters['scheduler'] is not None if was_scheduler: if training_parameters['scheduler_kwargs']['mode'] == 'max': scheduler_update_mode = 'max accuracy' elif training_parameters['scheduler_kwargs']['mode'] == 'min': scheduler_update_mode = 'min loss' else: upmode = training_parameters['scheduler_kwargs']['mode'] raise ValueError(f'unknown sheduler update method {upmode}') else: scheduler_update_mode = None crep_txt = (f"All dataset:\n" f"Accuracy: {crep_all['accuracy']:.3f}\n" f"F1 score: {crep_all['S']['f1-score']:.3f}\n" f"Precision: {crep_all['S']['precision']:.3f}\n" f"Recall: {crep_all['S']['recall']:.3f}\n" f"\nFirst in stage:\n" f"Accuracy: {crep_first['accuracy']:.3f}\n" f"F1 score: {crep_first['S']['f1-score']:.3f}\n" f"Precision: {crep_first['S']['precision']:.3f}\n" f"Recall: {crep_first['S']['recall']:.3f}\n" f"\nModel name: {training_parameters['model_name']}\n" f"Epochs trained: {epochs_trained}\n" f"Batch size: {training_parameters['batch_size']}\n" f"Learning rate: {training_parameters['learning_rate']}\n" f"Used scheduler: {was_scheduler}\n" f"Scheduler update mode: {scheduler_update_mode}\n" f"Used sampler: {training_parameters['is_use_sampler']}\n") fig.text(tbox.x0, tbox.y0 + tbox.height, crep_txt, verticalalignment='top', fontname='monospace', fontsize=12, wrap=True) # here goes the confusion matrix cmax = fig.add_subplot(gs[0, 1]) cm = confusion_matrix(all_labels, all_predictions) cmdisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_labels) cmdisp = cmdisp.plot(cmap='Blues', ax=cmax) # plot logs if log_path is not None: log_df = read_tb_log(log_path) lax = fig.add_subplot(gs[1, :]) for metric in ['train_epoch_loss', 'val_epoch_loss']: log_df.plot(y=metric, ax=lax) lax.set_ylim(0, 1) # examples grid gax = fig.add_subplot(gs[2:, :]) gax.set_visible(False) gbox = gax.get_position() grect = [gbox.x0, gbox.y0, gbox.width, gbox.height] # lets just use the "first in stage" as example imgs_first = imgs[isfirstinstage] labs_first = all_labels[isfirstinstage] preds_first = all_predictions[isfirstinstage] n_imgs = imgs_first.shape[0] if n_imgs > 18: n_imshows = 18 else: n_imshows = n_imgs # find factors. factors = [i for i in range(1, n_imshows + 1) if (n_imshows % i == 0)] n_imgs_per_row = factors[len(factors) // 2] n_rows = n_imshows // n_imgs_per_row # make grid image grid = ImageGrid(fig, grect, nrows_ncols=(n_rows, n_imgs_per_row), axes_pad=0.1) for ax, im, ytrue, ypred in zip(grid, imgs_first[:n_imshows], labs_first[:n_imshows], preds_first[:n_imshows]): edge_color = 'g' if (ytrue == ypred) else 'r' # Iterating over the grid returns the Axes. ax.imshow(im, cmap='gray') ax.text(0.05, 0.05, class_labels[ypred], color='w', fontweight='bold', verticalalignment='top') ax.set_xticks([]) ax.set_yticks([]) rect = patches.Rectangle( (0, 0), im.shape[1], im.shape[0], edgecolor=edge_color, linewidth=2, facecolor='none', ) ax.add_patch(rect) return fig
else: svc_disp = plot_roc_curve(model, X_test, y_test) svc_disp.plot() plt.title( f"ROC Curve of {PREDICTION_PHENO} by {model_name} (Scaled)") plt.savefig( f"finalplots/scaledroc_{PREDICTION_PHENO}_{model_name}.png", dpi=600, transparent=True, bbox_inches="tight", pad_inches=0.3) # Confusion Matrix if model_name == "elasticnetlinear": conf_mat = confusion_matrix(model[1], y_test) disp = ConfusionMatrixDisplay(confusion_matrix=conf_mat, display_labels=[0, 1]) disp.plot() plt.title( f"Confusion Matrix of {PREDICTION_PHENO} by {model_name} (Scaled)" ) plt.savefig( f"finalplots/scaledconfusion_mat_{PREDICTION_PHENO}_{model_name}.png", dpi=600, transparent=True, bbox_inches="tight", pad_inches=0.3) else: if model_name == "rbfsvmapprox": disp = plot_confusion_matrix( model,
Y_predicted, labels=labels) confmat = confusion_matrix(Y_test, Y_predicted, labels=labels, normalize='true') print(p) print(r) print(f1) display_confmat(confmat) print(classification_report(Y_test, Y_predicted, digits=3)) plt.figure(figsize=(4, 4)) cmatdisp = ConfusionMatrixDisplay(confmat, display_labels=['1, 2', '3', '4, 5']) cmatdisp.plot(cmap=plt.cm.Blues, ax=plt.gca()) plt.savefig('report/figures/confmat_CustomSentiCorefModel.pdf', bbox_inches='tight') print( textwrap.dedent(f""" EMBEDDING_DIM = {EMBEDDING_DIM} CNN_FILTERS = {CNN_FILTERS} DNN_UNITS = {DNN_UNITS} OUTPUT_CLASSES = {OUTPUT_CLASSES} DROPOUT_RATE = {DROPOUT_RATE} NB_EPOCHS = {NB_EPOCHS} BATCH_SIZE = {BATCH_SIZE} """))
# Get accuracy acc = accuracy_score(t_, y_) print("Accuracy: " + str(acc)) # F1 score f1 = f1_score(t_, y_, average=None) print("F1 score per class: " + str(f1)) # F1 score weighted f1_weighted = f1_score(t_, y_, average='weighted') print("Weighted F1 score: " + str(f1_weighted)) # Confusion matrix conf_matrix = confusion_matrix(t_, y_) classes = ['ambiguous', 'fw', 'lang1', 'lang2', 'mixed', 'ne', 'other', 'unk'] ConfusionMatrixDisplay(confusion_matrix=conf_matrix, display_labels=classes).plot(values_format='d') plt.savefig('./results/CM/mBERT_test.svg', format='svg') # RESULTS # Validation set # 0.9910626123503051 # [0. 0. 0.99140693 0.99081154 0. 0.99782942 0.] # Test set # 0.9859431603653248 # [0. 0.98548717 0.98404558 0. 0.99764318 0.] ########################################################################### ##################### RESULTS FOR WORKSHOP ################################ # Dev set
def main(): # argument parser parser = argparse.ArgumentParser(description='ESMH') parser.add_argument('--root_dir', type=str, default='/data/ESMH/cropped_patches/tcga_3p_ref') parser.add_argument('--result_dir', type=str, default='/data/ESMH/subtype_results/tcga_best') parser.add_argument('--net', type=str, default='resnet') parser.add_argument('--epochs', type=int, default=500) parser.add_argument('--env_name', type=str, default='nnunet') parser.add_argument('--data_dim', type=str, default='2d') parser.add_argument('--num_phase', type=int, default=2) args = parser.parse_args() scores_avg = np.zeros([]) trial_list = [2, 3, 4, 7, 8, 9, 57, 58, 76, 84] for i in range(10): for j in range(3): if j == 0: phases = [0, 1, 2] elif j == 1: phases = [0, 1, 3] elif j == 2: phases = [0, 2, 3] else: break true_labels, scores = eval_trial(trial=trial_list[i], mode='test', net=args.net, root_dir=args.root_dir, result_dir=args.result_dir, data_dim=args.data_dim, use_phases=phases) scores_avg = scores_avg + scores scores_avg /= (10 * 3) auc, opt_thr = get_roc_auc_fig(true_labels, scores_avg, find_opt_thr=True, save_roc=True, figure_name=os.path.join(args.result_dir)) print('auc: ', auc) print('opt_thr: ', opt_thr) auc = get_pr_auc_fig(true_labels, scores_avg, save_roc=True, figure_name=os.path.join(args.result_dir)) conf_mat = calc_conf_mat(true_labels, scores_avg, thr=[0.07, 0.34, 0.44, 0.24, 0.38]) conf_mat = conf_mat.astype(np.int) print(conf_mat) disp = ConfusionMatrixDisplay( conf_mat, ['ccRCC', 'pRCC', 'chRCC', 'AML', 'Oncocytoma']) disp.plot(figure_name=os.path.join(args.result_dir, 'conf_model.svg')) print("top-1 acc (conf):", calc_acc_from_conf_mat(conf_mat)) print("top-1 acc:", calc_top1_acc_from_scores(true_labels, scores_avg)) print("top-2 acc:", calc_top2_acc_from_scores(true_labels, scores_avg)) sen, spe, acc = calc_eval_metric(conf_mat) print('sen: ', sen) print('spe: ', spe) print('acc: ', acc) print('main done')