def _runtestset(self, model, test_loader, path_to_weights, fold): if path_to_weights: weights = torch.load(path_to_weights) model.load_state_dict(weights) model.to(self.device) # Make Predictions with torch.no_grad(): model.eval() y_truth, y_predicted = [], [] for data in test_loader: images, labels = data['image'].to( self.device, dtype=torch.float), data['label'].to(self.device, dtype=torch.long) output = model(images) output_pb = F.softmax(output.cpu(), dim=1) top_ps, top_class = output_pb.topk(1, dim=1) y_predicted.extend(list(top_class.flatten().numpy())) y_truth.extend(list(labels.cpu().flatten().numpy())) metrics = self._get_evaluation_metric(y_truth, y_predicted) logging.info("Test_Fold_{}_Metrics".format(fold)) for arg in metrics: print("{}: {}".format(arg, metrics[arg])) logging.info("{}: {}".format(arg, metrics[arg])) logging.info("=" * 40) bc = BinaryClassification(y_truth, y_predicted, labels=["Benign", "Malignant"]) bc.plot_roc_curve() auc_name = "Test_fold_{}_roc.png".format(fold) plt.savefig(auc_name) plt.clf()
def getROCCurve(model, X_test, y_test): y_pred = model.predict_proba(X_test)[:, 1] from plot_metric.functions import BinaryClassification bc = BinaryClassification(y_test, y_pred, labels=["Win", "Loss"]) plt.figure(figsize=(5, 5)) bc.plot_roc_curve() plt.show() return
def plot_roc_curve(self, yproba, name): '''Plots the ROC curve for given model (by definition requires a model that can gen probabilities)''' # Visualisation with plot_metric bc = BinaryClassification(self.y_test, yproba, labels=["No Disease", "Heart Disease"]) # Figures plt.figure(figsize=(6, 6)) bc.plot_roc_curve() pth = Path(self.graphics_path, 'roc_curve_'+name).with_suffix('.png') plt.savefig(pth) plt.close()
def drawCurve(results,target,TPR,FPR): #y_true = target #y_pred = results #this also works #fpr,tpr,thresholds = metrics.roc_curve(target,results,pos_label=0) #plt.plot(tpr,fpr) #plt.show() #this also works #fpr,tpr,thresholds = metrics.roc_curve(target,results,pos_label=0) #plt.plot(tpr,fpr) #plt.show() bc = BinaryClassification(target,results,labels=["Class1","Class2"],threshold=0.25) plt.figure(figsize=(5,5)) bc.plot_roc_curve() plt.show()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=2) # Building Classifier clf = RandomForestClassifier(n_estimators=50, random_state=23) # Train our classifier model = clf.fit(X_train, y_train) # Predict test set y_pred = clf.predict_proba(X_test)[:, 1] # Visualisation with plot_metric bc = BinaryClassification(y_test, y_pred, labels=["Class 1", "Class 2"]) # Figures plt.figure(figsize=(15, 10)) plt.subplot2grid(shape=(2, 6), loc=(0, 0), colspan=2) bc.plot_roc_curve() plt.subplot2grid((2, 6), (0, 2), colspan=2) bc.plot_precision_recall_curve() plt.subplot2grid((2, 6), (0, 4), colspan=2) bc.plot_class_distribution() plt.subplot2grid((2, 6), (1, 1), colspan=2) bc.plot_confusion_matrix() plt.subplot2grid((2, 6), (1, 3), colspan=2) bc.plot_confusion_matrix(normalize=True) # Save figure
def save_classification_plots( y_true: np.ndarray, y_proba: np.ndarray, threshold: np.float64, prefix: Optional[str] = None, destination: Optional[str] = None, ): """ Build and save binary classification performance evaluation plots. Args: y_true: Ground truth (correct) labels. y_pred: Predicted probabilities of the positive class returned by a classifier. threshold: Classification pipeline optimal threshold. prefix: Classification plots prefix i.e. pipeline name. Defaults to None. destination: Folder where the report should be saved. Defaults to ``METADATA_REGISTRY``. """ destination = cfg.METADATA_REGISTRY if destination is None else destination fname = ("classification_plots.png" if prefix is None else prefix + "_classification_plots.png") path = os.path.join(destination, fname) bc = BinaryClassification(y_true, y_proba, labels=["No fire", "Fire"]) plt.figure(figsize=(15, 10)) plt.subplot2grid(shape=(2, 6), loc=(0, 0), colspan=2) bc.plot_roc_curve(threshold=threshold) plt.subplot2grid((2, 6), (0, 2), colspan=2) bc.plot_precision_recall_curve(threshold=threshold) plt.subplot2grid((2, 6), (0, 4), colspan=2) bc.plot_class_distribution(threshold=threshold) plt.subplot2grid((2, 6), (1, 1), colspan=2) bc.plot_confusion_matrix(threshold=threshold) plt.subplot2grid((2, 6), (1, 3), colspan=2) bc.plot_confusion_matrix(threshold=threshold, normalize=True) plt.savefig(path)
# metrics are used to find accuracy or error from sklearn import metrics # using metrics module for accuracy calculation print("ACCURACY OF THE MODEL: ", metrics.accuracy_score(y_test, y_pred)) # classification report for precision, recall f1-score and accuracy: ,labels=[0,1] matrix = classification_report(y_test, y_pred) print('Classification report : \n', matrix) # In[291]: #AUC Curve from plot_metric.functions import BinaryClassification # Visualisation with plot_metric y_pred = clf.predict_proba(X_test)[:, 1] bc = BinaryClassification(y_test, y_pred, labels=["Class 1", "Class 2"]) # Figures plt.figure(figsize=(5, 5)) bc.plot_roc_curve() plt.show() # #### Model 2: KNN # In[292]: #Model 2: KNN from sklearn.neighbors import KNeighborsClassifier neighbors = np.arange(1, 30) train_accuracy = np.empty(len(neighbors)) test_accuracy = np.empty(len(neighbors))
correct_faces = sum(ensemble_vote_all(faces_ii_testing, classifiers)) incorrect_faces = len(faces_testing) - correct_faces correct_non_faces = len(non_faces_testing) - sum(ensemble_vote_all(non_faces_ii_testing, classifiers)) incorrect_non_faces = len(non_faces_testing) - correct_non_faces correct_faces_score = ensemble_score_all(faces_ii_testing, classifiers) incorrect_non_faces_score = ensemble_score_all(non_faces_ii_testing, classifiers) face_label = np.array([1] * len(correct_faces_score) + [0] * len(incorrect_non_faces_score)) face_predict = np.array(ensemble_vote_all(faces_ii_testing, classifiers) + ensemble_vote_all(non_faces_ii_testing, classifiers)) # plot confusion matrix cf = plot_confusion_matrix(correct_faces, incorrect_faces, correct_non_faces, incorrect_non_faces) cf.figure.savefig("results/cf_round_{}.png".format(num)) # plot roc curve # Visualisation with plot_metric bc = BinaryClassification(face_label, face_predict, labels=["Class 1", "Class 2"]) f1 = f1_score(face_label, face_predict) print("f1: ", f1) # Figures plt.figure(figsize=(5,5)) fpr, tpr, thres, auc = bc.plot_roc_curve() plt.savefig('results/round_{}_roc.jpg'.format(num)) print('..done.\n\nResult:\n Faces: ' + str(correct_faces) + '/' + str(len(faces_testing)) + ' (' + str((float(correct_faces) / len(faces_testing)) * 100) + '%)\n non-Faces: ' + str(correct_non_faces) + '/' + str(len(non_faces_testing)) + ' (' + str((float(correct_non_faces) / len(non_faces_testing)) * 100) + '%)') print('False Positive Rate: {}, True Positive Rate: {}'.format(fpr, tpr)) print('Classifier Results: ', ab.stats) haar_imgs = vis_haar(classifiers, faces_testing[0])
def metric_plots_2(self, actual=None, pred=None, threshold=0.5): ''' TODO : due to package issue this function is not used (need to check) Plotting the roc Curve, Precision recall curve, confusion matirx ''' if ((actual is not None) & (pred is not None)): bc = BinaryClassification(y_true=actual, y_pred=pred, labels=["Class 0", "Class 1"], threshold=threshold) # Figures plt.figure(figsize=(20, 15)) plt.subplot2grid(shape=(2, 6), loc=(0, 0), colspan=2) # Roc curve: bc.plot_roc_curve(threshold=threshold) plt.subplot2grid((2, 6), (0, 2), colspan=2) # precision recall curve: bc.plot_precision_recall_curve(threshold=threshold) plt.subplot2grid((2, 6), (0, 4), colspan=2) # class distribution curve: bc.plot_class_distribution(threshold=threshold) plt.subplot2grid((2, 6), (1, 1), colspan=2) # confusion matrix: bc.plot_confusion_matrix(threshold=threshold) plt.subplot2grid((2, 6), (1, 3), colspan=2) # normalised confusion matrix: bc.plot_confusion_matrix(threshold=threshold, normalize=True) plt.show() # Classification report: print( classification_report( actual, [0 if i <= threshold else 1 for i in pred]))