def specificity(label_gt, label_pred): mcm = multilabel_confusion_matrix(label_gt.flatten(), label_pred.flatten()) tn = mcm[:, 0, 0] tp = mcm[:, 1, 1] fn = mcm[:, 1, 0] fp = mcm[:, 0, 1] sp = tn / (tn + fp) return sp
def performance_measurement(target, prediction, scenario, performance): # Generate confusion matrix based on scenario if scenario == 1: cm = confusion_matrix(target, prediction, labels=['A', 'B']) tn, fp, fn, tp = cm.ravel() performance["tn"] = tn performance["fp"] = fp performance["fn"] = fn performance["tp"] = tp elif scenario == 2: cm = multilabel_confusion_matrix(target, prediction, labels=['A', 'B', 'C']) performance["tn"] = cm[:, 0, 0] performance["fn"] = cm[:, 1, 0] performance["tp"] = cm[:, 1, 1] performance["fp"] = cm[:, 0, 1] else: cm = multilabel_confusion_matrix(target, prediction, labels=['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'O', 'P', 'Q', 'R', 'S']) performance["tn"] = cm[:, 0, 0] performance["fn"] = cm[:, 1, 0] performance["tp"] = cm[:, 1, 1] performance["fp"] = cm[:, 0, 1] # Compute True Positive Rate (TPR) | Sensitivity performance['sensitivity'] += performance["tp"] / (performance["tp"] + performance["fn"]) # Compute True Negative Rate (TNR) | Specificity performance['specificity'] += performance["tn"] / (performance["tn"] + performance["fp"]) # Compute Accuracy accuracy = (performance["tp"] + performance["tn"]) / (performance["fp"] + performance["fn"] + performance["tp"] + performance["tn"]) performance['accuracy'] += accuracy # Compute Misclassification mc = (performance["fp"] + performance["fn"]) / (performance["fp"] + performance["fn"] + performance["tp"] + performance["tn"]) performance['avg_misclassification'] += mc # Save misclassification per fold and its average performance['misclassification_per_fold'].append(mc) performance['avg_misclassification_per_fold'].append(np.average(mc)) # print(cm) return performance
def test_multiclass_images(): num_classes = 3 cm = MultiLabelConfusionMatrix(num_classes=num_classes) y_true, y_pred = get_y_true_y_pred() # Compute confusion matrix with sklearn sklearn_CM = multilabel_confusion_matrix( y_true.transpose((0, 2, 3, 1)).reshape(-1, 3), y_pred.transpose((0, 2, 3, 1)).reshape(-1, 3)) # Update metric output = (torch.tensor(y_pred), torch.tensor(y_true)) cm.update(output) ignite_CM = cm.compute().cpu().numpy() assert np.all(ignite_CM == sklearn_CM) # Another test on batch of 2 images cm = MultiLabelConfusionMatrix(num_classes=num_classes) # Create a batch of two images: th_y_true1 = torch.tensor(y_true) th_y_true2 = torch.tensor(y_true.transpose(0, 1, 3, 2)) th_y_true = torch.cat([th_y_true1, th_y_true2], dim=0) th_y_pred1 = torch.tensor(y_pred) th_y_pred2 = torch.tensor(y_pred.transpose(0, 1, 3, 2)) th_y_pred = torch.cat([th_y_pred1, th_y_pred2], dim=0) # Update metric & compute output = (th_y_pred, th_y_true) cm.update(output) ignite_CM = cm.compute().cpu().numpy() # Compute confusion matrix with sklearn th_y_true = idist.all_gather(th_y_true) th_y_pred = idist.all_gather(th_y_pred) np_y_true = th_y_true.cpu().numpy().transpose((0, 2, 3, 1)).reshape(-1, 3) np_y_pred = th_y_pred.cpu().numpy().transpose((0, 2, 3, 1)).reshape(-1, 3) sklearn_CM = multilabel_confusion_matrix(np_y_true, np_y_pred) assert np.all(ignite_CM == sklearn_CM)
def evaluate_predictions(labels): labels_pred = ensemble_predictions(X_test, models) true_labels = np.argmax(y_test, axis=1) test_acc = accuracy_score(true_labels, labels_pred) matrix = multilabel_confusion_matrix(true_labels,labels_pred, labels=[6,5,4,3,2,1,0]) print('Confusion matrix : \n',matrix) # classification report for precision, recall f1-score and accuracy print('Classification report : \n', classification_report(true_labels,labels_pred, labels=[6,5,4,3,2,1,0])) return test_acc
def print_confm(test, predictions, model): if (model == 'b') or (model == 'g'): confm = confusion_matrix(test, predictions) print("Confusion Matrix:\n") print(confm) elif (model == 'r'): confm = multilabel_confusion_matrix(test, predictions) print("Confusion Matrix:\n") print(confm)
def accuracy(args): # Loading model with weights model = lu.select_model(args) # Saving image paths as a list test_image_paths, test_label_paths = lu.file_paths(args.csv_paths) # Calculating accuracy y, y_pred = [], [] rescale_value = lu.rescaling_value(args.rs) for i in range(len(test_image_paths)): image = gdal.Open(test_image_paths[i]) image_array = np.array(image.ReadAsArray()) / rescale_value image_array = image_array.transpose(1, 2, 0) label = gdal.Open(test_label_paths[i]) label_array = np.array(label.ReadAsArray()) / args.rs_label label_array = np.expand_dims(label_array, axis=-1) fm = np.expand_dims(image_array, axis=0) result_array = model.predict(fm) if args.num_classes != 1: result_array = np.argmax(result_array[0], axis=2) result_array = np.squeeze(result_array) if args.num_classes == 1: result_array = np.around(result_array.flatten()) y.append(np.around(label_array)) y_pred.append(result_array) print("Predicted " + str(i + 1) + " Patches") print("\n") print("list of classes from predictions: " + str(np.unique(np.array(y_pred)))) print("list of classes from labels: " + str(np.unique(np.array(y)))) print("\n") cm = confusion_matrix(np.array(y).flatten(), np.array(y_pred).flatten()) cm_multi = multilabel_confusion_matrix( np.array(y).flatten(), np.array(y_pred).flatten()) print("Confusion Matrix " + "\n") print(cm, "\n") accuracy = np.trace(cm / np.sum(cm)) print("Overal Accuracy: ", round(accuracy, 3), "\n") mean_iou = 0 mean_f1 = 0 for j in range(len(cm_multi)): print("Class: " + str(j)) iou = cm_multi[j][1][1] / (cm_multi[j][1][1] + cm_multi[j][0][1] + cm_multi[j][1][0]) f1 = (2 * cm_multi[j][1][1]) / (2 * cm_multi[j][1][1] + cm_multi[j][0][1] + cm_multi[j][1][0]) precision = cm_multi[j][1][1] / (cm_multi[j][1][1] + cm_multi[j][0][1]) recall = cm_multi[j][1][1] / (cm_multi[j][1][1] + cm_multi[j][1][0]) mean_iou += iou mean_f1 += f1 print("IoU Score: ", round(iou, 3)) print("F1-Measure: ", round(f1, 3)) print("Precision: ", round(precision, 3)) print("Recall: ", round(recall, 3), "\n") print("Mean IoU Score: ", round(mean_iou / len(cm_multi), 3)) print("Mean F1-Measure: ", round(mean_f1 / len(cm_multi), 3))
def f1_validation(y_true, y_pred): # performance print(metrics.classification_report(y_true, y_pred)) print(multilabel_confusion_matrix(y_true, y_pred)) #print("F1 micro: %1.4f\n" % f1_score(y_test, y_predicted, average='micro')) print("F1 macro: %1.4f\n" % f1_score(y_true, y_pred, average='macro')) #print("F1 weighted: %1.4f\n" % f1_score(y_test, y_predicted, average='weighted')) #print("Accuracy: %1.4f" % (accuracy_score(y_test, y_predicted))) return f1_score(y_true, y_pred, average='macro')
def train_model(classifier,X, y, max_feature = 1000, embedding= 'bow' ): #Train-test split print("... Performing train test split") X_train, X_test, y_train, y_test = model_selection.train_test_split(X,y, test_size=0.25,random_state=42) ## Features extraction with word embedding print("... Extracting features") Xv_train, Xv_test, vectorizer = get_embeddings(X_train, X_test, max_feature = max_feature , embedding_type= embedding) # train the model print('... Training {} model'.format(classifier.__class__.__name__)) clf = OneVsRestClassifier(classifier) clf.fit(Xv_train, y_train) # compute the test accuracy print("... Computing accuracy") prediction = clf.predict(Xv_test) ## Accuracy score score = (accuracy_score(y_test, prediction)) type2_score = j_score(y_test, prediction) f1_s = f1_score(y_test, prediction,average='macro') roc_auc = roc_auc_score(y_test, prediction) confusion_matrix = multilabel_confusion_matrix(y_test, prediction) score_sumry = [score, type2_score, f1_s, roc_auc] # ## Save model # print("... Saving model in model directory") # pkl_file = os.path.join(dir_path,'model', classifier.__class__.__name__) # file = open(pkl_file,"wb") # pickle.dump(clf,file) # file.close() #### Testing purpose only #### #### Prediction on comment ### # input_str = ["i'm going to kill you n***a, you are you sick or mad, i don't like you at all"] # input_str = clean(input_str[0]) # input_str = process_txt(input_str, stemm= True) # input_str = vectorizer.transform([input_str]) print('\n') print("Model evaluation") print("------") print(print_score(prediction,y_test, classifier)) print('Accuracy is {}'.format(score)) print("ROC_AUC - {}".format(roc_auc)) # print(print("check model accuracy on input_string {}".format(clf.predict(input_str)))) print("------") print("Multilabel confusion matrix \n {}".format(confusion_matrix)) return clf, vectorizer, score_sumry
def evaluate_model(model, X_test, Y_test, category_names): '''Evaluate outcome of model''' # create predictions Y_pred = model.predict(X_test) # create classification report and convert to dataframe report = classification_report(Y_test, Y_pred, target_names=category_names, output_dict=True) # transpose report so that scores are in columns report = pd.DataFrame(report).transpose() print('\n') print(report) print('\n') # multi lable confusion matrix confusion_matrix = multilabel_confusion_matrix(Y_test, Y_pred) # open text file confusion_matrix_report = open('./models/reports/confusion_matrix.txt', 'w') for i in range(len(category_names)): text = '{}\n{}\n{}\n'.format(category_names[i], '-' * 20, confusion_matrix[i]) print(text) confusion_matrix_report.write(text) # close text file confusion_matrix_report.close() # results matrix (select columns) cv_results = pd.DataFrame(model.cv_results_) cv_results.sort_values(by='rank_test_score', inplace=True) cv_results = cv_results[[ 'mean_fit_time', 'param_clf', 'mean_test_score', 'rank_test_score', 'mean_train_score' ]] print('\n') print(cv_results) # GridSearch output print('\nGridSearch result:\n' + '-' * 20) print('Best score:', model.best_estimator_) print('Best estimator:', model.best_estimator_) print('Best params:', model.best_estimator_) # TODO: write training reports to textfile.. with pd.ExcelWriter('./models/reports/report.xlsx') as writer: report.to_excel(writer, sheet_name='report') cv_results.to_excel(writer, sheet_name='cv_results') return report, confusion_matrix, cv_results
def multiLableClassification(x_train, y_train, x_test, y_test): clf = OneVsRestClassifier(SVC(kernel='linear')).fit(x_train, y_train) y_pred = clf.predict(x_test) scores = roc_auc_score(y_test, y_pred) print(scores) confMatrix = multilabel_confusion_matrix(y_test, y_pred, labels=['Yes', 'No']) print(confMatrix)
def build_row(X_test, y_test, y_pred): multi_matrix = multilabel_confusion_matrix(y_test, y_pred) result = [] for i in range(np.unique(y_test).shape[0]): result.extend(array_result(multi_matrix, i)) return result
def mcm_specificity(y, y_pred): mcm = multilabel_confusion_matrix(y, y_pred) metric = mcm[:, 0, 0] / (mcm[:, 0, 0] + mcm[:, 0, 1]) # Be rid of NANs cleaned_metric = [x for x in metric if np.isnan(x) == False] # Return average for all classes if not cleaned_metric: return 1 else: return sum(cleaned_metric) / len(cleaned_metric)
def tensor_confusion_matrix(y_pred, y_true, display_labels): """ Creates per class confusion matrices witch are later concatenated Args: y_pred (torch.Tensor ( N x C )): Sigmoid multi-label predictions y_true (torch.Tensor ( N x C )): Multi-label targets display_labels (list): The string representation of the labels Returns: (torch.Tensor): Image tensor of the concatenated confusion matrices """ # Creates an array of 2x2 arrays of size num_classes cm_array = multilabel_confusion_matrix(y_true.cpu(), y_pred.cpu()) # Create transform to convert PIL to Tensor for Tensorboard to_tensor = torch_transforms.ToTensor() ims = [] for i, cm in enumerate(cm_array): # Plotting one confusion matrix with sklearn disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['Negative', 'Positive']) disp.plot(include_values=True, cmap='Blues', ax=None, xticks_rotation='horizontal', values_format='d') # Add the corresponding label as titel of the plot plt.title(display_labels[i]) # Write figure into buffer and open it as PIL image buf = io.BytesIO() plt.savefig(buf, format='png') buf.seek(0) im = Image.open(buf) # Copy image because buffer is closed afterwards ims.append(im.copy()) buf.close() widths, heights = zip(*(i.size for i in ims)) total_width = sum(widths) max_height = max(heights) # Create new image with (num_classes * width) new_im = Image.new('RGB', (total_width, max_height)) x_offset = 0 # Pasting the confusion matrices into new_im for im in ims: new_im.paste(im, (x_offset, 0)) x_offset += im.size[0] # Transform to tensor for Tensorboard logger new_im = to_tensor(new_im) return new_im
def plot_confusion_matrix(results, class_names): # TODO: Make a confusion matrix plot. # TODO: You do not have to return the plots. # TODO: You can save plots as files by codes here or an interactive way according to your preference. results = np.array(results) matrices = multilabel_confusion_matrix(results[:, 0], results[:, 1]) matrices = multilabel_confusion_matrix(results[:, 0], results[:, 1]) for index, name in enumerate(class_names): matrix = normalize(matrices[index], norm="l1") fig, ax = plt.subplots() heatmap = ax.pcolor(matrix, cmap=plt.cm.Blues, vmin=0, vmax=1) ax.set_xticks(np.arange(len(["NO", "YES"])) + 0.5) ax.set_yticks(np.arange(len(["NO", "YES"])) + 0.5) ax.set_xticklabels(["NO", "YES"]) ax.set_yticklabels(["NO", "YES"]) plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor") for i in range(len(["NO", "YES"])): for j in range(len(["NO", "YES"])): text = ax.text(j + 0.5, i + 0.5, np.round(matrix[i, j], 2), ha="center", va="center") ax.invert_yaxis() plt.colorbar(heatmap) ax.set_title("Normalized Confusion Matrix ({})".format(name)) plt.ylabel('True') plt.xlabel('Predicted') fig.tight_layout() plt.savefig("CheXpert-model_confusion_matrix_{}.png".format(name))
def calculate_class_weights(y_true, y_pred, title): MCM = multilabel_confusion_matrix( y_true[title].values.astype(int), y_pred[title].values.astype(int) ) true_positives = MCM[:, 1, 1] true_sum = true_positives + MCM[:, 1, 0] class_recall = (true_positives / true_sum) class_recall = [(i, r) for i, r in zip([i for i in range(len(class_recall))], class_recall)] class_recall.sort(key=lambda x: x[0]) return class_recall
def resultsMultiClass(self): pred = [] for i in range(len(self.predictions)): temp = np.where( self.predictions[i] == np.amax(self.predictions[i])) pred.append(temp[0][0]) predictions = to_categorical(pred) tes_res = to_categorical(self.tes_res) cm = multilabel_confusion_matrix(predictions, tes_res) print(cm) print(classification_report(predictions, tes_res))
def confusion_metrics_basic(actuals, predictions): lbls = [*range(N_CLASSSES)] mcm = multilabel_confusion_matrix(actuals, predictions, labels=lbls) tp = mcm[:, 1, 1] tn = mcm[:, 0, 0] fn = mcm[:, 1, 0] fp = mcm[:, 0, 1] cm = confusion_matrix(actuals, predictions, labels=lbls, sample_weight=None) return cm, np.mean(tp), np.mean(tn), np.mean(fp), np.mean(fn)
def accuracy_multilabel(y_true, y_pred): Ny, dy = y_true.shape Cm = multilabel_confusion_matrix(y_true, y_pred) accuracy = 0 #print(Cm.shape) for i in range(dy): accuracy += (Cm[i, 0, 0] + Cm[i, 1, 1]) / (Ny) accuracy = accuracy / dy return accuracy
def __init__(self, y_true, y_pred): """ Creates object of class Metrics """ self.y_true = y_true self.y_pred = y_pred self.conf_mat = multilabel_confusion_matrix(self.y_true, self.y_pred) conf_mat_sum = np.zeros((2,2)) for mat in self.conf_mat: conf_mat_sum += mat self.true_negatives, self.false_positives, self.false_negatives, self.true_positives = conf_mat_sum.flatten()
def mcm_fnr(clf, X, y): y_pred = clf.predict(X) mcm = multilabel_confusion_matrix(y, y_pred) # Or the miss rate metric = mcm[:, 1, 0] / (mcm[:, 1, 0] + mcm[:, 1, 1]) # Be rid of NANs cleaned_metric = [x for x in metric if np.isnan(x) == False] # Return average for all classes if not cleaned_metric: return 1 else: return sum(cleaned_metric) / len(cleaned_metric)
def on_epoch_end(self, epoch, logs=None): """calculate confusion matrix at each epoch""" # print(f'validation x type: {type(self.validation_data[0])}') # print(f'validation y type: {type(self.validation_data[0])}') val_x = self.validation_data[0] # would not work val_y = self.validation_data[1] # would not work pred = self.model.predict(val_x) confusion = multilabel_confusion_matrix(val_y, pred) self.confusion.append(confusion) return
def create_confusion_matrix(labeled_image, ground_truth, verbal=False): labeled_image_list = image_to_list(labeled_image) ground_truth_list = image_to_list(ground_truth) if verbal: print() print("CONFUSION MATRIX") from sklearn.metrics import multilabel_confusion_matrix confusion_matrix = multilabel_confusion_matrix(ground_truth_list, labeled_image_list) if verbal: print(confusion_matrix) return confusion_matrix
def model_statistics(path_to_model, lr, momentum, epochs): model = PatternModel() model.load_state_dict(torch.load(path_to_model)) model = model.double() criterion = nn.CrossEntropyLoss() test_dataset = PatternDataset(test_data_dir, test_labels_path) test_loader = DataLoader(test_dataset, batch_size=32, shuffle=True, num_workers=4) test_loss = 0 data_size = len(test_dataset) correct, total = 0, 0 labels_top, preds_top = [], [] with torch.no_grad(): for data in test_loader: pixels, labels = data[0], data[1] outputs = model(pixels) _, predicted = torch.max(outputs.data, 1) test_loss += criterion(outputs, labels).item() labels_top.extend(labels.numpy()) preds_top.extend(predicted.numpy()) overall_accuracy = accuracy_score(labels_top, preds_top, normalize=True) cms = multilabel_confusion_matrix(np.array(labels_top, dtype=np.int32), np.array(preds_top, dtype=np.int32), labels=list(range(7))) # tn fp fn tp tn, tp, fn, fp = cms[:, 0, 0], cms[:, 1, 1], cms[:, 1, 0], cms[:, 0, 1] true_positive_rate = np.around(tp / (tp + fn), 3) true_negative_rate = np.around(tn / (tn + fp), 3) false_positive_rate = np.around(fp / (fp + tn), 3) false_negative_rate = np.around(fn / (fn + tp), 3) classwise_accuracies = np.around((tp + tn) / (tp + tn + fp + fn), 3) stat_file = statistics_path + "statistics_{}_{}_{}.txt".format( lr, momentum, epochs) with open(stat_file, "w") as f: f.write("True Positives: {} {} {} {} {} {} {}\n".format(*tp)) f.write("False Negatives: {} {} {} {} {} {} {}\n".format(*fn)) f.write("False Positives: {} {} {} {} {} {} {}\n".format(*fp)) f.write("True Negatives: {} {} {} {} {} {} {}\n".format(*tn)) f.write("Test Accuracy: {}\n".format(round(overall_accuracy, 3))) f.write("Test Loss: {}\n".format(round(test_loss / data_size, 3))) f.write("Classwise Accuracies: {} {} {} {} {} {} {}\n".format( *classwise_accuracies)) f.write("True Positive Rates: {} {} {} {} {} {} {}\n".format( *true_positive_rate)) f.write("True Negative Rates: {} {} {} {} {} {} {}\n".format( *true_negative_rate)) f.write("False Positive Rates: {} {} {} {} {} {} {}\n".format( *false_positive_rate)) f.write("False Negative Rates: {} {} {} {} {} {} {}\n".format( *false_negative_rate)) return overall_accuracy
def confusionmatrix_confusionmatrices(preds: List[str], labels: List[str], label_list: List[str]): def add_label(*args, label_list: List[str]): outs = [] for arg in args: outs.append(dict(zip(label_list, arg))) return tuple(out for out in outs) confusion_matricks = confusion_matrix(labels, preds, labels=label_list) confusion_matrix_img = plot_cm(confusion_matricks, label_list=label_list) confusion_matrices = multilabel_confusion_matrix(labels, preds, labels=label_list) confusion_matrices = confusion_matrices.tolist() confusion_matrices = add_label(confusion_matrices, label_list=label_list) return confusion_matrix_img, confusion_matrices[0]
def confusion_mat(pred, target, main_task=None, stl=False): if stl: assert main_task is not None, "please supply main task position" target = (target == main_task) * 1 pred = pred.argmax(1) # there are 2 outputs and we use cross-entropy, # so this is the same as taking the one with prob > .5 return confusion_matrix(target, pred) else: # mtl target = [[t] for t in target] one_hot_target = MultiLabelBinarizer().fit_transform(target) preds = (pred > 0) * 1 # this is equivalent to prob > .5 return multilabel_confusion_matrix(one_hot_target, preds)
def calculate_class_weights(y_true, y_pred, title, alpha): MCM = multilabel_confusion_matrix( y_true[title].values.astype(int), y_pred[title].values.astype(int) ) true_positives = MCM[:, 1, 1] true_sum = true_positives + MCM[:, 1, 0] class_recall = (true_positives / true_sum) + alpha class_recall = 1 / class_recall class_recall = class_recall / (true_sum ** (1/2)) class_recall = class_recall / class_recall.sum() class_index = [i for i in range(len(class_recall))] return pd.DataFrame([class_index, class_recall], index=[title, '{}_weight'.format(title)]).T, true_positives, true_sum, MCM
def plot_confusion_matrix(self, predictions): binary_confusion_matrices = multilabel_confusion_matrix(self.test_labels, predictions) print(f"{self.get_classifier_name()} - Confusion Matrix") emotions = ["Anger", "Anticipation", "Disgust", "Fear", "Joy", "Love", "Optimism", "Pessimism", "Sadness", "Surprise", "Trust"] for i in range(len(emotions)): conf_mat = binary_confusion_matrices[i] print(f"{emotions[i]}") print("| Positive | Negative |") print("|:---------:|:-------:|") print(f"| {conf_mat[0][0]} | {conf_mat[0][1]} |") print(f"| {conf_mat[1][0]} | {conf_mat[1][1]} |") print()
def calc_acc_n_loss(args, model, loader, log_matrix=False): """ Function to calculate the Accuracy and Loss given a loader Args: args (TrainOptions): TrainOptions class (refer options/train_options.py) model (Torch Model): Current model object to evaluate loader (DataLoader): DataLoader for dataset log_matrix (bool): Whether to log confusion matrix Returns: tuple: (Model Accuracy, Model Loss, F1 Score, Confusion Matrix, Precision, Recall) """ model.eval() device = args.device y_pred = [] y_true = [] criterion = nn.CrossEntropyLoss() loss = 0 for img, gt in tqdm(loader): img = img.to(device) gt = gt.to(device) out = model(img) loss += criterion(out, gt).item() out = torch.argmax(out, dim=-1) y_pred.extend(list(out.cpu().numpy())) y_true.extend(list(gt.cpu().numpy())) f1 = f1_score(y_true, y_pred, average='weighted') cm = multilabel_confusion_matrix(y_true, y_pred).tolist() precision = precision_score(y_true, y_pred, average='weighted') recall = recall_score(y_true, y_pred, average='weighted') if log_matrix == True: wandb_log_conf_matrix(y_true, y_pred) print(classification_report(y_true, y_pred)) # Calculating accuracy: Number of correct predictions / Number of samples acc = sum(1 for x, y in zip(y_true, y_pred) if x == y) * 100 / len(y_true) return acc, (loss / len(loader)), f1, cm, precision, recall
def calc_metrics_multilabel(self, y, y_hat, labels, type_prediction_label, sample_wise=False): if type_prediction_label in [ dn.TypePredictionLabel.MULTI_LABEL_CATEGORICAL, dn.TypePredictionLabel.SINGLE_LABEL_CATEGORICAL ]: y_hat = (y_hat > 0.5).astype('float32') y = (y > 0.5).astype('float32') else: # self.pp_data.type_prediction_label== 'dn.TypePredictionLabel.SINGLE_LABEL_CATEGORICAL' y_hat = np.argmax(y_hat, axis=1) y = np.argmax(y, axis=1) label_index = [index for index in range(len(labels))] rep_ml_dict = classification_report(y, y_hat, labels=label_index, target_names=labels, zero_division=0, output_dict=True) rep_sl, rep_sl_dict = self.calc_confusion_matrix_by_label(y, y_hat) final_metrics = dict() final_metrics.update({ 'Exact Match Ratio': self.exact_match_ratio(y, y_hat), 'Correct Prediction per Label': self.correct_prediction_by_label(y, y_hat), 'Hamming Loss': hamming_loss(y, y_hat), 'Multi-label Confusion Matrix': multilabel_confusion_matrix(y, y_hat, samplewise=sample_wise), 'Multi-label Report': classification_report(y, y_hat, labels=label_index, target_names=labels, zero_division=0), 'Single-label Report': rep_sl, 'Multi-label Report Dict': rep_ml_dict, 'Single-label Report Dict': rep_sl_dict }) return final_metrics
def evaluate(model, val_iter, epoch, early_stopping, ITERATION): model.eval() labels = [] preds = [] evals = [] imputations = [] save_impute = [] save_label = [] val_yloss = 0.0 for idx, data in enumerate(val_iter): data = utils.to_var(data) ret = model.run_on_batch(data, None) val_yloss += ret['yloss'].item() # save the imputation results which is used to test the improvement of traditional methods with imputed values save_impute.append(ret['imputations'].data.cpu().numpy()) save_label.append(ret['labels'].data.cpu().numpy()) pred = ret['predictions'].data.cpu().numpy() label = ret['labels'].data.cpu().numpy() eval_masks = ret['eval_masks'].data.cpu().numpy() eval_ = ret['evals'].data.cpu().numpy() imputation = ret['imputations'].data.cpu().numpy() evals += eval_[np.where(eval_masks == 1)].tolist() imputations += imputation[np.where(eval_masks == 1)].tolist() labels += label.tolist() preds += pred.tolist() labels = np.asarray(labels).astype('int32') preds = np.asarray(preds) preds = 1*(preds > 0.5) # early stopping based on yloss average_val_loss = val_yloss/len(val_iter) early_stopping(average_val_loss, model) report = metrics.classification_report(labels, preds) file = open('./{}/{}_run{}_val_report'.format(args.savepath, args.runname, ITERATION), "w") file.write(report) file.write("\n") file.close() save_confusion = metrics.multilabel_confusion_matrix(labels, preds) np.save('./{}/{}_run{}_val_conf'.format(args.savepath, args.runname,ITERATION), save_confusion) return early_stopping.early_stop, average_val_loss