def specificity(label_gt, label_pred):
mcm = multilabel_confusion_matrix(label_gt.flatten(), label_pred.flatten())
tn = mcm[:, 0, 0]
tp = mcm[:, 1, 1]
fn = mcm[:, 1, 0]
fp = mcm[:, 0, 1]
sp = tn / (tn + fp)
return sp
def performance_measurement(target, prediction, scenario, performance):
# Generate confusion matrix based on scenario
if scenario == 1:
cm = confusion_matrix(target, prediction, labels=['A', 'B'])
tn, fp, fn, tp = cm.ravel()
performance["tn"] = tn
performance["fp"] = fp
performance["fn"] = fn
performance["tp"] = tp
elif scenario == 2:
cm = multilabel_confusion_matrix(target, prediction, labels=['A', 'B', 'C'])
performance["tn"] = cm[:, 0, 0]
performance["fn"] = cm[:, 1, 0]
performance["tp"] = cm[:, 1, 1]
performance["fp"] = cm[:, 0, 1]
else:
cm = multilabel_confusion_matrix(target, prediction, labels=['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
'K', 'L', 'M', 'O', 'P', 'Q', 'R', 'S'])
performance["tn"] = cm[:, 0, 0]
performance["fn"] = cm[:, 1, 0]
performance["tp"] = cm[:, 1, 1]
performance["fp"] = cm[:, 0, 1]
# Compute True Positive Rate (TPR) | Sensitivity
performance['sensitivity'] += performance["tp"] / (performance["tp"] + performance["fn"])
# Compute True Negative Rate (TNR) | Specificity
performance['specificity'] += performance["tn"] / (performance["tn"] + performance["fp"])
# Compute Accuracy
accuracy = (performance["tp"] + performance["tn"]) / (performance["fp"] + performance["fn"] + performance["tp"] + performance["tn"])
performance['accuracy'] += accuracy
# Compute Misclassification
mc = (performance["fp"] + performance["fn"]) / (performance["fp"] + performance["fn"] + performance["tp"] + performance["tn"])
performance['avg_misclassification'] += mc
# Save misclassification per fold and its average
performance['misclassification_per_fold'].append(mc)
performance['avg_misclassification_per_fold'].append(np.average(mc))
# print(cm)
return performance
def test_multiclass_images():
num_classes = 3
cm = MultiLabelConfusionMatrix(num_classes=num_classes)
y_true, y_pred = get_y_true_y_pred()
# Compute confusion matrix with sklearn
sklearn_CM = multilabel_confusion_matrix(
y_true.transpose((0, 2, 3, 1)).reshape(-1, 3),
y_pred.transpose((0, 2, 3, 1)).reshape(-1, 3))
# Update metric
output = (torch.tensor(y_pred), torch.tensor(y_true))
cm.update(output)
ignite_CM = cm.compute().cpu().numpy()
assert np.all(ignite_CM == sklearn_CM)
# Another test on batch of 2 images
cm = MultiLabelConfusionMatrix(num_classes=num_classes)
# Create a batch of two images:
th_y_true1 = torch.tensor(y_true)
th_y_true2 = torch.tensor(y_true.transpose(0, 1, 3, 2))
th_y_true = torch.cat([th_y_true1, th_y_true2], dim=0)
th_y_pred1 = torch.tensor(y_pred)
th_y_pred2 = torch.tensor(y_pred.transpose(0, 1, 3, 2))
th_y_pred = torch.cat([th_y_pred1, th_y_pred2], dim=0)
# Update metric & compute
output = (th_y_pred, th_y_true)
cm.update(output)
ignite_CM = cm.compute().cpu().numpy()
# Compute confusion matrix with sklearn
th_y_true = idist.all_gather(th_y_true)
th_y_pred = idist.all_gather(th_y_pred)
np_y_true = th_y_true.cpu().numpy().transpose((0, 2, 3, 1)).reshape(-1, 3)
np_y_pred = th_y_pred.cpu().numpy().transpose((0, 2, 3, 1)).reshape(-1, 3)
sklearn_CM = multilabel_confusion_matrix(np_y_true, np_y_pred)
assert np.all(ignite_CM == sklearn_CM)
def evaluate_predictions(labels):
labels_pred = ensemble_predictions(X_test, models)
true_labels = np.argmax(y_test, axis=1)
test_acc = accuracy_score(true_labels, labels_pred)
matrix = multilabel_confusion_matrix(true_labels,labels_pred, labels=[6,5,4,3,2,1,0])
print('Confusion matrix : \n',matrix)
# classification report for precision, recall f1-score and accuracy
print('Classification report : \n', classification_report(true_labels,labels_pred, labels=[6,5,4,3,2,1,0]))
return test_acc
def print_confm(test, predictions, model):
if (model == 'b') or (model == 'g'):
confm = confusion_matrix(test, predictions)
print("Confusion Matrix:\n")
print(confm)
elif (model == 'r'):
confm = multilabel_confusion_matrix(test, predictions)
print("Confusion Matrix:\n")
print(confm)
def accuracy(args):
# Loading model with weights
model = lu.select_model(args)
# Saving image paths as a list
test_image_paths, test_label_paths = lu.file_paths(args.csv_paths)
# Calculating accuracy
y, y_pred = [], []
rescale_value = lu.rescaling_value(args.rs)
for i in range(len(test_image_paths)):
image = gdal.Open(test_image_paths[i])
image_array = np.array(image.ReadAsArray()) / rescale_value
image_array = image_array.transpose(1, 2, 0)
label = gdal.Open(test_label_paths[i])
label_array = np.array(label.ReadAsArray()) / args.rs_label
label_array = np.expand_dims(label_array, axis=-1)
fm = np.expand_dims(image_array, axis=0)
result_array = model.predict(fm)
if args.num_classes != 1:
result_array = np.argmax(result_array[0], axis=2)
result_array = np.squeeze(result_array)
if args.num_classes == 1:
result_array = np.around(result_array.flatten())
y.append(np.around(label_array))
y_pred.append(result_array)
print("Predicted " + str(i + 1) + " Patches")
print("\n")
print("list of classes from predictions: " +
str(np.unique(np.array(y_pred))))
print("list of classes from labels: " + str(np.unique(np.array(y))))
print("\n")
cm = confusion_matrix(np.array(y).flatten(), np.array(y_pred).flatten())
cm_multi = multilabel_confusion_matrix(
np.array(y).flatten(),
np.array(y_pred).flatten())
print("Confusion Matrix " + "\n")
print(cm, "\n")
accuracy = np.trace(cm / np.sum(cm))
print("Overal Accuracy: ", round(accuracy, 3), "\n")
mean_iou = 0
mean_f1 = 0
for j in range(len(cm_multi)):
print("Class: " + str(j))
iou = cm_multi[j][1][1] / (cm_multi[j][1][1] + cm_multi[j][0][1] +
cm_multi[j][1][0])
f1 = (2 * cm_multi[j][1][1]) / (2 * cm_multi[j][1][1] +
cm_multi[j][0][1] + cm_multi[j][1][0])
precision = cm_multi[j][1][1] / (cm_multi[j][1][1] + cm_multi[j][0][1])
recall = cm_multi[j][1][1] / (cm_multi[j][1][1] + cm_multi[j][1][0])
mean_iou += iou
mean_f1 += f1
print("IoU Score: ", round(iou, 3))
print("F1-Measure: ", round(f1, 3))
print("Precision: ", round(precision, 3))
print("Recall: ", round(recall, 3), "\n")
print("Mean IoU Score: ", round(mean_iou / len(cm_multi), 3))
print("Mean F1-Measure: ", round(mean_f1 / len(cm_multi), 3))
def f1_validation(y_true, y_pred):
# performance
print(metrics.classification_report(y_true, y_pred))
print(multilabel_confusion_matrix(y_true, y_pred))
#print("F1 micro: %1.4f\n" % f1_score(y_test, y_predicted, average='micro'))
print("F1 macro: %1.4f\n" % f1_score(y_true, y_pred, average='macro'))
#print("F1 weighted: %1.4f\n" % f1_score(y_test, y_predicted, average='weighted'))
#print("Accuracy: %1.4f" % (accuracy_score(y_test, y_predicted)))
return f1_score(y_true, y_pred, average='macro')
def train_model(classifier,X, y, max_feature = 1000, embedding= 'bow' ):
#Train-test split
print("... Performing train test split")
X_train, X_test, y_train, y_test = model_selection.train_test_split(X,y,
test_size=0.25,random_state=42)
## Features extraction with word embedding
print("... Extracting features")
Xv_train, Xv_test, vectorizer = get_embeddings(X_train, X_test,
max_feature = max_feature , embedding_type= embedding)
# train the model
print('... Training {} model'.format(classifier.__class__.__name__))
clf = OneVsRestClassifier(classifier)
clf.fit(Xv_train, y_train)
# compute the test accuracy
print("... Computing accuracy")
prediction = clf.predict(Xv_test)
## Accuracy score
score = (accuracy_score(y_test, prediction))
type2_score = j_score(y_test, prediction)
f1_s = f1_score(y_test, prediction,average='macro')
roc_auc = roc_auc_score(y_test, prediction)
confusion_matrix = multilabel_confusion_matrix(y_test, prediction)
score_sumry = [score, type2_score, f1_s, roc_auc]
# ## Save model
# print("... Saving model in model directory")
# pkl_file = os.path.join(dir_path,'model', classifier.__class__.__name__)
# file = open(pkl_file,"wb")
# pickle.dump(clf,file)
# file.close()
#### Testing purpose only ####
#### Prediction on comment ###
# input_str = ["i'm going to kill you n***a, you are you sick or mad, i don't like you at all"]
# input_str = clean(input_str[0])
# input_str = process_txt(input_str, stemm= True)
# input_str = vectorizer.transform([input_str])
print('\n')
print("Model evaluation")
print("------")
print(print_score(prediction,y_test, classifier))
print('Accuracy is {}'.format(score))
print("ROC_AUC - {}".format(roc_auc))
# print(print("check model accuracy on input_string {}".format(clf.predict(input_str))))
print("------")
print("Multilabel confusion matrix \n {}".format(confusion_matrix))
return clf, vectorizer, score_sumry
def evaluate_model(model, X_test, Y_test, category_names):
'''Evaluate outcome of model'''
# create predictions
Y_pred = model.predict(X_test)
# create classification report and convert to dataframe
report = classification_report(Y_test,
Y_pred,
target_names=category_names,
output_dict=True)
# transpose report so that scores are in columns
report = pd.DataFrame(report).transpose()
print('\n')
print(report)
print('\n')
# multi lable confusion matrix
confusion_matrix = multilabel_confusion_matrix(Y_test, Y_pred)
# open text file
confusion_matrix_report = open('./models/reports/confusion_matrix.txt',
'w')
for i in range(len(category_names)):
text = '{}\n{}\n{}\n'.format(category_names[i], '-' * 20,
confusion_matrix[i])
print(text)
confusion_matrix_report.write(text)
# close text file
confusion_matrix_report.close()
# results matrix (select columns)
cv_results = pd.DataFrame(model.cv_results_)
cv_results.sort_values(by='rank_test_score', inplace=True)
cv_results = cv_results[[
'mean_fit_time', 'param_clf', 'mean_test_score', 'rank_test_score',
'mean_train_score'
]]
print('\n')
print(cv_results)
# GridSearch output
print('\nGridSearch result:\n' + '-' * 20)
print('Best score:', model.best_estimator_)
print('Best estimator:', model.best_estimator_)
print('Best params:', model.best_estimator_)
# TODO: write training reports to textfile..
with pd.ExcelWriter('./models/reports/report.xlsx') as writer:
report.to_excel(writer, sheet_name='report')
cv_results.to_excel(writer, sheet_name='cv_results')
return report, confusion_matrix, cv_results
def multiLableClassification(x_train, y_train, x_test, y_test):
clf = OneVsRestClassifier(SVC(kernel='linear')).fit(x_train, y_train)
y_pred = clf.predict(x_test)
scores = roc_auc_score(y_test, y_pred)
print(scores)
confMatrix = multilabel_confusion_matrix(y_test,
y_pred,
labels=['Yes', 'No'])
print(confMatrix)
def build_row(X_test, y_test, y_pred):
multi_matrix = multilabel_confusion_matrix(y_test, y_pred)
result = []
for i in range(np.unique(y_test).shape[0]):
result.extend(array_result(multi_matrix, i))
return result
def mcm_specificity(y, y_pred):
mcm = multilabel_confusion_matrix(y, y_pred)
metric = mcm[:, 0, 0] / (mcm[:, 0, 0] + mcm[:, 0, 1])
# Be rid of NANs
cleaned_metric = [x for x in metric if np.isnan(x) == False]
# Return average for all classes
if not cleaned_metric:
return 1
else:
return sum(cleaned_metric) / len(cleaned_metric)
def tensor_confusion_matrix(y_pred, y_true, display_labels):
"""
Creates per class confusion matrices witch are later concatenated
Args:
y_pred (torch.Tensor ( N x C )): Sigmoid multi-label predictions
y_true (torch.Tensor ( N x C )): Multi-label targets
display_labels (list): The string representation of the labels
Returns:
(torch.Tensor): Image tensor of the concatenated confusion matrices
"""
# Creates an array of 2x2 arrays of size num_classes
cm_array = multilabel_confusion_matrix(y_true.cpu(), y_pred.cpu())
# Create transform to convert PIL to Tensor for Tensorboard
to_tensor = torch_transforms.ToTensor()
ims = []
for i, cm in enumerate(cm_array):
# Plotting one confusion matrix with sklearn
disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['Negative', 'Positive'])
disp.plot(include_values=True, cmap='Blues', ax=None, xticks_rotation='horizontal', values_format='d')
# Add the corresponding label as titel of the plot
plt.title(display_labels[i])
# Write figure into buffer and open it as PIL image
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
im = Image.open(buf)
# Copy image because buffer is closed afterwards
ims.append(im.copy())
buf.close()
widths, heights = zip(*(i.size for i in ims))
total_width = sum(widths)
max_height = max(heights)
# Create new image with (num_classes * width)
new_im = Image.new('RGB', (total_width, max_height))
x_offset = 0
# Pasting the confusion matrices into new_im
for im in ims:
new_im.paste(im, (x_offset, 0))
x_offset += im.size[0]
# Transform to tensor for Tensorboard logger
new_im = to_tensor(new_im)
return new_im
def plot_confusion_matrix(results, class_names):
# TODO: Make a confusion matrix plot.
# TODO: You do not have to return the plots.
# TODO: You can save plots as files by codes here or an interactive way according to your preference.
results = np.array(results)
matrices = multilabel_confusion_matrix(results[:, 0], results[:, 1])
matrices = multilabel_confusion_matrix(results[:, 0], results[:, 1])
for index, name in enumerate(class_names):
matrix = normalize(matrices[index], norm="l1")
fig, ax = plt.subplots()
heatmap = ax.pcolor(matrix, cmap=plt.cm.Blues, vmin=0, vmax=1)
ax.set_xticks(np.arange(len(["NO", "YES"])) + 0.5)
ax.set_yticks(np.arange(len(["NO", "YES"])) + 0.5)
ax.set_xticklabels(["NO", "YES"])
ax.set_yticklabels(["NO", "YES"])
plt.setp(ax.get_xticklabels(),
rotation=45,
ha="right",
rotation_mode="anchor")
for i in range(len(["NO", "YES"])):
for j in range(len(["NO", "YES"])):
text = ax.text(j + 0.5,
i + 0.5,
np.round(matrix[i, j], 2),
ha="center",
va="center")
ax.invert_yaxis()
plt.colorbar(heatmap)
ax.set_title("Normalized Confusion Matrix ({})".format(name))
plt.ylabel('True')
plt.xlabel('Predicted')
fig.tight_layout()
plt.savefig("CheXpert-model_confusion_matrix_{}.png".format(name))
def calculate_class_weights(y_true, y_pred, title):
MCM = multilabel_confusion_matrix(
y_true[title].values.astype(int),
y_pred[title].values.astype(int)
)
true_positives = MCM[:, 1, 1]
true_sum = true_positives + MCM[:, 1, 0]
class_recall = (true_positives / true_sum)
class_recall = [(i, r) for i, r in zip([i for i in range(len(class_recall))], class_recall)]
class_recall.sort(key=lambda x: x[0])
return class_recall
def resultsMultiClass(self):
pred = []
for i in range(len(self.predictions)):
temp = np.where(
self.predictions[i] == np.amax(self.predictions[i]))
pred.append(temp[0][0])
predictions = to_categorical(pred)
tes_res = to_categorical(self.tes_res)
cm = multilabel_confusion_matrix(predictions, tes_res)
print(cm)
print(classification_report(predictions, tes_res))
def confusion_metrics_basic(actuals, predictions):
lbls = [*range(N_CLASSSES)]
mcm = multilabel_confusion_matrix(actuals, predictions, labels=lbls)
tp = mcm[:, 1, 1]
tn = mcm[:, 0, 0]
fn = mcm[:, 1, 0]
fp = mcm[:, 0, 1]
cm = confusion_matrix(actuals,
predictions,
labels=lbls,
sample_weight=None)
return cm, np.mean(tp), np.mean(tn), np.mean(fp), np.mean(fn)
def accuracy_multilabel(y_true, y_pred):
Ny, dy = y_true.shape
Cm = multilabel_confusion_matrix(y_true, y_pred)
accuracy = 0
#print(Cm.shape)
for i in range(dy):
accuracy += (Cm[i, 0, 0] + Cm[i, 1, 1]) / (Ny)
accuracy = accuracy / dy
return accuracy
def __init__(self, y_true, y_pred):
""" Creates object of class Metrics """
self.y_true = y_true
self.y_pred = y_pred
self.conf_mat = multilabel_confusion_matrix(self.y_true,
self.y_pred)
conf_mat_sum = np.zeros((2,2))
for mat in self.conf_mat:
conf_mat_sum += mat
self.true_negatives, self.false_positives, self.false_negatives, self.true_positives = conf_mat_sum.flatten()
def mcm_fnr(clf, X, y):
y_pred = clf.predict(X)
mcm = multilabel_confusion_matrix(y, y_pred)
# Or the miss rate
metric = mcm[:, 1, 0] / (mcm[:, 1, 0] + mcm[:, 1, 1])
# Be rid of NANs
cleaned_metric = [x for x in metric if np.isnan(x) == False]
# Return average for all classes
if not cleaned_metric:
return 1
else:
return sum(cleaned_metric) / len(cleaned_metric)
def on_epoch_end(self, epoch, logs=None):
"""calculate confusion matrix at each epoch"""
# print(f'validation x type: {type(self.validation_data[0])}')
# print(f'validation y type: {type(self.validation_data[0])}')
val_x = self.validation_data[0] # would not work
val_y = self.validation_data[1] # would not work
pred = self.model.predict(val_x)
confusion = multilabel_confusion_matrix(val_y, pred)
self.confusion.append(confusion)
return
def create_confusion_matrix(labeled_image, ground_truth, verbal=False):
labeled_image_list = image_to_list(labeled_image)
ground_truth_list = image_to_list(ground_truth)
if verbal:
print()
print("CONFUSION MATRIX")
from sklearn.metrics import multilabel_confusion_matrix
confusion_matrix = multilabel_confusion_matrix(ground_truth_list, labeled_image_list)
if verbal:
print(confusion_matrix)
return confusion_matrix
def model_statistics(path_to_model, lr, momentum, epochs):
model = PatternModel()
model.load_state_dict(torch.load(path_to_model))
model = model.double()
criterion = nn.CrossEntropyLoss()
test_dataset = PatternDataset(test_data_dir, test_labels_path)
test_loader = DataLoader(test_dataset,
batch_size=32,
shuffle=True,
num_workers=4)
test_loss = 0
data_size = len(test_dataset)
correct, total = 0, 0
labels_top, preds_top = [], []
with torch.no_grad():
for data in test_loader:
pixels, labels = data[0], data[1]
outputs = model(pixels)
_, predicted = torch.max(outputs.data, 1)
test_loss += criterion(outputs, labels).item()
labels_top.extend(labels.numpy())
preds_top.extend(predicted.numpy())
overall_accuracy = accuracy_score(labels_top, preds_top, normalize=True)
cms = multilabel_confusion_matrix(np.array(labels_top, dtype=np.int32),
np.array(preds_top, dtype=np.int32),
labels=list(range(7))) # tn fp fn tp
tn, tp, fn, fp = cms[:, 0, 0], cms[:, 1, 1], cms[:, 1, 0], cms[:, 0, 1]
true_positive_rate = np.around(tp / (tp + fn), 3)
true_negative_rate = np.around(tn / (tn + fp), 3)
false_positive_rate = np.around(fp / (fp + tn), 3)
false_negative_rate = np.around(fn / (fn + tp), 3)
classwise_accuracies = np.around((tp + tn) / (tp + tn + fp + fn), 3)
stat_file = statistics_path + "statistics_{}_{}_{}.txt".format(
lr, momentum, epochs)
with open(stat_file, "w") as f:
f.write("True Positives: {} {} {} {} {} {} {}\n".format(*tp))
f.write("False Negatives: {} {} {} {} {} {} {}\n".format(*fn))
f.write("False Positives: {} {} {} {} {} {} {}\n".format(*fp))
f.write("True Negatives: {} {} {} {} {} {} {}\n".format(*tn))
f.write("Test Accuracy: {}\n".format(round(overall_accuracy, 3)))
f.write("Test Loss: {}\n".format(round(test_loss / data_size, 3)))
f.write("Classwise Accuracies: {} {} {} {} {} {} {}\n".format(
*classwise_accuracies))
f.write("True Positive Rates: {} {} {} {} {} {} {}\n".format(
*true_positive_rate))
f.write("True Negative Rates: {} {} {} {} {} {} {}\n".format(
*true_negative_rate))
f.write("False Positive Rates: {} {} {} {} {} {} {}\n".format(
*false_positive_rate))
f.write("False Negative Rates: {} {} {} {} {} {} {}\n".format(
*false_negative_rate))
return overall_accuracy
def confusionmatrix_confusionmatrices(preds: List[str], labels: List[str], label_list: List[str]):
def add_label(*args, label_list: List[str]):
outs = []
for arg in args:
outs.append(dict(zip(label_list, arg)))
return tuple(out for out in outs)
confusion_matricks = confusion_matrix(labels, preds, labels=label_list)
confusion_matrix_img = plot_cm(confusion_matricks, label_list=label_list)
confusion_matrices = multilabel_confusion_matrix(labels, preds, labels=label_list)
confusion_matrices = confusion_matrices.tolist()
confusion_matrices = add_label(confusion_matrices, label_list=label_list)
return confusion_matrix_img, confusion_matrices[0]
def confusion_mat(pred, target, main_task=None, stl=False):
if stl:
assert main_task is not None, "please supply main task position"
target = (target == main_task) * 1
pred = pred.argmax(1) # there are 2 outputs and we use cross-entropy,
# so this is the same as taking the one with prob > .5
return confusion_matrix(target, pred)
else: # mtl
target = [[t] for t in target]
one_hot_target = MultiLabelBinarizer().fit_transform(target)
preds = (pred > 0) * 1 # this is equivalent to prob > .5
return multilabel_confusion_matrix(one_hot_target, preds)
def calculate_class_weights(y_true, y_pred, title, alpha):
MCM = multilabel_confusion_matrix(
y_true[title].values.astype(int),
y_pred[title].values.astype(int)
)
true_positives = MCM[:, 1, 1]
true_sum = true_positives + MCM[:, 1, 0]
class_recall = (true_positives / true_sum) + alpha
class_recall = 1 / class_recall
class_recall = class_recall / (true_sum ** (1/2))
class_recall = class_recall / class_recall.sum()
class_index = [i for i in range(len(class_recall))]
return pd.DataFrame([class_index, class_recall], index=[title, '{}_weight'.format(title)]).T, true_positives, true_sum, MCM
def plot_confusion_matrix(self, predictions):
binary_confusion_matrices = multilabel_confusion_matrix(self.test_labels, predictions)
print(f"{self.get_classifier_name()} - Confusion Matrix")
emotions = ["Anger", "Anticipation", "Disgust", "Fear", "Joy", "Love", "Optimism", "Pessimism", "Sadness",
"Surprise", "Trust"]
for i in range(len(emotions)):
conf_mat = binary_confusion_matrices[i]
print(f"{emotions[i]}")
print("| Positive | Negative |")
print("|:---------:|:-------:|")
print(f"| {conf_mat[0][0]} | {conf_mat[0][1]} |")
print(f"| {conf_mat[1][0]} | {conf_mat[1][1]} |")
print()
def calc_acc_n_loss(args, model, loader, log_matrix=False):
"""
Function to calculate the Accuracy and Loss given a loader
Args:
args (TrainOptions): TrainOptions class (refer options/train_options.py)
model (Torch Model): Current model object to evaluate
loader (DataLoader): DataLoader for dataset
log_matrix (bool): Whether to log confusion matrix
Returns:
tuple: (Model Accuracy, Model Loss, F1 Score, Confusion Matrix, Precision, Recall)
"""
model.eval()
device = args.device
y_pred = []
y_true = []
criterion = nn.CrossEntropyLoss()
loss = 0
for img, gt in tqdm(loader):
img = img.to(device)
gt = gt.to(device)
out = model(img)
loss += criterion(out, gt).item()
out = torch.argmax(out, dim=-1)
y_pred.extend(list(out.cpu().numpy()))
y_true.extend(list(gt.cpu().numpy()))
f1 = f1_score(y_true, y_pred, average='weighted')
cm = multilabel_confusion_matrix(y_true, y_pred).tolist()
precision = precision_score(y_true, y_pred, average='weighted')
recall = recall_score(y_true, y_pred, average='weighted')
if log_matrix == True:
wandb_log_conf_matrix(y_true, y_pred)
print(classification_report(y_true, y_pred))
# Calculating accuracy: Number of correct predictions / Number of samples
acc = sum(1 for x, y in zip(y_true, y_pred) if x == y) * 100 / len(y_true)
return acc, (loss / len(loader)), f1, cm, precision, recall
def calc_metrics_multilabel(self,
y,
y_hat,
labels,
type_prediction_label,
sample_wise=False):
if type_prediction_label in [
dn.TypePredictionLabel.MULTI_LABEL_CATEGORICAL,
dn.TypePredictionLabel.SINGLE_LABEL_CATEGORICAL
]:
y_hat = (y_hat > 0.5).astype('float32')
y = (y > 0.5).astype('float32')
else: # self.pp_data.type_prediction_label== 'dn.TypePredictionLabel.SINGLE_LABEL_CATEGORICAL'
y_hat = np.argmax(y_hat, axis=1)
y = np.argmax(y, axis=1)
label_index = [index for index in range(len(labels))]
rep_ml_dict = classification_report(y,
y_hat,
labels=label_index,
target_names=labels,
zero_division=0,
output_dict=True)
rep_sl, rep_sl_dict = self.calc_confusion_matrix_by_label(y, y_hat)
final_metrics = dict()
final_metrics.update({
'Exact Match Ratio':
self.exact_match_ratio(y, y_hat),
'Correct Prediction per Label':
self.correct_prediction_by_label(y, y_hat),
'Hamming Loss':
hamming_loss(y, y_hat),
'Multi-label Confusion Matrix':
multilabel_confusion_matrix(y, y_hat, samplewise=sample_wise),
'Multi-label Report':
classification_report(y,
y_hat,
labels=label_index,
target_names=labels,
zero_division=0),
'Single-label Report':
rep_sl,
'Multi-label Report Dict':
rep_ml_dict,
'Single-label Report Dict':
rep_sl_dict
})
return final_metrics
def evaluate(model, val_iter, epoch, early_stopping, ITERATION):
model.eval()
labels = []
preds = []
evals = []
imputations = []
save_impute = []
save_label = []
val_yloss = 0.0
for idx, data in enumerate(val_iter):
data = utils.to_var(data)
ret = model.run_on_batch(data, None)
val_yloss += ret['yloss'].item()
# save the imputation results which is used to test the improvement of traditional methods with imputed values
save_impute.append(ret['imputations'].data.cpu().numpy())
save_label.append(ret['labels'].data.cpu().numpy())
pred = ret['predictions'].data.cpu().numpy()
label = ret['labels'].data.cpu().numpy()
eval_masks = ret['eval_masks'].data.cpu().numpy()
eval_ = ret['evals'].data.cpu().numpy()
imputation = ret['imputations'].data.cpu().numpy()
evals += eval_[np.where(eval_masks == 1)].tolist()
imputations += imputation[np.where(eval_masks == 1)].tolist()
labels += label.tolist()
preds += pred.tolist()
labels = np.asarray(labels).astype('int32')
preds = np.asarray(preds)
preds = 1*(preds > 0.5)
# early stopping based on yloss
average_val_loss = val_yloss/len(val_iter)
early_stopping(average_val_loss, model)
report = metrics.classification_report(labels, preds)
file = open('./{}/{}_run{}_val_report'.format(args.savepath, args.runname, ITERATION), "w")
file.write(report)
file.write("\n")
file.close()
save_confusion = metrics.multilabel_confusion_matrix(labels, preds)
np.save('./{}/{}_run{}_val_conf'.format(args.savepath, args.runname,ITERATION), save_confusion)
return early_stopping.early_stop, average_val_loss
