def ensemble_aug_eval(n_iter, class_model, with_temp_scal=False):
acc_test = 0
auc_test = 0
ens_preds = torch.zeros_like(class_model.calibration_variables[2][0])
start_time = time.time()
# data_loader, test_data_loader, valid_data_loader = get_dataloader(dname='isic2019_testwaugm', size=class_model.size,
# SRV=class_model.SRV,
# batch_size=class_model.batch_size,
# n_workers=class_model.n_workers,
# augm_config=class_model.augm_config,
# cutout_params=[class_model.cutout_nholes,
# class_model.cutout_pad_size])
data_loader, test_data_loader, valid_data_loader = class_model.data_loader, class_model.test_data_loader, class_model.valid_data_loader
for i in range(1, n_iter + 1):
acc_test_temp, w_acc_test_temp, conf_matrix_test_temp, acc_1_test_temp, pr_test_temp, rec_test_temp, fscore_test_temp, auc_test_temp, preds, true_lab = \
eval(class_model, test_data_loader, *class_model.calibration_variables[2], with_temp_scal)
# print('iteration ' + str(i) + ' completed in ' + str(time.time()-start_time) + ' seconds')
# print('Acc: ' + str(acc_test_temp) + ' | Weighted Acc: ' + str(w_acc_test_temp) + '\n')
# print(conf_matrix_temp)
acc_test += acc_test_temp
auc_test += auc_test_temp
ens_preds += preds
conf_matrix_test = ConfusionMatrix(class_model.num_classes)
temp_ens_preds = ens_preds / n_iter
check_output, res = torch.max(torch.tensor(temp_ens_preds, device='cuda'),
1)
conf_matrix_test.update_matrix(res, torch.tensor(true_lab, device='cuda'))
ens_acc, ens_w_acc = conf_matrix_test.get_metrics()
ens_acc_1, pr, rec, fscore, auc = compute_accuracy_metrics(
temp_ens_preds, true_lab)
print(
"\n|| took {:.1f} minutes \n"
"| Mean Accuracy statistics: Acc: {:.3f} AUC: {:.3f} \n"
"| Ensemble Accuracy statistics: Weighted Acc: {:.3f} AUC: {:.3f} Recall: {:.3f} Precision: {:.3f} Fscore: {:.3f} \n"
.format((time.time() - start_time) / 60., acc_test / i, auc_test / i,
ens_w_acc, auc, rec, pr, fscore))
print(conf_matrix_test.conf_matrix)
return ens_acc, ens_w_acc, conf_matrix_test, ens_acc, pr, rec, fscore, auc, temp_ens_preds, true_lab
def ensemble_aug_eval(n_iter, class_model, with_temp_scal=False):
acc_test = 0
w_acc_test = 0
ens_preds = torch.zeros_like(class_model.calibration_variables[2][0])
start_time = time.time()
data_loader, test_data_loader, valid_data_loader = class_model.data_loader, class_model.test_data_loader, class_model.valid_data_loader
for i in range(1, n_iter + 1):
acc_test_temp, w_acc_test_temp, calibration_statistics, conf_matrix_temp, _ = \
eval(class_model, test_data_loader, *class_model.calibration_variables[2], with_temp_scal)
acc_test += acc_test_temp
w_acc_test += w_acc_test_temp
_, preds, true_lab = calibration_statistics
ens_preds += preds
conf_matrix_test = ConfusionMatrix(class_model.num_classes)
temp_ens_preds = ens_preds / n_iter
check_output, res = torch.max(torch.tensor(temp_ens_preds, device='cuda'),
1)
conf_matrix_test.update_matrix(res, torch.tensor(true_lab, device='cuda'))
ens_acc, ens_w_acc = conf_matrix_test.get_metrics()
ECE_test, MCE_test, BRIER_test, NNL_test = compute_calibration_measures(
temp_ens_preds, true_lab, apply_softmax=False, bins=15)
print(
"\n|| took {:.1f} minutes \n"
"| Mean Accuracy statistics: weighted Acc test: {:.3f} Acc test: {:.3f} \n"
"| Ensemble Accuracy statistics: weighted Acc test: {:.3f} Acc test: {:.3f} \n"
"| Calibration test: ECE: {:.5f} MCE: {:.5f} BRIER: {:.5f} NNL: {:.5f}\n\n"
.format((time.time() - start_time) / 60., w_acc_test / i, acc_test / i,
ens_w_acc, ens_acc, ECE_test * 100, MCE_test * 100, BRIER_test,
NNL_test))
print(conf_matrix_test.conf_matrix)
return ens_acc, ens_w_acc, (ens_preds / n_iter), true_lab
def eval(class_model,
e_loader,
predictions,
labels,
with_temp_scal=False,
compute_separate_metrics_for_errors=False):
with torch.no_grad():
entropy_of_predictions = torch.zeros_like(labels).float()
'''to measure stuff on correct and incorrect classified samples'''
if compute_separate_metrics_for_errors:
corr_entropy = torch.zeros_like(labels).float()
incorr_entropy = torch.zeros_like(labels).float()
corr_labels = torch.zeros_like(labels).float()
incorr_labels = torch.zeros_like(labels).float()
corr_predictions = torch.zeros_like(predictions).float()
incorr_predictions = torch.zeros_like(predictions).float()
corr_count = 0
incorr_count = 0
class_model.n.eval()
sofmx = nn.Softmax(dim=-1)
conf_matrix = ConfusionMatrix(class_model.num_classes)
start_time = time.time()
for idx, (x, target, img_name) in enumerate(e_loader):
# measure data loading time
# print("data time: " + str(time.time() - start_time))
# compute output
x = x.to('cuda')
out = class_model.n(x)
if with_temp_scal:
out = class_model.temp_scal_model(out)
# output = torch.squeeze(out)
output = out
target = target.to('cuda', torch.long)
check_output_all = sofmx(output)
check_output, res = torch.max(check_output_all, -1)
aux = target.size(0)
predictions[idx *
class_model.batch_size:idx * class_model.batch_size +
aux, :] = check_output_all.data.cpu()
labels[idx * class_model.batch_size:idx * class_model.batch_size +
aux] = target.data.cpu()
entropy_of_predictions[idx * class_model.batch_size:idx *
class_model.batch_size +
aux] = entropy_categorical(
check_output_all).cpu()
# update the confusion matrix
conf_matrix.update_matrix(res, target)
# measure batch time
# print("batch " + str(idx) + " of " + str(len(e_loader)) + "; time: " + str(time.time() - start_time))
# start_time = time.time()
# if idx == 0:
# break
if compute_separate_metrics_for_errors:
# if true we compute the entropy and calibration measures on correct and incorrect samples separately
corr_idx = check_output_all.argmax(dim=1) == target
incorr_idx = check_output_all.argmax(dim=1) != target
corr_samples_prob = check_output_all[corr_idx, :]
incorr_samples_prob = check_output_all[incorr_idx, :]
corr_numel = corr_idx.sum().long()
incorr_numel = incorr_idx.sum().long()
corr_entropy[corr_count:corr_count +
corr_numel] = entropy_categorical(
corr_samples_prob).cpu()
incorr_entropy[incorr_count:incorr_count +
incorr_numel] = entropy_categorical(
incorr_samples_prob).cpu()
corr_predictions[corr_count:corr_count +
corr_numel] = corr_samples_prob.cpu()
incorr_predictions[incorr_count:incorr_count +
incorr_numel] = incorr_samples_prob.cpu()
corr_labels[corr_count:corr_count +
corr_numel] = target[corr_idx].cpu()
incorr_labels[incorr_count:incorr_count +
incorr_numel] = target[incorr_idx].cpu()
corr_count += corr_numel
incorr_count += incorr_numel
# filter out the zeros
per_samples_stats = None
if compute_separate_metrics_for_errors:
corr_entropy = corr_entropy[0:corr_count]
incorr_entropy = incorr_entropy[0:incorr_count]
corr_predictions = corr_predictions[0:corr_count]
incorr_predictions = incorr_predictions[0:incorr_count]
corr_labels = corr_labels[0:corr_count]
incorr_labels = incorr_labels[0:incorr_count]
per_samples_stats = {
'corr': [corr_entropy, corr_predictions, corr_labels],
'incorr': [incorr_entropy, incorr_predictions, incorr_labels]
}
acc, w_acc = conf_matrix.get_metrics()
acc_1, pr, rec, fscore, auc = compute_accuracy_metrics(
predictions, labels)
return acc, w_acc, conf_matrix.conf_matrix, acc_1, pr, rec, fscore, auc, predictions, labels
w_acc_test += w_acc
if ens_preds is None:
ens_preds = preds
else:
ens_preds += preds
except Exception as e:
print(f'Exception {e}')
conf_matrix_test = ConfusionMatrix(n.num_classes)
temp_ens_preds = ens_preds / counter
check_output, res = torch.max(torch.tensor(temp_ens_preds, device='cuda'),
1)
conf_matrix_test.update_matrix(res, torch.tensor(true_lab, device='cuda'))
ens_acc, ens_w_acc = conf_matrix_test.get_metrics()
ens_acc_1, pr, rec, fscore, auc = compute_accuracy_metrics(
temp_ens_preds, true_lab)
print("\n ----- FINAL PRINT ----- \n")
print(
"\n|| took {:.1f} minutes \n"
"| Mean Accuracy statistics: Weighted Acc: {:.3f} AUC: {:.3f} \n"
"| Ensemble Accuracy statistics: Weighted Acc: {:.3f} AUC: {:.3f} Recall: {:.3f} Precision: {:.3f} Fscore: {:.3f} \n"
.format((time.time() - start_time) / 60., w_acc_test / counter,
auc_test / counter, ens_w_acc, auc, rec, pr, fscore))
print(conf_matrix_test.conf_matrix)
np.save(os.path.join(output_path, "output_" + fname + ".npy"),
temp_ens_preds)