def log_results(trainer):
scheduler.step()
if trainer.state.epoch % 5 == 0:
evaluator.run(dl_val)
accuracy, roc_auc_mnist = get_fashionmnist_mnist_ood(model)
accuracy, roc_auc_notmnist = get_fashionmnist_notmnist_ood(model)
metrics = evaluator.state.metrics
print(f"Validation Results - Epoch: {trainer.state.epoch} "
f"Acc: {metrics['accuracy']:.4f} "
f"BCE: {metrics['bce']:.2f} "
f"GP: {metrics['gradient_penalty']:.6f} "
f"AUROC MNIST: {roc_auc_mnist:.2f} "
f"AUROC NotMNIST: {roc_auc_notmnist:.2f} ")
print(f"Sigma: {model.sigma}")
results = {}
for l_gradient_penalty in l_gradient_penalties:
for length_scale in length_scales:
val_accuracies = []
test_accuracies = []
roc_aucs_mnist = []
roc_aucs_notmnist = []
for _ in range(repetition):
print(" ### NEW MODEL ### ")
model, val_accuracy, test_accuracy = train_model(
l_gradient_penalty, length_scale, final_model, epochs,
input_dep_ls, use_grad_norm)
accuracy, roc_auc_mnist = get_fashionmnist_mnist_ood(model)
_, roc_auc_notmnist = get_fashionmnist_notmnist_ood(model)
val_accuracies.append(val_accuracy)
test_accuracies.append(test_accuracy)
roc_aucs_mnist.append(roc_auc_mnist)
roc_aucs_notmnist.append(roc_auc_notmnist)
# All stats
results[f"lgp{l_gradient_penalty}_ls{length_scale}"] = [
("val acc", np.mean(val_accuracies), np.std(val_accuracies)),
("test acc", np.mean(test_accuracies),
np.std(test_accuracies)),
("M auroc", np.mean(roc_aucs_mnist), np.std(roc_aucs_mnist)),
("NM auroc", np.mean(roc_aucs_notmnist),
np.std(roc_aucs_notmnist)),