def log_results(trainer): scheduler.step() if trainer.state.epoch % 5 == 0: evaluator.run(dl_val) accuracy, roc_auc_mnist = get_fashionmnist_mnist_ood(model) accuracy, roc_auc_notmnist = get_fashionmnist_notmnist_ood(model) metrics = evaluator.state.metrics print(f"Validation Results - Epoch: {trainer.state.epoch} " f"Acc: {metrics['accuracy']:.4f} " f"BCE: {metrics['bce']:.2f} " f"GP: {metrics['gradient_penalty']:.6f} " f"AUROC MNIST: {roc_auc_mnist:.2f} " f"AUROC NotMNIST: {roc_auc_notmnist:.2f} ") print(f"Sigma: {model.sigma}")
results = {} for l_gradient_penalty in l_gradient_penalties: for length_scale in length_scales: val_accuracies = [] test_accuracies = [] roc_aucs_mnist = [] roc_aucs_notmnist = [] for _ in range(repetition): print(" ### NEW MODEL ### ") model, val_accuracy, test_accuracy = train_model( l_gradient_penalty, length_scale, final_model, epochs, input_dep_ls, use_grad_norm) accuracy, roc_auc_mnist = get_fashionmnist_mnist_ood(model) _, roc_auc_notmnist = get_fashionmnist_notmnist_ood(model) val_accuracies.append(val_accuracy) test_accuracies.append(test_accuracy) roc_aucs_mnist.append(roc_auc_mnist) roc_aucs_notmnist.append(roc_auc_notmnist) # All stats results[f"lgp{l_gradient_penalty}_ls{length_scale}"] = [ ("val acc", np.mean(val_accuracies), np.std(val_accuracies)), ("test acc", np.mean(test_accuracies), np.std(test_accuracies)), ("M auroc", np.mean(roc_aucs_mnist), np.std(roc_aucs_mnist)), ("NM auroc", np.mean(roc_aucs_notmnist), np.std(roc_aucs_notmnist)),