def get_ROC_mnist_ensemble(ensemble):
_, _, _, fashionmnist_test_dataset = get_FashionMNIST()
_, _, _, mnist_test_dataset = get_MNIST()
dataloader, anomaly_targets = prepare_ood_datasets(fashionmnist_test_dataset, mnist_test_dataset)
scores = []
accuracies = []
with torch.no_grad():
for data, target in dataloader:
data = data.cuda()
target = target.cuda()
output_l=[]
for i,model in enumerate(ensemble):
model.eval()
output = model(data)
output_l.append(output)
output_l = torch.stack(output_l)
kernel_distance=output_l.exp().mean(0)
pred=kernel_distance.argmax(1)
kernel_distance=-(kernel_distance*torch.log(kernel_distance)).sum(1)
accuracy = pred.eq(target)
accuracies.append(accuracy.cpu().numpy())
scores.append(kernel_distance.cpu().numpy())
scores = np.concatenate(scores)
accuracies = np.concatenate(accuracies)
accuracy = np.mean(accuracies[: len(fashionmnist_test_dataset)])
roc = roc_curve(anomaly_targets, scores)
return roc
def get_ROC_mnist(model):
_, _, _, fashionmnist_test_dataset = get_FashionMNIST()
_, _, _, mnist_test_dataset = get_MNIST()
dataloader, anomaly_targets = prepare_ood_datasets(
fashionmnist_test_dataset, mnist_test_dataset)
scores, accuracies = loop_over_dataloader(model, dataloader)
accuracy = np.mean(accuracies[:len(fashionmnist_test_dataset)])
roc = roc_curve(anomaly_targets, scores)
return roc
def get_fashionmnist_mnist_ood(model):
_, _, _, fashionmnist_test_dataset = get_FashionMNIST()
_, _, _, mnist_test_dataset = get_MNIST()
return get_auroc_ood(fashionmnist_test_dataset, mnist_test_dataset, model)