py_out = predict_dkl(test_loader_adv_nonasymp, model_dkl,
likelihood).cpu().numpy()
conf_farway = get_confidence(py_out)
mmc = conf_farway.mean()
auroc = get_auroc(py_in, py_out)
save_res_ood(tab_ood['CIFAR10 - Adversarial'], mmc, auroc)
print(f'[Out-Adversarial, DKL] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
# Out-distribution - FarAwayAdversarial
py_out = predict_dkl(test_loader_adv_asymp, model_dkl,
likelihood).cpu().numpy()
conf_farway = get_confidence(py_out)
mmc = conf_farway.mean()
auroc = get_auroc(py_in, py_out)
save_res_ood(tab_ood['CIFAR10 - FarAwayAdv'], mmc, auroc)
print(f'[Out-FarAwayAdv, DKL] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
print()
print()
if args.generate_histograms:
plot_histogram([conf_in, conf_emnist, conf_farway],
['In - CIFAR10', 'Out - SVHN', 'Out - FarAway'],
'hist_confs_cifar10_dkl')
with open(f'results/tab_ood_cifar10_dkl_{args.randseed}.pkl', 'wb') as f:
pickle.dump(tab_ood, f)
with open(f'results/tab_cal_cifar10_dkl_{args.randseed}.pkl', 'wb') as f:
pickle.dump(tab_cal, f)
save_res_ood(tab_ood['CIFAR10 - Adversarial'], mmc, auroc)
print(f'[Out-Adversarial, MAP] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
# Out-distribution - FarAwayAdversarial
py_out = predict(test_loader_adv_asymp, model).cpu().numpy()
conf_farway = get_confidence(py_out)
mmc = conf_farway.mean()
auroc = get_auroc(py_in, py_out)
save_res_ood(tab_ood['CIFAR10 - FarAwayAdv'], mmc, auroc)
print(f'[Out-FarAwayAdv, MAP] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
print()
if args.generate_histograms:
plot_histogram([conf_in, conf_svhn, conf_farway],
['In - CIFAR10', 'Out - SVHN', 'Out - FarAway'],
'hist_confs_cifar10_map')
"""
=============================================================================================
Temperature scaling
=============================================================================================
"""
model = load_model()
X = predict_logit(val_loader, model).cpu().numpy()
y = torch.cat([y for x, y in val_loader], dim=0).numpy()
T, time = timing(lambda: TemperatureScaling().fit(X, y).T)
print(f'T = {T:.3f}. Done in {time:.1f}s')
# In-distribution
save_res_ood(tab_ood['SVHN - LSUN'], mmc, auroc)
print(f'[Out-LSUN, MAP] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
# Out-distribution - FarAway
py_out = predict_binary(noise_loader, model, delta=deltas).cpu().numpy()
conf_farway = get_confidence(py_out, binary=True)
mmc = conf_farway.mean()
auroc = get_auroc_binary(py_in, py_out)
save_res_ood(tab_ood['SVHN - FarAway'], mmc, auroc)
print(f'[Out-FarAway, MAP] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
print()
if args.generate_histograms:
plot_histogram([conf_in, conf_cifar10, conf_farway],
['In - SVHN', 'Out - CIFAR10', 'Out - FarAway'],
'hist_confs_svhn_binary_map')
"""
=============================================================================================
Temperature scaling
=============================================================================================
"""
model = load_model()
logits = predict_logit(val_loader, model).squeeze().cpu().numpy()
T, time = timing(lambda: temp_scaling_binary(logits, targets_val))
print(f'T = {T:.3f}. Done in {time:.1f}s')
# In-distribution
py_in, time_pred = timing(
py_out = predict_dkl(test_loader_adv_nonasymp, model_dkl,
likelihood).cpu().numpy()
conf_farway = get_confidence(py_out)
mmc = conf_farway.mean()
auroc = get_auroc(py_in, py_out)
save_res_ood(tab_ood['MNIST - Adversarial'], mmc, auroc)
print(f'[Out-Adversarial, DKL] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
# Out-distribution - FarAwayAdversarial
py_out = predict_dkl(test_loader_adv_asymp, model_dkl,
likelihood).cpu().numpy()
conf_farway = get_confidence(py_out)
mmc = conf_farway.mean()
auroc = get_auroc(py_in, py_out)
save_res_ood(tab_ood['MNIST - FarAwayAdv'], mmc, auroc)
print(f'[Out-FarAwayAdv, DKL] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
print()
print()
if args.generate_histograms:
plot_histogram([conf_in, conf_emnist, conf_farway],
['In - MNIST', 'Out - EMNIST', 'Out - FarAway'],
'hist_confs_mnist_dkl')
with open(f'results/tab_ood_mnist_dkl_{args.randseed}.pkl', 'wb') as f:
pickle.dump(tab_ood, f)
with open(f'results/tab_cal_mnist_dkl_{args.randseed}.pkl', 'wb') as f:
pickle.dump(tab_cal, f)
save_res_ood(tab_ood['MNIST - Adversarial'], mmc, auroc)
print(f'[Out-Adversarial, MAP] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
# Out-distribution - FarAwayAdversarial
py_out = predict(test_loader_adv_asymp, model).cpu().numpy()
conf_farway = get_confidence(py_out)
mmc = conf_farway.mean()
auroc = get_auroc(py_in, py_out)
save_res_ood(tab_ood['MNIST - FarAwayAdv'], mmc, auroc)
print(f'[Out-FarAwayAdv, MAP] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
print()
if args.generate_histograms:
plot_histogram([conf_in, conf_emnist, conf_farway],
['In - MNIST', 'Out - EMNIST', 'Out - FarAway'],
'hist_confs_mnist_map')
"""
=============================================================================================
Temperature scaling
=============================================================================================
"""
model = load_model()
X = predict_logit(val_loader, model).cpu().numpy()
y = torch.cat([y for x, y in val_loader], dim=0).numpy()
T, time = timing(lambda: TemperatureScaling().fit(X, y).T)
print(f'T = {T:.3f}. Done in {time:.1f}s')
# In-distribution
save_res_ood(tab_ood['MNIST - FMNIST'], mmc, auroc)
print(f'[Out-FMNIST, MAP] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
# Out-distribution - FarAway
py_out = predict_binary(noise_loader, model, delta=deltas).cpu().numpy()
conf_farway = get_confidence(py_out, binary=True)
mmc = conf_farway.mean()
auroc = get_auroc_binary(py_in, py_out)
save_res_ood(tab_ood['MNIST - FarAway'], mmc, auroc)
print(f'[Out-FarAway, MAP] MMC: {mmc:.3f}; AUROC: {auroc:.3f}')
print()
if args.generate_histograms:
plot_histogram([conf_in, conf_fmnist, conf_farway],
['In - MNIST', 'Out - FMNIST', 'Out - FarAway'],
'hist_confs_mnist_binary_map')
"""
=============================================================================================
Temperature scaling
=============================================================================================
"""
model = load_model()
logits = predict_logit(val_loader, model).squeeze().cpu().numpy()
T, time = timing(lambda: temp_scaling_binary(logits, targets_val))
print(f'T = {T:.3f}. Done in {time:.1f}s')
# In-distribution
py_in, time_pred = timing(