def get_cifar10_svhn_ood_ensemble(ensemble):
_, _, _, cifar_test_dataset = get_CIFAR10()
_, _, _, svhn_test_dataset = get_SVHN()
dataloader, anomaly_targets = prepare_ood_datasets(cifar_test_dataset, svhn_test_dataset)
scores = []
accuracies = []
with torch.no_grad():
for data, target in dataloader:
data = data.cuda()
target = target.cuda()
output_l=[]
for i,model in enumerate(ensemble):
model.eval()
output = model(data)
output_l.append(output)
output_l = torch.stack(output_l)
kernel_distance=output_l.exp().mean(0)
pred=kernel_distance.argmax(1)
kernel_distance=-(kernel_distance*torch.log(kernel_distance)).sum(1)
accuracy = pred.eq(target)
accuracies.append(accuracy.cpu().numpy())
scores.append(kernel_distance.cpu().numpy())
scores = np.concatenate(scores)
accuracies = np.concatenate(accuracies)
accuracy = np.mean(accuracies[: len(cifar_test_dataset)])
auroc = roc_auc_score(anomaly_targets, scores)
return accuracy, auroc
def get_cifar_svhn_ood(model):
_, _, _, cifar_test_dataset = get_CIFAR10()
_, _, _, svhn_test_dataset = get_SVHN()
return get_auroc_ood(cifar_test_dataset, svhn_test_dataset, model)
log_dir = os.path.join(logs_root_dir, model_name)
start_epoch, optim_state_dict = rh.load_model_checkpoint(
model, model_dir, device, hps)
model = Cifar10Wrapper(model).to(device)
msda_config = rh.create_msda_config(hps)
#load dataset
od_bs = int(hps.od_bs_factor * hps.bs)
id_config = {}
if hps.dataset == 'cifar10':
train_loader = dl.get_CIFAR10(train=True,
batch_size=hps.bs,
augm_type=hps.augm,
size=img_size,
config_dict=id_config)
elif hps.dataset == 'semi-cifar10':
train_loader = dl.get_CIFAR10_ti_500k(train=True,
batch_size=hps.bs,
augm_type=hps.augm,
fraction=0.7,
size=img_size,
config_dict=id_config)
else:
raise ValueError(f'Dataset {hps.datset} not supported')
if hps.train_type.lower() in [
'ceda', 'acet', 'advacet', 'tradesacet', 'tradesceda'
]:
folder,
checkpoint,
temperature,
device,
load_temp=temp,
dataset=dataset)
model.to(device)
if len(hps.gpu) > 1:
model = nn.DataParallel(model, device_ids=device_ids)
model.eval()
print(f'\n\n{folder} {checkpoint}\n ')
if dataset == 'cifar10':
dataloader = dl.get_CIFAR10(False, batch_size=bs, augm_type='none')
elif dataset == 'cifar100':
dataloader = dl.get_CIFAR100(False, batch_size=bs, augm_type='none')
else:
raise NotImplementedError()
acc = 0.0
with torch.no_grad():
for data, target in dataloader:
data = data.to(device)
target = target.to(device)
out = model(data)
_, pred = torch.max(out, dim=1)
acc += torch.sum(pred == target).item() / len(dataloader.dataset)
print(f'Clean accuracy {acc}')
device = torch.device('cuda:' + str(min(device_ids)))
bs = bs * len(device_ids)
big_model_bs = big_model_bs * len(device_ids)
model_descriptions = [
('WideResNet34x10', 'cifar10_pgd', 'best_avg', None, False),
('WideResNet34x10', 'cifar10_apgd', 'best_avg', None, False),
('WideResNet34x10', 'cifar10_500k_pgd', 'best_avg', None, False),
('WideResNet34x10', 'cifar10_500k_apgd', 'best_avg', None, False),
('WideResNet34x10', 'cifar10_500k_apgd_asam', 'best_avg', None, False),
]
model_batchsize = bs * np.ones(len(model_descriptions), dtype=np.int)
num_examples = 16
dataloader = dl.get_CIFAR10(False, bs, augm_type='none')
num_datapoints = len(dataloader.dataset)
class_labels = dl.cifar.get_CIFAR10_labels()
eval_dir = 'Cifar10Eval/'
norm = 'l2'
if norm == 'l1':
radii = np.linspace(15, 90, 6)
visual_counterfactuals(model_descriptions, radii, dataloader, model_batchsize, num_examples, class_labels, device,
eval_dir, 'cifar10', norm='l1', stepsize=5, device_ids=device_ids)
else:
radii = np.linspace(0.5, 3, 6)
visual_counterfactuals(model_descriptions, radii, dataloader, model_batchsize, num_examples, class_labels, device, eval_dir, 'cifar10', device_ids=device_ids)