def adversarial_attack(args, model, inv_factors, results_path, fig_path):
print("Loading data")
if args.data == 'cifar10':
test_loader = datasets.cifar10(args.torch_data, splits='test')
elif args.data == 'gtsrb':
test_loader = datasets.gtsrb(args.data_dir, batch_size=args.batch_size, splits='test')
if args.data == 'mnist':
test_loader = datasets.mnist(args.torch_data, splits='test')
elif args.data == 'tiny':
test_loader = datasets.imagenet(args.data_dir, img_size=64, batch_size=args.batch_size, splits='test',
tiny=True)
elif args.data == 'imagenet':
img_size = 224
if args.model in ['googlenet', 'inception_v3']:
img_size = 299
test_loader = datasets.imagenet(args.data_dir, img_size, args.batch_size, workers=args.workers, splits='test')
if args.epsilon > 0:
print(eval_fgsm(model, test_loader, args.epsilon, args.device)[-1])
else:
stats_dict = {"eps": [], "acc": [], "ece1": [], "ece2": [], "nll": [], "ent": []}
bnn_stats_dict = {"eps": [], "acc": [], "ece1": [], "ece2": [], "nll": [], "ent": []}
steps = np.concatenate([np.linspace(0, 0.2, 11), np.linspace(0.3, 1, 8)])
for step in steps:
stats = eval_fgsm(model, test_loader, step, args.device, verbose=False)[-1]
bnn_stats = eval_fgsm_bnn(model, test_loader, inv_factors, args.estimator, args.samples, step,
device=args.device)[-1]
for (k1, v1), (k2, v2) in zip(stats.items(), bnn_stats.items()):
stats_dict[k1].append(v1)
bnn_stats_dict[k2].append(v2)
np.savez(results_path + "_fgsm.npz", stats=stats_dict, bnn_stats=bnn_stats_dict)
print(tabulate.tabulate(stats_dict, headers="keys"))
print(tabulate.tabulate(bnn_stats_dict, headers="keys"))
plot.adversarial_results(steps, stats_dict, bnn_stats_dict, fig_path)
def main():
args = setup()
print("Loading model")
"""
model_class = getattr(torchvision.models, args.model)
if args.model in ['googlenet', 'inception_v3']:
model = model_class(pretrained=True, aux_logits=False)
else:
model = model_class(pretrained=True)
model.fc = torch.nn.Linear(model.fc.in_features, 43)
"""
model = resnet18(num_classes=43)
model.to(args.device).train()
if args.parallel:
model = torch.nn.parallel.DataParallel(model)
train_loader, val_loader = gtsrb(args.data_dir, batch_size=args.batch_size, workers=args.workers)
if args.optimizer == 'sgd':
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum,
weight_decay=args.l2)
elif args.optimizer == 'adam':
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.l2)
criterion = torch.nn.CrossEntropyLoss()
train(model, train_loader, val_loader, optimizer, criterion, args.epochs, args.lr, args.device)
path = os.path.join(args.root_dir, 'weights', f"{args.model}_{args.data}.pth")
torch.save(model.state_dict(), path)
def test(args, model, fig_path=""):
print("Loading data")
if args.data == 'cifar10':
test_loader = datasets.cifar10(args.torch_data, splits='test')
elif args.data == 'gtsrb':
test_loader = datasets.gtsrb(args.data_dir, batch_size=args.batch_size, splits='test')
if args.data == 'mnist':
test_loader = datasets.mnist(args.torch_data, splits='test')
elif args.data == 'tiny':
test_loader = datasets.imagenet(args.data_dir, img_size=64, batch_size=args.batch_size, splits='test',
tiny=True)
elif args.data == 'imagenet':
img_size = 224
if args.model in ['googlenet', 'inception_v3']:
img_size = 299
test_loader = datasets.imagenet(args.data_dir, img_size, args.batch_size, workers=args.workers, splits='test')
predictions, labels = eval_nn(model, test_loader, args.device, args.verbose)
print("Plotting results")
plot.reliability_diagram(predictions, labels, path=fig_path + "_reliability.pdf")
def main():
args = setup()
print("Preparing directories")
os.makedirs(os.path.join(args.root_dir, "factors"), exist_ok=True)
filename = f"{args.prefix}{args.model}_{args.data}_{args.estimator}{args.suffix}"
factors_path = os.path.join(args.root_dir, "factors", filename)
print("Loading model")
if args.model == 'lenet5':
model = lenet5.lenet5(pretrained=args.data, device=args.device)
elif args.model == 'resnet18' and args.data != 'imagenet':
model = resnet.resnet18(pretrained=os.path.join(
args.root_dir, 'weights', f"{args.model}_{args.data}.pth"),
num_classes=43 if args.data == 'gtsrb' else 10,
device=args.device)
else:
model_class = getattr(torchvision.models, args.model)
if args.model in ['googlenet', 'inception_v3']:
model = model_class(pretrained=True, aux_logits=False)
else:
model = model_class(pretrained=True)
model.to(args.device).train()
if args.parallel:
model = torch.nn.parallel.DataParallel(model)
if args.estimator != 'inf':
print(f"Loading data")
if args.data == 'cifar10':
data = datasets.cifar10(args.torch_data,
args.batch_size,
args.workers,
args.augment,
splits='train')
elif args.data == 'mnist':
data = datasets.mnist(args.torch_data,
args.batch_size,
args.workers,
args.augment,
splits='train')
elif args.data == 'gtsrb':
data = datasets.gtsrb(args.data_dir,
batch_size=args.batch_size,
workers=args.workers,
splits='train')
elif args.data == 'tiny':
img_size = 64
data = datasets.imagenet(args.data_dir,
img_size,
args.batch_size,
splits='train',
tiny=True)
elif args.data == 'imagenet':
img_size = 224
if args.model in ['googlenet', 'inception_v3']:
img_size = 299
data = datasets.imagenet(args.data_dir,
img_size,
args.batch_size,
workers=args.workers,
splits='train')
torch.backends.cudnn.benchmark = True
print("Computing factors")
if args.estimator == 'inf':
est = compute_inf(args)
elif args.estimator == 'efb':
factors = torch.load(factors_path.replace("efb", "kfac") + '.pth')
est = compute_factors(args, model, data, factors)
else:
est = compute_factors(args, model, data)
print("Saving factors")
if args.estimator == "inf":
torch.save(est.state, f"{factors_path}{args.rank}.pth")
elif args.estimator == "efb":
torch.save(list(est.state.values()), factors_path + '.pth')
torch.save(list(est.diags.values()),
factors_path.replace("efb", "diag") + '.pth')
else:
torch.save(list(est.state.values()), factors_path + '.pth')
def out_of_domain(args, model, inv_factors, results_path="", fig_path=""):
"""Evaluates the model on in- and out-of-domain data.
Each dataset has its own out-of-domain dataset which is loaded automatically alongside the in-domain dataset
specified in `args.data`. For each image (batch) in the in- and out-of-domain data a forward pass through the
provided `model` is performed and the predictions are stored under `results_path`. This is repeated for the Bayesian
variant of the model (Laplace approximation).
Parameters
----------
args : Todo: Check type
The arguments provided to the script on execution.
model : torch.nn.Module Todo: Verify
A `torchvision` or custom neural network (a `torch.nn.Module` or `torch.nn.Sequential` instance)
inv_factors : list
A list KFAC factors, Eigenvectors of KFAC factors or diagonal terms. Todo: INF
results_path : string, optional
The path where results (in- and out-of-domain predictions) should be stored. Results are not stored if
argument `args.no_results` is provided.
fig_path : string, optional
The path where figures should be stored. Figures are only generated if argument `args.plot` is provided.
"""
print("Loading data")
if args.data == 'cifar10':
in_data = datasets.cifar10(args.torch_data, splits='test')
out_data = datasets.svhn(args.torch_data, splits='test')
elif args.data == 'mnist':
in_data = datasets.mnist(args.torch_data, splits='test')
out_data = datasets.kmnist(args.torch_data, splits='test')
elif args.data == 'gtsrb':
in_data = datasets.gtsrb(args.data_dir, batch_size=args.batch_size, splits='test')
out_data = datasets.cifar10(args.torch_data, splits='test')
elif args.data == 'tiny':
in_data = datasets.imagenet(args.data_dir, img_size=64, batch_size=args.batch_size, splits='test', tiny=True,
use_cache=True)
out_data = datasets.art(args.data_dir, img_size=64, batch_size=args.batch_size, use_cache=True)
elif args.data == 'imagenet':
img_size = 224
if args.model in ['googlenet', 'inception_v3']:
img_size = 299
in_data = datasets.imagenet(args.data_dir, img_size, args.batch_size, workers=args.workers, splits='test',
use_cache=True)
out_data = datasets.art(args.data_dir, img_size, args.batch_size, workers=args.workers, use_cache=True)
# Compute NN and BNN predictions on validation set of training data
predictions, bnn_predictions, labels, stats = eval_nn_and_bnn(model, in_data, inv_factors, args.estimator,
args.samples, args.stats, args.device, verbose=True)
# Compute NN and BNN predictions on out-of-distribution data
ood_predictions, bnn_ood_predictions, _, _ = eval_nn_and_bnn(model, out_data, inv_factors, args.estimator,
args.samples, False, args.device, verbose=True)
if not args.no_results:
print("Saving results")
np.savez_compressed(results_path,
stats=stats,
labels=labels,
predictions=predictions,
bnn_predictions=bnn_predictions,
ood_predictions=ood_predictions,
bnn_ood_predictions=bnn_ood_predictions)
if args.plot:
print("Plotting results")
fig, ax = plt.subplots(figsize=(12, 7), tight_layout=True)
plot.inv_ecdf_vs_pred_entropy(predictions, color='dodgerblue', linestyle='--', axis=ax)
plot.inv_ecdf_vs_pred_entropy(ood_predictions, color='crimson', linestyle='--', axis=ax)
plot.inv_ecdf_vs_pred_entropy(bnn_predictions, color='dodgerblue', axis=ax)
plot.inv_ecdf_vs_pred_entropy(bnn_ood_predictions, color='crimson', axis=ax)
ax.legend([f"NN {args.data.upper()} | Acc.: {accuracy(predictions, labels):.2f}%",
f"NN OOD",
f"BNN {args.data.upper()} | Acc.: {accuracy(bnn_predictions, labels):.2f}%",
f"BNN OOD"], fontsize=16, frameon=False)
plt.savefig(fig_path + "_ecdf.pdf", forma='pdf', dpi=1200)
plot.reliability_diagram(predictions, labels, path=fig_path + "_reliability.pdf")
plot.reliability_diagram(bnn_predictions, labels, path=fig_path + "_bnn_reliability.pdf")
plot.entropy_hist(predictions, ood_predictions, path=fig_path + "_entropy.pdf")
plot.entropy_hist(bnn_predictions, bnn_ood_predictions, path=fig_path + "_bnn_entropy.pdf")
def main():
args = setup()
print("Preparing directories")
filename = f"{args.prefix}{args.model}_{args.data}_{args.estimator}{args.suffix}"
factors_path = os.path.join(args.root_dir, "factors", filename)
weights_path = os.path.join(args.root_dir, "weights",
f"{args.model}_{args.data}.pth")
if args.exp_id == -1:
if not args.no_results:
os.makedirs(os.path.join(args.results_dir, args.model, "data",
args.estimator, args.optimizer),
exist_ok=True)
if args.plot:
os.makedirs(os.path.join(args.results_dir, args.model, "figures",
args.estimator, args.optimizer),
exist_ok=True)
results_path = os.path.join(args.results_dir, args.model, "data",
args.estimator, args.optimizer, filename)
else:
if not args.no_results:
os.makedirs(os.path.join(args.results_dir, args.model, "data",
args.estimator, args.optimizer,
args.exp_id),
exist_ok=True)
if args.plot:
os.makedirs(os.path.join(args.results_dir, args.model, "figures",
args.estimator, args.optimizer,
args.exp_id),
exist_ok=True)
results_path = os.path.join(args.results_dir, args.model, "data",
args.estimator, args.optimizer,
args.exp_id, filename)
print("Loading model")
if args.model == 'lenet5':
model = lenet5(pretrained=args.data, device=args.device)
elif args.model == 'resnet18' and args.data != 'imagenet':
model = resnet18(pretrained=weights_path,
num_classes=43 if args.data == 'gtsrb' else 10,
device=args.device)
else:
model_class = getattr(torchvision.models, args.model)
if args.model in ['googlenet', 'inception_v3']:
model = model_class(pretrained=True, aux_logits=False)
else:
model = model_class(pretrained=True)
model.to(args.device).eval()
if args.parallel:
model = torch.nn.parallel.DataParallel(model)
print("Loading data")
if args.data == 'mnist':
val_loader = datasets.mnist(args.torch_data, splits='val')
elif args.data == 'cifar10':
val_loader = datasets.cifar10(args.torch_data, splits='val')
elif args.data == 'gtsrb':
val_loader = datasets.gtsrb(args.data_dir,
batch_size=args.batch_size,
splits='val')
elif args.data == 'imagenet':
img_size = 224
if args.model in ['googlenet', 'inception_v3']:
img_size = 299
val_loader = datasets.imagenet(args.data_dir,
img_size,
args.batch_size,
args.workers,
splits='val',
use_cache=True,
pre_cache=True)
print("Loading factors")
if args.estimator in ["diag", "kfac"]:
factors = torch.load(factors_path + '.pth')
elif args.estimator == 'efb':
kfac_factors = torch.load(factors_path.replace("efb", "kfac") + '.pth')
lambdas = torch.load(factors_path + '.pth')
factors = list()
eigvecs = get_eigenvectors(kfac_factors)
for eigvec, lambda_ in zip(eigvecs, lambdas):
factors.append((eigvec[0], eigvec[1], lambda_))
elif args.estimator == 'inf':
factors = torch.load(f"{factors_path}{args.rank}.pth")
torch.backends.cudnn.benchmark = True
norm_min = -10
norm_max = 10
scale_min = -10
scale_max = 10
if args.boundaries:
x0 = list()
boundaries = [[norm_min, scale_min], [norm_max, scale_max],
[norm_min, scale_max], [norm_max, scale_min],
[norm_min / 2., scale_min], [norm_max / 2., scale_max],
[norm_min, scale_max / 2.], [norm_max, scale_min / 2.],
[norm_min / 2., scale_min / 2.],
[norm_max / 2., scale_max / 2.],
[norm_min / 2., scale_max / 2.],
[norm_max / 2., scale_min / 2.]]
for b in boundaries:
tmp = list()
for _ in range(3 if args.layer else 1):
tmp.extend(b)
x0.append(tmp)
else:
x0 = None
f_norms = np.array([factor.norm().cpu().numpy() for factor in factors])
space = list()
for i in range(3 if args.layer else 1):
space.append(
skopt.space.Real(norm_min,
norm_max,
name=f"norm{i}",
prior='uniform'))
space.append(
skopt.space.Real(scale_min,
scale_max,
name=f"scale{i}",
prior='uniform'))
stats = {
"norms": [],
"scales": [],
"acc": [],
"ece": [],
"nll": [],
"ent": [],
"cost": []
}
@skopt.utils.use_named_args(dimensions=space)
def objective(**params):
norms = list()
scales = list()
for f in f_norms:
if args.layer:
# Closest to max
if abs(f_norms.max() - f) < abs(f_norms.min() - f) and abs(
f_norms.max() - f) < abs(f_norms.mean() - f):
norms.append(10**params['norm0'])
scales.append(10**params['scale0'])
# Closest to min
elif abs(f_norms.min() - f) < abs(f_norms.max() - f) and abs(
f_norms.min() - f) < abs(f_norms.mean() - f):
norms.append(10**params['norm1'])
scales.append(10**params['scale1'])
# Closest to mean
else:
norms.append(10**params['norm2'])
scales.append(10**params['scale2'])
else:
norms.append(10**params['norm0'])
scales.append(10**params['scale0'])
if args.layer:
print(
tabulate.tabulate(
{
'Layer': np.arange(len(factors)),
'F-Norm:': f_norms,
'Norms': norms,
'Scales': scales
},
headers='keys',
numalign='right'))
else:
print("Norm:", norms[0], "Scale:", scales[0])
try:
inv_factors = invert_factors(factors, norms,
args.pre_scale * scales,
args.estimator)
except (RuntimeError, np.linalg.LinAlgError):
print(f"Error: Singular matrix")
return 200
predictions, labels, _ = eval_bnn(model,
val_loader,
inv_factors,
args.estimator,
args.samples,
stats=False,
device=args.device,
verbose=False)
err = 100 - accuracy(predictions, labels)
ece = 100 * expected_calibration_error(predictions, labels)[0]
nll = negative_log_likelihood(predictions, labels)
ent = predictive_entropy(predictions, mean=True)
stats["norms"].append(norms)
stats["scales"].append(scales)
stats["acc"].append(100 - err)
stats["ece"].append(ece)
stats["nll"].append(nll)
stats["ent"].append(ent)
stats["cost"].append(err + ece)
print(
f"Err.: {err:.2f}% | ECE: {ece:.2f}% | NLL: {nll:.3f} | Ent.: {ent:.3f}"
)
return err + ece
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
if args.optimizer == "gbrt":
res = skopt.gbrt_minimize(func=objective,
dimensions=space,
n_calls=args.calls,
x0=x0,
verbose=True,
n_jobs=args.workers,
n_random_starts=0 if x0 else 10,
acq_func='EI')
# EI (neg. expected improvement)
# LCB (lower confidence bound)
# PI (neg. prob. of improvement): Usually favours exploitation over exploration
# gp_hedge (choose probabilistically between all)
if args.optimizer == "gp":
res = skopt.gp_minimize(func=objective,
dimensions=space,
n_calls=args.calls,
x0=x0,
verbose=True,
n_jobs=args.workers,
n_random_starts=0 if x0 else 1,
acq_func='gp_hedge')
# acq_func: EI (neg. expected improvement), LCB (lower confidence bound), PI (neg. prob. of improvement)
# xi: how much improvement one wants over the previous best values.
# kappa: Importance of variance of predicted values. High: exploration > exploitation
# base_estimator: RF (random forest), ET (extra trees)
elif args.optimizer == "forest":
res = skopt.forest_minimize(func=objective,
dimensions=space,
n_calls=args.calls,
x0=x0,
verbose=True,
n_jobs=args.workers,
n_random_starts=0 if x0 else 1,
acq_func='EI')
elif args.optimizer == "random":
res = skopt.dummy_minimize(func=objective,
dimensions=space,
n_calls=args.calls,
x0=x0,
verbose=True)
elif args.optimizer == "grid":
space = [
np.arange(norm_min, norm_max + 1, 10),
np.arange(scale_min, scale_max + 1, 10)
]
res = grid(func=objective, dimensions=space)
print(f"Minimal cost of {min(stats['cost'])} found at:")
if args.layer:
print(
tabulate.tabulate(
{
'Layer': np.arange(len(factors)),
'F-Norm:': f_norms,
'Norms': stats['norms'][np.argmin(stats['cost'])],
'Scales': stats['scales'][np.argmin(stats['cost'])]
},
headers='keys',
numalign='right'))
else:
print("Norm:", stats['norms'][np.argmin(stats['cost'])][0],
"Scale:", stats['scales'][np.argmin(stats['cost'])][0])
if not args.no_results:
print("Saving results")
del res.specs['args']['func']
np.save(
results_path +
f"_best_params{'_layer.npy' if args.layer else '.npy'}", [
stats['norms'][np.argmin(stats['cost'])],
stats['scales'][np.argmin(stats['cost'])]
])
np.save(
results_path +
f"_hyperopt_stats{'_layer.npy' if args.layer else '.npy'}", stats)
skopt.dump(
res, results_path +
f"_hyperopt_dump{'_layer.pkl' if args.layer else '.pkl'}")
if args.plot:
print("Plotting results")
hyperparameters(args)