def eval_fgsm_bnn(model,
data,
estimator,
samples=30,
epsilon=0.1,
stats=True,
device=torch.device('cuda'),
verbose=True):
model.eval()
mean_predictions = 0
samples = tqdm.tqdm(range(samples), disable=not verbose)
for _ in samples:
samples.set_postfix({'RAM': ram(), 'VRAM': vram()})
estimator.sample_and_replace()
predictions, labels, _ = eval_fgsm(model, data, epsilon, stats=False, device=device, verbose=False)
mean_predictions += predictions
mean_predictions /= len(samples)
if stats:
acc = accuracy(mean_predictions, labels)
ece1 = 100 * expected_calibration_error(mean_predictions, labels)[0]
ece2 = 100 * calibration_curve(mean_predictions, labels)[0]
nll = negative_log_likelihood(mean_predictions, labels)
ent = predictive_entropy(mean_predictions, mean=True)
stats_dict = {"eps": epsilon, "acc": acc, "ece1": ece1, "ece2": ece2, "nll": nll, "ent": ent}
if verbose:
print(f"Step: {epsilon:.2f} | Adv. Entropy: {stats_dict['ent']:.2f} | Adv. Accuracy: {stats_dict['acc']:.2f}%")
return mean_predictions, labels, stats_dict
def eval_bnn(model,
dataset,
estimator,
samples=30,
stats=False,
device=torch.device('cuda'),
verbose=True):
model.eval()
mean_predictions = 0
stats_list = {"acc": [], "ece": [], "nll": [], "ent": []}
with torch.no_grad():
samples = tqdm.tqdm(range(samples), disable=not verbose)
for sample in samples:
samples.set_postfix({'RAM': ram(), 'VRAM': vram()})
estimator.sample_and_replace()
predictions, labels = eval_nn(model, dataset, device)
mean_predictions += predictions
if stats:
running_mean = mean_predictions / (sample + 1)
stats_list["acc"].append(accuracy(running_mean, labels))
stats_list["ece"].append(100 * expected_calibration_error(running_mean, labels)[0])
stats_list["nll"].append(negative_log_likelihood(predictions, labels))
stats_list["ent"].append(predictive_entropy(running_mean, mean=True))
mean_predictions /= len(samples)
if verbose:
print(f"Accuracy: {accuracy(mean_predictions, labels):.2f}% | ECE: {100 * expected_calibration_error(mean_predictions, labels)[0]:.2f}%")
return mean_predictions, labels, stats_list
def eval_fgsm(model,
data,
epsilon=0.1,
stats=True,
device=torch.device('cuda'),
verbose=True):
model.eval()
logits_list = torch.Tensor().to(device)
labels_list = torch.LongTensor()
stats_dict = None
data = tqdm.tqdm(data, disable=not verbose or len(data) == 1)
for images, labels in data:
data.set_postfix({'RAM': ram(), 'VRAM': vram()})
adv_images = datasets.fgsm(model, images.to(device, non_blocking=True), labels.to(device, non_blocking=True),
epsilon=epsilon)
with torch.no_grad():
adv_logits = model(adv_images)
logits_list = torch.cat([logits_list, adv_logits])
labels_list = torch.cat([labels_list, labels])
adv_predictions = torch.nn.functional.softmax(logits_list, dim=1).detach().cpu().numpy()
labels = labels_list.numpy()
if stats:
acc = accuracy(adv_predictions, labels)
ece1 = 100 * expected_calibration_error(adv_predictions, labels)[0]
ece2 = 100 * calibration_curve(adv_predictions, labels)[0]
nll = negative_log_likelihood(adv_predictions, labels)
ent = predictive_entropy(adv_predictions, mean=True)
stats_dict = {"eps": epsilon, "acc": acc, "ece1": ece1, "ece2": ece2, "nll": nll, "ent": ent}
if verbose:
print(f"Step: {epsilon:.2f} | Adv. Entropy: {stats_dict['ent']:.2f} | Adv. Accuracy: {stats_dict['acc']:.2f}%")
return adv_predictions, labels, stats_dict
def entropy_histogram(args):
if args.estimator in ['swa', 'swag']:
labels, _, predictions = load_data(args, args.model, args.data,
args.estimator)
_, _, ood_predictions = load_data(args,
args.model,
args.data,
args.estimator,
ood=True)
elif args.estimator == 'sgd':
labels, predictions, _ = load_data(args, args.model, args.data, 'kfac')
_, ood_predictions, _ = load_data(args,
args.model,
args.data,
'kfac',
ood=True)
else:
labels, _, predictions = load_data(args, args.model, args.data,
args.estimator)
_, _, ood_predictions = load_data(args,
args.model,
args.data,
args.estimator,
ood=True)
acc = accuracy(predictions, labels)
ece = expected_calibration_error(predictions, labels)[0]
label = f"In Class | Acc.: {acc:.2f}% | ECE: {100 * ece:.2f}%"
fig_path = os.path.join(args.results_dir, args.model, "figures",
args.estimator)
os.makedirs(fig_path, exist_ok=True)
filename = f"{args.prefix}{args.model}_{args.data}{args.suffix}"
plot.entropy_hist(predictions,
ood_predictions,
path=os.path.join(fig_path, f"{filename}_entropy.pdf"),
label=label)
def objective(**params):
norms = [10**params["norm"]] * len(factors)
scales = [10**params["scale"]] * len(factors)
print("Norm:", norms[0], "Scale:", scales[0])
try:
estimator.invert(norms, args.pre_scale * scales)
except (RuntimeError, np.linalg.LinAlgError):
print(f"Error: Singular matrix")
return 200
predictions, labels, _ = eval_bnn(model,
val_loader,
estimator,
args.samples,
stats=False,
device=args.device,
verbose=args.verbose)
err = 100 - accuracy(predictions, labels)
ece = 100 * expected_calibration_error(predictions, labels)[0]
nll = negative_log_likelihood(predictions, labels)
ent = predictive_entropy(predictions, mean=True)
stats["norms"].append(norms)
stats["scales"].append(scales)
stats["acc"].append(100 - err)
stats["ece"].append(ece)
stats["nll"].append(nll)
stats["ent"].append(ent)
stats["cost"].append(err + ece)
print(
f"Err.: {err:.2f}% | ECE: {ece:.2f}% | NLL: {nll:.3f} | Ent.: {ent:.3f}"
)
np.save(
results_path +
f"_hyperopt_stats{'_layer.npy' if args.layer else '.npy'}", stats)
return err + ece
def reliability_diagram(probabilities, labels, path="", bins=10, axis=None):
ece, bin_aces, bin_accs, bin_confs = expected_calibration_error(
probabilities, labels, bins=bins)
if axis is None:
text = offsetbox.AnchoredText(
f"ECE: {(ece * 100):.2f}%\nAccuracy: {accuracy(probabilities, labels):.2f}%\nConfidence: {100 * confidence(probabilities):.2f}%",
loc="upper left",
frameon=False,
prop=dict(fontsize=12))
fig, ax = plt.subplots(figsize=(9, 9), tight_layout=True)
ax.add_artist(text)
else:
ax = axis
ax.bar(x=np.arange(0, 1, 0.1),
height=bin_accs,
width=0.1,
linewidth=1,
edgecolor='black',
align='edge',
color='dodgerblue')
ax.bar(x=np.arange(0, 1, 0.1),
height=bin_aces,
bottom=bin_accs,
width=0.1,
linewidth=1,
edgecolor='crimson',
align='edge',
color='crimson',
fill=False,
hatch='/')
ax.bar(x=np.arange(0, 1, 0.1),
height=bin_aces,
bottom=bin_accs,
width=0.1,
linewidth=1,
edgecolor='crimson',
align='edge',
color='crimson',
alpha=0.3)
if axis is None:
ax.set_ylim(0, 1)
ax.set_xlim(0, 1)
ax.plot(ax.get_xlim(),
ax.get_ylim(),
color='black',
linestyle='dashed',
linewidth=1.0)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.tick_params(direction='out', labelsize=12, right=False, top=False)
ax.set_ylabel('Accuracy', fontsize=14)
ax.set_xlabel('Confidence', fontsize=14)
plt.savefig(path if path else 'reliability_diagram.pdf',
format='pdf',
dpi=1200)
else:
ax.tick_params(right=False,
left=False,
top=False,
bottom=False,
labelright=False,
labelleft=False,
labeltop=False,
labelbottom=False)
ax.set_frame_on(False)