def inv_ecdf_vs_pred_entropy(probabilities,
label=None,
color='b',
linestyle='-',
axis=None):
pred_ent = predictive_entropy(probabilities)
ecdf = ECDF(pred_ent)
x_lim = np.log(probabilities.shape[1])
entropy_range = np.linspace(0.0, x_lim, probabilities.shape[1] * 100)
if axis is None:
fig, ax = plt.subplots(figsize=(12, 7), tight_layout=True)
else:
ax = axis
ax.plot(entropy_range,
1 - ecdf(entropy_range),
c=color,
ls=linestyle,
lw=3,
label=label,
clip_on=False)
ax.set_xlim(ax.get_xlim()[0], np.ceil(x_lim))
ax.set_ylim(ax.get_ylim()[0], 1)
ax.tick_params(direction='out', labelsize=14)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.tick_params(direction='out', labelsize=14, right=False, top=False)
ax.set_ylabel('1-ecdf', fontsize=16)
ax.set_xlabel('Predictive Entropy', fontsize=16)
def eval_fgsm_bnn(model,
data,
estimator,
samples=30,
epsilon=0.1,
stats=True,
device=torch.device('cuda'),
verbose=True):
model.eval()
mean_predictions = 0
samples = tqdm.tqdm(range(samples), disable=not verbose)
for _ in samples:
samples.set_postfix({'RAM': ram(), 'VRAM': vram()})
estimator.sample_and_replace()
predictions, labels, _ = eval_fgsm(model, data, epsilon, stats=False, device=device, verbose=False)
mean_predictions += predictions
mean_predictions /= len(samples)
if stats:
acc = accuracy(mean_predictions, labels)
ece1 = 100 * expected_calibration_error(mean_predictions, labels)[0]
ece2 = 100 * calibration_curve(mean_predictions, labels)[0]
nll = negative_log_likelihood(mean_predictions, labels)
ent = predictive_entropy(mean_predictions, mean=True)
stats_dict = {"eps": epsilon, "acc": acc, "ece1": ece1, "ece2": ece2, "nll": nll, "ent": ent}
if verbose:
print(f"Step: {epsilon:.2f} | Adv. Entropy: {stats_dict['ent']:.2f} | Adv. Accuracy: {stats_dict['acc']:.2f}%")
return mean_predictions, labels, stats_dict
def eval_bnn(model,
dataset,
estimator,
samples=30,
stats=False,
device=torch.device('cuda'),
verbose=True):
model.eval()
mean_predictions = 0
stats_list = {"acc": [], "ece": [], "nll": [], "ent": []}
with torch.no_grad():
samples = tqdm.tqdm(range(samples), disable=not verbose)
for sample in samples:
samples.set_postfix({'RAM': ram(), 'VRAM': vram()})
estimator.sample_and_replace()
predictions, labels = eval_nn(model, dataset, device)
mean_predictions += predictions
if stats:
running_mean = mean_predictions / (sample + 1)
stats_list["acc"].append(accuracy(running_mean, labels))
stats_list["ece"].append(100 * expected_calibration_error(running_mean, labels)[0])
stats_list["nll"].append(negative_log_likelihood(predictions, labels))
stats_list["ent"].append(predictive_entropy(running_mean, mean=True))
mean_predictions /= len(samples)
if verbose:
print(f"Accuracy: {accuracy(mean_predictions, labels):.2f}% | ECE: {100 * expected_calibration_error(mean_predictions, labels)[0]:.2f}%")
return mean_predictions, labels, stats_list
def true_false_ecdf(probabilities, labels, path="", axis=None):
true_preds = probabilities[labels == np.argmax(probabilities, axis=1)]
false_preds = probabilities[labels != np.argmax(probabilities, axis=1)]
true_ent = predictive_entropy(true_preds)
false_ent = predictive_entropy(false_preds)
true_ecdf = ECDF(true_ent)
false_ecdf = ECDF(false_ent)
x_lim = np.log(probabilities.shape[1])
entropy_range = np.linspace(0.0, x_lim, probabilities.shape[1] * 100)
if axis is None:
fig, ax = plt.subplots(figsize=(12, 7), tight_layout=True)
else:
ax = axis
ax.plot(entropy_range,
1 - true_ecdf(entropy_range),
color='blueviolet',
linewidth=2,
label="Correct classification")
ax.plot(entropy_range,
1 - false_ecdf(entropy_range),
color='blueviolet',
linestyle='--',
linewidth=2,
label="Misclassification")
ax.tick_params(direction='out', labelsize=14)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.tick_params(direction='out', labelsize=14, right=False, top=False)
ax.set_ylabel('1-ecdf', fontsize=16)
ax.set_xlabel('Predictive Entropy', fontsize=16)
if axis is None:
ax.legend(fontsize=16)
plt.savefig(path if path else 'true_false_ecdf.pdf',
format='pdf',
dpi=1200)
def eval_fgsm(model,
data,
epsilon=0.1,
stats=True,
device=torch.device('cuda'),
verbose=True):
model.eval()
logits_list = torch.Tensor().to(device)
labels_list = torch.LongTensor()
stats_dict = None
data = tqdm.tqdm(data, disable=not verbose or len(data) == 1)
for images, labels in data:
data.set_postfix({'RAM': ram(), 'VRAM': vram()})
adv_images = datasets.fgsm(model, images.to(device, non_blocking=True), labels.to(device, non_blocking=True),
epsilon=epsilon)
with torch.no_grad():
adv_logits = model(adv_images)
logits_list = torch.cat([logits_list, adv_logits])
labels_list = torch.cat([labels_list, labels])
adv_predictions = torch.nn.functional.softmax(logits_list, dim=1).detach().cpu().numpy()
labels = labels_list.numpy()
if stats:
acc = accuracy(adv_predictions, labels)
ece1 = 100 * expected_calibration_error(adv_predictions, labels)[0]
ece2 = 100 * calibration_curve(adv_predictions, labels)[0]
nll = negative_log_likelihood(adv_predictions, labels)
ent = predictive_entropy(adv_predictions, mean=True)
stats_dict = {"eps": epsilon, "acc": acc, "ece1": ece1, "ece2": ece2, "nll": nll, "ent": ent}
if verbose:
print(f"Step: {epsilon:.2f} | Adv. Entropy: {stats_dict['ent']:.2f} | Adv. Accuracy: {stats_dict['acc']:.2f}%")
return adv_predictions, labels, stats_dict
def objective(**params):
norms = [10**params["norm"]] * len(factors)
scales = [10**params["scale"]] * len(factors)
print("Norm:", norms[0], "Scale:", scales[0])
try:
estimator.invert(norms, args.pre_scale * scales)
except (RuntimeError, np.linalg.LinAlgError):
print(f"Error: Singular matrix")
return 200
predictions, labels, _ = eval_bnn(model,
val_loader,
estimator,
args.samples,
stats=False,
device=args.device,
verbose=args.verbose)
err = 100 - accuracy(predictions, labels)
ece = 100 * expected_calibration_error(predictions, labels)[0]
nll = negative_log_likelihood(predictions, labels)
ent = predictive_entropy(predictions, mean=True)
stats["norms"].append(norms)
stats["scales"].append(scales)
stats["acc"].append(100 - err)
stats["ece"].append(ece)
stats["nll"].append(nll)
stats["ent"].append(ent)
stats["cost"].append(err + ece)
print(
f"Err.: {err:.2f}% | ECE: {ece:.2f}% | NLL: {nll:.3f} | Ent.: {ent:.3f}"
)
np.save(
results_path +
f"_hyperopt_stats{'_layer.npy' if args.layer else '.npy'}", stats)
return err + ece
def entropy_hist(inclass,
outclass,
bins=100,
norm=True,
log=False,
kde=False,
jsd=False,
path="",
axis=None,
label=""):
"""Makes a predictive entropy histogram or density plot for in- and out-of-domain data.
Args:
inclass (Numpy array): The predicted probabilities of the in-domain data.
outclass (Numpy array): The predicted probabilities of the out-of-domain data.
bins (int, optional): The number of bins to use for the histogram. Default: 100.
norm (bool, optional): If True, entropy values are normalized between 0 and 1.
log (bool, optional): If True, the x-axis is shown in log-scale. Default: False.
kde (bool, optional): If True, plots a density instead of a histogram. Default: True.
jsd (bool, optional): If True, calculates and prints the symmetric, discretized Kullback-Leibler divergence.
path (string, optional): Where to save the figure. Default: Current directory.
axis (matplotlib.Axis, optional): If provided, plots the figure on this axis.
"""
if axis is None:
fig, ax = plt.subplots(figsize=(9, 9), tight_layout=True)
else:
ax = axis
inclass_entropy = predictive_entropy(inclass)
outclass_entropy = predictive_entropy(outclass)
xlim = np.log(inclass.shape[1])
if norm:
inclass_entropy /= xlim
outclass_entropy /= xlim
xlim = 1
bins = np.linspace(0, xlim, num=bins)
kwargs = dict(hist_kws={'alpha': .5}, kde_kws={'linewidth': 3})
if kde:
ax = distplot(inclass_entropy,
color='dodgerblue',
label=label if label else 'In Class',
bins=bins,
hist=False,
ax=ax,
**kwargs)
ax = distplot(outclass_entropy,
color='crimson',
label='Out of Class',
bins=bins,
hist=False,
ax=ax,
**kwargs)
l1 = ax.lines[0]
l2 = ax.lines[1]
x1 = l1.get_xydata()[:, 0]
y1 = l1.get_xydata()[:, 1]
x2 = l2.get_xydata()[:, 0]
y2 = l2.get_xydata()[:, 1]
ax.fill_between(x1, y1, color="dodgerblue", alpha=0.5)
ax.fill_between(x2, y2, color="crimson", alpha=0.5)
ax.set_ylim(0.0, ax.get_ylim()[1])
ax.set_ylabel('Density', fontsize=18)
else:
kwargs['hist_kws']['histtype'] = 'stepfilled'
kwargs['hist_kws']['edgecolor'] = 'black'
distplot(inclass_entropy,
color='dodgerblue',
label=label if label else 'In Class',
bins=bins,
hist=True,
kde=False,
ax=ax,
**kwargs)
distplot(outclass_entropy,
color='crimson',
label='Out of Class',
bins=bins,
hist=True,
kde=False,
ax=ax,
**kwargs)
ax.set_ylabel('Frequency', fontsize=20)
ax.set_xlim(0, xlim)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.tick_params(direction='out', labelsize=18, right=False, top=False)
ax.legend(fontsize=20, loc='upper right', frameon=False)
ax.set_xlabel('Entropy', fontsize=20)
if log:
ax.set_xscale('log')
if not kde:
ax.set_xlim(np.min(bins), np.max(bins))
if jsd:
jsd = binned_kl_distance(inclass, outclass)
text = offsetbox.AnchoredText(f"JSD: {jsd:.3f}",
loc="upper center",
frameon=True,
prop=dict(fontsize=20))
ax.add_artist(text)
if axis is None:
plt.savefig(path if path else "entropy_hist.pdf",
format='pdf',
dpi=1200)