def std_over_epsilon(bnn_df: pandas.DataFrame, config: Configuration):
corr_y = []
wro_y = []
eps_y = []
epsilons = config.epsilons
for epsilon in epsilons:
corr_eps = bnn_df.loc[
(bnn_df["epsilon"] == epsilon) & (bnn_df["y"] == bnn_df["y_"])
] # type: pandas.DataFrame
wro_eps = bnn_df.loc[
(bnn_df["epsilon"] == epsilon) & (bnn_df["y"] != bnn_df["y_"])
] # type: pandas.DataFrame
eps = bnn_df.loc[bnn_df["epsilon"] == epsilon] # type: pandas.DataFrame
corr_y.append(corr_eps["std"].mean())
wro_y.append(wro_eps["std"].mean())
eps_y.append(eps["std"].mean())
config.correct_std = float(np.mean(corr_y))
config.wrong_std = float(np.mean(wro_y))
plt.plot(epsilons, corr_y, label="Average correct STD")
plt.plot(epsilons, wro_y, label="Average wrong STD")
plt.plot(epsilons, eps_y, label="Average STD")
plt.xlabel("Epsilon")
plt.ylabel("Standard Deviation")
plt.legend()
plot_path = Path(f"data/{config.id:02}_std.svg")
plt.savefig(plot_path)
plt.show()
plt.close()
def visualize():
config = Configuration()
bnn_df = load_dict(f"{config.id:02}_bnn_result.csv")
nn_df = load_dict(f"{config.id:02}_nn_result.csv")
accuracy_over_epsilon(bnn_df, nn_df, config)
std_over_epsilon(bnn_df, config)
accuracy_over_epsilon_with_rejection(nn_df, bnn_df, config)
plot_images(config)
config.save()
def accuracy_over_epsilon_with_rejection(
nn_df: pandas.DataFrame, bnn_df: pandas.DataFrame, config: Configuration
):
bnn_y = []
num_classified = []
nn_y = []
std_df = bnn_df.loc[(bnn_df["epsilon"] == 0.0) & (bnn_df["y"] == bnn_df["y_"])][
"std"
] # type: pandas.Series
threshold = float(std_df.mean() + std_df.std())
config.threshold_std = threshold
epsilons = config.epsilons
for epsilon in epsilons:
eps_df = bnn_df.loc[bnn_df["epsilon"] == epsilon]
classified_df = eps_df.loc[eps_df["std"] < threshold]
cor_df = classified_df.loc[classified_df["y"] == classified_df["y_"]]
bnn_y.append(cor_df.shape[0] / classified_df.shape[0])
num_classified.append(classified_df.shape[0] / eps_df.shape[0])
eps_df = nn_df.loc[nn_df["epsilon"] == epsilon]
cor_df = eps_df.loc[eps_df["y"] == eps_df["y_"]]
nn_y.append(cor_df.shape[0] / eps_df.shape[0])
plt.plot(epsilons, bnn_y, label="BNN")
plt.plot(epsilons, nn_y, label="NN")
plt.plot(epsilons, num_classified, label="Percentage of classified examples")
plt.xlabel("Epsilon")
plt.ylabel("Accuracy")
plt.legend()
plot_path = Path(f"data/{config.id:02}_accuracy_with_rejection.svg")
plt.savefig(plot_path)
plt.show()
plt.close()
def __init__(self, device: Optional[str] = "cpu"):
super(Network, self).__init__()
self.fc1 = nn.Linear(28 * 28, 1024)
self.out = nn.Linear(1024, 10)
if torch.cuda.is_available():
device = "cuda:0"
self.to(device)
self.device = device
self.config = Configuration()
def training():
config = Configuration()
net = Network()
optimizer = optim.Adam(net.parameters(), lr=0.01)
train_loader = get_train_loader()
test_loader = get_test_loader()
for epoch in range(config.nn_training_epochs):
train(net, optimizer, train_loader, epoch)
test(net, test_loader)
net.save_model()
def main():
config = Configuration()
image_id = 1362
bnn = BNNWrapper()
bnn.load_model()
loss_fn = pyro.infer.Trace_ELBO(
num_particles=config.bnn_adversary_samples
).differentiable_loss
nn = Network()
nn.load_model()
test_loader = get_test_loader(1, shuffle=False)
x, y = test_loader.dataset[image_id]
y = torch.tensor([y])
bnn_d, bnn_imgs, bnn_pertubation_imgs = bnn_adversary.run_attack(
bnn, loss_fn, x, y, config.epsilons, image_id
)
nn_d, nn_imgs, nn_pertubation_imgs = nn_adversary.run_attack(
nn, x, y, config.epsilons, 3
)
ids = [2, 5]
utils.img_show(
x,
f"Image, BNN Prediction: {bnn_d.iloc[0]['y_']}, NN Prediction: {nn_d.iloc[0]['y_']}",
)
for id in ids:
utils.img_two_show(
bnn_pertubation_imgs[id].cpu(),
f"BNN Noise (epsilon: {bnn_d.iloc[id]['epsilon']})",
nn_pertubation_imgs[id].cpu(),
f"NN Noise (epsilon: {bnn_d.iloc[id]['epsilon']})",
)
utils.img_two_show(
bnn_imgs[id].cpu(),
f"Noise added to image, BNN Prediction: {bnn_d.iloc[id]['y_']}",
nn_imgs[id].cpu(),
f"Noise added to image, NN Prediction: {nn_d.iloc[id]['y_']}",
)
def main():
config = Configuration()
image_id = 1362
nn = Network()
nn.load_model()
test_loader = get_test_loader(1, shuffle=False)
x, y = test_loader.dataset[image_id]
y = torch.tensor([y])
nn_d, nn_imgs, nn_pertubation_imgs = nn_adversary.run_attack(
nn, x, y, config.epsilons, 3)
id = 2
utils.img_show(x, f"Image, NN Prediction: {nn_d.iloc[0]['y_']}")
utils.img_show(nn_pertubation_imgs[id].cpu(),
f"Noise (epsilon: {nn_d.iloc[id]['epsilon']})")
utils.img_show(nn_imgs[id].cpu(),
f"Noise added to image, Prediction: {nn_d.iloc[id]['y_']}")
pertubation.append(pert)
pred = net(pertubed_image.view(-1, 28 * 28))
y_ = pred.data.max(1).indices.item()
tmp_dict["id"].append(batch_id)
tmp_dict["epsilon"].append(epsilon)
tmp_dict["y"].append(y.item())
tmp_dict["y_"].append(y_)
return pandas.DataFrame.from_dict(tmp_dict), pertubed_images, pertubation
if __name__ == "__main__":
config = Configuration()
net = Network()
net.load_model()
net.eval()
test_loader = get_test_loader(batch_size=1, shuffle=False)
result = []
for batch_id, (x, y) in enumerate(test_loader):
df, _ = run_attack(net, x, y, config.epsilons, batch_id)
result.append(df)
if batch_id % 100 == 0:
print(f"Step {batch_id}/{len(test_loader.dataset)}")