def exp_over_grid_eval(model,
attack,
sess,
config,
attack_type,
eval_on_train=False):
num_eval_examples = config.eval.num_eval_examples
eval_batch_size = config.eval.batch_size
if config.data.dataset_name == "cifar-10":
data_iterator = cifar10_input.CIFAR10Data(config.data.data_path)
elif config.data.dataset_name == "cifar-100":
data_iterator = cifar100_input.CIFAR100Data(config.data.data_path)
elif config.data.dataset_name == "svhn":
data_iterator = svhn_input.SVHNData(config.data.data_path)
else:
raise ValueError("Unknown dataset name.")
# Iterate over the samples batch-by-batch
num_batches = int(math.ceil(num_eval_examples / eval_batch_size))
n_grid = np.prod(config.attack.grid_granularity)
results = np.zeros([num_eval_examples, n_grid])
for ibatch in trange(num_batches):
bstart = ibatch * eval_batch_size
bend = min(bstart + eval_batch_size, num_eval_examples)
if eval_on_train:
x_batch = data_iterator.train_data.xs[bstart:bend, :]
y_batch = data_iterator.train_data.ys[bstart:bend]
else:
x_batch = data_iterator.eval_data.xs[bstart:bend, :]
y_batch = data_iterator.eval_data.ys[bstart:bend]
grid = product(*list(
np.linspace(-l, l, num=g) for l, g in zip(
config.attack.spatial_limits, config.attack.grid_granularity)))
n_batch = len(x_batch)
for i, (tx, ty, r) in enumerate(grid):
trans = np.stack(repeat([tx, ty, r], n_batch))
dict_nat = {
model.x_input: x_batch,
model.y_input: y_batch,
model.transform: trans,
model.is_training: False
}
results[bstart:bend, i] = sess.run(model.correct_prediction,
feed_dict=dict_nat)
return results
def exp_over_grid_eval_random_sample(model,
sess,
config,
eval_on_train=False,
num_eval_examples=100,
seed=1):
np.random.seed(seed)
if config.data.dataset_name == "cifar-10":
data_iterator = cifar10_input.CIFAR10Data(config.data.data_path)
elif config.data.dataset_name == "cifar-100":
data_iterator = cifar100_input.CIFAR100Data(config.data.data_path)
elif config.data.dataset_name == "svhn":
data_iterator = svhn_input.SVHNData(config.data.data_path)
else:
raise ValueError("Unknown dataset name.")
if eval_on_train:
n = data_iterator.train_data.n
indices = np.random.choice(n, num_eval_examples)
x_batch = data_iterator.train_data.xs[indices, :]
y_batch = data_iterator.train_data.ys[indices]
else:
n = data_iterator.eval_data.n
indices = np.random.choice(n, num_eval_examples)
x_batch = data_iterator.eval_data.xs[indices, :]
y_batch = data_iterator.eval_data.ys[indices]
grid = product(*list(
np.linspace(-l, l, num=g) for l, g in zip(
config.attack.spatial_limits, config.attack.grid_granularity)))
n_grid = np.prod(config.attack.grid_granularity)
results = np.zeros([len(x_batch), n_grid])
n_batch = len(x_batch)
for i, (tx, ty, r) in enumerate(grid):
trans = np.stack(repeat([tx, ty, r], n_batch))
dict_nat = {
model.x_input: x_batch,
model.y_input: y_batch,
model.transform: trans,
model.is_training: False
}
results[:, i] = sess.run(model.correct_prediction, feed_dict=dict_nat)
return results
def get_correctly_classified_angles(model,
sess,
config,
eval_on_train=False,
num_eval_examples=200,
seed=1):
np.random.seed(seed)
if config.data.dataset_name == "cifar-10":
data_iterator = cifar10_input.CIFAR10Data(config.data.data_path)
elif config.data.dataset_name == "cifar-100":
data_iterator = cifar100_input.CIFAR100Data(config.data.data_path)
elif config.data.dataset_name == "svhn":
data_iterator = svhn_input.SVHNData(config.data.data_path)
if eval_on_train:
n = data_iterator.train_data.n
indices = np.random.choice(n, num_eval_examples)
x_batch = data_iterator.train_data.xs[indices, :]
y_batch = data_iterator.train_data.ys[indices]
else:
n = data_iterator.eval_data.n
indices = np.random.choice(n, num_eval_examples)
x_batch = data_iterator.eval_data.xs[indices, :]
y_batch = data_iterator.eval_data.ys[indices]
trans = np.zeros([len(x_batch), 3])
results = np.zeros([len(x_batch), 61])
predictions = np.zeros([len(x_batch), 61])
angles = np.arange(-30, 31)
for i, j in enumerate(range(-30, 31)):
trans[:, 2] = j
dict_nat = {
model.x_input: x_batch,
model.y_input: y_batch,
model.transform: trans,
model.is_training: False
}
results[:, i], predictions[:, i] = sess.run(
[model.correct_prediction, model.predictions], feed_dict=dict_nat)
cor_class_means = np.mean(results, axis=1)
# get images that fooled the model for all angles
indices_all_fooled = np.where(cor_class_means == 0)[0]
if len(indices_all_fooled) > 0:
fooled_tuple = get_images(indices_all_fooled, config, data_iterator,
model, sess, x_batch, y_batch, results,
predictions, angles)
else:
fooled_tuple = None
# get images that were correctly classified for all angles
indices_all_correct = np.where(cor_class_means == 1)[0]
if len(indices_all_correct) > 0:
correct_tuple = get_images(indices_all_correct, config, data_iterator,
model, sess, x_batch, y_batch, results,
predictions, angles)
else:
correct_tuple = None
# get images that were correctly classified for a large proportion of angles
indices_some_correct = np.where(
np.logical_and(cor_class_means >= .9, cor_class_means < 1))[0]
if len(indices_some_correct) > 0:
some_correct_tuple = get_images(indices_some_correct, config,
data_iterator, model, sess, x_batch,
y_batch, results, predictions, angles)
else:
some_correct_tuple = None
return results, predictions, fooled_tuple, correct_tuple, some_correct_tuple
dataset, split, args.method, args.k, args.exp_id)))
plt.close()
if config.data.dataset_name == "cifar-10":
bins = 10
data_iterator = cifar10_input.CIFAR10Data(
config.data.data_path)
label_names = data_iterator.label_names
elif config.data.dataset_name == "cifar-100":
bins = 100
data_iterator = cifar100_input.CIFAR100Data(
config.data.data_path)
label_names = data_iterator.label_names
elif config.data.dataset_name == "svhn":
bins = 10
data_iterator = svhn_input.SVHNData(config.data.data_path)
label_names = np.arange(bins)
# images fooled
if fool_tup is not None:
imgs_fooled, y_fooled, pred_fooled, imgs_f_per, pred_f_per = fool_tup
plot_images(imgs_fooled, y_fooled, pred_fooled, imgs_f_per,
pred_f_per, bins, label_names, save_plot_subdir,
'misclassified_all_angles', args.method,
args.exp_id, args.k)
# images all correct
if cor_tup is not None:
imgs_correct, y_correct, pred_correct, imgs_c_per, pred_c_per = cor_tup
plot_images(imgs_correct, y_correct, pred_correct, imgs_c_per,
pred_c_per, bins, label_names, save_plot_subdir,
