def mean_filter_benchmark(
classifier,
filter_hw,
weightings,
advex_subdir='alexnet_val_2k_top1_correct/deepfool_oblivious/'):
"""
evaluates weighted mean filter performances on oblivious adversarials
"""
log_list = []
advex_matches = advex_match_paths(
images_file='alexnet_val_2k_top1_correct.txt',
advex_subdir=advex_subdir)
print('number of matches:', len(advex_matches))
count = 0
with tf.Graph().as_default():
smoothed_img, img_pl, mean_filter_pl, filter_feed_op = mean_filter_model(
filter_hw)
_, logit_tsr, _ = get_classifier_io(classifier,
input_init=smoothed_img,
input_type='tensor')
# ref_in, ref_out = get_classifier_io(classifier, input_type='placeholder')
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for img_path, adv_path in advex_matches:
count += 1
print('match no.', count)
image = np.expand_dims(
load_image(img_path).astype(dtype=np.float32), axis=0)
advex = np.expand_dims(
load_image(adv_path).astype(dtype=np.float32), axis=0)
# ref_img = sess.run(ref_out, feed_dict={ref_in: image})
# ref_adv = sess.run(ref_out, feed_dict={ref_in: advex})
img_log_list = []
adv_log_list = []
for weight in weightings:
filter_mat = make_weighted_mean_filter(weight, filter_hw)
sess.run(filter_feed_op,
feed_dict={mean_filter_pl: filter_mat})
img_smoothed_pred, img_smoothed = sess.run(
[logit_tsr, smoothed_img], feed_dict={img_pl: image})
img_smoothed_label = np.argmax(img_smoothed_pred)
img_log_list.append(img_smoothed_label)
adv_smoothed_pred, adv_smoothed = sess.run(
[logit_tsr, smoothed_img], feed_dict={img_pl: advex})
adv_smoothed_label = np.argmax(adv_smoothed_pred)
adv_log_list.append(adv_smoothed_label)
log_list.append([img_log_list, adv_log_list])
log_mat = np.asarray(log_list)
print(log_mat)
np.save('smooth_log.npy', log_mat)
def get_score_from_files(self, src_file, rec_file):
src_mat = load_image(src_file)
if src_mat.shape[0] != 1:
src_mat = np.expand_dims(src_mat, axis=0)
rec_mat = load_image(rec_file, resize=False)
if rec_mat.shape[0] != 1:
rec_mat = np.expand_dims(rec_mat, axis=0)
self.get_score(src_mat, rec_mat)
def inv_mse_and_vgg_scores(classifier):
tgt_paths = subset10_paths(classifier)
_, img_hw, layer_names = classifier_stats(classifier)
log_path = '../logs/opt_inversion/{}/image_rec/'.format(classifier)
layer_subdirs = [n.replace('/', '_') for n in layer_names]
img_subdirs = ['val{}'.format(i) for i in selected_img_ids()]
tgt_images = [np.expand_dims(load_image(p), axis=0) for p in tgt_paths]
rec_filename = 'imgs/rec_5000.png'
vgg_loss = VggScoreLoss(('tgt_224:0', 'rec_224:0'), weighting=1.0, name=None, input_scaling=1.0)
mse_loss = MSELoss('tgt_pl:0', 'rec_pl:0')
nmse_loss = NormedMSELoss('tgt_pl:0', 'rec_pl:0')
loss_mods = [vgg_loss, mse_loss, nmse_loss]
found_layers = []
score_list = []
with tf.Graph().as_default():
tgt_pl = tf.placeholder(dtype=tf.float32, shape=(1, img_hw, img_hw, 3), name='tgt_pl')
rec_pl = tf.placeholder(dtype=tf.float32, shape=(1, img_hw, img_hw, 3), name='rec_pl')
_ = tf.slice(tgt_pl, begin=[0, 0, 0, 0], size=[-1, 224, 224, -1], name='tgt_224')
_ = tf.slice(rec_pl, begin=[0, 0, 0, 0], size=[-1, 224, 224, -1], name='rec_224')
for lmod in loss_mods:
lmod.build()
loss_tsr_list = [m.get_loss() for m in loss_mods]
with tf.Session() as sess:
for layer_subdir in layer_subdirs:
layer_log_path = '{}{}/imgs/'.format(log_path, layer_subdir)
if not os.path.exists(layer_log_path):
continue
found_layers.append(layer_subdir)
layer_score_list = []
for idx, img_subdir in enumerate(img_subdirs):
img_log_path = '{}{}/'.format(layer_log_path, img_subdir)
rec_image = np.expand_dims(load_image(img_log_path + rec_filename), axis=0)
if np.max(rec_image) < 2.:
rec_image = rec_image * 255.
scores = sess.run(loss_tsr_list, feed_dict={tgt_pl: tgt_images[idx], rec_pl: rec_image})
layer_score_list.append(scores)
score_list.append(layer_score_list)
score_mat = np.asarray(score_list)
print(score_mat.shape)
print(found_layers)
np.save('{}score_mat.npy'.format(log_path), score_mat)
def db_collect_rec_images(classifier,
select_modules=None,
select_images=None,
merged=False):
# makes one [layers, imgs, h, w, c] mat for all rec images
_, img_hw, layer_names = classifier_stats(classifier)
log_path = '../logs/cnn_inversion/{}/'.format(classifier)
stack_mode = 'merged' if merged else 'stacked'
start_module_ids = select_modules or (1, 4, 7, 8, 9)
img_subdirs = select_images or selected_img_ids()
img_list = []
for module_id in start_module_ids:
layer_log_path = '{}stack_{}_to_1/{}/'.format(log_path, module_id,
stack_mode)
layer_list = []
for idx, img_subdir in enumerate(img_subdirs):
rec_image = load_image(layer_log_path +
'img_rec_{}.png'.format(img_subdir))
if rec_image.shape[0] == 1:
rec_image = np.squeeze(rec_image, axis=0)
layer_list.append(rec_image)
img_list.append(layer_list)
return np.asarray(img_list)
def foe_collect_rec_images(classifier, select_layers=None, select_images=None, rescaled=False):
# makes one [layers, imgs, h, w, c] mat for all rec3500 images
tgt_paths = subset10_paths(classifier)
_, img_hw, layer_names = classifier_stats(classifier)
log_path = '../logs/opt_inversion/{}/image_rec/'.format(classifier)
layer_subdirs = select_layers or [l.replace('/', '_') for l in layer_names]
img_subdirs = select_images or [p.split('/')[-1].split('.')[0] for p in tgt_paths]
# tgt_images = [np.expand_dims(load_image(p), axis=0) for p in tgt_paths]
scale_subdir = 'imgs_rescaled' if rescaled else 'imgs'
rec_filename = 'full512/{}/rec_10000.png'.format(scale_subdir)
img_list = []
for layer_subdir in layer_subdirs:
layer_log_path = '{}{}/'.format(log_path, layer_subdir)
layer_list = []
for idx, img_subdir in enumerate(img_subdirs):
img_log_path = '{}{}/'.format(layer_log_path, img_subdir)
rec_image = load_image(img_log_path + rec_filename)
if rec_image.shape[0] == 1:
rec_image = np.squeeze(rec_image, axis=0)
layer_list.append(rec_image)
img_list.append(layer_list)
return np.asarray(img_list)
def mv_script_fun(src_layer, img_path, log_path, classifier,
n_iterations=3500, lr_lower_points=None, jitter_stop_point=2750, range_b=80, alpha=6, beta=2):
if not src_layer.endswith(':0'):
src_layer = src_layer + ':0'
imagenet_mean, img_hw, _ = classifier_stats(classifier)
mse_weight, jitter_t = get_jitter_and_mse_weight(classifier, src_layer)
sr_weight = 1. / (img_hw ** 2 * range_b ** alpha)
tv_weight = 1. / (img_hw ** 2 * (range_b / 6.5) ** beta)
split = SplitModule(name_to_split=src_layer, img_slice_name='img_rep', rec_slice_name='rec_rep')
norm = NormModule(name_to_norm='pre_featmap/read:0', out_name='pre_featmap_normed', offset=imagenet_mean, scale=1.0)
mse = NormedMSELoss(target='img_rep:0', reconstruction='rec_rep:0', weighting=mse_weight)
sr_prior = SoftRangeLoss(tensor='pre_featmap_normed:0', alpha=6, weighting=sr_weight)
tv_prior = TotalVariationLoss(tensor='pre_featmap_normed:0', beta=2, weighting=tv_weight)
modules = [split, norm, mse, sr_prior, tv_prior]
lr_factor = range_b ** 2 / alpha
lr_lower_points = lr_lower_points or ((0, 1e-2 * lr_factor), (1000, 3e-3 * lr_factor), (2000, 1e-3 * lr_factor))
ni = NetInversion(modules, log_path, classifier=classifier, print_freq=100, log_freq=1750)
pre_img_init = np.expand_dims(load_image(img_path), axis=0).astype(np.float32)
ni.train_pre_featmap(img_path, n_iterations=n_iterations, optim_name='adam',
jitter_t=jitter_t, jitter_stop_point=jitter_stop_point, range_clip=True, scale_pre_img=1.,
range_b=range_b,
lr_lower_points=lr_lower_points,
pre_featmap_init=pre_img_init, ckpt_offset=0, save_as_plot=True)
def get_batch_generator(self, batch_size, mode, resize=False,
data_path='../data/imagenet2012-validationset/',
train_images_file='train_48k_images.txt',
validation_images_file='validate_2k_images.txt'):
if mode == 'train':
img_file = train_images_file
elif mode == 'validate':
img_file = validation_images_file
else:
raise NotImplementedError
if self.classifier.lower() == 'alexnet':
subdir = 'images_resized_227/'
elif self.classifier.lower() == 'vgg16':
subdir = 'images_resized_224/'
else:
raise NotImplementedError
with open(data_path + img_file) as f:
image_files = [k.rstrip() for k in f.readlines()]
begin = 0
while True:
end = begin + batch_size
if end < len(image_files):
batch_files = image_files[begin:end]
else:
end = end - len(image_files)
batch_files = image_files[begin:] + image_files[:end]
begin = end
if resize:
batch_paths = [data_path + 'images/' + k for k in batch_files]
images = []
for img_path in batch_paths:
image = load_image(img_path, resize=True)
if len(image.shape) == 2:
image = grey2rgb(image)
images.append(image)
mat = np.stack(images, axis=0)
else:
batch_paths = [data_path + subdir + k[:-len('JPEG')] + 'bmp' for k in batch_files]
images = []
for img_path in batch_paths:
image = load_image(img_path, resize=False)
images.append(image)
mat = np.stack(images, axis=0)
yield mat
def get_orig_featmaps():
with tf.Graph().as_default() as graph:
img = load_image(
'/home/frederik/PycharmProjects/deep_restoration/data/selected/images_resized_227/red-fox.bmp'
)
image = tf.constant(img, dtype=tf.float32, shape=[1, 227, 227, 3])
net = AlexNet()
net.build(image, rescale=1.0)
feat_map = graph.get_tensor_by_name('conv1/lin' + ':0')
with tf.Session() as sess:
feat_map_mat = sess.run(feat_map)
return feat_map_mat
def mv_collect_rec_images(classifier, select_layers=None, select_images=None):
# makes one [layers, imgs, h, w, c] mat for all rec3500 images
tgt_paths = subset10_paths(classifier)
_, img_hw, layer_names = classifier_stats(classifier)
log_path = '../logs/mahendran_vedaldi/2016/{}/'.format(classifier)
layer_subdirs = select_layers or [l.replace('/', '_') for l in layer_names]
img_subdirs = select_images or [p.split('/')[-1].split('.')[0] for p in tgt_paths]
# tgt_images = [np.expand_dims(load_image(p), axis=0) for p in tgt_paths]
rec_filename = 'imgs/rec_3500.png'
img_list = []
for layer_subdir in layer_subdirs:
layer_log_path = '{}{}/'.format(log_path, layer_subdir)
layer_list = []
for idx, img_subdir in enumerate(img_subdirs):
img_log_path = '{}{}/'.format(layer_log_path, img_subdir)
rec_image = load_image(img_log_path + rec_filename)
if rec_image.shape[0] == 1:
rec_image = np.squeeze(rec_image, axis=0)
layer_list.append(rec_image)
img_list.append(layer_list)
return np.asarray(img_list)
def db_img_mse_and_vgg_scores(classifier,
select_modules=None,
select_images=None,
merged=True):
tgt_paths = subset10_paths(classifier)
_, img_hw, layer_names = classifier_stats(classifier)
tgt_images = [np.expand_dims(load_image(p), axis=0) for p in tgt_paths]
_, img_hw, layer_names = classifier_stats(classifier)
log_path = '../logs/cnn_inversion/{}/'.format(classifier)
stack_mode = 'merged' if merged else 'stacked'
start_module_ids = select_modules or (1, 4, 7, 8, 9)
img_subdirs = select_images or selected_img_ids()
vgg_loss = VggScoreLoss(('tgt_224:0', 'rec_224:0'),
weighting=1.0,
name=None,
input_scaling=1.0)
mse_loss = MSELoss('tgt_pl:0', 'rec_pl:0')
nmse_loss = NormedMSELoss('tgt_pl:0', 'rec_pl:0')
loss_mods = [vgg_loss, mse_loss, nmse_loss]
found_layers = []
score_list = []
with tf.Graph().as_default():
tgt_pl = tf.placeholder(dtype=tf.float32,
shape=(1, img_hw, img_hw, 3),
name='tgt_pl')
rec_pl = tf.placeholder(dtype=tf.float32,
shape=(1, img_hw, img_hw, 3),
name='rec_pl')
_ = tf.slice(tgt_pl,
begin=[0, 0, 0, 0],
size=[-1, 224, 224, -1],
name='tgt_224')
_ = tf.slice(rec_pl,
begin=[0, 0, 0, 0],
size=[-1, 224, 224, -1],
name='rec_224')
for lmod in loss_mods:
lmod.build()
loss_tsr_list = [m.get_loss() for m in loss_mods]
with tf.Session() as sess:
img_list = []
for module_id in start_module_ids:
layer_log_path = '{}stack_{}_to_1/{}/'.format(
log_path, module_id, stack_mode)
layer_list = []
for idx, img_subdir in enumerate(img_subdirs):
rec_image = load_image(layer_log_path +
'img_rec_{}.png'.format(img_subdir))
if rec_image.shape[0] != 1:
rec_image = np.expand_dims(rec_image, axis=0)
scores = sess.run(loss_tsr_list,
feed_dict={
tgt_pl: tgt_images[idx],
rec_pl: rec_image
})
layer_list.append(scores)
score_list.append(layer_list)
score_mat = np.asarray(score_list)
print(score_mat.shape)
print(found_layers)
np.save('../logs/cnn_inversion/{}/score_mat.npy'.format(classifier),
score_mat)
def load_and_stack_imgs(img_paths):
return np.stack([load_image(p) for p in img_paths], axis=0)
def train_pre_featmap(self, image_path, n_iterations, grad_clip=100.0, lr_lower_points=((0, 1e-4),),
range_b=80, jitter_t=0, optim_name='adam',
range_clip=False, save_as_plot=False, bound_plots=False, jitter_stop_point=-1,
scale_pre_img=1.0, pre_featmap_init=None, ckpt_offset=0,
pre_featmap_name='input', classifier_cutoff=None,
tensor_names_to_save=(), featmap_names_to_plot=(), max_n_featmaps_to_plot=5):
"""
like mahendran & vedaldi, optimizes pre-image based on a single other image
"""
if not os.path.exists(self.log_path + 'mats/'):
os.makedirs(self.log_path + 'mats/')
if save_as_plot and not os.path.exists(self.log_path + 'imgs/'):
os.makedirs(self.log_path + 'imgs/')
img_mat = np.expand_dims(load_image(image_path, resize=False), axis=0)
target_featmap_mat = self.get_target_featmap(img_mat, pre_featmap_name)
print('target_shape:', target_featmap_mat.shape)
with tf.Graph().as_default() as graph:
with tf.Session() as sess:
target_featmap = tf.get_variable(name='target_featmap', dtype=tf.float32, trainable=False,
initializer=target_featmap_mat)
if pre_featmap_init is None and scale_pre_img == 2.7098e+4:
print('using special initializer') # remnant from m&v settings
pre_featmap_init = tf.abs(tf.random_normal([1, self.img_hw, self.img_hw, 3], mean=0, stddev=0.0001))
elif pre_featmap_init is None and pre_featmap_name == 'input':
pre_featmap_init = np.random.normal(loc=np.mean(self.imagenet_mean), scale=1.1,
size=([1, self.img_hw, self.img_hw, 3])).astype(np.float32)
pre_featmap_init = np.maximum(pre_featmap_init, 100.)
pre_featmap_init = np.minimum(pre_featmap_init, 155.)
elif pre_featmap_init is None:
pre_featmap_init = np.random.normal(loc=0, scale=0.1,
size=target_featmap_mat.shape).astype(np.float32)
# think about taking mean and sdev from target feature map layers
pre_featmap = tf.get_variable('pre_featmap', dtype=tf.float32, initializer=pre_featmap_init)
jitter_x_pl = tf.placeholder(dtype=tf.int32, shape=[], name='jitter_x_pl')
jitter_y_pl = tf.placeholder(dtype=tf.int32, shape=[], name='jitter_y_pl')
# old jitter: always a positive offset
# rec_part = tf.slice(pre_featmap, [0, jitter_x_pl, jitter_y_pl, 0], [-1, -1, -1, -1])
# rec_padded = tf.pad(rec_part, paddings=[[0, 0], [jitter_x_pl, 0], [jitter_y_pl, 0], [0, 0]])
# new jitter: expected offset is 0
p1, p2 = (jitter_t // 2), jitter_t - (jitter_t // 2)
rec_padded = tf.pad(pre_featmap, paddings=[[0, 0], [p1, p2], [p1, p2], [0, 0]])
rec_padded = rec_padded[:, jitter_x_pl:jitter_x_pl + self.img_hw,
jitter_y_pl:jitter_y_pl + self.img_hw, :]
use_jitter_pl = tf.placeholder(dtype=tf.bool, shape=[], name='use_jitter')
rec_input = tf.cond(use_jitter_pl, lambda: rec_padded, lambda: pre_featmap, name='jitter_cond')
net_input = tf.concat([target_featmap, rec_input * scale_pre_img], axis=0)
self.load_partial_classifier(net_input, pre_featmap_name, classifier_cutoff)
loss = self.build_model()
tensors_to_save = [graph.get_tensor_by_name(k) for k in tensor_names_to_save]
featmaps_to_plot = [graph.get_tensor_by_name(k) for k in featmap_names_to_plot]
if optim_name.lower() == 'l-bfgs-b':
options = {'maxiter': n_iterations}
scipy_opt = tf.contrib.opt.ScipyOptimizerInterface(loss, method='L-BFGS-B',
options=options)
def loss_cb(*args):
print(args)
sess.run(tf.global_variables_initializer())
scipy_opt.minimize(session=sess, feed_dict={jitter_x_pl: 0, jitter_y_pl: 0, use_jitter_pl: False},
loss_callback=loss_cb)
rec_mat = sess.run(pre_featmap)
np.save(self.log_path + 'mats/rec_{}.npy'.format(n_iterations), rec_mat)
else:
lr_pl = tf.placeholder(dtype=tf.float32, shape=[])
optimizer = get_optimizer(optim_name, lr_pl)
tvars = tf.trainable_variables()
grads = tf.gradients(loss, tvars)
tg_pairs = [k for k in zip(grads, tvars) if k[0] is not None]
tg_clipped = [(tf.clip_by_value(k[0], -grad_clip, grad_clip), k[1])
for k in tg_pairs]
train_op = optimizer.apply_gradients(tg_clipped)
if range_clip == 'img_bounds':
clipped_img = tf.maximum(tf.minimum(pre_featmap, 255.), 0.)
clip_op = tf.assign(pre_featmap, clipped_img)
elif range_clip and pre_featmap_name == 'input':
position_norm = tf.sqrt(tf.reduce_sum((pre_featmap - self.imagenet_mean) ** 2, axis=3))
box_rescale = tf.minimum(2 * range_b / position_norm, 1.)
box_rescale = tf.stack([box_rescale] * 3, axis=3)
clip_op = tf.assign(pre_featmap,
(pre_featmap - self.imagenet_mean) * box_rescale + self.imagenet_mean)
else:
clip_op = None
train_summary_op, summary_writer, saver, val_loss, val_summary_op = self.build_logging(loss)
# summary_writer.add_graph(graph)
sess.run(tf.global_variables_initializer())
for mod in self.modules:
if isinstance(mod, (TrainedModule, LearnedPriorLoss)):
mod.load_weights(sess)
use_jitter = False if jitter_t == 0 else True
lr = lr_lower_points[0][1]
start_time = time.time()
for count in range(ckpt_offset + 1, ckpt_offset + n_iterations + 1):
jitter = np.random.randint(low=0, high=jitter_t + 1, dtype=int, size=(2,))
feed = {lr_pl: lr,
jitter_x_pl: jitter[0],
jitter_y_pl: jitter[1],
use_jitter_pl: use_jitter}
batch_loss, _, summary_string = sess.run([loss, train_op, train_summary_op], feed_dict=feed)
if range_clip:
sess.run(clip_op)
if count % self.summary_freq == 0:
summary_writer.add_summary(summary_string, count)
if count % self.print_freq == 0:
print(('Iteration: {0:6d} Training Error: {1:8.5f} ' +
'Time: {2:5.1f} min').format(count, batch_loss, (time.time() - start_time) / 60))
if count % self.log_freq == 0:
summary_writer.flush()
rec_mat = sess.run(pre_featmap)
np.save(self.log_path + 'mats/rec_' + str(count) + '.npy', rec_mat)
if save_as_plot:
if bound_plots:
rec_mat = np.minimum(np.maximum(rec_mat / 255., 0.), 1.)
else:
rec_mat = (rec_mat - np.min(rec_mat)) / (np.max(rec_mat) - np.min(rec_mat))
if rec_mat.shape[0] == 1:
rec_mat = np.squeeze(rec_mat, axis=0)
imsave(self.log_path + 'imgs/rec_' + str(count) + '.png', rec_mat)
# fig = plt.figure(frameon=False)
# fig.set_size_inches(1, 1)
# ax = plt.Axes(fig, [0., 0., 1., 1.])
# ax.set_axis_off()
# fig.add_axes(ax)
# if rec_mat.shape[0] == 1:
# rec_mat = np.squeeze(rec_mat, axis=0)
# ax.imshow(rec_mat, aspect='auto')
# plt.savefig(self.log_path + 'imgs/rec_' + str(count) + '.png',
# format='png', dpi=self.img_hw)
# plt.close()
if tensors_to_save:
mats_to_save = sess.run(tensors_to_save, feed_dict=feed)
for idx, fmap in enumerate(mats_to_save):
file_name = tensor_names_to_save[idx] + '-' + str(count) + '.npy'
np.save(self.log_path + 'mats/' + file_name, fmap)
if featmaps_to_plot:
fmaps_to_plot = sess.run(featmaps_to_plot, feed_dict=feed)
for fmap, name in zip(fmaps_to_plot, featmap_names_to_plot):
name = name.replace('/', '_').rstrip(':0')
file_path = '{}mats/{}-{}.npy'.format(self.log_path, name, count)
np.save(file_path, fmap)
if save_as_plot:
file_path = '{}imgs/{}-{}.png'.format(self.log_path, name, count)
plot_feat_map_diffs(fmap, file_path, max_n_featmaps_to_plot)
if use_jitter is True and jitter_stop_point <= count:
print('Jittering stopped at ', count)
use_jitter = False
if lr_lower_points and lr_lower_points[0][0] <= count:
lr = lr_lower_points[0][1]
print('new learning rate: ', lr)
lr_lower_points = lr_lower_points[1:]
summary_writer.flush()
weighting=mse_weight)
sr_prior = SoftRangeLoss(tensor='pre_featmap_normed:0',
alpha=6,
weighting=sr_weight)
tv_prior = TotalVariationLoss(tensor='pre_featmap_normed:0',
beta=2,
weighting=tv_weight)
modules = [split, norm, mse, sr_prior, tv_prior]
lr_factor = 80.**2 / 6.
lr_lower_points = ((0, 1e-2 * lr_factor), (1000, 3e-3 * lr_factor),
(2000, 1e-3 * lr_factor))
ni = NetInversion(modules, log_path, classifier='alexnet')
pre_img_init = np.expand_dims(load_image(img_path), axis=0).astype(np.float32)
# pre_img_init = np.load('../logs/net_inversion/alexnet/c1l_tests_16_08/init_helper.npy')
# pre_img_init = None
# ni.train_pre_image('../data/selected/images_resized/val13_monkey.bmp', jitter_t=0, optim_name='adam',
# lr_lower_points=((0, 0.01), (500, 0.001), (1000, 0.0001), (3000, 0.00003), (7000, 0.00001)))
ni.train_pre_featmap(img_path,
n_iterations=n_iterations,
optim_name='adam',
jitter_t=jitter_t,
jitter_stop_point=jitter_stop_point,
range_clip=True,
scale_pre_img=1.,
range_b=range_b,
def weighted_mean_make_adaptive_adv(attack_name='deepfool',
mean_weight=0.2,
filter_hw=2,
attack_keys=None,
verbose=True):
"""
creates adaptive adversarials for mean filter defense
"""
log_path = '../logs/adversarial_examples/alexnet_top1/deepfool/adaptive_mean_filter/'
if not os.path.exists(log_path):
os.makedirs(log_path)
img_log = np.load(
'../logs/adversarial_examples/alexnet_top1/deepfool/oblivious_fullprior512/lr06/img_log.npy'
)
classifier = 'alexnet'
# _, img_hw, _ = classifier_stats(classifier)
images_file = 'alexnet_val_2k_top1_correct.txt'
advex_subdir = 'alexnet_val_2k_top1_correct/deepfool_oblivious/'
advex_matches = advex_match_paths(images_file=images_file,
advex_subdir=advex_subdir)
filter_mat = make_weighted_mean_filter(mean_weight, filter_hw=filter_hw)
with tf.Graph().as_default():
smoothed_img, img_pl, mean_filter_pl, filter_feed_op = mean_filter_model(
filter_hw=2)
input_var, logit_tsr, _ = get_classifier_io(classifier,
input_init=smoothed_img,
input_type='tensor')
with tf.Session() as sess:
model = foolbox.models.TensorFlowModel(img_pl,
logit_tsr,
bounds=(0, 255))
criterion = foolbox.criteria.Misclassification()
attack = get_attack(attack_name, model, criterion)
if attack_keys is None:
if attack == 'deepfool':
attack_keys = {'steps': 300}
else:
attack_keys = dict()
init_op = tf.global_variables_initializer()
sess.run(init_op)
sess.run(filter_feed_op, feed_dict={mean_filter_pl: filter_mat})
noise_norms = []
src_invariant = []
adv_path = advex_matches[0][1]
adaptive_save_path = adv_path.replace('oblivious',
'adaptive_mean_filter')
adaptive_save_dir = '/'.join(adaptive_save_path.split('/')[:-1])
if not os.path.exists(adaptive_save_dir):
os.makedirs(adaptive_save_dir)
for idx, match in enumerate(advex_matches):
img_path, adv_path = match
src_label = img_log[idx][0]
img = load_image(img_path)
adv = load_image(adv_path)
oblivious_norm = np.linalg.norm((img - adv).flatten(), ord=2)
print('oblivious norm', oblivious_norm)
img_pred = model.predictions(img)
img_pred_label = np.argmax(img_pred)
adv_pred = model.predictions(adv)
adv_pred_label = np.argmax(adv_pred)
if verbose:
noisy_label_name = get_class_name(img_pred_label)
if img_pred_label == src_label:
print('noisy label {} same as source: {}'.format(
img_pred_label, noisy_label_name))
src_invariant.append(1)
if adv_pred_label == img_pred_label:
print('oblivious attack averted')
else:
print('WARNING: oblivious attack succeeded!')
else:
print(
'image with prior misclassified as {}. (label {})'.
format(noisy_label_name, img_pred_label))
src_invariant.append(0)
try:
adversarial = attack(image=img,
label=img_pred_label,
**attack_keys)
if adversarial is None:
adaptive_norm = None
if verbose:
print(
'no adversary found for source label {} using {}'
.format(img_pred_label, attack_name))
else:
fooled_pred = model.predictions(adversarial)
fooled_label = np.argmax(fooled_pred)
fooled_label_name = get_class_name(fooled_label)
adaptive_norm = np.linalg.norm(
(img - adversarial).flatten(), ord=2)
if verbose:
print(
'adversarial image classified as {}. (label {}) '
'Necessary perturbation: {}'.format(
fooled_label_name, fooled_label,
adaptive_norm))
adaptive_save_path = adv_path.replace(
'oblivious', 'adaptive_mean_filter')
np.save(adaptive_save_path, adversarial)
except AssertionError as err:
adaptive_norm = -np.inf
print('FoolBox failed Assertion: {}'.format(err))
noise_norms.append((oblivious_norm, adaptive_norm))
if idx + 1 % 100 == 0:
np.save(log_path + 'noise_norms.npy',
np.asarray(noise_norms))
np.save(log_path + 'src_invariants.npy',
np.asarray(src_invariant))
np.save(log_path + 'noise_norms.npy', np.asarray(noise_norms))
np.save(log_path + 'src_invariants.npy', np.asarray(src_invariant))
def mean_adaptive_attacks_200(attack_name='deepfool',
attack_keys=None,
verbose=True):
"""
creates adaptive adversarials for mean filter defense
:param attack_name:
:param attack_keys:
:param verbose:
:return:
"""
path = '../logs/adversarial_examples/deepfool_oblivious_198/'
img_log = np.load(path + 'img_log_198_fine.npy')
# adv_log = np.load(path + 'adv_log_198_fine.npy')
classifier = 'alexnet'
advex_matches = advex_match_paths(
images_file='subset_cutoff_200_images.txt',
advex_subdir='200_dataset/deepfool_oblivious/')
with tf.Graph().as_default():
net_input, img_pl, _ = mean_filter_model(make_switch=False)
input_var, logit_tsr = get_classifier_io(classifier,
input_init=net_input,
input_type='tensor')
with tf.Session() as sess:
model = foolbox.models.TensorFlowModel(img_pl,
logit_tsr,
bounds=(0, 255))
criterion = foolbox.criteria.Misclassification()
attack = get_attack(attack_name, model, criterion)
if attack_keys is None:
attack_keys = dict()
init_op = tf.global_variables_initializer()
sess.run(init_op)
noise_norms = []
src_invariant = []
for idx, match in enumerate(advex_matches[:20]):
img_path, adv_path = match
src_label = img_log[idx][0]
img = load_image(img_path)
adv = load_image(adv_path)
oblivious_norm = np.linalg.norm((img - adv).flatten(), ord=2)
print('oblivious norm', oblivious_norm)
img_pred = model.predictions(img)
img_pred_label = np.argmax(img_pred)
adv_pred = model.predictions(adv)
adv_pred_label = np.argmax(adv_pred)
if verbose:
noisy_label_name = get_class_name(img_pred_label)
if img_pred_label == src_label:
print('noisy label {} same as source: {}'.format(
img_pred_label, noisy_label_name))
src_invariant.append(1)
if adv_pred_label == img_pred_label:
print('oblivious attack averted')
else:
print('WARNING: oblivious attack succeeded!')
else:
print(
'image with prior misclassified as {}. (label {})'.
format(noisy_label_name, img_pred_label))
src_invariant.append(0)
adversarial = attack(image=img, label=src_label, **attack_keys)
if adversarial is None:
adaptive_norm = None
if verbose:
print(
'no adversary found for source label {} using {}'.
format(img_pred_label, attack_name))
else:
fooled_pred = model.predictions(adversarial)
fooled_label = np.argmax(fooled_pred)
fooled_label_name = get_class_name(fooled_label)
adaptive_norm = np.linalg.norm(
(img - adversarial).flatten(), ord=2)
if verbose:
print('adversarial image classified as {}. (label {}) '
'Necessary perturbation: {}'.format(
fooled_label_name, fooled_label,
adaptive_norm))
# adaptive_save_path = adv_path.replace('oblivious', 'adaptive'
# np.save(adaptive_save_path, adversarial)
noise_norms.append((oblivious_norm, adaptive_norm))
def raw_patch_data_mat(map_name,
classifier,
num_patches,
ph,
pw,
batch_size,
n_channels,
save_dir,
file_name='raw_mat.npy'):
"""
create (num_patches, n_channels, feats per channel) matrix of extracted patches
"""
assert num_patches % batch_size == 0
if classifier.lower() == 'vgg16':
classifier = Vgg16()
image_subdir = 'images_resized_224/'
img_dims = [batch_size, 224, 224, 3]
elif classifier.lower() == 'alexnet':
classifier = AlexNet()
image_subdir = 'images_resized_227/'
img_dims = [batch_size, 227, 227, 3]
else:
raise NotImplementedError
file_path = save_dir + file_name
with tf.Graph().as_default() as graph:
with tf.Session() as sess:
img_pl = tf.placeholder(dtype=tf.float32,
shape=img_dims,
name='img_pl')
classifier.build(img_pl, rescale=1.0)
feat_map = graph.get_tensor_by_name(map_name)
map_dims = [d.value for d in feat_map.get_shape()]
n_feats_per_channel = ph * pw
# n_features = n_feats_per_channel * map_dims[3]
data_path = '../data/imagenet2012-validationset/'
img_file = 'train_48k_images.txt'
raw_mat = np.memmap(file_path,
dtype=np.float32,
mode='w+',
shape=(num_patches, n_channels,
n_feats_per_channel))
max_h = map_dims[1] - ph
max_w = map_dims[2] - pw
with open(data_path + img_file) as f:
image_files = [k.rstrip() for k in f.readlines()]
image_paths = [
data_path + image_subdir + k[:-len('JPEG')] + 'bmp'
for k in image_files
]
img_mat = np.zeros(shape=img_dims)
for count in range(num_patches // batch_size):
for idx in range(batch_size):
img_path = image_paths[idx + (count * batch_size) %
len(image_paths)]
img_mat[idx, :, :, :] = load_image(img_path, resize=False)
if count == 0:
print('Verifying scale - this should be around 255: ',
np.max(img_mat))
map_mat = sess.run(feat_map, feed_dict={img_pl: img_mat})
for idx in range(batch_size):
h = np.random.randint(0, max_h)
w = np.random.randint(0, max_w)
map_patch = np.transpose(map_mat[idx, h:h + ph,
w:w + pw, :],
axes=(2, 0, 1))
map_patch = map_patch.reshape([n_channels,
-1]).astype(np.float32)
raw_mat[idx + (count * batch_size), :, :] = map_patch
del raw_mat
raw_mat = np.memmap(file_path,
dtype=np.float32,
mode='r',
shape=(num_patches, n_channels, n_feats_per_channel))
return raw_mat
