def _get_siamese_selected_blobs(net, layer_def, siamese_view_mode,
blob_selector):
'''
function used to extract both blobs according to the specified layer and siamese input mode and
blob selector.
this is the main function which contains logic on the siamese network internal format structure
:param net: network containing the blob to extract
:param layer_def: layer requested
:param siamese_view_mode: siamese view mode
:param blob_selector:
:return: first_blob, second_blob
'''
if layer_def['format'] == 'normal':
raise Exception(
'function get_siamese_blobs() should not be called when layer is in normal format'
)
elif layer_def['format'] == 'siamese_layer_pair':
return blob_selector(net.blobs[layer_name_to_top_name(
net, layer_def['name/s'][0])])[0], blob_selector(
net.blobs[layer_name_to_top_name(
net, layer_def['name/s'][1])])[0]
elif layer_def['format'] == 'siamese_batch_pair':
return blob_selector(net.blobs[layer_name_to_top_name(
net, layer_def['name/s'])])[0], blob_selector(
net.blobs[layer_name_to_top_name(net,
layer_def['name/s'])])[1]
else:
raise Exception(
"get_siamese_blobs() got invalid layer_def['format']=%s" %
layer_def['format'])
def move_selection(self, direction, dist=1):
default_layer_name = self.get_default_layer_name()
default_top_name = layer_name_to_top_name(self.net, default_layer_name)
if direction == 'left':
if self.cursor_area == 'top':
self.layer_idx = max(0, self.layer_idx - dist)
else:
self.selected_unit -= dist
elif direction == 'right':
if self.cursor_area == 'top':
self.layer_idx = min(
len(self.settings.layers_list) - 1, self.layer_idx + dist)
else:
self.selected_unit += dist
elif direction == 'down':
if self.cursor_area == 'top':
self.cursor_area = 'bottom'
else:
self.selected_unit += self.net_blob_info[default_top_name][
'tile_cols'] * dist
elif direction == 'up':
if self.cursor_area == 'top':
pass
else:
self.selected_unit -= self.net_blob_info[default_top_name][
'tile_cols'] * dist
if self.selected_unit < 0:
self.selected_unit += self.net_blob_info[default_top_name][
'tile_cols']
self.cursor_area = 'top'
self.validate_state_for_summary_only_patterns()
def _get_single_selected_blob(net, layer_def, siamese_view_mode,
blob_selector):
'''
function used to extract the single selected blob according to the specified layer and siamese input mode and
blob selector.
note that it is invalid to call this function when siamese input mode is BOTH
this is the main function which contains logic on the siamese network internal format structure
:param net: network containing the blob to extract
:param layer_def: layer requested
:param siamese_view_mode: siamese view mode
:param blob_selector: lambda function which lets us choose between data and diff blobs
:return: requested single blob
'''
if layer_def['format'] == 'normal':
return blob_selector(net.blobs[layer_name_to_top_name(
net, layer_def['name/s'])])[0]
elif layer_def['format'] == 'siamese_layer_pair':
if siamese_view_mode == SiameseViewMode.FIRST_IMAGE:
selected_layer_name = layer_def['name/s'][0]
elif siamese_view_mode == SiameseViewMode.SECOND_IMAGE:
selected_layer_name = layer_def['name/s'][1]
else:
raise Exception(
'in get_single_selected_blob() siamese_view_mode cant be BOTH'
)
return blob_selector(net.blobs[layer_name_to_top_name(
net, selected_layer_name)])[0]
elif layer_def['format'] == 'siamese_batch_pair':
if siamese_view_mode == SiameseViewMode.FIRST_IMAGE:
selected_batch_index = 0
elif siamese_view_mode == SiameseViewMode.SECOND_IMAGE:
selected_batch_index = 1
else:
raise Exception(
'in get_single_selected_blob() siamese_view_mode cant be BOTH'
)
return blob_selector(net.blobs[layer_name_to_top_name(
net, layer_def['name/s'])])[selected_batch_index]
else:
raise Exception(
"get_single_selected_blob() got invalid layer_def['format']=%s"
% layer_def['format'])
def get_receptive_field(settings, net, layer_name):
# flag which indicates whether the dictionary was changed hence we need to write it to cache
should_save_to_cache = False
# check if dictionary exists
if not hasattr(settings, '_receptive_field_per_layer'):
# if it doesn't, try load it from file
receptive_fields_cache_filename = _get_receptive_fields_cache_filename(
settings)
if os.path.isfile(receptive_fields_cache_filename):
try:
with open(receptive_fields_cache_filename,
'rb') as receptive_fields_cache_file:
settings._receptive_field_per_layer = pickle.load(
receptive_fields_cache_file)
except:
settings._receptive_field_per_layer = dict()
should_save_to_cache = True
else:
settings._receptive_field_per_layer = dict()
should_save_to_cache = True
# calculate lazy
if layer_name not in settings._receptive_field_per_layer:
print("Calculating receptive fields for layer %s" % (layer_name))
top_name = layer_name_to_top_name(net, layer_name)
if top_name is not None:
blob = net.blobs[top_name].data
is_spatial = (len(blob.shape) == 4)
layer_receptive_field = get_max_data_extent(
net, settings, layer_name, is_spatial)
settings._receptive_field_per_layer[
layer_name] = layer_receptive_field
should_save_to_cache = True
if should_save_to_cache:
try:
receptive_fields_cache_filename = _get_receptive_fields_cache_filename(
settings)
mkdir_p(settings.caffevis_outputs_dir)
with open(receptive_fields_cache_filename,
'wb') as receptive_fields_cache_file:
pickle.dump(settings._receptive_field_per_layer,
receptive_fields_cache_file, -1)
except IOError:
# ignore problems in cache saving
pass
return settings._receptive_field_per_layer[layer_name]
def _ensure_valid_selected(self):
default_layer_name = self.get_default_layer_name()
default_top_name = layer_name_to_top_name(self.net, default_layer_name)
n_tiles = self.net_blob_info[default_top_name]['n_tiles']
# Forward selection
self.selected_unit = max(0, self.selected_unit)
self.selected_unit = min(n_tiles - 1, self.selected_unit)
# Backward selection
if not self.backprop_selection_frozen:
# If backprop_selection is not frozen, backprop layer/unit follows selected unit
if not (self.backprop_layer_idx == self.layer_idx
and self.backprop_unit == self.selected_unit and
self.backprop_siamese_view_mode == self.siamese_view_mode):
self.backprop_layer_idx = self.layer_idx
self.backprop_unit = self.selected_unit
self.backprop_siamese_view_mode = self.siamese_view_mode
self.back_stale = True # If there is any change, back diffs are now stale
def _optimize(self, params, x0, prefix_template):
xx = x0.copy()
results = [
FindResults(batch_index)
for batch_index in range(params.batch_size)
]
# check if all required outputs exist, in which case skip this optimization
all_outputs = [
self.generate_output_names(batch_index, params, results,
prefix_template,
self.settings.caffevis_outputs_dir)
for batch_index in range(params.batch_size)
]
relevant_outputs = [
best_X_name for [
best_X_name, best_Xpm_name, majority_X_name, majority_Xpm_name,
info_name, info_pkl_name, info_big_pkl_name
] in all_outputs
]
relevant_outputs_exist = [
os.path.exists(best_X_name) for best_X_name in relevant_outputs
]
if all(relevant_outputs_exist):
return xx, results, False
# Whether or not the unit being optimized corresponds to a label (e.g. one of the 1000 imagenet classes)
is_labeled_unit = params.push_layer in self.label_layers
# Sanity checks for conv vs FC layers
top_name = layer_name_to_top_name(self.net, params.push_layer)
data_shape = self.net.blobs[top_name].data.shape
assert len(data_shape) in (
2, 4
), 'Expected shape of length 2 (for FC) or 4 (for conv) layers but shape is %s' % repr(
data_shape)
is_spatial = (len(data_shape) == 4)
if is_spatial:
if params.push_spatial == (0, 0):
recommended_spatial = (data_shape[2] / 2, data_shape[3] / 2)
print(
'WARNING: A unit on a conv layer (%s) is being optimized, but push_spatial\n'
'is %s, so the upper-left unit in the channel is selected. To avoid edge\n'
'effects, you might want to optimize a non-edge unit instead, e.g. the center\n'
'unit by using `--push_spatial "%s"`\n' %
(params.push_layer, params.push_spatial,
recommended_spatial))
else:
assert params.push_spatial == (
0, 0), 'For FC layers, spatial indices must be (0,0)'
if is_labeled_unit:
# Sanity check
push_label = self.labels[params.push_unit[0]]
else:
push_label = None
old_obj = np.zeros(params.batch_size)
obj = np.zeros(params.batch_size)
for ii in range(params.max_iter):
# 0. Crop data
if self.batched_data_mean is not None:
xx = minimum(
255.0, maximum(0.0, xx + self.batched_data_mean)
) - self.batched_data_mean # Crop all values to [0,255]
else:
xx = minimum(255.0, maximum(0.0,
xx)) # Crop all values to [0,255]
# 1. Push data through net
out = self.net.forward_all(data=xx)
#shownet(net)
top_name = layer_name_to_top_name(self.net, params.push_layer)
acts = self.net.blobs[top_name].data
layer_format = self.siamese_helper.get_layer_format_by_layer_name(
params.push_layer)
# note: no batch support in 'siamese_batch_pair'
if self.settings.is_siamese and layer_format == 'siamese_batch_pair' and acts.shape[
0] == 2:
if not is_spatial:
# promote to 4D
acts = np.reshape(acts, (2, -1, 1, 1))
reshaped_acts = np.reshape(acts, (2, -1))
idxmax = unravel_index(reshaped_acts.argmax(axis=1),
acts.shape[1:])
valmax = reshaped_acts.max(axis=1)
# idxmax for fc or prob layer will be like: (batch,278, 0, 0)
# idxmax for conv layer will be like: (batch,37, 4, 37)
obj[0] = acts[0, params.push_unit[0], params.push_unit[1],
params.push_unit[2]]
elif self.settings.is_siamese and layer_format == 'siamese_batch_pair' and acts.shape[
0] == 1:
if not is_spatial:
# promote to 4D
acts = np.reshape(acts, (1, -1, 1, 1))
reshaped_acts = np.reshape(acts, (1, -1))
idxmax = unravel_index(reshaped_acts.argmax(axis=1),
acts.shape[1:])
valmax = reshaped_acts.max(axis=1)
# idxmax for fc or prob layer will be like: (batch,278, 0, 0)
# idxmax for conv layer will be like: (batch,37, 4, 37)
obj[0] = acts[0, params.push_unit[0], params.push_unit[1],
params.push_unit[2]]
else:
if not is_spatial:
# promote to 4D
acts = np.reshape(acts, (params.batch_size, -1, 1, 1))
reshaped_acts = np.reshape(acts, (params.batch_size, -1))
idxmax = unravel_index(reshaped_acts.argmax(axis=1),
acts.shape[1:])
valmax = reshaped_acts.max(axis=1)
# idxmax for fc or prob layer will be like: (batch,278, 0, 0)
# idxmax for conv layer will be like: (batch,37, 4, 37)
obj = acts[np.arange(params.batch_size), params.push_unit[0],
params.push_unit[1], params.push_unit[2]]
# 2. Update results
for batch_index in range(params.batch_size):
results[batch_index].update(params, ii, acts[batch_index], \
(idxmax[0][batch_index],idxmax[1][batch_index],idxmax[2][batch_index]), \
xx[batch_index], x0[batch_index])
# 3. Print progress
if ii > 0:
if params.lr_policy == 'progress':
print 'iter %-4d batch_index %d progress predicted: %g, actual: %g' % (
ii, batch_index, pred_prog[batch_index],
obj[batch_index] - old_obj[batch_index])
else:
print 'iter %-4d batch_index %d progress: %g' % (
ii, batch_index,
obj[batch_index] - old_obj[batch_index])
else:
print 'iter %d batch_index %d' % (ii, batch_index)
old_obj[batch_index] = obj[batch_index]
push_label_str = ('(%s)' %
push_label) if is_labeled_unit else ''
max_label_str = ('(%s)' % self.labels[idxmax[0][batch_index]]
) if is_labeled_unit else ''
print ' push unit: %16s with value %g %s' % (
params.push_unit, acts[batch_index][params.push_unit],
push_label_str)
print ' Max idx: %16s with value %g %s' % (
(idxmax[0][batch_index], idxmax[1][batch_index],
idxmax[2][batch_index]), valmax[batch_index],
max_label_str)
print ' X:', xx[batch_index].min(
), xx[batch_index].max(), norm(xx[batch_index])
# 4. Do backward pass to get gradient
top_name = layer_name_to_top_name(self.net, params.push_layer)
diffs = self.net.blobs[top_name].diff * 0
if not is_spatial:
# Promote bc -> bc01
diffs = diffs[:, :, np.newaxis, np.newaxis]
if self.settings.is_siamese and layer_format == 'siamese_batch_pair' and acts.shape[
0] == 2:
diffs[0, params.push_unit[0], params.push_unit[1],
params.push_unit[2]] = params.push_dir
elif self.settings.is_siamese and layer_format == 'siamese_batch_pair' and acts.shape[
0] == 1:
diffs[0, params.push_unit[0], params.push_unit[1],
params.push_unit[2]] = params.push_dir
else:
diffs[np.arange(params.batch_size), params.push_unit[0],
params.push_unit[1],
params.push_unit[2]] = params.push_dir
backout = self.net.backward_from_layer(
params.push_layer, diffs if is_spatial else diffs[:, :, 0, 0])
grad = backout['data'].copy()
reshaped_grad = np.reshape(grad, (params.batch_size, -1))
norm_grad = np.linalg.norm(reshaped_grad, axis=1)
min_grad = np.amin(reshaped_grad, axis=1)
max_grad = np.amax(reshaped_grad, axis=1)
for batch_index in range(params.batch_size):
print ' layer: %s, channel: %d, batch_index: %d min grad: %f, max grad: %f, norm grad: %f' % (
params.push_layer, params.push_unit[0], batch_index,
min_grad[batch_index], max_grad[batch_index],
norm_grad[batch_index])
if norm_grad[batch_index] == 0:
print ' batch_index: %d, Grad exactly 0, failed' % batch_index
results[
batch_index].meta_result = 'Metaresult: grad 0 failure'
break
# 5. Pick gradient update per learning policy
if params.lr_policy == 'progress01':
# Useful for softmax layer optimization, taper off near 1
late_prog = params.lr_params['late_prog_mult'] * (1 - obj)
desired_prog = np.amin(np.stack(
(np.repeat(params.lr_params['early_prog'],
params.batch_size), late_prog),
axis=1),
axis=1)
prog_lr = desired_prog / np.square(norm_grad)
lr = np.amin(np.stack((np.repeat(params.lr_params['max_lr'],
params.batch_size), prog_lr),
axis=1),
axis=1)
print ' entire batch, desired progress:', desired_prog, 'prog_lr:', prog_lr, 'lr:', lr
pred_prog = lr * np.sum(np.abs(reshaped_grad)**2, axis=-1)
elif params.lr_policy == 'progress':
# straight progress-based lr
prog_lr = params.lr_params['desired_prog'] / (norm_grad**2)
lr = np.amin(np.stack((np.repeat(params.lr_params['max_lr'],
params.batch_size), prog_lr),
axis=1),
axis=1)
print ' entire batch, desired progress:', params.lr_params[
'desired_prog'], 'prog_lr:', prog_lr, 'lr:', lr
pred_prog = lr * np.sum(np.abs(reshaped_grad)**2, axis=-1)
elif params.lr_policy == 'constant':
# constant fixed learning rate
lr = np.repeat(params.lr_params['lr'], params.batch_size)
else:
raise Exception('Unimplemented lr_policy')
for batch_index in range(params.batch_size):
# 6. Apply gradient update and regularizations
if ii < params.max_iter - 1:
# Skip gradient and regularizations on the very last step (so the above printed info is valid for the last step)
xx[batch_index] += lr[batch_index] * grad[batch_index]
xx[batch_index] *= (1 - params.decay)
channels = xx.shape[1]
if params.blur_every is not 0 and params.blur_radius > 0:
if params.blur_radius < .3:
print 'Warning: blur-radius of .3 or less works very poorly'
#raise Exception('blur-radius of .3 or less works very poorly')
if ii % params.blur_every == 0:
for channel in range(channels):
cimg = gaussian_filter(
xx[batch_index, channel],
params.blur_radius)
xx[batch_index, channel] = cimg
if params.small_val_percentile > 0:
small_entries = (abs(xx[batch_index]) < percentile(
abs(xx[batch_index]), params.small_val_percentile))
xx[batch_index] = xx[batch_index] - xx[
batch_index] * small_entries # e.g. set smallest 50% of xx to zero
if params.small_norm_percentile > 0:
pxnorms = norm(xx[batch_index, np.newaxis, :, :, :],
axis=1)
smallpx = pxnorms < percentile(
pxnorms, params.small_norm_percentile)
smallpx3 = tile(smallpx[:, newaxis, :, :],
(1, channels, 1, 1))
xx[batch_index, :, :, :] = xx[
batch_index, np.newaxis, :, :, :] - xx[
batch_index, np.newaxis, :, :, :] * smallpx3
if params.px_benefit_percentile > 0:
pred_0_benefit = grad[
batch_index,
np.newaxis, :, :, :] * -xx[batch_index,
np.newaxis, :, :, :]
px_benefit = pred_0_benefit.sum(
1) # sum over color channels
smallben = px_benefit < percentile(
px_benefit, params.px_benefit_percentile)
smallben3 = tile(smallben[:, newaxis, :, :],
(1, channels, 1, 1))
xx[batch_index, :, :, :] = xx[
batch_index, np.newaxis, :, :, :] - xx[
batch_index, np.newaxis, :, :, :] * smallben3
if params.px_abs_benefit_percentile > 0:
pred_0_benefit = grad[
batch_index,
np.newaxis, :, :, :] * -xx[batch_index,
np.newaxis, :, :, :]
px_benefit = pred_0_benefit.sum(
1) # sum over color channels
smallaben = abs(px_benefit) < percentile(
abs(px_benefit), params.px_abs_benefit_percentile)
smallaben3 = tile(smallaben[:, newaxis, :, :],
(1, channels, 1, 1))
xx[batch_index, :, :, :] = xx[
batch_index, np.newaxis, :, :, :] - xx[
batch_index, np.newaxis, :, :, :] * smallaben3
print ' timestamp:', datetime.datetime.now()
for batch_index in range(params.batch_size):
if results[batch_index].meta_result is None:
if results[batch_index].majority_obj is not None:
results[
batch_index].meta_result = 'batch_index: %d, Metaresult: majority success' % batch_index
else:
results[
batch_index].meta_result = 'batch_index: %d, Metaresult: majority failure' % batch_index
return xx, results, True
def handle_mouse_left_click(self, x, y, flags, param, panes, header_boxes,
buttons_boxes):
for pane_name, pane in panes.items():
if pane.j_begin <= x < pane.j_end and pane.i_begin <= y < pane.i_end:
if pane_name == 'caffevis_control': # layers list
# search for layer clicked on
for box_idx, box in enumerate(header_boxes):
start_x, end_x, start_y, end_y, text = box
if start_x <= x - pane.j_begin < end_x and start_y <= y - pane.i_begin <= end_y:
# print 'layers list clicked on layer %d (%s,%s)' % (box_idx, x, y)
self.layer_idx = box_idx
self.cursor_area = 'top'
self._ensure_valid_selected()
self.drawing_stale = True # Request redraw any time we handled the mouse
return
# print 'layers list clicked on (%s,%s)' % (x, y)
elif pane_name == 'caffevis_layers': # channels list
# print 'channels list clicked on (%s,%s)' % (x, y)
default_layer_name = self.get_default_layer_name()
default_top_name = layer_name_to_top_name(
self.net, default_layer_name)
tile_rows, tile_cols = self.net_blob_info[
default_top_name]['tiles_rc']
dy_per_channel = (pane.data.shape[0] +
1) / float(tile_rows)
dx_per_channel = (pane.data.shape[1] +
1) / float(tile_cols)
tile_x = int(((x - pane.j_begin) / dx_per_channel) + 1)
tile_y = int(((y - pane.i_begin) / dy_per_channel) + 1)
channel_id = (tile_y - 1) * tile_cols + (tile_x - 1)
self.selected_unit = channel_id
self.cursor_area = 'bottom'
self.validate_state_for_summary_only_patterns()
self._ensure_valid_selected()
self.drawing_stale = True # Request redraw any time we handled the mouse
return
elif pane_name == 'caffevis_buttons':
# print 'buttons!'
# search for layer clicked on
for box_idx, box in enumerate(buttons_boxes):
start_x, end_x, start_y, end_y, text = box
if start_x <= x - pane.j_begin < end_x and start_y <= y - pane.i_begin <= end_y:
# print 'DEBUG: pressed %s' % text
if text == 'File':
self.live_vis.input_updater.set_mode_static()
pass
elif text == 'Prev':
self.live_vis.input_updater.prev_image()
pass
elif text == 'Next':
self.live_vis.input_updater.next_image()
pass
elif text == 'Camera':
self.live_vis.input_updater.set_mode_cam()
pass
elif text == 'Modes':
pass
elif text == 'Activations':
self.set_show_back(False)
pass
elif text == 'Gradients':
self.set_show_back(True)
pass
elif text == 'Maximal Optimized':
with self.lock:
self.set_pattern_mode(
PatternMode.MAXIMAL_OPTIMIZED_IMAGE)
pass
elif text == 'Maximal Input':
with self.lock:
self.set_pattern_mode(
PatternMode.MAXIMAL_INPUT_IMAGE)
pass
elif text == 'Weights Histogram':
with self.lock:
self.set_pattern_mode(
PatternMode.WEIGHTS_HISTOGRAM)
pass
elif text == 'Activations Histogram':
with self.lock:
self.set_pattern_mode(
PatternMode.MAX_ACTIVATIONS_HISTOGRAM)
pass
elif text == 'Weights Correlation':
with self.lock:
self.set_pattern_mode(
PatternMode.WEIGHTS_CORRELATION)
pass
elif text == 'Activations Correlation':
with self.lock:
self.set_pattern_mode(
PatternMode.ACTIVATIONS_CORRELATION)
pass
elif text == 'Input Overlay':
pass
elif text == 'No Overlay':
self.set_input_overlay(InputOverlayOption.OFF)
pass
elif text == 'Over Active':
self.set_input_overlay(
InputOverlayOption.OVER_ACTIVE)
pass
elif text == 'Over Inactive':
self.set_input_overlay(
InputOverlayOption.OVER_INACTIVE)
pass
elif text == 'Backprop Modes':
pass
elif text == 'No Backprop':
self.set_back_mode(BackpropMode.OFF)
pass
elif text == 'Gradient':
self.set_back_mode(BackpropMode.GRAD)
pass
elif text == 'ZF Deconv':
self.set_back_mode(BackpropMode.DECONV_ZF)
pass
elif text == 'Guided Backprop':
self.set_back_mode(BackpropMode.DECONV_GB)
pass
elif text == 'Freeze Origin':
self.toggle_freeze_back_unit()
pass
elif text == 'Backprop Views':
pass
elif text == 'Raw':
self.set_back_view_option(
BackpropViewOption.RAW)
pass
elif text == 'Gray':
self.set_back_view_option(
BackpropViewOption.GRAY)
pass
elif text == 'Norm':
self.set_back_view_option(
BackpropViewOption.NORM)
pass
elif text == 'Blurred Norm':
self.set_back_view_option(
BackpropViewOption.NORM_BLUR)
pass
elif text == 'Sum > 0':
self.set_back_view_option(
BackpropViewOption.POS_SUM)
pass
elif text == 'Gradient Histogram':
self.set_back_view_option(
BackpropViewOption.HISTOGRAM)
pass
elif text == 'Help':
self.live_vis.toggle_help_mode()
pass
elif text == 'Quit':
self.live_vis.set_quit_flag()
pass
self._ensure_valid_selected()
self.drawing_stale = True
return
else:
# print "Clicked: %s - %s" % (x, y)
pass
break
pass
def main():
try:
# if model in command line change to it
if '--model' in sys.argv:
change_model_to_load(sys.argv[sys.argv.index('--model') + 1])
import settings
parser = argparse.ArgumentParser(
description=
'Script to find, with or without regularization, images that cause high or low activations of specific neurons in a network via numerical optimization. Settings are read from settings.py, overridden in settings_MODEL.py and settings_user.py, and may be further overridden on the command line.',
formatter_class=lambda prog: argparse.
ArgumentDefaultsHelpFormatter(prog, width=100))
# Network and data options
parser.add_argument('--caffe-root',
type=str,
default=settings.caffevis_caffe_root,
help='Path to caffe root directory.')
parser.add_argument('--deploy-proto',
type=str,
default=settings.caffevis_deploy_prototxt,
help='Path to caffe network prototxt.')
parser.add_argument('--net-weights',
type=str,
default=settings.caffevis_network_weights,
help='Path to caffe network weights.')
parser.add_argument(
'--channel-swap-to-rgb',
type=str,
default='(2,1,0)',
help=
'Permutation to apply to channels to change to RGB space for plotting. Hint: (0,1,2) if your network is trained for RGB, (2,1,0) if it is trained for BGR.'
)
parser.add_argument('--data-size',
type=str,
default='(227,227)',
help='Size of network input.')
#### FindParams
# Where to start
parser.add_argument(
'--start-at',
type=str,
default='mean_plus_rand',
choices=('mean_plus_rand', 'randu', 'mean'),
help='How to generate x0, the initial point used in optimization.')
parser.add_argument(
'--rand-seed',
type=int,
default=settings.optimize_image_rand_seed,
help=
'Random seed used for generating the start-at image (use different seeds to generate different images).'
)
parser.add_argument(
'--batch-size',
type=int,
default=settings.optimize_image_batch_size,
help=
'Batch size used for generating several images, each index will be used as random seed'
)
# What to optimize
parser.add_argument(
'--push-layers',
type=list,
default=settings.layers_to_output_in_offline_scripts,
help=
'Name of layers that contains the desired neuron whose value is optimized.'
)
parser.add_argument(
'--push-channel',
type=int,
default='130',
help=
'Channel number for desired neuron whose value is optimized (channel for conv, neuron index for FC).'
)
parser.add_argument(
'--push-spatial',
type=str,
default='None',
help=
'Which spatial location to push for conv layers. For FC layers, set this to None. For conv layers, set it to a tuple, e.g. when using `--push-layer conv5` on AlexNet, --push-spatial (6,6) will maximize the center unit of the 13x13 spatial grid.'
)
parser.add_argument(
'--push-dir',
type=float,
default=1,
help=
'Which direction to push the activation of the selected neuron, that is, the value used to begin backprop. For example, use 1 to maximize the selected neuron activation and -1 to minimize it.'
)
# Use regularization?
parser.add_argument('--decay',
type=float,
default=settings.optimize_image_decay,
help='Amount of L2 decay to use.')
parser.add_argument(
'--blur-radius',
type=float,
default=settings.optimize_image_blur_radius,
help=
'Radius in pixels of blur to apply after each BLUR_EVERY steps. If 0, perform no blurring. Blur sizes between 0 and 0.3 work poorly.'
)
parser.add_argument(
'--blur-every',
type=int,
default=settings.optimize_image_blue_every,
help='Blur every BLUR_EVERY steps. If 0, perform no blurring.')
parser.add_argument(
'--small-val-percentile',
type=float,
default=0,
help=
'Induce sparsity by setting pixels with absolute value under SMALL_VAL_PERCENTILE percentile to 0. Not discussed in paper. 0 to disable.'
)
parser.add_argument(
'--small-norm-percentile',
type=float,
default=0,
help=
'Induce sparsity by setting pixels with norm under SMALL_NORM_PERCENTILE percentile to 0. \\theta_{n_pct} from the paper. 0 to disable.'
)
parser.add_argument(
'--px-benefit-percentile',
type=float,
default=0,
help=
'Induce sparsity by setting pixels with contribution under PX_BENEFIT_PERCENTILE percentile to 0. Mentioned briefly in paper but not used. 0 to disable.'
)
parser.add_argument(
'--px-abs-benefit-percentile',
type=float,
default=0,
help=
'Induce sparsity by setting pixels with contribution under PX_BENEFIT_PERCENTILE percentile to 0. \\theta_{c_pct} from the paper. 0 to disable.'
)
# How much to optimize
parser.add_argument(
'--lr-policy',
type=str,
default=settings.optimize_image_lr_policy,
choices=LR_POLICY_CHOICES,
help='Learning rate policy. See description in lr-params.')
parser.add_argument(
'--lr-params',
type=str,
default=settings.optimize_image_lr_params,
help=
'Learning rate params, specified as a string that evalutes to a Python dict. Params that must be provided dependon which lr-policy is selected. The "constant" policy requires the "lr" key and uses the constant given learning rate. The "progress" policy requires the "max_lr" and "desired_prog" keys and scales the learning rate such that the objective function will change by an amount equal to DESIRED_PROG under a linear objective assumption, except the LR is limited to MAX_LR. The "progress01" policy requires the "max_lr", "early_prog", and "late_prog_mult" keys and is tuned for optimizing neurons with outputs in the [0,1] range, e.g. neurons on a softmax layer. Under this policy optimization slows down as the output approaches 1 (see code for details).'
)
parser.add_argument(
'--max-iters',
type=list,
default=settings.optimize_image_max_iters,
help='List of number of iterations of the optimization loop.')
# Where to save results
parser.add_argument(
'--output-prefix',
type=str,
default=settings.optimize_image_output_prefix,
help=
'Output path and filename prefix (default: outputs/%(p.push_layer)s/unit_%(p.push_channel)04d/opt_%(r.batch_index)03d)'
)
parser.add_argument(
'--brave',
action='store_true',
default=True,
help=
'Allow overwriting existing results files. Default: off, i.e. cowardly refuse to overwrite existing files.'
)
parser.add_argument(
'--skipbig',
action='store_true',
default=True,
help=
'Skip outputting large *info_big.pkl files (contains pickled version of x0, last x, best x, first x that attained max on the specified layer.'
)
parser.add_argument(
'--skipsmall',
action='store_true',
default=True,
help=
'Skip outputting small *info.pkl files (contains pickled version of..'
)
parser.add_argument(
'--model',
type=str,
default=None,
help=
'Name of the model you want to change to. This overwrites the settings made in files.'
)
args = parser.parse_args()
# Finish parsing args
lr_params = parse_and_validate_lr_params(parser, args.lr_policy,
args.lr_params)
push_spatial = parse_and_validate_push_spatial(parser,
args.push_spatial)
settings.caffevis_deploy_prototxt = args.deploy_proto
settings.caffevis_network_weights = args.net_weights
net, data_mean = load_network(settings)
# validate batch size
if settings.is_siamese and settings.siamese_network_format == 'siamese_batch_pair':
# currently, no batch support for siamese_batch_pair networks
# it can be added by simply handle the batch indexes properly, but it should be thoroughly tested
assert (settings.max_tracker_batch_size == 1)
current_data_shape = net.blobs['data'].shape
net.blobs['data'].reshape(args.batch_size, current_data_shape[1],
current_data_shape[2], current_data_shape[3])
net.reshape()
labels = None
if settings.caffevis_labels:
labels = read_label_file(settings.caffevis_labels)
if data_mean is not None:
if len(data_mean.shape) == 3:
batched_data_mean = np.repeat(data_mean[np.newaxis, :, :, :],
args.batch_size,
axis=0)
elif len(data_mean.shape) == 1:
data_mean = data_mean[np.newaxis, :, np.newaxis, np.newaxis]
batched_data_mean = np.tile(
data_mean, (args.batch_size, 1, current_data_shape[2],
current_data_shape[3]))
else:
batched_data_mean = data_mean
optimizer = GradientOptimizer(
settings,
net,
batched_data_mean,
labels=labels,
label_layers=settings.caffevis_label_layers,
channel_swap_to_rgb=settings.caffe_net_channel_swap)
if not args.push_layers:
print "ERROR: No layers to work on, please set layers_to_output_in_offline_scripts to list of layers"
return
# go over push layers
for count, push_layer in enumerate(args.push_layers):
top_name = layer_name_to_top_name(net, push_layer)
blob = net.blobs[top_name].data
is_spatial = (len(blob.shape) == 4)
channels = blob.shape[1]
# get layer definition
layer_def = settings._layer_name_to_record[push_layer]
if is_spatial:
push_spatial = (layer_def.filter[0] / 2,
layer_def.filter[1] / 2)
else:
push_spatial = (0, 0)
# if channels defined in settings file, use them
if settings.optimize_image_channels:
channels_list = settings.optimize_image_channels
else:
channels_list = range(channels)
# go over channels
for current_channel in channels_list:
params = FindParams(
start_at=args.start_at,
rand_seed=args.rand_seed,
batch_size=args.batch_size,
push_layer=push_layer,
push_channel=current_channel,
push_spatial=push_spatial,
push_dir=args.push_dir,
decay=args.decay,
blur_radius=args.blur_radius,
blur_every=args.blur_every,
small_val_percentile=args.small_val_percentile,
small_norm_percentile=args.small_norm_percentile,
px_benefit_percentile=args.px_benefit_percentile,
px_abs_benefit_percentile=args.px_abs_benefit_percentile,
lr_policy=args.lr_policy,
lr_params=lr_params,
max_iter=args.max_iters[count % len(args.max_iters)],
is_spatial=is_spatial,
)
optimizer.run_optimize(params,
prefix_template=args.output_prefix,
brave=args.brave,
skipbig=args.skipbig,
skipsmall=args.skipsmall)
finally:
clean_temp_file()
def main():
parser = get_parser()
args = parser.parse_args()
# Finish parsing args
lr_params = parse_and_validate_lr_params(parser, args.lr_policy,
args.lr_params)
push_spatial = parse_and_validate_push_spatial(parser, args.push_spatial)
settings.caffevis_deploy_prototxt = args.deploy_proto
settings.caffevis_network_weights = args.net_weights
net, data_mean = load_network(settings)
# validate batch size
if settings.is_siamese and settings.siamese_network_format == 'siamese_batch_pair':
# currently, no batch support for siamese_batch_pair networks
# it can be added by simply handle the batch indexes properly, but it should be thoroughly tested
assert (settings.max_tracker_batch_size == 1)
current_data_shape = net.blobs['data'].shape
net.blobs['data'].reshape(args.batch_size, current_data_shape[1],
current_data_shape[2], current_data_shape[3])
net.reshape()
labels = None
if settings.caffevis_labels:
labels = read_label_file(settings.caffevis_labels)
if data_mean is not None:
if len(data_mean.shape) == 3:
batched_data_mean = np.repeat(data_mean[np.newaxis, :, :, :],
args.batch_size,
axis=0)
elif len(data_mean.shape) == 1:
data_mean = data_mean[np.newaxis, :, np.newaxis, np.newaxis]
batched_data_mean = np.tile(
data_mean, (args.batch_size, 1, current_data_shape[2],
current_data_shape[3]))
else:
batched_data_mean = data_mean
optimizer = GradientOptimizer(
settings,
net,
batched_data_mean,
labels=labels,
label_layers=settings.caffevis_label_layers,
channel_swap_to_rgb=settings.caffe_net_channel_swap)
if not args.push_layers:
print "ERROR: No layers to work on, please set layers_to_output_in_offline_scripts to list of layers"
return
# go over push layers
for count, push_layer in enumerate(args.push_layers):
top_name = layer_name_to_top_name(net, push_layer)
blob = net.blobs[top_name].data
is_spatial = (len(blob.shape) == 4)
channels = blob.shape[1]
# get layer definition
layer_def = settings._layer_name_to_record[push_layer]
if is_spatial:
push_spatial = (layer_def.filter[0] / 2, layer_def.filter[1] / 2)
else:
push_spatial = (0, 0)
# if channels defined in settings file, use them
if settings.optimize_image_channels:
channels_list = settings.optimize_image_channels
else:
channels_list = range(channels)
# go over channels
for current_channel in channels_list:
params = FindParams(
start_at=args.start_at,
rand_seed=args.rand_seed,
batch_size=args.batch_size,
push_layer=push_layer,
push_channel=current_channel,
push_spatial=push_spatial,
push_dir=args.push_dir,
decay=args.decay,
blur_radius=args.blur_radius,
blur_every=args.blur_every,
small_val_percentile=args.small_val_percentile,
small_norm_percentile=args.small_norm_percentile,
px_benefit_percentile=args.px_benefit_percentile,
px_abs_benefit_percentile=args.px_abs_benefit_percentile,
lr_policy=args.lr_policy,
lr_params=lr_params,
max_iter=args.max_iters[count % len(args.max_iters)],
is_spatial=is_spatial,
)
optimizer.run_optimize(params,
prefix_template=args.output_prefix,
brave=args.brave,
skipbig=args.skipbig,
skipsmall=args.skipsmall)