def __init__(self, net, settings, bindings, live_vis):
self.lock = Lock() # State is accessed in multiple threads
self.settings = settings
self.bindings = bindings
self.net = net
self.live_vis = live_vis
self.fill_layers_list(net)
self.siamese_helper = SiameseHelper(settings.layers_list)
self._populate_net_blob_info(net)
self.layer_boost_indiv_choices = self.settings.caffevis_boost_indiv_choices # 0-1, 0 is noop
self.layer_boost_gamma_choices = self.settings.caffevis_boost_gamma_choices # 0-inf, 1 is noop
self.caffe_net_state = 'free' # 'free', 'proc', or 'draw'
self.extra_msg = ''
self.back_stale = True # back becomes stale whenever the last back diffs were not computed using the current backprop unit and method (bprop or deconv)
self.next_frame = None
self.next_label = None
self.next_filename = None
self.last_frame = None
self.jpgvis_to_load_key = None
self.last_key_at = 0
self.quit = False
self._reset_user_state()
def __init__(self,
settings,
net,
batched_data_mean,
labels=None,
label_layers=[],
channel_swap_to_rgb=None):
self.settings = settings
self.net = net
self.batched_data_mean = batched_data_mean
self.labels = labels if labels else [
'labels not provided' for ii in range(1000)
]
self.label_layers = label_layers if label_layers else list()
self.siamese_helper = SiameseHelper(self.settings.layers_list)
if channel_swap_to_rgb:
self.channel_swap_to_rgb = array(channel_swap_to_rgb)
else:
if settings._calculated_is_gray_model:
self.channel_swap_to_rgb = arange(1)
else:
self.channel_swap_to_rgb = arange(3) # Don't change order
# since we have a batch of same data mean images, we can just take the first
if batched_data_mean is not None:
self._data_mean_rgb_img = self.batched_data_mean[
0, self.channel_swap_to_rgb].transpose(
(1, 2, 0)) # Store as (227,227,3) in RGB order.
else:
self._data_mean_rgb_img = None
def get_headers(self):
headers = list()
for layer_def in self.settings.layers_list:
headers.append(SiameseHelper.get_header_from_layer_def(layer_def))
return headers
def get_layer_output_size(self, layer_def=None):
# if no layer specified, get current layer
if layer_def is None:
layer_def = self.get_current_layer_definition()
return SiameseHelper.get_layer_output_size(self.net,
self.settings.is_siamese,
layer_def,
self.siamese_view_mode)
def siamese_view_mode_has_two_images(self, layer_def=None):
'''
helper function which checks whether the input mode is two images, and the provided layer contains two layer names
:param layer: can be a single string layer name, or a pair of layer names
:return: True if both the input mode is BOTH_IMAGES and layer contains two layer names, False oherwise
'''
# if no layer specified, get current layer
if layer_def is None:
layer_def = self.get_current_layer_definition()
return SiameseHelper.siamese_view_mode_has_two_images(
layer_def, self.siamese_view_mode)
def backward_from_layer(self, net, backprop_layer_def, backprop_unit):
try:
return SiameseHelper.backward_from_layer(net, backprop_layer_def,
backprop_unit,
self.siamese_view_mode)
except AttributeError:
print 'ERROR: required bindings (backward_from_layer) not found! Try using the deconv-deep-vis-toolbox branch as described here: https://github.com/yosinski/deep-visualization-toolbox'
raise
except ValueError:
print "ERROR: probably impossible to backprop layer %s, ignoring to avoid crash" % (
str(backprop_layer_def['name/s']))
with self.lock:
self.back_enabled = False
def main():
parser = argparse.ArgumentParser(description='Loads a pickled NetMaxTracker and outputs one or more of {the patches of the image, a deconv patch, a backprop patch} associated with the maxes.')
parser.add_argument('--N', type = int, default = 9, help = 'Note and save top N activations.')
parser.add_argument('--gpu', action = 'store_true', default=settings.caffevis_mode_gpu, help = 'Use gpu.')
parser.add_argument('--do-maxes', action = 'store_true', default=settings.max_tracker_do_maxes, help = 'Output max patches.')
parser.add_argument('--do-deconv', action = 'store_true', default=settings.max_tracker_do_deconv, help = 'Output deconv patches.')
parser.add_argument('--do-deconv-norm', action = 'store_true', default=settings.max_tracker_do_deconv_norm, help = 'Output deconv-norm patches.')
parser.add_argument('--do-backprop', action = 'store_true', default=settings.max_tracker_do_backprop, help = 'Output backprop patches.')
parser.add_argument('--do-backprop-norm', action = 'store_true', default=settings.max_tracker_do_backprop_norm, help = 'Output backprop-norm patches.')
parser.add_argument('--do-info', action = 'store_true', default=settings.max_tracker_do_info, help = 'Output info file containing max filenames and labels.')
parser.add_argument('--idx-begin', type = int, default = None, help = 'Start at this unit (default: all units).')
parser.add_argument('--idx-end', type = int, default = None, help = 'End at this unit (default: all units).')
parser.add_argument('--nmt_pkl', type = str, default = os.path.join(settings.caffevis_outputs_dir, 'find_max_acts_output.pickled'), help = 'Which pickled NetMaxTracker to load.')
parser.add_argument('--net_prototxt', type = str, default = settings.caffevis_deploy_prototxt, help = 'network prototxt to load')
parser.add_argument('--net_weights', type = str, default = settings.caffevis_network_weights, help = 'network weights to load')
parser.add_argument('--datadir', type = str, default = settings.static_files_dir, help = 'directory to look for files in')
parser.add_argument('--filelist', type = str, default = settings.static_files_input_file, help = 'List of image files to consider, one per line. Must be the same filelist used to produce the NetMaxTracker!')
parser.add_argument('--outdir', type = str, default = settings.caffevis_outputs_dir, help = 'Which output directory to use. Files are output into outdir/layer/unit_%%04d/{maxes,deconv,backprop}_%%03d.png')
parser.add_argument('--search-min', action='store_true', default=False, help='Should we also search for minimal activations?')
args = parser.parse_args()
settings.caffevis_deploy_prototxt = args.net_prototxt
settings.caffevis_network_weights = args.net_weights
net, data_mean = load_network(settings)
# validate batch size
if settings.is_siamese and settings._calculated_siamese_network_format == 'siamese_batch_pair':
# currently, no batch support for siamese_batch_pair networks
# it can be added by simply handle the batch indexes properly, but it should be thoroughly tested
assert (settings.max_tracker_batch_size == 1)
# set network batch size
current_input_shape = net.blobs[net.inputs[0]].shape
current_input_shape[0] = settings.max_tracker_batch_size
net.blobs[net.inputs[0]].reshape(*current_input_shape)
net.reshape()
assert args.do_maxes or args.do_deconv or args.do_deconv_norm or args.do_backprop or args.do_backprop_norm or args.do_info, 'Specify at least one do_* option to output.'
siamese_helper = SiameseHelper(settings.layers_list)
nmt = load_max_tracker_from_file(args.nmt_pkl)
for layer_name in settings.layers_to_output_in_offline_scripts:
print 'Started work on layer %s' % (layer_name)
normalized_layer_name = siamese_helper.normalize_layer_name_for_max_tracker(layer_name)
mt = nmt.max_trackers[normalized_layer_name]
if args.idx_begin is None:
idx_begin = 0
if args.idx_end is None:
idx_end = mt.max_vals.shape[0]
with WithTimer('Saved %d images per unit for %s units %d:%d.' % (args.N, normalized_layer_name, idx_begin, idx_end)):
output_max_patches(settings, mt, net, normalized_layer_name, idx_begin, idx_end,
args.N, args.datadir, args.filelist, args.outdir, False,
(args.do_maxes, args.do_deconv, args.do_deconv_norm, args.do_backprop, args.do_backprop_norm, args.do_info))
if args.search_min:
output_max_patches(settings, mt, net, normalized_layer_name, idx_begin, idx_end,
args.N, args.datadir, args.filelist, args.outdir, True,
(args.do_maxes, args.do_deconv, args.do_deconv_norm, args.do_backprop, args.do_backprop_norm, args.do_info))
class GradientOptimizer(object):
'''Finds images by gradient.'''
def __init__(self,
settings,
net,
batched_data_mean,
labels=None,
label_layers=[],
channel_swap_to_rgb=None):
self.settings = settings
self.net = net
self.batched_data_mean = batched_data_mean
self.labels = labels if labels else [
'labels not provided' for ii in range(1000)
]
self.label_layers = label_layers if label_layers else list()
self.siamese_helper = SiameseHelper(self.settings.layers_list)
if channel_swap_to_rgb:
self.channel_swap_to_rgb = array(channel_swap_to_rgb)
else:
if settings._calculated_is_gray_model:
self.channel_swap_to_rgb = arange(1)
else:
self.channel_swap_to_rgb = arange(3) # Don't change order
# since we have a batch of same data mean images, we can just take the first
if batched_data_mean is not None:
self._data_mean_rgb_img = self.batched_data_mean[
0, self.channel_swap_to_rgb].transpose(
(1, 2, 0)) # Store as (227,227,3) in RGB order.
else:
self._data_mean_rgb_img = None
def run_optimize(self,
params,
prefix_template=None,
brave=False,
skipbig=False,
skipsmall=False):
'''All images are in Caffe format, e.g. shape (3, 227, 227) in BGR order.'''
print '\n\nStarting optimization with the following parameters:'
print params
x0 = self._get_x0(params)
xx, results, results_generated = self._optimize(
params, x0, prefix_template)
if results_generated:
self.save_results(params,
results,
prefix_template,
brave=brave,
skipbig=skipbig,
skipsmall=skipsmall)
print str([
results[batch_index].meta_result
for batch_index in range(params.batch_size)
])
return xx
def _get_x0(self, params):
'''Chooses a starting location'''
np.random.seed(params.rand_seed)
input_shape = self.net.blobs['data'].data.shape
if params.start_at == 'mean_plus_rand':
x0 = np.random.normal(0, 10, input_shape)
elif params.start_at == 'randu':
if self.batched_data_mean is not None:
x0 = uniform(0, 255, input_shape) - self.batched_data_mean
else:
x0 = uniform(0, 255, input_shape)
elif params.start_at == 'mean':
x0 = zeros(input_shape)
else:
raise Exception('Unknown start conditions: %s' % params.start_at)
return x0
def _optimize(self, params, x0, prefix_template):
xx = x0.copy()
results = [
FindResults(batch_index)
for batch_index in range(params.batch_size)
]
# check if all required outputs exist, in which case skip this optimization
all_outputs = [
self.generate_output_names(batch_index, params, results,
prefix_template,
self.settings.caffevis_outputs_dir)
for batch_index in range(params.batch_size)
]
relevant_outputs = [
best_X_name for [
best_X_name, best_Xpm_name, majority_X_name, majority_Xpm_name,
info_name, info_pkl_name, info_big_pkl_name
] in all_outputs
]
relevant_outputs_exist = [
os.path.exists(best_X_name) for best_X_name in relevant_outputs
]
if all(relevant_outputs_exist):
return xx, results, False
# Whether or not the unit being optimized corresponds to a label (e.g. one of the 1000 imagenet classes)
is_labeled_unit = params.push_layer in self.label_layers
# Sanity checks for conv vs FC layers
top_name = layer_name_to_top_name(self.net, params.push_layer)
data_shape = self.net.blobs[top_name].data.shape
assert len(data_shape) in (
2, 4
), 'Expected shape of length 2 (for FC) or 4 (for conv) layers but shape is %s' % repr(
data_shape)
is_spatial = (len(data_shape) == 4)
if is_spatial:
if params.push_spatial == (0, 0):
recommended_spatial = (data_shape[2] / 2, data_shape[3] / 2)
print(
'WARNING: A unit on a conv layer (%s) is being optimized, but push_spatial\n'
'is %s, so the upper-left unit in the channel is selected. To avoid edge\n'
'effects, you might want to optimize a non-edge unit instead, e.g. the center\n'
'unit by using `--push_spatial "%s"`\n' %
(params.push_layer, params.push_spatial,
recommended_spatial))
else:
assert params.push_spatial == (
0, 0), 'For FC layers, spatial indices must be (0,0)'
if is_labeled_unit:
# Sanity check
push_label = self.labels[params.push_unit[0]]
else:
push_label = None
old_obj = np.zeros(params.batch_size)
obj = np.zeros(params.batch_size)
for ii in range(params.max_iter):
# 0. Crop data
if self.batched_data_mean is not None:
xx = minimum(
255.0, maximum(0.0, xx + self.batched_data_mean)
) - self.batched_data_mean # Crop all values to [0,255]
else:
xx = minimum(255.0, maximum(0.0,
xx)) # Crop all values to [0,255]
# 1. Push data through net
out = self.net.forward_all(data=xx)
#shownet(net)
top_name = layer_name_to_top_name(self.net, params.push_layer)
acts = self.net.blobs[top_name].data
layer_format = self.siamese_helper.get_layer_format_by_layer_name(
params.push_layer)
# note: no batch support in 'siamese_batch_pair'
if self.settings.is_siamese and layer_format == 'siamese_batch_pair' and acts.shape[
0] == 2:
if not is_spatial:
# promote to 4D
acts = np.reshape(acts, (2, -1, 1, 1))
reshaped_acts = np.reshape(acts, (2, -1))
idxmax = unravel_index(reshaped_acts.argmax(axis=1),
acts.shape[1:])
valmax = reshaped_acts.max(axis=1)
# idxmax for fc or prob layer will be like: (batch,278, 0, 0)
# idxmax for conv layer will be like: (batch,37, 4, 37)
obj[0] = acts[0, params.push_unit[0], params.push_unit[1],
params.push_unit[2]]
elif self.settings.is_siamese and layer_format == 'siamese_batch_pair' and acts.shape[
0] == 1:
if not is_spatial:
# promote to 4D
acts = np.reshape(acts, (1, -1, 1, 1))
reshaped_acts = np.reshape(acts, (1, -1))
idxmax = unravel_index(reshaped_acts.argmax(axis=1),
acts.shape[1:])
valmax = reshaped_acts.max(axis=1)
# idxmax for fc or prob layer will be like: (batch,278, 0, 0)
# idxmax for conv layer will be like: (batch,37, 4, 37)
obj[0] = acts[0, params.push_unit[0], params.push_unit[1],
params.push_unit[2]]
else:
if not is_spatial:
# promote to 4D
acts = np.reshape(acts, (params.batch_size, -1, 1, 1))
reshaped_acts = np.reshape(acts, (params.batch_size, -1))
idxmax = unravel_index(reshaped_acts.argmax(axis=1),
acts.shape[1:])
valmax = reshaped_acts.max(axis=1)
# idxmax for fc or prob layer will be like: (batch,278, 0, 0)
# idxmax for conv layer will be like: (batch,37, 4, 37)
obj = acts[np.arange(params.batch_size), params.push_unit[0],
params.push_unit[1], params.push_unit[2]]
# 2. Update results
for batch_index in range(params.batch_size):
results[batch_index].update(params, ii, acts[batch_index], \
(idxmax[0][batch_index],idxmax[1][batch_index],idxmax[2][batch_index]), \
xx[batch_index], x0[batch_index])
# 3. Print progress
if ii > 0:
if params.lr_policy == 'progress':
print 'iter %-4d batch_index %d progress predicted: %g, actual: %g' % (
ii, batch_index, pred_prog[batch_index],
obj[batch_index] - old_obj[batch_index])
else:
print 'iter %-4d batch_index %d progress: %g' % (
ii, batch_index,
obj[batch_index] - old_obj[batch_index])
else:
print 'iter %d batch_index %d' % (ii, batch_index)
old_obj[batch_index] = obj[batch_index]
push_label_str = ('(%s)' %
push_label) if is_labeled_unit else ''
max_label_str = ('(%s)' % self.labels[idxmax[0][batch_index]]
) if is_labeled_unit else ''
print ' push unit: %16s with value %g %s' % (
params.push_unit, acts[batch_index][params.push_unit],
push_label_str)
print ' Max idx: %16s with value %g %s' % (
(idxmax[0][batch_index], idxmax[1][batch_index],
idxmax[2][batch_index]), valmax[batch_index],
max_label_str)
print ' X:', xx[batch_index].min(
), xx[batch_index].max(), norm(xx[batch_index])
# 4. Do backward pass to get gradient
top_name = layer_name_to_top_name(self.net, params.push_layer)
diffs = self.net.blobs[top_name].diff * 0
if not is_spatial:
# Promote bc -> bc01
diffs = diffs[:, :, np.newaxis, np.newaxis]
if self.settings.is_siamese and layer_format == 'siamese_batch_pair' and acts.shape[
0] == 2:
diffs[0, params.push_unit[0], params.push_unit[1],
params.push_unit[2]] = params.push_dir
elif self.settings.is_siamese and layer_format == 'siamese_batch_pair' and acts.shape[
0] == 1:
diffs[0, params.push_unit[0], params.push_unit[1],
params.push_unit[2]] = params.push_dir
else:
diffs[np.arange(params.batch_size), params.push_unit[0],
params.push_unit[1],
params.push_unit[2]] = params.push_dir
backout = self.net.backward_from_layer(
params.push_layer, diffs if is_spatial else diffs[:, :, 0, 0])
grad = backout['data'].copy()
reshaped_grad = np.reshape(grad, (params.batch_size, -1))
norm_grad = np.linalg.norm(reshaped_grad, axis=1)
min_grad = np.amin(reshaped_grad, axis=1)
max_grad = np.amax(reshaped_grad, axis=1)
for batch_index in range(params.batch_size):
print ' layer: %s, channel: %d, batch_index: %d min grad: %f, max grad: %f, norm grad: %f' % (
params.push_layer, params.push_unit[0], batch_index,
min_grad[batch_index], max_grad[batch_index],
norm_grad[batch_index])
if norm_grad[batch_index] == 0:
print ' batch_index: %d, Grad exactly 0, failed' % batch_index
results[
batch_index].meta_result = 'Metaresult: grad 0 failure'
break
# 5. Pick gradient update per learning policy
if params.lr_policy == 'progress01':
# Useful for softmax layer optimization, taper off near 1
late_prog = params.lr_params['late_prog_mult'] * (1 - obj)
desired_prog = np.amin(np.stack(
(np.repeat(params.lr_params['early_prog'],
params.batch_size), late_prog),
axis=1),
axis=1)
prog_lr = desired_prog / np.square(norm_grad)
lr = np.amin(np.stack((np.repeat(params.lr_params['max_lr'],
params.batch_size), prog_lr),
axis=1),
axis=1)
print ' entire batch, desired progress:', desired_prog, 'prog_lr:', prog_lr, 'lr:', lr
pred_prog = lr * np.sum(np.abs(reshaped_grad)**2, axis=-1)
elif params.lr_policy == 'progress':
# straight progress-based lr
prog_lr = params.lr_params['desired_prog'] / (norm_grad**2)
lr = np.amin(np.stack((np.repeat(params.lr_params['max_lr'],
params.batch_size), prog_lr),
axis=1),
axis=1)
print ' entire batch, desired progress:', params.lr_params[
'desired_prog'], 'prog_lr:', prog_lr, 'lr:', lr
pred_prog = lr * np.sum(np.abs(reshaped_grad)**2, axis=-1)
elif params.lr_policy == 'constant':
# constant fixed learning rate
lr = np.repeat(params.lr_params['lr'], params.batch_size)
else:
raise Exception('Unimplemented lr_policy')
for batch_index in range(params.batch_size):
# 6. Apply gradient update and regularizations
if ii < params.max_iter - 1:
# Skip gradient and regularizations on the very last step (so the above printed info is valid for the last step)
xx[batch_index] += lr[batch_index] * grad[batch_index]
xx[batch_index] *= (1 - params.decay)
channels = xx.shape[1]
if params.blur_every is not 0 and params.blur_radius > 0:
if params.blur_radius < .3:
print 'Warning: blur-radius of .3 or less works very poorly'
#raise Exception('blur-radius of .3 or less works very poorly')
if ii % params.blur_every == 0:
for channel in range(channels):
cimg = gaussian_filter(
xx[batch_index, channel],
params.blur_radius)
xx[batch_index, channel] = cimg
if params.small_val_percentile > 0:
small_entries = (abs(xx[batch_index]) < percentile(
abs(xx[batch_index]), params.small_val_percentile))
xx[batch_index] = xx[batch_index] - xx[
batch_index] * small_entries # e.g. set smallest 50% of xx to zero
if params.small_norm_percentile > 0:
pxnorms = norm(xx[batch_index, np.newaxis, :, :, :],
axis=1)
smallpx = pxnorms < percentile(
pxnorms, params.small_norm_percentile)
smallpx3 = tile(smallpx[:, newaxis, :, :],
(1, channels, 1, 1))
xx[batch_index, :, :, :] = xx[
batch_index, np.newaxis, :, :, :] - xx[
batch_index, np.newaxis, :, :, :] * smallpx3
if params.px_benefit_percentile > 0:
pred_0_benefit = grad[
batch_index,
np.newaxis, :, :, :] * -xx[batch_index,
np.newaxis, :, :, :]
px_benefit = pred_0_benefit.sum(
1) # sum over color channels
smallben = px_benefit < percentile(
px_benefit, params.px_benefit_percentile)
smallben3 = tile(smallben[:, newaxis, :, :],
(1, channels, 1, 1))
xx[batch_index, :, :, :] = xx[
batch_index, np.newaxis, :, :, :] - xx[
batch_index, np.newaxis, :, :, :] * smallben3
if params.px_abs_benefit_percentile > 0:
pred_0_benefit = grad[
batch_index,
np.newaxis, :, :, :] * -xx[batch_index,
np.newaxis, :, :, :]
px_benefit = pred_0_benefit.sum(
1) # sum over color channels
smallaben = abs(px_benefit) < percentile(
abs(px_benefit), params.px_abs_benefit_percentile)
smallaben3 = tile(smallaben[:, newaxis, :, :],
(1, channels, 1, 1))
xx[batch_index, :, :, :] = xx[
batch_index, np.newaxis, :, :, :] - xx[
batch_index, np.newaxis, :, :, :] * smallaben3
print ' timestamp:', datetime.datetime.now()
for batch_index in range(params.batch_size):
if results[batch_index].meta_result is None:
if results[batch_index].majority_obj is not None:
results[
batch_index].meta_result = 'batch_index: %d, Metaresult: majority success' % batch_index
else:
results[
batch_index].meta_result = 'batch_index: %d, Metaresult: majority failure' % batch_index
return xx, results, True
def find_selected_input_index(self, layer_name):
for item in self.settings.layers_list:
# if we have only a single layer, the header is the layer name
if item['format'] == 'normal' and item['name/s'] == layer_name:
return -1
# if we got a pair of layers
elif item['format'] == 'siamese_layer_pair':
if item['name/s'][0] == layer_name:
return 0
if item['name/s'][1] == layer_name:
return 1
elif item['format'] == 'siamese_batch_pair' and item[
'name/s'] == layer_name:
return 0
return -1
def generate_output_names(self, batch_index, params, results,
prefix_template, output_dir):
results_and_params = combine_dicts(
(('p.', params.__dict__), ('r.', results[batch_index].__dict__)))
prefix = prefix_template % results_and_params
prefix = os.path.join(output_dir, prefix)
if os.path.isdir(prefix):
if prefix[-1] != '/':
prefix += '/' # append slash for dir-only template
else:
dirname = os.path.dirname(prefix)
if dirname:
mkdir_p(dirname)
best_X_name = '%s_best_X.jpg' % prefix
best_Xpm_name = '%s_best_Xpm.jpg' % prefix
majority_X_name = '%s_majority_X.jpg' % prefix
majority_Xpm_name = '%s_majority_Xpm.jpg' % prefix
info_name = '%s_info.txt' % prefix
info_pkl_name = '%s_info.pkl' % prefix
info_big_pkl_name = '%s_info_big.pkl' % prefix
return [
best_X_name, best_Xpm_name, majority_X_name, majority_Xpm_name,
info_name, info_pkl_name, info_big_pkl_name
]
def save_results(self,
params,
results,
prefix_template,
brave=False,
skipbig=False,
skipsmall=False):
if prefix_template is None:
return
for batch_index in range(params.batch_size):
[best_X_name, best_Xpm_name, majority_X_name, majority_Xpm_name, info_name, info_pkl_name, info_big_pkl_name] = \
self.generate_output_names(batch_index, params, results, prefix_template, self.settings.caffevis_outputs_dir)
# Don't overwrite previous results
if os.path.exists(info_name) and not brave:
raise Exception('Cowardly refusing to overwrite ' + info_name)
output_majority = False
if output_majority:
# NOTE: this section wasn't tested after changes to code, so some minor index tweaking are in order
if results[batch_index].majority_xx is not None:
asimg = results[batch_index].majority_xx[
self.channel_swap_to_rgb].transpose((1, 2, 0))
saveimagescc(majority_X_name, asimg, 0)
saveimagesc(majority_Xpm_name,
asimg + self._data_mean_rgb_img) # PlusMean
if results[batch_index].best_xx is not None:
# results[batch_index].best_xx.shape is (6,224,224)
def save_output(data, channel_swap_to_rgb, best_X_image_name):
# , best_Xpm_image_name, data_mean_rgb_img):
asimg = data[channel_swap_to_rgb].transpose((1, 2, 0))
saveimagescc(best_X_image_name, asimg, 0)
# get center position, relative to layer, of best maximum
[temp_ii, temp_jj] = results[batch_index].idxmax[
results[batch_index].best_ii][1:3]
is_spatial = params.is_spatial
layer_name = params.push_layer
size_ii, size_jj = get_max_data_extent(self.net, self.settings,
layer_name, is_spatial)
data_size_ii, data_size_jj = self.net.blobs['data'].data.shape[
2:4]
[out_ii_start, out_ii_end, out_jj_start, out_jj_end, data_ii_start, data_ii_end, data_jj_start, data_jj_end] = \
compute_data_layer_focus_area(is_spatial, temp_ii, temp_jj, self.settings, layer_name, size_ii, size_jj, data_size_ii, data_size_jj)
selected_input_index = self.find_selected_input_index(
layer_name)
out_arr = extract_patch_from_image(
results[batch_index].best_xx, self.net,
selected_input_index, self.settings, data_ii_end,
data_ii_start, data_jj_end, data_jj_start, out_ii_end,
out_ii_start, out_jj_end, out_jj_start, size_ii, size_jj)
if self.settings.is_siamese:
save_output(
out_arr,
channel_swap_to_rgb=self.channel_swap_to_rgb[[0, 1,
2]],
best_X_image_name=best_X_name)
else:
save_output(out_arr,
channel_swap_to_rgb=self.channel_swap_to_rgb,
best_X_image_name=best_X_name)
if self.settings.optimize_image_generate_plus_mean:
out_arr_pm = extract_patch_from_image(
results[batch_index].best_xx + self.batched_data_mean,
self.net, selected_input_index, self.settings,
data_ii_end, data_ii_start, data_jj_end, data_jj_start,
out_ii_end, out_ii_start, out_jj_end, out_jj_start,
size_ii, size_jj)
if self.settings.is_siamese:
save_output(
out_arr_pm,
channel_swap_to_rgb=self.channel_swap_to_rgb[[
0, 1, 2
]],
best_X_image_name=best_Xpm_name)
else:
save_output(
out_arr_pm,
channel_swap_to_rgb=self.channel_swap_to_rgb,
best_X_image_name=best_Xpm_name)
with open(info_name, 'w') as ff:
print >> ff, params
print >> ff
print >> ff, results[batch_index]
if not skipbig:
with open(info_big_pkl_name, 'w') as ff:
pickle.dump((params, results[batch_index]),
ff,
protocol=-1)
if not skipsmall:
results[batch_index].trim_arrays()
with open(info_pkl_name, 'w') as ff:
pickle.dump((params, results[batch_index]),
ff,
protocol=-1)
def convert_image_pair_to_network_input_format(self, settings, frame_pair,
resize_shape):
return SiameseHelper.convert_image_pair_to_network_input_format(
frame_pair, resize_shape, settings.siamese_input_mode)
class CaffeVisAppState(object):
'''State of CaffeVis app.'''
def __init__(self, net, settings, bindings, live_vis):
self.lock = Lock() # State is accessed in multiple threads
self.settings = settings
self.bindings = bindings
self.net = net
self.live_vis = live_vis
self.fill_layers_list(net)
self.siamese_helper = SiameseHelper(settings.layers_list)
self._populate_net_blob_info(net)
self.layer_boost_indiv_choices = self.settings.caffevis_boost_indiv_choices # 0-1, 0 is noop
self.layer_boost_gamma_choices = self.settings.caffevis_boost_gamma_choices # 0-inf, 1 is noop
self.caffe_net_state = 'free' # 'free', 'proc', or 'draw'
self.extra_msg = ''
self.back_stale = True # back becomes stale whenever the last back diffs were not computed using the current backprop unit and method (bprop or deconv)
self.next_frame = None
self.next_label = None
self.next_filename = None
self.last_frame = None
self.jpgvis_to_load_key = None
self.last_key_at = 0
self.quit = False
self._reset_user_state()
def _populate_net_blob_info(self, net):
'''For each blob, save the number of filters and precompute
tile arrangement (needed by CaffeVisAppState to handle keyboard navigation).
'''
self.net_blob_info = {}
for key in net.blobs.keys():
self.net_blob_info[key] = {}
# Conv example: (1, 96, 55, 55)
# FC example: (1, 1000)
blob_shape = net.blobs[key].data.shape
# handle case when output is a single number per image in the batch
if (len(blob_shape) == 1):
blob_shape = (blob_shape[0], 1)
self.net_blob_info[key]['isconv'] = (len(blob_shape) == 4)
self.net_blob_info[key]['data_shape'] = blob_shape[
1:] # Chop off batch size
self.net_blob_info[key]['n_tiles'] = blob_shape[1]
self.net_blob_info[key]['tiles_rc'] = get_tiles_height_width_ratio(
blob_shape[1], self.settings.caffevis_layers_aspect_ratio)
self.net_blob_info[key]['tile_rows'] = self.net_blob_info[key][
'tiles_rc'][0]
self.net_blob_info[key]['tile_cols'] = self.net_blob_info[key][
'tiles_rc'][1]
def get_headers(self):
headers = list()
for layer_def in self.settings.layers_list:
headers.append(SiameseHelper.get_header_from_layer_def(layer_def))
return headers
def _reset_user_state(self):
self.show_maximal_score = True
self.input_overlay_option = InputOverlayOption.OFF
self.layer_idx = 0
self.layer_boost_indiv_idx = self.settings.caffevis_boost_indiv_default_idx
self.layer_boost_indiv = self.layer_boost_indiv_choices[
self.layer_boost_indiv_idx]
self.layer_boost_gamma_idx = self.settings.caffevis_boost_gamma_default_idx
self.layer_boost_gamma = self.layer_boost_gamma_choices[
self.layer_boost_gamma_idx]
self.cursor_area = 'top' # 'top' or 'bottom'
self.selected_unit = 0
self.siamese_view_mode = SiameseViewMode.BOTH_IMAGES
# Which layer and unit (or channel) to use for backprop
self.backprop_layer_idx = self.layer_idx
self.backprop_unit = self.selected_unit
self.backprop_selection_frozen = False # If false, backprop unit tracks selected unit
self.backprop_siamese_view_mode = SiameseViewMode.BOTH_IMAGES
self.back_enabled = False
self.back_mode = BackpropMode.OFF
self.back_view_option = BackpropViewOption.RAW
self.color_map_option = ColorMapOption.JET
self.pattern_mode = PatternMode.OFF # type of patterns to show instead of activations in layers pane: maximal optimized image, maximal input image, maximal histogram, off
self.pattern_first_only = True # should we load only the first pattern image for each neuron, or all the relevant images per neuron
self.layers_pane_zoom_mode = 0 # 0: off, 1: zoom selected (and show pref in small pane), 2: zoom backprop
self.layers_show_back = False # False: show forward activations. True: show backward diffs
self.show_label_predictions = self.settings.caffevis_init_show_label_predictions
self.show_unit_jpgs = self.settings.caffevis_init_show_unit_jpgs
self.drawing_stale = True
kh, _ = self.bindings.get_key_help('help_mode')
self.extra_msg = '%s for help' % kh[0]
def handle_key(self, key):
#print 'Ignoring key:', key
if key == -1:
return key
with self.lock:
key_handled = True
self.last_key_at = time.time()
tag = self.bindings.get_tag(key)
if tag == 'reset_state':
self._reset_user_state()
elif tag == 'sel_layer_left':
#hh,ww = self.tiles_height_width
#self.selected_unit = self.selected_unit % ww # equivalent to scrolling all the way to the top row
#self.cursor_area = 'top' # Then to the control pane
self.layer_idx = max(0, self.layer_idx - 1)
elif tag == 'sel_layer_right':
#hh,ww = self.tiles_height_width
#self.selected_unit = self.selected_unit % ww # equivalent to scrolling all the way to the top row
#self.cursor_area = 'top' # Then to the control pane
self.layer_idx = min(
len(self.settings.layers_list) - 1, self.layer_idx + 1)
elif tag == 'sel_left':
self.move_selection('left')
elif tag == 'sel_right':
self.move_selection('right')
elif tag == 'sel_down':
self.move_selection('down')
elif tag == 'sel_up':
self.move_selection('up')
elif tag == 'sel_left_fast':
self.move_selection('left',
self.settings.caffevis_fast_move_dist)
elif tag == 'sel_right_fast':
self.move_selection('right',
self.settings.caffevis_fast_move_dist)
elif tag == 'sel_down_fast':
self.move_selection('down',
self.settings.caffevis_fast_move_dist)
elif tag == 'sel_up_fast':
self.move_selection('up',
self.settings.caffevis_fast_move_dist)
elif tag == 'boost_individual':
self.layer_boost_indiv_idx = (
self.layer_boost_indiv_idx + 1) % len(
self.layer_boost_indiv_choices)
self.layer_boost_indiv = self.layer_boost_indiv_choices[
self.layer_boost_indiv_idx]
elif tag == 'boost_gamma':
self.layer_boost_gamma_idx = (
self.layer_boost_gamma_idx + 1) % len(
self.layer_boost_gamma_choices)
self.layer_boost_gamma = self.layer_boost_gamma_choices[
self.layer_boost_gamma_idx]
elif tag == 'next_pattern_mode':
self.set_pattern_mode(
(self.pattern_mode + 1) % PatternMode.NUMBER_OF_MODES)
elif tag == 'prev_pattern_mode':
self.set_pattern_mode(
(self.pattern_mode - 1 + PatternMode.NUMBER_OF_MODES) %
PatternMode.NUMBER_OF_MODES)
elif tag == 'pattern_first_only':
self.pattern_first_only = not self.pattern_first_only
elif tag == 'show_back':
self.set_show_back(not self.layers_show_back)
elif tag == 'next_ez_back_mode_loop':
self.set_back_mode(
(self.back_mode + 1) % BackpropMode.NUMBER_OF_MODES)
elif tag == 'prev_ez_back_mode_loop':
self.set_back_mode(
(self.back_mode - 1 + BackpropMode.NUMBER_OF_MODES) %
BackpropMode.NUMBER_OF_MODES)
elif tag == 'next_back_view_option':
self.set_back_view_option((self.back_view_option + 1) %
BackpropViewOption.NUMBER_OF_OPTIONS)
elif tag == 'prev_back_view_option':
self.set_back_view_option(
(self.back_view_option - 1 +
BackpropViewOption.NUMBER_OF_OPTIONS) %
BackpropViewOption.NUMBER_OF_OPTIONS)
elif tag == 'next_color_map':
self.color_map_option = (self.color_map_option +
1) % ColorMapOption.NUMBER_OF_OPTIONS
elif tag == 'prev_color_map':
self.color_map_option = (self.color_map_option - 1 +
ColorMapOption.NUMBER_OF_OPTIONS
) % ColorMapOption.NUMBER_OF_OPTIONS
elif tag == 'freeze_back_unit':
self.toggle_freeze_back_unit()
elif tag == 'zoom_mode':
self.layers_pane_zoom_mode = (self.layers_pane_zoom_mode +
1) % 3
if self.layers_pane_zoom_mode == 2 and not self.back_enabled:
# Skip zoom into backprop pane when backprop is off
self.layers_pane_zoom_mode = 0
elif tag == 'toggle_label_predictions':
self.show_label_predictions = not self.show_label_predictions
elif tag == 'toggle_unit_jpgs':
self.show_unit_jpgs = not self.show_unit_jpgs
elif tag == 'siamese_view_mode':
self.siamese_view_mode = (self.siamese_view_mode +
1) % SiameseViewMode.NUMBER_OF_MODES
elif tag == 'toggle_maximal_score':
self.show_maximal_score = not self.show_maximal_score
elif tag == 'next_input_overlay':
self.set_input_overlay((self.input_overlay_option + 1) %
InputOverlayOption.NUMBER_OF_OPTIONS)
elif tag == 'prev_input_overlay':
self.set_input_overlay((self.input_overlay_option - 1 +
InputOverlayOption.NUMBER_OF_OPTIONS) %
InputOverlayOption.NUMBER_OF_OPTIONS)
else:
key_handled = False
self._ensure_valid_selected()
self.drawing_stale = key_handled # Request redraw any time we handled the key
return (None if key_handled else key)
def handle_mouse_left_click(self, x, y, flags, param, panes, header_boxes,
buttons_boxes):
for pane_name, pane in panes.items():
if pane.j_begin <= x < pane.j_end and pane.i_begin <= y < pane.i_end:
if pane_name == 'caffevis_control': # layers list
# search for layer clicked on
for box_idx, box in enumerate(header_boxes):
start_x, end_x, start_y, end_y, text = box
if start_x <= x - pane.j_begin < end_x and start_y <= y - pane.i_begin <= end_y:
# print 'layers list clicked on layer %d (%s,%s)' % (box_idx, x, y)
self.layer_idx = box_idx
self.cursor_area = 'top'
self._ensure_valid_selected()
self.drawing_stale = True # Request redraw any time we handled the mouse
return
# print 'layers list clicked on (%s,%s)' % (x, y)
elif pane_name == 'caffevis_layers': # channels list
# print 'channels list clicked on (%s,%s)' % (x, y)
default_layer_name = self.get_default_layer_name()
default_top_name = layer_name_to_top_name(
self.net, default_layer_name)
tile_rows, tile_cols = self.net_blob_info[
default_top_name]['tiles_rc']
dy_per_channel = (pane.data.shape[0] +
1) / float(tile_rows)
dx_per_channel = (pane.data.shape[1] +
1) / float(tile_cols)
tile_x = int(((x - pane.j_begin) / dx_per_channel) + 1)
tile_y = int(((y - pane.i_begin) / dy_per_channel) + 1)
channel_id = (tile_y - 1) * tile_cols + (tile_x - 1)
self.selected_unit = channel_id
self.cursor_area = 'bottom'
self.validate_state_for_summary_only_patterns()
self._ensure_valid_selected()
self.drawing_stale = True # Request redraw any time we handled the mouse
return
elif pane_name == 'caffevis_buttons':
# print 'buttons!'
# search for layer clicked on
for box_idx, box in enumerate(buttons_boxes):
start_x, end_x, start_y, end_y, text = box
if start_x <= x - pane.j_begin < end_x and start_y <= y - pane.i_begin <= end_y:
# print 'DEBUG: pressed %s' % text
if text == 'File':
self.live_vis.input_updater.set_mode_static()
pass
elif text == 'Prev':
self.live_vis.input_updater.prev_image()
pass
elif text == 'Next':
self.live_vis.input_updater.next_image()
pass
elif text == 'Camera':
self.live_vis.input_updater.set_mode_cam()
pass
elif text == 'Modes':
pass
elif text == 'Activations':
self.set_show_back(False)
pass
elif text == 'Gradients':
self.set_show_back(True)
pass
elif text == 'Maximal Optimized':
with self.lock:
self.set_pattern_mode(
PatternMode.MAXIMAL_OPTIMIZED_IMAGE)
pass
elif text == 'Maximal Input':
with self.lock:
self.set_pattern_mode(
PatternMode.MAXIMAL_INPUT_IMAGE)
pass
elif text == 'Weights Histogram':
with self.lock:
self.set_pattern_mode(
PatternMode.WEIGHTS_HISTOGRAM)
pass
elif text == 'Activations Histogram':
with self.lock:
self.set_pattern_mode(
PatternMode.MAX_ACTIVATIONS_HISTOGRAM)
pass
elif text == 'Weights Correlation':
with self.lock:
self.set_pattern_mode(
PatternMode.WEIGHTS_CORRELATION)
pass
elif text == 'Activations Correlation':
with self.lock:
self.set_pattern_mode(
PatternMode.ACTIVATIONS_CORRELATION)
pass
elif text == 'Input Overlay':
pass
elif text == 'No Overlay':
self.set_input_overlay(InputOverlayOption.OFF)
pass
elif text == 'Over Active':
self.set_input_overlay(
InputOverlayOption.OVER_ACTIVE)
pass
elif text == 'Over Inactive':
self.set_input_overlay(
InputOverlayOption.OVER_INACTIVE)
pass
elif text == 'Backprop Modes':
pass
elif text == 'No Backprop':
self.set_back_mode(BackpropMode.OFF)
pass
elif text == 'Gradient':
self.set_back_mode(BackpropMode.GRAD)
pass
elif text == 'ZF Deconv':
self.set_back_mode(BackpropMode.DECONV_ZF)
pass
elif text == 'Guided Backprop':
self.set_back_mode(BackpropMode.DECONV_GB)
pass
elif text == 'Freeze Origin':
self.toggle_freeze_back_unit()
pass
elif text == 'Backprop Views':
pass
elif text == 'Raw':
self.set_back_view_option(
BackpropViewOption.RAW)
pass
elif text == 'Gray':
self.set_back_view_option(
BackpropViewOption.GRAY)
pass
elif text == 'Norm':
self.set_back_view_option(
BackpropViewOption.NORM)
pass
elif text == 'Blurred Norm':
self.set_back_view_option(
BackpropViewOption.NORM_BLUR)
pass
elif text == 'Sum > 0':
self.set_back_view_option(
BackpropViewOption.POS_SUM)
pass
elif text == 'Gradient Histogram':
self.set_back_view_option(
BackpropViewOption.HISTOGRAM)
pass
elif text == 'Help':
self.live_vis.toggle_help_mode()
pass
elif text == 'Quit':
self.live_vis.set_quit_flag()
pass
self._ensure_valid_selected()
self.drawing_stale = True
return
else:
# print "Clicked: %s - %s" % (x, y)
pass
break
pass
def redraw_needed(self):
with self.lock:
return self.drawing_stale
def get_current_layer_definition(self):
return self.settings.layers_list[self.layer_idx]
def get_current_backprop_layer_definition(self):
return self.settings.layers_list[self.backprop_layer_idx]
def get_single_selected_data_blob(self, net, layer_def=None):
# if no layer specified, get current layer
if layer_def is None:
layer_def = self.get_current_layer_definition()
return self.siamese_helper.get_single_selected_data_blob(
net, layer_def, self.siamese_view_mode)
def get_single_selected_diff_blob(self, net, layer_def=None):
# if no layer specified, get current layer
if layer_def is None:
layer_def = self.get_current_layer_definition()
return self.siamese_helper.get_single_selected_diff_blob(
net, layer_def, self.siamese_view_mode)
def get_siamese_selected_data_blobs(self, net, layer_def=None):
# if no layer specified, get current layer
if layer_def is None:
layer_def = self.get_current_layer_definition()
return self.siamese_helper.get_siamese_selected_data_blobs(
net, layer_def, self.siamese_view_mode)
def get_siamese_selected_diff_blobs(self, net, layer_def=None):
# if no layer specified, get current layer
if layer_def is None:
layer_def = self.get_current_layer_definition()
return self.siamese_helper.get_siamese_selected_diff_blobs(
net, layer_def, self.siamese_view_mode)
def backward_from_layer(self, net, backprop_layer_def, backprop_unit):
try:
return SiameseHelper.backward_from_layer(net, backprop_layer_def,
backprop_unit,
self.siamese_view_mode)
except AttributeError:
print 'ERROR: required bindings (backward_from_layer) not found! Try using the deconv-deep-vis-toolbox branch as described here: https://github.com/yosinski/deep-visualization-toolbox'
raise
except ValueError:
print "ERROR: probably impossible to backprop layer %s, ignoring to avoid crash" % (
str(backprop_layer_def['name/s']))
with self.lock:
self.back_enabled = False
def deconv_from_layer(self, net, backprop_layer_def, backprop_unit,
deconv_type):
try:
return SiameseHelper.deconv_from_layer(net, backprop_layer_def,
backprop_unit,
self.siamese_view_mode,
deconv_type)
except AttributeError:
print 'ERROR: required bindings (deconv_from_layer) not found! Try using the deconv-deep-vis-toolbox branch as described here: https://github.com/yosinski/deep-visualization-toolbox'
raise
except ValueError:
print "ERROR: probably impossible to deconv layer %s, ignoring to avoid crash" % (
str(backprop_layer_def['name/s']))
with self.lock:
self.back_enabled = False
def get_default_layer_name(self, layer_def=None):
# if no layer specified, get current layer
if layer_def is None:
layer_def = self.get_current_layer_definition()
return self.siamese_helper.get_default_layer_name(layer_def)
def siamese_view_mode_has_two_images(self, layer_def=None):
'''
helper function which checks whether the input mode is two images, and the provided layer contains two layer names
:param layer: can be a single string layer name, or a pair of layer names
:return: True if both the input mode is BOTH_IMAGES and layer contains two layer names, False oherwise
'''
# if no layer specified, get current layer
if layer_def is None:
layer_def = self.get_current_layer_definition()
return SiameseHelper.siamese_view_mode_has_two_images(
layer_def, self.siamese_view_mode)
def move_selection(self, direction, dist=1):
default_layer_name = self.get_default_layer_name()
default_top_name = layer_name_to_top_name(self.net, default_layer_name)
if direction == 'left':
if self.cursor_area == 'top':
self.layer_idx = max(0, self.layer_idx - dist)
else:
self.selected_unit -= dist
elif direction == 'right':
if self.cursor_area == 'top':
self.layer_idx = min(
len(self.settings.layers_list) - 1, self.layer_idx + dist)
else:
self.selected_unit += dist
elif direction == 'down':
if self.cursor_area == 'top':
self.cursor_area = 'bottom'
else:
self.selected_unit += self.net_blob_info[default_top_name][
'tile_cols'] * dist
elif direction == 'up':
if self.cursor_area == 'top':
pass
else:
self.selected_unit -= self.net_blob_info[default_top_name][
'tile_cols'] * dist
if self.selected_unit < 0:
self.selected_unit += self.net_blob_info[default_top_name][
'tile_cols']
self.cursor_area = 'top'
self.validate_state_for_summary_only_patterns()
def _ensure_valid_selected(self):
default_layer_name = self.get_default_layer_name()
default_top_name = layer_name_to_top_name(self.net, default_layer_name)
n_tiles = self.net_blob_info[default_top_name]['n_tiles']
# Forward selection
self.selected_unit = max(0, self.selected_unit)
self.selected_unit = min(n_tiles - 1, self.selected_unit)
# Backward selection
if not self.backprop_selection_frozen:
# If backprop_selection is not frozen, backprop layer/unit follows selected unit
if not (self.backprop_layer_idx == self.layer_idx
and self.backprop_unit == self.selected_unit and
self.backprop_siamese_view_mode == self.siamese_view_mode):
self.backprop_layer_idx = self.layer_idx
self.backprop_unit = self.selected_unit
self.backprop_siamese_view_mode = self.siamese_view_mode
self.back_stale = True # If there is any change, back diffs are now stale
def fill_layers_list(self, net):
# if layers list is empty, fill it with layer names
if not self.settings.layers_list:
# go over layers
self.settings.layers_list = []
for layer_name in list(net._layer_names):
# skip inplace layers
if len(net.top_names[layer_name]) == 1 and len(
net.bottom_names[layer_name]) == 1 and net.top_names[
layer_name][0] == net.bottom_names[layer_name][0]:
continue
self.settings.layers_list.append({
'format': 'normal',
'name/s': layer_name
})
# filter layers if needed
if hasattr(self.settings, 'caffevis_filter_layers'):
for layer_def in self.settings.layers_list:
if self.settings.caffevis_filter_layers(layer_def['name/s']):
print ' Layer filtered out by caffevis_filter_layers: %s' % str(
layer_def['name/s'])
self.settings.layers_list = filter(
lambda layer_def: not self.settings.caffevis_filter_layers(
layer_def['name/s']), self.settings.layers_list)
pass
def gray_to_colormap(self, gray_image):
if self.color_map_option == ColorMapOption.GRAY:
return gray_image
elif self.color_map_option == ColorMapOption.JET:
return gray_to_colormap('jet', gray_image)
elif self.color_map_option == ColorMapOption.PLASMA:
return gray_to_colormap('plasma', gray_image)
def convert_image_pair_to_network_input_format(self, settings, frame_pair,
resize_shape):
return SiameseHelper.convert_image_pair_to_network_input_format(
frame_pair, resize_shape, settings.siamese_input_mode)
def get_layer_output_size(self, layer_def=None):
# if no layer specified, get current layer
if layer_def is None:
layer_def = self.get_current_layer_definition()
return SiameseHelper.get_layer_output_size(self.net,
self.settings.is_siamese,
layer_def,
self.siamese_view_mode)
def get_layer_output_size_string(self, layer_def=None):
layer_output_size = self.get_layer_output_size(layer_def)
if len(layer_output_size) == 1:
return '(%d)' % (layer_output_size[0])
elif len(layer_output_size) == 2:
return '(%d,%d)' % (layer_output_size[0], layer_output_size[1])
elif len(layer_output_size) == 3:
return '(%d,%d,%d)' % (layer_output_size[0], layer_output_size[1],
layer_output_size[2])
else:
return str(layer_output_size)
def validate_state_for_summary_only_patterns(self):
if self.pattern_mode in [
PatternMode.ACTIVATIONS_CORRELATION,
PatternMode.WEIGHTS_CORRELATION
]:
self.cursor_area = 'top'
def set_pattern_mode(self, pattern_mode):
self.pattern_mode = pattern_mode
self.validate_state_for_summary_only_patterns()
def set_show_back(self, show_back):
# If in pattern mode: switch to fwd/back. Else toggle fwd/back mode
if self.pattern_mode != PatternMode.OFF:
self.set_pattern_mode(PatternMode.OFF)
self.layers_show_back = show_back
if self.layers_show_back:
if not self.back_enabled:
if self.back_mode == BackpropMode.OFF:
self.back_mode = BackpropMode.GRAD
self.back_enabled = True
self.back_stale = True
def set_input_overlay(self, input_overlay):
self.input_overlay_option = input_overlay
def set_back_mode(self, back_mode):
self.back_mode = back_mode
self.back_enabled = (self.back_mode != BackpropMode.OFF)
self.back_stale = True
def toggle_freeze_back_unit(self):
# Freeze selected layer/unit as backprop unit
self.backprop_selection_frozen = not self.backprop_selection_frozen
if self.backprop_selection_frozen:
# Grap layer/selected_unit upon transition from non-frozen -> frozen
self.backprop_layer_idx = self.layer_idx
self.backprop_unit = self.selected_unit
self.backprop_siamese_view_mode = self.siamese_view_mode
def set_back_view_option(self, back_view_option):
self.back_view_option = back_view_option