def _draw_layer_pane(self, pane):
'''Returns the data shown in highres format, b01c order.'''
if self.state.layers_show_back:
layer_dat_3D = self.net.blobs[self.state.layer].diff[0]
else:
layer_dat_3D = self.net.blobs[self.state.layer].data[0]
# Promote FC layers with shape (n) to have shape (n,1,1)
if len(layer_dat_3D.shape) == 1:
layer_dat_3D = layer_dat_3D[:,np.newaxis,np.newaxis]
n_tiles = layer_dat_3D.shape[0]
tile_rows,tile_cols = self.net_layer_info[self.state.layer]['tiles_rc']
display_3D_highres = None
if self.state.pattern_mode:
# Show desired patterns loaded from disk
load_layer = self.state.layer
if self.settings.caffevis_jpgvis_remap and self.state.layer in self.settings.caffevis_jpgvis_remap:
load_layer = self.settings.caffevis_jpgvis_remap[self.state.layer]
if self.settings.caffevis_jpgvis_layers and load_layer in self.settings.caffevis_jpgvis_layers:
jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
'regularized_opt', load_layer, 'whole_layer.jpg')
# Get highres version
#cache_before = str(self.img_cache)
display_3D_highres = self.img_cache.get((jpg_path, 'whole'), None)
#else:
# display_3D_highres = None
if display_3D_highres is None:
try:
with WithTimer('CaffeVisApp:load_sprite_image', quiet = self.debug_level < 1):
display_3D_highres = load_square_sprite_image(jpg_path, n_sprites = n_tiles)
except IOError:
# File does not exist, so just display disabled.
pass
else:
self.img_cache.set((jpg_path, 'whole'), display_3D_highres)
#cache_after = str(self.img_cache)
#print 'Cache was / is:\n %s\n %s' % (cache_before, cache_after)
if display_3D_highres is not None:
# Get lowres version, maybe. Assume we want at least one pixel for selection border.
row_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[1]) / (pane.data.shape[0] / tile_rows - 2)))
col_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[2]) / (pane.data.shape[1] / tile_cols - 2)))
ds = max(row_downsamp_factor, col_downsamp_factor)
if ds > 1:
#print 'Downsampling by', ds
display_3D = display_3D_highres[:,::ds,::ds,:]
else:
display_3D = display_3D_highres
else:
display_3D = layer_dat_3D * 0 # nothing to show
else:
# Show data from network (activations or diffs)
if self.state.layers_show_back:
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
layer_dat_3D_normalized = np.tile(self.settings.window_background, layer_dat_3D.shape + (1,))
elif back_what_to_disp == 'stale':
layer_dat_3D_normalized = np.tile(self.settings.stale_background, layer_dat_3D.shape + (1,))
else:
layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
boost_indiv = self.state.layer_boost_indiv,
boost_gamma = self.state.layer_boost_gamma,
neg_pos_colors = ((1,0,0), (0,1,0)))
else:
layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
boost_indiv = self.state.layer_boost_indiv,
boost_gamma = self.state.layer_boost_gamma)
#print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()
display_3D = layer_dat_3D_normalized
# Convert to float if necessary:
display_3D = ensure_float01(display_3D)
# Upsample gray -> color if necessary
# e.g. (1000,32,32) -> (1000,32,32,3)
if len(display_3D.shape) == 3:
display_3D = display_3D[:,:,:,np.newaxis]
if display_3D.shape[3] == 1:
display_3D = np.tile(display_3D, (1, 1, 1, 3))
# Upsample unit length tiles to give a more sane tile / highlight ratio
# e.g. (1000,1,1,3) -> (1000,3,3,3)
if display_3D.shape[1] == 1:
display_3D = np.tile(display_3D, (1, 3, 3, 1))
if self.state.layers_show_back and not self.state.pattern_mode:
padval = self.settings.caffevis_layer_clr_back_background
else:
padval = self.settings.window_background
highlights = [None] * n_tiles
with self.state.lock:
if self.state.cursor_area == 'bottom':
highlights[self.state.selected_unit] = self.settings.caffevis_layer_clr_cursor # in [0,1] range
if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
highlights[self.state.backprop_unit] = self.settings.caffevis_layer_clr_back_sel # in [0,1] range
_, display_2D = tile_images_make_tiles(display_3D, hw = (tile_rows,tile_cols), padval = padval, highlights = highlights)
if display_3D_highres is None:
display_3D_highres = display_3D
# Display pane based on layers_pane_zoom_mode
state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
assert state_layers_pane_zoom_mode in (0,1,2)
if state_layers_pane_zoom_mode == 0:
# Mode 0: normal display (activations or patterns)
display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape)
elif state_layers_pane_zoom_mode == 1:
# Mode 1: zoomed selection
unit_data = display_3D_highres[self.state.selected_unit]
display_2D_resize = ensure_uint255_and_resize_to_fit(unit_data, pane.data.shape)
else:
# Mode 2: zoomed backprop pane
display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape) * 0
pane.data[:] = to_255(self.settings.window_background)
pane.data[0:display_2D_resize.shape[0], 0:display_2D_resize.shape[1], :] = display_2D_resize
if self.settings.caffevis_label_layers and self.state.layer in self.settings.caffevis_label_layers and self.labels and self.state.cursor_area == 'bottom':
# Display label annotation atop layers pane (e.g. for fc8/prob)
defaults = {'face': getattr(cv2, self.settings.caffevis_label_face),
'fsize': self.settings.caffevis_label_fsize,
'clr': to_255(self.settings.caffevis_label_clr),
'thick': self.settings.caffevis_label_thick}
loc_base = self.settings.caffevis_label_loc[::-1] # Reverse to OpenCV c,r order
lines = [FormattedString(self.labels[self.state.selected_unit], defaults)]
cv2_typeset_text(pane.data, lines, loc_base)
return display_3D_highres
def _draw_layer_pane(self, pane):
'''Returns the data shown in highres format, b01c order.'''
if not hasattr(self.net, 'intermediate_predictions') or \
self.net.intermediate_predictions is None:
return None, None
display_3D_highres, selected_unit_highres = None, None
out = self.net.intermediate_predictions[self.state.layer_idx]
if self.state.layers_pane_filter_mode in (
4, 5) and self.state.extra_info is None:
self.state.layers_pane_filter_mode = 0
state_layers_pane_filter_mode = self.state.layers_pane_filter_mode
assert state_layers_pane_filter_mode in (0, 1, 2, 3, 4)
# Display pane based on layers_pane_zoom_mode
state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
assert state_layers_pane_zoom_mode in (0, 1, 2)
layer_dat_3D = out[0].T
n_tiles = layer_dat_3D.shape[0]
tile_rows, tile_cols = self.net_layer_info[
self.state.layer]['tiles_rc']
if state_layers_pane_filter_mode == 0:
if len(layer_dat_3D.shape) > 1:
img_width, img_height = get_tiles_height_width_ratio(
layer_dat_3D.shape[1],
self.settings.kerasvis_layers_aspect_ratio)
pad = np.zeros(
(layer_dat_3D.shape[0],
((img_width * img_height) - layer_dat_3D.shape[1])))
layer_dat_3D = np.concatenate((layer_dat_3D, pad), axis=1)
layer_dat_3D = np.reshape(
layer_dat_3D,
(layer_dat_3D.shape[0], img_width, img_height))
elif state_layers_pane_filter_mode == 1:
if len(layer_dat_3D.shape) > 1:
layer_dat_3D = np.average(layer_dat_3D, axis=1)
elif state_layers_pane_filter_mode == 2:
if len(layer_dat_3D.shape) > 1:
layer_dat_3D = np.max(layer_dat_3D, axis=1)
elif state_layers_pane_filter_mode == 3:
if len(layer_dat_3D.shape) > 1:
title, r, c, hide_axis = None, tile_rows, tile_cols, True
x_axis_label, y_axis_label = None, None
if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 1:
r, c, hide_axis = 1, 1, False
layer_dat_3D = layer_dat_3D[self.state.selected_unit:self.
state.selected_unit + 1]
title = 'Layer {}, Filter {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit)
x_axis_label, y_axis_label = 'Time', 'Activation'
display_3D = plt_plot_filters_blit(
y=layer_dat_3D,
x=None,
shape=(pane.data.shape[0], pane.data.shape[1]),
rows=r,
cols=c,
title=title,
log_scale=self.state.log_scale,
hide_axis=hide_axis,
x_axis_label=x_axis_label,
y_axis_label=y_axis_label)
if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 0:
selected_unit_highres = plt_plot_filter(
x=None,
y=layer_dat_3D[self.state.selected_unit],
title='Layer {}, Filter {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit),
log_scale=self.state.log_scale,
x_axis_label='Time',
y_axis_label='Activation')
else:
state_layers_pane_filter_mode = 0
elif state_layers_pane_filter_mode == 4:
if self.state.extra_info is not None:
extra = self.state.extra_info.item()
is_heatmap = True if 'type' in extra and extra[
'type'] == 'heatmap' else False
if is_heatmap:
layer_dat_3D = extra['data'][self.state.layer_idx]
if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 1:
display_3D = plt_plot_heatmap(
data=layer_dat_3D[self.state.selected_unit:self.
state.selected_unit + 1],
shape=(pane.data.shape[0], pane.data.shape[1]),
rows=1,
cols=1,
x_axis_label=extra['x_axis'],
y_axis_label=extra['y_axis'],
title='Layer {}, Filter {} \n {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit, extra['title']),
hide_axis=False,
x_axis_values=extra['x_axis_values'],
y_axis_values=extra['y_axis_values'],
vmin=layer_dat_3D.min(),
vmax=layer_dat_3D.max())
else:
display_3D = plt_plot_heatmap(
data=layer_dat_3D,
shape=(pane.data.shape[0], pane.data.shape[1]),
rows=tile_rows,
cols=tile_cols,
x_axis_label=extra['x_axis'],
y_axis_label=extra['y_axis'],
title=extra['title'],
x_axis_values=extra['x_axis_values'],
y_axis_values=extra['y_axis_values'])
if self.state.cursor_area == 'bottom':
selected_unit_highres = plt_plot_heatmap(
data=layer_dat_3D[self.state.selected_unit:self.
state.selected_unit + 1],
shape=(300, 300),
rows=1,
cols=1,
x_axis_label=extra['x_axis'],
y_axis_label=extra['y_axis'],
title='Layer {}, Filter {} \n {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit, extra['title']),
x_axis_values=extra['x_axis_values'],
y_axis_values=extra['y_axis_values'],
hide_axis=False,
vmin=layer_dat_3D.min(),
vmax=layer_dat_3D.max())[0]
else:
layer_dat_3D = extra['x'][self.state.layer_idx]
title, x_axis_label, y_axis_label, r, c, hide_axis = None, None, None, tile_rows, tile_cols, True
if self.state.cursor_area == 'bottom':
if state_layers_pane_zoom_mode == 1:
r, c, hide_axis = 1, 1, False
layer_dat_3D = layer_dat_3D[self.state.
selected_unit:self.
state.selected_unit +
1]
title = 'Layer {}, Filter {} \n {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit, extra['title'])
x_axis_label, y_axis_label = extra[
'x_axis'], extra['y_axis']
if self.state.log_scale == 1:
y_axis_label = y_axis_label + ' (log-scale)'
# start_time = timeit.default_timer()
display_3D = plt_plot_filters_blit(
y=layer_dat_3D,
x=extra['y'],
shape=(pane.data.shape[0], pane.data.shape[1]),
rows=r,
cols=c,
title=title,
log_scale=self.state.log_scale,
x_axis_label=x_axis_label,
y_axis_label=y_axis_label,
hide_axis=hide_axis)
if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 0:
selected_unit_highres = plt_plot_filter(
x=extra['y'],
y=layer_dat_3D[self.state.selected_unit],
title='Layer {}, Filter {} \n {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit, extra['title']),
log_scale=self.state.log_scale,
x_axis_label=extra['x_axis'],
y_axis_label=extra['y_axis'])
# TODO
# if hasattr(self.settings, 'static_files_extra_fn'):
# self.data = self.settings.static_files_extra_fn(self.latest_static_file)
# self.state.layer_idx
if len(layer_dat_3D.shape) == 1:
layer_dat_3D = layer_dat_3D[:, np.newaxis, np.newaxis]
if self.state.layers_show_back and not self.state.pattern_mode:
padval = self.settings.kerasvis_layer_clr_back_background
else:
padval = self.settings.window_background
if self.state.pattern_mode:
# Show desired patterns loaded from disk
load_layer = self.state.layer
if self.settings.kerasvis_jpgvis_remap and self.state.layer in self.settings.kerasvis_jpgvis_remap:
load_layer = self.settings.kerasvis_jpgvis_remap[
self.state.layer]
if self.settings.kerasvis_jpgvis_layers and load_layer in self.settings.kerasvis_jpgvis_layers:
jpg_path = os.path.join(self.settings.kerasvis_unit_jpg_dir,
'regularized_opt', load_layer,
'whole_layer.jpg')
# Get highres version
# cache_before = str(self.img_cache)
display_3D_highres = self.img_cache.get((jpg_path, 'whole'),
None)
# else:
# display_3D_highres = None
if display_3D_highres is None:
try:
with WithTimer('KerasVisApp:load_sprite_image',
quiet=self.debug_level < 1):
display_3D_highres = load_square_sprite_image(
jpg_path, n_sprites=n_tiles)
except IOError:
# File does not exist, so just display disabled.
pass
else:
self.img_cache.set((jpg_path, 'whole'),
display_3D_highres)
# cache_after = str(self.img_cache)
# print 'Cache was / is:\n %s\n %s' % (cache_before, cache_after)
if display_3D_highres is not None:
# Get lowres version, maybe. Assume we want at least one pixel for selection border.
row_downsamp_factor = int(
np.ceil(
float(display_3D_highres.shape[1]) /
(pane.data.shape[0] / tile_rows - 2)))
col_downsamp_factor = int(
np.ceil(
float(display_3D_highres.shape[2]) /
(pane.data.shape[1] / tile_cols - 2)))
ds = max(row_downsamp_factor, col_downsamp_factor)
if ds > 1:
# print 'Downsampling by', ds
display_3D = display_3D_highres[:, ::ds, ::ds, :]
else:
display_3D = display_3D_highres
else:
display_3D = layer_dat_3D * 0 # nothing to show
else:
# Show data from network (activations or diffs)
if self.state.layers_show_back:
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
layer_dat_3D_normalized = np.tile(
self.settings.window_background,
layer_dat_3D.shape + (1, ))
elif back_what_to_disp == 'stale':
layer_dat_3D_normalized = np.tile(
self.settings.stale_background,
layer_dat_3D.shape + (1, ))
else:
layer_dat_3D_normalized = tile_images_normalize(
layer_dat_3D,
boost_indiv=self.state.layer_boost_indiv,
boost_gamma=self.state.layer_boost_gamma,
neg_pos_colors=((1, 0, 0), (0, 1, 0)))
else:
layer_dat_3D_normalized = tile_images_normalize(
layer_dat_3D,
boost_indiv=self.state.layer_boost_indiv,
boost_gamma=self.state.layer_boost_gamma)
# print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()
if state_layers_pane_filter_mode in (0, 1, 2):
display_3D = layer_dat_3D_normalized
# Convert to float if necessary:
display_3D = ensure_float01(display_3D)
# Upsample gray -> color if necessary
# e.g. (1000,32,32) -> (1000,32,32,3)
if len(display_3D.shape) == 3:
display_3D = display_3D[:, :, :, np.newaxis]
if display_3D.shape[3] == 1:
display_3D = np.tile(display_3D, (1, 1, 1, 3))
# Upsample unit length tiles to give a more sane tile / highlight ratio
# e.g. (1000,1,1,3) -> (1000,3,3,3)
if display_3D.shape[1] == 1:
display_3D = np.tile(display_3D, (1, 3, 3, 1))
if state_layers_pane_zoom_mode in (0, 2):
highlights = [None] * n_tiles
with self.state.lock:
if self.state.cursor_area == 'bottom':
highlights[
self.state.
selected_unit] = self.settings.kerasvis_layer_clr_cursor # in [0,1] range
if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
highlights[
self.state.
backprop_unit] = self.settings.kerasvis_layer_clr_back_sel # in [0,1] range
if self.state.cursor_area == 'bottom' and state_layers_pane_filter_mode in (
3, 4):
# pane.data[0:display_2D_resize.shape[0], 0:2, :] = to_255(self.settings.window_background)
# pane.data[0:2, 0:display_2D_resize.shape[1], :] = to_255(self.settings.window_background)
display_3D[self.state.selected_unit, 0:display_3D.shape[1],
0:2, :] = self.settings.kerasvis_layer_clr_cursor
display_3D[
self.state.selected_unit, 0:2, 0:display_3D.
shape[2], :] = self.settings.kerasvis_layer_clr_cursor
display_3D[self.state.selected_unit, 0:display_3D.shape[1],
-2:, :] = self.settings.kerasvis_layer_clr_cursor
display_3D[
self.state.selected_unit, -2:, 0:display_3D.
shape[2], :] = self.settings.kerasvis_layer_clr_cursor
_, display_2D = tile_images_make_tiles(display_3D,
hw=(tile_rows, tile_cols),
padval=padval,
highlights=highlights)
# Mode 0: normal display (activations or patterns)
display_2D_resize = ensure_uint255_and_resize_to_fit(
display_2D, pane.data.shape)
if state_layers_pane_zoom_mode == 2:
display_2D_resize = display_2D_resize * 0
if display_3D_highres is None:
display_3D_highres = display_3D
elif state_layers_pane_zoom_mode == 1:
if display_3D_highres is None:
display_3D_highres = display_3D
# Mode 1: zoomed selection
if state_layers_pane_filter_mode in (0, 1, 2):
unit_data = display_3D_highres[self.state.selected_unit]
else:
unit_data = display_3D_highres[0]
display_2D_resize = ensure_uint255_and_resize_to_fit(
unit_data, pane.data.shape)
pane.data[:] = to_255(self.settings.window_background)
pane.data[0:display_2D_resize.shape[0],
0:display_2D_resize.shape[1], :] = display_2D_resize
# # Add background strip around the top and left edges
# pane.data[0:display_2D_resize.shape[0], 0:2, :] = to_255(self.settings.window_background)
# pane.data[0:2, 0:display_2D_resize.shape[1], :] = to_255(self.settings.window_background)
if self.settings.kerasvis_label_layers and \
self.state.layer in self.settings.kerasvis_label_layers and \
self.labels and self.state.cursor_area == 'bottom':
# Display label annotation atop layers pane (e.g. for fc8/prob)
defaults = {
'face': getattr(cv2, self.settings.kerasvis_label_face),
'fsize': self.settings.kerasvis_label_fsize,
'clr': to_255(self.settings.kerasvis_label_clr),
'thick': self.settings.kerasvis_label_thick
}
loc_base = self.settings.kerasvis_label_loc[::
-1] # Reverse to OpenCV c,r order
lines = [
FormattedString(self.labels[self.state.selected_unit],
defaults)
]
cv2_typeset_text(pane.data, lines, loc_base)
return display_3D_highres, selected_unit_highres
def _draw_layer_pane(self, pane):
'''Returns the data shown in highres format, b01c order.'''
if self.state.layers_show_back:
layer_dat_3D = self.net.blobs[self.state.layer].diff[0]
else:
layer_dat_3D = self.net.blobs[self.state.layer].data[0]
# Promote FC layers with shape (n) to have shape (n,1,1)
if len(layer_dat_3D.shape) == 1:
layer_dat_3D = layer_dat_3D[:, np.newaxis, np.newaxis]
n_tiles = layer_dat_3D.shape[0]
tile_rows, tile_cols = self.net_layer_info[
self.state.layer]['tiles_rc']
display_3D_highres = None
if self.state.pattern_mode:
# Show desired patterns loaded from disk
load_layer = self.state.layer
if self.settings.caffevis_jpgvis_remap and self.state.layer in self.settings.caffevis_jpgvis_remap:
load_layer = self.settings.caffevis_jpgvis_remap[
self.state.layer]
if self.settings.caffevis_jpgvis_layers and load_layer in self.settings.caffevis_jpgvis_layers:
jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
'regularized_opt', load_layer,
'whole_layer.jpg')
# Get highres version
#cache_before = str(self.img_cache)
display_3D_highres = self.img_cache.get((jpg_path, 'whole'),
None)
#else:
# display_3D_highres = None
if display_3D_highres is None:
try:
with WithTimer('CaffeVisApp:load_sprite_image',
quiet=self.debug_level < 1):
display_3D_highres = load_square_sprite_image(
jpg_path, n_sprites=n_tiles)
except IOError:
# File does not exist, so just display disabled.
pass
else:
self.img_cache.set((jpg_path, 'whole'),
display_3D_highres)
#cache_after = str(self.img_cache)
#print 'Cache was / is:\n %s\n %s' % (cache_before, cache_after)
if display_3D_highres is not None:
# Get lowres version, maybe. Assume we want at least one pixel for selection border.
row_downsamp_factor = int(
np.ceil(
float(display_3D_highres.shape[1]) /
(pane.data.shape[0] / tile_rows - 2)))
col_downsamp_factor = int(
np.ceil(
float(display_3D_highres.shape[2]) /
(pane.data.shape[1] / tile_cols - 2)))
ds = max(row_downsamp_factor, col_downsamp_factor)
if ds > 1:
#print 'Downsampling by', ds
display_3D = display_3D_highres[:, ::ds, ::ds, :]
else:
display_3D = display_3D_highres
else:
display_3D = layer_dat_3D * 0 # nothing to show
else:
# Show data from network (activations or diffs)
if self.state.layers_show_back:
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
layer_dat_3D_normalized = np.tile(
self.settings.window_background,
layer_dat_3D.shape + (1, ))
elif back_what_to_disp == 'stale':
layer_dat_3D_normalized = np.tile(
self.settings.stale_background,
layer_dat_3D.shape + (1, ))
else:
layer_dat_3D_normalized = tile_images_normalize(
layer_dat_3D,
boost_indiv=self.state.layer_boost_indiv,
boost_gamma=self.state.layer_boost_gamma,
neg_pos_colors=((1, 0, 0), (0, 1, 0)))
else:
layer_dat_3D_normalized = tile_images_normalize(
layer_dat_3D,
boost_indiv=self.state.layer_boost_indiv,
boost_gamma=self.state.layer_boost_gamma)
#print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()
display_3D = layer_dat_3D_normalized
# Convert to float if necessary:
display_3D = ensure_float01(display_3D)
# Upsample gray -> color if necessary
# e.g. (1000,32,32) -> (1000,32,32,3)
if len(display_3D.shape) == 3:
display_3D = display_3D[:, :, :, np.newaxis]
if display_3D.shape[3] == 1:
display_3D = np.tile(display_3D, (1, 1, 1, 3))
# Upsample unit length tiles to give a more sane tile / highlight ratio
# e.g. (1000,1,1,3) -> (1000,3,3,3)
if display_3D.shape[1] == 1:
display_3D = np.tile(display_3D, (1, 3, 3, 1))
if self.state.layers_show_back and not self.state.pattern_mode:
padval = self.settings.caffevis_layer_clr_back_background
else:
padval = self.settings.window_background
highlights = [None] * n_tiles
with self.state.lock:
if self.state.cursor_area == 'bottom':
highlights[
self.state.
selected_unit] = self.settings.caffevis_layer_clr_cursor # in [0,1] range
if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
highlights[
self.state.
backprop_unit] = self.settings.caffevis_layer_clr_back_sel # in [0,1] range
_, display_2D = tile_images_make_tiles(display_3D,
hw=(tile_rows, tile_cols),
padval=padval,
highlights=highlights)
if display_3D_highres is None:
display_3D_highres = display_3D
# Display pane based on layers_pane_zoom_mode
state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
assert state_layers_pane_zoom_mode in (0, 1, 2)
if state_layers_pane_zoom_mode == 0:
# Mode 0: normal display (activations or patterns)
display_2D_resize = ensure_uint255_and_resize_to_fit(
display_2D, pane.data.shape)
elif state_layers_pane_zoom_mode == 1:
# Mode 1: zoomed selection
unit_data = display_3D_highres[self.state.selected_unit]
display_2D_resize = ensure_uint255_and_resize_to_fit(
unit_data, pane.data.shape)
else:
# Mode 2: zoomed backprop pane
display_2D_resize = ensure_uint255_and_resize_to_fit(
display_2D, pane.data.shape) * 0
pane.data[:] = to_255(self.settings.window_background)
pane.data[0:display_2D_resize.shape[0],
0:display_2D_resize.shape[1], :] = display_2D_resize
if self.settings.caffevis_label_layers and self.state.layer in self.settings.caffevis_label_layers and self.labels and self.state.cursor_area == 'bottom':
# Display label annotation atop layers pane (e.g. for fc8/prob)
defaults = {
'face': getattr(cv2, self.settings.caffevis_label_face),
'fsize': self.settings.caffevis_label_fsize,
'clr': to_255(self.settings.caffevis_label_clr),
'thick': self.settings.caffevis_label_thick
}
loc_base = self.settings.caffevis_label_loc[::
-1] # Reverse to OpenCV c,r order
lines = [
FormattedString(self.labels[self.state.selected_unit],
defaults)
]
cv2_typeset_text(pane.data, lines, loc_base)
return display_3D_highres
def _draw_layer_pane(self, pane):
'''Returns the data shown in highres format, b01c order.'''
if self.state.layers_show_back:
layer_dat_3D = self.net.blobs[self.state.layer].diff[0]
else:
layer_dat_3D = self.net.blobs[self.state.layer].data[0]
# Promote FC layers with shape (n) to have shape (n,1,1)
if len(layer_dat_3D.shape) == 1:
layer_dat_3D = layer_dat_3D[:,np.newaxis,np.newaxis]
n_tiles = layer_dat_3D.shape[0]
tile_rows,tile_cols = get_tiles_height_width(n_tiles)
display_3D_highres = None
if self.state.pattern_mode:
# Show desired patterns loaded from disk
#available = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'fc6', 'fc7', 'fc8', 'prob']
jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
'regularized_opt', self.state.layer, 'whole_layer.jpg')
# Get highres version
cache_before = str(self.img_cache)
display_3D_highres = self.img_cache.get((jpg_path, 'whole'), None)
if display_3D_highres is None:
try:
with WithTimer('CaffeVisApp:load_sprite_image', quiet = self.debug_level < 1):
display_3D_highres = load_sprite_image(jpg_path, (tile_rows, tile_cols), n_sprites = n_tiles)
except IOError:
# File does not exist, so just display disabled.
pass
else:
self.img_cache.set((jpg_path, 'whole'), display_3D_highres)
cache_after = str(self.img_cache)
#print 'Cache was / is:\n %s\n %s' % (cache_before, cache_after)
if display_3D_highres is not None:
# Get lowres version, maybe. Assume we want at least one pixel for selection border.
row_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[1]) / (pane.data.shape[0] / tile_rows - 2)))
col_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[2]) / (pane.data.shape[1] / tile_cols - 2)))
ds = max(row_downsamp_factor, col_downsamp_factor)
if ds > 1:
#print 'Downsampling by', ds
display_3D = display_3D_highres[:,::ds,::ds,:]
else:
display_3D = display_3D_highres
else:
display_3D = layer_dat_3D * 0 # nothing to show
else:
# Show data from network (activations or diffs)
if self.state.layers_show_back:
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
layer_dat_3D_normalized = np.tile(self.settings.window_background, layer_dat_3D.shape + (1,))
elif back_what_to_disp == 'stale':
layer_dat_3D_normalized = np.tile(self.settings.stale_background, layer_dat_3D.shape + (1,))
else:
layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
boost_indiv = self.state.layer_boost_indiv,
boost_gamma = self.state.layer_boost_gamma,
neg_pos_colors = ((1,0,0), (0,1,0)))
else:
layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
boost_indiv = self.state.layer_boost_indiv,
boost_gamma = self.state.layer_boost_gamma)
#print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()
display_3D = layer_dat_3D_normalized
# Convert to float if necessary:
display_3D = ensure_float01(display_3D)
# Upsample gray -> color if necessary
# (1000,32,32) -> (1000,32,32,3)
if len(display_3D.shape) == 3:
display_3D = display_3D[:,:,:,np.newaxis]
if display_3D.shape[3] == 1:
display_3D = np.tile(display_3D, (1, 1, 1, 3))
# Upsample unit length tiles to give a more sane tile / highlight ratio
# (1000,1,1,3) -> (1000,3,3,3)
if display_3D.shape[1] == 1:
display_3D = np.tile(display_3D, (1, 3, 3, 1))
if self.state.layers_show_back and not self.state.pattern_mode:
padval = self.settings.caffevis_layer_clr_back_background
else:
padval = self.settings.window_background
# Tell the state about the updated (height,width) tile display (ensures valid selection)
self.state.update_tiles_height_width((tile_rows,tile_cols), display_3D.shape[0])
#if self.state.layers_show_back:
# highlights = [(.5, .5, 1)] * n_tiles
#else:
highlights = [None] * n_tiles
with self.state.lock:
if self.state.cursor_area == 'bottom':
highlights[self.state.selected_unit] = self.settings.caffevis_layer_clr_cursor # in [0,1] range
if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
highlights[self.state.backprop_unit] = self.settings.caffevis_layer_clr_back_sel # in [0,1] range
_, display_2D = tile_images_make_tiles(display_3D, padval = padval, highlights = highlights)
#print ' ===tile_conv dtype', tile_conv.dtype, 'range', tile_conv.min(), tile_conv.max()
if display_3D_highres is None:
display_3D_highres = display_3D
# Display pane based on layers_pane_zoom_mode
state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
assert state_layers_pane_zoom_mode in (0,1,2)
if state_layers_pane_zoom_mode == 0:
# Mode 0: base case
display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape)
elif state_layers_pane_zoom_mode == 1:
# Mode 1: zoomed selection
unit_data = display_3D_highres[self.state.selected_unit]
display_2D_resize = ensure_uint255_and_resize_to_fit(unit_data, pane.data.shape)
else:
# Mode 2: ??? backprop ???
display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape) * 0
pane.data[0:display_2D_resize.shape[0], 0:display_2D_resize.shape[1], :] = display_2D_resize
return display_3D_highres