def _draw_aux_pane(self,
pane,
layer_data_normalized,
selected_unit_highres=None):
pane.data[:] = to_255(self.settings.window_background)
with self.state.lock:
if self.state.layers_pane_zoom_mode == 1:
mode = 'prob_labels'
elif self.state.cursor_area == 'bottom' and layer_data_normalized is not None:
mode = 'selected'
elif self.state.layers_pane_filter_mode in (0, 1, 2, 3):
mode = 'prob_labels'
else:
mode = 'none'
# if mode == 'selected' and layer_data_normalized is None and selected_unit_highres is None:
# mode = 'prob_labels'
if mode == 'selected':
if selected_unit_highres is not None:
unit_data = selected_unit_highres
else:
unit_data = layer_data_normalized[self.state.selected_unit]
unit_data_resize = ensure_uint255_and_resize_to_fit(
unit_data, pane.data.shape)
pane.data[0:unit_data_resize.shape[0],
0:unit_data_resize.shape[1], :] = unit_data_resize
elif mode == 'prob_labels':
self._draw_prob_labels_pane(pane)
def _draw_aux_pane(self, pane, layer_data_normalized):
pane.data[:] = to_255(self.settings.window_background)
mode = None
with self.state.lock:
if self.state.cursor_area == 'bottom':
mode = 'selected'
else:
mode = 'prob_labels'
if mode == 'selected':
unit_data = layer_data_normalized[self.state.selected_unit]
# # Edited
# # -----------------------
# print '*'*100
# print unit_data.mean(axis=(0,1))
unit_data_resize = ensure_uint255_and_resize_to_fit(unit_data, pane.data.shape)
pane.data[0:unit_data_resize.shape[0], 0:unit_data_resize.shape[1], :] = unit_data_resize
elif mode == 'prob_labels':
self._draw_prob_labels_pane(pane)
def _draw_back_pane(self, pane):
mode = None
with self.state.lock:
back_enabled = self.state.back_enabled
back_mode = self.state.back_mode
back_filt_mode = self.state.back_filt_mode
state_layer = self.state.layer
selected_unit = self.state.selected_unit
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
pane.data[:] = to_255(self.settings.window_background)
elif back_what_to_disp == 'stale':
pane.data[:] = to_255(self.settings.stale_background)
else:
# One of the backprop modes is enabled and the back computation (gradient or deconv) is up to date
grad_blob = self.net.blobs['data'].diff
# Manually deprocess (skip mean subtraction and rescaling)
#grad_img = self.net.deprocess('data', diff_blob)
grad_blob = grad_blob[0] # bc01 -> c01
grad_blob = grad_blob.transpose((1, 2, 0)) # c01 -> 01c
grad_img = grad_blob[:, :,
self._net_channel_swap_inv] # e.g. BGR -> RGB
# Mode-specific processing
assert back_mode in ('grad', 'deconv')
assert back_filt_mode in ('raw', 'gray', 'norm', 'normblur')
if back_filt_mode == 'raw':
grad_img = norm01c(grad_img, 0)
elif back_filt_mode == 'gray':
grad_img = grad_img.mean(axis=2)
grad_img = norm01c(grad_img, 0)
elif back_filt_mode == 'norm':
grad_img = np.linalg.norm(grad_img, axis=2)
grad_img = norm01(grad_img)
else:
grad_img = np.linalg.norm(grad_img, axis=2)
cv2.GaussianBlur(grad_img, (0, 0),
self.settings.caffevis_grad_norm_blur_radius,
grad_img)
grad_img = norm01(grad_img)
# If necessary, re-promote from grayscale to color
if len(grad_img.shape) == 2:
grad_img = np.tile(grad_img[:, :, np.newaxis], 3)
grad_img_resize = ensure_uint255_and_resize_to_fit(
grad_img, pane.data.shape)
pane.data[0:grad_img_resize.shape[0],
0:grad_img_resize.shape[1], :] = grad_img_resize
def _draw_back_pane(self, pane):
mode = None
with self.state.lock:
back_enabled = self.state.back_enabled
back_mode = self.state.back_mode
back_filt_mode = self.state.back_filt_mode
state_layer = self.state.layer
selected_unit = self.state.selected_unit
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
pane.data[:] = to_255(self.settings.window_background)
elif back_what_to_disp == 'stale':
pane.data[:] = to_255(self.settings.stale_background)
else:
# One of the backprop modes is enabled and the back computation (gradient or deconv) is up to date
grad_blob = self.net.blobs['data'].diff
# Manually deprocess (skip mean subtraction and rescaling)
#grad_img = self.net.deprocess('data', diff_blob)
grad_blob = grad_blob[0] # bc01 -> c01
grad_blob = grad_blob.transpose((1,2,0)) # c01 -> 01c
grad_img = grad_blob[:, :, self._net_channel_swap_inv] # e.g. BGR -> RGB
# Mode-specific processing
assert back_mode in ('grad', 'deconv')
assert back_filt_mode in ('raw', 'gray', 'norm', 'normblur')
if back_filt_mode == 'raw':
grad_img = norm01c(grad_img, 0)
elif back_filt_mode == 'gray':
grad_img = grad_img.mean(axis=2)
grad_img = norm01c(grad_img, 0)
elif back_filt_mode == 'norm':
grad_img = np.linalg.norm(grad_img, axis=2)
grad_img = norm01(grad_img)
else:
grad_img = np.linalg.norm(grad_img, axis=2)
cv2.GaussianBlur(grad_img, (0,0), self.settings.caffevis_grad_norm_blur_radius, grad_img)
grad_img = norm01(grad_img)
# If necessary, re-promote from grayscale to color
if len(grad_img.shape) == 2:
grad_img = np.tile(grad_img[:,:,np.newaxis], 3)
grad_img_resize = ensure_uint255_and_resize_to_fit(grad_img, pane.data.shape)
pane.data[0:grad_img_resize.shape[0], 0:grad_img_resize.shape[1], :] = grad_img_resize
def _draw_back_pane(self, pane):
mode = None
with self.state.lock:
back_enabled = self.state.back_enabled
back_mode = self.state.back_mode
back_filt_mode = self.state.back_filt_mode
state_layer = self.state.layer
selected_unit = self.state.selected_unit
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
pane.data[:] = to_255(self.settings.window_background)
elif back_what_to_disp == 'stale':
pane.data[:] = to_255(self.settings.stale_background)
else:
# One of the backprop modes is enabled and the back computation (gradient or deconv) is up to date
grad_img = self.my_net.get_input_gradient_as_image()
# Mode-specific processing
assert back_mode in ('grad', 'deconv')
assert back_filt_mode in ('raw', 'gray', 'norm', 'normblur')
if back_filt_mode == 'raw':
grad_img = norm01c(grad_img, 0)
elif back_filt_mode == 'gray':
grad_img = grad_img.mean(axis=2)
grad_img = norm01c(grad_img, 0)
elif back_filt_mode == 'norm':
grad_img = np.linalg.norm(grad_img, axis=2)
grad_img = norm01(grad_img)
else:
grad_img = np.linalg.norm(grad_img, axis=2)
cv2.GaussianBlur(grad_img, (0, 0),
self.settings.caffevis_grad_norm_blur_radius,
grad_img)
grad_img = norm01(grad_img)
# If necessary, re-promote from grayscale to color
if len(grad_img.shape) == 2:
grad_img = np.tile(grad_img[:, :, np.newaxis], 3)
grad_img_resize = ensure_uint255_and_resize_to_fit(
grad_img, pane.data.shape)
pane.data[0:grad_img_resize.shape[0],
0:grad_img_resize.shape[1], :] = grad_img_resize
def _draw_aux_pane(self, pane, layer_data_normalized):
pane.data[:] = to_255(self.settings.window_background)
mode = None
with self.state.lock:
if self.state.cursor_area == 'bottom':
mode = 'selected'
else:
mode = 'prob_labels'
if mode == 'selected':
unit_data = layer_data_normalized[self.state.selected_unit]
unit_data_resize = ensure_uint255_and_resize_to_fit(unit_data, pane.data.shape)
pane.data[0:unit_data_resize.shape[0], 0:unit_data_resize.shape[1], :] = unit_data_resize
elif mode == 'prob_labels':
self._draw_prob_labels_pane(pane)
def _draw_selected_pane(self,
pane,
layer_data_normalized,
selected_unit_highres=None):
pane.data[:] = to_255(self.settings.window_background)
with self.state.lock:
mode = 'selected' if self.state.cursor_area == 'bottom' else 'none'
if mode == 'selected':
unit_data = None
if selected_unit_highres is not None:
unit_data = selected_unit_highres
else:
if self.state.selected_unit < len(layer_data_normalized):
unit_data = layer_data_normalized[self.state.selected_unit]
elif len(layer_data_normalized) == 1:
unit_data = layer_data_normalized[0]
if unit_data is not None:
unit_data_resize = ensure_uint255_and_resize_to_fit(
unit_data, pane.data.shape)
pane.data[0:unit_data_resize.shape[0],
0:unit_data_resize.shape[1], :] = unit_data_resize
def run(self):
print 'JPGVisLoadingThread.run called'
while not self.is_timed_out():
with self.state.lock:
if self.state.quit:
break
#print 'JPGVisLoadingThread.run: caffe_net_state is:', self.state.caffe_net_state
#print 'JPGVisLoadingThread.run loop: next_frame: %s, caffe_net_state: %s, back_enabled: %s' % (
# 'None' if self.state.next_frame is None else 'Avail',
# self.state.caffe_net_state,
# self.state.back_enabled)
jpgvis_to_load_key = self.state.jpgvis_to_load_key
if jpgvis_to_load_key is None:
time.sleep(self.loop_sleep)
continue
state_layer_name, state_selected_unit, data_shape, show_maximal_score = jpgvis_to_load_key
# Load three images:
images = [None] * 3
# Resize each component images only using one direction as
# a constraint. This is straightforward but could be very
# wasteful (making an image much larger then much smaller)
# if the proportions of the stacked image are very
# different from the proportions of the data pane.
#resize_shape = (None, data_shape[1]) if self.settings.caffevis_jpgvis_stack_vert else (data_shape[0], None)
# As a heuristic, instead just assume the three images are of the same shape.
if self.settings.caffevis_jpgvis_stack_vert:
resize_shape = (data_shape[0]/3, data_shape[1])
else:
resize_shape = (data_shape[0], data_shape[1]/3)
if self.settings.caffevis_outputs_dir_folder_format == 'original_combined_single_image':
# 0. e.g. regularized_opt/conv1/conv1_0037_montage.jpg
self.load_image_into_pane_original_format(state_layer_name, state_selected_unit, resize_shape, images,
sub_folder='regularized_opt',
file_pattern='%s_%04d_montage.jpg',
image_index_to_set=0,
should_crop_to_corner=True)
elif self.settings.caffevis_outputs_dir_folder_format == 'max_tracker_output':
self.load_image_into_pane_max_tracker_format(state_layer_name, state_selected_unit, resize_shape, images,
file_search_pattern='opt*.jpg',
image_index_to_set=0)
if self.settings.caffevis_outputs_dir_folder_format == 'original_combined_single_image':
# 1. e.g. max_im/conv1/conv1_0037.jpg
self.load_image_into_pane_original_format(state_layer_name, state_selected_unit, resize_shape, images,
sub_folder='max_im',
file_pattern='%s_%04d.jpg',
image_index_to_set=1)
elif self.settings.caffevis_outputs_dir_folder_format == 'max_tracker_output':
# convert to string with 2 decimal digits
values = self.get_score_values_for_max_input_images(state_layer_name, state_selected_unit)
if self.state.show_maximal_score:
captions = [('%.2f' % value) for value in values]
else:
captions = []
self.load_image_into_pane_max_tracker_format(state_layer_name, state_selected_unit, resize_shape, images,
file_search_pattern='maxim*.png',
image_index_to_set=1, captions=captions, values=values)
if self.settings.caffevis_outputs_dir_folder_format == 'original_combined_single_image':
# 2. e.g. max_deconv/conv1/conv1_0037.jpg
self.load_image_into_pane_original_format(state_layer_name, state_selected_unit, resize_shape, images,
sub_folder='max_deconv',
file_pattern='%s_%04d.jpg',
image_index_to_set=2)
elif self.settings.caffevis_outputs_dir_folder_format == 'max_tracker_output':
# convert to string with 2 decimal digits
values = self.get_score_values_for_max_input_images(state_layer_name, state_selected_unit)
self.load_image_into_pane_max_tracker_format(state_layer_name, state_selected_unit, resize_shape, images,
file_search_pattern='deconv*.png',
image_index_to_set=2, values=values)
# Prune images that were not found:
images = [im for im in images if im is not None]
# Stack together
if len(images) > 0:
#print 'Stacking:', [im.shape for im in images]
stack_axis = 0 if self.settings.caffevis_jpgvis_stack_vert else 1
img_stacked = np.concatenate(images, axis = stack_axis)
#print 'Stacked:', img_stacked.shape
img_resize = ensure_uint255_and_resize_to_fit(img_stacked, data_shape)
#print 'Resized:', img_resize.shape
else:
img_resize = np.zeros(shape=(0,)) # Sentinal value when image is not found.
self.cache.set(jpgvis_to_load_key, img_resize)
with self.state.lock:
self.state.jpgvis_to_load_key = None
self.state.drawing_stale = True
print 'JPGVisLoadingThread.run: finished'
def run(self):
print 'JPGVisLoadingThread.run called'
while not self.is_timed_out():
with self.state.lock:
if self.state.quit:
break
#print 'JPGVisLoadingThread.run: caffe_net_state is:', self.state.caffe_net_state
#print 'JPGVisLoadingThread.run loop: next_frame: %s, caffe_net_state: %s, back_enabled: %s' % (
# 'None' if self.state.next_frame is None else 'Avail',
# self.state.caffe_net_state,
# self.state.back_enabled)
jpgvis_to_load_key = self.state.jpgvis_to_load_key
if jpgvis_to_load_key is None:
time.sleep(self.loop_sleep)
continue
state_layer, state_selected_unit, data_shape = jpgvis_to_load_key
# Load three images:
images = [None] * 3
# Resize each component images only using one direction as
# a constraint. This is straightforward but could be very
# wasteful (making an image much larger then much smaller)
# if the proportions of the stacked image are very
# different from the proportions of the data pane.
#resize_shape = (None, data_shape[1]) if self.settings.caffevis_jpgvis_stack_vert else (data_shape[0], None)
# As a heuristic, instead just assume the three images are of the same shape.
if self.settings.caffevis_jpgvis_stack_vert:
resize_shape = (data_shape[0]/3, data_shape[1])
else:
resize_shape = (data_shape[0], data_shape[1]/3)
# 0. e.g. regularized_opt/conv1/conv1_0037_montage.jpg
jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
'regularized_opt',
state_layer,
'%s_%04d_montage.jpg' % (state_layer, state_selected_unit))
try:
img = caffe_load_image(jpg_path, color = True)
img_corner = crop_to_corner(img, 2)
images[0] = ensure_uint255_and_resize_to_fit(img_corner, resize_shape)
except IOError:
print '\nAttempted to load file %s but failed. To supress this warning, remove layer "%s" from settings.caffevis_jpgvis_layers' % (jpg_path, state_layer)
pass
# 1. e.g. max_im/conv1/conv1_0037.jpg
jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
'max_im',
state_layer,
'%s_%04d.jpg' % (state_layer, state_selected_unit))
try:
img = caffe_load_image(jpg_path, color=True)
images[1] = ensure_uint255_and_resize_to_fit(img, resize_shape)
except IOError:
print '\nAttempted to load file %s but failed.' % (jpg_path)
pass
# 2. e.g. max_deconv/conv1/conv1_0037.jpg
try:
jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
'max_deconv',
state_layer,
'%s_%04d.jpg' % (state_layer, state_selected_unit))
img = caffe_load_image(jpg_path, color = True)
images[2] = ensure_uint255_and_resize_to_fit(img, resize_shape)
except IOError:
pass
# Prune images that were not found:
images = [im for im in images if im is not None]
# Stack together
if len(images) > 0:
print 'Stacking:', [im.shape for im in images]
stack_axis = 0 if self.settings.caffevis_jpgvis_stack_vert else 1
img_stacked = np.concatenate(images, axis = stack_axis)
print 'Stacked:', img_stacked.shape
img_resize = ensure_uint255_and_resize_to_fit(img_stacked, data_shape)
print 'Resized:', img_resize.shape
else:
img_resize = np.zeros(shape=(0,)) # Sentinal value when image is not found.
self.cache.set(jpgvis_to_load_key, img_resize)
with self.state.lock:
self.state.jpgvis_to_load_key = None
self.state.drawing_stale = True
print 'JPGVisLoadingThread.run: finished'
def _draw_layer_pane(self, pane):
'''Returns the data shown in highres format, b01c order.'''
if self.state.layers_show_back:
layer_dat_3D = self.net.blobs[self.state.layer].diff[0]
else:
layer_dat_3D = self.net.blobs[self.state.layer].data[0]
# Promote FC layers with shape (n) to have shape (n,1,1)
if len(layer_dat_3D.shape) == 1:
layer_dat_3D = layer_dat_3D[:,np.newaxis,np.newaxis]
n_tiles = layer_dat_3D.shape[0]
tile_rows,tile_cols = self.net_layer_info[self.state.layer]['tiles_rc']
display_3D_highres = None
if self.state.pattern_mode:
# Show desired patterns loaded from disk
load_layer = self.state.layer
if self.settings.caffevis_jpgvis_remap and self.state.layer in self.settings.caffevis_jpgvis_remap:
load_layer = self.settings.caffevis_jpgvis_remap[self.state.layer]
if self.settings.caffevis_jpgvis_layers and load_layer in self.settings.caffevis_jpgvis_layers:
jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
'regularized_opt', load_layer, 'whole_layer.jpg')
# Get highres version
#cache_before = str(self.img_cache)
display_3D_highres = self.img_cache.get((jpg_path, 'whole'), None)
#else:
# display_3D_highres = None
if display_3D_highres is None:
try:
with WithTimer('CaffeVisApp:load_sprite_image', quiet = self.debug_level < 1):
display_3D_highres = load_square_sprite_image(jpg_path, n_sprites = n_tiles)
except IOError:
# File does not exist, so just display disabled.
pass
else:
self.img_cache.set((jpg_path, 'whole'), display_3D_highres)
#cache_after = str(self.img_cache)
#print 'Cache was / is:\n %s\n %s' % (cache_before, cache_after)
if display_3D_highres is not None:
# Get lowres version, maybe. Assume we want at least one pixel for selection border.
row_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[1]) / (pane.data.shape[0] / tile_rows - 2)))
col_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[2]) / (pane.data.shape[1] / tile_cols - 2)))
ds = max(row_downsamp_factor, col_downsamp_factor)
if ds > 1:
#print 'Downsampling by', ds
display_3D = display_3D_highres[:,::ds,::ds,:]
else:
display_3D = display_3D_highres
else:
display_3D = layer_dat_3D * 0 # nothing to show
else:
# Show data from network (activations or diffs)
if self.state.layers_show_back:
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
layer_dat_3D_normalized = np.tile(self.settings.window_background, layer_dat_3D.shape + (1,))
elif back_what_to_disp == 'stale':
layer_dat_3D_normalized = np.tile(self.settings.stale_background, layer_dat_3D.shape + (1,))
else:
layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
boost_indiv = self.state.layer_boost_indiv,
boost_gamma = self.state.layer_boost_gamma,
neg_pos_colors = ((1,0,0), (0,1,0)))
else:
layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
boost_indiv = self.state.layer_boost_indiv,
boost_gamma = self.state.layer_boost_gamma)
#print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()
display_3D = layer_dat_3D_normalized
# Convert to float if necessary:
display_3D = ensure_float01(display_3D)
# Upsample gray -> color if necessary
# e.g. (1000,32,32) -> (1000,32,32,3)
if len(display_3D.shape) == 3:
display_3D = display_3D[:,:,:,np.newaxis]
if display_3D.shape[3] == 1:
display_3D = np.tile(display_3D, (1, 1, 1, 3))
# Upsample unit length tiles to give a more sane tile / highlight ratio
# e.g. (1000,1,1,3) -> (1000,3,3,3)
if display_3D.shape[1] == 1:
display_3D = np.tile(display_3D, (1, 3, 3, 1))
if self.state.layers_show_back and not self.state.pattern_mode:
padval = self.settings.caffevis_layer_clr_back_background
else:
padval = self.settings.window_background
highlights = [None] * n_tiles
with self.state.lock:
if self.state.cursor_area == 'bottom':
highlights[self.state.selected_unit] = self.settings.caffevis_layer_clr_cursor # in [0,1] range
if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
highlights[self.state.backprop_unit] = self.settings.caffevis_layer_clr_back_sel # in [0,1] range
_, display_2D = tile_images_make_tiles(display_3D, hw = (tile_rows,tile_cols), padval = padval, highlights = highlights)
if display_3D_highres is None:
display_3D_highres = display_3D
# Display pane based on layers_pane_zoom_mode
state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
assert state_layers_pane_zoom_mode in (0,1,2)
if state_layers_pane_zoom_mode == 0:
# Mode 0: normal display (activations or patterns)
display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape)
elif state_layers_pane_zoom_mode == 1:
# Mode 1: zoomed selection
unit_data = display_3D_highres[self.state.selected_unit]
display_2D_resize = ensure_uint255_and_resize_to_fit(unit_data, pane.data.shape)
else:
# Mode 2: zoomed backprop pane
display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape) * 0
pane.data[:] = to_255(self.settings.window_background)
pane.data[0:display_2D_resize.shape[0], 0:display_2D_resize.shape[1], :] = display_2D_resize
if self.settings.caffevis_label_layers and self.state.layer in self.settings.caffevis_label_layers and self.labels and self.state.cursor_area == 'bottom':
# Display label annotation atop layers pane (e.g. for fc8/prob)
defaults = {'face': getattr(cv2, self.settings.caffevis_label_face),
'fsize': self.settings.caffevis_label_fsize,
'clr': to_255(self.settings.caffevis_label_clr),
'thick': self.settings.caffevis_label_thick}
loc_base = self.settings.caffevis_label_loc[::-1] # Reverse to OpenCV c,r order
lines = [FormattedString(self.labels[self.state.selected_unit], defaults)]
cv2_typeset_text(pane.data, lines, loc_base)
return display_3D_highres
def run(self):
print 'JPGVisLoadingThread.run called'
while not self.is_timed_out():
with self.state.lock:
if self.state.quit:
break
#print 'JPGVisLoadingThread.run: caffe_net_state is:', self.state.caffe_net_state
#print 'JPGVisLoadingThread.run loop: next_frame: %s, caffe_net_state: %s, back_enabled: %s' % (
# 'None' if self.state.next_frame is None else 'Avail',
# self.state.caffe_net_state,
# self.state.back_enabled)
jpgvis_to_load_key = self.state.jpgvis_to_load_key
if jpgvis_to_load_key is None:
time.sleep(self.loop_sleep)
continue
state_layer, state_selected_unit, data_shape = jpgvis_to_load_key
# Load three images:
images = [None] * 3
# Resize each component images only using one direction as
# a constraint. This is straightforward but could be very
# wasteful (making an image much larger then much smaller)
# if the proportions of the stacked image are very
# different from the proportions of the data pane.
#resize_shape = (None, data_shape[1]) if self.settings.caffevis_jpgvis_stack_vert else (data_shape[0], None)
# As a heuristic, instead just assume the three images are of the same shape.
if self.settings.caffevis_jpgvis_stack_vert:
resize_shape = (data_shape[0] / 3, data_shape[1])
else:
resize_shape = (data_shape[0], data_shape[1] / 3)
# 0. e.g. regularized_opt/conv1/conv1_0037_montage.jpg
jpg_path = os.path.join(
self.settings.caffevis_unit_jpg_dir, 'regularized_opt',
state_layer,
'%s_%04d_montage.jpg' % (state_layer, state_selected_unit))
try:
img = caffe_load_image(jpg_path, color=True)
img_corner = crop_to_corner(img, 2)
images[0] = ensure_uint255_and_resize_to_fit(
img_corner, resize_shape)
except IOError:
print '\nAttempted to load file %s but failed. To supress this warning, remove layer "%s" from settings.caffevis_jpgvis_layers' % (
jpg_path, state_layer)
pass
# 1. e.g. max_im/conv1/conv1_0037.jpg
jpg_path = os.path.join(
self.settings.caffevis_unit_jpg_dir, 'max_im', state_layer,
'%s_%04d.jpg' % (state_layer, state_selected_unit))
try:
img = caffe_load_image(jpg_path, color=True)
images[1] = ensure_uint255_and_resize_to_fit(img, resize_shape)
except IOError:
print '\nAttempted to load file %s but failed.' % (jpg_path)
pass
# 2. e.g. max_deconv/conv1/conv1_0037.jpg
try:
jpg_path = os.path.join(
self.settings.caffevis_unit_jpg_dir, 'max_deconv',
state_layer,
'%s_%04d.jpg' % (state_layer, state_selected_unit))
img = caffe_load_image(jpg_path, color=True)
images[2] = ensure_uint255_and_resize_to_fit(img, resize_shape)
except IOError:
pass
# Prune images that were not found:
images = [im for im in images if im is not None]
# Stack together
if len(images) > 0:
print 'Stacking:', [im.shape for im in images]
stack_axis = 0 if self.settings.caffevis_jpgvis_stack_vert else 1
img_stacked = np.concatenate(images, axis=stack_axis)
print 'Stacked:', img_stacked.shape
img_resize = ensure_uint255_and_resize_to_fit(
img_stacked, data_shape)
print 'Resized:', img_resize.shape
else:
img_resize = np.zeros(
shape=(0, )) # Sentinal value when image is not found.
self.cache.set(jpgvis_to_load_key, img_resize)
with self.state.lock:
self.state.jpgvis_to_load_key = None
self.state.drawing_stale = True
print 'JPGVisLoadingThread.run: finished'
def _draw_layer_pane(self, pane):
'''Returns the data shown in highres format, b01c order.'''
if self.state.layers_show_back:
layer_dat_3D = self.net.blobs[self.state.layer].diff[0]
else:
layer_dat_3D = self.net.blobs[self.state.layer].data[0]
# Promote FC layers with shape (n) to have shape (n,1,1)
if len(layer_dat_3D.shape) == 1:
layer_dat_3D = layer_dat_3D[:, np.newaxis, np.newaxis]
n_tiles = layer_dat_3D.shape[0]
tile_rows, tile_cols = self.net_layer_info[
self.state.layer]['tiles_rc']
display_3D_highres = None
if self.state.pattern_mode:
# Show desired patterns loaded from disk
load_layer = self.state.layer
if self.settings.caffevis_jpgvis_remap and self.state.layer in self.settings.caffevis_jpgvis_remap:
load_layer = self.settings.caffevis_jpgvis_remap[
self.state.layer]
if self.settings.caffevis_jpgvis_layers and load_layer in self.settings.caffevis_jpgvis_layers:
jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
'regularized_opt', load_layer,
'whole_layer.jpg')
# Get highres version
#cache_before = str(self.img_cache)
display_3D_highres = self.img_cache.get((jpg_path, 'whole'),
None)
#else:
# display_3D_highres = None
if display_3D_highres is None:
try:
with WithTimer('CaffeVisApp:load_sprite_image',
quiet=self.debug_level < 1):
display_3D_highres = load_square_sprite_image(
jpg_path, n_sprites=n_tiles)
except IOError:
# File does not exist, so just display disabled.
pass
else:
self.img_cache.set((jpg_path, 'whole'),
display_3D_highres)
#cache_after = str(self.img_cache)
#print 'Cache was / is:\n %s\n %s' % (cache_before, cache_after)
if display_3D_highres is not None:
# Get lowres version, maybe. Assume we want at least one pixel for selection border.
row_downsamp_factor = int(
np.ceil(
float(display_3D_highres.shape[1]) /
(pane.data.shape[0] / tile_rows - 2)))
col_downsamp_factor = int(
np.ceil(
float(display_3D_highres.shape[2]) /
(pane.data.shape[1] / tile_cols - 2)))
ds = max(row_downsamp_factor, col_downsamp_factor)
if ds > 1:
#print 'Downsampling by', ds
display_3D = display_3D_highres[:, ::ds, ::ds, :]
else:
display_3D = display_3D_highres
else:
display_3D = layer_dat_3D * 0 # nothing to show
else:
# Show data from network (activations or diffs)
if self.state.layers_show_back:
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
layer_dat_3D_normalized = np.tile(
self.settings.window_background,
layer_dat_3D.shape + (1, ))
elif back_what_to_disp == 'stale':
layer_dat_3D_normalized = np.tile(
self.settings.stale_background,
layer_dat_3D.shape + (1, ))
else:
layer_dat_3D_normalized = tile_images_normalize(
layer_dat_3D,
boost_indiv=self.state.layer_boost_indiv,
boost_gamma=self.state.layer_boost_gamma,
neg_pos_colors=((1, 0, 0), (0, 1, 0)))
else:
layer_dat_3D_normalized = tile_images_normalize(
layer_dat_3D,
boost_indiv=self.state.layer_boost_indiv,
boost_gamma=self.state.layer_boost_gamma)
#print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()
display_3D = layer_dat_3D_normalized
# Convert to float if necessary:
display_3D = ensure_float01(display_3D)
# Upsample gray -> color if necessary
# e.g. (1000,32,32) -> (1000,32,32,3)
if len(display_3D.shape) == 3:
display_3D = display_3D[:, :, :, np.newaxis]
if display_3D.shape[3] == 1:
display_3D = np.tile(display_3D, (1, 1, 1, 3))
# Upsample unit length tiles to give a more sane tile / highlight ratio
# e.g. (1000,1,1,3) -> (1000,3,3,3)
if display_3D.shape[1] == 1:
display_3D = np.tile(display_3D, (1, 3, 3, 1))
if self.state.layers_show_back and not self.state.pattern_mode:
padval = self.settings.caffevis_layer_clr_back_background
else:
padval = self.settings.window_background
highlights = [None] * n_tiles
with self.state.lock:
if self.state.cursor_area == 'bottom':
highlights[
self.state.
selected_unit] = self.settings.caffevis_layer_clr_cursor # in [0,1] range
if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
highlights[
self.state.
backprop_unit] = self.settings.caffevis_layer_clr_back_sel # in [0,1] range
_, display_2D = tile_images_make_tiles(display_3D,
hw=(tile_rows, tile_cols),
padval=padval,
highlights=highlights)
if display_3D_highres is None:
display_3D_highres = display_3D
# Display pane based on layers_pane_zoom_mode
state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
assert state_layers_pane_zoom_mode in (0, 1, 2)
if state_layers_pane_zoom_mode == 0:
# Mode 0: normal display (activations or patterns)
display_2D_resize = ensure_uint255_and_resize_to_fit(
display_2D, pane.data.shape)
elif state_layers_pane_zoom_mode == 1:
# Mode 1: zoomed selection
unit_data = display_3D_highres[self.state.selected_unit]
display_2D_resize = ensure_uint255_and_resize_to_fit(
unit_data, pane.data.shape)
else:
# Mode 2: zoomed backprop pane
display_2D_resize = ensure_uint255_and_resize_to_fit(
display_2D, pane.data.shape) * 0
pane.data[:] = to_255(self.settings.window_background)
pane.data[0:display_2D_resize.shape[0],
0:display_2D_resize.shape[1], :] = display_2D_resize
if self.settings.caffevis_label_layers and self.state.layer in self.settings.caffevis_label_layers and self.labels and self.state.cursor_area == 'bottom':
# Display label annotation atop layers pane (e.g. for fc8/prob)
defaults = {
'face': getattr(cv2, self.settings.caffevis_label_face),
'fsize': self.settings.caffevis_label_fsize,
'clr': to_255(self.settings.caffevis_label_clr),
'thick': self.settings.caffevis_label_thick
}
loc_base = self.settings.caffevis_label_loc[::
-1] # Reverse to OpenCV c,r order
lines = [
FormattedString(self.labels[self.state.selected_unit],
defaults)
]
cv2_typeset_text(pane.data, lines, loc_base)
return display_3D_highres
def _draw_layer_pane(self, pane):
'''Returns the data shown in highres format, b01c order.'''
if not hasattr(self.net, 'intermediate_predictions') or \
self.net.intermediate_predictions is None:
return None, None
display_3D_highres, selected_unit_highres = None, None
out = self.net.intermediate_predictions[self.state.layer_idx]
if self.state.layers_pane_filter_mode in (
4, 5) and self.state.extra_info is None:
self.state.layers_pane_filter_mode = 0
state_layers_pane_filter_mode = self.state.layers_pane_filter_mode
assert state_layers_pane_filter_mode in (0, 1, 2, 3, 4)
# Display pane based on layers_pane_zoom_mode
state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
assert state_layers_pane_zoom_mode in (0, 1, 2)
layer_dat_3D = out[0].T
n_tiles = layer_dat_3D.shape[0]
tile_rows, tile_cols = self.net_layer_info[
self.state.layer]['tiles_rc']
if state_layers_pane_filter_mode == 0:
if len(layer_dat_3D.shape) > 1:
img_width, img_height = get_tiles_height_width_ratio(
layer_dat_3D.shape[1],
self.settings.kerasvis_layers_aspect_ratio)
pad = np.zeros(
(layer_dat_3D.shape[0],
((img_width * img_height) - layer_dat_3D.shape[1])))
layer_dat_3D = np.concatenate((layer_dat_3D, pad), axis=1)
layer_dat_3D = np.reshape(
layer_dat_3D,
(layer_dat_3D.shape[0], img_width, img_height))
elif state_layers_pane_filter_mode == 1:
if len(layer_dat_3D.shape) > 1:
layer_dat_3D = np.average(layer_dat_3D, axis=1)
elif state_layers_pane_filter_mode == 2:
if len(layer_dat_3D.shape) > 1:
layer_dat_3D = np.max(layer_dat_3D, axis=1)
elif state_layers_pane_filter_mode == 3:
if len(layer_dat_3D.shape) > 1:
title, r, c, hide_axis = None, tile_rows, tile_cols, True
x_axis_label, y_axis_label = None, None
if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 1:
r, c, hide_axis = 1, 1, False
layer_dat_3D = layer_dat_3D[self.state.selected_unit:self.
state.selected_unit + 1]
title = 'Layer {}, Filter {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit)
x_axis_label, y_axis_label = 'Time', 'Activation'
display_3D = plt_plot_filters_blit(
y=layer_dat_3D,
x=None,
shape=(pane.data.shape[0], pane.data.shape[1]),
rows=r,
cols=c,
title=title,
log_scale=self.state.log_scale,
hide_axis=hide_axis,
x_axis_label=x_axis_label,
y_axis_label=y_axis_label)
if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 0:
selected_unit_highres = plt_plot_filter(
x=None,
y=layer_dat_3D[self.state.selected_unit],
title='Layer {}, Filter {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit),
log_scale=self.state.log_scale,
x_axis_label='Time',
y_axis_label='Activation')
else:
state_layers_pane_filter_mode = 0
elif state_layers_pane_filter_mode == 4:
if self.state.extra_info is not None:
extra = self.state.extra_info.item()
is_heatmap = True if 'type' in extra and extra[
'type'] == 'heatmap' else False
if is_heatmap:
layer_dat_3D = extra['data'][self.state.layer_idx]
if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 1:
display_3D = plt_plot_heatmap(
data=layer_dat_3D[self.state.selected_unit:self.
state.selected_unit + 1],
shape=(pane.data.shape[0], pane.data.shape[1]),
rows=1,
cols=1,
x_axis_label=extra['x_axis'],
y_axis_label=extra['y_axis'],
title='Layer {}, Filter {} \n {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit, extra['title']),
hide_axis=False,
x_axis_values=extra['x_axis_values'],
y_axis_values=extra['y_axis_values'],
vmin=layer_dat_3D.min(),
vmax=layer_dat_3D.max())
else:
display_3D = plt_plot_heatmap(
data=layer_dat_3D,
shape=(pane.data.shape[0], pane.data.shape[1]),
rows=tile_rows,
cols=tile_cols,
x_axis_label=extra['x_axis'],
y_axis_label=extra['y_axis'],
title=extra['title'],
x_axis_values=extra['x_axis_values'],
y_axis_values=extra['y_axis_values'])
if self.state.cursor_area == 'bottom':
selected_unit_highres = plt_plot_heatmap(
data=layer_dat_3D[self.state.selected_unit:self.
state.selected_unit + 1],
shape=(300, 300),
rows=1,
cols=1,
x_axis_label=extra['x_axis'],
y_axis_label=extra['y_axis'],
title='Layer {}, Filter {} \n {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit, extra['title']),
x_axis_values=extra['x_axis_values'],
y_axis_values=extra['y_axis_values'],
hide_axis=False,
vmin=layer_dat_3D.min(),
vmax=layer_dat_3D.max())[0]
else:
layer_dat_3D = extra['x'][self.state.layer_idx]
title, x_axis_label, y_axis_label, r, c, hide_axis = None, None, None, tile_rows, tile_cols, True
if self.state.cursor_area == 'bottom':
if state_layers_pane_zoom_mode == 1:
r, c, hide_axis = 1, 1, False
layer_dat_3D = layer_dat_3D[self.state.
selected_unit:self.
state.selected_unit +
1]
title = 'Layer {}, Filter {} \n {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit, extra['title'])
x_axis_label, y_axis_label = extra[
'x_axis'], extra['y_axis']
if self.state.log_scale == 1:
y_axis_label = y_axis_label + ' (log-scale)'
# start_time = timeit.default_timer()
display_3D = plt_plot_filters_blit(
y=layer_dat_3D,
x=extra['y'],
shape=(pane.data.shape[0], pane.data.shape[1]),
rows=r,
cols=c,
title=title,
log_scale=self.state.log_scale,
x_axis_label=x_axis_label,
y_axis_label=y_axis_label,
hide_axis=hide_axis)
if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 0:
selected_unit_highres = plt_plot_filter(
x=extra['y'],
y=layer_dat_3D[self.state.selected_unit],
title='Layer {}, Filter {} \n {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit, extra['title']),
log_scale=self.state.log_scale,
x_axis_label=extra['x_axis'],
y_axis_label=extra['y_axis'])
# TODO
# if hasattr(self.settings, 'static_files_extra_fn'):
# self.data = self.settings.static_files_extra_fn(self.latest_static_file)
# self.state.layer_idx
if len(layer_dat_3D.shape) == 1:
layer_dat_3D = layer_dat_3D[:, np.newaxis, np.newaxis]
if self.state.layers_show_back and not self.state.pattern_mode:
padval = self.settings.kerasvis_layer_clr_back_background
else:
padval = self.settings.window_background
if self.state.pattern_mode:
# Show desired patterns loaded from disk
load_layer = self.state.layer
if self.settings.kerasvis_jpgvis_remap and self.state.layer in self.settings.kerasvis_jpgvis_remap:
load_layer = self.settings.kerasvis_jpgvis_remap[
self.state.layer]
if self.settings.kerasvis_jpgvis_layers and load_layer in self.settings.kerasvis_jpgvis_layers:
jpg_path = os.path.join(self.settings.kerasvis_unit_jpg_dir,
'regularized_opt', load_layer,
'whole_layer.jpg')
# Get highres version
# cache_before = str(self.img_cache)
display_3D_highres = self.img_cache.get((jpg_path, 'whole'),
None)
# else:
# display_3D_highres = None
if display_3D_highres is None:
try:
with WithTimer('KerasVisApp:load_sprite_image',
quiet=self.debug_level < 1):
display_3D_highres = load_square_sprite_image(
jpg_path, n_sprites=n_tiles)
except IOError:
# File does not exist, so just display disabled.
pass
else:
self.img_cache.set((jpg_path, 'whole'),
display_3D_highres)
# cache_after = str(self.img_cache)
# print 'Cache was / is:\n %s\n %s' % (cache_before, cache_after)
if display_3D_highres is not None:
# Get lowres version, maybe. Assume we want at least one pixel for selection border.
row_downsamp_factor = int(
np.ceil(
float(display_3D_highres.shape[1]) /
(pane.data.shape[0] / tile_rows - 2)))
col_downsamp_factor = int(
np.ceil(
float(display_3D_highres.shape[2]) /
(pane.data.shape[1] / tile_cols - 2)))
ds = max(row_downsamp_factor, col_downsamp_factor)
if ds > 1:
# print 'Downsampling by', ds
display_3D = display_3D_highres[:, ::ds, ::ds, :]
else:
display_3D = display_3D_highres
else:
display_3D = layer_dat_3D * 0 # nothing to show
else:
# Show data from network (activations or diffs)
if self.state.layers_show_back:
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
layer_dat_3D_normalized = np.tile(
self.settings.window_background,
layer_dat_3D.shape + (1, ))
elif back_what_to_disp == 'stale':
layer_dat_3D_normalized = np.tile(
self.settings.stale_background,
layer_dat_3D.shape + (1, ))
else:
layer_dat_3D_normalized = tile_images_normalize(
layer_dat_3D,
boost_indiv=self.state.layer_boost_indiv,
boost_gamma=self.state.layer_boost_gamma,
neg_pos_colors=((1, 0, 0), (0, 1, 0)))
else:
layer_dat_3D_normalized = tile_images_normalize(
layer_dat_3D,
boost_indiv=self.state.layer_boost_indiv,
boost_gamma=self.state.layer_boost_gamma)
# print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()
if state_layers_pane_filter_mode in (0, 1, 2):
display_3D = layer_dat_3D_normalized
# Convert to float if necessary:
display_3D = ensure_float01(display_3D)
# Upsample gray -> color if necessary
# e.g. (1000,32,32) -> (1000,32,32,3)
if len(display_3D.shape) == 3:
display_3D = display_3D[:, :, :, np.newaxis]
if display_3D.shape[3] == 1:
display_3D = np.tile(display_3D, (1, 1, 1, 3))
# Upsample unit length tiles to give a more sane tile / highlight ratio
# e.g. (1000,1,1,3) -> (1000,3,3,3)
if display_3D.shape[1] == 1:
display_3D = np.tile(display_3D, (1, 3, 3, 1))
if state_layers_pane_zoom_mode in (0, 2):
highlights = [None] * n_tiles
with self.state.lock:
if self.state.cursor_area == 'bottom':
highlights[
self.state.
selected_unit] = self.settings.kerasvis_layer_clr_cursor # in [0,1] range
if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
highlights[
self.state.
backprop_unit] = self.settings.kerasvis_layer_clr_back_sel # in [0,1] range
if self.state.cursor_area == 'bottom' and state_layers_pane_filter_mode in (
3, 4):
# pane.data[0:display_2D_resize.shape[0], 0:2, :] = to_255(self.settings.window_background)
# pane.data[0:2, 0:display_2D_resize.shape[1], :] = to_255(self.settings.window_background)
display_3D[self.state.selected_unit, 0:display_3D.shape[1],
0:2, :] = self.settings.kerasvis_layer_clr_cursor
display_3D[
self.state.selected_unit, 0:2, 0:display_3D.
shape[2], :] = self.settings.kerasvis_layer_clr_cursor
display_3D[self.state.selected_unit, 0:display_3D.shape[1],
-2:, :] = self.settings.kerasvis_layer_clr_cursor
display_3D[
self.state.selected_unit, -2:, 0:display_3D.
shape[2], :] = self.settings.kerasvis_layer_clr_cursor
_, display_2D = tile_images_make_tiles(display_3D,
hw=(tile_rows, tile_cols),
padval=padval,
highlights=highlights)
# Mode 0: normal display (activations or patterns)
display_2D_resize = ensure_uint255_and_resize_to_fit(
display_2D, pane.data.shape)
if state_layers_pane_zoom_mode == 2:
display_2D_resize = display_2D_resize * 0
if display_3D_highres is None:
display_3D_highres = display_3D
elif state_layers_pane_zoom_mode == 1:
if display_3D_highres is None:
display_3D_highres = display_3D
# Mode 1: zoomed selection
if state_layers_pane_filter_mode in (0, 1, 2):
unit_data = display_3D_highres[self.state.selected_unit]
else:
unit_data = display_3D_highres[0]
display_2D_resize = ensure_uint255_and_resize_to_fit(
unit_data, pane.data.shape)
pane.data[:] = to_255(self.settings.window_background)
pane.data[0:display_2D_resize.shape[0],
0:display_2D_resize.shape[1], :] = display_2D_resize
# # Add background strip around the top and left edges
# pane.data[0:display_2D_resize.shape[0], 0:2, :] = to_255(self.settings.window_background)
# pane.data[0:2, 0:display_2D_resize.shape[1], :] = to_255(self.settings.window_background)
if self.settings.kerasvis_label_layers and \
self.state.layer in self.settings.kerasvis_label_layers and \
self.labels and self.state.cursor_area == 'bottom':
# Display label annotation atop layers pane (e.g. for fc8/prob)
defaults = {
'face': getattr(cv2, self.settings.kerasvis_label_face),
'fsize': self.settings.kerasvis_label_fsize,
'clr': to_255(self.settings.kerasvis_label_clr),
'thick': self.settings.kerasvis_label_thick
}
loc_base = self.settings.kerasvis_label_loc[::
-1] # Reverse to OpenCV c,r order
lines = [
FormattedString(self.labels[self.state.selected_unit],
defaults)
]
cv2_typeset_text(pane.data, lines, loc_base)
return display_3D_highres, selected_unit_highres
def _draw_layer_pane(self, pane):
'''Returns the data shown in highres format, b01c order.'''
if self.state.layers_show_back:
layer_dat_3D = self.net.blobs[self.state.layer].diff[0]
else:
layer_dat_3D = self.net.blobs[self.state.layer].data[0]
# Promote FC layers with shape (n) to have shape (n,1,1)
if len(layer_dat_3D.shape) == 1:
layer_dat_3D = layer_dat_3D[:,np.newaxis,np.newaxis]
n_tiles = layer_dat_3D.shape[0]
tile_rows,tile_cols = get_tiles_height_width(n_tiles)
display_3D_highres = None
if self.state.pattern_mode:
# Show desired patterns loaded from disk
#available = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'fc6', 'fc7', 'fc8', 'prob']
jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
'regularized_opt', self.state.layer, 'whole_layer.jpg')
# Get highres version
cache_before = str(self.img_cache)
display_3D_highres = self.img_cache.get((jpg_path, 'whole'), None)
if display_3D_highres is None:
try:
with WithTimer('CaffeVisApp:load_sprite_image', quiet = self.debug_level < 1):
display_3D_highres = load_sprite_image(jpg_path, (tile_rows, tile_cols), n_sprites = n_tiles)
except IOError:
# File does not exist, so just display disabled.
pass
else:
self.img_cache.set((jpg_path, 'whole'), display_3D_highres)
cache_after = str(self.img_cache)
#print 'Cache was / is:\n %s\n %s' % (cache_before, cache_after)
if display_3D_highres is not None:
# Get lowres version, maybe. Assume we want at least one pixel for selection border.
row_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[1]) / (pane.data.shape[0] / tile_rows - 2)))
col_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[2]) / (pane.data.shape[1] / tile_cols - 2)))
ds = max(row_downsamp_factor, col_downsamp_factor)
if ds > 1:
#print 'Downsampling by', ds
display_3D = display_3D_highres[:,::ds,::ds,:]
else:
display_3D = display_3D_highres
else:
display_3D = layer_dat_3D * 0 # nothing to show
else:
# Show data from network (activations or diffs)
if self.state.layers_show_back:
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
layer_dat_3D_normalized = np.tile(self.settings.window_background, layer_dat_3D.shape + (1,))
elif back_what_to_disp == 'stale':
layer_dat_3D_normalized = np.tile(self.settings.stale_background, layer_dat_3D.shape + (1,))
else:
layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
boost_indiv = self.state.layer_boost_indiv,
boost_gamma = self.state.layer_boost_gamma,
neg_pos_colors = ((1,0,0), (0,1,0)))
else:
layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
boost_indiv = self.state.layer_boost_indiv,
boost_gamma = self.state.layer_boost_gamma)
#print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()
display_3D = layer_dat_3D_normalized
# Convert to float if necessary:
display_3D = ensure_float01(display_3D)
# Upsample gray -> color if necessary
# (1000,32,32) -> (1000,32,32,3)
if len(display_3D.shape) == 3:
display_3D = display_3D[:,:,:,np.newaxis]
if display_3D.shape[3] == 1:
display_3D = np.tile(display_3D, (1, 1, 1, 3))
# Upsample unit length tiles to give a more sane tile / highlight ratio
# (1000,1,1,3) -> (1000,3,3,3)
if display_3D.shape[1] == 1:
display_3D = np.tile(display_3D, (1, 3, 3, 1))
if self.state.layers_show_back and not self.state.pattern_mode:
padval = self.settings.caffevis_layer_clr_back_background
else:
padval = self.settings.window_background
# Tell the state about the updated (height,width) tile display (ensures valid selection)
self.state.update_tiles_height_width((tile_rows,tile_cols), display_3D.shape[0])
#if self.state.layers_show_back:
# highlights = [(.5, .5, 1)] * n_tiles
#else:
highlights = [None] * n_tiles
with self.state.lock:
if self.state.cursor_area == 'bottom':
highlights[self.state.selected_unit] = self.settings.caffevis_layer_clr_cursor # in [0,1] range
if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
highlights[self.state.backprop_unit] = self.settings.caffevis_layer_clr_back_sel # in [0,1] range
_, display_2D = tile_images_make_tiles(display_3D, padval = padval, highlights = highlights)
#print ' ===tile_conv dtype', tile_conv.dtype, 'range', tile_conv.min(), tile_conv.max()
if display_3D_highres is None:
display_3D_highres = display_3D
# Display pane based on layers_pane_zoom_mode
state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
assert state_layers_pane_zoom_mode in (0,1,2)
if state_layers_pane_zoom_mode == 0:
# Mode 0: base case
display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape)
elif state_layers_pane_zoom_mode == 1:
# Mode 1: zoomed selection
unit_data = display_3D_highres[self.state.selected_unit]
display_2D_resize = ensure_uint255_and_resize_to_fit(unit_data, pane.data.shape)
else:
# Mode 2: ??? backprop ???
display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape) * 0
pane.data[0:display_2D_resize.shape[0], 0:display_2D_resize.shape[1], :] = display_2D_resize
return display_3D_highres
