def _draw_input_label_pane(self, pane, label):
defaults = {
'face': getattr(cv2, self.settings.kerasvis_class_face),
'fsize': self.settings.kerasvis_class_fsize,
'clr': to_255(self.settings.kerasvis_class_clr_0),
'thick': self.settings.kerasvis_class_thick
}
loc = self.settings.kerasvis_class_loc[::
-1] # Reverse to OpenCV c,r order
clr_0 = to_255(self.settings.kerasvis_class_clr_0)
clr_1 = to_255(self.settings.kerasvis_class_clr_1)
strings = []
fs = FormattedString('Actual Label:', defaults)
fs.clr = clr_0
strings.append([fs])
if hasattr(self, 'labels') and self.labels is not None:
lbl = self.labels[np.argmax(label)]
else:
lbl = np.argmax(label)
fs = FormattedString(' {}'.format(lbl), defaults)
fs.clr = clr_1
strings.append([fs])
pane.data[:] = to_255(self.settings.window_background)
cv2_typeset_text(
pane.data,
strings,
loc,
line_spacing=self.settings.kerasvis_class_line_spacing)
def _OLDDEP_draw_control_pane(self, pane):
pane.data[:] = to_255(self.settings.window_background)
with self.state.lock:
layer_idx = self.state.layer_idx
face = getattr(cv2, self.settings.caffevis_control_face)
loc = self.settings.caffevis_control_loc[::-1] # Reverse to OpenCV c,r order
clr = to_255(self.settings.caffevis_control_clr)
clr_sel = to_255(self.settings.caffevis_control_clr_selected)
clr_high = to_255(self.settings.caffevis_control_clr_cursor)
fsize = self.settings.caffevis_control_fsize
thick = self.settings.caffevis_control_thick
thick_sel = self.settings.caffevis_control_thick_selected
thick_high = self.settings.caffevis_control_thick_cursor
st1 = ' '.join(self.layer_print_names[:layer_idx])
st3 = ' '.join(self.layer_print_names[layer_idx+1:])
st2 = ((' ' if len(st1) > 0 else '')
+ self.layer_print_names[layer_idx]
+ (' ' if len(st3) > 0 else ''))
st1 = ' ' + st1
cv2.putText(pane.data, st1, loc, face, fsize, clr, thick)
boxsize1, _ = cv2.getTextSize(st1, face, fsize, thick)
loc = (loc[0] + boxsize1[0], loc[1])
if self.state.cursor_area == 'top':
clr_this, thick_this = clr_high, thick_high
else:
clr_this, thick_this = clr_sel, thick_sel
cv2.putText(pane.data, st2, loc, face, fsize, clr_this, thick_this)
boxsize2, _ = cv2.getTextSize(st2, face, fsize, thick_this)
loc = (loc[0] + boxsize2[0], loc[1])
cv2.putText(pane.data, st3, loc, face, fsize, clr, thick)
def _draw_prob_labels_pane(self, pane):
'''Adds text label annotation atop the given pane.'''
if not self.labels or not self.state.show_label_predictions or not self.settings.caffevis_prob_layer:
return
#pane.data[:] = to_255(self.settings.window_background)
defaults = {'face': getattr(cv2, self.settings.caffevis_class_face),
'fsize': self.settings.caffevis_class_fsize,
'clr': to_255(self.settings.caffevis_class_clr_0),
'thick': self.settings.caffevis_class_thick}
loc = self.settings.caffevis_class_loc[::-1] # Reverse to OpenCV c,r order
clr_0 = to_255(self.settings.caffevis_class_clr_0)
clr_1 = to_255(self.settings.caffevis_class_clr_1)
probs_flat = self.net.blobs[self.settings.caffevis_prob_layer].data.flatten()
top_5 = probs_flat.argsort()[-1:-6:-1]
strings = []
pmax = probs_flat[top_5[0]]
for idx in top_5:
prob = probs_flat[idx]
text = '%.2f %s' % (prob, self.labels[idx])
fs = FormattedString(text, defaults)
#fs.clr = tuple([clr_1[ii]*prob/pmax + clr_0[ii]*(1-prob/pmax) for ii in range(3)])
fs.clr = tuple([max(0,min(255,clr_1[ii]*prob + clr_0[ii]*(1-prob))) for ii in range(3)])
strings.append([fs]) # Line contains just fs
cv2_typeset_text(pane.data, strings, loc,
line_spacing = self.settings.caffevis_class_line_spacing)
def _draw_status_pane(self, pane):
pane.data[:] = to_255(self.settings.window_background)
defaults = {'face': getattr(cv2, self.settings.caffevis_status_face),
'fsize': self.settings.caffevis_status_fsize,
'clr': to_255(self.settings.caffevis_status_clr),
'thick': self.settings.caffevis_status_thick}
loc = self.settings.caffevis_status_loc[::-1] # Reverse to OpenCV c,r order
status = StringIO.StringIO()
fps = self.proc_thread.approx_fps()
with self.state.lock:
print >>status, 'pattern' if self.state.pattern_mode else ('back' if self.state.layers_show_back else 'fwd'),
print >>status, '%s:%d |' % (self.state.layer, self.state.selected_unit),
if not self.state.back_enabled:
print >>status, 'Back: off',
else:
print >>status, 'Back: %s' % ('deconv' if self.state.back_mode == 'deconv' else 'bprop'),
print >>status, '(from %s_%d, disp %s)' % (self.state.backprop_layer,
self.state.backprop_unit,
self.state.back_filt_mode),
print >>status, '|',
print >>status, 'Boost: %g/%g' % (self.state.layer_boost_indiv, self.state.layer_boost_gamma)
if fps > 0:
print >>status, '| FPS: %.01f' % fps
if self.state.extra_msg:
print >>status, '|', self.state.extra_msg
self.state.extra_msg = ''
strings = [FormattedString(line, defaults) for line in status.getvalue().split('\n')]
cv2_typeset_text(pane.data, strings, loc,
line_spacing = self.settings.caffevis_status_line_spacing)
def _draw_prob_labels_pane(self, pane):
'''Adds text label annotation atop the given pane.'''
if not self.labels or not self.state.show_label_predictions or not self.settings.caffevis_prob_layer:
return
#pane.data[:] = to_255(self.settings.window_background)
defaults = {'face': getattr(cv2, self.settings.caffevis_class_face),
'fsize': self.settings.caffevis_class_fsize,
'clr': to_255(self.settings.caffevis_class_clr_0),
'thick': self.settings.caffevis_class_thick}
loc = self.settings.caffevis_class_loc[::-1] # Reverse to OpenCV c,r order
clr_0 = to_255(self.settings.caffevis_class_clr_0)
clr_1 = to_255(self.settings.caffevis_class_clr_1)
probs_flat = self.net.blobs[self.settings.caffevis_prob_layer].data.flatten()
top_5 = probs_flat.argsort()[-1:-6:-1]
strings = []
pmax = probs_flat[top_5[0]]
for idx in top_5:
prob = probs_flat[idx]
text = '%.2f %s' % (prob, self.labels[idx])
fs = FormattedString(text, defaults)
#fs.clr = tuple([clr_1[ii]*prob/pmax + clr_0[ii]*(1-prob/pmax) for ii in range(3)])
fs.clr = tuple([max(0,min(255,clr_1[ii]*prob + clr_0[ii]*(1-prob))) for ii in range(3)])
strings.append([fs]) # Line contains just fs
cv2_typeset_text(pane.data, strings, loc,
line_spacing = self.settings.caffevis_class_line_spacing)
def _draw_control_pane(self, pane):
pane.data[:] = to_255(self.settings.window_background)
with self.state.lock:
layer_idx = self.state.layer_idx
loc = self.settings.caffevis_control_loc[::-1] # Reverse to OpenCV c,r order
strings = []
defaults = {'face': getattr(cv2, self.settings.caffevis_control_face),
'fsize': self.settings.caffevis_control_fsize,
'clr': to_255(self.settings.caffevis_control_clr),
'thick': self.settings.caffevis_control_thick}
for ii in range(len(self.layer_print_names)):
fs = FormattedString(self.layer_print_names[ii], defaults)
this_layer = self.state._layers[ii]
if self.state.backprop_selection_frozen and this_layer == self.state.backprop_layer:
fs.clr = to_255(self.settings.caffevis_control_clr_bp)
fs.thick = self.settings.caffevis_control_thick_bp
if this_layer == self.state.layer:
if self.state.cursor_area == 'top':
fs.clr = to_255(self.settings.caffevis_control_clr_cursor)
fs.thick = self.settings.caffevis_control_thick_cursor
else:
if not (self.state.backprop_selection_frozen and this_layer == self.state.backprop_layer):
fs.clr = to_255(self.settings.caffevis_control_clr_selected)
fs.thick = self.settings.caffevis_control_thick_selected
strings.append(fs)
cv2_typeset_text(pane.data, strings, loc,
line_spacing = self.settings.caffevis_control_line_spacing,
wrap = True)
def _draw_status_pane(self, pane):
pane.data[:] = to_255(self.settings.window_background)
defaults = {'face': getattr(cv2, self.settings.caffevis_status_face),
'fsize': self.settings.caffevis_status_fsize,
'clr': to_255(self.settings.caffevis_status_clr),
'thick': self.settings.caffevis_status_thick}
loc = self.settings.caffevis_status_loc[::-1] # Reverse to OpenCV c,r order
status = StringIO.StringIO()
fps = self.proc_thread.approx_fps()
with self.state.lock:
print >>status, 'pattern' if self.state.pattern_mode else ('back' if self.state.layers_show_back else 'fwd'),
print >>status, '%s:%d |' % (self.state.layer, self.state.selected_unit),
if not self.state.back_enabled:
print >>status, 'Back: off',
else:
print >>status, 'Back: %s' % ('deconv' if self.state.back_mode == 'deconv' else 'bprop'),
print >>status, '(from %s_%d, disp %s)' % (self.state.backprop_layer,
self.state.backprop_unit,
self.state.back_filt_mode),
print >>status, '|',
print >>status, 'Boost: %g/%g' % (self.state.layer_boost_indiv, self.state.layer_boost_gamma)
if fps > 0:
print >>status, '| FPS: %.01f' % fps
if self.state.extra_msg:
print >>status, '|', self.state.extra_msg
self.state.extra_msg = ''
strings = [FormattedString(line, defaults) for line in status.getvalue().split('\n')]
cv2_typeset_text(pane.data, strings, loc,
line_spacing = self.settings.caffevis_status_line_spacing)
def _draw_control_pane(self, pane):
pane.data[:] = to_255(self.settings.window_background)
with self.state.lock:
layer_idx = self.state.layer_idx
loc = self.settings.caffevis_control_loc[::-1] # Reverse to OpenCV c,r order
strings = []
defaults = {'face': getattr(cv2, self.settings.caffevis_control_face),
'fsize': self.settings.caffevis_control_fsize,
'clr': to_255(self.settings.caffevis_control_clr),
'thick': self.settings.caffevis_control_thick}
for ii in range(len(self.layer_print_names)):
fs = FormattedString(self.layer_print_names[ii], defaults)
this_layer = self.state._layers[ii]
if self.state.backprop_selection_frozen and this_layer == self.state.backprop_layer:
fs.clr = to_255(self.settings.caffevis_control_clr_bp)
fs.thick = self.settings.caffevis_control_thick_bp
if this_layer == self.state.layer:
if self.state.cursor_area == 'top':
fs.clr = to_255(self.settings.caffevis_control_clr_cursor)
fs.thick = self.settings.caffevis_control_thick_cursor
else:
if not (self.state.backprop_selection_frozen and this_layer == self.state.backprop_layer):
fs.clr = to_255(self.settings.caffevis_control_clr_selected)
fs.thick = self.settings.caffevis_control_thick_selected
strings.append(fs)
cv2_typeset_text(pane.data, strings, loc,
line_spacing = self.settings.caffevis_control_line_spacing,
wrap = True)
def _draw_jpgvis_pane(self, pane):
pane.data[:] = to_255(self.settings.window_background)
with self.state.lock:
state_layer, state_selected_unit, cursor_area, show_unit_jpgs = self.state.layer, self.state.selected_unit, self.state.cursor_area, self.state.show_unit_jpgs
available = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'fc6', 'fc7', 'fc8', 'prob']
if state_layer in available and cursor_area == 'bottom' and show_unit_jpgs:
img_key = (state_layer, state_selected_unit, pane.data.shape)
img_resize = self.img_cache.get(img_key, None)
if img_resize is None:
# If img_resize is None, loading has not yet been attempted, so show stale image and request load by JPGVisLoadingThread
with self.state.lock:
self.state.jpgvis_to_load_key = img_key
pane.data[:] = to_255(self.settings.stale_background)
elif img_resize.nbytes == 0:
# This is the sentinal value when the image is not
# found, i.e. loading was already attempted but no jpg
# assets were found. Just display disabled.
pane.data[:] = to_255(self.settings.window_background)
else:
# Show image
pane.data[:img_resize.shape[0], :img_resize.shape[1], :] = img_resize
else:
# Will never be available
pane.data[:] = to_255(self.settings.window_background)
def _draw_back_pane(self, pane):
mode = None
with self.state.lock:
back_enabled = self.state.back_enabled
back_mode = self.state.back_mode
back_filt_mode = self.state.back_filt_mode
state_layer = self.state.layer
selected_unit = self.state.selected_unit
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
pane.data[:] = to_255(self.settings.window_background)
elif back_what_to_disp == 'stale':
pane.data[:] = to_255(self.settings.stale_background)
else:
# One of the backprop modes is enabled and the back computation (gradient or deconv) is up to date
grad_blob = self.net.blobs['data'].diff
# Manually deprocess (skip mean subtraction and rescaling)
#grad_img = self.net.deprocess('data', diff_blob)
grad_blob = grad_blob[0] # bc01 -> c01
grad_blob = grad_blob.transpose((1, 2, 0)) # c01 -> 01c
grad_img = grad_blob[:, :,
self._net_channel_swap_inv] # e.g. BGR -> RGB
# Mode-specific processing
assert back_mode in ('grad', 'deconv')
assert back_filt_mode in ('raw', 'gray', 'norm', 'normblur')
if back_filt_mode == 'raw':
grad_img = norm01c(grad_img, 0)
elif back_filt_mode == 'gray':
grad_img = grad_img.mean(axis=2)
grad_img = norm01c(grad_img, 0)
elif back_filt_mode == 'norm':
grad_img = np.linalg.norm(grad_img, axis=2)
grad_img = norm01(grad_img)
else:
grad_img = np.linalg.norm(grad_img, axis=2)
cv2.GaussianBlur(grad_img, (0, 0),
self.settings.caffevis_grad_norm_blur_radius,
grad_img)
grad_img = norm01(grad_img)
# If necessary, re-promote from grayscale to color
if len(grad_img.shape) == 2:
grad_img = np.tile(grad_img[:, :, np.newaxis], 3)
grad_img_resize = ensure_uint255_and_resize_to_fit(
grad_img, pane.data.shape)
pane.data[0:grad_img_resize.shape[0],
0:grad_img_resize.shape[1], :] = grad_img_resize
def _draw_back_pane(self, pane):
mode = None
with self.state.lock:
back_enabled = self.state.back_enabled
back_mode = self.state.back_mode
back_filt_mode = self.state.back_filt_mode
state_layer = self.state.layer
selected_unit = self.state.selected_unit
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
pane.data[:] = to_255(self.settings.window_background)
elif back_what_to_disp == 'stale':
pane.data[:] = to_255(self.settings.stale_background)
else:
# One of the backprop modes is enabled and the back computation (gradient or deconv) is up to date
grad_blob = self.net.blobs['data'].diff
# Manually deprocess (skip mean subtraction and rescaling)
#grad_img = self.net.deprocess('data', diff_blob)
grad_blob = grad_blob[0] # bc01 -> c01
grad_blob = grad_blob.transpose((1,2,0)) # c01 -> 01c
grad_img = grad_blob[:, :, self._net_channel_swap_inv] # e.g. BGR -> RGB
# Mode-specific processing
assert back_mode in ('grad', 'deconv')
assert back_filt_mode in ('raw', 'gray', 'norm', 'normblur')
if back_filt_mode == 'raw':
grad_img = norm01c(grad_img, 0)
elif back_filt_mode == 'gray':
grad_img = grad_img.mean(axis=2)
grad_img = norm01c(grad_img, 0)
elif back_filt_mode == 'norm':
grad_img = np.linalg.norm(grad_img, axis=2)
grad_img = norm01(grad_img)
else:
grad_img = np.linalg.norm(grad_img, axis=2)
cv2.GaussianBlur(grad_img, (0,0), self.settings.caffevis_grad_norm_blur_radius, grad_img)
grad_img = norm01(grad_img)
# If necessary, re-promote from grayscale to color
if len(grad_img.shape) == 2:
grad_img = np.tile(grad_img[:,:,np.newaxis], 3)
grad_img_resize = ensure_uint255_and_resize_to_fit(grad_img, pane.data.shape)
pane.data[0:grad_img_resize.shape[0], 0:grad_img_resize.shape[1], :] = grad_img_resize
def _draw_back_pane(self, pane):
mode = None
with self.state.lock:
back_enabled = self.state.back_enabled
back_mode = self.state.back_mode
back_filt_mode = self.state.back_filt_mode
state_layer = self.state.layer
selected_unit = self.state.selected_unit
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
pane.data[:] = to_255(self.settings.window_background)
elif back_what_to_disp == 'stale':
pane.data[:] = to_255(self.settings.stale_background)
else:
# One of the backprop modes is enabled and the back computation (gradient or deconv) is up to date
grad_img = self.my_net.get_input_gradient_as_image()
# Mode-specific processing
assert back_mode in ('grad', 'deconv')
assert back_filt_mode in ('raw', 'gray', 'norm', 'normblur')
if back_filt_mode == 'raw':
grad_img = norm01c(grad_img, 0)
elif back_filt_mode == 'gray':
grad_img = grad_img.mean(axis=2)
grad_img = norm01c(grad_img, 0)
elif back_filt_mode == 'norm':
grad_img = np.linalg.norm(grad_img, axis=2)
grad_img = norm01(grad_img)
else:
grad_img = np.linalg.norm(grad_img, axis=2)
cv2.GaussianBlur(grad_img, (0, 0),
self.settings.caffevis_grad_norm_blur_radius,
grad_img)
grad_img = norm01(grad_img)
# If necessary, re-promote from grayscale to color
if len(grad_img.shape) == 2:
grad_img = np.tile(grad_img[:, :, np.newaxis], 3)
grad_img_resize = ensure_uint255_and_resize_to_fit(
grad_img, pane.data.shape)
pane.data[0:grad_img_resize.shape[0],
0:grad_img_resize.shape[1], :] = grad_img_resize
def __init__(self, settings):
self.settings = settings
self.bindings = bindings
self.app_classes = OrderedDict()
self.apps = OrderedDict()
for module_path, app_name in settings.installed_apps:
module = importlib.import_module(module_path)
print 'got module', module
app_class = getattr(module, app_name)
print 'got app', app_class
self.app_classes[app_name] = app_class
for app_name, app_class in self.app_classes.iteritems():
app = app_class(settings, self.bindings)
self.apps[app_name] = app
self.help_mode = False
self.window_name = 'Deep Visualization Toolbox'
self.quit = False
self.debug_level = 0
self.debug_pane_defaults = {
'face': getattr(cv2, self.settings.help_face),
'fsize': self.settings.help_fsize,
'clr': pane_debug_clr,
'thick': self.settings.help_thick
}
self.help_pane_defaults = {
'face': getattr(cv2, self.settings.help_face),
'fsize': self.settings.help_fsize,
'clr': to_255(self.settings.help_clr),
'thick': self.settings.help_thick
}
def __init__(self, settings):
self.settings = settings
self.bindings = bindings
self.app_classes = OrderedDict()
self.apps = OrderedDict()
for module_path, app_name in settings.installed_apps:
module = importlib.import_module(module_path)
print('got module: {}'.format(module))
app_class = getattr(module, app_name)
print('got app: {}'.format(app_class))
self.app_classes[app_name] = app_class
for app_name, app_class in iter(self.app_classes.items()):
app = app_class(settings, self.bindings)
self.apps[app_name] = app
self.help_mode = False
self.window_name = 'Deep Visualization Toolbox | Model: %s' % (settings.model_to_load)
self.quit = False
self.debug_level = 0
self.debug_pane_defaults = {
'face': getattr(cv2, self.settings.help_face),
'fsize': self.settings.help_fsize,
'clr': pane_debug_clr,
'thick': self.settings.help_thick
}
self.help_pane_defaults = {
'face': getattr(cv2, self.settings.help_face),
'fsize': self.settings.help_fsize,
'clr': to_255(self.settings.help_clr),
'thick': self.settings.help_thick
}
def _draw_aux_pane(self, pane, layer_data_normalized):
pane.data[:] = to_255(self.settings.window_background)
mode = None
with self.state.lock:
if self.state.cursor_area == 'bottom':
mode = 'selected'
else:
mode = 'prob_labels'
if mode == 'selected':
unit_data = layer_data_normalized[self.state.selected_unit]
# # Edited
# # -----------------------
# print '*'*100
# print unit_data.mean(axis=(0,1))
unit_data_resize = ensure_uint255_and_resize_to_fit(unit_data, pane.data.shape)
pane.data[0:unit_data_resize.shape[0], 0:unit_data_resize.shape[1], :] = unit_data_resize
elif mode == 'prob_labels':
self._draw_prob_labels_pane(pane)
def __init__(self, settings):
self.settings = settings
self.bindings = bindings
self.app_classes = OrderedDict()
self.apps = OrderedDict()
for module_path, app_name in settings.installed_apps:
module = importlib.import_module(module_path)
print 'debug[app]: LiveVis.__init__: got module', module
app_class = getattr(module, app_name)
print 'debug[app]: LiveVis.__init__: got app', app_class
self.app_classes[app_name] = app_class
for app_name, app_class in self.app_classes.iteritems():
app = app_class(settings, self.bindings)
print 'debug[app]: LiveVis.__init__: initialized app', app_name, 'as class', app_class
self.apps[app_name] = app
self.help_mode = False
self.window_name = 'Deep Visualization Toolbox'
self.quit = False
self.debug_level = 0
self.debug_pane_defaults = {
'face': getattr(cv2, self.settings.help_face),
'fsize': self.settings.help_fsize,
'clr': pane_debug_clr,
'thick': self.settings.help_thick
}
self.help_pane_defaults = {
'face': getattr(cv2, self.settings.help_face),
'fsize': self.settings.help_fsize,
'clr': to_255(self.settings.help_clr),
'thick': self.settings.help_thick
}
def _draw_aux_pane(self,
pane,
layer_data_normalized,
selected_unit_highres=None):
pane.data[:] = to_255(self.settings.window_background)
with self.state.lock:
if self.state.layers_pane_zoom_mode == 1:
mode = 'prob_labels'
elif self.state.cursor_area == 'bottom' and layer_data_normalized is not None:
mode = 'selected'
elif self.state.layers_pane_filter_mode in (0, 1, 2, 3):
mode = 'prob_labels'
else:
mode = 'none'
# if mode == 'selected' and layer_data_normalized is None and selected_unit_highres is None:
# mode = 'prob_labels'
if mode == 'selected':
if selected_unit_highres is not None:
unit_data = selected_unit_highres
else:
unit_data = layer_data_normalized[self.state.selected_unit]
unit_data_resize = ensure_uint255_and_resize_to_fit(
unit_data, pane.data.shape)
pane.data[0:unit_data_resize.shape[0],
0:unit_data_resize.shape[1], :] = unit_data_resize
elif mode == 'prob_labels':
self._draw_prob_labels_pane(pane)
def draw_help(self, help_pane, locy):
'''Tells the app to draw its help screen in the given pane.
:param help_pane:
a Pane to use for displaying the help for this application.
:param locy:
the vertical position within the help_pane.
'''
defaults = {
'face': getattr(cv2, self.settings.help_face),
'fsize': self.settings.help_fsize,
'clr': to_255(self.settings.help_clr),
'thick': self.settings.help_thick
}
loc_base = self.settings.help_loc[::-1] # Reverse to OpenCV c,r order
locx = loc_base[0]
lines = []
lines.append([FormattedString('', defaults)])
lines.append([FormattedString('Caffevis keys', defaults)])
kl, _ = self.bindings.get_key_help('sel_left')
kr, _ = self.bindings.get_key_help('sel_right')
ku, _ = self.bindings.get_key_help('sel_up')
kd, _ = self.bindings.get_key_help('sel_down')
klf, _ = self.bindings.get_key_help('sel_left_fast')
krf, _ = self.bindings.get_key_help('sel_right_fast')
kuf, _ = self.bindings.get_key_help('sel_up_fast')
kdf, _ = self.bindings.get_key_help('sel_down_fast')
keys_nav_0 = ','.join([kk[0] for kk in (kl, kr, ku, kd)])
keys_nav_1 = ''
if len(kl) > 1 and len(kr) > 1 and len(ku) > 1 and len(kd) > 1:
keys_nav_1 += ' or '
keys_nav_1 += ','.join([kk[1] for kk in (kl, kr, ku, kd)])
keys_nav_f = ','.join([kk[0] for kk in (klf, krf, kuf, kdf)])
nav_string = 'Navigate with %s%s. Use %s to move faster.' % (
keys_nav_0, keys_nav_1, keys_nav_f)
lines.append([
FormattedString('', defaults, width=120, align='right'),
FormattedString(nav_string, defaults)
])
for tag in ('sel_layer_left', 'sel_layer_right', 'zoom_mode',
'pattern_mode', 'ez_back_mode_loop', 'freeze_back_unit',
'show_back', 'back_mode', 'back_filt_mode', 'boost_gamma',
'boost_individual', 'reset_state'):
key_strings, help_string = self.bindings.get_key_help(tag)
label = '%10s:' % (','.join(key_strings))
lines.append([
FormattedString(label, defaults, width=120, align='right'),
FormattedString(help_string, defaults)
])
locy = cv2_typeset_text(help_pane.data,
lines, (locx, locy),
line_spacing=self.settings.help_line_spacing)
return locy
def draw_help(self, help_pane, locy):
defaults = {'face': getattr(cv2, self.settings.caffevis_help_face),
'fsize': self.settings.caffevis_help_fsize,
'clr': to_255(self.settings.caffevis_help_clr),
'thick': self.settings.caffevis_help_thick}
loc_base = self.settings.caffevis_help_loc[::-1] # Reverse to OpenCV c,r order
locx = loc_base[0]
lines = []
lines.append([FormattedString('', defaults)])
lines.append([FormattedString('Caffevis keys', defaults)])
kl,_ = self.bindings.get_key_help('sel_left')
kr,_ = self.bindings.get_key_help('sel_right')
ku,_ = self.bindings.get_key_help('sel_up')
kd,_ = self.bindings.get_key_help('sel_down')
klf,_ = self.bindings.get_key_help('sel_left_fast')
krf,_ = self.bindings.get_key_help('sel_right_fast')
kuf,_ = self.bindings.get_key_help('sel_up_fast')
kdf,_ = self.bindings.get_key_help('sel_down_fast')
keys_nav_0 = ','.join([kk[0] for kk in (kl, kr, ku, kd)])
keys_nav_1 = ''
if len(kl)>1 and len(kr)>1 and len(ku)>1 and len(kd)>1:
keys_nav_1 += ' or '
keys_nav_1 += ','.join([kk[1] for kk in (kl, kr, ku, kd)])
keys_nav_f = ','.join([kk[0] for kk in (klf, krf, kuf, kdf)])
nav_string = 'Navigate with %s%s. Use %s to move faster.' % (keys_nav_0, keys_nav_1, keys_nav_f)
lines.append([FormattedString('', defaults, width=120, align='right'),
FormattedString(nav_string, defaults)])
#label = '%10s:' % (
#help_string = 'Move cursor left, right, up, or down'
#lines.append([FormattedString(label, defaults, width=120, align='right'),
# FormattedString(help_string, defaults)])
#if len(kl)>1 and len(kr)>1 and len(ku)>1 and len(kd)>1:
# label = '%10s:' % (','.join([kk[1] for kk in (kl, kr, ku, kd)]))
# help_string = 'Move cursor left, right, up, or down'
# lines.append([FormattedString(label, defaults, width=120, align='right'),
# FormattedString(help_string, defaults)])
#label = '%10s:' % (','.join([kk[0] for kk in (klf, krf, kuf, kdf)]))
#help_string = 'Move cursor left, right, up, or down (faster)'
#lines.append([FormattedString(label, defaults, width=120, align='right'),
# FormattedString(help_string, defaults)])
for tag in ('sel_layer_left', 'sel_layer_right', 'zoom_mode', 'pattern_mode',
'ez_back_mode_loop', 'freeze_back_unit', 'show_back', 'back_mode', 'back_filt_mode',
'boost_gamma', 'boost_individual', 'reset_state'):
key_strings, help_string = self.bindings.get_key_help(tag)
label = '%10s:' % (','.join(key_strings))
lines.append([FormattedString(label, defaults, width=120, align='right'),
FormattedString(help_string, defaults)])
locy = cv2_typeset_text(help_pane.data, lines, (locx, locy),
line_spacing = self.settings.caffevis_help_line_spacing)
return locy
def draw_help(self):
self.help_buffer[:] *= 0.7
self.help_pane.data *= 0.7
# pane.data[:] = to_255(self.settings.window_background)
defaults = {
"face": getattr(cv2, self.settings.caffevis_help_face),
"fsize": self.settings.caffevis_help_fsize,
"clr": to_255(self.settings.caffevis_help_clr),
"thick": self.settings.caffevis_help_thick,
}
loc = self.settings.caffevis_help_loc[::-1] # Reverse to OpenCV c,r order
lines = []
lines.append(
[
FormattedString(
"~ ~ ~ Deep Visualization Toolbox ~ ~ ~", defaults, align="center", width=self.help_pane.j_size
)
]
)
lines.append([FormattedString("", defaults)])
lines.append([FormattedString("Base keys", defaults)])
# lines.append([FormattedString('lllll', defaults), FormattedString('WWWW', defaults)])
# lines.append([FormattedString('WWWWW', defaults), FormattedString('llll', defaults)])
# lines.append([FormattedString('lllll', defaults, width=150), FormattedString('WWWW', defaults, width=150)])
# lines.append([FormattedString('WWWWW', defaults, width=150), FormattedString('llll', defaults, width=150)])
# lines.append([FormattedString('AAA', defaults),
# FormattedString('left', defaults, width = 300),
# FormattedString('BBB', defaults)])
# lines.append([FormattedString('AAA', defaults),
# FormattedString('center', defaults, width = 300, align='center'),
# FormattedString('BBB', defaults)])
# lines.append([FormattedString('AAA', defaults),
# FormattedString('right', defaults, width = 300, align='right'),
# FormattedString('BBB', defaults)])
for tag in (
"help_mode",
"freeze_cam",
"toggle_input_mode",
"static_file_increment",
"static_file_decrement",
"stretch_mode",
"quit",
):
key_strings, help_string = self.bindings.get_key_help(tag)
label = "%10s:" % (",".join(key_strings))
lines.append(
[FormattedString(label, defaults, width=120, align="right"), FormattedString(help_string, defaults)]
)
locy = cv2_typeset_text(self.help_pane.data, lines, loc, line_spacing=self.settings.caffevis_help_line_spacing)
for app_name, app in self.apps.iteritems():
locy = app.draw_help(self.help_pane, locy)
def draw_help(self):
self.help_buffer[:] *= .7
self.help_pane.data *= .7
#pane.data[:] = to_255(self.settings.window_background)
defaults = {
'face': getattr(cv2, self.settings.caffevis_help_face),
'fsize': self.settings.caffevis_help_fsize,
'clr': to_255(self.settings.caffevis_help_clr),
'thick': self.settings.caffevis_help_thick
}
loc = self.settings.caffevis_help_loc[::
-1] # Reverse to OpenCV c,r order
lines = []
lines.append([
FormattedString(
'~ ~ ~ Deep Visualization Toolbox ~ ~ ~',
defaults,
align='center',
width=self.help_pane.j_size)
])
lines.append([FormattedString('', defaults)])
lines.append([FormattedString('Base keys', defaults)])
#lines.append([FormattedString('lllll', defaults), FormattedString('WWWW', defaults)])
#lines.append([FormattedString('WWWWW', defaults), FormattedString('llll', defaults)])
#lines.append([FormattedString('lllll', defaults, width=150), FormattedString('WWWW', defaults, width=150)])
#lines.append([FormattedString('WWWWW', defaults, width=150), FormattedString('llll', defaults, width=150)])
#lines.append([FormattedString('AAA', defaults),
# FormattedString('left', defaults, width = 300),
# FormattedString('BBB', defaults)])
#lines.append([FormattedString('AAA', defaults),
# FormattedString('center', defaults, width = 300, align='center'),
# FormattedString('BBB', defaults)])
#lines.append([FormattedString('AAA', defaults),
# FormattedString('right', defaults, width = 300, align='right'),
# FormattedString('BBB', defaults)])
for tag in ('help_mode', 'freeze_cam', 'toggle_input_mode',
'static_file_increment', 'static_file_decrement',
'stretch_mode', 'quit'):
key_strings, help_string = self.bindings.get_key_help(tag)
label = '%10s:' % (','.join(key_strings))
lines.append([
FormattedString(label, defaults, width=120, align='right'),
FormattedString(help_string, defaults)
])
locy = cv2_typeset_text(
self.help_pane.data,
lines,
loc,
line_spacing=self.settings.caffevis_help_line_spacing)
for app_name, app in self.apps.iteritems():
locy = app.draw_help(self.help_pane, locy)
def draw_help(self):
self.help_buffer[:] *= .7
self.help_pane.data *= .7
#pane.data[:] = to_255(self.settings.window_background)
defaults = {
'face': getattr(cv2, self.settings.caffevis_help_face),
'fsize': self.settings.caffevis_help_fsize,
'clr': to_255(self.settings.caffevis_help_clr),
'thick': self.settings.caffevis_help_thick
}
loc = self.settings.caffevis_help_loc[::
-1] # Reverse to OpenCV c,r order
lines = []
lines.append([
FormattedString('~ ~ ~ Deep Visualization Toolbox ~ ~ ~',
defaults,
align='center',
width=self.help_pane.j_size)
])
lines.append([FormattedString('', defaults)])
lines.append([FormattedString('Base keys', defaults)])
#lines.append([FormattedString('lllll', defaults), FormattedString('WWWW', defaults)])
#lines.append([FormattedString('WWWWW', defaults), FormattedString('llll', defaults)])
#lines.append([FormattedString('lllll', defaults, width=150), FormattedString('WWWW', defaults, width=150)])
#lines.append([FormattedString('WWWWW', defaults, width=150), FormattedString('llll', defaults, width=150)])
#lines.append([FormattedString('AAA', defaults),
# FormattedString('left', defaults, width = 300),
# FormattedString('BBB', defaults)])
#lines.append([FormattedString('AAA', defaults),
# FormattedString('center', defaults, width = 300, align='center'),
# FormattedString('BBB', defaults)])
#lines.append([FormattedString('AAA', defaults),
# FormattedString('right', defaults, width = 300, align='right'),
# FormattedString('BBB', defaults)])
for tag in ('help_mode', 'freeze_cam', 'toggle_input_mode',
'static_file_increment', 'static_file_decrement',
'stretch_mode', 'quit'):
key_strings, help_string = self.bindings.get_key_help(tag)
label = '%10s:' % (','.join(key_strings))
lines.append([
FormattedString(label, defaults, width=120, align='right'),
FormattedString(help_string, defaults)
])
locy = cv2_typeset_text(
self.help_pane.data,
lines,
loc,
line_spacing=self.settings.caffevis_help_line_spacing)
for app_name, app in self.apps.iteritems():
locy = app.draw_help(self.help_pane, locy)
def _draw_jpgvis_pane(self, pane):
pane.data[:] = to_255(self.settings.window_background)
with self.state.lock:
state_layer, state_selected_unit, cursor_area, show_unit_jpgs = self.state.layer, self.state.selected_unit, self.state.cursor_area, self.state.show_unit_jpgs
try:
# Some may be missing this setting
self.settings.caffevis_jpgvis_layers
except:
print(
'\n\nNOTE: you need to upgrade your settings.py and settings_local.py files. See README.md.\n\n'
)
raise
if self.settings.caffevis_jpgvis_remap and state_layer in self.settings.caffevis_jpgvis_remap:
img_key_layer = self.settings.caffevis_jpgvis_remap[state_layer]
else:
img_key_layer = state_layer
if self.settings.caffevis_jpgvis_layers and img_key_layer in self.settings.caffevis_jpgvis_layers and cursor_area == 'bottom' and show_unit_jpgs:
img_key = (img_key_layer, state_selected_unit, pane.data.shape)
img_resize = self.img_cache.get(img_key, None)
if img_resize is None:
# If img_resize is None, loading has not yet been attempted, so show stale image and request load by JPGVisLoadingThread
with self.state.lock:
self.state.jpgvis_to_load_key = img_key
pane.data[:] = to_255(self.settings.stale_background)
elif img_resize.nbytes == 0:
# This is the sentinal value when the image is not
# found, i.e. loading was already attempted but no jpg
# assets were found. Just display disabled.
pane.data[:] = to_255(self.settings.window_background)
else:
# Show image
pane.data[:img_resize.shape[0], :img_resize.
shape[1], :] = img_resize
else:
# Will never be available
pane.data[:] = to_255(self.settings.window_background)
def _draw_jpgvis_pane(self, pane):
pane.data[:] = to_255(self.settings.window_background)
with self.state.lock:
state_layer, state_selected_unit, cursor_area, show_unit_jpgs = self.state.layer, self.state.selected_unit, self.state.cursor_area, self.state.show_unit_jpgs
try:
# Some may be missing this setting
self.settings.caffevis_jpgvis_layers
except:
print '\n\nNOTE: you need to upgrade your settings.py and settings_local.py files. See README.md.\n\n'
raise
if self.settings.caffevis_jpgvis_remap and state_layer in self.settings.caffevis_jpgvis_remap:
img_key_layer = self.settings.caffevis_jpgvis_remap[state_layer]
else:
img_key_layer = state_layer
if self.settings.caffevis_jpgvis_layers and img_key_layer in self.settings.caffevis_jpgvis_layers and cursor_area == 'bottom' and show_unit_jpgs:
img_key = (img_key_layer, state_selected_unit, pane.data.shape)
img_resize = self.img_cache.get(img_key, None)
if img_resize is None:
# If img_resize is None, loading has not yet been attempted, so show stale image and request load by JPGVisLoadingThread
with self.state.lock:
self.state.jpgvis_to_load_key = img_key
pane.data[:] = to_255(self.settings.stale_background)
elif img_resize.nbytes == 0:
# This is the sentinal value when the image is not
# found, i.e. loading was already attempted but no jpg
# assets were found. Just display disabled.
pane.data[:] = to_255(self.settings.window_background)
else:
# Show image
pane.data[:img_resize.shape[0], :img_resize.shape[1], :] = img_resize
else:
# Will never be available
pane.data[:] = to_255(self.settings.window_background)
def _draw_aux_pane(self, pane, layer_data_normalized):
pane.data[:] = to_255(self.settings.window_background)
mode = None
with self.state.lock:
if self.state.cursor_area == 'bottom':
mode = 'selected'
else:
mode = 'prob_labels'
if mode == 'selected':
unit_data = layer_data_normalized[self.state.selected_unit]
unit_data_resize = ensure_uint255_and_resize_to_fit(unit_data, pane.data.shape)
pane.data[0:unit_data_resize.shape[0], 0:unit_data_resize.shape[1], :] = unit_data_resize
elif mode == 'prob_labels':
self._draw_prob_labels_pane(pane)
def draw_help(self, help_pane, locy):
defaults = {'face': getattr(cv2, self.settings.help_face),
'fsize': self.settings.help_fsize,
'clr': to_255(self.settings.help_clr),
'thick': self.settings.help_thick}
loc_base = self.settings.help_loc[::-1] # Reverse to OpenCV c,r order
locx = loc_base[0]
lines = []
lines.append([FormattedString('', defaults)])
lines.append([FormattedString('Caffevis keys', defaults)])
kl,_ = self.bindings.get_key_help('sel_left')
kr,_ = self.bindings.get_key_help('sel_right')
ku,_ = self.bindings.get_key_help('sel_up')
kd,_ = self.bindings.get_key_help('sel_down')
klf,_ = self.bindings.get_key_help('sel_left_fast')
krf,_ = self.bindings.get_key_help('sel_right_fast')
kuf,_ = self.bindings.get_key_help('sel_up_fast')
kdf,_ = self.bindings.get_key_help('sel_down_fast')
keys_nav_0 = ','.join([kk[0] for kk in (kl, kr, ku, kd)])
keys_nav_1 = ''
if len(kl)>1 and len(kr)>1 and len(ku)>1 and len(kd)>1:
keys_nav_1 += ' or '
keys_nav_1 += ','.join([kk[1] for kk in (kl, kr, ku, kd)])
keys_nav_f = ','.join([kk[0] for kk in (klf, krf, kuf, kdf)])
nav_string = 'Navigate with %s%s. Use %s to move faster.' % (keys_nav_0, keys_nav_1, keys_nav_f)
lines.append([FormattedString('', defaults, width=120, align='right'),
FormattedString(nav_string, defaults)])
for tag in ('sel_layer_left', 'sel_layer_right', 'zoom_mode', 'pattern_mode',
'ez_back_mode_loop', 'freeze_back_unit', 'show_back', 'back_mode', 'back_filt_mode',
'boost_gamma', 'boost_individual', 'reset_state'):
key_strings, help_string = self.bindings.get_key_help(tag)
label = '%10s:' % (','.join(key_strings))
lines.append([FormattedString(label, defaults, width=120, align='right'),
FormattedString(help_string, defaults)])
locy = cv2_typeset_text(help_pane.data, lines, (locx, locy),
line_spacing = self.settings.help_line_spacing)
return locy
def _draw_selected_pane(self,
pane,
layer_data_normalized,
selected_unit_highres=None):
pane.data[:] = to_255(self.settings.window_background)
with self.state.lock:
mode = 'selected' if self.state.cursor_area == 'bottom' else 'none'
if mode == 'selected':
unit_data = None
if selected_unit_highres is not None:
unit_data = selected_unit_highres
else:
if self.state.selected_unit < len(layer_data_normalized):
unit_data = layer_data_normalized[self.state.selected_unit]
elif len(layer_data_normalized) == 1:
unit_data = layer_data_normalized[0]
if unit_data is not None:
unit_data_resize = ensure_uint255_and_resize_to_fit(
unit_data, pane.data.shape)
pane.data[0:unit_data_resize.shape[0],
0:unit_data_resize.shape[1], :] = unit_data_resize
def _draw_layer_pane(self, pane):
'''Returns the data shown in highres format, b01c order.'''
if not hasattr(self.net, 'intermediate_predictions') or \
self.net.intermediate_predictions is None:
return None, None
display_3D_highres, selected_unit_highres = None, None
out = self.net.intermediate_predictions[self.state.layer_idx]
if self.state.layers_pane_filter_mode in (
4, 5) and self.state.extra_info is None:
self.state.layers_pane_filter_mode = 0
state_layers_pane_filter_mode = self.state.layers_pane_filter_mode
assert state_layers_pane_filter_mode in (0, 1, 2, 3, 4)
# Display pane based on layers_pane_zoom_mode
state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
assert state_layers_pane_zoom_mode in (0, 1, 2)
layer_dat_3D = out[0].T
n_tiles = layer_dat_3D.shape[0]
tile_rows, tile_cols = self.net_layer_info[
self.state.layer]['tiles_rc']
if state_layers_pane_filter_mode == 0:
if len(layer_dat_3D.shape) > 1:
img_width, img_height = get_tiles_height_width_ratio(
layer_dat_3D.shape[1],
self.settings.kerasvis_layers_aspect_ratio)
pad = np.zeros(
(layer_dat_3D.shape[0],
((img_width * img_height) - layer_dat_3D.shape[1])))
layer_dat_3D = np.concatenate((layer_dat_3D, pad), axis=1)
layer_dat_3D = np.reshape(
layer_dat_3D,
(layer_dat_3D.shape[0], img_width, img_height))
elif state_layers_pane_filter_mode == 1:
if len(layer_dat_3D.shape) > 1:
layer_dat_3D = np.average(layer_dat_3D, axis=1)
elif state_layers_pane_filter_mode == 2:
if len(layer_dat_3D.shape) > 1:
layer_dat_3D = np.max(layer_dat_3D, axis=1)
elif state_layers_pane_filter_mode == 3:
if len(layer_dat_3D.shape) > 1:
title, r, c, hide_axis = None, tile_rows, tile_cols, True
x_axis_label, y_axis_label = None, None
if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 1:
r, c, hide_axis = 1, 1, False
layer_dat_3D = layer_dat_3D[self.state.selected_unit:self.
state.selected_unit + 1]
title = 'Layer {}, Filter {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit)
x_axis_label, y_axis_label = 'Time', 'Activation'
display_3D = plt_plot_filters_blit(
y=layer_dat_3D,
x=None,
shape=(pane.data.shape[0], pane.data.shape[1]),
rows=r,
cols=c,
title=title,
log_scale=self.state.log_scale,
hide_axis=hide_axis,
x_axis_label=x_axis_label,
y_axis_label=y_axis_label)
if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 0:
selected_unit_highres = plt_plot_filter(
x=None,
y=layer_dat_3D[self.state.selected_unit],
title='Layer {}, Filter {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit),
log_scale=self.state.log_scale,
x_axis_label='Time',
y_axis_label='Activation')
else:
state_layers_pane_filter_mode = 0
elif state_layers_pane_filter_mode == 4:
if self.state.extra_info is not None:
extra = self.state.extra_info.item()
is_heatmap = True if 'type' in extra and extra[
'type'] == 'heatmap' else False
if is_heatmap:
layer_dat_3D = extra['data'][self.state.layer_idx]
if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 1:
display_3D = plt_plot_heatmap(
data=layer_dat_3D[self.state.selected_unit:self.
state.selected_unit + 1],
shape=(pane.data.shape[0], pane.data.shape[1]),
rows=1,
cols=1,
x_axis_label=extra['x_axis'],
y_axis_label=extra['y_axis'],
title='Layer {}, Filter {} \n {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit, extra['title']),
hide_axis=False,
x_axis_values=extra['x_axis_values'],
y_axis_values=extra['y_axis_values'],
vmin=layer_dat_3D.min(),
vmax=layer_dat_3D.max())
else:
display_3D = plt_plot_heatmap(
data=layer_dat_3D,
shape=(pane.data.shape[0], pane.data.shape[1]),
rows=tile_rows,
cols=tile_cols,
x_axis_label=extra['x_axis'],
y_axis_label=extra['y_axis'],
title=extra['title'],
x_axis_values=extra['x_axis_values'],
y_axis_values=extra['y_axis_values'])
if self.state.cursor_area == 'bottom':
selected_unit_highres = plt_plot_heatmap(
data=layer_dat_3D[self.state.selected_unit:self.
state.selected_unit + 1],
shape=(300, 300),
rows=1,
cols=1,
x_axis_label=extra['x_axis'],
y_axis_label=extra['y_axis'],
title='Layer {}, Filter {} \n {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit, extra['title']),
x_axis_values=extra['x_axis_values'],
y_axis_values=extra['y_axis_values'],
hide_axis=False,
vmin=layer_dat_3D.min(),
vmax=layer_dat_3D.max())[0]
else:
layer_dat_3D = extra['x'][self.state.layer_idx]
title, x_axis_label, y_axis_label, r, c, hide_axis = None, None, None, tile_rows, tile_cols, True
if self.state.cursor_area == 'bottom':
if state_layers_pane_zoom_mode == 1:
r, c, hide_axis = 1, 1, False
layer_dat_3D = layer_dat_3D[self.state.
selected_unit:self.
state.selected_unit +
1]
title = 'Layer {}, Filter {} \n {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit, extra['title'])
x_axis_label, y_axis_label = extra[
'x_axis'], extra['y_axis']
if self.state.log_scale == 1:
y_axis_label = y_axis_label + ' (log-scale)'
# start_time = timeit.default_timer()
display_3D = plt_plot_filters_blit(
y=layer_dat_3D,
x=extra['y'],
shape=(pane.data.shape[0], pane.data.shape[1]),
rows=r,
cols=c,
title=title,
log_scale=self.state.log_scale,
x_axis_label=x_axis_label,
y_axis_label=y_axis_label,
hide_axis=hide_axis)
if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 0:
selected_unit_highres = plt_plot_filter(
x=extra['y'],
y=layer_dat_3D[self.state.selected_unit],
title='Layer {}, Filter {} \n {}'.format(
self.state._layers[self.state.layer_idx],
self.state.selected_unit, extra['title']),
log_scale=self.state.log_scale,
x_axis_label=extra['x_axis'],
y_axis_label=extra['y_axis'])
# TODO
# if hasattr(self.settings, 'static_files_extra_fn'):
# self.data = self.settings.static_files_extra_fn(self.latest_static_file)
# self.state.layer_idx
if len(layer_dat_3D.shape) == 1:
layer_dat_3D = layer_dat_3D[:, np.newaxis, np.newaxis]
if self.state.layers_show_back and not self.state.pattern_mode:
padval = self.settings.kerasvis_layer_clr_back_background
else:
padval = self.settings.window_background
if self.state.pattern_mode:
# Show desired patterns loaded from disk
load_layer = self.state.layer
if self.settings.kerasvis_jpgvis_remap and self.state.layer in self.settings.kerasvis_jpgvis_remap:
load_layer = self.settings.kerasvis_jpgvis_remap[
self.state.layer]
if self.settings.kerasvis_jpgvis_layers and load_layer in self.settings.kerasvis_jpgvis_layers:
jpg_path = os.path.join(self.settings.kerasvis_unit_jpg_dir,
'regularized_opt', load_layer,
'whole_layer.jpg')
# Get highres version
# cache_before = str(self.img_cache)
display_3D_highres = self.img_cache.get((jpg_path, 'whole'),
None)
# else:
# display_3D_highres = None
if display_3D_highres is None:
try:
with WithTimer('KerasVisApp:load_sprite_image',
quiet=self.debug_level < 1):
display_3D_highres = load_square_sprite_image(
jpg_path, n_sprites=n_tiles)
except IOError:
# File does not exist, so just display disabled.
pass
else:
self.img_cache.set((jpg_path, 'whole'),
display_3D_highres)
# cache_after = str(self.img_cache)
# print 'Cache was / is:\n %s\n %s' % (cache_before, cache_after)
if display_3D_highres is not None:
# Get lowres version, maybe. Assume we want at least one pixel for selection border.
row_downsamp_factor = int(
np.ceil(
float(display_3D_highres.shape[1]) /
(pane.data.shape[0] / tile_rows - 2)))
col_downsamp_factor = int(
np.ceil(
float(display_3D_highres.shape[2]) /
(pane.data.shape[1] / tile_cols - 2)))
ds = max(row_downsamp_factor, col_downsamp_factor)
if ds > 1:
# print 'Downsampling by', ds
display_3D = display_3D_highres[:, ::ds, ::ds, :]
else:
display_3D = display_3D_highres
else:
display_3D = layer_dat_3D * 0 # nothing to show
else:
# Show data from network (activations or diffs)
if self.state.layers_show_back:
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
layer_dat_3D_normalized = np.tile(
self.settings.window_background,
layer_dat_3D.shape + (1, ))
elif back_what_to_disp == 'stale':
layer_dat_3D_normalized = np.tile(
self.settings.stale_background,
layer_dat_3D.shape + (1, ))
else:
layer_dat_3D_normalized = tile_images_normalize(
layer_dat_3D,
boost_indiv=self.state.layer_boost_indiv,
boost_gamma=self.state.layer_boost_gamma,
neg_pos_colors=((1, 0, 0), (0, 1, 0)))
else:
layer_dat_3D_normalized = tile_images_normalize(
layer_dat_3D,
boost_indiv=self.state.layer_boost_indiv,
boost_gamma=self.state.layer_boost_gamma)
# print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()
if state_layers_pane_filter_mode in (0, 1, 2):
display_3D = layer_dat_3D_normalized
# Convert to float if necessary:
display_3D = ensure_float01(display_3D)
# Upsample gray -> color if necessary
# e.g. (1000,32,32) -> (1000,32,32,3)
if len(display_3D.shape) == 3:
display_3D = display_3D[:, :, :, np.newaxis]
if display_3D.shape[3] == 1:
display_3D = np.tile(display_3D, (1, 1, 1, 3))
# Upsample unit length tiles to give a more sane tile / highlight ratio
# e.g. (1000,1,1,3) -> (1000,3,3,3)
if display_3D.shape[1] == 1:
display_3D = np.tile(display_3D, (1, 3, 3, 1))
if state_layers_pane_zoom_mode in (0, 2):
highlights = [None] * n_tiles
with self.state.lock:
if self.state.cursor_area == 'bottom':
highlights[
self.state.
selected_unit] = self.settings.kerasvis_layer_clr_cursor # in [0,1] range
if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
highlights[
self.state.
backprop_unit] = self.settings.kerasvis_layer_clr_back_sel # in [0,1] range
if self.state.cursor_area == 'bottom' and state_layers_pane_filter_mode in (
3, 4):
# pane.data[0:display_2D_resize.shape[0], 0:2, :] = to_255(self.settings.window_background)
# pane.data[0:2, 0:display_2D_resize.shape[1], :] = to_255(self.settings.window_background)
display_3D[self.state.selected_unit, 0:display_3D.shape[1],
0:2, :] = self.settings.kerasvis_layer_clr_cursor
display_3D[
self.state.selected_unit, 0:2, 0:display_3D.
shape[2], :] = self.settings.kerasvis_layer_clr_cursor
display_3D[self.state.selected_unit, 0:display_3D.shape[1],
-2:, :] = self.settings.kerasvis_layer_clr_cursor
display_3D[
self.state.selected_unit, -2:, 0:display_3D.
shape[2], :] = self.settings.kerasvis_layer_clr_cursor
_, display_2D = tile_images_make_tiles(display_3D,
hw=(tile_rows, tile_cols),
padval=padval,
highlights=highlights)
# Mode 0: normal display (activations or patterns)
display_2D_resize = ensure_uint255_and_resize_to_fit(
display_2D, pane.data.shape)
if state_layers_pane_zoom_mode == 2:
display_2D_resize = display_2D_resize * 0
if display_3D_highres is None:
display_3D_highres = display_3D
elif state_layers_pane_zoom_mode == 1:
if display_3D_highres is None:
display_3D_highres = display_3D
# Mode 1: zoomed selection
if state_layers_pane_filter_mode in (0, 1, 2):
unit_data = display_3D_highres[self.state.selected_unit]
else:
unit_data = display_3D_highres[0]
display_2D_resize = ensure_uint255_and_resize_to_fit(
unit_data, pane.data.shape)
pane.data[:] = to_255(self.settings.window_background)
pane.data[0:display_2D_resize.shape[0],
0:display_2D_resize.shape[1], :] = display_2D_resize
# # Add background strip around the top and left edges
# pane.data[0:display_2D_resize.shape[0], 0:2, :] = to_255(self.settings.window_background)
# pane.data[0:2, 0:display_2D_resize.shape[1], :] = to_255(self.settings.window_background)
if self.settings.kerasvis_label_layers and \
self.state.layer in self.settings.kerasvis_label_layers and \
self.labels and self.state.cursor_area == 'bottom':
# Display label annotation atop layers pane (e.g. for fc8/prob)
defaults = {
'face': getattr(cv2, self.settings.kerasvis_label_face),
'fsize': self.settings.kerasvis_label_fsize,
'clr': to_255(self.settings.kerasvis_label_clr),
'thick': self.settings.kerasvis_label_thick
}
loc_base = self.settings.kerasvis_label_loc[::
-1] # Reverse to OpenCV c,r order
lines = [
FormattedString(self.labels[self.state.selected_unit],
defaults)
]
cv2_typeset_text(pane.data, lines, loc_base)
return display_3D_highres, selected_unit_highres
def _draw_layer_pane(self, pane):
'''Returns the data shown in highres format, b01c order.'''
if self.state.layers_show_back:
layer_dat_3D = self.net.blobs[self.state.layer].diff[0]
else:
layer_dat_3D = self.net.blobs[self.state.layer].data[0]
# Promote FC layers with shape (n) to have shape (n,1,1)
if len(layer_dat_3D.shape) == 1:
layer_dat_3D = layer_dat_3D[:,np.newaxis,np.newaxis]
n_tiles = layer_dat_3D.shape[0]
tile_rows,tile_cols = self.net_layer_info[self.state.layer]['tiles_rc']
display_3D_highres = None
if self.state.pattern_mode:
# Show desired patterns loaded from disk
load_layer = self.state.layer
if self.settings.caffevis_jpgvis_remap and self.state.layer in self.settings.caffevis_jpgvis_remap:
load_layer = self.settings.caffevis_jpgvis_remap[self.state.layer]
if self.settings.caffevis_jpgvis_layers and load_layer in self.settings.caffevis_jpgvis_layers:
jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
'regularized_opt', load_layer, 'whole_layer.jpg')
# Get highres version
#cache_before = str(self.img_cache)
display_3D_highres = self.img_cache.get((jpg_path, 'whole'), None)
#else:
# display_3D_highres = None
if display_3D_highres is None:
try:
with WithTimer('CaffeVisApp:load_sprite_image', quiet = self.debug_level < 1):
display_3D_highres = load_square_sprite_image(jpg_path, n_sprites = n_tiles)
except IOError:
# File does not exist, so just display disabled.
pass
else:
self.img_cache.set((jpg_path, 'whole'), display_3D_highres)
#cache_after = str(self.img_cache)
#print 'Cache was / is:\n %s\n %s' % (cache_before, cache_after)
if display_3D_highres is not None:
# Get lowres version, maybe. Assume we want at least one pixel for selection border.
row_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[1]) / (pane.data.shape[0] / tile_rows - 2)))
col_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[2]) / (pane.data.shape[1] / tile_cols - 2)))
ds = max(row_downsamp_factor, col_downsamp_factor)
if ds > 1:
#print 'Downsampling by', ds
display_3D = display_3D_highres[:,::ds,::ds,:]
else:
display_3D = display_3D_highres
else:
display_3D = layer_dat_3D * 0 # nothing to show
else:
# Show data from network (activations or diffs)
if self.state.layers_show_back:
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
layer_dat_3D_normalized = np.tile(self.settings.window_background, layer_dat_3D.shape + (1,))
elif back_what_to_disp == 'stale':
layer_dat_3D_normalized = np.tile(self.settings.stale_background, layer_dat_3D.shape + (1,))
else:
layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
boost_indiv = self.state.layer_boost_indiv,
boost_gamma = self.state.layer_boost_gamma,
neg_pos_colors = ((1,0,0), (0,1,0)))
else:
layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
boost_indiv = self.state.layer_boost_indiv,
boost_gamma = self.state.layer_boost_gamma)
#print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()
display_3D = layer_dat_3D_normalized
# Convert to float if necessary:
display_3D = ensure_float01(display_3D)
# Upsample gray -> color if necessary
# e.g. (1000,32,32) -> (1000,32,32,3)
if len(display_3D.shape) == 3:
display_3D = display_3D[:,:,:,np.newaxis]
if display_3D.shape[3] == 1:
display_3D = np.tile(display_3D, (1, 1, 1, 3))
# Upsample unit length tiles to give a more sane tile / highlight ratio
# e.g. (1000,1,1,3) -> (1000,3,3,3)
if display_3D.shape[1] == 1:
display_3D = np.tile(display_3D, (1, 3, 3, 1))
if self.state.layers_show_back and not self.state.pattern_mode:
padval = self.settings.caffevis_layer_clr_back_background
else:
padval = self.settings.window_background
highlights = [None] * n_tiles
with self.state.lock:
if self.state.cursor_area == 'bottom':
highlights[self.state.selected_unit] = self.settings.caffevis_layer_clr_cursor # in [0,1] range
if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
highlights[self.state.backprop_unit] = self.settings.caffevis_layer_clr_back_sel # in [0,1] range
_, display_2D = tile_images_make_tiles(display_3D, hw = (tile_rows,tile_cols), padval = padval, highlights = highlights)
if display_3D_highres is None:
display_3D_highres = display_3D
# Display pane based on layers_pane_zoom_mode
state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
assert state_layers_pane_zoom_mode in (0,1,2)
if state_layers_pane_zoom_mode == 0:
# Mode 0: normal display (activations or patterns)
display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape)
elif state_layers_pane_zoom_mode == 1:
# Mode 1: zoomed selection
unit_data = display_3D_highres[self.state.selected_unit]
display_2D_resize = ensure_uint255_and_resize_to_fit(unit_data, pane.data.shape)
else:
# Mode 2: zoomed backprop pane
display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape) * 0
pane.data[:] = to_255(self.settings.window_background)
pane.data[0:display_2D_resize.shape[0], 0:display_2D_resize.shape[1], :] = display_2D_resize
if self.settings.caffevis_label_layers and self.state.layer in self.settings.caffevis_label_layers and self.labels and self.state.cursor_area == 'bottom':
# Display label annotation atop layers pane (e.g. for fc8/prob)
defaults = {'face': getattr(cv2, self.settings.caffevis_label_face),
'fsize': self.settings.caffevis_label_fsize,
'clr': to_255(self.settings.caffevis_label_clr),
'thick': self.settings.caffevis_label_thick}
loc_base = self.settings.caffevis_label_loc[::-1] # Reverse to OpenCV c,r order
lines = [FormattedString(self.labels[self.state.selected_unit], defaults)]
cv2_typeset_text(pane.data, lines, loc_base)
return display_3D_highres
def _draw_layer_pane(self, pane):
'''Returns the data shown in highres format, b01c order.'''
if self.state.layers_show_back:
layer_dat_3D = self.net.blobs[self.state.layer].diff[0]
else:
layer_dat_3D = self.net.blobs[self.state.layer].data[0]
# Promote FC layers with shape (n) to have shape (n,1,1)
if len(layer_dat_3D.shape) == 1:
layer_dat_3D = layer_dat_3D[:, np.newaxis, np.newaxis]
n_tiles = layer_dat_3D.shape[0]
tile_rows, tile_cols = self.net_layer_info[
self.state.layer]['tiles_rc']
display_3D_highres = None
if self.state.pattern_mode:
# Show desired patterns loaded from disk
load_layer = self.state.layer
if self.settings.caffevis_jpgvis_remap and self.state.layer in self.settings.caffevis_jpgvis_remap:
load_layer = self.settings.caffevis_jpgvis_remap[
self.state.layer]
if self.settings.caffevis_jpgvis_layers and load_layer in self.settings.caffevis_jpgvis_layers:
jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
'regularized_opt', load_layer,
'whole_layer.jpg')
# Get highres version
#cache_before = str(self.img_cache)
display_3D_highres = self.img_cache.get((jpg_path, 'whole'),
None)
#else:
# display_3D_highres = None
if display_3D_highres is None:
try:
with WithTimer('CaffeVisApp:load_sprite_image',
quiet=self.debug_level < 1):
display_3D_highres = load_square_sprite_image(
jpg_path, n_sprites=n_tiles)
except IOError:
# File does not exist, so just display disabled.
pass
else:
self.img_cache.set((jpg_path, 'whole'),
display_3D_highres)
#cache_after = str(self.img_cache)
#print 'Cache was / is:\n %s\n %s' % (cache_before, cache_after)
if display_3D_highres is not None:
# Get lowres version, maybe. Assume we want at least one pixel for selection border.
row_downsamp_factor = int(
np.ceil(
float(display_3D_highres.shape[1]) /
(pane.data.shape[0] / tile_rows - 2)))
col_downsamp_factor = int(
np.ceil(
float(display_3D_highres.shape[2]) /
(pane.data.shape[1] / tile_cols - 2)))
ds = max(row_downsamp_factor, col_downsamp_factor)
if ds > 1:
#print 'Downsampling by', ds
display_3D = display_3D_highres[:, ::ds, ::ds, :]
else:
display_3D = display_3D_highres
else:
display_3D = layer_dat_3D * 0 # nothing to show
else:
# Show data from network (activations or diffs)
if self.state.layers_show_back:
back_what_to_disp = self.get_back_what_to_disp()
if back_what_to_disp == 'disabled':
layer_dat_3D_normalized = np.tile(
self.settings.window_background,
layer_dat_3D.shape + (1, ))
elif back_what_to_disp == 'stale':
layer_dat_3D_normalized = np.tile(
self.settings.stale_background,
layer_dat_3D.shape + (1, ))
else:
layer_dat_3D_normalized = tile_images_normalize(
layer_dat_3D,
boost_indiv=self.state.layer_boost_indiv,
boost_gamma=self.state.layer_boost_gamma,
neg_pos_colors=((1, 0, 0), (0, 1, 0)))
else:
layer_dat_3D_normalized = tile_images_normalize(
layer_dat_3D,
boost_indiv=self.state.layer_boost_indiv,
boost_gamma=self.state.layer_boost_gamma)
#print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()
display_3D = layer_dat_3D_normalized
# Convert to float if necessary:
display_3D = ensure_float01(display_3D)
# Upsample gray -> color if necessary
# e.g. (1000,32,32) -> (1000,32,32,3)
if len(display_3D.shape) == 3:
display_3D = display_3D[:, :, :, np.newaxis]
if display_3D.shape[3] == 1:
display_3D = np.tile(display_3D, (1, 1, 1, 3))
# Upsample unit length tiles to give a more sane tile / highlight ratio
# e.g. (1000,1,1,3) -> (1000,3,3,3)
if display_3D.shape[1] == 1:
display_3D = np.tile(display_3D, (1, 3, 3, 1))
if self.state.layers_show_back and not self.state.pattern_mode:
padval = self.settings.caffevis_layer_clr_back_background
else:
padval = self.settings.window_background
highlights = [None] * n_tiles
with self.state.lock:
if self.state.cursor_area == 'bottom':
highlights[
self.state.
selected_unit] = self.settings.caffevis_layer_clr_cursor # in [0,1] range
if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
highlights[
self.state.
backprop_unit] = self.settings.caffevis_layer_clr_back_sel # in [0,1] range
_, display_2D = tile_images_make_tiles(display_3D,
hw=(tile_rows, tile_cols),
padval=padval,
highlights=highlights)
if display_3D_highres is None:
display_3D_highres = display_3D
# Display pane based on layers_pane_zoom_mode
state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
assert state_layers_pane_zoom_mode in (0, 1, 2)
if state_layers_pane_zoom_mode == 0:
# Mode 0: normal display (activations or patterns)
display_2D_resize = ensure_uint255_and_resize_to_fit(
display_2D, pane.data.shape)
elif state_layers_pane_zoom_mode == 1:
# Mode 1: zoomed selection
unit_data = display_3D_highres[self.state.selected_unit]
display_2D_resize = ensure_uint255_and_resize_to_fit(
unit_data, pane.data.shape)
else:
# Mode 2: zoomed backprop pane
display_2D_resize = ensure_uint255_and_resize_to_fit(
display_2D, pane.data.shape) * 0
pane.data[:] = to_255(self.settings.window_background)
pane.data[0:display_2D_resize.shape[0],
0:display_2D_resize.shape[1], :] = display_2D_resize
if self.settings.caffevis_label_layers and self.state.layer in self.settings.caffevis_label_layers and self.labels and self.state.cursor_area == 'bottom':
# Display label annotation atop layers pane (e.g. for fc8/prob)
defaults = {
'face': getattr(cv2, self.settings.caffevis_label_face),
'fsize': self.settings.caffevis_label_fsize,
'clr': to_255(self.settings.caffevis_label_clr),
'thick': self.settings.caffevis_label_thick
}
loc_base = self.settings.caffevis_label_loc[::
-1] # Reverse to OpenCV c,r order
lines = [
FormattedString(self.labels[self.state.selected_unit],
defaults)
]
cv2_typeset_text(pane.data, lines, loc_base)
return display_3D_highres
def get_image_from_files(settings, unit_folder_path, should_crop_to_corner, resize_shape, first_only, captions = [], values = []):
try:
# list unit images
unit_images_path = sorted(glob.glob(unit_folder_path))
mega_image = np.zeros((resize_shape[0], resize_shape[1], 3), dtype=np.uint8)
# if no images
if not unit_images_path:
return mega_image
if first_only:
unit_images_path = [unit_images_path[0]]
# load all images
unit_images = [caffe_load_image(unit_image_path, color=True, as_uint=True) for unit_image_path in
unit_images_path]
if settings.caffevis_clear_negative_activations:
# clear images with 0 value
if values:
for i in range(len(values)):
if values[i] < float_info.epsilon:
unit_images[i] *= 0
if should_crop_to_corner:
unit_images = [crop_to_corner(img, 2) for img in unit_images]
num_images = len(unit_images)
images_per_axis = int(np.math.ceil(np.math.sqrt(num_images)))
padding_pixel = 1
if first_only:
single_resized_image_shape = (resize_shape[0] - 2*padding_pixel, resize_shape[1] - 2*padding_pixel)
else:
single_resized_image_shape = ((resize_shape[0] / images_per_axis) - 2*padding_pixel, (resize_shape[1] / images_per_axis) - 2*padding_pixel)
unit_images = [ensure_uint255_and_resize_without_fit(unit_image, single_resized_image_shape) for unit_image in unit_images]
# build mega image
should_add_caption = (len(captions) == num_images)
defaults = {'face': settings.caffevis_score_face,
'fsize': settings.caffevis_score_fsize,
'clr': to_255(settings.caffevis_score_clr),
'thick': settings.caffevis_score_thick}
for i in range(num_images):
# add caption if we have exactly one for each image
if should_add_caption:
loc = settings.caffevis_score_loc[::-1] # Reverse to OpenCV c,r order
fs = FormattedString(captions[i], defaults)
cv2_typeset_text(unit_images[i], [[fs]], loc)
cell_row = i / images_per_axis
cell_col = i % images_per_axis
mega_image_height_start = 1 + cell_row * (single_resized_image_shape[0] + 2 * padding_pixel)
mega_image_height_end = mega_image_height_start + single_resized_image_shape[0]
mega_image_width_start = 1 + cell_col * (single_resized_image_shape[1] + 2 * padding_pixel)
mega_image_width_end = mega_image_width_start + single_resized_image_shape[1]
mega_image[mega_image_height_start:mega_image_height_end, mega_image_width_start:mega_image_width_end,:] = unit_images[i]
return mega_image
except:
print '\nAttempted to load files from %s but failed. ' % unit_folder_path
# set black image as place holder
return np.zeros((resize_shape[0], resize_shape[1], 3), dtype=np.uint8)
pass
return
