예제 #1
0
    def _draw_layer_pane(self, pane):
        '''Returns the data shown in highres format, b01c order.'''
        
        if self.state.layers_show_back:
            layer_dat_3D = self.net.blobs[self.state.layer].diff[0]
        else:
            layer_dat_3D = self.net.blobs[self.state.layer].data[0]
        # Promote FC layers with shape (n) to have shape (n,1,1)
        if len(layer_dat_3D.shape) == 1:
            layer_dat_3D = layer_dat_3D[:,np.newaxis,np.newaxis]

        n_tiles = layer_dat_3D.shape[0]
        tile_rows,tile_cols = self.net_layer_info[self.state.layer]['tiles_rc']

        display_3D_highres = None
        if self.state.pattern_mode:
            # Show desired patterns loaded from disk

            load_layer = self.state.layer
            if self.settings.caffevis_jpgvis_remap and self.state.layer in self.settings.caffevis_jpgvis_remap:
                load_layer = self.settings.caffevis_jpgvis_remap[self.state.layer]

            
            if self.settings.caffevis_jpgvis_layers and load_layer in self.settings.caffevis_jpgvis_layers:
                jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
                                        'regularized_opt', load_layer, 'whole_layer.jpg')

                # Get highres version
                #cache_before = str(self.img_cache)
                display_3D_highres = self.img_cache.get((jpg_path, 'whole'), None)
                #else:
                #    display_3D_highres = None

                if display_3D_highres is None:
                    try:
                        with WithTimer('CaffeVisApp:load_sprite_image', quiet = self.debug_level < 1):
                            display_3D_highres = load_square_sprite_image(jpg_path, n_sprites = n_tiles)
                    except IOError:
                        # File does not exist, so just display disabled.
                        pass
                    else:
                        self.img_cache.set((jpg_path, 'whole'), display_3D_highres)
                #cache_after = str(self.img_cache)
                #print 'Cache was / is:\n  %s\n  %s' % (cache_before, cache_after)

            if display_3D_highres is not None:
                # Get lowres version, maybe. Assume we want at least one pixel for selection border.
                row_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[1]) / (pane.data.shape[0] / tile_rows - 2)))
                col_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[2]) / (pane.data.shape[1] / tile_cols - 2)))
                ds = max(row_downsamp_factor, col_downsamp_factor)
                if ds > 1:
                    #print 'Downsampling by', ds
                    display_3D = display_3D_highres[:,::ds,::ds,:]
                else:
                    display_3D = display_3D_highres
            else:
                display_3D = layer_dat_3D * 0  # nothing to show

        else:

            # Show data from network (activations or diffs)
            if self.state.layers_show_back:
                back_what_to_disp = self.get_back_what_to_disp()
                if back_what_to_disp == 'disabled':
                    layer_dat_3D_normalized = np.tile(self.settings.window_background, layer_dat_3D.shape + (1,))
                elif back_what_to_disp == 'stale':
                    layer_dat_3D_normalized = np.tile(self.settings.stale_background, layer_dat_3D.shape + (1,))
                else:
                    layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
                                                                    boost_indiv = self.state.layer_boost_indiv,
                                                                    boost_gamma = self.state.layer_boost_gamma,
                                                                    neg_pos_colors = ((1,0,0), (0,1,0)))
            else:
                layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
                                                                boost_indiv = self.state.layer_boost_indiv,
                                                                boost_gamma = self.state.layer_boost_gamma)
            #print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()

            display_3D         = layer_dat_3D_normalized

        # Convert to float if necessary:
        display_3D = ensure_float01(display_3D)
        # Upsample gray -> color if necessary
        #   e.g. (1000,32,32) -> (1000,32,32,3)
        if len(display_3D.shape) == 3:
            display_3D = display_3D[:,:,:,np.newaxis]
        if display_3D.shape[3] == 1:
            display_3D = np.tile(display_3D, (1, 1, 1, 3))
        # Upsample unit length tiles to give a more sane tile / highlight ratio
        #   e.g. (1000,1,1,3) -> (1000,3,3,3)
        if display_3D.shape[1] == 1:
            display_3D = np.tile(display_3D, (1, 3, 3, 1))
        if self.state.layers_show_back and not self.state.pattern_mode:
            padval = self.settings.caffevis_layer_clr_back_background
        else:
            padval = self.settings.window_background

        highlights = [None] * n_tiles
        with self.state.lock:
            if self.state.cursor_area == 'bottom':
                highlights[self.state.selected_unit] = self.settings.caffevis_layer_clr_cursor  # in [0,1] range
            if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
                highlights[self.state.backprop_unit] = self.settings.caffevis_layer_clr_back_sel  # in [0,1] range

        _, display_2D = tile_images_make_tiles(display_3D, hw = (tile_rows,tile_cols), padval = padval, highlights = highlights)

        if display_3D_highres is None:
            display_3D_highres = display_3D
        
        # Display pane based on layers_pane_zoom_mode
        state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
        assert state_layers_pane_zoom_mode in (0,1,2)
        if state_layers_pane_zoom_mode == 0:
            # Mode 0: normal display (activations or patterns)
            display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape)
        elif state_layers_pane_zoom_mode == 1:
            # Mode 1: zoomed selection
            unit_data = display_3D_highres[self.state.selected_unit]
            display_2D_resize = ensure_uint255_and_resize_to_fit(unit_data, pane.data.shape)
        else:
            # Mode 2: zoomed backprop pane
            display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape) * 0

        pane.data[:] = to_255(self.settings.window_background)
        pane.data[0:display_2D_resize.shape[0], 0:display_2D_resize.shape[1], :] = display_2D_resize
        
        if self.settings.caffevis_label_layers and self.state.layer in self.settings.caffevis_label_layers and self.labels and self.state.cursor_area == 'bottom':
            # Display label annotation atop layers pane (e.g. for fc8/prob)
            defaults = {'face':  getattr(cv2, self.settings.caffevis_label_face),
                        'fsize': self.settings.caffevis_label_fsize,
                        'clr':   to_255(self.settings.caffevis_label_clr),
                        'thick': self.settings.caffevis_label_thick}
            loc_base = self.settings.caffevis_label_loc[::-1]   # Reverse to OpenCV c,r order
            lines = [FormattedString(self.labels[self.state.selected_unit], defaults)]
            cv2_typeset_text(pane.data, lines, loc_base)
            
        return display_3D_highres
예제 #2
0
    def _draw_layer_pane(self, pane):
        '''Returns the data shown in highres format, b01c order.'''

        if not hasattr(self.net, 'intermediate_predictions') or \
                self.net.intermediate_predictions is None:
            return None, None

        display_3D_highres, selected_unit_highres = None, None
        out = self.net.intermediate_predictions[self.state.layer_idx]

        if self.state.layers_pane_filter_mode in (
                4, 5) and self.state.extra_info is None:
            self.state.layers_pane_filter_mode = 0

        state_layers_pane_filter_mode = self.state.layers_pane_filter_mode
        assert state_layers_pane_filter_mode in (0, 1, 2, 3, 4)

        # Display pane based on layers_pane_zoom_mode
        state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
        assert state_layers_pane_zoom_mode in (0, 1, 2)

        layer_dat_3D = out[0].T
        n_tiles = layer_dat_3D.shape[0]
        tile_rows, tile_cols = self.net_layer_info[
            self.state.layer]['tiles_rc']

        if state_layers_pane_filter_mode == 0:
            if len(layer_dat_3D.shape) > 1:
                img_width, img_height = get_tiles_height_width_ratio(
                    layer_dat_3D.shape[1],
                    self.settings.kerasvis_layers_aspect_ratio)

                pad = np.zeros(
                    (layer_dat_3D.shape[0],
                     ((img_width * img_height) - layer_dat_3D.shape[1])))
                layer_dat_3D = np.concatenate((layer_dat_3D, pad), axis=1)
                layer_dat_3D = np.reshape(
                    layer_dat_3D,
                    (layer_dat_3D.shape[0], img_width, img_height))

        elif state_layers_pane_filter_mode == 1:
            if len(layer_dat_3D.shape) > 1:
                layer_dat_3D = np.average(layer_dat_3D, axis=1)

        elif state_layers_pane_filter_mode == 2:
            if len(layer_dat_3D.shape) > 1:
                layer_dat_3D = np.max(layer_dat_3D, axis=1)

        elif state_layers_pane_filter_mode == 3:

            if len(layer_dat_3D.shape) > 1:
                title, r, c, hide_axis = None, tile_rows, tile_cols, True
                x_axis_label, y_axis_label = None, None
                if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 1:
                    r, c, hide_axis = 1, 1, False
                    layer_dat_3D = layer_dat_3D[self.state.selected_unit:self.
                                                state.selected_unit + 1]
                    title = 'Layer {}, Filter {}'.format(
                        self.state._layers[self.state.layer_idx],
                        self.state.selected_unit)
                    x_axis_label, y_axis_label = 'Time', 'Activation'

                display_3D = plt_plot_filters_blit(
                    y=layer_dat_3D,
                    x=None,
                    shape=(pane.data.shape[0], pane.data.shape[1]),
                    rows=r,
                    cols=c,
                    title=title,
                    log_scale=self.state.log_scale,
                    hide_axis=hide_axis,
                    x_axis_label=x_axis_label,
                    y_axis_label=y_axis_label)

                if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 0:
                    selected_unit_highres = plt_plot_filter(
                        x=None,
                        y=layer_dat_3D[self.state.selected_unit],
                        title='Layer {}, Filter {}'.format(
                            self.state._layers[self.state.layer_idx],
                            self.state.selected_unit),
                        log_scale=self.state.log_scale,
                        x_axis_label='Time',
                        y_axis_label='Activation')

            else:
                state_layers_pane_filter_mode = 0

        elif state_layers_pane_filter_mode == 4:

            if self.state.extra_info is not None:
                extra = self.state.extra_info.item()
                is_heatmap = True if 'type' in extra and extra[
                    'type'] == 'heatmap' else False

                if is_heatmap:
                    layer_dat_3D = extra['data'][self.state.layer_idx]

                    if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 1:
                        display_3D = plt_plot_heatmap(
                            data=layer_dat_3D[self.state.selected_unit:self.
                                              state.selected_unit + 1],
                            shape=(pane.data.shape[0], pane.data.shape[1]),
                            rows=1,
                            cols=1,
                            x_axis_label=extra['x_axis'],
                            y_axis_label=extra['y_axis'],
                            title='Layer {}, Filter {} \n {}'.format(
                                self.state._layers[self.state.layer_idx],
                                self.state.selected_unit, extra['title']),
                            hide_axis=False,
                            x_axis_values=extra['x_axis_values'],
                            y_axis_values=extra['y_axis_values'],
                            vmin=layer_dat_3D.min(),
                            vmax=layer_dat_3D.max())
                    else:
                        display_3D = plt_plot_heatmap(
                            data=layer_dat_3D,
                            shape=(pane.data.shape[0], pane.data.shape[1]),
                            rows=tile_rows,
                            cols=tile_cols,
                            x_axis_label=extra['x_axis'],
                            y_axis_label=extra['y_axis'],
                            title=extra['title'],
                            x_axis_values=extra['x_axis_values'],
                            y_axis_values=extra['y_axis_values'])

                    if self.state.cursor_area == 'bottom':
                        selected_unit_highres = plt_plot_heatmap(
                            data=layer_dat_3D[self.state.selected_unit:self.
                                              state.selected_unit + 1],
                            shape=(300, 300),
                            rows=1,
                            cols=1,
                            x_axis_label=extra['x_axis'],
                            y_axis_label=extra['y_axis'],
                            title='Layer {}, Filter {} \n {}'.format(
                                self.state._layers[self.state.layer_idx],
                                self.state.selected_unit, extra['title']),
                            x_axis_values=extra['x_axis_values'],
                            y_axis_values=extra['y_axis_values'],
                            hide_axis=False,
                            vmin=layer_dat_3D.min(),
                            vmax=layer_dat_3D.max())[0]

                else:

                    layer_dat_3D = extra['x'][self.state.layer_idx]
                    title, x_axis_label, y_axis_label, r, c, hide_axis = None, None, None, tile_rows, tile_cols, True

                    if self.state.cursor_area == 'bottom':
                        if state_layers_pane_zoom_mode == 1:
                            r, c, hide_axis = 1, 1, False
                            layer_dat_3D = layer_dat_3D[self.state.
                                                        selected_unit:self.
                                                        state.selected_unit +
                                                        1]
                            title = 'Layer {}, Filter {} \n {}'.format(
                                self.state._layers[self.state.layer_idx],
                                self.state.selected_unit, extra['title'])
                            x_axis_label, y_axis_label = extra[
                                'x_axis'], extra['y_axis']

                            if self.state.log_scale == 1:
                                y_axis_label = y_axis_label + ' (log-scale)'

                    # start_time = timeit.default_timer()
                    display_3D = plt_plot_filters_blit(
                        y=layer_dat_3D,
                        x=extra['y'],
                        shape=(pane.data.shape[0], pane.data.shape[1]),
                        rows=r,
                        cols=c,
                        title=title,
                        log_scale=self.state.log_scale,
                        x_axis_label=x_axis_label,
                        y_axis_label=y_axis_label,
                        hide_axis=hide_axis)

                    if self.state.cursor_area == 'bottom' and state_layers_pane_zoom_mode == 0:
                        selected_unit_highres = plt_plot_filter(
                            x=extra['y'],
                            y=layer_dat_3D[self.state.selected_unit],
                            title='Layer {}, Filter {} \n {}'.format(
                                self.state._layers[self.state.layer_idx],
                                self.state.selected_unit, extra['title']),
                            log_scale=self.state.log_scale,
                            x_axis_label=extra['x_axis'],
                            y_axis_label=extra['y_axis'])

            # TODO

            # if hasattr(self.settings, 'static_files_extra_fn'):
            #     self.data = self.settings.static_files_extra_fn(self.latest_static_file)
            #      self.state.layer_idx

        if len(layer_dat_3D.shape) == 1:
            layer_dat_3D = layer_dat_3D[:, np.newaxis, np.newaxis]

        if self.state.layers_show_back and not self.state.pattern_mode:
            padval = self.settings.kerasvis_layer_clr_back_background
        else:
            padval = self.settings.window_background

        if self.state.pattern_mode:
            # Show desired patterns loaded from disk

            load_layer = self.state.layer
            if self.settings.kerasvis_jpgvis_remap and self.state.layer in self.settings.kerasvis_jpgvis_remap:
                load_layer = self.settings.kerasvis_jpgvis_remap[
                    self.state.layer]

            if self.settings.kerasvis_jpgvis_layers and load_layer in self.settings.kerasvis_jpgvis_layers:
                jpg_path = os.path.join(self.settings.kerasvis_unit_jpg_dir,
                                        'regularized_opt', load_layer,
                                        'whole_layer.jpg')

                # Get highres version
                # cache_before = str(self.img_cache)
                display_3D_highres = self.img_cache.get((jpg_path, 'whole'),
                                                        None)
                # else:
                #    display_3D_highres = None

                if display_3D_highres is None:
                    try:
                        with WithTimer('KerasVisApp:load_sprite_image',
                                       quiet=self.debug_level < 1):
                            display_3D_highres = load_square_sprite_image(
                                jpg_path, n_sprites=n_tiles)
                    except IOError:
                        # File does not exist, so just display disabled.
                        pass
                    else:
                        self.img_cache.set((jpg_path, 'whole'),
                                           display_3D_highres)
                        # cache_after = str(self.img_cache)
                        # print 'Cache was / is:\n  %s\n  %s' % (cache_before, cache_after)

            if display_3D_highres is not None:
                # Get lowres version, maybe. Assume we want at least one pixel for selection border.
                row_downsamp_factor = int(
                    np.ceil(
                        float(display_3D_highres.shape[1]) /
                        (pane.data.shape[0] / tile_rows - 2)))
                col_downsamp_factor = int(
                    np.ceil(
                        float(display_3D_highres.shape[2]) /
                        (pane.data.shape[1] / tile_cols - 2)))
                ds = max(row_downsamp_factor, col_downsamp_factor)
                if ds > 1:
                    # print 'Downsampling by', ds
                    display_3D = display_3D_highres[:, ::ds, ::ds, :]
                else:
                    display_3D = display_3D_highres
            else:
                display_3D = layer_dat_3D * 0  # nothing to show

        else:

            # Show data from network (activations or diffs)
            if self.state.layers_show_back:
                back_what_to_disp = self.get_back_what_to_disp()
                if back_what_to_disp == 'disabled':
                    layer_dat_3D_normalized = np.tile(
                        self.settings.window_background,
                        layer_dat_3D.shape + (1, ))
                elif back_what_to_disp == 'stale':
                    layer_dat_3D_normalized = np.tile(
                        self.settings.stale_background,
                        layer_dat_3D.shape + (1, ))
                else:
                    layer_dat_3D_normalized = tile_images_normalize(
                        layer_dat_3D,
                        boost_indiv=self.state.layer_boost_indiv,
                        boost_gamma=self.state.layer_boost_gamma,
                        neg_pos_colors=((1, 0, 0), (0, 1, 0)))
            else:
                layer_dat_3D_normalized = tile_images_normalize(
                    layer_dat_3D,
                    boost_indiv=self.state.layer_boost_indiv,
                    boost_gamma=self.state.layer_boost_gamma)
            # print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()

            if state_layers_pane_filter_mode in (0, 1, 2):
                display_3D = layer_dat_3D_normalized

        # Convert to float if necessary:
        display_3D = ensure_float01(display_3D)

        # Upsample gray -> color if necessary
        #   e.g. (1000,32,32) -> (1000,32,32,3)
        if len(display_3D.shape) == 3:
            display_3D = display_3D[:, :, :, np.newaxis]

        if display_3D.shape[3] == 1:
            display_3D = np.tile(display_3D, (1, 1, 1, 3))
        # Upsample unit length tiles to give a more sane tile / highlight ratio
        #   e.g. (1000,1,1,3) -> (1000,3,3,3)
        if display_3D.shape[1] == 1:
            display_3D = np.tile(display_3D, (1, 3, 3, 1))

        if state_layers_pane_zoom_mode in (0, 2):

            highlights = [None] * n_tiles
            with self.state.lock:
                if self.state.cursor_area == 'bottom':
                    highlights[
                        self.state.
                        selected_unit] = self.settings.kerasvis_layer_clr_cursor  # in [0,1] range
                if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
                    highlights[
                        self.state.
                        backprop_unit] = self.settings.kerasvis_layer_clr_back_sel  # in [0,1] range

            if self.state.cursor_area == 'bottom' and state_layers_pane_filter_mode in (
                    3, 4):
                # pane.data[0:display_2D_resize.shape[0], 0:2, :] = to_255(self.settings.window_background)
                # pane.data[0:2, 0:display_2D_resize.shape[1], :] = to_255(self.settings.window_background)
                display_3D[self.state.selected_unit, 0:display_3D.shape[1],
                           0:2, :] = self.settings.kerasvis_layer_clr_cursor
                display_3D[
                    self.state.selected_unit, 0:2, 0:display_3D.
                    shape[2], :] = self.settings.kerasvis_layer_clr_cursor

                display_3D[self.state.selected_unit, 0:display_3D.shape[1],
                           -2:, :] = self.settings.kerasvis_layer_clr_cursor
                display_3D[
                    self.state.selected_unit, -2:, 0:display_3D.
                    shape[2], :] = self.settings.kerasvis_layer_clr_cursor

            _, display_2D = tile_images_make_tiles(display_3D,
                                                   hw=(tile_rows, tile_cols),
                                                   padval=padval,
                                                   highlights=highlights)

            # Mode 0: normal display (activations or patterns)
            display_2D_resize = ensure_uint255_and_resize_to_fit(
                display_2D, pane.data.shape)
            if state_layers_pane_zoom_mode == 2:
                display_2D_resize = display_2D_resize * 0

            if display_3D_highres is None:
                display_3D_highres = display_3D

        elif state_layers_pane_zoom_mode == 1:
            if display_3D_highres is None:
                display_3D_highres = display_3D

            # Mode 1: zoomed selection
            if state_layers_pane_filter_mode in (0, 1, 2):
                unit_data = display_3D_highres[self.state.selected_unit]
            else:
                unit_data = display_3D_highres[0]

            display_2D_resize = ensure_uint255_and_resize_to_fit(
                unit_data, pane.data.shape)

        pane.data[:] = to_255(self.settings.window_background)
        pane.data[0:display_2D_resize.shape[0],
                  0:display_2D_resize.shape[1], :] = display_2D_resize

        # # Add background strip around the top and left edges
        # pane.data[0:display_2D_resize.shape[0], 0:2, :] = to_255(self.settings.window_background)
        # pane.data[0:2, 0:display_2D_resize.shape[1], :] = to_255(self.settings.window_background)

        if self.settings.kerasvis_label_layers and \
                self.state.layer in self.settings.kerasvis_label_layers and \
                self.labels and self.state.cursor_area == 'bottom':
            # Display label annotation atop layers pane (e.g. for fc8/prob)
            defaults = {
                'face': getattr(cv2, self.settings.kerasvis_label_face),
                'fsize': self.settings.kerasvis_label_fsize,
                'clr': to_255(self.settings.kerasvis_label_clr),
                'thick': self.settings.kerasvis_label_thick
            }
            loc_base = self.settings.kerasvis_label_loc[::
                                                        -1]  # Reverse to OpenCV c,r order
            lines = [
                FormattedString(self.labels[self.state.selected_unit],
                                defaults)
            ]
            cv2_typeset_text(pane.data, lines, loc_base)

        return display_3D_highres, selected_unit_highres
예제 #3
0
    def _draw_layer_pane(self, pane):
        '''Returns the data shown in highres format, b01c order.'''

        if self.state.layers_show_back:
            layer_dat_3D = self.net.blobs[self.state.layer].diff[0]
        else:
            layer_dat_3D = self.net.blobs[self.state.layer].data[0]
        # Promote FC layers with shape (n) to have shape (n,1,1)
        if len(layer_dat_3D.shape) == 1:
            layer_dat_3D = layer_dat_3D[:, np.newaxis, np.newaxis]

        n_tiles = layer_dat_3D.shape[0]
        tile_rows, tile_cols = self.net_layer_info[
            self.state.layer]['tiles_rc']

        display_3D_highres = None
        if self.state.pattern_mode:
            # Show desired patterns loaded from disk

            load_layer = self.state.layer
            if self.settings.caffevis_jpgvis_remap and self.state.layer in self.settings.caffevis_jpgvis_remap:
                load_layer = self.settings.caffevis_jpgvis_remap[
                    self.state.layer]

            if self.settings.caffevis_jpgvis_layers and load_layer in self.settings.caffevis_jpgvis_layers:
                jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
                                        'regularized_opt', load_layer,
                                        'whole_layer.jpg')

                # Get highres version
                #cache_before = str(self.img_cache)
                display_3D_highres = self.img_cache.get((jpg_path, 'whole'),
                                                        None)
                #else:
                #    display_3D_highres = None

                if display_3D_highres is None:
                    try:
                        with WithTimer('CaffeVisApp:load_sprite_image',
                                       quiet=self.debug_level < 1):
                            display_3D_highres = load_square_sprite_image(
                                jpg_path, n_sprites=n_tiles)
                    except IOError:
                        # File does not exist, so just display disabled.
                        pass
                    else:
                        self.img_cache.set((jpg_path, 'whole'),
                                           display_3D_highres)
                #cache_after = str(self.img_cache)
                #print 'Cache was / is:\n  %s\n  %s' % (cache_before, cache_after)

            if display_3D_highres is not None:
                # Get lowres version, maybe. Assume we want at least one pixel for selection border.
                row_downsamp_factor = int(
                    np.ceil(
                        float(display_3D_highres.shape[1]) /
                        (pane.data.shape[0] / tile_rows - 2)))
                col_downsamp_factor = int(
                    np.ceil(
                        float(display_3D_highres.shape[2]) /
                        (pane.data.shape[1] / tile_cols - 2)))
                ds = max(row_downsamp_factor, col_downsamp_factor)
                if ds > 1:
                    #print 'Downsampling by', ds
                    display_3D = display_3D_highres[:, ::ds, ::ds, :]
                else:
                    display_3D = display_3D_highres
            else:
                display_3D = layer_dat_3D * 0  # nothing to show

        else:

            # Show data from network (activations or diffs)
            if self.state.layers_show_back:
                back_what_to_disp = self.get_back_what_to_disp()
                if back_what_to_disp == 'disabled':
                    layer_dat_3D_normalized = np.tile(
                        self.settings.window_background,
                        layer_dat_3D.shape + (1, ))
                elif back_what_to_disp == 'stale':
                    layer_dat_3D_normalized = np.tile(
                        self.settings.stale_background,
                        layer_dat_3D.shape + (1, ))
                else:
                    layer_dat_3D_normalized = tile_images_normalize(
                        layer_dat_3D,
                        boost_indiv=self.state.layer_boost_indiv,
                        boost_gamma=self.state.layer_boost_gamma,
                        neg_pos_colors=((1, 0, 0), (0, 1, 0)))
            else:
                layer_dat_3D_normalized = tile_images_normalize(
                    layer_dat_3D,
                    boost_indiv=self.state.layer_boost_indiv,
                    boost_gamma=self.state.layer_boost_gamma)
            #print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()

            display_3D = layer_dat_3D_normalized

        # Convert to float if necessary:
        display_3D = ensure_float01(display_3D)
        # Upsample gray -> color if necessary
        #   e.g. (1000,32,32) -> (1000,32,32,3)
        if len(display_3D.shape) == 3:
            display_3D = display_3D[:, :, :, np.newaxis]
        if display_3D.shape[3] == 1:
            display_3D = np.tile(display_3D, (1, 1, 1, 3))
        # Upsample unit length tiles to give a more sane tile / highlight ratio
        #   e.g. (1000,1,1,3) -> (1000,3,3,3)
        if display_3D.shape[1] == 1:
            display_3D = np.tile(display_3D, (1, 3, 3, 1))
        if self.state.layers_show_back and not self.state.pattern_mode:
            padval = self.settings.caffevis_layer_clr_back_background
        else:
            padval = self.settings.window_background

        highlights = [None] * n_tiles
        with self.state.lock:
            if self.state.cursor_area == 'bottom':
                highlights[
                    self.state.
                    selected_unit] = self.settings.caffevis_layer_clr_cursor  # in [0,1] range
            if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
                highlights[
                    self.state.
                    backprop_unit] = self.settings.caffevis_layer_clr_back_sel  # in [0,1] range

        _, display_2D = tile_images_make_tiles(display_3D,
                                               hw=(tile_rows, tile_cols),
                                               padval=padval,
                                               highlights=highlights)

        if display_3D_highres is None:
            display_3D_highres = display_3D

        # Display pane based on layers_pane_zoom_mode
        state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
        assert state_layers_pane_zoom_mode in (0, 1, 2)
        if state_layers_pane_zoom_mode == 0:
            # Mode 0: normal display (activations or patterns)
            display_2D_resize = ensure_uint255_and_resize_to_fit(
                display_2D, pane.data.shape)
        elif state_layers_pane_zoom_mode == 1:
            # Mode 1: zoomed selection
            unit_data = display_3D_highres[self.state.selected_unit]
            display_2D_resize = ensure_uint255_and_resize_to_fit(
                unit_data, pane.data.shape)
        else:
            # Mode 2: zoomed backprop pane
            display_2D_resize = ensure_uint255_and_resize_to_fit(
                display_2D, pane.data.shape) * 0

        pane.data[:] = to_255(self.settings.window_background)
        pane.data[0:display_2D_resize.shape[0],
                  0:display_2D_resize.shape[1], :] = display_2D_resize

        if self.settings.caffevis_label_layers and self.state.layer in self.settings.caffevis_label_layers and self.labels and self.state.cursor_area == 'bottom':
            # Display label annotation atop layers pane (e.g. for fc8/prob)
            defaults = {
                'face': getattr(cv2, self.settings.caffevis_label_face),
                'fsize': self.settings.caffevis_label_fsize,
                'clr': to_255(self.settings.caffevis_label_clr),
                'thick': self.settings.caffevis_label_thick
            }
            loc_base = self.settings.caffevis_label_loc[::
                                                        -1]  # Reverse to OpenCV c,r order
            lines = [
                FormattedString(self.labels[self.state.selected_unit],
                                defaults)
            ]
            cv2_typeset_text(pane.data, lines, loc_base)

        return display_3D_highres
예제 #4
0
    def _draw_layer_pane(self, pane):
        '''Returns the data shown in highres format, b01c order.'''

        if self.state.layers_show_back:
            layer_dat_3D = self.net.blobs[self.state.layer].diff[0]
        else:
            layer_dat_3D = self.net.blobs[self.state.layer].data[0]
        # Promote FC layers with shape (n) to have shape (n,1,1)
        if len(layer_dat_3D.shape) == 1:
            layer_dat_3D = layer_dat_3D[:,np.newaxis,np.newaxis]

        n_tiles = layer_dat_3D.shape[0]
        tile_rows,tile_cols = get_tiles_height_width(n_tiles)

        display_3D_highres = None
        if self.state.pattern_mode:
            # Show desired patterns loaded from disk

            #available = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'fc6', 'fc7', 'fc8', 'prob']
            jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir,
                                    'regularized_opt', self.state.layer, 'whole_layer.jpg')

            # Get highres version
            cache_before = str(self.img_cache)
            display_3D_highres = self.img_cache.get((jpg_path, 'whole'), None)
            if display_3D_highres is None:
                try:
                    with WithTimer('CaffeVisApp:load_sprite_image', quiet = self.debug_level < 1):
                        display_3D_highres = load_sprite_image(jpg_path, (tile_rows, tile_cols), n_sprites = n_tiles)
                except IOError:
                    # File does not exist, so just display disabled.
                    pass
                else:
                    self.img_cache.set((jpg_path, 'whole'), display_3D_highres)
            cache_after = str(self.img_cache)
            #print 'Cache was / is:\n  %s\n  %s' % (cache_before, cache_after)

            if display_3D_highres is not None:
                # Get lowres version, maybe. Assume we want at least one pixel for selection border.
                row_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[1]) / (pane.data.shape[0] / tile_rows - 2)))
                col_downsamp_factor = int(np.ceil(float(display_3D_highres.shape[2]) / (pane.data.shape[1] / tile_cols - 2)))
                ds = max(row_downsamp_factor, col_downsamp_factor)
                if ds > 1:
                    #print 'Downsampling by', ds
                    display_3D = display_3D_highres[:,::ds,::ds,:]
                else:
                    display_3D = display_3D_highres
            else:
                display_3D = layer_dat_3D * 0  # nothing to show

        else:

            # Show data from network (activations or diffs)
            if self.state.layers_show_back:
                back_what_to_disp = self.get_back_what_to_disp()
                if back_what_to_disp == 'disabled':
                    layer_dat_3D_normalized = np.tile(self.settings.window_background, layer_dat_3D.shape + (1,))
                elif back_what_to_disp == 'stale':
                    layer_dat_3D_normalized = np.tile(self.settings.stale_background, layer_dat_3D.shape + (1,))
                else:
                    layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
                                                                    boost_indiv = self.state.layer_boost_indiv,
                                                                    boost_gamma = self.state.layer_boost_gamma,
                                                                    neg_pos_colors = ((1,0,0), (0,1,0)))
            else:
                layer_dat_3D_normalized = tile_images_normalize(layer_dat_3D,
                                                                boost_indiv = self.state.layer_boost_indiv,
                                                                boost_gamma = self.state.layer_boost_gamma)
            #print ' ===layer_dat_3D_normalized.shape', layer_dat_3D_normalized.shape, 'layer_dat_3D_normalized dtype', layer_dat_3D_normalized.dtype, 'range', layer_dat_3D_normalized.min(), layer_dat_3D_normalized.max()

            display_3D         = layer_dat_3D_normalized

        # Convert to float if necessary:
        display_3D = ensure_float01(display_3D)
        # Upsample gray -> color if necessary
        #   (1000,32,32) -> (1000,32,32,3)
        if len(display_3D.shape) == 3:
            display_3D = display_3D[:,:,:,np.newaxis]
        if display_3D.shape[3] == 1:
            display_3D = np.tile(display_3D, (1, 1, 1, 3))
        # Upsample unit length tiles to give a more sane tile / highlight ratio
        #   (1000,1,1,3) -> (1000,3,3,3)
        if display_3D.shape[1] == 1:
            display_3D = np.tile(display_3D, (1, 3, 3, 1))
        if self.state.layers_show_back and not self.state.pattern_mode:
            padval = self.settings.caffevis_layer_clr_back_background
        else:
            padval = self.settings.window_background
        # Tell the state about the updated (height,width) tile display (ensures valid selection)
        self.state.update_tiles_height_width((tile_rows,tile_cols), display_3D.shape[0])

        #if self.state.layers_show_back:
        #    highlights = [(.5, .5, 1)] * n_tiles
        #else:
        highlights = [None] * n_tiles
        with self.state.lock:
            if self.state.cursor_area == 'bottom':
                highlights[self.state.selected_unit] = self.settings.caffevis_layer_clr_cursor  # in [0,1] range
            if self.state.backprop_selection_frozen and self.state.layer == self.state.backprop_layer:
                highlights[self.state.backprop_unit] = self.settings.caffevis_layer_clr_back_sel  # in [0,1] range

        _, display_2D = tile_images_make_tiles(display_3D, padval = padval, highlights = highlights)
        #print ' ===tile_conv dtype', tile_conv.dtype, 'range', tile_conv.min(), tile_conv.max()

        if display_3D_highres is None:
            display_3D_highres = display_3D

        # Display pane based on layers_pane_zoom_mode
        state_layers_pane_zoom_mode = self.state.layers_pane_zoom_mode
        assert state_layers_pane_zoom_mode in (0,1,2)
        if state_layers_pane_zoom_mode == 0:
            # Mode 0: base case
            display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape)
        elif state_layers_pane_zoom_mode == 1:
            # Mode 1: zoomed selection
            unit_data = display_3D_highres[self.state.selected_unit]
            display_2D_resize = ensure_uint255_and_resize_to_fit(unit_data, pane.data.shape)
        else:
            # Mode 2: ??? backprop ???
            display_2D_resize = ensure_uint255_and_resize_to_fit(display_2D, pane.data.shape) * 0

        pane.data[0:display_2D_resize.shape[0], 0:display_2D_resize.shape[1], :] = display_2D_resize

        return display_3D_highres