def __init__(self, net_id, img_fname, gpu_id, multiplier_mode='auto_1'): ''' # Args net_id: Network to inspect (lib/recon/config.py) img_fname: Image to inspect (data/gallery/) gpu_id: Id of GPU to use multiplier_mode: see self._filter_feature() ''' self.net_id = net_id self.gpu_id = gpu_id self.multiplier_mode = multiplier_mode self.config = config.nets[self.net_id] self.net_param = self._load_param(with_data=False) self.mean = load_mean_image(self.config.mean_fname) self._labels = load_ilsvrc12_labels(self.config.labels_fname) self._set_image(img_fname) self.net.forward() self.dag = nx.DiGraph() self._reconstructions = {}
def build_max_act_db(self, blob_name, k=5): # don't use self.net, which a deploy net (data comes from python) def _get_blob_layer(net_param, blob_name): ''' Return the name of the last layer to output a blob. ''' layer_name = None for layer in net_param.layer: if blob_name in layer.top: layer_name = layer.name if layer_name == None: raise Exception('could not find a layer that outputs ' \ 'blob {}'.format(blob_name)) return layer_name def _len_lmdb(fname): logger.info('computing lmdb size...') db = lmdb.open(fname) n_examples = sum(1 for _ in db.begin().cursor()) del db logger.info('found lmdb size: {} examples'.format(n_examples)) return n_examples def _showable(img, rescale=False): # NOTE: assumes images in the net are BGR img = img.transpose([1, 2, 0]) img = (img + mean)[:, :, ::-1] img = img.clip(0, 255).astype(np.uint8) if rescale: img = rescale_intensity(img) return img def _reconstruct_backward(net, net_param, blob_name, blob_idxs, act_vals=None, act_mult=1): blob = net.blobs[blob_name] blob.diff[:] = 0 for i, blob_idx in enumerate(blob_idxs): if act_vals == None: blob.diff[blob_idx] = act_mult * blob.data[blob_idx] else: blob.diff[blob_idx] = act_mult * act_vals[i] layer_name = _get_blob_layer(net_param, blob_name) net.backward(start=layer_name, end=img_layer_name) def _to_bbox(img, blob_idx): ''' Take an image which is mostly 0s and return the smallest bounding box which contains all non-0 entries. img array of size (c, h, w) blob_idx tuple with (num, channels, height, width) or (num, channels) ''' # (num, channel, height, width) if len(blob_idx) == 4: row, col = blob_idx[-2:] m = abs(img).max(axis=0) linear_idx_map = np.arange(np.prod(m.shape)).reshape(m.shape) linear_idxs = linear_idx_map[m > 0] rows = (linear_idxs // m.shape[0]) cols = (linear_idxs % m.shape[0]) if np.prod(rows.shape) == 0 or np.prod(cols.shape) == 0: top_left = row, col bottom_right = row, col else: top_left = (rows.min(), cols.min()) bottom_right = (rows.max(), cols.max()) return (top_left, bottom_right) # (num, channel) elif len(blob_idx) == 2: return ((0, 0), (img.shape[1]-1, img.shape[2]-1)) else: raise Exception('do not know how to create a bounding box from ' \ 'blob_idx {}'.format(blob_idx)) mean = load_mean_image(self.config.mean_fname) data_net_param = self._load_param(with_data=True) net = self._load_net(data_net_param) img_layer_name = _get_blob_layer(data_net_param, self.config.image_blob_name) batch_size = self.config.batch_size n_batches = int(math.ceil(self.config.num_examples / float(batch_size))) layers = {l.name: l for l in data_net_param.layer} dbname = layers[self.config.data_layer_name].data_param.source n_db_examples = _len_lmdb(dbname) assert n_db_examples == self.config.num_examples assert n_db_examples % batch_size == 0 example_offset = 0 # TODO: load the top_k lists from the db and append to them if they already exist maxes = defaultdict(lambda: [{'activation': -np.inf} for _ in range(k)]) logger.info('blob {}'.format(blob_name)) img_blob = net.blobs[self.config.image_blob_name] blob = net.blobs[blob_name] assert blob.data.shape[0] == batch_size n_features = blob.data.shape[1] for batch_i in xrange(n_batches): net.forward() logger.info('batch {}'.format(batch_i)) # for each example for num in xrange(batch_size): # examples logger.info('example {}'.format(example_offset + num)) for chan in xrange(n_features): # channel key = self._get_key(blob_name, chan) top_k = maxes[key] fmap_idx = blob.data[num, chan].argmax() if len(blob.data.shape) > 2: fmap_idx = np.unravel_index(fmap_idx, blob.data.shape[2:]) else: fmap_idx = tuple() blob_idx = (num, chan) + fmap_idx act = blob.data[blob_idx] # is this example's best patch in the topk? if act > top_k[0]['activation']: img = img_blob.data[num, :, :, :] entry = { 'example_idx': example_offset + num, 'feature_map_loc': blob_idx[2:] if len(blob_idx) > 2 else tuple(), 'img': _showable(img), #'reconstruction': self._showable(reconstruction), 'reconstruct_idx': blob_idx, 'batch_idx': batch_i, 'num': num, 'activation': act, #'patch_bbox': bbox, } top_k_idx = bisect([v['activation'] for v in top_k], act) top_k.insert(top_k_idx, entry) del top_k[0] example_offset += batch_size entries_per_example = [[] for _ in range(n_db_examples)] for chan in range(n_features): key = self._get_key(blob_name, chan) top_k = maxes[key] for entry in top_k: max_act = top_k[-1]['activation'] ex_idx = entry['example_idx'] entries_per_example[ex_idx].append((entry['reconstruct_idx'], max_act, entry)) # for each example, list the reconstructions which must be computed example_offset = 0 # compute those reconstructions for batch_i in xrange(n_batches): net.forward() logger.info('rec batch {}'.format(batch_i)) entries_in_batch = entries_per_example[example_offset:example_offset+batch_size] def total_entries(): total = 0 return sum(len(e) for e in entries_in_batch) while total_entries(): entries_to_process = [ent.pop() for ent in entries_in_batch if ent] blob_idxs, act_vals, blob_entries = zip(*entries_to_process) # one idx per example assert len(set(zip(*blob_idxs)[0])) == len(entries_to_process) _reconstruct_backward(net, data_net_param, blob_name, blob_idxs, act_vals, act_mult=self.config.blob_multipliers[blob_name]) for blob_idx, entry in zip(blob_idxs, blob_entries): num = entry['num'] reconstruction = img_blob.diff[num, :, :, :] bbox = _to_bbox(reconstruction, blob_idx) entry['reconstruction'] = _showable(reconstruction, rescale=True) entry['patch_bbox'] = bbox example_offset += batch_size logger.info('finished computing maximum activations... writing to db') act_env = lmdb.open(self.config.max_activation_dbname, map_size=recon.config.config.lmdb_map_size) with act_env.begin(write=True) as txn: for key, top_k in maxes.iteritems(): s = pkl.dumps(top_k) txn.put(key, s)
def build_max_act_db(self, blob_name, k=5): # don't use self.net, which a deploy net (data comes from python) def _get_blob_layer(net_param, blob_name): ''' Return the name of the last layer to output a blob. ''' layer_name = None for layer in net_param.layer: if blob_name in layer.top: layer_name = layer.name if layer_name == None: raise Exception('could not find a layer that outputs ' \ 'blob {}'.format(blob_name)) return layer_name def _len_lmdb(fname): logger.info('computing lmdb size...') db = lmdb.open(fname) n_examples = sum(1 for _ in db.begin().cursor()) del db logger.info('found lmdb size: {} examples'.format(n_examples)) return n_examples def _showable(img, rescale=False): # NOTE: assumes images in the net are BGR img = img.transpose([1, 2, 0]) img = (img + mean)[:, :, ::-1] img = img.clip(0, 255).astype(np.uint8) if rescale: img = rescale_intensity(img) return img def _reconstruct_backward(net, net_param, blob_name, blob_idxs, act_vals=None, act_mult=1): blob = net.blobs[blob_name] blob.diff[:] = 0 for i, blob_idx in enumerate(blob_idxs): if act_vals == None: blob.diff[blob_idx] = act_mult * blob.data[blob_idx] else: blob.diff[blob_idx] = act_mult * act_vals[i] layer_name = _get_blob_layer(net_param, blob_name) net.backward(start=layer_name, end=img_layer_name) def _to_bbox(img, blob_idx): ''' Take an image which is mostly 0s and return the smallest bounding box which contains all non-0 entries. img array of size (c, h, w) blob_idx tuple with (num, channels, height, width) or (num, channels) ''' # (num, channel, height, width) if len(blob_idx) == 4: row, col = blob_idx[-2:] m = abs(img).max(axis=0) linear_idx_map = np.arange(np.prod(m.shape)).reshape(m.shape) linear_idxs = linear_idx_map[m > 0] rows = (linear_idxs // m.shape[0]) cols = (linear_idxs % m.shape[0]) if np.prod(rows.shape) == 0 or np.prod(cols.shape) == 0: top_left = row, col bottom_right = row, col else: top_left = (rows.min(), cols.min()) bottom_right = (rows.max(), cols.max()) return (top_left, bottom_right) # (num, channel) elif len(blob_idx) == 2: return ((0, 0), (img.shape[1] - 1, img.shape[2] - 1)) else: raise Exception('do not know how to create a bounding box from ' \ 'blob_idx {}'.format(blob_idx)) mean = load_mean_image(self.config.mean_fname) data_net_param = self._load_param(with_data=True) net = self._load_net(data_net_param) img_layer_name = _get_blob_layer(data_net_param, self.config.image_blob_name) batch_size = self.config.batch_size n_batches = int(math.ceil(self.config.num_examples / float(batch_size))) layers = {l.name: l for l in data_net_param.layer} dbname = layers[self.config.data_layer_name].data_param.source n_db_examples = _len_lmdb(dbname) assert n_db_examples == self.config.num_examples assert n_db_examples % batch_size == 0 example_offset = 0 # TODO: load the top_k lists from the db and append to them if they already exist maxes = defaultdict(lambda: [{ 'activation': -np.inf } for _ in range(k)]) logger.info('blob {}'.format(blob_name)) img_blob = net.blobs[self.config.image_blob_name] blob = net.blobs[blob_name] assert blob.data.shape[0] == batch_size n_features = blob.data.shape[1] for batch_i in xrange(n_batches): net.forward() logger.info('batch {}'.format(batch_i)) # for each example for num in xrange(batch_size): # examples logger.info('example {}'.format(example_offset + num)) for chan in xrange(n_features): # channel key = self._get_key(blob_name, chan) top_k = maxes[key] fmap_idx = blob.data[num, chan].argmax() if len(blob.data.shape) > 2: fmap_idx = np.unravel_index(fmap_idx, blob.data.shape[2:]) else: fmap_idx = tuple() blob_idx = (num, chan) + fmap_idx act = blob.data[blob_idx] # is this example's best patch in the topk? if act > top_k[0]['activation']: img = img_blob.data[num, :, :, :] entry = { 'example_idx': example_offset + num, 'feature_map_loc': blob_idx[2:] if len(blob_idx) > 2 else tuple(), 'img': _showable(img), #'reconstruction': self._showable(reconstruction), 'reconstruct_idx': blob_idx, 'batch_idx': batch_i, 'num': num, 'activation': act, #'patch_bbox': bbox, } top_k_idx = bisect([v['activation'] for v in top_k], act) top_k.insert(top_k_idx, entry) del top_k[0] example_offset += batch_size entries_per_example = [[] for _ in range(n_db_examples)] for chan in range(n_features): key = self._get_key(blob_name, chan) top_k = maxes[key] for entry in top_k: max_act = top_k[-1]['activation'] ex_idx = entry['example_idx'] entries_per_example[ex_idx].append( (entry['reconstruct_idx'], max_act, entry)) # for each example, list the reconstructions which must be computed example_offset = 0 # compute those reconstructions for batch_i in xrange(n_batches): net.forward() logger.info('rec batch {}'.format(batch_i)) entries_in_batch = entries_per_example[ example_offset:example_offset + batch_size] def total_entries(): total = 0 return sum(len(e) for e in entries_in_batch) while total_entries(): entries_to_process = [ ent.pop() for ent in entries_in_batch if ent ] blob_idxs, act_vals, blob_entries = zip(*entries_to_process) # one idx per example assert len(set(zip(*blob_idxs)[0])) == len(entries_to_process) _reconstruct_backward( net, data_net_param, blob_name, blob_idxs, act_vals, act_mult=self.config.blob_multipliers[blob_name]) for blob_idx, entry in zip(blob_idxs, blob_entries): num = entry['num'] reconstruction = img_blob.diff[num, :, :, :] bbox = _to_bbox(reconstruction, blob_idx) entry['reconstruction'] = _showable(reconstruction, rescale=True) entry['patch_bbox'] = bbox example_offset += batch_size logger.info('finished computing maximum activations... writing to db') act_env = lmdb.open(self.config.max_activation_dbname, map_size=recon.config.config.lmdb_map_size) with act_env.begin(write=True) as txn: for key, top_k in maxes.iteritems(): s = pkl.dumps(top_k) txn.put(key, s)