def __init__(self, net_id, img_fname, gpu_id, multiplier_mode='auto_1'):
'''
# Args
net_id: Network to inspect (lib/recon/config.py)
img_fname: Image to inspect (data/gallery/)
gpu_id: Id of GPU to use
multiplier_mode: see self._filter_feature()
'''
self.net_id = net_id
self.gpu_id = gpu_id
self.multiplier_mode = multiplier_mode
self.config = config.nets[self.net_id]
self.net_param = self._load_param(with_data=False)
self.mean = load_mean_image(self.config.mean_fname)
self._labels = load_ilsvrc12_labels(self.config.labels_fname)
self._set_image(img_fname)
self.net.forward()
self.dag = nx.DiGraph()
self._reconstructions = {}
def __init__(self, net_id, img_fname, gpu_id, multiplier_mode='auto_1'):
'''
# Args
net_id: Network to inspect (lib/recon/config.py)
img_fname: Image to inspect (data/gallery/)
gpu_id: Id of GPU to use
multiplier_mode: see self._filter_feature()
'''
self.net_id = net_id
self.gpu_id = gpu_id
self.multiplier_mode = multiplier_mode
self.config = config.nets[self.net_id]
self.net_param = self._load_param(with_data=False)
self.mean = load_mean_image(self.config.mean_fname)
self._labels = load_ilsvrc12_labels(self.config.labels_fname)
self._set_image(img_fname)
self.net.forward()
self.dag = nx.DiGraph()
self._reconstructions = {}
def build_max_act_db(self, blob_name, k=5):
# don't use self.net, which a deploy net (data comes from python)
def _get_blob_layer(net_param, blob_name):
'''
Return the name of the last layer to output a blob.
'''
layer_name = None
for layer in net_param.layer:
if blob_name in layer.top:
layer_name = layer.name
if layer_name == None:
raise Exception('could not find a layer that outputs ' \
'blob {}'.format(blob_name))
return layer_name
def _len_lmdb(fname):
logger.info('computing lmdb size...')
db = lmdb.open(fname)
n_examples = sum(1 for _ in db.begin().cursor())
del db
logger.info('found lmdb size: {} examples'.format(n_examples))
return n_examples
def _showable(img, rescale=False):
# NOTE: assumes images in the net are BGR
img = img.transpose([1, 2, 0])
img = (img + mean)[:, :, ::-1]
img = img.clip(0, 255).astype(np.uint8)
if rescale:
img = rescale_intensity(img)
return img
def _reconstruct_backward(net, net_param, blob_name, blob_idxs, act_vals=None, act_mult=1):
blob = net.blobs[blob_name]
blob.diff[:] = 0
for i, blob_idx in enumerate(blob_idxs):
if act_vals == None:
blob.diff[blob_idx] = act_mult * blob.data[blob_idx]
else:
blob.diff[blob_idx] = act_mult * act_vals[i]
layer_name = _get_blob_layer(net_param, blob_name)
net.backward(start=layer_name, end=img_layer_name)
def _to_bbox(img, blob_idx):
'''
Take an image which is mostly 0s and return the smallest
bounding box which contains all non-0 entries.
img array of size (c, h, w)
blob_idx tuple with (num, channels, height, width) or (num, channels)
'''
# (num, channel, height, width)
if len(blob_idx) == 4:
row, col = blob_idx[-2:]
m = abs(img).max(axis=0)
linear_idx_map = np.arange(np.prod(m.shape)).reshape(m.shape)
linear_idxs = linear_idx_map[m > 0]
rows = (linear_idxs // m.shape[0])
cols = (linear_idxs % m.shape[0])
if np.prod(rows.shape) == 0 or np.prod(cols.shape) == 0:
top_left = row, col
bottom_right = row, col
else:
top_left = (rows.min(), cols.min())
bottom_right = (rows.max(), cols.max())
return (top_left, bottom_right)
# (num, channel)
elif len(blob_idx) == 2:
return ((0, 0), (img.shape[1]-1, img.shape[2]-1))
else:
raise Exception('do not know how to create a bounding box from ' \
'blob_idx {}'.format(blob_idx))
mean = load_mean_image(self.config.mean_fname)
data_net_param = self._load_param(with_data=True)
net = self._load_net(data_net_param)
img_layer_name = _get_blob_layer(data_net_param, self.config.image_blob_name)
batch_size = self.config.batch_size
n_batches = int(math.ceil(self.config.num_examples / float(batch_size)))
layers = {l.name: l for l in data_net_param.layer}
dbname = layers[self.config.data_layer_name].data_param.source
n_db_examples = _len_lmdb(dbname)
assert n_db_examples == self.config.num_examples
assert n_db_examples % batch_size == 0
example_offset = 0
# TODO: load the top_k lists from the db and append to them if they already exist
maxes = defaultdict(lambda: [{'activation': -np.inf} for _ in range(k)])
logger.info('blob {}'.format(blob_name))
img_blob = net.blobs[self.config.image_blob_name]
blob = net.blobs[blob_name]
assert blob.data.shape[0] == batch_size
n_features = blob.data.shape[1]
for batch_i in xrange(n_batches):
net.forward()
logger.info('batch {}'.format(batch_i))
# for each example
for num in xrange(batch_size): # examples
logger.info('example {}'.format(example_offset + num))
for chan in xrange(n_features): # channel
key = self._get_key(blob_name, chan)
top_k = maxes[key]
fmap_idx = blob.data[num, chan].argmax()
if len(blob.data.shape) > 2:
fmap_idx = np.unravel_index(fmap_idx, blob.data.shape[2:])
else:
fmap_idx = tuple()
blob_idx = (num, chan) + fmap_idx
act = blob.data[blob_idx]
# is this example's best patch in the topk?
if act > top_k[0]['activation']:
img = img_blob.data[num, :, :, :]
entry = {
'example_idx': example_offset + num,
'feature_map_loc': blob_idx[2:] if len(blob_idx) > 2 else tuple(),
'img': _showable(img),
#'reconstruction': self._showable(reconstruction),
'reconstruct_idx': blob_idx,
'batch_idx': batch_i,
'num': num,
'activation': act,
#'patch_bbox': bbox,
}
top_k_idx = bisect([v['activation'] for v in top_k], act)
top_k.insert(top_k_idx, entry)
del top_k[0]
example_offset += batch_size
entries_per_example = [[] for _ in range(n_db_examples)]
for chan in range(n_features):
key = self._get_key(blob_name, chan)
top_k = maxes[key]
for entry in top_k:
max_act = top_k[-1]['activation']
ex_idx = entry['example_idx']
entries_per_example[ex_idx].append((entry['reconstruct_idx'], max_act, entry))
# for each example, list the reconstructions which must be computed
example_offset = 0
# compute those reconstructions
for batch_i in xrange(n_batches):
net.forward()
logger.info('rec batch {}'.format(batch_i))
entries_in_batch = entries_per_example[example_offset:example_offset+batch_size]
def total_entries():
total = 0
return sum(len(e) for e in entries_in_batch)
while total_entries():
entries_to_process = [ent.pop() for ent in entries_in_batch if ent]
blob_idxs, act_vals, blob_entries = zip(*entries_to_process)
# one idx per example
assert len(set(zip(*blob_idxs)[0])) == len(entries_to_process)
_reconstruct_backward(net,
data_net_param,
blob_name,
blob_idxs,
act_vals,
act_mult=self.config.blob_multipliers[blob_name])
for blob_idx, entry in zip(blob_idxs, blob_entries):
num = entry['num']
reconstruction = img_blob.diff[num, :, :, :]
bbox = _to_bbox(reconstruction, blob_idx)
entry['reconstruction'] = _showable(reconstruction, rescale=True)
entry['patch_bbox'] = bbox
example_offset += batch_size
logger.info('finished computing maximum activations... writing to db')
act_env = lmdb.open(self.config.max_activation_dbname, map_size=recon.config.config.lmdb_map_size)
with act_env.begin(write=True) as txn:
for key, top_k in maxes.iteritems():
s = pkl.dumps(top_k)
txn.put(key, s)
def build_max_act_db(self, blob_name, k=5):
# don't use self.net, which a deploy net (data comes from python)
def _get_blob_layer(net_param, blob_name):
'''
Return the name of the last layer to output a blob.
'''
layer_name = None
for layer in net_param.layer:
if blob_name in layer.top:
layer_name = layer.name
if layer_name == None:
raise Exception('could not find a layer that outputs ' \
'blob {}'.format(blob_name))
return layer_name
def _len_lmdb(fname):
logger.info('computing lmdb size...')
db = lmdb.open(fname)
n_examples = sum(1 for _ in db.begin().cursor())
del db
logger.info('found lmdb size: {} examples'.format(n_examples))
return n_examples
def _showable(img, rescale=False):
# NOTE: assumes images in the net are BGR
img = img.transpose([1, 2, 0])
img = (img + mean)[:, :, ::-1]
img = img.clip(0, 255).astype(np.uint8)
if rescale:
img = rescale_intensity(img)
return img
def _reconstruct_backward(net,
net_param,
blob_name,
blob_idxs,
act_vals=None,
act_mult=1):
blob = net.blobs[blob_name]
blob.diff[:] = 0
for i, blob_idx in enumerate(blob_idxs):
if act_vals == None:
blob.diff[blob_idx] = act_mult * blob.data[blob_idx]
else:
blob.diff[blob_idx] = act_mult * act_vals[i]
layer_name = _get_blob_layer(net_param, blob_name)
net.backward(start=layer_name, end=img_layer_name)
def _to_bbox(img, blob_idx):
'''
Take an image which is mostly 0s and return the smallest
bounding box which contains all non-0 entries.
img array of size (c, h, w)
blob_idx tuple with (num, channels, height, width) or (num, channels)
'''
# (num, channel, height, width)
if len(blob_idx) == 4:
row, col = blob_idx[-2:]
m = abs(img).max(axis=0)
linear_idx_map = np.arange(np.prod(m.shape)).reshape(m.shape)
linear_idxs = linear_idx_map[m > 0]
rows = (linear_idxs // m.shape[0])
cols = (linear_idxs % m.shape[0])
if np.prod(rows.shape) == 0 or np.prod(cols.shape) == 0:
top_left = row, col
bottom_right = row, col
else:
top_left = (rows.min(), cols.min())
bottom_right = (rows.max(), cols.max())
return (top_left, bottom_right)
# (num, channel)
elif len(blob_idx) == 2:
return ((0, 0), (img.shape[1] - 1, img.shape[2] - 1))
else:
raise Exception('do not know how to create a bounding box from ' \
'blob_idx {}'.format(blob_idx))
mean = load_mean_image(self.config.mean_fname)
data_net_param = self._load_param(with_data=True)
net = self._load_net(data_net_param)
img_layer_name = _get_blob_layer(data_net_param,
self.config.image_blob_name)
batch_size = self.config.batch_size
n_batches = int(math.ceil(self.config.num_examples /
float(batch_size)))
layers = {l.name: l for l in data_net_param.layer}
dbname = layers[self.config.data_layer_name].data_param.source
n_db_examples = _len_lmdb(dbname)
assert n_db_examples == self.config.num_examples
assert n_db_examples % batch_size == 0
example_offset = 0
# TODO: load the top_k lists from the db and append to them if they already exist
maxes = defaultdict(lambda: [{
'activation': -np.inf
} for _ in range(k)])
logger.info('blob {}'.format(blob_name))
img_blob = net.blobs[self.config.image_blob_name]
blob = net.blobs[blob_name]
assert blob.data.shape[0] == batch_size
n_features = blob.data.shape[1]
for batch_i in xrange(n_batches):
net.forward()
logger.info('batch {}'.format(batch_i))
# for each example
for num in xrange(batch_size): # examples
logger.info('example {}'.format(example_offset + num))
for chan in xrange(n_features): # channel
key = self._get_key(blob_name, chan)
top_k = maxes[key]
fmap_idx = blob.data[num, chan].argmax()
if len(blob.data.shape) > 2:
fmap_idx = np.unravel_index(fmap_idx,
blob.data.shape[2:])
else:
fmap_idx = tuple()
blob_idx = (num, chan) + fmap_idx
act = blob.data[blob_idx]
# is this example's best patch in the topk?
if act > top_k[0]['activation']:
img = img_blob.data[num, :, :, :]
entry = {
'example_idx':
example_offset + num,
'feature_map_loc':
blob_idx[2:] if len(blob_idx) > 2 else tuple(),
'img':
_showable(img),
#'reconstruction': self._showable(reconstruction),
'reconstruct_idx':
blob_idx,
'batch_idx':
batch_i,
'num':
num,
'activation':
act,
#'patch_bbox': bbox,
}
top_k_idx = bisect([v['activation'] for v in top_k],
act)
top_k.insert(top_k_idx, entry)
del top_k[0]
example_offset += batch_size
entries_per_example = [[] for _ in range(n_db_examples)]
for chan in range(n_features):
key = self._get_key(blob_name, chan)
top_k = maxes[key]
for entry in top_k:
max_act = top_k[-1]['activation']
ex_idx = entry['example_idx']
entries_per_example[ex_idx].append(
(entry['reconstruct_idx'], max_act, entry))
# for each example, list the reconstructions which must be computed
example_offset = 0
# compute those reconstructions
for batch_i in xrange(n_batches):
net.forward()
logger.info('rec batch {}'.format(batch_i))
entries_in_batch = entries_per_example[
example_offset:example_offset + batch_size]
def total_entries():
total = 0
return sum(len(e) for e in entries_in_batch)
while total_entries():
entries_to_process = [
ent.pop() for ent in entries_in_batch if ent
]
blob_idxs, act_vals, blob_entries = zip(*entries_to_process)
# one idx per example
assert len(set(zip(*blob_idxs)[0])) == len(entries_to_process)
_reconstruct_backward(
net,
data_net_param,
blob_name,
blob_idxs,
act_vals,
act_mult=self.config.blob_multipliers[blob_name])
for blob_idx, entry in zip(blob_idxs, blob_entries):
num = entry['num']
reconstruction = img_blob.diff[num, :, :, :]
bbox = _to_bbox(reconstruction, blob_idx)
entry['reconstruction'] = _showable(reconstruction,
rescale=True)
entry['patch_bbox'] = bbox
example_offset += batch_size
logger.info('finished computing maximum activations... writing to db')
act_env = lmdb.open(self.config.max_activation_dbname,
map_size=recon.config.config.lmdb_map_size)
with act_env.begin(write=True) as txn:
for key, top_k in maxes.iteritems():
s = pkl.dumps(top_k)
txn.put(key, s)
