def main():
parser = argparse.ArgumentParser(description='Finds images in a training set that cause max activation for a network; saves results in a pickled NetMaxTracker.')
parser.add_argument('--N', type = int, default = 9, help = 'note and save top N activations')
parser.add_argument('--gpu', action = 'store_true', help = 'use gpu')
parser.add_argument('net_prototxt', type = str, default = '', help = 'network prototxt to load')
parser.add_argument('net_weights', type = str, default = '', help = 'network weights to load')
parser.add_argument('datadir', type = str, default = '.', help = 'directory to look for files in')
parser.add_argument('filelist', type = str, help = 'list of image files to consider, one per line')
parser.add_argument('outfile', type = str, help = 'output filename for pkl')
#parser.add_argument('--mean', type = str, default = '', help = 'data mean to load')
args = parser.parse_args()
imagenet_mean = load_imagenet_mean()
net = caffe.Classifier(args.net_prototxt, args.net_weights,
mean=imagenet_mean,
channel_swap=(2,1,0),
raw_scale=255,
image_dims=(256, 256))
if args.gpu:
caffe.set_mode_gpu()
else:
caffe.set_mode_cpu()
with WithTimer('Scanning images'):
max_tracker = scan_images_for_maxes(net, args.datadir, args.filelist, args.N)
with WithTimer('Saving maxes'):
with open(args.outfile, 'wb') as ff:
pickle.dump(max_tracker, ff, -1)
def scan_images_for_maxes(net, n_top, images_df=None, images_dir=None, filelist_path=None,
layers=default_layers, is_conv_layer=default_is_conv):
if images_df is None:
image_filenames, image_labels = load_file_list(filelist_path)
image_pathes = [os.path.join(images_dir, p) for p in image_filenames]
else:
image_pathes = images_df['path'].values
image_labels = images_df['label'].values
print 'Scanning %d files' % len(image_pathes)
print ' First file', image_pathes[0]
tracker = NetMaxTracker(n_top = n_top, layers=layers, is_conv=is_conv_layer)
for image_idx in xrange(len(image_pathes)):
img_path = image_pathes[image_idx]
image_class = image_labels[image_idx]
do_print = (image_idx % 100 == 0)
if do_print:
print '%s Image %d/%d' % (datetime.now().ctime(), image_idx, len(image_pathes))
with WithTimer('Load image', quiet=(not do_print)):
im = caffe.io.load_image(img_path)
with WithTimer('Predict ', quiet=(not do_print)):
net.predict([im], oversample=False) # Just take center crop
with WithTimer('Update ', quiet=(not do_print)):
tracker.update(net, image_idx, image_class)
print 'done!'
return tracker
def save_representations(net, datadir, filelist, layer, first_N=None):
image_filenames, image_labels = load_file_list(filelist)
if first_N is None:
first_N = len(image_filenames)
assert first_N <= len(image_filenames)
image_indices = range(first_N)
print 'Scanning %d files' % len(image_indices)
assert len(image_indices) > 0
print ' First file', os.path.join(datadir, image_filenames[image_indices[0]])
indices = None
rep = None
for ii, image_idx in enumerate(image_indices):
filename = image_filenames[image_idx]
image_class = image_labels[image_idx]
do_print = (image_idx % 10 == 0)
if do_print:
print '%s Image %d/%d' % (datetime.now().ctime(), image_idx, len(image_indices))
with WithTimer('Load image', quiet=not do_print):
im = caffe.io.load_image(os.path.join(datadir, filename))
with WithTimer('Predict ', quiet=not do_print):
net.predict([im], oversample=False) # Just take center crop
with WithTimer('Store ', quiet=not do_print):
if rep is None:
rep_shape = net.blobs[layer].data[0].shape # e.g. (256,13,13)
rep = np.zeros((len(image_indices),) + rep_shape) # e.g. (1000,256,13,13)
indices = [0] * len(image_indices)
indices[ii] = image_idx
rep[ii] = net.blobs[layer].data[0]
print 'done!'
return indices, rep
def scan_images_for_maxes(net, datadir, filelist, n_top):
image_filenames, image_xmls, image_labels = load_file_list(filelist)
print('Scanning %d files' % len(image_filenames))
print(' First file', os.path.join(datadir, image_filenames[0]))
tracker = NetMaxTracker(n_top=n_top)
for image_idx in xrange(len(image_filenames)):
filename = image_filenames[image_idx]
roiname = image_xmls[image_idx]
image_class = image_labels[image_idx]
do_print = (image_idx % 100 == 0)
if do_print:
print('%s Image %d/%d' %
(datetime.now().ctime(), image_idx, len(image_filenames)))
with WithTimer('Load image', quiet=not do_print):
im = load_image(os.path.join(datadir, filename))
_, _, rois = parse_roi_xml(os.path.join(datadir, roiname))
rois = np.array(rois)
with WithTimer('Predict ', quiet=not do_print):
net.blobs['data'].data[...] = im
net.blobs['rois'].reshape(1, *rois.shape)
net.blobs['rois'].data[...] = rois
net.forward()
with WithTimer('Update ', quiet=not do_print):
tracker.update(net, image_idx, image_class)
print('done!')
return tracker
def main():
parser = argparse.ArgumentParser(description='Finds images in a training set that cause max activation for a network; saves results in a pickled NetMaxTracker.')
parser.add_argument('--N', type=int, default=9, help='note and save top N activations')
parser.add_argument('--gpu-id', type=int, default=0, help='GPU id. Set -1 to use CPU')
parser.add_argument('net_prototxt', type=str, default='', help='network prototxt to load')
parser.add_argument('net_weights', type=str, default='', help='network weights to load')
parser.add_argument('output_path', type=str, help='output filename for pkl')
parser.add_argument('--mean-path', type=str, default=None, help='path to mean file')
parser.add_argument('--images-dir', type=str, default='.', help='directory to look for files in. Used only together with --filelist-path')
group = parser.add_mutually_exclusive_group()
group.add_argument('--filelist-path', type=str, default=None,
help='list of image files to consider, one per line')
group.add_argument('--images-csv-path', type=str, default=None,
help='path to csv file containing image pathes and labels')
args = parser.parse_args()
if args.mean_path is not None:
print 'Loading RGB HWC mean and convert it to BRG CWH from', args.mean_path
mean = load_rgb_hwc_mean_and_convert(args.mean_path)
else:
print 'Loading Imagenet mean'
mean = load_imagenet_mean()
if args.gpu_id >= 0:
print 'Using GPU: {}'.format(args.gpu_id)
caffe.set_device(args.gpu_id)
caffe.set_mode_gpu()
else:
caffe.set_mode_cpu()
net = caffe.Classifier(args.net_prototxt, args.net_weights,
mean=mean,
channel_swap=(2, 1, 0),
raw_scale=255,
image_dims=(256, 256))
layers = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'fc6', 'fc7', 'fc8_exh_all', 'prob']
is_conv_layer = [('conv' in ll) for ll in layers]
images_df = None
if args.images_csv_path is not None:
images_df = pd.read_csv(args.images_csv_path, index_col='index')
with WithTimer('Scanning images'):
max_tracker = scan_images_for_maxes(net, args.N,
datadir=args.images_dir,
filelist_path=args.filelist_path,
images_df=images_df,
layers=layers, is_conv_layer=is_conv_layer)
with WithTimer('Saving maxes'):
with open(args.output_path, 'wb') as ff:
pickle.dump(max_tracker, ff, -1)
def main():
parser = argparse.ArgumentParser(
description='Finds images in a training set that cause max activation for a network; saves results in a pickled NetMaxTracker.')
parser.add_argument('--N', type=int, default=9, help='note and save top N activations')
parser.add_argument('--gpu', action='store_true', help='use gpu')
parser.add_argument('net_prototxt', type=str, default='', help='network prototxt to load')
parser.add_argument('net_weights', type=str, default='', help='network weights to load')
parser.add_argument('datadir', type=str, default='.', help='directory to look for files in')
parser.add_argument('filelist', type=str, help='list of image files to consider, one per line')
parser.add_argument('outfile', type=str, help='output filename for pkl')
parser.add_argument('net_layers', type=str, help='list of layers to compute, separated by comma')
parser.add_argument('net_is_conv', type=str,
help='list of 1 (True) and 0 (False), separated by comma, same length as net_layers')
# parser.add_argument('--mean', type = str, default = '', help = 'data mean to load')
args = parser.parse_args()
# layers, is_conv
layers = [x.strip() for x in args.net_layers.split(',')]
is_conv = [x.strip() for x in args.net_is_conv.split(',')]
is_conv = [True if x == '1' else False for x in is_conv]
assert len(layers) == len(is_conv)
imagenet_mean = load_imagenet_mean()
# this should be rewritten in transformer + net, and some batch operation should be supported.
# whatever, just make it usable first.
# also, this restricts to images with labels, since `filelist` must accept something with label
# and restrict the dataset to those with size 256x256.
net = caffe.Classifier(args.net_prototxt, args.net_weights,
mean=imagenet_mean,
channel_swap=(2, 1, 0),
raw_scale=255,
image_dims=(256, 256))
if args.gpu:
caffe.set_mode_gpu()
else:
caffe.set_mode_cpu()
with WithTimer('Scanning images'):
max_tracker = scan_images_for_maxes(net, args.datadir, args.filelist, args.N,
layers=layers, is_conv=is_conv)
with WithTimer('Saving maxes'):
with open(args.outfile, 'wb') as ff:
pickle.dump(max_tracker, ff, -1)
def save_max_tracker_to_file(filename, net_max_tracker):
dir_name = os.path.dirname(filename)
mkdir_p(dir_name)
with WithTimer('Saving maxes'):
with open(filename, 'wb') as ff:
pickle.dump(net_max_tracker, ff, -1)
# save text version of pickle file for easier debugging
pickle_to_text(filename)
def save_representations(net, datadir, filelist, layer, first_N=None):
image_filenames, image_labels = load_file_list(filelist)
if first_N is None:
first_N = len(image_filenames)
assert first_N <= len(image_filenames)
image_indices = range(first_N)
print('Scanning %d files' % len(image_indices))
assert len(image_indices) > 0
print(' First file',
os.path.join(datadir, image_filenames[image_indices[0]]))
indices = None
rep = None
for ii, image_idx in enumerate(image_indices):
filename = image_filenames[image_idx]
roiname = image_xmls[image_idx]
image_class = image_labels[image_idx]
do_print = (image_idx % 10 == 0)
if do_print:
print('%s Image %d/%d' %
(datetime.now().ctime(), image_idx, len(image_indices)))
with WithTimer('Load image & ROIs', quiet=not do_print):
im = load_image(os.path.join(datadir, filename))
_, _, rois = parse_roi_xml(os.path.join(datadir, roiname))
rois = np.array(rois)
with WithTimer('Predict ', quiet=not do_print):
net.blobs['data'].data[...] = im
net.blobs['rois'].reshape(1, *rois.shape)
net.blobs['rois'].data[...] = rois
net.forward()
with WithTimer('Store ', quiet=not do_print):
if rep is None:
rep_shape = net.blobs[layer].data[0].shape # e.g. (256,13,13)
rep = np.zeros((len(image_indices), ) +
rep_shape) # e.g. (1000,256,13,13)
indices = [0] * len(image_indices)
indices[ii] = image_idx
rep[ii] = net.blobs[layer].data[0]
print('done!')
return indices, rep
def scan_images_for_maxes(net, datadir, filelist, n_top, layers=None, is_conv=None):
image_filenames, image_labels = load_file_list(filelist)
print 'Scanning %d files' % len(image_filenames)
print ' First file', os.path.join(datadir, image_filenames[0])
tracker = NetMaxTracker(n_top=n_top, layers=layers, is_conv=is_conv)
for image_idx in xrange(len(image_filenames)):
filename = image_filenames[image_idx]
image_class = image_labels[image_idx]
# im = caffe.io.load_image('../../data/ilsvrc12/mini_ilsvrc_valid/sized/ILSVRC2012_val_00000610.JPEG')
do_print = (image_idx % 100 == 0)
if do_print:
print '%s Image %d/%d' % (datetime.now().ctime(), image_idx, len(image_filenames))
with WithTimer('Load image', quiet=not do_print):
im = caffe.io.load_image(os.path.join(datadir, filename))
with WithTimer('Predict ', quiet=not do_print):
net.predict([im], oversample=False) # Just take center crop
with WithTimer('Update ', quiet=not do_print):
tracker.update(net, image_idx, image_class)
print 'done!'
return tracker
def main():
parser = argparse.ArgumentParser(
description='Loads a pickled NetMaxTracker and outputs one or more of {the patches of the image, a deconv patch, a backprop patch} associated with the maxes.')
parser.add_argument('--model')
parser.add_argument('--idx-begin', type=int, default=None, help='Start at this unit (default: all units).')
parser.add_argument('--idx-end', type=int, default=None, help='End at this unit (default: all units).')
args = parser.parse_args()
settings = Settings.Settings()
settings.load_settings(args.model)
net = load_network(settings)
# set network batch size
current_input_shape = net.blobs[net.inputs[0]].shape
current_input_shape[0] = settings.max_tracker_batch_size
net.blobs[net.inputs[0]].reshape(*current_input_shape)
net.reshape()
assert settings.max_tracker_do_maxes or settings.max_tracker_do_deconv or settings.max_tracker_do_deconv_norm or settings.max_tracker_do_backprop or settings.max_tracker_do_backprop_norm, 'Specify at least one do_* option to output.'
nmt = load_max_tracker_from_file(settings.find_maxes_output_file)
for layer_name in settings.layers_to_output_in_offline_scripts:
print 'Started work on layer %s' % (layer_name)
mt = nmt.max_trackers[layer_name]
if args.idx_begin is None:
idx_begin = 0
if args.idx_end is None:
idx_end = mt.max_vals.shape[0]
with WithTimer('Saved %d images per unit for %s units %d:%d.' % (settings.N, layer_name, idx_begin, idx_end)):
output_max_patches(settings, mt, net, layer_name, idx_begin, idx_end,
settings.N, settings.data_dir, settings.deepvis_outputs_path, False,
(settings.max_tracker_do_maxes, settings.max_tracker_do_deconv,
settings.max_tracker_do_deconv_norm, settings.max_tracker_do_backprop,
settings.max_tracker_do_backprop_norm, True))
if settings.search_min:
output_max_patches(settings, mt, net, layer_name, idx_begin, idx_end,
settings.N, settings.data_dir, settings.deepvis_outputs_path, True,
(settings.max_tracker_do_maxes, settings.max_tracker_do_deconv,
settings.max_tracker_do_deconv_norm, settings.max_tracker_do_backprop,
settings.max_tracker_do_backprop_norm, True))
def main():
#pdb.set_trace()
parser = argparse.ArgumentParser(description='Finds images in a training set that cause max activation for a network; saves results in a pickled NetMaxTracker.')
parser.add_argument('--N', type = int, default = 9, help = 'note and save top N activations')
parser.add_argument('--gpu', action = 'store_true', default = settings.caffevis_mode_gpu, help = 'use gpu')
parser.add_argument('--net_prototxt', type = str, default = settings.caffevis_deploy_prototxt, help = 'network prototxt to load')
parser.add_argument('--net_weights', type = str, default = settings.caffevis_network_weights, help = 'network weights to load')
parser.add_argument('--datadir', type = str, default = settings.static_files_dir, help = 'directory to look for files in')
parser.add_argument('--outfile', type=str, default = os.path.join(settings.caffevis_outputs_dir, 'find_max_acts_output.pickled'), help='output filename for pkl')
parser.add_argument('--outdir', type = str, default = settings.caffevis_outputs_dir, help = 'Which output directory to use. Files are output into outdir/layer/unit_%%04d/{max_histogram}.png')
parser.add_argument('--do-histograms', action = 'store_true', default = settings.max_tracker_do_histograms, help = 'Output histogram image file containing histogrma of max values per channel')
parser.add_argument('--do-correlation', action = 'store_true', default = settings.max_tracker_do_correlation, help = 'Output correlation image file containing correlation of channels per layer')
parser.add_argument('--search-min', action='store_true', default=True, help='Should we also search for minimal activations?')
args = parser.parse_args()
settings.caffevis_deploy_prototxt = args.net_prototxt
settings.caffevis_network_weights = args.net_weights
net, data_mean = load_network(settings)
#data_mean = np.load('/home/mbm/Desktop/Aux/models/alexnet/places365CNN_mean.npy')
# validate batch size
if settings.is_siamese and settings._calculated_siamese_network_format == 'siamese_batch_pair':
# currently, no batch support for siamese_batch_pair networks
# it can be added by simply handle the batch indexes properly, but it should be thoroughly tested
assert (settings.max_tracker_batch_size == 1)
# set network batch size
current_input_shape = net.blobs[net.inputs[0]].shape
current_input_shape[0] = settings.max_tracker_batch_size
net.blobs[net.inputs[0]].reshape(*current_input_shape)
net.reshape()
with WithTimer('Scanning images'):
if settings.is_siamese:
net_max_tracker = scan_pairs_for_maxes(settings, net, args.datadir, args.N, args.outdir, args.search_min)
else: # normal operation
net_max_tracker = scan_images_for_maxes(settings, net, args.datadir, args.N, args.outdir, args.search_min)
save_max_tracker_to_file(args.outfile, net_max_tracker)
if args.do_correlation:
net_max_tracker.calculate_correlation(args.outdir)
if args.do_histograms:
net_max_tracker.calculate_histograms(args.outdir)
def save_max_tracker_to_file(filename, net_max_tracker):
dir_name = os.path.dirname(filename)
mkdir_p(dir_name)
with WithTimer('Saving maxes'):
# Save unpickleable list of ndarrays with all maximal values
all_max_vals_dict = dict()
for layer_name in net_max_tracker.layers:
all_max_vals_dict[layer_name] = net_max_tracker.max_trackers[
layer_name].all_max_vals
np.save(os.path.join(dir_name, 'all_max_vals.npy'), all_max_vals_dict)
del all_max_vals_dict
# Pickle pickleable Net_Max_Tracker parameters
with open(filename, 'wb') as ff:
pickle.dump(net_max_tracker, ff, -1)
# save text version of pickle file for easier debugging
pickle_to_text(filename)
def main():
parser = argparse.ArgumentParser(
description=
'Finds images in a training set that cause max activation for a network; saves results in a pickled NetMaxTracker.'
)
parser.add_argument('--model')
args = parser.parse_args()
settings = Settings.Settings()
settings.load_settings(args.model)
net = load_network(settings)
# set network batch size
current_input_shape = net.blobs[net.inputs[0]].shape
current_input_shape[0] = settings.max_tracker_batch_size
net.blobs[net.inputs[0]].reshape(*current_input_shape)
net.reshape()
with WithTimer('Scanning images'):
net_max_tracker = scan_images_for_maxes(settings, net,
settings.data_dir, settings.N,
settings.deepvis_outputs_path,
settings.search_min)
save_max_tracker_to_file(settings.find_maxes_output_file, net_max_tracker)
def output_max_patches(max_tracker, net, layer, idx_begin, idx_end, num_top, datadir, filelist, outdir, do_which,
layer_info):
do_maxes, do_deconv, do_deconv_norm, do_backprop, do_backprop_norm, do_info = do_which
assert do_maxes or do_deconv or do_deconv_norm or do_backprop or do_backprop_norm or do_info, 'nothing to do'
mt = max_tracker
rc = RegionComputer(layer_info)
image_filenames, image_labels = load_file_list(filelist)
print 'Loaded filenames and labels for %d files' % len(image_filenames)
print ' First file', os.path.join(datadir, image_filenames[0])
num_top_in_mt = mt.max_locs.shape[1]
assert num_top <= num_top_in_mt, 'Requested %d top images but MaxTracker contains only %d' % (
num_top, num_top_in_mt)
assert idx_end >= idx_begin, 'Range error'
size_ii, size_jj = get_max_data_extent(net, layer, rc, mt.is_conv)
data_size_ii, data_size_jj = net.blobs['data'].data.shape[2:4]
assert 0 < size_ii <= data_size_ii
assert 0 < size_jj <= data_size_jj
n_total_images = (idx_end - idx_begin) * num_top
for cc, channel_idx in enumerate(range(idx_begin, idx_end)):
unit_dir = os.path.join(outdir, layer, 'unit_%04d' % channel_idx)
mkdir_p(unit_dir)
if do_info:
info_filename = os.path.join(unit_dir, 'info.txt')
info_file = open(info_filename, 'w')
print >> info_file, '# is_conv val image_idx image_class i(if is_conv) j(if is_conv) filename'
# iterate through maxes from highest (at end) to lowest
for max_idx_0 in range(num_top):
max_idx = num_top_in_mt - 1 - max_idx_0
if mt.is_conv:
im_idx, im_class, ii, jj = mt.max_locs[channel_idx, max_idx]
else:
im_idx, im_class = mt.max_locs[channel_idx, max_idx]
recorded_val = mt.max_vals[channel_idx, max_idx]
filename = image_filenames[im_idx]
do_print = (max_idx_0 == 0)
if do_print:
print '%s Output file/image(s) %d/%d' % (datetime.now().ctime(), cc * num_top, n_total_images)
if mt.is_conv:
# Compute the focus area of the data layer
layer_indices = (ii, ii + 1, jj, jj + 1)
data_indices = rc.convert_region(layer, 'data', layer_indices, normalize_last=True)
data_ii_start, data_ii_end, data_jj_start, data_jj_end = data_indices
# why == 0. I think it's also possible for it to be < 0
# if we have padding in the first layer. such as VGG-16/19.
touching_imin = (data_ii_start <= 0)
touching_jmin = (data_jj_start <= 0)
# Compute how much of the data slice falls outside the actual data [0,max] range
ii_outside = size_ii - (data_ii_end - data_ii_start) # possibly 0
jj_outside = size_jj - (data_jj_end - data_jj_start) # possibly 0
# this should be true, since usually, the central unit (with extent size_ii, size_jj)
# should cover larger or equal region compared to all others, since there's less effect of padding.
assert ii_outside >= 0 and jj_outside >= 0
# if ii_outside (or jj_outside) is > 0,
# then that means more data got chopped off due to padding.
# if touching_imin, that means the chopping is due to close to left side
# if touching_jmin, chopping due to close to top.
# so we should pad those data there.
if touching_imin:
out_ii_start = ii_outside
out_ii_end = size_ii
else:
out_ii_start = 0
out_ii_end = size_ii - ii_outside
if touching_jmin:
out_jj_start = jj_outside
out_jj_end = size_jj
else:
out_jj_start = 0
out_jj_end = size_jj - jj_outside
else:
ii, jj = 0, 0
data_ii_start, out_ii_start, data_jj_start, out_jj_start = 0, 0, 0, 0
data_ii_end, out_ii_end, data_jj_end, out_jj_end = size_ii, size_ii, size_jj, size_jj
if do_info:
print >> info_file, 1 if mt.is_conv else 0, '%.6f' % mt.max_vals[channel_idx, max_idx],
if mt.is_conv:
print >> info_file, '%d %d %d %d' % tuple(mt.max_locs[channel_idx, max_idx]),
else:
print >> info_file, '%d %d' % tuple(mt.max_locs[channel_idx, max_idx]),
print >> info_file, filename
if not (do_maxes or do_deconv or do_deconv_norm or do_backprop or do_backprop_norm):
continue
with WithTimer('Load image', quiet=not do_print):
im = caffe.io.load_image(os.path.join(datadir, filename))
with WithTimer('Predict ', quiet=not do_print):
net.predict([im], oversample=False) # Just take center crop, same as in scan_images_for_maxes
if len(net.blobs[layer].data.shape) == 4:
reproduced_val = net.blobs[layer].data[0, channel_idx, ii, jj]
else:
reproduced_val = net.blobs[layer].data[0, channel_idx]
if abs(reproduced_val - recorded_val) > .1:
print 'Warning: recorded value %s is suspiciously different from reproduced value %s. Is the filelist the same?' % (
recorded_val, reproduced_val)
if do_maxes:
# grab image from data layer, not from im (to ensure preprocessing / center crop details match between image and deconv/backprop)
out_arr = np.zeros((3, size_ii, size_jj), dtype='float32')
out_arr[:, out_ii_start:out_ii_end, out_jj_start:out_jj_end] = net.blobs['data'].data[0, :,
data_ii_start:data_ii_end,
data_jj_start:data_jj_end]
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(out_arr, os.path.join(unit_dir, 'maxim_%03d.png' % max_idx_0),
autoscale=False, autoscale_center=0)
if do_deconv or do_deconv_norm:
diffs = net.blobs[layer].diff * 0
if len(diffs.shape) == 4:
diffs[0, channel_idx, ii, jj] = 1.0
else:
assert len(diffs.shape) == 2
diffs[0, channel_idx] = 1.0
with WithTimer('Deconv ', quiet=not do_print):
net.deconv_from_layer(layer, diffs)
out_arr = np.zeros((3, size_ii, size_jj), dtype='float32')
out_arr[:, out_ii_start:out_ii_end, out_jj_start:out_jj_end] = net.blobs['data'].diff[0, :,
data_ii_start:data_ii_end,
data_jj_start:data_jj_end]
if out_arr.max() == 0:
print 'Warning: Deconv out_arr in range', out_arr.min(), 'to', out_arr.max(), 'ensure force_backward: true in prototxt'
if do_deconv:
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(out_arr, os.path.join(unit_dir, 'deconv_%03d.png' % max_idx_0),
autoscale=False, autoscale_center=0)
if do_deconv_norm:
out_arr = np.linalg.norm(out_arr, axis=0)
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(out_arr, os.path.join(unit_dir, 'deconvnorm_%03d.png' % max_idx_0))
if do_backprop or do_backprop_norm:
diffs = net.blobs[layer].diff * 0
if len(diffs.shape) == 4:
diffs[0, channel_idx, ii, jj] = 1.0
else:
assert len(diffs.shape) == 2
diffs[0, channel_idx] = 1.0
with WithTimer('Backward ', quiet=not do_print):
net.backward_from_layer(layer, diffs)
out_arr = np.zeros((3, size_ii, size_jj), dtype='float32')
out_arr[:, out_ii_start:out_ii_end, out_jj_start:out_jj_end] = net.blobs['data'].diff[0, :,
data_ii_start:data_ii_end,
data_jj_start:data_jj_end]
if out_arr.max() == 0:
print 'Warning: Deconv out_arr in range', out_arr.min(), 'to', out_arr.max(), 'ensure force_backward: true in prototxt'
if do_backprop:
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(out_arr, os.path.join(unit_dir, 'backprop_%03d.png' % max_idx_0),
autoscale=False, autoscale_center=0)
if do_backprop_norm:
out_arr = np.linalg.norm(out_arr, axis=0)
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(out_arr, os.path.join(unit_dir, 'backpropnorm_%03d.png' % max_idx_0))
if do_info:
info_file.close()
def output_max_patches(settings, max_tracker, net, layer_name, idx_begin,
idx_end, num_top, datadir, outdir, search_min,
do_which):
'''
:param settings:
:param max_tracker:
:param net:
:param layer_name:
:param idx_begin:
:param idx_end:
:param num_top:
:param datadir:
:param outdir:
:param search_min:
:param do_which: do_info must be True
:return:
'''
do_maxes, do_deconv, do_deconv_norm, do_backprop, do_backprop_norm, do_info = do_which
assert do_maxes or do_deconv or do_deconv_norm or do_backprop or do_backprop_norm or do_info, 'nothing to do'
sys.path.insert(0, os.path.join(settings.caffevis_caffe_root, 'python'))
import caffe
mt = max_tracker
locs = mt.min_locs if search_min else mt.max_locs
vals = mt.min_vals if search_min else mt.max_vals
image_filenames, image_labels = get_files_list(datadir)
print 'Loaded filenames and labels for %d files' % len(image_filenames)
print ' First file', os.path.join(datadir, image_filenames[0])
num_top_in_mt = locs.shape[1]
assert num_top <= num_top_in_mt, 'Requested %d top images but MaxTracker contains only %d' % (
num_top, num_top_in_mt)
assert idx_end >= idx_begin, 'Range error'
# minor fix for backwards compatability
if hasattr(mt, 'is_conv'):
mt.is_spatial = mt.is_conv
size_ii, size_jj = get_max_data_extent(net, settings, layer_name,
mt.is_spatial)
data_size_ii, data_size_jj = net.blobs['data'].data.shape[2:4]
net_input_dims = net.blobs['data'].data.shape[2:4]
# prepare variables used for batches
batch = [None] * settings.max_tracker_batch_size
for i in range(0, settings.max_tracker_batch_size):
batch[i] = MaxTrackerCropBatchRecord()
batch_index = 0
channel_to_info_file = dict()
n_total_images = (idx_end - idx_begin) * num_top
for cc, channel_idx in enumerate(range(idx_begin, idx_end)):
unit_dir = os.path.join(outdir, layer_name, 'unit_%04d' % channel_idx)
mkdir_p(unit_dir)
# check if all required outputs exist, in which case skip this iteration
[
info_filename, maxim_filenames, deconv_filenames,
deconvnorm_filenames, backprop_filenames, backpropnorm_filenames
] = generate_output_names(unit_dir, num_top, do_info, do_maxes,
do_deconv, do_deconv_norm, do_backprop,
do_backprop_norm, search_min)
relevant_outputs = info_filename + \
maxim_filenames + \
deconv_filenames + \
deconvnorm_filenames + \
backprop_filenames + \
backpropnorm_filenames
# we skip generation if:
# 1. all outputs exist, AND
# 2.1. (not last iteration OR
# 2.2. last iteration, but batch is empty)
relevant_outputs_exist = [
os.path.exists(file_name) for file_name in relevant_outputs
]
if all(relevant_outputs_exist) and \
((channel_idx != idx_end - 1) or ((channel_idx == idx_end - 1) and (batch_index == 0))):
print "skipped generation of channel %d in layer %s since files already exist" % (
channel_idx, layer_name)
continue
if do_info:
channel_to_info_file[channel_idx] = InfoFileMetadata()
channel_to_info_file[channel_idx].info_file = open(
info_filename[0], 'w')
channel_to_info_file[channel_idx].ref_count = num_top
print >> channel_to_info_file[
channel_idx].info_file, '# is_spatial val image_idx selected_input_index i(if is_spatial) j(if is_spatial) filename'
# iterate through maxes from highest (at end) to lowest
for max_idx_0 in range(num_top):
batch[batch_index].cc = cc
batch[batch_index].channel_idx = channel_idx
batch[batch_index].info_filename = info_filename
batch[batch_index].maxim_filenames = maxim_filenames
batch[batch_index].deconv_filenames = deconv_filenames
batch[batch_index].deconvnorm_filenames = deconvnorm_filenames
batch[batch_index].backprop_filenames = backprop_filenames
batch[batch_index].backpropnorm_filenames = backpropnorm_filenames
batch[batch_index].info_file = channel_to_info_file[
channel_idx].info_file
batch[batch_index].max_idx_0 = max_idx_0
batch[batch_index].max_idx = num_top_in_mt - 1 - batch[
batch_index].max_idx_0
if mt.is_spatial:
# fix for backward compatability
if locs.shape[2] == 5:
# remove second column
locs = np.delete(locs, 1, 2)
batch[batch_index].im_idx, batch[
batch_index].selected_input_index, batch[
batch_index].ii, batch[batch_index].jj = locs[
batch[batch_index].channel_idx,
batch[batch_index].max_idx]
else:
# fix for backward compatability
if locs.shape[2] == 3:
# remove second column
locs = np.delete(locs, 1, 2)
batch[batch_index].im_idx, batch[
batch_index].selected_input_index = locs[
batch[batch_index].channel_idx,
batch[batch_index].max_idx]
batch[batch_index].ii, batch[batch_index].jj = 0, 0
# if ii and jj are invalid then there is no data for this "top" image, so we can skip it
if (batch[batch_index].ii, batch[batch_index].jj) == (-1, -1):
continue
batch[batch_index].recorded_val = vals[
batch[batch_index].channel_idx, batch[batch_index].max_idx]
batch[batch_index].filename = image_filenames[
batch[batch_index].im_idx]
do_print = (batch[batch_index].max_idx_0 == 0)
if do_print:
print '%s Output file/image(s) %d/%d layer %s channel %d' % (
datetime.now().ctime(), batch[batch_index].cc * num_top,
n_total_images, layer_name, batch[batch_index].channel_idx)
# print "DEBUG: (mt.is_spatial, batch[batch_index].ii, batch[batch_index].jj, layer_name, size_ii, size_jj, data_size_ii, data_size_jj)", str((mt.is_spatial, batch[batch_index].ii, batch[batch_index].jj, rc, layer_name, size_ii, size_jj, data_size_ii, data_size_jj))
[batch[batch_index].out_ii_start,
batch[batch_index].out_ii_end,
batch[batch_index].out_jj_start,
batch[batch_index].out_jj_end,
batch[batch_index].data_ii_start,
batch[batch_index].data_ii_end,
batch[batch_index].data_jj_start,
batch[batch_index].data_jj_end] = \
compute_data_layer_focus_area(mt.is_spatial, batch[batch_index].ii, batch[batch_index].jj, settings,
layer_name,
size_ii, size_jj, data_size_ii, data_size_jj)
# print "DEBUG: channel:%d out_ii_start:%d out_ii_end:%d out_jj_start:%d out_jj_end:%d data_ii_start:%d data_ii_end:%d data_jj_start:%d data_jj_end:%d" % \
# (channel_idx,
# batch[batch_index].out_ii_start, batch[batch_index].out_ii_end,
# batch[batch_index].out_jj_start, batch[batch_index].out_jj_end,
# batch[batch_index].data_ii_start, batch[batch_index].data_ii_end,
# batch[batch_index].data_jj_start, batch[batch_index].data_jj_end)
if do_info:
print >> batch[
batch_index].info_file, 1 if mt.is_spatial else 0, '%.6f' % vals[
batch[batch_index].channel_idx,
batch[batch_index].max_idx],
if mt.is_spatial:
print >> batch[
batch_index].info_file, '%d %d %d %d' % tuple(
locs[batch[batch_index].channel_idx,
batch[batch_index].max_idx]),
else:
print >> batch[batch_index].info_file, '%d %d' % tuple(
locs[batch[batch_index].channel_idx,
batch[batch_index].max_idx]),
print >> batch[batch_index].info_file, batch[
batch_index].filename
if not (do_maxes or do_deconv or do_deconv_norm or do_backprop
or do_backprop_norm):
continue
with WithTimer('Load image', quiet=not do_print):
# load image
batch[batch_index].im = caffe.io.load_image(os.path.join(
datadir, batch[batch_index].filename),
color=True)
# resize images according to input dimension
batch[batch_index].im = resize_without_fit(
batch[batch_index].im, net_input_dims)
# convert to float to avoid caffe destroying the image in the scaling phase
batch[batch_index].im = batch[batch_index].im.astype(
np.float32)
batch_index += 1
# if current batch is full
if batch_index == settings.max_tracker_batch_size \
or ((channel_idx == idx_end - 1) and (max_idx_0 == num_top - 1)): # or last iteration
with WithTimer('Predict on batch ', quiet=not do_print):
im_batch = [record.im for record in batch]
net.predict(im_batch, oversample=False)
# go over batch and update statistics
for i in range(0, batch_index):
batch[i].denormalized_layer_name = layer_name
batch[i].denormalized_top_name = layer_name_to_top_name(
net, batch[i].denormalized_layer_name)
batch[i].layer_format = 'normal' # non-siamese
if len(net.blobs[
batch[i].denormalized_top_name].data.shape) == 4:
reproduced_val = net.blobs[
batch[i].denormalized_top_name].data[
i, batch[i].channel_idx, batch[i].ii,
batch[i].jj]
else:
reproduced_val = net.blobs[
batch[i].denormalized_top_name].data[
i, batch[i].channel_idx]
if abs(reproduced_val - batch[i].recorded_val) > .1:
print 'Warning: recorded value %s is suspiciously different from reproduced value %s. Is the filelist the same?' % (
batch[i].recorded_val, reproduced_val)
if do_maxes:
# grab image from data layer, not from im (to ensure preprocessing / center crop details match between image and deconv/backprop)
out_arr = extract_patch_from_image(
net.blobs['data'].data[i], net,
batch[i].selected_input_index, settings,
batch[i].data_ii_end, batch[i].data_ii_start,
batch[i].data_jj_end, batch[i].data_jj_start,
batch[i].out_ii_end, batch[i].out_ii_start,
batch[i].out_jj_end, batch[i].out_jj_start,
size_ii, size_jj)
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(
out_arr,
batch[i].maxim_filenames[batch[i].max_idx_0],
autoscale=False,
autoscale_center=0,
channel_swap=settings.channel_swap)
if do_deconv or do_deconv_norm:
# TODO: we can improve performance by doing batch of deconv_from_layer, but only if we group
# together instances which have the same selected_input_index, this can be done by holding two
# separate batches
for i in range(0, batch_index):
diffs = net.blobs[
batch[i].denormalized_top_name].diff * 0
if len(diffs.shape) == 4:
diffs[i, batch[i].channel_idx, batch[i].ii,
batch[i].jj] = 1.0
else:
diffs[i, batch[i].channel_idx] = 1.0
with WithTimer('Deconv ', quiet=not do_print):
net.deconv_from_layer(
batch[i].denormalized_layer_name,
diffs,
zero_higher=True,
deconv_type='Guided Backprop')
out_arr = extract_patch_from_image(
net.blobs['data'].diff[i], net,
batch[i].selected_input_index, settings,
batch[i].data_ii_end, batch[i].data_ii_start,
batch[i].data_jj_end, batch[i].data_jj_start,
batch[i].out_ii_end, batch[i].out_ii_start,
batch[i].out_jj_end, batch[i].out_jj_start,
size_ii, size_jj)
if out_arr.max() == 0:
print 'Warning: Deconv out_arr in range', out_arr.min(
), 'to', out_arr.max(
), 'ensure force_backward: true in prototxt'
if do_deconv:
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(
out_arr,
batch[i].deconv_filenames[
batch[i].max_idx_0],
autoscale=False,
autoscale_center=0,
channel_swap=settings.channel_swap)
if do_deconv_norm:
out_arr = np.linalg.norm(out_arr, axis=0)
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(
out_arr,
batch[i].deconvnorm_filenames[
batch[i].max_idx_0],
channel_swap=settings.channel_swap)
if do_backprop or do_backprop_norm:
for i in range(0, batch_index):
diffs = net.blobs[
batch[i].denormalized_top_name].diff * 0
if len(diffs.shape) == 4:
diffs[i, batch[i].channel_idx, batch[i].ii,
batch[i].jj] = 1.0
else:
diffs[i, batch[i].channel_idx] = 1.0
with WithTimer('Backward batch ', quiet=not do_print):
net.backward_from_layer(
batch[i].denormalized_layer_name, diffs)
for i in range(0, batch_index):
out_arr = extract_patch_from_image(
net.blobs['data'].diff[i], net,
batch[i].selected_input_index, settings,
batch[i].data_ii_end, batch[i].data_ii_start,
batch[i].data_jj_end, batch[i].data_jj_start,
batch[i].out_ii_end, batch[i].out_ii_start,
batch[i].out_jj_end, batch[i].out_jj_start,
size_ii, size_jj)
if out_arr.max() == 0:
print 'Warning: Deconv out_arr in range', out_arr.min(
), 'to', out_arr.max(
), 'ensure force_backward: true in prototxt'
if do_backprop:
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(
out_arr,
batch[i].backprop_filenames[
batch[i].max_idx_0],
autoscale=False,
autoscale_center=0,
channel_swap=settings.channel_swap)
if do_backprop_norm:
out_arr = np.linalg.norm(out_arr, axis=0)
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(
out_arr,
batch[i].backpropnorm_filenames[
batch[i].max_idx_0],
channel_swap=settings.channel_swap)
# close info files
for i in range(0, batch_index):
channel_to_info_file[batch[i].channel_idx].ref_count -= 1
if channel_to_info_file[
batch[i].channel_idx].ref_count == 0:
if do_info:
channel_to_info_file[
batch[i].channel_idx].info_file.close()
batch_index = 0
def scan_images_for_maxes(settings, net, datadir, n_top, outdir, search_min):
image_filenames, image_labels = get_files_list(datadir)
print 'Scanning %d files' % len(image_filenames)
print ' First file', os.path.join(datadir, image_filenames[0])
sys.path.insert(0, os.path.join(settings.caffevis_caffe_root, 'python'))
import caffe
tracker = NetMaxTracker(
settings,
n_top=n_top,
layers=settings.layers_to_output_in_offline_scripts,
search_min=search_min)
net_input_dims = net.blobs['data'].data.shape[2:4]
# prepare variables used for batches
batch = [None] * settings.max_tracker_batch_size
for i in range(0, settings.max_tracker_batch_size):
batch[i] = MaxTrackerBatchRecord()
batch_index = 0
for image_idx in xrange(len(image_filenames)):
batch[batch_index].image_idx = image_idx
batch[batch_index].filename = image_filenames[image_idx]
do_print = (batch[batch_index].image_idx % 100 == 0)
if do_print:
print '%s Image %d/%d' % (datetime.now().ctime(),
batch[batch_index].image_idx,
len(image_filenames))
with WithTimer('Load image', quiet=not do_print):
try:
batch[batch_index].im = caffe.io.load_image(os.path.join(
datadir, batch[batch_index].filename),
color=True)
batch[batch_index].im = resize_without_fit(
batch[batch_index].im, net_input_dims)
batch[batch_index].im = batch[batch_index].im.astype(
np.float32)
except:
# skip bad/missing inputs
print "WARNING: skipping bad/missing input:", batch[
batch_index].filename
continue
batch_index += 1
# if current batch is full
if batch_index == settings.max_tracker_batch_size \
or image_idx == len(image_filenames) - 1: # or last iteration
# batch predict
with WithTimer('Predict on batch ', quiet=not do_print):
im_batch = [record.im for record in batch]
net.predict(im_batch,
oversample=False) # Just take center crop
# go over batch and update statistics
for i in range(0, batch_index):
with WithTimer('Update ', quiet=not do_print):
tracker.update(net,
batch[i].image_idx,
net_unique_input_source=batch[i].filename,
batch_index=i)
batch_index = 0
print 'done!'
return tracker
def main():
parser = argparse.ArgumentParser(
description=
'Loads a pickled NetMaxTracker and outputs one or more of {the patches of the image, a deconv patch, a backprop patch} associated with the maxes.'
)
parser.add_argument('--N',
type=int,
default=9,
help='Note and save top N activations.')
parser.add_argument('--gpu-id',
type=int,
default=0,
help='GPU id. Set -1 to use CPU')
parser.add_argument('--do-maxes',
action='store_true',
help='Output max patches.')
parser.add_argument('--do-deconv',
action='store_true',
help='Output deconv patches.')
parser.add_argument('--do-deconv-norm',
action='store_true',
help='Output deconv-norm patches.')
parser.add_argument('--do-backprop',
action='store_true',
help='Output backprop patches.')
parser.add_argument('--do-backprop-norm',
action='store_true',
help='Output backprop-norm patches.')
parser.add_argument(
'--do-info',
action='store_true',
help='Output info file containing max filenames and labels.')
parser.add_argument('--idx-begin',
type=int,
default=None,
help='Start at this unit (default: all units).')
parser.add_argument('--idx-end',
type=int,
default=None,
help='End at this unit (default: all units).')
parser.add_argument('nmt_pkl',
type=str,
help='Which pickled NetMaxTracker to load.')
parser.add_argument('net_prototxt',
type=str,
help='Network prototxt to load')
parser.add_argument('net_weights',
type=str,
help='Network weights to load')
parser.add_argument('--mean-path',
type=str,
default=None,
help='path to mean file')
parser.add_argument(
'--images-dir',
type=str,
default=None,
help=
'directory to look for files in. Used only together with --filelist-path'
)
parser.add_argument(
'--output-dir',
type=str,
help=
r'Which output directory to use. Files are output into outdir/layer/unit_%%04d/{maxes,deconv,backprop}_%%03d.png'
)
parser.add_argument('--layer', type=str, help='Which layer to output')
group = parser.add_mutually_exclusive_group()
group.add_argument(
'--images-csv-path',
type=str,
default=None,
help='path to csv file containing image pathes and labels')
group.add_argument(
'--filelist-path',
type=str,
help=
'List of image files to consider, one per line. Must be the same filelist used to produce the NetMaxTracker!'
)
args = parser.parse_args()
print 'Caffe file path:', caffe.__file__
if args.gpu_id >= 0:
print 'Using GPU: {}'.format(args.gpu_id)
caffe.set_device(args.gpu_id)
caffe.set_mode_gpu()
else:
caffe.set_mode_cpu()
if args.mean_path is not None:
print 'Loading RGB HWC mean and convert it to BRG CWH from', args.mean_path
mean = load_rgb_hwc_mean_and_convert(args.mean_path)
else:
print 'Loading Imagenet mean'
mean = load_imagenet_mean()
net = caffe.Classifier(args.net_prototxt,
args.net_weights,
mean=mean,
channel_swap=(2, 1, 0),
raw_scale=255,
image_dims=(256, 256))
assert args.do_maxes or args.do_deconv or args.do_deconv_norm or args.do_backprop or args.do_backprop_norm or args.do_info, 'Specify at least one do_* option to output.'
with open(args.nmt_pkl, 'rb') as ff:
nmt = pkl.load(ff)
mt = nmt.max_trackers[args.layer]
if args.idx_begin is None:
args.idx_begin = 0
if args.idx_end is None:
args.idx_end = mt.max_vals.shape[0]
images_df = None
if args.images_csv_path is not None:
images_df = pd.read_csv(args.images_csv_path, index_col='index')
with WithTimer('Saved %d images per unit for %s units %d:%d.' %
(args.N, args.layer, args.idx_begin, args.idx_end)):
output_max_patches(
mt,
net,
args.layer,
args.idx_begin,
args.idx_end,
args.N,
args.output_dir,
(args.do_maxes, args.do_deconv, args.do_deconv_norm,
args.do_backprop, args.do_backprop_norm, args.do_info),
filelist_path=args.filelist_path,
images_dir=args.images_dir,
images_df=images_df)
def main():
parser = argparse.ArgumentParser(description='Loads a pickled NetMaxTracker and outputs one or more of {the patches of the image, a deconv patch, a backprop patch} associated with the maxes.')
parser.add_argument('--N', type = int, default = 9, help = 'Note and save top N activations.')
parser.add_argument('--gpu', action = 'store_true', default=settings.caffevis_mode_gpu, help = 'Use gpu.')
parser.add_argument('--do-maxes', action = 'store_true', default=settings.max_tracker_do_maxes, help = 'Output max patches.')
parser.add_argument('--do-deconv', action = 'store_true', default=settings.max_tracker_do_deconv, help = 'Output deconv patches.')
parser.add_argument('--do-deconv-norm', action = 'store_true', default=settings.max_tracker_do_deconv_norm, help = 'Output deconv-norm patches.')
parser.add_argument('--do-backprop', action = 'store_true', default=settings.max_tracker_do_backprop, help = 'Output backprop patches.')
parser.add_argument('--do-backprop-norm', action = 'store_true', default=settings.max_tracker_do_backprop_norm, help = 'Output backprop-norm patches.')
parser.add_argument('--do-info', action = 'store_true', default=settings.max_tracker_do_info, help = 'Output info file containing max filenames and labels.')
parser.add_argument('--idx-begin', type = int, default = None, help = 'Start at this unit (default: all units).')
parser.add_argument('--idx-end', type = int, default = None, help = 'End at this unit (default: all units).')
parser.add_argument('--nmt_pkl', type = str, default = os.path.join(settings.caffevis_outputs_dir, 'find_max_acts_output.pickled'), help = 'Which pickled NetMaxTracker to load.')
parser.add_argument('--net_prototxt', type = str, default = settings.caffevis_deploy_prototxt, help = 'network prototxt to load')
parser.add_argument('--net_weights', type = str, default = settings.caffevis_network_weights, help = 'network weights to load')
parser.add_argument('--datadir', type = str, default = settings.static_files_dir, help = 'directory to look for files in')
parser.add_argument('--filelist', type = str, default = settings.static_files_input_file, help = 'List of image files to consider, one per line. Must be the same filelist used to produce the NetMaxTracker!')
parser.add_argument('--outdir', type = str, default = settings.caffevis_outputs_dir, help = 'Which output directory to use. Files are output into outdir/layer/unit_%%04d/{maxes,deconv,backprop}_%%03d.png')
parser.add_argument('--search-min', action='store_true', default=False, help='Should we also search for minimal activations?')
args = parser.parse_args()
settings.caffevis_deploy_prototxt = args.net_prototxt
settings.caffevis_network_weights = args.net_weights
net, data_mean = load_network(settings)
# validate batch size
if settings.is_siamese and settings._calculated_siamese_network_format == 'siamese_batch_pair':
# currently, no batch support for siamese_batch_pair networks
# it can be added by simply handle the batch indexes properly, but it should be thoroughly tested
assert (settings.max_tracker_batch_size == 1)
# set network batch size
current_input_shape = net.blobs[net.inputs[0]].shape
current_input_shape[0] = settings.max_tracker_batch_size
net.blobs[net.inputs[0]].reshape(*current_input_shape)
net.reshape()
assert args.do_maxes or args.do_deconv or args.do_deconv_norm or args.do_backprop or args.do_backprop_norm or args.do_info, 'Specify at least one do_* option to output.'
siamese_helper = SiameseHelper(settings.layers_list)
nmt = load_max_tracker_from_file(args.nmt_pkl)
for layer_name in settings.layers_to_output_in_offline_scripts:
print 'Started work on layer %s' % (layer_name)
normalized_layer_name = siamese_helper.normalize_layer_name_for_max_tracker(layer_name)
mt = nmt.max_trackers[normalized_layer_name]
if args.idx_begin is None:
idx_begin = 0
if args.idx_end is None:
idx_end = mt.max_vals.shape[0]
with WithTimer('Saved %d images per unit for %s units %d:%d.' % (args.N, normalized_layer_name, idx_begin, idx_end)):
output_max_patches(settings, mt, net, normalized_layer_name, idx_begin, idx_end,
args.N, args.datadir, args.filelist, args.outdir, False,
(args.do_maxes, args.do_deconv, args.do_deconv_norm, args.do_backprop, args.do_backprop_norm, args.do_info))
if args.search_min:
output_max_patches(settings, mt, net, normalized_layer_name, idx_begin, idx_end,
args.N, args.datadir, args.filelist, args.outdir, True,
(args.do_maxes, args.do_deconv, args.do_deconv_norm, args.do_backprop, args.do_backprop_norm, args.do_info))
def output_max_patches(max_tracker, net, layer, idx_begin, idx_end, num_top,
datadir, filelist, outdir, do_which):
do_maxes, do_deconv, do_deconv_norm, do_backprop, do_backprop_norm, do_info = do_which
assert do_maxes or do_deconv or do_deconv_norm or do_backprop or do_backprop_norm or do_info, 'nothing to do'
mt = max_tracker
rc = RegionComputer()
image_filenames, image_xmls, image_labels = load_file_list(filelist)
print('Loaded filenames and labels for %d files' % len(image_filenames))
print(' First file', os.path.join(datadir, image_filenames[0]))
num_top_in_mt = mt.max_locs.shape[1]
assert num_top <= num_top_in_mt, 'Requested %d top images but MaxTracker contains only %d' % (
num_top, num_top_in_mt)
assert idx_end >= idx_begin, 'Range error'
size_ii, size_jj = get_max_data_extent(net, layer, rc, mt.is_conv)
data_size_ii, data_size_jj = net.blobs['data'].data.shape[2:4]
n_total_images = (idx_end - idx_begin) * num_top
for cc, channel_idx in enumerate(range(idx_begin, idx_end)):
unit_dir = os.path.join(outdir, layer, 'unit_%04d' % channel_idx)
mkdir_p(unit_dir)
if do_info:
info_filename = os.path.join(unit_dir, 'info.txt')
info_file = open(info_filename, 'w')
info_file.write(
'# is_conv val image_idx image_class i(if is_conv) j(if is_conv) filename'
)
# iterate through maxes from highest (at end) to lowest
for max_idx_0 in range(num_top):
max_idx = num_top_in_mt - 1 - max_idx_0
if mt.is_conv:
im_idx, im_class, ii, jj = mt.max_locs[channel_idx, max_idx]
else:
im_idx, im_class = mt.max_locs[channel_idx, max_idx]
recorded_val = mt.max_vals[channel_idx, max_idx]
filename = image_filenames[im_idx]
roiname = image_xmls[im_idx]
do_print = (max_idx_0 == 0)
if do_print:
print('%s Output file/image(s) %d/%d' %
(datetime.now().ctime(), cc * num_top, n_total_images))
if mt.is_conv:
# Compute the focus area of the data layer
layer_indices = (ii, ii + 1, jj, jj + 1)
data_indices = rc.convert_region(layer, 'data', layer_indices)
data_ii_start, data_ii_end, data_jj_start, data_jj_end = data_indices
touching_imin = (data_ii_start == 0)
touching_jmin = (data_jj_start == 0)
# Compute how much of the data slice falls outside the actual data [0,max] range
ii_outside = size_ii - (data_ii_end - data_ii_start
) # possibly 0
jj_outside = size_jj - (data_jj_end - data_jj_start
) # possibly 0
if touching_imin:
out_ii_start = ii_outside
out_ii_end = size_ii
else:
out_ii_start = 0
out_ii_end = size_ii - ii_outside
if touching_jmin:
out_jj_start = jj_outside
out_jj_end = size_jj
else:
out_jj_start = 0
out_jj_end = size_jj - jj_outside
else:
ii, jj = 0, 0
data_ii_start, out_ii_start, data_jj_start, out_jj_start = 0, 0, 0, 0
data_ii_end, out_ii_end, data_jj_end, out_jj_end = size_ii, size_ii, size_jj, size_jj
if do_info:
info_file.write(str(1 if mt.is_conv else 0))
info_file.write('%.6f' % mt.max_vals[channel_idx, max_idx])
if mt.is_conv:
info_file.write('%d %d %d %d' %
tuple(mt.max_locs[channel_idx, max_idx]))
else:
info_file.write('%d %d' %
tuple(mt.max_locs[channel_idx, max_idx]))
info_file.write(filename)
if not (do_maxes or do_deconv or do_deconv_norm or do_backprop
or do_backprop_norm):
continue
with WithTimer('Load image & ROIs', quiet=not do_print):
im = load_image(os.path.join(datadir, filename))
_, _, rois = parse_roi_xml(os.path.join(datadir, roiname))
rois = np.array(rois)
with WithTimer('Predict ', quiet=not do_print):
net.blobs['data'].data[...] = im
net.blobs['rois'].reshape(1, *rois.shape)
net.blobs['rois'].data[...] = rois
net.forward()
if len(net.blobs[layer].data.shape) == 4:
reproduced_val = net.blobs[layer].data[0, channel_idx, ii, jj]
else:
reproduced_val = net.blobs[layer].data[0, channel_idx]
if abs(reproduced_val - recorded_val) > .1:
print(
'Warning: recorded value %s is suspiciously different from reproduced value %s. Is the filelist the same?'
% (recorded_val, reproduced_val))
if do_maxes:
#grab image from data layer, not from im (to ensure preprocessing / center crop details match between image and deconv/backprop)
out_arr = np.zeros((3, size_ii, size_jj), dtype='float32')
out_arr[:, out_ii_start:out_ii_end,
out_jj_start:out_jj_end] = net.blobs['data'].data[
0, :, data_ii_start:data_ii_end,
data_jj_start:data_jj_end]
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(out_arr,
os.path.join(unit_dir, 'maxim_%03d.png' %
max_idx_0),
autoscale=False,
autoscale_center=0)
if do_deconv or do_deconv_norm:
diffs = net.blobs[layer].diff * 0
if len(diffs.shape) == 4:
diffs[0, channel_idx, ii, jj] = 1.0
else:
diffs[0, channel_idx] = 1.0
with WithTimer('Deconv ', quiet=not do_print):
net.deconv_from_layer(layer, diffs)
out_arr = np.zeros((3, size_ii, size_jj), dtype='float32')
out_arr[:, out_ii_start:out_ii_end,
out_jj_start:out_jj_end] = net.blobs['data'].diff[
0, :, data_ii_start:data_ii_end,
data_jj_start:data_jj_end]
if out_arr.max() == 0:
print('Warning: Deconv out_arr in range', out_arr.min(),
'to', out_arr.max(),
'ensure force_backward: true in prototxt')
if do_deconv:
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(out_arr,
os.path.join(
unit_dir,
'deconv_%03d.png' % max_idx_0),
autoscale=False,
autoscale_center=0)
if do_deconv_norm:
out_arr = np.linalg.norm(out_arr, axis=0)
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(
out_arr,
os.path.join(unit_dir,
'deconvnorm_%03d.png' % max_idx_0))
if do_backprop or do_backprop_norm:
diffs = net.blobs[layer].diff * 0
diffs[0, channel_idx, ii, jj] = 1.0
with WithTimer('Backward ', quiet=not do_print):
net.backward_from_layer(layer, diffs)
out_arr = np.zeros((3, size_ii, size_jj), dtype='float32')
out_arr[:, out_ii_start:out_ii_end,
out_jj_start:out_jj_end] = net.blobs['data'].diff[
0, :, data_ii_start:data_ii_end,
data_jj_start:data_jj_end]
if out_arr.max() == 0:
print('Warning: Deconv out_arr in range', out_arr.min(),
'to', out_arr.max(),
'ensure force_backward: true in prototxt')
if do_backprop:
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(out_arr,
os.path.join(
unit_dir,
'backprop_%03d.png' % max_idx_0),
autoscale=False,
autoscale_center=0)
if do_backprop_norm:
out_arr = np.linalg.norm(out_arr, axis=0)
with WithTimer('Save img ', quiet=not do_print):
save_caffe_image(
out_arr,
os.path.join(unit_dir,
'backpropnorm_%03d.png' % max_idx_0))
if do_info:
info_file.close()
def main():
parser = argparse.ArgumentParser(
description=
'Loads a pickled NetMaxTracker and outputs one or more of {the patches of the image, a deconv patch, a backprop patch} associated with the maxes.'
)
parser.add_argument('--N',
type=int,
default=9,
help='Note and save top N activations.')
parser.add_argument('--gpu', action='store_true', help='Use gpu.')
parser.add_argument('--do-maxes',
action='store_true',
help='Output max patches.')
parser.add_argument('--do-deconv',
action='store_true',
help='Output deconv patches.')
parser.add_argument('--do-deconv-norm',
action='store_true',
help='Output deconv-norm patches.')
parser.add_argument('--do-backprop',
action='store_true',
help='Output backprop patches.')
parser.add_argument('--do-backprop-norm',
action='store_true',
help='Output backprop-norm patches.')
parser.add_argument(
'--do-info',
action='store_true',
help='Output info file containing max filenames and labels.')
parser.add_argument('--idx-begin',
type=int,
default=None,
help='Start at this unit (default: all units).')
parser.add_argument('--idx-end',
type=int,
default=None,
help='End at this unit (default: all units).')
parser.add_argument('nmt_pkl',
type=str,
help='Which pickled NetMaxTracker to load.')
parser.add_argument('net_prototxt',
type=str,
help='Network prototxt to load')
parser.add_argument('net_weights',
type=str,
help='Network weights to load')
parser.add_argument('datadir',
type=str,
help='Directory to look for files in')
parser.add_argument(
'filelist',
type=str,
help=
'List of image files to consider, one per line. Must be the same filelist used to produce the NetMaxTracker!'
)
parser.add_argument(
'outdir',
type=str,
help=
r'Which output directory to use. Files are output into outdir/layer/unit_%%04d/{maxes,deconv,backprop}_%%03d.png'
)
parser.add_argument('layer', type=str, help='Which layer to output')
#parser.add_argument('--mean', type = str, default = '', help = 'data mean to load')
args = parser.parse_args()
if args.gpu:
caffe.set_mode_gpu()
else:
caffe.set_mode_cpu()
imagenet_mean = load_imagenet_mean()
net = caffe.Classifier(args.net_prototxt,
args.net_weights,
mean=imagenet_mean,
channel_swap=(2, 1, 0),
raw_scale=255,
image_dims=(256, 256))
assert args.do_maxes or args.do_deconv or args.do_deconv_norm or args.do_backprop or args.do_backprop_norm or args.do_info, 'Specify at least one do_* option to output.'
with open(args.nmt_pkl, 'rb') as ff:
nmt = pickle.load(ff)
mt = nmt.max_trackers[args.layer]
if args.idx_begin is None:
args.idx_begin = 0
if args.idx_end is None:
args.idx_end = mt.max_vals.shape[0]
with WithTimer('Saved %d images per unit for %s units %d:%d.' %
(args.N, args.layer, args.idx_begin, args.idx_end)):
output_max_patches(
mt, net, args.layer, args.idx_begin, args.idx_end, args.N,
args.datadir, args.filelist, args.outdir,
(args.do_maxes, args.do_deconv, args.do_deconv_norm,
args.do_backprop, args.do_backprop_norm, args.do_info))
