def quant_probe(directory, blob, quantiles=None, batch_size=None):
'''
Adds a [blob]-sort directory to a probe such as conv5-sort,
where the directory contains [blob]-[unit].mmap files for
each unit such as conv5-0143.mmap, where the contents are
sorted. Also creates a [blob]-quant-[num].mmap file
such as conv5-quantile-1000.mmap, where the higest seen
value for each quantile of each unit is summarized in a
single (units, quantiles) matrix.
'''
# Make sure we have a directory to work in
ed = expdir.ExperimentDirectory(directory)
# If it's already computed, then skip it!!!
print "Checking", ed.mmap_filename(blob=blob, part='quant-%d' % quantiles)
if ed.has_mmap(blob=blob, part='quant-%d' % quantiles):
print "Already have %s-quant-%d.mmap, skipping." % (blob, quantiles)
return
# Read about the blob shape
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
print 'Computing quantiles for %s shape %r' % (blob, shape)
data = ed.open_mmap(blob=blob, shape=shape)
qmap = ed.open_mmap(blob=blob, part='quant-%d' % quantiles,
mode='w+', shape=(data.shape[1], quantiles))
linmap = ed.open_mmap(blob=blob, part='minmax',
mode='w+', shape=(data.shape[1], 2))
# Automatic batch size selection: 64mb batches
if batch_size is None:
batch_size = max(1, int(16 * 1024 * 1024 / numpy.prod(shape[1:])))
# Algorithm: one pass over the data with a quantile counter for each unit
quant = vecquantile.QuantileVector(depth=data.shape[1], seed=1)
start_time = time.time()
last_batch_time = start_time
for i in range(0, data.shape[0], batch_size):
batch_time = time.time()
rate = i / (batch_time - start_time + 1e-15)
batch_rate = batch_size / (batch_time - last_batch_time + 1e-15)
last_batch_time = batch_time
print 'Processing %s index %d: %f %f' % (blob, i, rate, batch_rate)
sys.stdout.flush()
batch = data[i:i+batch_size]
if len(batch.shape) == 4:
batch = numpy.transpose(batch,axes=(0,2,3,1)).reshape(-1, data.shape[1])
elif len(batch.shape) != 2:
assert(False)
quant.add(batch)
print 'Writing quantiles'
sys.stdout.flush()
# Reverse the quantiles, largest first.
qmap[...] = quant.readout(quantiles)[:,::-1]
linmap[...] = quant.minmax()
if qmap is not None:
ed.finish_mmap(qmap)
if linmap is not None:
ed.finish_mmap(linmap)
def create_image_dataset_label_index(directory, batch_size=64, ahead=16):
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
ds = loadseg.SegmentationData(info.dataset)
categories = ds.category_names()
shape = (ds.size(), len(DATASETS), len(ds.label))
index = np.zeros(shape)
pf = loadseg.SegmentationPrefetcher(ds,
categories=categories,
once=True,
batch_size=batch_size,
ahead=ahead,
thread=True)
batch_count = 0
for batch in pf.batches():
if batch_count % 100 == 0:
print('Processing batch %d ...' % batch_count)
for rec in batch:
dataset_index = get_dataset_index(rec['fn'])
image_index = rec['i']
for cat in categories:
if ((type(rec[cat]) is np.ndarray and rec[cat].size > 0)
or type(rec[cat]) is list and len(rec[cat]) > 0):
index[image_index][dataset_index][np.unique(
rec[cat])] = True
batch_count += 1
mmap = ed.open_mmap(part='image_dataset_label',
mode='w+',
dtype=bool,
shape=shape)
mmap[:] = index[:]
ed.finish_mmap(mmap)
print('Finished and saved at %s' %
ed.mmap_filename(part='image_dataset_label'))
def compute_alpha(directory):
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
ds = loadseg.SegmentationData(info.dataset)
L = ds.label_size()
if not has_image_to_label(directory):
create_image_to_label(directory)
image_to_label = load_image_to_label(directory)
label_names = np.array([ds.label[i]['name'] for i in range(L)])
alphas = np.zeros((L, ))
for label_i in range(1, L):
label_categories = ds.label[label_i]['category'].keys()
label_idx = np.where(image_to_label[:, label_i])[0]
train_loader = loadseg.SegmentationPrefetcher(
ds,
categories=label_categories,
split='train',
indexes=label_idx,
once=True,
batch_size=64,
ahead=4,
thread=True)
train_idx = np.array(train_loader.indexes)
#sw = 0
#sh = 0
perc_label = []
#train_label_categories = []
for batch in train_loader.batches():
for rec in batch:
sw, sh = [rec[k] for k in ['sw', 'sh']]
#sw_r, sh_r = [rec[k] for k in ['sw', 'sh']]
#if sw == 0 and sh == 0:
# sw = sw_r
# sh = sh_r
#else:
# assert(sw == sw_r and sh == sh_r)
for cat in label_categories:
if rec[cat] != []:
#train_label_categories.append(cat)
if type(rec[cat]) is np.ndarray:
perc_label.append(
np.sum(rec[cat] == label_i) / float(sw * sh))
else:
perc_label.append(1.)
break
assert (len(perc_label) == len(train_idx))
alphas[label_i] = float(1. - np.mean(perc_label))
print label_i, label_names[label_i], alphas[label_i]
train_loader.close()
alphas_mmap = ed.open_mmap(part='train_alphas',
mode='w+',
dtype='float32',
shape=alphas.shape)
alphas_mmap[:] = alphas[:]
ed.finish_mmap(alphas_mmap)
def load_image_dataset_label_index(directory):
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
ds = loadseg.SegmentationData(info.dataset)
shape = (ds.size(), len(DATASETS), len(ds.label))
return ed.open_mmap(part='image_dataset_label',
mode='r',
dtype=bool,
shape=shape)
def __init__(self, directory, blobs=None):
self.ed = expdir.ExperimentDirectory(directory)
# Load probe metadata
info = self.ed.load_info()
self.ih, self.iw = info.input_dim
self.layers = info.blobs if blobs is None else blobs
self.ds = loadseg.SegmentationData(info.dataset)
self.layer = {}
for blob in self.layers:
self.layer[blob] = LayerProbe(self.ed, blob, self.ds)
def create_image_to_label(directory, batch_size=16, ahead=4):
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
print info.dataset
if 'broden' in info.dataset:
ds = loadseg.SegmentationData(info.dataset)
categories = ds.category_names()
shape = (ds.size(), len(ds.label))
pf = loadseg.SegmentationPrefetcher(ds,
categories=categories,
once=True,
batch_size=batch_size,
ahead=ahead,
thread=False)
image_to_label = np.zeros(shape, dtype='int32')
batch_count = 0
for batch in pf.batches():
if batch_count % 100 == 0:
print('Processing batch %d ...' % batch_count)
for rec in batch:
image_index = rec['i']
for cat in categories:
if ((type(rec[cat]) is np.ndarray and rec[cat].size > 0)
or type(rec[cat]) is list and len(rec[cat]) > 0):
image_to_label[image_index][np.unique(rec[cat])] = True
batch_count += 1
elif 'imagenet' in info.dataset or 'ILSVRC' in info.dataset:
classes, class_to_idx = find_classes(info.dataset)
imgs = make_dataset(info.dataset, class_to_idx)
_, labels = zip(*imgs)
labels = np.array(labels)
L = 1000
shape = (len(labels), L)
image_to_label = np.zeros(shape)
for i in range(L):
image_to_label[labels == i, i] = 1
else:
assert (False)
mmap = ed.open_mmap(part='image_to_label',
mode='w+',
dtype=bool,
shape=shape)
mmap[:] = image_to_label[:]
ed.finish_mmap(mmap)
f = ed.mmap_filename(part='image_to_label')
print('Finished and saved index_to_label at %s' % f)
def extract_concept_data(directory, batch_size=64, ahead=16, verbose=True):
ed = expdir.ExperimentDirectory(directory)
if ed.has_mmap(part='concept_data'):
print('%s already has %s, so skipping' %
(directory, ed.mmap_filename(part='concept_data')))
return
info = ed.load_info()
(sh, sw) = get_seg_size(info.input_dim)
ds = loadseg.SegmentationData(info.dataset)
categories = np.array(ds.category_names())
L = ds.label_size()
N = ds.size()
pf = loadseg.SegmentationPrefetcher(ds,
categories=categories,
once=True,
batch_size=batch_size,
ahead=ahead,
thread=True)
if verbose:
print 'Creating new mmap at %s' % ed.mmap_filename(part='concept_data')
data = ed.open_mmap(part='concept_data', mode='w+', shape=(N, L, sh, sw))
start_time = time.time()
last_batch_time = start_time
index = 0
for batch in pf.batches():
batch_time = time.time()
rate = index / (batch_time - start_time + 1e-15)
batch_rate = batch_size / (batch_time - last_batch_time + 1e-15)
last_batch_time = batch_time
if verbose:
print 'extract_concept_data index %d/%d (%.2f)\titems per sec %.2f\t%.2f' % (
index, N, index / float(N), batch_rate, rate)
for rec in batch:
for cat in categories:
if len(rec[cat]) == 0:
continue
if cat == 'texture' or cat == 'scene':
for i in range(len(rec[cat])):
data[index][rec[cat][i] - 1, :, :] = 1
else:
for i in range(len(rec[cat])):
ys, xs = np.where(rec[cat][i])
for j in range(len(xs)):
data[index][rec[cat][i][ys[j]][xs[j]] -
1][ys[j]][xs[j]] = 1
index += 1
assert index == N, ("Data source should return every item once %d %d." %
(index, N))
if verbose:
print 'Renaming mmap.'
ed.finish_mmap(data)
def load_image_to_label(directory, blob=None):
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
if 'broden' in info.dataset:
ds = loadseg.SegmentationData(info.dataset)
shape = (ds.size(), len(ds.label))
elif 'imagenet' in info.dataset or 'ILSVRC' in info.dataset:
assert (blob is not None)
blob_info = ed.load_info(blob=blob)
shape = (blob_info.shape[0], 1000)
else:
assert (False)
return ed.open_mmap(part='image_to_label',
mode='r',
dtype=bool,
shape=shape)
def max_probe(directory, blob, batch_size=None):
# Make sure we have a directory to work in
ed = expdir.ExperimentDirectory(directory)
# If it's already computed, then skip it!!!
print "Checking", ed.mmap_filename(blob=blob, part='imgmax')
if ed.has_mmap(blob=blob, part='imgmax'):
print "Already have %s-imgmax.mmap, skipping." % (blob)
return
# Read about the blob shape
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
print 'Computing imgmax for %s shape %r' % (blob, shape)
data = ed.open_mmap(blob=blob, shape=shape)
imgmax = ed.open_mmap(blob=blob,
part='imgmax',
mode='w+',
shape=data.shape[:2])
# Automatic batch size selection: 64mb batches
if batch_size is None:
batch_size = max(1, int(128 * 1024 * 1024 / numpy.prod(shape[1:])))
# Algorithm: one pass over the data
start_time = time.time()
last_batch_time = start_time
for i in range(0, data.shape[0], batch_size):
batch_time = time.time()
rate = i / (batch_time - start_time + 1e-15)
batch_rate = batch_size / (batch_time - last_batch_time + 1e-15)
last_batch_time = batch_time
print 'Imgmax %s index %d: %f %f' % (blob, i, rate, batch_rate)
sys.stdout.flush()
batch = data[i:i + batch_size]
imgmax[i:i + batch_size] = batch.max(axis=(2, 3))
print 'Writing imgmax'
sys.stdout.flush()
# Save as mat file
filename = ed.filename('imgmax.mat', blob=blob)
savemat(filename, {'imgmax': imgmax})
# And as mmap
ed.finish_mmap(imgmax)
def upsample_blob_data(directory, blob, batch_size=64, verbose=True):
ed = expdir.ExperimentDirectory(directory)
if ed.has_mmap(blob=blob, part='upsampled'):
print('%s already has %s, so skipping.' %
(directory, ed.mmap_filename(blob=blob, part='upsampled')))
info = ed.load_info()
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
N = shape[0]
seg_size = get_seg_size(info.input_dim)
blobdata = ed.open_mmap(blob=blob, mode='r', shape=shape)
if verbose:
print 'Creating new mmap at %s' % ed.mmap_filename(blob=blob,
part='upsampled')
upsampled_data = ed.open_mmap(blob=blob,
part='upsampled',
mode='w+',
shape=(shape[0], shape[1], seg_size[0],
seg_size[1]))
up = nn.Upsample(size=seg_size, mode='bilinear')
start_time = time.time()
last_batch_time = start_time
for i in range(int(np.ceil(np.true_divide(N, batch_size)))):
if (i + 1) * batch_size < N:
idx = range(i * batch_size, (i + 1) * batch_size)
else:
idx = range(i * batch_size, N)
batch_time = time.time()
rate = idx[-1] / (batch_time - start_time + 1e-15)
batch_rate = batch_size / (batch_time - last_batch_time + 1e-15)
last_batch_time = batch_time
if verbose:
print 'upsample_blob_data %d/%d (%.2f)\titems per sec %.2f\t%.2f' % (
idx[-1], N, idx[-1] / float(N), batch_rate, rate)
inp = Variable(torch.Tensor(blobdata[idx]))
out = up(inp).data.cpu().numpy()
upsampled_data[idx] = np.copy(out)
if verbose:
print 'Renaming mmap.'
ed.finish_mmap(upsampled_data)
def loadviz(directory, blob):
ed = expdir.ExperimentDirectory(directory)
html_fn = ed.filename(['html', '%s.html' % expdir.fn_safe(blob)])
with open(html_fn) as f:
lines = f.readlines()
result = OrderedDict()
for line in lines:
if 'unit ' in line:
u, v = line.split(':', 1)
unit = int(re.search(r'unit (\d+)', u).group(1)) - 1
u_result = []
for w in v.split(';'):
m = re.search(r'(\w.*\w)? \((\w+), ([.\d]+)\)', w)
if not m:
print 'On line', v
print 'Error with', w
label = m.group(1)
category = m.group(2)
score = float(m.group(3))
u_result.append((label, category, score))
result[unit] = u_result
return result
default=0,
help='set to 1 to force image regeneration')
parser.add_argument('--imscale',
type=int,
default=72,
help='thumbnail dimensions')
parser.add_argument('--imcount',
type=int,
default=50,
help='number of thumbnails to include')
parser.add_argument('--threshold',
type=float,
default=0.04,
help='minimum IoU to count as a detector')
args = parser.parse_args()
ed = expdir.ExperimentDirectory(args.directory)
ds = open_dataset(ed)
for blob in args.blobs:
generate_html_summary(ed,
ds,
blob,
imsize=args.imsize,
imscale=args.imscale,
imcount=args.imcount,
gridwidth=args.gridwidth,
force=args.force,
threshold=args.threshold,
verbose=True)
except:
traceback.print_exc(file=sys.stdout)
sys.exit(1)
def save_neural_statistics(directory, blob, split=None):
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
dataset = info.dataset
blob_info = ed.load_info(blob=blob)
if 'broden' in dataset:
assert (split is not None)
suffix = '_%s' % split
else:
suffix = ''
if ed.has_mmap(blob=blob, part='act_probs%s' % suffix):
print('%s already exists' %
ed.mmap_filename(blob=blob, part='act_probs%s' % suffix))
return
acts = ed.open_mmap(blob=blob,
mode='r',
dtype='float32',
shape=blob_info.shape)
if 'broden' in dataset:
ds = loadseg.SegmentationData(dataset)
split_ind = np.array([
True if ds.split(i) == split else False for i in range(ds.size())
])
L = ds.label_size()
elif 'imagenet' in dataset or 'ILSVRC' in dataset:
assert (split is None)
L = 1000
split_ind = True
image_to_label = load_image_to_label(directory, blob=blob)
K = blob_info.shape[1]
probs = np.zeros((L, K))
mus = np.zeros((L, K))
sigmas = np.zeros((L, K))
for class_i in range(L):
class_idx = np.where(split_ind & image_to_label[:, class_i])[0]
if len(class_idx) == 0:
probs[class_i, :] = np.nan
mus[class_i, :] = np.nan
sigmas[class_i, :] = np.nan
print class_i, 'no examples'
continue
print class_i
max_acts = np.amax(acts[class_idx], axis=(2, 3))
for filter_i in range(blob_info.shape[1]):
nz_idx = np.where(max_acts[:, filter_i] > 0)[0]
probs[class_i][filter_i] = len(nz_idx) / float(
len(max_acts[:, filter_i]))
try:
mus[class_i][filter_i] = np.mean(max_acts[nz_idx, filter_i])
sigmas[class_i][filter_i] = np.std(max_acts[nz_idx, filter_i])
except:
mus[class_i][filter_i] = np.nan
sigmas[class_i][filter_i] = np.nan
print class_i, filter_i, 'no nonzero idx'
probs_mmap = ed.open_mmap(blob=blob,
part='act_probs%s' % suffix,
mode='w+',
dtype='float32',
shape=probs.shape)
mus_mmap = ed.open_mmap(blob=blob,
part='act_mus%s' % suffix,
mode='w+',
dtype='float32',
shape=mus.shape)
sigmas_mmap = ed.open_mmap(blob=blob,
part='act_sigmas%s' % suffix,
mode='w+',
dtype='float32',
shape=sigmas.shape)
probs_mmap[:] = probs[:]
mus_mmap[:] = mus[:]
sigmas_mmap[:] = sigmas[:]
ed.finish_mmap(probs_mmap)
ed.finish_mmap(mus_mmap)
ed.finish_mmap(sigmas_mmap)
print('Finished saving neural statistics for %s' % blob)
def create_probe(directory,
dataset,
definition,
weights,
mean,
blobs,
colordepth=3,
rotation_seed=None,
rotation_power=1,
limit=None,
split=None,
batch_size=16,
ahead=4,
cl_args=None,
verbose=True):
# If we're already done, skip it!
ed = expdir.ExperimentDirectory(directory)
if all(ed.has_mmap(blob=b) for b in blobs):
return
'''
directory: where to place the probe_conv5.mmap files.
data: the AbstractSegmentation data source to draw upon
definition: the filename for the caffe prototxt
weights: the filename for the caffe model weights
mean: to use to normalize rgb values for the network
blobs: ['conv3', 'conv4', 'conv5'] to probe
'''
if verbose:
print 'Opening dataset', dataset
data = loadseg.SegmentationData(args.dataset)
# the network to dissect
if args.weights == None:
# load the imagenet pretrained model
net = torchvision.models.__dict__[args.definition](pretrained=True)
else:
# load your own model
net = torchvision.models.__dict__[args.definition](
num_classes=args.num_classes)
checkpoint = torch.load(args.weights)
# the data parallel layer will add 'module' before each layer name
state_dict = {
str.replace(k, 'module.', ''): v
for k, v in checkpoint['state_dict'].iteritems()
}
net.load_state_dict(state_dict)
net.eval()
# hook up to get the information for each selected layer
layers = net._modules.keys()
size_blobs_output = []
def hook_size(module, input, output):
size_blobs_output.append(output.data.size())
input_sample = V(torch.randn(1, 3, args.input_size, args.input_size))
for blob in blobs:
net._modules.get(blob).register_forward_hook(hook_size)
output_sample = net(input_sample)
input_dim = [args.input_size, args.input_size]
data_size = data.size(split) # the image size
if limit is not None:
data_size = min(data_size, limit)
# Make sure we have a directory to work in
ed.ensure_dir()
# Step 0: write a README file with generated information.
ed.save_info(
dict(dataset=dataset,
split=split,
definition=definition,
weights=weights,
mean=mean,
blobs=blobs,
input_dim=input_dim,
rotation_seed=rotation_seed,
rotation_power=rotation_power))
# Clear old probe data
ed.remove_all('*.mmap*')
# Create new (empty) mmaps
if verbose:
print 'Creating new mmaps.'
out = {}
rot = None
if rotation_seed is not None:
rot = {}
for idx, blob in enumerate(blobs):
#shape = (data_size, ) + net.blobs[blob].data.shape[1:]
shape = (data_size, int(size_blobs_output[idx][1]),
int(size_blobs_output[idx][2]),
int(size_blobs_output[idx][3]))
out[blob] = ed.open_mmap(blob=blob, mode='w+', shape=shape)
# Rather than use the exact RF, here we use some heuristics to compute the approximate RF
size_RF = (args.input_size / size_blobs_output[idx][2],
args.input_size / size_blobs_output[idx][3])
fieldmap = ((0, 0), size_RF, size_RF)
ed.save_info(blob=blob,
data=dict(name=blob, shape=shape, fieldmap=fieldmap))
# The main loop
if verbose:
print 'Beginning work.'
pf = loadseg.SegmentationPrefetcher(data,
categories=['image'],
split=split,
once=True,
batch_size=batch_size,
ahead=ahead)
index = 0
start_time = time.time()
last_batch_time = start_time
batch_size = 0
net.cuda()
# hook the feature extractor
features_blobs = []
def hook_feature(module, input, output):
features_blobs.append(output.data.cpu().numpy())
for blob in blobs:
net._modules.get(blob).register_forward_hook(hook_feature)
for batch in pf.tensor_batches(bgr_mean=mean):
del features_blobs[:] # clear up the feature basket
batch_time = time.time()
rate = index / (batch_time - start_time + 1e-15)
batch_rate = batch_size / (batch_time - last_batch_time + 1e-15)
last_batch_time = batch_time
if verbose:
print 'netprobe index', index, 'items per sec', batch_rate, rate
sys.stdout.flush()
inp = batch[0]
batch_size = len(inp)
if limit is not None and index + batch_size > limit:
# Truncate last if limited
batch_size = limit - index
inp = inp[:batch_size]
if colordepth == 1:
inp = numpy.mean(inp, axis=1, keepdims=True)
# previous feedforward case
inp = inp[:, ::-1, :, :]
inp_tensor = V(torch.from_numpy(inp.copy()))
# approximately normalize the input to make the images scaled at around 1.
inp_tensor.div_(255.0 * 0.224)
inp_tensor = inp_tensor.cuda()
result = net.forward(inp_tensor)
# output the hooked feature
for i, key in enumerate(blobs):
out[key][index:index + batch_size] = numpy.copy(
features_blobs[i][:batch_size])
# print 'Recording data in mmap done'
index += batch_size
if index >= data_size:
break
assert index == data_size, (
"Data source should return evey item once %d %d." % (index, data_size))
if verbose:
print 'Renaming mmaps.'
for blob in blobs:
ed.finish_mmap(out[blob])
# Final step: write the README file
write_readme_file([('cl_args', cl_args), ('data', data),
('definition', definition), ('weight', weights),
('mean', mean), ('blobs', blobs)],
ed,
verbose=verbose)
blob_names = {
'features.1': 'relu1',
'features.4': 'relu2',
'features.7': 'relu3',
'features.9': 'relu4',
'features.11': 'relu5',
'features': 'pool5'
}
directory='/home/ruthfong/NetDissect/probes/pytorch_alexnet_imagenet'
if not os.path.exists(directory):
directory='/scratch/shared/slow/ruthfong/pytorch_alexnet_imagenet'
assert(os.path.exists(directory))
blob='features.11'
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
ds = loadseg.SegmentationData(info.dataset)
categories = ds.category_names()
K = shape[1]
L = ds.label_size()
N = ds.size()
label_names = np.array([ds.label[i]['name'] for i in range(L)])
blobs = ['features.1', 'features.4', 'features.7', 'features.9', 'features.11']
def max_probe(directory,
blob,
batch_size=None,
quantile=0.005,
results=None,
num_components=None,
use_y_weights=False,
suffix='',
new_suffix='',
normalize=True,
pool='max_pool',
should_thresh=False,
disc=False):
# Make sure we have a directory to work in
ed = expdir.ExperimentDirectory(directory)
# If it's already computed, then skip it!!!
if disc:
suffix = '_disc%s' % suffix
print "Checking", ed.mmap_filename(blob=blob,
part='linear_imgmax%s%s' %
(suffix, new_suffix))
if ed.has_mmap(blob=blob, part='linear_imgmax%s%s' % (suffix, new_suffix)):
print "Already have %s-imgmax.mmap, skipping." % (blob)
return
info = ed.load_info()
ds = loadseg.SegmentationData(info.dataset)
# Read about the blob shape
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
N = shape[0]
K = shape[1]
L = ds.label_size()
if should_thresh:
if quantile == 1:
thresh = np.zeros((1, 1, K, 1, 1))
else:
quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(K, -1))
threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))]
thresh = threshold[numpy.newaxis, numpy.newaxis, :, numpy.newaxis,
numpy.newaxis]
if use_y_weights:
shape = (
N, L, 113, 113
) # TODO: don't hardcode segmentation size, this only works for alexnet
data = ed.open_mmap(part='concept_data', mode='r', shape=shape)
else:
data = ed.open_mmap(blob=blob, shape=shape)
print 'Computing imgmax for %s shape %r' % (blob, shape)
if results is not None:
imgmax = ed.open_mmap(blob=blob,
part='linear_imgmax%s%s' % (suffix, new_suffix),
mode='w+',
shape=(N, num_components))
if use_y_weights:
all_weights = pkl.load(open(results, 'rb'))['model'].y_weights_.T
else:
all_weights = pkl.load(open(results, 'rb'))['model'].x_weights_.T
else:
imgmax = ed.open_mmap(blob=blob,
part='linear_imgmax%s%s' % (suffix, new_suffix),
mode='w+',
shape=(N, L))
if disc:
assert (ed.has_mmap(blob=blob, part='linear_weights%s' % suffix))
all_weights = ed.open_mmap(blob=blob,
part='linear_weights%s' % suffix,
mode='r',
dtype='float32',
shape=(L, 2, K))
all_weights = all_weights[:, -1, :]
else:
assert (ed.has_mmap(blob=blob, part='linear_weights%s' % suffix))
all_weights = ed.open_mmap(blob=blob,
part='linear_weights%s' % suffix,
mode='r',
dtype='float32',
shape=(L, K))
if normalize:
all_weights = numpy.array([
numpy.true_divide(all_weights[i],
numpy.linalg.norm(all_weights[i]))
for i in range(L)
])
# Automatic batch size selection: 64mb batches
if batch_size is None:
batch_size = max(1, int(128 * 1024 * 1024 / numpy.prod(shape[1:])))
# Algorithm: one pass over the data
start_time = time.time()
last_batch_time = start_time
for i in range(0, data.shape[0], batch_size):
batch_time = time.time()
rate = i / (batch_time - start_time + 1e-15)
batch_rate = batch_size / (batch_time - last_batch_time + 1e-15)
last_batch_time = batch_time
print 'Imgmax %s index %d: %f %f' % (blob, i, rate, batch_rate)
sys.stdout.flush()
batch = data[i:i +
batch_size][:, numpy.
newaxis, :, :, :] # (batch_size, L, K, S, S)
if should_thresh:
batch = (batch > thresh).astype(float)
if pool == 'max_pool':
imgmax[i:i + batch_size, :] = (
batch * all_weights[numpy.newaxis, :, :, numpy.newaxis,
numpy.newaxis]).sum(axis=2).max(axis=(2,
3))
elif pool == 'avg_pool':
imgmax[i:i + batch_size, :] = (
batch * all_weights[numpy.newaxis, :, :, numpy.newaxis,
numpy.newaxis]).sum(axis=2).mean(axis=(2,
3))
print 'Writing imgmax'
sys.stdout.flush()
# Save as mat file
filename = ed.filename('linear_imgmax%s.mat' % suffix, blob=blob)
savemat(filename, {'linear_imgmax': imgmax})
# And as mmap
ed.finish_mmap(imgmax)
def compute_correlation(directory, blob, num_samples=None, num_components=1, out_file=None,
verbose=False):
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
ds = loadseg.SegmentationData(info.dataset)
L = ds.label_size()
N = ds.size()
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
K = shape[1]
categories = np.array(ds.category_names())
label_names = np.array([ds.label[i]['name'] for i in range(L)])
(Hs, Ws) = get_seg_size(info.input_dim)
if verbose:
start = time.time()
print 'Loading data...'
upsampled_data = ed.open_mmap(blob=blob, part='upsampled', mode='r',
shape=(N,K,Hs,Ws))
concept_data = ed.open_mmap(part='concept_data', mode='r',
shape=(N,L,Hs,Ws))
if verbose:
print 'Finished loading data in %d secs.' % (time.time() - start)
if verbose:
start = time.time()
print 'Selecting data...'
if num_samples is not None:
rand_idx = np.random.choice(N, num_samples, replace=False)
X = upsampled_data[rand_idx,:,Hs/2,Ws/2]
Y = concept_data[rand_idx,:,Hs/2,Ws/2]
else:
X = upsampled_data[:,:,Hs/2,Ws/2]
Y = concept_data[:,:,Hs/2,Ws/2]
if verbose:
print 'Finished selecting data in %d secs.' % (time.time() - start)
cca = CCA(n_components=num_components)
if verbose:
start = time.time()
if num_samples is None:
num_samples = N
print 'Fitting %d-component CCA with N = %d samples...' % (num_components, num_samples)
cca.fit(X,Y)
if verbose:
print 'Fitted %d-component CCA with N = %d samples in %d secs.' % (num_components,
num_samples, time.time() - start)
X_c, Y_c = cca.transform(X,Y)
score = cca.score(X,Y)
results = {}
if out_file is not None:
if verbose:
start = time.time()
print 'Saving results...'
results['model'] = cca
try:
results['idx'] = rand_idx
except:
results['idx'] = None
results['directory'] = directory
results['blob'] = blob
results['num_samples'] = num_samples
results['num_components'] = num_components
results['score'] = score
pkl.dump(results, open(out_file, 'wb'))
if verbose:
print 'Saved results at %s in %d secs.' % (out_file, time.time() - start)
return results
def linear_probe_discriminative(directory, blob, label_i, suffix='', batch_size=16,
quantile=0.005, bias=False, pool='avg_pool', positive=False,
num_filters=None, init_suffix='',
num_epochs=30, epoch_iter=10, lr=1e-4, momentum=0.9,
l1_weight_decay=0, l2_weight_decay=0, nesterov=False,
lower_bound=None, optimizer_type='sgd', cuda=False,
fig_path=None, show_fig=True):
ed = expdir.ExperimentDirectory(directory)
try:
info = ed.load_info()
except:
ed = expdir.ExperimentDirectory(os.path.join(directory, 'train'))
ed_val = expdir.ExperimentDirectory(os.path.join(directory, 'val'))
info = ed.load_info()
assert('imagenet' in info.dataset or 'ILSVRC' in info.dataset)
# Check if linear weights have already been learned
if ed.has_mmap(blob=blob, part='label_i_%d_weights_disc%s' % (label_i, suffix)):
print('%s already has %s, so skipping.' % (directory,
ed.mmap_filename(blob=blob, part='label_i_%d_weights_disc%s' % (
label_i, suffix))))
return
# Load probe metadata
info = ed.load_info()
ih, iw = info.input_dim
# Load blob metadata
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
unit_size = shape[1]
fieldmap = blob_info.fieldmap
# Load the blob quantile data and grab thresholds
if quantile == 1:
if len(shape) == 4:
thresh = np.zeros((unit_size, 1, 1))
elif len(shape) == 2:
thresh = np.zeros(unit_size)
else:
assert(False)
else:
quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(unit_size, -1))
threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))]
if len(shape) == 4:
thresh = threshold[:, np.newaxis, np.newaxis]
elif len(shape) == 2:
thresh = threshold
else:
assert(False)
# print np.max(thresh), thresh.shape, type(thresh)
# Map the blob activation data for reading
fn_read = ed.mmap_filename(blob=blob)
blobdata = cached_memmap(fn_read, mode='r', dtype='float32', shape=shape)
if 'broden' in info.dataset:
# Load the dataset
ds = loadseg.SegmentationData(info.dataset)
# Get all the categories the label is a part of
label_categories = ds.label[label_i]['category'].keys()
num_categories = len(label_categories)
# Get label name
label_name = ds.name(category=None, j=label_i)
# Get indices of images containing the given label
if not has_image_to_label(directory):
print('image_to_label does not exist in %s; creating it now...' % directory)
create_image_to_label(directory, batch_size=batch_size)
image_to_label = load_image_to_label(directory)
label_idx = np.where(image_to_label[:, label_i])[0]
non_label_idx = np.where(image_to_label[:, label_i] == 0)[0]
print('Number of positive and negative examples of label %d (%s): %d %d' % (
label_i, label_name, len(label_idx), len(non_label_idx)))
val_blobdata = blobdata
elif 'imagenet' in info.dataset or 'ILSVRC' in info.dataset:
label_desc = np.loadtxt('synset_words.txt', str, delimiter='\t')
label_name = ' '.join(label_desc[label_i].split(',')[0].split()[1:])
image_to_label_train = load_image_to_label(os.path.join(directory, 'train'), blob=blob)
train_label_idx = np.where(image_to_label_train[:, label_i])[0]
train_non_label_idx = np.where(1-image_to_label_train[:, label_i])[0]
image_to_label_val = load_image_to_label(os.path.join(directory, 'val'), blob=blob)
val_label_idx = np.where(image_to_label_val[:, label_i])[0]
val_non_label_idx = np.where(1-image_to_label_val[:, label_i])[0]
print('Number of positive and negative examples of label %d (%s): %d %d %d %d' %
(label_i, label_name, len(train_label_idx), len(train_non_label_idx),
len(val_label_idx), len(val_non_label_idx)))
val_blob_info = ed_val.load_info(blob=blob)
val_shape = val_blob_info.shape
val_fn_read = ed_val.mmap_filename(blob=blob)
val_blobdata = cached_memmap(val_fn_read, mode='r', dtype='float32', shape=val_shape)
else:
assert(False)
criterion = torch.nn.BCEWithLogitsLoss()
if num_filters is not None:
init_weights_mmap = ed.open_mmap(blob=blob, part='label_i_%d_weights_disc%s' % (label_i, init_suffix),
mode='r', dtype='float32', shape=unit_size)
sorted_idx = np.argsort(np.abs(init_weights_mmap))[::-1]
mask_idx = np.zeros(unit_size, dtype=int)
mask_idx[sorted_idx[:num_filters]] = 1
layer = CustomDiscLayer(unit_size, act=False, pool=pool, bias=bias, positive=positive,
mask_idx=torch.ByteTensor(mask_idx), cuda=cuda)
else:
layer = CustomDiscLayer(unit_size, act=False, pool=pool, bias=bias, positive=positive, cuda=cuda)
if cuda:
criterion.cuda()
layer.cuda()
if optimizer_type == 'sgd':
optimizer = Custom_SGD(layer.parameters(), lr, momentum,
l1_weight_decay=l1_weight_decay, l2_weight_decay=l2_weight_decay,
nesterov=nesterov, lower_bound=lower_bound)
elif optimizer_type == 'adam':
optimizer = Custom_Adam(layer.parameters(), lr, l1_weight_decay=l1_weight_decay,
l2_weight_decay=l2_weight_decay, lower_bound=lower_bound)
if 'broden' in info.dataset:
train_label_idx = []
val_label_idx = []
for ind in label_idx:
if ds.split(ind) == 'train':
train_label_idx.append(ind)
elif ds.split(ind) == 'val':
val_label_idx.append(ind)
train_non_label_idx = []
val_non_label_idx = []
for ind in non_label_idx:
if ds.split(ind) == 'train':
train_non_label_idx.append(ind)
elif ds.split(ind) == 'val':
val_non_label_idx.append(ind)
train_label_idx = np.array(train_label_idx)
val_label_idx = np.array(val_label_idx)
train_non_label_idx = np.array(train_non_label_idx)
val_non_label_idx = np.array(val_non_label_idx)
num_train_labels = len(train_label_idx)
num_train_non_labels = len(train_non_label_idx)
train_neg_greater = num_train_labels < num_train_non_labels
if len(val_label_idx) < len(val_non_label_idx):
pos_val_idx = np.arange(len(val_label_idx))
neg_val_idx = np.random.choice(len(val_non_label_idx), len(val_label_idx), replace=False)
else:
pos_val_idx = np.random.choice(len(val_label_idx), len(val_non_label_idx), replace=False)
neg_val_idx = np.arange(len(val_non_label_idx))
val_indexes = np.concatenate((val_label_idx[pos_val_idx], val_non_label_idx[neg_val_idx]))
val_targets = np.concatenate((np.ones(len(pos_val_idx)), np.zeros(len(neg_val_idx))))
print 'Number of train and val examples: %d %d' % (min(2*num_train_non_labels, 2*num_train_labels),
len(val_targets))
train_losses = []
train_accs = []
val_losses = []
val_accs = []
if 'imagenet' in info.dataset or 'ILSVRC' in info.dataset:
if train_neg_greater:
pos_train_idx = np.arange(num_train_labels)
neg_train_idx = np.random.choice(num_train_non_labels, num_train_labels, replace=False)
else:
pos_train_idx = np.random.choice(num_train_labels, num_train_non_labels, replace=False)
neg_train_idx = np.arange(num_train_non_labels)
train_indexes = np.concatenate((train_label_idx[pos_train_idx], train_non_label_idx[neg_train_idx]))
train_targets = np.concatenate((np.ones(len(pos_train_idx)), np.zeros(len(neg_train_idx))))
rand_idx = np.random.permutation(len(train_targets))
for t in range(num_epochs):
if epoch_iter is not None:
adjust_learning_rate(lr, optimizer, t, epoch_iter=epoch_iter)
if 'broden' in info.dataset:
if train_neg_greater:
pos_train_idx = np.arange(num_train_labels)
neg_train_idx = np.random.choice(num_train_non_labels, num_train_labels, replace=False)
else:
pos_train_idx = np.random.choice(num_train_labels, num_train_non_labels, replace=False)
neg_train_idx = np.arange(num_train_non_labels)
train_indexes = np.concatenate((train_label_idx[pos_train_idx], train_non_label_idx[neg_train_idx]))
train_targets = np.concatenate((np.ones(len(pos_train_idx)), np.zeros(len(neg_train_idx))))
rand_idx = np.random.permutation(len(train_targets))
(train_loss, train_acc) = run_epoch(blobdata, train_targets[rand_idx], train_indexes[rand_idx], thresh, layer, criterion,
optimizer, t, batch_size=batch_size, train=True, cuda=cuda)
(val_loss, val_acc) = run_epoch(val_blobdata, val_targets, val_indexes, thresh, layer, criterion,
optimizer, t, batch_size=batch_size, train=False, cuda=cuda)
train_losses.append(train_loss)
train_accs.append(train_acc)
val_losses.append(val_loss)
val_accs.append(val_acc)
if fig_path is not None or show_fig:
f, ax = plt.subplots(1,2, figsize=(8,4))
ax[0].plot(range(num_epochs), train_losses, label='train')
ax[0].plot(range(num_epochs), val_losses, label='val')
ax[0].set_title('BCE Loss (train=%d, val=%d)' % (len(train_targets), len(val_targets)))
ax[1].plot(range(num_epochs), train_accs, label='train')
ax[1].plot(range(num_epochs), val_accs, label='val')
ax[1].set_title('Accuracy (%s, %s)' % (label_name, blob))
plt.legend()
if fig_path is not None:
plt.savefig(fig_path)
if show_fig:
plt.show()
plt.close()
# Save weights
weights = (layer.mask * layer.weight).data.cpu().numpy()
print np.sum(weights != 0)
weights_mmap = ed.open_mmap(blob=blob, part='label_i_%d_weights_disc%s' % (label_i, suffix),
mode='w+', dtype='float32', shape=weights.shape)
weights_mmap[:] = weights[:]
ed.finish_mmap(weights_mmap)
if bias:
bias_v = layer.bias.data.cpu().numpy()
bias_mmap = ed.open_mmap(blob=blob, part='label_i_%d_bias_disc%s' % (label_i, suffix),
mode='w+', dtype='float32', shape=(1,))
bias_mmap[:] = bias_v[:]
ed.finish_mmap(bias_mmap)
results_mmap = ed.open_mmap(blob=blob, part='label_i_%d_results_disc%s' % (label_i, suffix),
mode='w+', dtype='float32', shape=(4,num_epochs))
results_mmap[0] = train_losses
results_mmap[1] = val_losses
results_mmap[2] = train_accs
results_mmap[3] = val_accs
ed.finish_mmap(results_mmap)
def has_image_to_label(directory):
ed = expdir.ExperimentDirectory(directory)
return ed.has_mmap(part='image_to_label')
def probe_linear(directory,
blob,
suffix='',
start=None,
end=None,
batch_size=16,
ahead=4,
quantile=0.005,
bias=False,
positive=False,
cuda=False,
force=False):
qcode = ('%f' % quantile).replace('0.', '.').rstrip('0')
ed = expdir.ExperimentDirectory(directory)
if (ed.has_mmap(blob=blob, part='linear_ind_ious%s' % suffix)
and ed.has_mmap(blob=blob, part='linear_set_ious%s' % suffix)):
print('Linear weights have already been probed.')
print ed.mmap_filename(blob=blob,
part='linear_set_val_ious%s' % suffix)
if not force:
return
else:
print('Forcefully continuing...')
info = ed.load_info()
seg_size = get_seg_size(info.input_dim)
blob_info = ed.load_info(blob=blob)
ds = loadseg.SegmentationData(info.dataset)
shape = blob_info.shape
N = shape[0] # number of total images
K = shape[1] # number of units in given blob
L = ds.label_size() # number of labels
if quantile == 1:
thresh = np.zeros((K, 1, 1))
else:
quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(K, -1))
threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))]
thresh = threshold[:, np.newaxis, np.newaxis]
fn_read = ed.mmap_filename(blob=blob)
blobdata = cached_memmap(fn_read, mode='r', dtype='float32', shape=shape)
image_to_label = load_image_to_label(directory)
if ed.has_mmap(blob=blob, part='linear_ind_ious%s' % suffix,
inc=True) and not force:
assert (ed.has_mmap(blob=blob,
part='linear_set_ious%s' % suffix,
inc=True))
ind_ious = ed.open_mmap(blob=blob,
part='linear_ind_ious%s' % suffix,
mode='r+',
inc=True,
dtype='float32',
shape=(L, N))
set_ious = ed.open_mmap(blob=blob,
part='linear_set_ious%s' % suffix,
mode='r+',
inc=True,
dtype='float32',
shape=(L, ))
set_ious_train = ed.open_mmap(blob=blob,
part='linear_set_train_ious%s' % suffix,
mode='r+',
inc=True,
dtype='float32',
shape=(L, ))
try:
set_ious_val = ed.open_mmap(blob=blob,
part='linear_set_val_ious%s' % suffix,
mode='r+',
inc=True,
dtype='float32',
shape=(L, ))
except:
set_ious_val = ed.open_mmap(blob=blob,
part='linear_set_val_ious%s' % suffix,
mode='r+',
dtype='float32',
shape=(L, ))
else:
ind_ious = ed.open_mmap(blob=blob,
part='linear_ind_ious%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, N))
set_ious = ed.open_mmap(blob=blob,
part='linear_set_ious%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, ))
set_ious_train = ed.open_mmap(blob=blob,
part='linear_set_train_ious%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, ))
set_ious_val = ed.open_mmap(blob=blob,
part='linear_set_val_ious%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, ))
if start is None:
start = 1
if end is None:
end = L
for label_i in range(start, end):
if ed.has_mmap(blob=blob,
part='label_i_%d_weights%s' % (label_i, suffix)):
try:
weights = ed.open_mmap(blob=blob,
part='label_i_%d_weights%s' %
(label_i, suffix),
mode='r',
dtype='float32',
shape=(K, ))
except ValueError:
# SUPPORTING LEGACY CODE (TODO: Remove)
weights = ed.open_mmap(blob=blob,
part='label_i_%d_weights%s' %
(label_i, suffix),
mode='r',
dtype=float,
shape=(K, ))
elif ed.has_mmap(blob=blob, part='linear_weights%s' % suffix):
all_weights = ed.open_mmap(blob=blob,
part='linear_weights%s' % suffix,
mode='r',
dtype='float32',
shape=(L, K))
weights = all_weights[label_i]
if not np.any(weights):
print(
'Label %d does not have associated weights to it, so skipping.'
% label_i)
continue
else:
print(
'Label %d does not have associated weights to it, so skipping.'
% label_i)
continue
if bias:
if ed.has_mmap(blob=blob,
part='label_i_%d_bias%s' % (label_i, suffix)):
bias_v = ed.open_mmap(blob=blob,
part='label_i_%d_bias%s' %
(label_i, suffix),
mode='r',
dtype='float32',
shape=(1, ))
else:
assert (ed.has_mmap(blob=blob, part='linear_bias%s' % suffix))
all_bias_v = ed.open_mmap(blob=blob,
part='linear_bias%s' % suffix,
mode='r',
dtype='float32',
shape=(L, ))
bias_v = np.array([all_bias_v[label_i]])
label_categories = ds.label[label_i]['category'].keys()
label_name = ds.name(category=None, j=label_i)
label_idx = np.where(image_to_label[:, label_i])[0]
loader = loadseg.SegmentationPrefetcher(ds,
categories=label_categories,
indexes=label_idx,
once=True,
batch_size=batch_size,
ahead=ahead,
thread=True)
loader_idx = loader.indexes
num_imgs = len(loader.indexes)
print(
'Probing with learned weights for label %d (%s) with %d images...'
% (label_i, label_name, num_imgs))
model = CustomLayer(K,
upsample=True,
up_size=seg_size,
act=True,
bias=bias,
positive=positive,
cuda=cuda)
model.weight.data[...] = torch.Tensor(weights)
if bias:
model.bias.data[...] = torch.Tensor(bias_v)
if cuda:
model.cuda()
model.eval()
iou_intersects = np.zeros(num_imgs)
iou_unions = np.zeros(num_imgs)
i = 0
for batch in loader.batches():
start_t = time.time()
if (i + 1) * batch_size < num_imgs:
idx = range(i * batch_size, (i + 1) * batch_size)
else:
idx = range(i * batch_size, num_imgs)
i += 1
input = torch.Tensor(
(blobdata[loader_idx[idx]] > thresh).astype(float))
input_var = (Variable(input.cuda(), volatile=True)
if cuda else Variable(input, volatile=True))
target_ = []
for rec in batch:
for cat in label_categories:
if rec[cat] != []:
if type(rec[cat]) is np.ndarray:
target_.append(
np.max((rec[cat] == label_i).astype(float),
axis=0))
else:
target_.append(np.ones(seg_size))
break
target = torch.Tensor(target_)
target_var = (Variable(target.cuda(), volatile=True)
if cuda else Variable(target, volatile=True))
#target_var = Variable(target.unsqueeze(1).expand_as(
# input_var).cuda() if cuda
# else target.unsqueeze(1).expand_as(input_var))
output_var = model(input_var)
iou_intersects[idx] = np.squeeze(
iou_intersect_d(output_var, target_var).data.cpu().numpy())
iou_unions[idx] = np.squeeze(
iou_union_d(output_var, target_var).data.cpu().numpy())
print('Batch: %d/%d\tTime: %f secs\tAvg Ind IOU: %f' %
(i, num_imgs / batch_size, time.time() - start_t,
np.mean(
np.true_divide(iou_intersects[idx],
iou_unions[idx] + 1e-20))))
loader.close()
label_ind_ious = np.true_divide(iou_intersects, iou_unions + 1e-20)
label_set_iou = np.true_divide(np.sum(iou_intersects),
np.sum(iou_unions) + 1e-20)
ind_ious[label_i, loader_idx] = label_ind_ious
set_ious[label_i] = label_set_iou
train_idx = [
i for i in range(len(loader_idx))
if ds.split(loader_idx[i]) == 'train'
]
val_idx = [
i for i in range(len(loader_idx))
if ds.split(loader_idx[i]) == 'val'
]
set_ious_train[label_i] = np.true_divide(
np.sum(iou_intersects[train_idx]),
np.sum(iou_unions[train_idx]) + 1e-20)
set_ious_val[label_i] = np.true_divide(
np.sum(iou_intersects[val_idx]),
np.sum(iou_unions[val_idx]) + 1e-20)
print(
'Label %d (%s) Set IOU: %f, Train Set IOU: %f, Val Set IOU: %f, Max Ind IOU: %f'
% (label_i, label_name, label_set_iou, set_ious_train[label_i],
set_ious_val[label_i], np.max(label_ind_ious)))
ed.finish_mmap(ind_ious)
ed.finish_mmap(set_ious)
ed.finish_mmap(set_ious_train)
ed.finish_mmap(set_ious_val)
def label_probe(directory,
blob,
batch_size=5,
ahead=4,
tally_depth=2048,
quantile=0.01,
categories=None,
verbose=False,
parallel=0):
'''
Evaluates the given blob from the probe stored in the given directory.
Note that as part of the evaluation, the dataset is loaded again.
'''
# Make sure we have a directory to work in
qcode = ('%f' % quantile).replace('0.', '').rstrip('0')
ed = expdir.ExperimentDirectory(directory)
# If the tally file already exists, return.
if ed.has_mmap(blob=blob, part='tally-%s' % qcode):
if verbose:
print 'Have', (ed.mmap_filename(blob=blob, part='tally-%s' %
qcode)), 'already. Skipping.'
return
# Load probe metadata
info = ed.load_info()
ih, iw = info.input_dim
# Load blob metadata
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
unit_size = shape[1]
fieldmap = blob_info.fieldmap
# Load the blob quantile data and grab thresholds
quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(shape[1], -1))
threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))]
thresh = threshold[:, numpy.newaxis, numpy.newaxis]
# Map the blob activation data for reading
fn_read = ed.mmap_filename(blob=blob)
# Load the dataset
ds = loadseg.SegmentationData(info.dataset)
if categories is None:
categories = ds.category_names()
labelcat = onehot(primary_categories_per_index(ds, categories))
# Be sure to zero out the background label - it belongs to no category.
labelcat[0, :] = 0
# Set up output data
# G = ground truth counts
# A = activation counts
# I = intersection counts
# TODO: consider activation_quantile and intersect_quantile
# fn_s = os.path.join(directory, '%s-%s-scores.txt' % (blob, qcode))
# T = tally, a compressed per-sample record of intersection counts.
fn_t = ed.mmap_filename(blob=blob, part='tally-%s' % qcode, inc=True)
# Create the file so that it can be mapped 'r+'
ed.open_mmap(blob=blob,
part='tally-%s' % qcode,
mode='w+',
dtype='int32',
shape=(ds.size(), tally_depth, 3))
if parallel > 1:
fast_process(fn_t, fn_read, shape, tally_depth, ds, iw, ih, categories,
fieldmap, thresh, labelcat, batch_size, ahead, verbose,
parallel)
else:
process_data(fn_t, fn_read, shape, tally_depth, ds, iw, ih, categories,
fieldmap, thresh, labelcat, batch_size, ahead, verbose,
False, 0, ds.size())
fn_final = re.sub('-inc$', '', fn_t)
if verbose:
print 'Renaming', fn_t, 'to', fn_final
os.rename(fn_t, fn_final)
def compute_distance_correlation(directory,
blob,
out_file=None,
learning_rate=1e-4,
momentum=0.9,
nesterov=False,
num_epochs=10,
batch_size=128,
randomize=True,
cuda=False,
verbose=False):
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
ds = loadseg.SegmentationData(info.dataset)
L = ds.label_size()
N = ds.size()
blob_info = ed.load_info(blob=blob)
K = blob_info.shape[1]
categories = np.array(ds.category_names())
label_names = np.array([ds.label[i]['name'] for i in range(L)])
(Hs, Ws) = get_seg_size(info.input_dim)
#concept_data = ed.open_mmap(part='concept_data', mode='r',
# shape=(N,L,Hs,Ws))
#upsampled_data = ed.open_mmap(blob=blob, part='upsampled', mode='r',
# shape=(N,K,Hs,Ws))
concept_fn = ed.mmap_filename(part='concept_data')
upsampled_fn = ed.mmap_filename(blob=blob, part='upsampled')
concept_data = cached_memmap(concept_fn,
mode='r',
dtype='float32',
shape=(N, L, Hs, Ws))
upsampled_data = cached_memmap(upsampled_fn,
mode='r',
dtype='float32',
shape=(N, K, Hs, Ws))
concept_weights = WeightedData(L)
act_weights = WeightedData(K)
if cuda:
concept_weights = concept_weights.cuda()
act_weights = act_weights.cuda()
criterion = lambda x, y: -1 * distcorr(x, y, use_quad=False)
optimizer = torch.optim.SGD([{
'params': concept_weights.parameters()
}, {
'params': act_weights.parameters()
}],
lr=args.learning_rate,
momentum=args.momentum,
nesterov=args.nesterov)
train_idx = np.array([i for i in range(N) if ds.split(i) == 'train'])
val_idx = np.array([i for i in range(N) if ds.split(i) == 'val'])
train_losses = np.zeros(num_epochs)
val_losses = np.zeros(num_epochs)
for t in range(num_epochs):
train_loss = run_epoch(concept_data[:, :, Hs / 2, Ws / 2],
upsampled_data[:, :, Hs / 2, Ws / 2],
concept_weights,
act_weights,
train_idx,
criterion,
optimizer,
t + 1,
batch_size=batch_size,
train=True,
randomize=randomize,
cuda=cuda,
verbose=verbose)
val_loss = run_epoch(concept_data[:, :, Hs / 2, Ws / 2],
upsampled_data[:, :, Hs / 2, Ws / 2],
concept_weights,
act_weights,
val_idx,
criterion,
optimizer,
t + 1,
batch_size=batch_size,
train=False,
randomize=randomize,
cuda=cuda,
verbose=verbose)
train_losses[t] = train_loss
val_losses[t] = val_loss
#print(distcorr(x_weights(Variable(torch.Tensor(x).cuda())),
# y_weights(Variable(torch.Tensor(y).cuda())).data.cpu().numpy()))
print(concept_weights.weight)
print(act_weights.weight)
results = {}
if out_file is not None:
if verbose:
start = time.time()
print('Saving results...')
results['directory'] = directory
results['blob'] = blob
results['learning_rate'] = learning_rate
results['momentum'] = momentum
results['nesterov'] = nesterov
results['batch_size'] = batch_size
results['train_losses'] = train_losses
results['val_losses'] = val_losses
results['num_epochs'] = num_epochs
results['randomize'] = randomize
results['concept_weights'] = concept_weights.weight.data.cpu().numpy()
results['act_weights'] = act_weights.weight.data.cpu().numpy()
pkl.dump(results, open(out_file, 'wb'))
if verbose:
print('Saved results at %s in %d secs.' %
(out_file, time.time() - start))
def reverse_linear_probe(directory,
blob,
filter_i,
suffix='',
prev_suffix=None,
batch_size=64,
quantile=0.005,
bias=False,
positive=False,
use_svm=True,
num_examples=1000,
num_epochs=30,
lr=1e-4,
momentum=0.9,
l1_weight_decay=0,
l2_weight_decay=0,
validation=False,
nesterov=False,
lower_bound=None,
cuda=False):
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
ds = loadseg.SegmentationData(info.dataset)
L = ds.label_size()
N = ds.size()
(Hs, Ws) = get_seg_size(info.input_dim)
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
K = shape[1]
if quantile == 1:
thresh = np.zeros((1, 1))
else:
quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(K, -1))
#threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))]
threshold = quantdata[filter_i,
int(round(quantdata.shape[1] * quantile))]
#thresh = threshold[:, np.newaxis, np.newaxis]
thresh = threshold
blobdata = ed.open_mmap(blob=blob, mode='r', shape=shape)
#upsampled_data = ed.open_mmap(blob=blob, part='upsampled', mode='r')
conceptdata = ed.open_mmap(part='concept_data',
mode='r',
shape=(N, L, Hs, Ws))
train_idx = np.array([i for i in range(N) if ds.split(i) == 'train'])
val_idx = np.array([i for i in range(N) if ds.split(i) == 'val'])
# calculate mean non-negative filter activation
perc_label = []
thresh_idx = []
for i in train_idx:
num_thresh = np.sum((blobdata[i, filter_i] > thresh).astype(float))
perc_label.append(num_thresh / float(np.prod(shape[2:])))
if num_thresh > 0:
thresh_idx.append(i)
#perc_label.append(num_nz / float(np.prod(shape[2:])))
perc_label = np.array(perc_label)
alpha_unnorm = float(1. - np.mean(perc_label))
alpha = float(1. - np.mean(perc_label[thresh_idx]))
train_idx = train_idx[thresh_idx]
print('Alpha for filter %d: %f (%f unnorm)' %
(filter_i, alpha, alpha_unnorm))
print('# above thresh train examples for filter %d: %d' %
(filter_i, len(thresh_idx)))
if use_svm:
train_subset_idx = range(num_examples)
val_subset_idx = range(num_examples)
start = time.time()
print 'Selecting training data...'
X_train = np.mean(np.mean(conceptdata[train_idx[train_subset_idx], :,
52:62, 52:62],
axis=-1),
axis=-1)
Y_train = (blobdata[train_idx[train_subset_idx], filter_i, 5, 5] >
thresh).astype(int)
print 'Finished selecting training data in %d secs. Fitting SVM...' % (
time.time() - start)
start = time.time()
clf = svm.SVC(kernel="linear", class_weight="balanced")
clf.fit(X_train, Y_train)
print 'Finished fitting SVM in %d secs. Selecting validation data...' % (
time.time() - start)
start = time.time()
X_val = np.mean(np.mean(conceptdata[train_idx[val_subset_idx], :,
52:62, 52:62],
axis=-1),
axis=-1)
Y_val = (blobdata[val_idx[val_subset_idx], filter_i, 5, 5] >
thresh).astype(float)
print 'Finished selecting validation data in %d secs. Predicting for validation data...' % (
time.time() - start)
start = time.time()
Y_pred = clf.predict(X_val)
print 'Finished predicting validation in %d secs.' % (time.time() -
start)
print 'Val MSE:', np.mean((Y_pred - Y_val)**2)
joblib.dump(
clf,
os.path.join(
directory, "%s-filter_i_%d_num_examples_%d.pkl" %
(blob, filter_i, num_examples)))
else:
layer = CustomLayer(num_features=L - 1,
upsample=False,
act=False,
positive=positive,
bias=bias,
cuda=cuda)
if prev_suffix is not None:
prev_weights_mmap = ed.open_mmap(blob=blob,
part='filter_i_%d_weights%s' %
(filter_i, prev_suffix),
mode='r',
shape=layer.weights.shape)
layer.weights.data[:] = torch.Tensor(prev_weights_mmap[:])
if cuda:
layer = layer.cuda()
#if quantile == 1:
# criterion = nn.MSELoss()
#else:
# criterion = nn.BCEWithLogitsLoss()
#if cuda:
# criterion = criterion.cuda()
criterion = lambda x, y: BCELoss2d(x, y, alpha)
optimizer = Custom_SGD(layer.parameters(),
lr,
momentum,
l1_weight_decay=l1_weight_decay,
l2_weight_decay=l2_weight_decay,
nesterov=nesterov,
lower_bound=lower_bound)
results = np.zeros((8, num_epochs))
for t in range(num_epochs):
(trn_loss, trn_set_iou, trn_ind_ious, trn_c_given_f,
trn_f_given_c) = run_epoch(blobdata,
conceptdata,
train_idx,
filter_i,
thresh,
layer,
batch_size,
criterion,
optimizer,
t + 1,
train=True,
cuda=cuda,
iou_threshold=0)
(val_loss, val_set_iou, val_ind_ious, val_c_given_f,
val_f_given_c) = run_epoch(blobdata,
conceptdata,
val_idx,
filter_i,
thresh,
layer,
batch_size,
criterion,
optimizer,
t + 1,
train=False,
cuda=cuda,
iou_threshold=0)
results[0][t] = trn_loss
results[1][t] = val_loss
results[2][t] = trn_set_iou
results[3][t] = val_set_iou
results[4][t] = trn_c_given_f
results[5][t] = val_c_given_f
results[6][t] = trn_f_given_c
results[7][t] = val_f_given_c
ind_ious = np.zeros(N)
ind_ious[train_idx] = trn_ind_ious
ind_ious[val_idx] = val_ind_ious
ind_ious_mmap = ed.open_mmap(blob=blob,
part='filter_i_%d_ind_ious%s' %
(filter_i, suffix),
mode='w+',
shape=ind_ious.shape)
ind_ious_mmap[:] = ind_ious[:]
ed.finish_mmap(ind_ious_mmap)
results_mmap = ed.open_mmap(blob=blob,
part='filter_i_%d_results%s' %
(filter_i, suffix),
mode='w+',
shape=results.shape)
results_mmap[:] = results[:]
ed.finish_mmap(results_mmap)
# save learned weights (and bias)
weights = layer.weight.data.cpu().numpy()
weights_mmap = ed.open_mmap(blob=blob,
part='filter_i_%d_weights%s' %
(filter_i, suffix),
mode='w+',
shape=weights.shape)
weights_mmap[:] = weights[:]
ed.finish_mmap(weights_mmap)
if bias:
bias_v = layer.bias.data.cpu().numpy()
bias_mmap = ed.open_mmap(blob=blob,
part='filter_i_%d_bias%s' %
(filter_i, suffix),
mode='w+',
shape=(1, ))
bias_mmap[:] = bias_v[:]
ed.finish_mmap(bias_mmap)
def consolidate_disc_probe(directory,
blob,
bias=False,
num_filters=None,
suffix='',
epochs=30,
delete=False):
ed = expdir.ExperimentDirectory(directory)
try:
info = ed.load_info()
except:
ed = expdir.ExperimentDirectory(os.path.join(directory, 'train'))
#ed_val = expdir.ExperimentDirectory(os.path.join(directory, 'val'))
info = ed.load_info()
assert ('imagenet' in info.dataset or 'ILSVRC' in info.dataset)
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
if 'broden' in info.dataset:
ds = loadseg.SegmentationData(info.dataset)
L = ds.label_size()
elif 'imagenet' in info.dataset or 'ILSVRC' in info.dataset:
L = 1000
N = shape[0] # total number of images in the dataset
K = shape[1] # number of units for the given blob
F = 1 if num_filters is None else len(num_filters) + 1
if num_filters is None:
suffixes = [suffix]
else:
suffixes = ['%s_num_filters_%d' % (suffix, n) for n in num_filters]
suffixes.append(suffix)
suffix = '%s_num_filters_%d' % (suffix, F)
if (ed.has_mmap(blob=blob, part='linear_weights_disc%s' % suffix) and
(not bias
or ed.has_mmap(blob=blob, part='linear_bias_disc%s' % suffix))):
print('Linear weights (and bias) have already been consolidated')
print ed.mmap_filename(blob=blob,
part='linear_weights_disc%s' % suffix)
else:
weights_mmap = ed.open_mmap(blob=blob,
part='linear_weights_disc%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, F, K))
if bias:
bias_mmap = ed.open_mmap(blob=blob,
part='linear_bias_disc%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, F))
results_mmap = ed.open_mmap(blob=blob,
part='linear_results_disc%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, F, 4, epochs))
missing_idx = []
for l in range(L):
if not ed.has_mmap(blob=blob,
part='label_i_%d_weights_disc%s' %
(l, suffixes[0])):
missing_idx.append(l)
continue
for i in range(len(suffixes)):
suffix = suffixes[i]
label_weights_mmap = ed.open_mmap(
blob=blob,
part='label_i_%d_weights_disc%s' % (l, suffix),
mode='r',
dtype='float32',
shape=(K, ))
weights_mmap[l, i, :] = label_weights_mmap[:]
if bias:
label_bias_mmap = ed.open_mmap(
blob=blob,
part='label_i_%d_bias_disc%s' % (l, suffix),
mode='r',
dtype='float32',
shape=(1, ))
bias_mmap[l, i] = label_bias_mmap[:]
label_results_mmap = ed.open_mmap(
blob=blob,
part='label_i_%d_results_disc%s' % (l, suffix),
mode='r',
dtype='float32',
shape=(4, epochs))
results_mmap[l, i, :, :] = label_results_mmap[:, :]
print l
ed.finish_mmap(weights_mmap)
if bias:
ed.finish_mmap(bias_mmap)
ed.finish_mmap(results_mmap)
print('Finished consolidating existing weights files ' +
'(Files for %d labels were missing).' % len(missing_idx))
print(missing_idx)
if delete:
c_w = 0
c_b = 0
c_r = 0
print('Deleting all unnecessary label weights (and bias) files...')
for l in range(L):
for i in range(F):
suffix = suffixes[i]
if ed.has_mmap(blob=blob,
part='label_i_%d_weights_disc%s' % (l, suffix)):
fn = ed.mmap_filename(blob=blob,
part='label_i_%d_weights_disc%s' %
(l, suffix))
os.remove(fn)
c_w += 1
if bias and ed.has_mmap(blob=blob,
part='label_i_%d_bias_disc%s' %
(l, suffix)):
fn = ed.mmap_filename(blob=blob,
part='label_i_%d_bias_disc%s' %
(l, suffix))
os.remove(fn)
c_b += 1
if ed.has_mmap(blob=blob,
part='label_i_%d_results_disc%s' % (l, suffix)):
fn = ed.mmap_filename(blob=blob,
part='label_i_%d_results_disc%s' %
(l, suffix))
os.remove(fn)
c_r += 1
print('Removed %d weights, %d bias, and %d results files.' %
(c_w, c_b, c_r))
def create_probe(directory,
dataset,
definition,
weights,
mean,
blobs,
colordepth=3,
rotation_seed=None,
rotation_power=1,
limit=None,
split=None,
batch_size=16,
ahead=4,
cl_args=None,
verbose=True,
cuda=False):
# If we're already done, skip it!
ed = expdir.ExperimentDirectory(directory)
if all(ed.has_mmap(blob=b) for b in blobs):
return
'''
directory: where to place the probe_conv5.mmap files.
data: the AbstractSegmentation data source to draw upon
definition: the filename for the caffe prototxt
weights: the filename for the caffe model weights
mean: to use to normalize rgb values for the network
blobs: ['conv3', 'conv4', 'conv5'] to probe
'''
if verbose:
print 'Opening dataset', dataset
#data = loadseg.SegmentationData(args.dataset)
# the network to dissect
if args.weights == None:
# load the imagenet pretrained model
try:
net = torchvision.models.__dict__[args.definition](pretrained=True)
except URLError:
print('Manually loading pretrained weights...')
import torch.utils.model_zoo as model_zoo
model_urls = {
'vgg19':
'http://download.pytorch.org/models/vgg19-dcbb9e9d.pth'
}
net = torchvision.models.__dict__[args.definition]()
net.load_state_dict(model_zoo.load_url(
model_urls[args.definition]))
else:
# load your own model
net = torchvision.models.__dict__[args.definition](
num_classes=args.num_classes)
checkpoint = torch.load(args.weights)
state_dict = {
str.replace(k, 'module.', ''): v
for k, v in checkpoint['state_dict'].iteritems()
} # the data parallel layer will add 'module' before each layer name
net.load_state_dict(state_dict)
net.eval()
# hook up to get the information for each selected layer
layers = net._modules.keys()
size_blobs_output = []
def hook_size(module, input, output):
size_blobs_output.append(output.data.size())
input_sample = V(torch.randn(1, 3, args.input_size, args.input_size))
def get_pytorch_module(net, blob):
modules = blob.split('.')
if len(modules) == 1:
return net._modules.get(blob)
else:
curr_m = net
for m in modules:
curr_m = curr_m._modules.get(m)
return curr_m
for blob in blobs:
get_pytorch_module(net, blob).register_forward_hook(hook_size)
#net._modules.get(blob).register_forward_hook(hook_size)
output_sample = net(input_sample)
input_dim = [args.input_size, args.input_size]
#data_size = data.size(split) # the image size
#if limit is not None:
# data_size = min(data_size, limit)
transform = trn.Compose([
#transforms.RandomSizedCrop(args.input_size),
#transforms.RandomHorizontalFlip(),
trn.Scale(args.input_size),
trn.CenterCrop(args.input_size),
trn.ToTensor(),
trn.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
#traindir = os.path.join(dataset, 'train')
#valdir = os.path.join(dataset, 'val')
data = datasets.ImageFolder(dataset, transform)
data_size = len(data)
# Make sure we have a directory to work in
ed.ensure_dir()
# Step 0: write a README file with generated information.
ed.save_info(
dict(dataset=dataset,
split=split,
definition=definition,
weights=weights,
mean=mean,
blobs=blobs,
input_dim=input_dim,
rotation_seed=rotation_seed,
rotation_power=rotation_power))
# Clear old probe data
ed.remove_all('*.mmap*')
# Create new (empty) mmaps
if verbose:
print 'Creating new mmaps.'
out = {}
rot = None
if rotation_seed is not None:
rot = {}
for idx, blob in enumerate(blobs):
#shape = (data_size, ) + net.blobs[blob].data.shape[1:]
shape = (data_size, int(size_blobs_output[idx][1]),
int(size_blobs_output[idx][2]),
int(size_blobs_output[idx][3]))
out[blob] = ed.open_mmap(blob=blob, mode='w+', shape=shape)
# Rather than use the exact RF, here we use some heuristics to compute the approximate RF
size_RF = (args.input_size / size_blobs_output[idx][2],
args.input_size / size_blobs_output[idx][3])
fieldmap = ((0, 0), size_RF, size_RF)
ed.save_info(blob=blob,
data=dict(name=blob, shape=shape, fieldmap=fieldmap))
# The main loop
if verbose:
print 'Beginning work.'
#pf = loadseg.SegmentationPrefetcher(data, categories=['image'],
# split=split, once=True, batch_size=batch_size, ahead=ahead)
# Data loading code
#val = datasets.ImageFolder(valdir, transform)
loader = torch.utils.data.DataLoader(data,
batch_size=batch_size,
shuffle=False,
num_workers=4)
index = 0
start_time = time.time()
last_batch_time = start_time
batch_size = 0
if cuda:
net.cuda()
# hook the feature extractor
features_blobs = []
def hook_feature(module, input, output):
features_blobs.append(output.data.cpu().numpy())
for blob in blobs:
get_pytorch_module(net, blob).register_forward_hook(hook_feature)
#net._modules.get(blob).register_forward_hook(hook_feature)
for i, (batch, target) in enumerate(loader):
del features_blobs[:] # clear up the feature basket
batch_time = time.time()
rate = i / (batch_time - start_time + 1e-15)
batch_size = len(batch)
batch_rate = batch_size / (batch_time - last_batch_time + 1e-15)
last_batch_time = batch_time
if verbose:
print 'netprobe index', i, 'items per sec', batch_rate, rate
sys.stdout.flush()
#inp = batch[0]
#batch_size = len(inp)
#if limit is not None and index + batch_size > limit:
# Truncate last if limited
# batch_size = limit - index
# inp = inp[:batch_size]
#if colordepth == 1:
# inp = numpy.mean(inp, axis=1, keepdims=True)
# previous feedforward case
#inp = inp[:,::-1,:,:]
#inp_tensor = V(torch.from_numpy(inp.copy()))
#inp_tensor.div_(255.0*0.224) # approximately normalize the input to make the images scaled at around 1.
inp_tensor = V(batch)
if cuda:
inp_tensor = inp_tensor.cuda()
result = net.forward(inp_tensor)
# output the hooked feature
for i, key in enumerate(blobs):
out[key][index:index + batch_size] = numpy.copy(
features_blobs[i][:batch_size])
# print 'Recording data in mmap done'
index += batch_size
if index >= data_size:
break
assert index == data_size, (
"Data source should return evey item once %d %d." % (index, data_size))
if verbose:
print 'Renaming mmaps.'
for blob in blobs:
ed.finish_mmap(out[blob])
# Final step: write the README file
write_readme_file([('cl_args', cl_args), ('data', data),
('definition', definition), ('weight', weights),
('mean', mean), ('blobs', blobs)],
ed,
verbose=verbose)
def consolidate_probe(directory,
blob,
bias=False,
num_filters=None,
suffix='',
delete=False):
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
ds = loadseg.SegmentationData(info.dataset)
L = ds.label_size() # number of labels (including background at index 0)
N = ds.size() # total number of images in the dataset
K = shape[1] # number of units for the given blob
F = 1 if num_filters is None else len(num_filters) + 1
if num_filters is None:
suffixes = [suffix]
else:
suffixes = ['%s_num_filters_%d' % (suffix, n) for n in num_filters]
suffixes.append(suffix)
suffix = '%s_num_filters_%d' % (suffix, F)
if (ed.has_mmap(blob=blob, part='linear_weights%s' % suffix) and
(not bias or ed.has_mmap(blob=blob, part='linear_bias%s' % suffix))):
print('Linear weights (and bias) have already been consolidated')
else:
weights_mmap = ed.open_mmap(blob=blob,
part='linear_weights%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, F, K))
if bias:
bias_mmap = ed.open_mmap(blob=blob,
part='linear_bias%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, F))
missing_idx = []
for l in range(L):
if not ed.has_mmap(blob=blob,
part='label_i_%d_weights%s' % (l, suffixes[0])):
missing_idx.append(l)
continue
for i in range(F):
suffix = suffixes[i]
if ed.has_mmap(blob=blob,
part='label_i_%d_weights%s' % (l, suffix)):
try:
label_weights_mmap = ed.open_mmap(
blob=blob,
part='label_i_%d_weights%s' % (l, suffix),
mode='r',
dtype='float32',
shape=(K, ))
except ValueError:
print('here')
# SUPPORT LEGACY CODE, TODO: remove eventually
label_weights_mmap = ed.open_mmap(
blob=blob,
part='label_i_%d_weights%s' % (l, suffix),
mode='r',
dtype=float,
shape=(K, ))
else:
all_weights_mmap = ed.open_mmap(blob=blob,
part='linear_weights%s' %
suffix,
mode='r',
dtype='float32',
shape=(L, K))
label_weights_mmap = all_weights_mmap[l]
weights_mmap[l, i, :] = label_weights_mmap[:]
if bias:
if ed.has_mmap(blob=blob,
part='label_i_%d_bias' % (l, suffix)):
label_bias_mmap = ed.open_mmap(
blob=blob,
part='label_i_%d_bias%s' % (l, suffix),
mode='r',
dtype='float32',
shape=(1, ))
else:
all_bias_mmap = ed.open_mmap(blob=blob,
part='linear_bias%s' %
suffix,
mode='r',
dtype='float32',
shape=(K, ))
label_bias_mmap = all_bias_mmap[l]
bias_mmap[l, i] = label_bias_mmap[:]
ed.finish_mmap(weights_mmap)
if bias:
ed.finish_mmap(bias_mmap)
print('Finished consolidating existing weights files ' +
'(Files for %d labels were missing).' % len(missing_idx))
print(missing_idx)
if delete:
c_w = 0
c_b = 0
print('Deleting all unnecessary label weights (and bias) files...')
for l in range(L):
for i in range(F):
suffix = suffixes[i]
if ed.has_mmap(blob=blob,
part='label_i_%d_weights%s' % (l, suffix)):
fn = ed.mmap_filename(blob=blob,
part='label_i_%d_weights%s' %
(l, suffix))
os.remove(fn)
c_w += 1
if bias and ed.has_mmap(
blob=blob, part='label_i_%d_bias%s' % (l, suffix)):
fn = ed.mmap_filename(blob=blob,
part='label_i_%d_bias%s' %
(l, suffix))
os.remove(fn)
c_b += 1
print('Removed %d weights and %d bias files.' % (c_w, c_b))
def linear_probe(directory, blob, label_i, suffix='', init_suffix='', num_filters=None, batch_size=16, ahead=4,
quantile=0.005, bias=False, positive=False, num_epochs=30, lr=1e-4, momentum=0.9,
l1_weight_decay=0, l2_weight_decay=0, validation=False, nesterov=False, lower_bound=None,
min_train=None, max_train=None, max_val=None,
cuda=False):
# Make sure we have a directory to work in
#qcode = ('%f' % quantile).replace('0.','').rstrip('0')
ed = expdir.ExperimentDirectory(directory)
# Check if linear weights have already been learned
print ed.mmap_filename(blob=blob, part='label_i_%s_weights%s' % (label_i, suffix))
if ed.has_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix)):
print('%s already has %s, so skipping.' % (directory,
ed.mmap_filename(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix))))
return
# Load probe metadata
info = ed.load_info()
ih, iw = info.input_dim
# Load blob metadata
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
unit_size = shape[1]
fieldmap = blob_info.fieldmap
# Load the blob quantile data and grab thresholds
if quantile == 1:
thresh = np.zeros((unit_size,1,1))
else:
quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(unit_size, -1))
threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))]
thresh = threshold[:, np.newaxis, np.newaxis]
#print np.max(thresh), thresh.shape, type(thresh)
# Map the blob activation data for reading
fn_read = ed.mmap_filename(blob=blob)
# Load the dataset
ds = loadseg.SegmentationData(info.dataset)
# Get all the categories the label is a part of
label_categories = ds.label[label_i]['category'].keys()
num_categories = len(label_categories)
# Get label name
label_name = ds.name(category=None, j=label_i)
blobdata = cached_memmap(fn_read, mode='r', dtype='float32', shape=shape)
# Get indices of images containing the given label
if not has_image_to_label(directory):
print('image_to_label does not exist in %s; creating it now...' % directory)
create_image_to_label(directory, batch_size=batch_size, ahead=ahead)
image_to_label = load_image_to_label(directory)
label_idx = np.where(image_to_label[:, label_i])[0]
train_idx = np.array([i for i in label_idx if ds.split(i) == 'train'])
val_idx = np.array([i for i in label_idx if ds.split(i) == 'val'])
if min_train is not None and len(train_idx) < min_train:
print('Number of training examples for label %d (%s) is %d, which is less than the minimum of %d so skipping.'
% (label_i, label_name, len(train_idx), min_train))
if max_train is not None and len(train_idx) > max_train:
train_idx = train_idx[:max_train]
if max_val is not None and len(val_idx) > max_val:
val_idx = val_idx[:max_val]
print('Total number of images containing label %d (%s): %d' % (
label_i, label_name, len(label_idx)))
try:
train_loader = loadseg.SegmentationPrefetcher(ds, categories=label_categories,
indexes=train_idx, once=False,
batch_size=batch_size,
ahead=ahead, thread=True)
except IndexError as err:
print(err.args)
return
sw = 0
sh = 0
perc_label = []
train_label_categories = []
for batch in train_loader.batches():
for rec in batch:
# Check that the same segmentation dimensions are used for all
# examples
sw_r, sh_r = [rec[k] for k in ['sw', 'sh']]
if sw == 0 and sh == 0:
sw = sw_r
sh = sh_r
else:
assert(sw == sw_r and sh == sh_r)
for cat in label_categories:
if rec[cat] != []:
train_label_categories.append(cat)
if type(rec[cat]) is np.ndarray:
perc_label.append(np.sum(rec[cat] == label_i) / float(sw * sh))
else:
perc_label.append(1.)
break
assert(len(perc_label) == len(train_idx))
# Compute reduction from segmentation dimensions to image dimensions
reduction = int(round(iw / float(sw)))
# Calculate class-weighting alpha parameter for segmentation loss
# (Note: float typecast is necessary)
alpha = float(1. - np.mean(perc_label))
if alpha == 0:
alpha = None
print('Not using class-weighting because no pixel-level annotations')
else:
print('Alpha for label %d (%s): %f' % (label_i, label_name, alpha))
# Prepare segmentation loss function using class-weight alpha
criterion = lambda x,y: BCELoss2d(x,y,alpha)
# Prepare to learn linear weights with a sigmoid activation after
# the linear layer
#layer = CustomLayer(unit_size, upsample=False, act=True, positive=False)
if num_filters is not None:
if ed.has_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, init_suffix)):
init_weights_mmap = ed.open_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, init_suffix),
mode='r', dtype='float32', shape=unit_size)
elif ed.has_mmap(blob=blob, part='linear_weights%s' % (init_suffix)):
all_weights_mmap = ed.open_mmap(blob=blob, part='linear_weights%s' % init_suffix,
mode='r', dtype='float32', shape=(ds.label_size(),unit_size))
init_weights_mmap = all_weights_mmap[label_i]
else:
assert(False)
sorted_idx = np.argsort(np.abs(init_weights_mmap))[::-1]
mask_idx = np.zeros(unit_size, dtype=int)
mask_idx[sorted_idx[:num_filters]] = 1
layer = CustomLayer(unit_size, upsample=True, up_size=(sh,sw), act=True,
bias=bias, positive=positive, mask_idx=torch.ByteTensor(mask_idx), cuda=cuda)
else:
layer = CustomLayer(unit_size, upsample=True, up_size=(sh,sw), act=True,
bias=bias, positive=positive, cuda=cuda)
if cuda:
layer.cuda()
optimizer = Custom_SGD(layer.parameters(), lr, momentum,
l1_weight_decay=l1_weight_decay, l2_weight_decay=l2_weight_decay,
nesterov=nesterov, lower_bound=lower_bound)
if not validation:
try:
val_loader = loadseg.SegmentationPrefetcher(ds, categories=label_categories,
indexes=val_idx, once=False, batch_size=batch_size,
ahead=ahead, thread=True)
except IndexError as err:
print(err.args)
train_loader.close()
return
val_label_categories = []
for batch in val_loader.batches():
for rec in batch:
for cat in label_categories:
if rec[cat] != []:
val_label_categories.append(cat)
break
assert(len(val_label_categories) == len(val_idx))
for t in range(num_epochs):
(_, iou) = run_epoch(blobdata, train_idx, train_label_categories, label_i,
fieldmap, thresh, sh, sw, reduction, train_loader, layer, criterion,
optimizer, t+1, train=True, cuda=cuda, iou_threshold=0.5)
if not validation:
(_, iou) = run_epoch(blobdata, val_idx, val_label_categories, label_i,
fieldmap, thresh, sh, sw, reduction, val_loader, layer, criterion,
optimizer, t+1, train=False, cuda=cuda, iou_threshold=0.5)
# Close segmentation prefetcher (i.e. close pools)
train_loader.close()
if not validation:
val_loader.close()
# Save weights
weights = (layer.mask * layer.weight).data.cpu().numpy()
weights_mmap = ed.open_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix),
mode='w+', dtype='float32', shape=weights.shape)
weights_mmap[:] = weights[:]
ed.finish_mmap(weights_mmap)
if bias:
bias_v = layer.bias.data.cpu().numpy()
bias_mmap = ed.open_mmap(blob=blob, part='label_i_%d_bias%s' % (label_i, suffix),
mode='w+', dtype='float32', shape=(1,))
bias_mmap[:] = bias_v[:]
ed.finish_mmap(bias_mmap)
print '%s finished' % ed.mmap_filename(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix))
def create_probe(
directory, dataset, weights, mean, blobs):
data = loadseg.SegmentationData(dataset)
ed = expdir.ExperimentDirectory(directory)
def create_probe(directory,
dataset,
definition,
weights,
mean,
blobs,
colordepth=3,
rotation_seed=None,
rotation_power=1,
rotation_unpermute=False,
limit=None,
split=None,
batch_size=16,
ahead=4,
cl_args=None,
verbose=True):
# If we're already done, skip it!
ed = expdir.ExperimentDirectory(directory)
if all(ed.has_mmap(blob=b) for b in blobs):
return
'''
directory: where to place the probe_conv5.mmap files.
data: the AbstractSegmentation data source to draw upon
definition: the filename for the caffe prototxt
weights: the filename for the caffe model weights
mean: to use to normalize rgb values for the network
blobs: ['conv3', 'conv4', 'conv5'] to probe
'''
if verbose:
print 'Opening dataset', dataset
data = loadseg.SegmentationData(args.dataset)
if verbose:
print 'Opening network', definition
np = caffe_pb2.NetParameter()
with open(definition, 'r') as dfn_file:
text_format.Merge(dfn_file.read(), np)
net = caffe.Net(definition, weights, caffe.TEST)
input_blob = net.inputs[0]
input_dim = net.blobs[input_blob].data.shape[2:]
data_size = data.size(split)
if limit is not None:
data_size = min(data_size, limit)
# Make sure we have a directory to work in
ed.ensure_dir()
# Step 0: write a README file with generated information.
ed.save_info(
dict(dataset=dataset,
split=split,
definition=definition,
weights=weights,
mean=mean,
blobs=blobs,
input_dim=input_dim,
rotation_seed=rotation_seed,
rotation_power=rotation_power))
# Clear old probe data
ed.remove_all('*.mmap*')
# Create new (empty) mmaps
if verbose:
print 'Creating new mmaps.'
out = {}
rot = None
if rotation_seed is not None:
rot = {}
for blob in blobs:
shape = (data_size, ) + net.blobs[blob].data.shape[1:]
out[blob] = ed.open_mmap(blob=blob, mode='w+', shape=shape)
# Find the shortest path through the network to the target blob
fieldmap, _ = upsample.composed_fieldmap(np.layer, blob)
# Compute random rotation for each blob, if needed
if rot is not None:
rot[blob] = rotate.randomRotationPowers(
shape[1], [rotation_power],
rotation_seed,
unpermute=rotation_unpermute)[0]
ed.save_info(blob=blob,
data=dict(name=blob, shape=shape, fieldmap=fieldmap))
# The main loop
if verbose:
print 'Beginning work.'
pf = loadseg.SegmentationPrefetcher(data,
categories=['image'],
split=split,
once=True,
batch_size=batch_size,
ahead=ahead)
index = 0
start_time = time.time()
last_batch_time = start_time
batch_size = 0
for batch in pf.tensor_batches(bgr_mean=mean):
batch_time = time.time()
rate = index / (batch_time - start_time + 1e-15)
batch_rate = batch_size / (batch_time - last_batch_time + 1e-15)
last_batch_time = batch_time
if verbose:
print 'netprobe index', index, 'items per sec', batch_rate, rate
sys.stdout.flush()
inp = batch[0]
batch_size = len(inp)
if limit is not None and index + batch_size > limit:
# Truncate last if limited
batch_size = limit - index
inp = inp[:batch_size]
if colordepth == 1:
inp = numpy.mean(inp, axis=1, keepdims=True)
net.blobs[input_blob].reshape(*(inp.shape))
net.blobs[input_blob].data[...] = inp
result = net.forward(blobs=blobs)
if rot is not None:
for key in out.keys():
result[key] = numpy.swapaxes(
numpy.tensordot(rot[key], result[key],
axes=((1, ), (1, ))), 0, 1)
# print 'Computation done'
for key in out.keys():
out[key][index:index + batch_size] = result[key]
# print 'Recording data in mmap done'
index += batch_size
if index >= data_size:
break
assert index == data_size, (
"Data source should return evey item once %d %d." % (index, data_size))
if verbose:
print 'Renaming mmaps.'
for blob in blobs:
ed.finish_mmap(out[blob])
# Final step: write the README file
write_readme_file([('cl_args', cl_args), ('data', data),
('definition', definition), ('weight', weights),
('mean', mean), ('blobs', blobs)],
ed,
verbose=verbose)
def label_probe(directory, blob, quantile=0.005, batch_size=16, ahead=4, start=None,
end=None, suffix='', cuda=False):
# Make sure we have a directory to work in
qcode = ('%f' % quantile).replace('0.','').rstrip('0')
ed = expdir.ExperimentDirectory(directory)
# Check if label probe has already been created
if (ed.has_mmap(blob=blob, part='single_set_ious%s' % suffix) and
ed.has_mmap(blob=blob, part='single_ind_ious%s' % suffix)):
print('label_probe_pytorch.py has already been run.')
return
# Load probe metadata
info = ed.load_info()
seg_size = get_seg_size(info.input_dim)
# Load blob metadata
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
tot_imgs = shape[0]
unit_size = shape[1]
# Load the blob quantile data and grab thresholds
quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(unit_size, -1))
threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))]
thresh = threshold[:, np.newaxis, np.newaxis]
# Load the dataset
ds = loadseg.SegmentationData(info.dataset)
# Map the blob activation data for reading
#fn_read = ed.mmap_filename(blob=blob)
#blobdata = cached_memmap(fn_read, mode='r', dtype='float32', shape=shape)
blobdata = ed.open_mmap(blob=blob, mode='r', shape=shape)
# Get image-to-labels mapping
if not has_image_to_label(directory):
print('image_to_label does not exist in %s; creating it now...' % directory)
create_image_to_label(directory, batch_size=batch_size, ahead=ahead)
image_to_label = load_image_to_label(directory)
num_labels = ds.label_size()
upsample = nn.Upsample(size=seg_size, mode='bilinear')
set_ious_train_mmap = ed.open_mmap(blob=blob, part='single_set_train_ious%s' % suffix,
mode='w+', dtype='float32', shape=(num_labels, unit_size))
set_ious_val_mmap = ed.open_mmap(blob=blob, part='single_set_val_ious%s' % suffix,
mode='w+', dtype='float32', shape=(num_labels, unit_size))
set_ious_mmap = ed.open_mmap(blob=blob, part='single_set_ious%s' % suffix, mode='w+',
dtype='float32', shape=(num_labels, unit_size))
ind_ious_mmap = ed.open_mmap(blob=blob, part='single_ind_ious%s' % suffix, mode='w+',
dtype='float32', shape=(num_labels, tot_imgs, unit_size))
if start is None:
start = 1
if end is None:
end = num_labels
#for label_i in range(1, num_labels):
for label_i in range(start, end):
print('Starting for label %d (%s)' % (label_i, ds.name(category=None,
j=label_i)))
label_categories = ds.label[label_i]['category'].keys()
num_cats = len(label_categories)
label_idx = np.where(image_to_label[:, label_i])[0]
loader = loadseg.SegmentationPrefetcher(ds, categories=label_categories,
indexes=label_idx, once=False, batch_size=batch_size,
ahead=ahead, thread=True)
loader_idx = loader.indexes
N = len(loader_idx)
iou_intersects = np.zeros((N, unit_size))
iou_unions = np.zeros((N, unit_size))
if num_cats > 1:
rec_labcat = []
for batch in loader.batches():
for rec in batch:
for cat in label_categories:
if rec[cat] != []:
rec_labcat.append(cat)
break
else:
rec_labcat = [label_categories[0] for i in range(N)]
i = 0
for batch in loader.batches():
start_t = time.time()
if (i+1)*batch_size < N:
idx = range(i*batch_size, (i+1)*batch_size)
else:
idx = range(i*batch_size, N)
i += 1
input = torch.Tensor((blobdata[loader_idx[idx]] > thresh).astype(float))
input_var = upsample(Variable(input.cuda()) if cuda else
Variable(input))
target = torch.Tensor([np.max((rec[rec_labcat[j]]
== label_i).astype(float), axis=0)
if type(rec[rec_labcat[j]]) is np.ndarray
else np.ones(seg_size) for j, rec in enumerate(batch)])
target_var = Variable(target.unsqueeze(1).expand_as(
input_var).cuda() if cuda
else target.unsqueeze(1).expand_as(input_var))
iou_intersects[idx] = np.squeeze(iou_intersect_d(input_var,
target_var).data.cpu().numpy())
iou_unions[idx] = np.squeeze(iou_union_d(input_var,
target_var).data.cpu().numpy())
print('Batch %d/%d\tTime %f secs\tAvg Ind IOU %f\t' % (i, N/batch_size,
time.time()-start_t, np.mean(np.true_divide(iou_intersects[idx],
iou_unions[idx] + 1e-20))))
set_ious = np.true_divide(np.sum(iou_intersects, axis=0),
np.sum(iou_unions, axis=0) + 1e-20)
loader.close()
best_filter = np.argmax(set_ious)
print('Label %d (%s): best set IOU = %f (filter %d)' % (label_i,
ds.name(category=None,j=label_i), set_ious[best_filter], best_filter))
ind_ious = np.true_divide(iou_intersects, iou_unions + 1e-20)
set_ious_mmap[label_i] = set_ious
ind_ious_mmap[label_i, loader_idx] = ind_ious
train_idx = [i for i in range(len(loader_idx)) if ds.split(loader_idx[i]) == 'train']
val_idx = [i for i in range(len(loader_idx)) if ds.split(loader_idx[i]) == 'val']
set_ious_train_mmap[label_i] = np.true_divide(np.sum(iou_intersects[train_idx], axis=0),
np.sum(iou_unions[train_idx], axis=0) + 1e-20)
set_ious_val_mmap[label_i] = np.true_divide(np.sum(iou_intersects[val_idx], axis=0),
np.sum(iou_unions[val_idx], axis=0) + 1e-20)
#set_ious_mmap.flush()
#ind_ious_mmap.flush()
ed.finish_mmap(set_ious_train_mmap)
ed.finish_mmap(set_ious_val_mmap)
ed.finish_mmap(set_ious_mmap)
ed.finish_mmap(ind_ious_mmap)
