def compute_alpha(directory):
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
ds = loadseg.SegmentationData(info.dataset)
L = ds.label_size()
if not has_image_to_label(directory):
create_image_to_label(directory)
image_to_label = load_image_to_label(directory)
label_names = np.array([ds.label[i]['name'] for i in range(L)])
alphas = np.zeros((L, ))
for label_i in range(1, L):
label_categories = ds.label[label_i]['category'].keys()
label_idx = np.where(image_to_label[:, label_i])[0]
train_loader = loadseg.SegmentationPrefetcher(
ds,
categories=label_categories,
split='train',
indexes=label_idx,
once=True,
batch_size=64,
ahead=4,
thread=True)
train_idx = np.array(train_loader.indexes)
#sw = 0
#sh = 0
perc_label = []
#train_label_categories = []
for batch in train_loader.batches():
for rec in batch:
sw, sh = [rec[k] for k in ['sw', 'sh']]
#sw_r, sh_r = [rec[k] for k in ['sw', 'sh']]
#if sw == 0 and sh == 0:
# sw = sw_r
# sh = sh_r
#else:
# assert(sw == sw_r and sh == sh_r)
for cat in label_categories:
if rec[cat] != []:
#train_label_categories.append(cat)
if type(rec[cat]) is np.ndarray:
perc_label.append(
np.sum(rec[cat] == label_i) / float(sw * sh))
else:
perc_label.append(1.)
break
assert (len(perc_label) == len(train_idx))
alphas[label_i] = float(1. - np.mean(perc_label))
print label_i, label_names[label_i], alphas[label_i]
train_loader.close()
alphas_mmap = ed.open_mmap(part='train_alphas',
mode='w+',
dtype='float32',
shape=alphas.shape)
alphas_mmap[:] = alphas[:]
ed.finish_mmap(alphas_mmap)
def save_neural_statistics(directory, blob, split=None):
ed = expdir.ExperimentDirectory(directory)
info = ed.load_info()
dataset = info.dataset
blob_info = ed.load_info(blob=blob)
if 'broden' in dataset:
assert (split is not None)
suffix = '_%s' % split
else:
suffix = ''
if ed.has_mmap(blob=blob, part='act_probs%s' % suffix):
print('%s already exists' %
ed.mmap_filename(blob=blob, part='act_probs%s' % suffix))
return
acts = ed.open_mmap(blob=blob,
mode='r',
dtype='float32',
shape=blob_info.shape)
if 'broden' in dataset:
ds = loadseg.SegmentationData(dataset)
split_ind = np.array([
True if ds.split(i) == split else False for i in range(ds.size())
])
L = ds.label_size()
elif 'imagenet' in dataset or 'ILSVRC' in dataset:
assert (split is None)
L = 1000
split_ind = True
image_to_label = load_image_to_label(directory, blob=blob)
K = blob_info.shape[1]
probs = np.zeros((L, K))
mus = np.zeros((L, K))
sigmas = np.zeros((L, K))
for class_i in range(L):
class_idx = np.where(split_ind & image_to_label[:, class_i])[0]
if len(class_idx) == 0:
probs[class_i, :] = np.nan
mus[class_i, :] = np.nan
sigmas[class_i, :] = np.nan
print class_i, 'no examples'
continue
print class_i
max_acts = np.amax(acts[class_idx], axis=(2, 3))
for filter_i in range(blob_info.shape[1]):
nz_idx = np.where(max_acts[:, filter_i] > 0)[0]
probs[class_i][filter_i] = len(nz_idx) / float(
len(max_acts[:, filter_i]))
try:
mus[class_i][filter_i] = np.mean(max_acts[nz_idx, filter_i])
sigmas[class_i][filter_i] = np.std(max_acts[nz_idx, filter_i])
except:
mus[class_i][filter_i] = np.nan
sigmas[class_i][filter_i] = np.nan
print class_i, filter_i, 'no nonzero idx'
probs_mmap = ed.open_mmap(blob=blob,
part='act_probs%s' % suffix,
mode='w+',
dtype='float32',
shape=probs.shape)
mus_mmap = ed.open_mmap(blob=blob,
part='act_mus%s' % suffix,
mode='w+',
dtype='float32',
shape=mus.shape)
sigmas_mmap = ed.open_mmap(blob=blob,
part='act_sigmas%s' % suffix,
mode='w+',
dtype='float32',
shape=sigmas.shape)
probs_mmap[:] = probs[:]
mus_mmap[:] = mus[:]
sigmas_mmap[:] = sigmas[:]
ed.finish_mmap(probs_mmap)
ed.finish_mmap(mus_mmap)
ed.finish_mmap(sigmas_mmap)
print('Finished saving neural statistics for %s' % blob)
for blob in blobs:
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
K = shape[1]
quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(K, -1))
threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))]
thresh[blob] = np.copy(threshold[:, np.newaxis, np.newaxis])
train_idx = np.array([ds.split(i) == 'train' for i in range(N)])
val_idx = np.array([ds.split(i) == 'val' for i in range(N)])
train_ind = np.array([True if ds.split(i) == 'train' else False for i in range(N)])
val_ind = np.array([True if ds.split(i) == 'val' else False for i in range(N)])
image_to_label = load_image_to_label(directory)
pc = ds.primary_categories_per_index()
pc[0] = -1
categories = np.array(ds.category_names())
print categories
suffix = '_lr_1e-1_sgd_quant_1_epochs_30_iter_15_num_filters_2'
#num_filters = [1, 2, 3, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 80, 100, 128]
#F = len(num_filters) + 1
F = 2
epochs = 30
disc_weights = {}
disc_bias = {}
disc_results = {}
def linear_probe_discriminative(directory, blob, label_i, suffix='', batch_size=16,
quantile=0.005, bias=False, pool='avg_pool', positive=False,
num_filters=None, init_suffix='',
num_epochs=30, epoch_iter=10, lr=1e-4, momentum=0.9,
l1_weight_decay=0, l2_weight_decay=0, nesterov=False,
lower_bound=None, optimizer_type='sgd', cuda=False,
fig_path=None, show_fig=True):
ed = expdir.ExperimentDirectory(directory)
try:
info = ed.load_info()
except:
ed = expdir.ExperimentDirectory(os.path.join(directory, 'train'))
ed_val = expdir.ExperimentDirectory(os.path.join(directory, 'val'))
info = ed.load_info()
assert('imagenet' in info.dataset or 'ILSVRC' in info.dataset)
# Check if linear weights have already been learned
if ed.has_mmap(blob=blob, part='label_i_%d_weights_disc%s' % (label_i, suffix)):
print('%s already has %s, so skipping.' % (directory,
ed.mmap_filename(blob=blob, part='label_i_%d_weights_disc%s' % (
label_i, suffix))))
return
# Load probe metadata
info = ed.load_info()
ih, iw = info.input_dim
# Load blob metadata
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
unit_size = shape[1]
fieldmap = blob_info.fieldmap
# Load the blob quantile data and grab thresholds
if quantile == 1:
if len(shape) == 4:
thresh = np.zeros((unit_size, 1, 1))
elif len(shape) == 2:
thresh = np.zeros(unit_size)
else:
assert(False)
else:
quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(unit_size, -1))
threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))]
if len(shape) == 4:
thresh = threshold[:, np.newaxis, np.newaxis]
elif len(shape) == 2:
thresh = threshold
else:
assert(False)
# print np.max(thresh), thresh.shape, type(thresh)
# Map the blob activation data for reading
fn_read = ed.mmap_filename(blob=blob)
blobdata = cached_memmap(fn_read, mode='r', dtype='float32', shape=shape)
if 'broden' in info.dataset:
# Load the dataset
ds = loadseg.SegmentationData(info.dataset)
# Get all the categories the label is a part of
label_categories = ds.label[label_i]['category'].keys()
num_categories = len(label_categories)
# Get label name
label_name = ds.name(category=None, j=label_i)
# Get indices of images containing the given label
if not has_image_to_label(directory):
print('image_to_label does not exist in %s; creating it now...' % directory)
create_image_to_label(directory, batch_size=batch_size)
image_to_label = load_image_to_label(directory)
label_idx = np.where(image_to_label[:, label_i])[0]
non_label_idx = np.where(image_to_label[:, label_i] == 0)[0]
print('Number of positive and negative examples of label %d (%s): %d %d' % (
label_i, label_name, len(label_idx), len(non_label_idx)))
val_blobdata = blobdata
elif 'imagenet' in info.dataset or 'ILSVRC' in info.dataset:
label_desc = np.loadtxt('synset_words.txt', str, delimiter='\t')
label_name = ' '.join(label_desc[label_i].split(',')[0].split()[1:])
image_to_label_train = load_image_to_label(os.path.join(directory, 'train'), blob=blob)
train_label_idx = np.where(image_to_label_train[:, label_i])[0]
train_non_label_idx = np.where(1-image_to_label_train[:, label_i])[0]
image_to_label_val = load_image_to_label(os.path.join(directory, 'val'), blob=blob)
val_label_idx = np.where(image_to_label_val[:, label_i])[0]
val_non_label_idx = np.where(1-image_to_label_val[:, label_i])[0]
print('Number of positive and negative examples of label %d (%s): %d %d %d %d' %
(label_i, label_name, len(train_label_idx), len(train_non_label_idx),
len(val_label_idx), len(val_non_label_idx)))
val_blob_info = ed_val.load_info(blob=blob)
val_shape = val_blob_info.shape
val_fn_read = ed_val.mmap_filename(blob=blob)
val_blobdata = cached_memmap(val_fn_read, mode='r', dtype='float32', shape=val_shape)
else:
assert(False)
criterion = torch.nn.BCEWithLogitsLoss()
if num_filters is not None:
init_weights_mmap = ed.open_mmap(blob=blob, part='label_i_%d_weights_disc%s' % (label_i, init_suffix),
mode='r', dtype='float32', shape=unit_size)
sorted_idx = np.argsort(np.abs(init_weights_mmap))[::-1]
mask_idx = np.zeros(unit_size, dtype=int)
mask_idx[sorted_idx[:num_filters]] = 1
layer = CustomDiscLayer(unit_size, act=False, pool=pool, bias=bias, positive=positive,
mask_idx=torch.ByteTensor(mask_idx), cuda=cuda)
else:
layer = CustomDiscLayer(unit_size, act=False, pool=pool, bias=bias, positive=positive, cuda=cuda)
if cuda:
criterion.cuda()
layer.cuda()
if optimizer_type == 'sgd':
optimizer = Custom_SGD(layer.parameters(), lr, momentum,
l1_weight_decay=l1_weight_decay, l2_weight_decay=l2_weight_decay,
nesterov=nesterov, lower_bound=lower_bound)
elif optimizer_type == 'adam':
optimizer = Custom_Adam(layer.parameters(), lr, l1_weight_decay=l1_weight_decay,
l2_weight_decay=l2_weight_decay, lower_bound=lower_bound)
if 'broden' in info.dataset:
train_label_idx = []
val_label_idx = []
for ind in label_idx:
if ds.split(ind) == 'train':
train_label_idx.append(ind)
elif ds.split(ind) == 'val':
val_label_idx.append(ind)
train_non_label_idx = []
val_non_label_idx = []
for ind in non_label_idx:
if ds.split(ind) == 'train':
train_non_label_idx.append(ind)
elif ds.split(ind) == 'val':
val_non_label_idx.append(ind)
train_label_idx = np.array(train_label_idx)
val_label_idx = np.array(val_label_idx)
train_non_label_idx = np.array(train_non_label_idx)
val_non_label_idx = np.array(val_non_label_idx)
num_train_labels = len(train_label_idx)
num_train_non_labels = len(train_non_label_idx)
train_neg_greater = num_train_labels < num_train_non_labels
if len(val_label_idx) < len(val_non_label_idx):
pos_val_idx = np.arange(len(val_label_idx))
neg_val_idx = np.random.choice(len(val_non_label_idx), len(val_label_idx), replace=False)
else:
pos_val_idx = np.random.choice(len(val_label_idx), len(val_non_label_idx), replace=False)
neg_val_idx = np.arange(len(val_non_label_idx))
val_indexes = np.concatenate((val_label_idx[pos_val_idx], val_non_label_idx[neg_val_idx]))
val_targets = np.concatenate((np.ones(len(pos_val_idx)), np.zeros(len(neg_val_idx))))
print 'Number of train and val examples: %d %d' % (min(2*num_train_non_labels, 2*num_train_labels),
len(val_targets))
train_losses = []
train_accs = []
val_losses = []
val_accs = []
if 'imagenet' in info.dataset or 'ILSVRC' in info.dataset:
if train_neg_greater:
pos_train_idx = np.arange(num_train_labels)
neg_train_idx = np.random.choice(num_train_non_labels, num_train_labels, replace=False)
else:
pos_train_idx = np.random.choice(num_train_labels, num_train_non_labels, replace=False)
neg_train_idx = np.arange(num_train_non_labels)
train_indexes = np.concatenate((train_label_idx[pos_train_idx], train_non_label_idx[neg_train_idx]))
train_targets = np.concatenate((np.ones(len(pos_train_idx)), np.zeros(len(neg_train_idx))))
rand_idx = np.random.permutation(len(train_targets))
for t in range(num_epochs):
if epoch_iter is not None:
adjust_learning_rate(lr, optimizer, t, epoch_iter=epoch_iter)
if 'broden' in info.dataset:
if train_neg_greater:
pos_train_idx = np.arange(num_train_labels)
neg_train_idx = np.random.choice(num_train_non_labels, num_train_labels, replace=False)
else:
pos_train_idx = np.random.choice(num_train_labels, num_train_non_labels, replace=False)
neg_train_idx = np.arange(num_train_non_labels)
train_indexes = np.concatenate((train_label_idx[pos_train_idx], train_non_label_idx[neg_train_idx]))
train_targets = np.concatenate((np.ones(len(pos_train_idx)), np.zeros(len(neg_train_idx))))
rand_idx = np.random.permutation(len(train_targets))
(train_loss, train_acc) = run_epoch(blobdata, train_targets[rand_idx], train_indexes[rand_idx], thresh, layer, criterion,
optimizer, t, batch_size=batch_size, train=True, cuda=cuda)
(val_loss, val_acc) = run_epoch(val_blobdata, val_targets, val_indexes, thresh, layer, criterion,
optimizer, t, batch_size=batch_size, train=False, cuda=cuda)
train_losses.append(train_loss)
train_accs.append(train_acc)
val_losses.append(val_loss)
val_accs.append(val_acc)
if fig_path is not None or show_fig:
f, ax = plt.subplots(1,2, figsize=(8,4))
ax[0].plot(range(num_epochs), train_losses, label='train')
ax[0].plot(range(num_epochs), val_losses, label='val')
ax[0].set_title('BCE Loss (train=%d, val=%d)' % (len(train_targets), len(val_targets)))
ax[1].plot(range(num_epochs), train_accs, label='train')
ax[1].plot(range(num_epochs), val_accs, label='val')
ax[1].set_title('Accuracy (%s, %s)' % (label_name, blob))
plt.legend()
if fig_path is not None:
plt.savefig(fig_path)
if show_fig:
plt.show()
plt.close()
# Save weights
weights = (layer.mask * layer.weight).data.cpu().numpy()
print np.sum(weights != 0)
weights_mmap = ed.open_mmap(blob=blob, part='label_i_%d_weights_disc%s' % (label_i, suffix),
mode='w+', dtype='float32', shape=weights.shape)
weights_mmap[:] = weights[:]
ed.finish_mmap(weights_mmap)
if bias:
bias_v = layer.bias.data.cpu().numpy()
bias_mmap = ed.open_mmap(blob=blob, part='label_i_%d_bias_disc%s' % (label_i, suffix),
mode='w+', dtype='float32', shape=(1,))
bias_mmap[:] = bias_v[:]
ed.finish_mmap(bias_mmap)
results_mmap = ed.open_mmap(blob=blob, part='label_i_%d_results_disc%s' % (label_i, suffix),
mode='w+', dtype='float32', shape=(4,num_epochs))
results_mmap[0] = train_losses
results_mmap[1] = val_losses
results_mmap[2] = train_accs
results_mmap[3] = val_accs
ed.finish_mmap(results_mmap)
def linear_probe(directory, blob, label_i, suffix='', init_suffix='', num_filters=None, batch_size=16, ahead=4,
quantile=0.005, bias=False, positive=False, num_epochs=30, lr=1e-4, momentum=0.9,
l1_weight_decay=0, l2_weight_decay=0, validation=False, nesterov=False, lower_bound=None,
min_train=None, max_train=None, max_val=None,
cuda=False):
# Make sure we have a directory to work in
#qcode = ('%f' % quantile).replace('0.','').rstrip('0')
ed = expdir.ExperimentDirectory(directory)
# Check if linear weights have already been learned
print ed.mmap_filename(blob=blob, part='label_i_%s_weights%s' % (label_i, suffix))
if ed.has_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix)):
print('%s already has %s, so skipping.' % (directory,
ed.mmap_filename(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix))))
return
# Load probe metadata
info = ed.load_info()
ih, iw = info.input_dim
# Load blob metadata
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
unit_size = shape[1]
fieldmap = blob_info.fieldmap
# Load the blob quantile data and grab thresholds
if quantile == 1:
thresh = np.zeros((unit_size,1,1))
else:
quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(unit_size, -1))
threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))]
thresh = threshold[:, np.newaxis, np.newaxis]
#print np.max(thresh), thresh.shape, type(thresh)
# Map the blob activation data for reading
fn_read = ed.mmap_filename(blob=blob)
# Load the dataset
ds = loadseg.SegmentationData(info.dataset)
# Get all the categories the label is a part of
label_categories = ds.label[label_i]['category'].keys()
num_categories = len(label_categories)
# Get label name
label_name = ds.name(category=None, j=label_i)
blobdata = cached_memmap(fn_read, mode='r', dtype='float32', shape=shape)
# Get indices of images containing the given label
if not has_image_to_label(directory):
print('image_to_label does not exist in %s; creating it now...' % directory)
create_image_to_label(directory, batch_size=batch_size, ahead=ahead)
image_to_label = load_image_to_label(directory)
label_idx = np.where(image_to_label[:, label_i])[0]
train_idx = np.array([i for i in label_idx if ds.split(i) == 'train'])
val_idx = np.array([i for i in label_idx if ds.split(i) == 'val'])
if min_train is not None and len(train_idx) < min_train:
print('Number of training examples for label %d (%s) is %d, which is less than the minimum of %d so skipping.'
% (label_i, label_name, len(train_idx), min_train))
if max_train is not None and len(train_idx) > max_train:
train_idx = train_idx[:max_train]
if max_val is not None and len(val_idx) > max_val:
val_idx = val_idx[:max_val]
print('Total number of images containing label %d (%s): %d' % (
label_i, label_name, len(label_idx)))
try:
train_loader = loadseg.SegmentationPrefetcher(ds, categories=label_categories,
indexes=train_idx, once=False,
batch_size=batch_size,
ahead=ahead, thread=True)
except IndexError as err:
print(err.args)
return
sw = 0
sh = 0
perc_label = []
train_label_categories = []
for batch in train_loader.batches():
for rec in batch:
# Check that the same segmentation dimensions are used for all
# examples
sw_r, sh_r = [rec[k] for k in ['sw', 'sh']]
if sw == 0 and sh == 0:
sw = sw_r
sh = sh_r
else:
assert(sw == sw_r and sh == sh_r)
for cat in label_categories:
if rec[cat] != []:
train_label_categories.append(cat)
if type(rec[cat]) is np.ndarray:
perc_label.append(np.sum(rec[cat] == label_i) / float(sw * sh))
else:
perc_label.append(1.)
break
assert(len(perc_label) == len(train_idx))
# Compute reduction from segmentation dimensions to image dimensions
reduction = int(round(iw / float(sw)))
# Calculate class-weighting alpha parameter for segmentation loss
# (Note: float typecast is necessary)
alpha = float(1. - np.mean(perc_label))
if alpha == 0:
alpha = None
print('Not using class-weighting because no pixel-level annotations')
else:
print('Alpha for label %d (%s): %f' % (label_i, label_name, alpha))
# Prepare segmentation loss function using class-weight alpha
criterion = lambda x,y: BCELoss2d(x,y,alpha)
# Prepare to learn linear weights with a sigmoid activation after
# the linear layer
#layer = CustomLayer(unit_size, upsample=False, act=True, positive=False)
if num_filters is not None:
if ed.has_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, init_suffix)):
init_weights_mmap = ed.open_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, init_suffix),
mode='r', dtype='float32', shape=unit_size)
elif ed.has_mmap(blob=blob, part='linear_weights%s' % (init_suffix)):
all_weights_mmap = ed.open_mmap(blob=blob, part='linear_weights%s' % init_suffix,
mode='r', dtype='float32', shape=(ds.label_size(),unit_size))
init_weights_mmap = all_weights_mmap[label_i]
else:
assert(False)
sorted_idx = np.argsort(np.abs(init_weights_mmap))[::-1]
mask_idx = np.zeros(unit_size, dtype=int)
mask_idx[sorted_idx[:num_filters]] = 1
layer = CustomLayer(unit_size, upsample=True, up_size=(sh,sw), act=True,
bias=bias, positive=positive, mask_idx=torch.ByteTensor(mask_idx), cuda=cuda)
else:
layer = CustomLayer(unit_size, upsample=True, up_size=(sh,sw), act=True,
bias=bias, positive=positive, cuda=cuda)
if cuda:
layer.cuda()
optimizer = Custom_SGD(layer.parameters(), lr, momentum,
l1_weight_decay=l1_weight_decay, l2_weight_decay=l2_weight_decay,
nesterov=nesterov, lower_bound=lower_bound)
if not validation:
try:
val_loader = loadseg.SegmentationPrefetcher(ds, categories=label_categories,
indexes=val_idx, once=False, batch_size=batch_size,
ahead=ahead, thread=True)
except IndexError as err:
print(err.args)
train_loader.close()
return
val_label_categories = []
for batch in val_loader.batches():
for rec in batch:
for cat in label_categories:
if rec[cat] != []:
val_label_categories.append(cat)
break
assert(len(val_label_categories) == len(val_idx))
for t in range(num_epochs):
(_, iou) = run_epoch(blobdata, train_idx, train_label_categories, label_i,
fieldmap, thresh, sh, sw, reduction, train_loader, layer, criterion,
optimizer, t+1, train=True, cuda=cuda, iou_threshold=0.5)
if not validation:
(_, iou) = run_epoch(blobdata, val_idx, val_label_categories, label_i,
fieldmap, thresh, sh, sw, reduction, val_loader, layer, criterion,
optimizer, t+1, train=False, cuda=cuda, iou_threshold=0.5)
# Close segmentation prefetcher (i.e. close pools)
train_loader.close()
if not validation:
val_loader.close()
# Save weights
weights = (layer.mask * layer.weight).data.cpu().numpy()
weights_mmap = ed.open_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix),
mode='w+', dtype='float32', shape=weights.shape)
weights_mmap[:] = weights[:]
ed.finish_mmap(weights_mmap)
if bias:
bias_v = layer.bias.data.cpu().numpy()
bias_mmap = ed.open_mmap(blob=blob, part='label_i_%d_bias%s' % (label_i, suffix),
mode='w+', dtype='float32', shape=(1,))
bias_mmap[:] = bias_v[:]
ed.finish_mmap(bias_mmap)
print '%s finished' % ed.mmap_filename(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix))
def probe_linear(directory,
blob,
suffix='',
start=None,
end=None,
batch_size=16,
ahead=4,
quantile=0.005,
bias=False,
positive=False,
cuda=False,
force=False):
qcode = ('%f' % quantile).replace('0.', '.').rstrip('0')
ed = expdir.ExperimentDirectory(directory)
if (ed.has_mmap(blob=blob, part='linear_ind_ious%s' % suffix)
and ed.has_mmap(blob=blob, part='linear_set_ious%s' % suffix)):
print('Linear weights have already been probed.')
print ed.mmap_filename(blob=blob,
part='linear_set_val_ious%s' % suffix)
if not force:
return
else:
print('Forcefully continuing...')
info = ed.load_info()
seg_size = get_seg_size(info.input_dim)
blob_info = ed.load_info(blob=blob)
ds = loadseg.SegmentationData(info.dataset)
shape = blob_info.shape
N = shape[0] # number of total images
K = shape[1] # number of units in given blob
L = ds.label_size() # number of labels
if quantile == 1:
thresh = np.zeros((K, 1, 1))
else:
quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(K, -1))
threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))]
thresh = threshold[:, np.newaxis, np.newaxis]
fn_read = ed.mmap_filename(blob=blob)
blobdata = cached_memmap(fn_read, mode='r', dtype='float32', shape=shape)
image_to_label = load_image_to_label(directory)
if ed.has_mmap(blob=blob, part='linear_ind_ious%s' % suffix,
inc=True) and not force:
assert (ed.has_mmap(blob=blob,
part='linear_set_ious%s' % suffix,
inc=True))
ind_ious = ed.open_mmap(blob=blob,
part='linear_ind_ious%s' % suffix,
mode='r+',
inc=True,
dtype='float32',
shape=(L, N))
set_ious = ed.open_mmap(blob=blob,
part='linear_set_ious%s' % suffix,
mode='r+',
inc=True,
dtype='float32',
shape=(L, ))
set_ious_train = ed.open_mmap(blob=blob,
part='linear_set_train_ious%s' % suffix,
mode='r+',
inc=True,
dtype='float32',
shape=(L, ))
try:
set_ious_val = ed.open_mmap(blob=blob,
part='linear_set_val_ious%s' % suffix,
mode='r+',
inc=True,
dtype='float32',
shape=(L, ))
except:
set_ious_val = ed.open_mmap(blob=blob,
part='linear_set_val_ious%s' % suffix,
mode='r+',
dtype='float32',
shape=(L, ))
else:
ind_ious = ed.open_mmap(blob=blob,
part='linear_ind_ious%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, N))
set_ious = ed.open_mmap(blob=blob,
part='linear_set_ious%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, ))
set_ious_train = ed.open_mmap(blob=blob,
part='linear_set_train_ious%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, ))
set_ious_val = ed.open_mmap(blob=blob,
part='linear_set_val_ious%s' % suffix,
mode='w+',
dtype='float32',
shape=(L, ))
if start is None:
start = 1
if end is None:
end = L
for label_i in range(start, end):
if ed.has_mmap(blob=blob,
part='label_i_%d_weights%s' % (label_i, suffix)):
try:
weights = ed.open_mmap(blob=blob,
part='label_i_%d_weights%s' %
(label_i, suffix),
mode='r',
dtype='float32',
shape=(K, ))
except ValueError:
# SUPPORTING LEGACY CODE (TODO: Remove)
weights = ed.open_mmap(blob=blob,
part='label_i_%d_weights%s' %
(label_i, suffix),
mode='r',
dtype=float,
shape=(K, ))
elif ed.has_mmap(blob=blob, part='linear_weights%s' % suffix):
all_weights = ed.open_mmap(blob=blob,
part='linear_weights%s' % suffix,
mode='r',
dtype='float32',
shape=(L, K))
weights = all_weights[label_i]
if not np.any(weights):
print(
'Label %d does not have associated weights to it, so skipping.'
% label_i)
continue
else:
print(
'Label %d does not have associated weights to it, so skipping.'
% label_i)
continue
if bias:
if ed.has_mmap(blob=blob,
part='label_i_%d_bias%s' % (label_i, suffix)):
bias_v = ed.open_mmap(blob=blob,
part='label_i_%d_bias%s' %
(label_i, suffix),
mode='r',
dtype='float32',
shape=(1, ))
else:
assert (ed.has_mmap(blob=blob, part='linear_bias%s' % suffix))
all_bias_v = ed.open_mmap(blob=blob,
part='linear_bias%s' % suffix,
mode='r',
dtype='float32',
shape=(L, ))
bias_v = np.array([all_bias_v[label_i]])
label_categories = ds.label[label_i]['category'].keys()
label_name = ds.name(category=None, j=label_i)
label_idx = np.where(image_to_label[:, label_i])[0]
loader = loadseg.SegmentationPrefetcher(ds,
categories=label_categories,
indexes=label_idx,
once=True,
batch_size=batch_size,
ahead=ahead,
thread=True)
loader_idx = loader.indexes
num_imgs = len(loader.indexes)
print(
'Probing with learned weights for label %d (%s) with %d images...'
% (label_i, label_name, num_imgs))
model = CustomLayer(K,
upsample=True,
up_size=seg_size,
act=True,
bias=bias,
positive=positive,
cuda=cuda)
model.weight.data[...] = torch.Tensor(weights)
if bias:
model.bias.data[...] = torch.Tensor(bias_v)
if cuda:
model.cuda()
model.eval()
iou_intersects = np.zeros(num_imgs)
iou_unions = np.zeros(num_imgs)
i = 0
for batch in loader.batches():
start_t = time.time()
if (i + 1) * batch_size < num_imgs:
idx = range(i * batch_size, (i + 1) * batch_size)
else:
idx = range(i * batch_size, num_imgs)
i += 1
input = torch.Tensor(
(blobdata[loader_idx[idx]] > thresh).astype(float))
input_var = (Variable(input.cuda(), volatile=True)
if cuda else Variable(input, volatile=True))
target_ = []
for rec in batch:
for cat in label_categories:
if rec[cat] != []:
if type(rec[cat]) is np.ndarray:
target_.append(
np.max((rec[cat] == label_i).astype(float),
axis=0))
else:
target_.append(np.ones(seg_size))
break
target = torch.Tensor(target_)
target_var = (Variable(target.cuda(), volatile=True)
if cuda else Variable(target, volatile=True))
#target_var = Variable(target.unsqueeze(1).expand_as(
# input_var).cuda() if cuda
# else target.unsqueeze(1).expand_as(input_var))
output_var = model(input_var)
iou_intersects[idx] = np.squeeze(
iou_intersect_d(output_var, target_var).data.cpu().numpy())
iou_unions[idx] = np.squeeze(
iou_union_d(output_var, target_var).data.cpu().numpy())
print('Batch: %d/%d\tTime: %f secs\tAvg Ind IOU: %f' %
(i, num_imgs / batch_size, time.time() - start_t,
np.mean(
np.true_divide(iou_intersects[idx],
iou_unions[idx] + 1e-20))))
loader.close()
label_ind_ious = np.true_divide(iou_intersects, iou_unions + 1e-20)
label_set_iou = np.true_divide(np.sum(iou_intersects),
np.sum(iou_unions) + 1e-20)
ind_ious[label_i, loader_idx] = label_ind_ious
set_ious[label_i] = label_set_iou
train_idx = [
i for i in range(len(loader_idx))
if ds.split(loader_idx[i]) == 'train'
]
val_idx = [
i for i in range(len(loader_idx))
if ds.split(loader_idx[i]) == 'val'
]
set_ious_train[label_i] = np.true_divide(
np.sum(iou_intersects[train_idx]),
np.sum(iou_unions[train_idx]) + 1e-20)
set_ious_val[label_i] = np.true_divide(
np.sum(iou_intersects[val_idx]),
np.sum(iou_unions[val_idx]) + 1e-20)
print(
'Label %d (%s) Set IOU: %f, Train Set IOU: %f, Val Set IOU: %f, Max Ind IOU: %f'
% (label_i, label_name, label_set_iou, set_ious_train[label_i],
set_ious_val[label_i], np.max(label_ind_ious)))
ed.finish_mmap(ind_ious)
ed.finish_mmap(set_ious)
ed.finish_mmap(set_ious_train)
ed.finish_mmap(set_ious_val)
def label_probe(directory, blob, quantile=0.005, batch_size=16, ahead=4, start=None,
end=None, suffix='', cuda=False):
# Make sure we have a directory to work in
qcode = ('%f' % quantile).replace('0.','').rstrip('0')
ed = expdir.ExperimentDirectory(directory)
# Check if label probe has already been created
if (ed.has_mmap(blob=blob, part='single_set_ious%s' % suffix) and
ed.has_mmap(blob=blob, part='single_ind_ious%s' % suffix)):
print('label_probe_pytorch.py has already been run.')
return
# Load probe metadata
info = ed.load_info()
seg_size = get_seg_size(info.input_dim)
# Load blob metadata
blob_info = ed.load_info(blob=blob)
shape = blob_info.shape
tot_imgs = shape[0]
unit_size = shape[1]
# Load the blob quantile data and grab thresholds
quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(unit_size, -1))
threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))]
thresh = threshold[:, np.newaxis, np.newaxis]
# Load the dataset
ds = loadseg.SegmentationData(info.dataset)
# Map the blob activation data for reading
#fn_read = ed.mmap_filename(blob=blob)
#blobdata = cached_memmap(fn_read, mode='r', dtype='float32', shape=shape)
blobdata = ed.open_mmap(blob=blob, mode='r', shape=shape)
# Get image-to-labels mapping
if not has_image_to_label(directory):
print('image_to_label does not exist in %s; creating it now...' % directory)
create_image_to_label(directory, batch_size=batch_size, ahead=ahead)
image_to_label = load_image_to_label(directory)
num_labels = ds.label_size()
upsample = nn.Upsample(size=seg_size, mode='bilinear')
set_ious_train_mmap = ed.open_mmap(blob=blob, part='single_set_train_ious%s' % suffix,
mode='w+', dtype='float32', shape=(num_labels, unit_size))
set_ious_val_mmap = ed.open_mmap(blob=blob, part='single_set_val_ious%s' % suffix,
mode='w+', dtype='float32', shape=(num_labels, unit_size))
set_ious_mmap = ed.open_mmap(blob=blob, part='single_set_ious%s' % suffix, mode='w+',
dtype='float32', shape=(num_labels, unit_size))
ind_ious_mmap = ed.open_mmap(blob=blob, part='single_ind_ious%s' % suffix, mode='w+',
dtype='float32', shape=(num_labels, tot_imgs, unit_size))
if start is None:
start = 1
if end is None:
end = num_labels
#for label_i in range(1, num_labels):
for label_i in range(start, end):
print('Starting for label %d (%s)' % (label_i, ds.name(category=None,
j=label_i)))
label_categories = ds.label[label_i]['category'].keys()
num_cats = len(label_categories)
label_idx = np.where(image_to_label[:, label_i])[0]
loader = loadseg.SegmentationPrefetcher(ds, categories=label_categories,
indexes=label_idx, once=False, batch_size=batch_size,
ahead=ahead, thread=True)
loader_idx = loader.indexes
N = len(loader_idx)
iou_intersects = np.zeros((N, unit_size))
iou_unions = np.zeros((N, unit_size))
if num_cats > 1:
rec_labcat = []
for batch in loader.batches():
for rec in batch:
for cat in label_categories:
if rec[cat] != []:
rec_labcat.append(cat)
break
else:
rec_labcat = [label_categories[0] for i in range(N)]
i = 0
for batch in loader.batches():
start_t = time.time()
if (i+1)*batch_size < N:
idx = range(i*batch_size, (i+1)*batch_size)
else:
idx = range(i*batch_size, N)
i += 1
input = torch.Tensor((blobdata[loader_idx[idx]] > thresh).astype(float))
input_var = upsample(Variable(input.cuda()) if cuda else
Variable(input))
target = torch.Tensor([np.max((rec[rec_labcat[j]]
== label_i).astype(float), axis=0)
if type(rec[rec_labcat[j]]) is np.ndarray
else np.ones(seg_size) for j, rec in enumerate(batch)])
target_var = Variable(target.unsqueeze(1).expand_as(
input_var).cuda() if cuda
else target.unsqueeze(1).expand_as(input_var))
iou_intersects[idx] = np.squeeze(iou_intersect_d(input_var,
target_var).data.cpu().numpy())
iou_unions[idx] = np.squeeze(iou_union_d(input_var,
target_var).data.cpu().numpy())
print('Batch %d/%d\tTime %f secs\tAvg Ind IOU %f\t' % (i, N/batch_size,
time.time()-start_t, np.mean(np.true_divide(iou_intersects[idx],
iou_unions[idx] + 1e-20))))
set_ious = np.true_divide(np.sum(iou_intersects, axis=0),
np.sum(iou_unions, axis=0) + 1e-20)
loader.close()
best_filter = np.argmax(set_ious)
print('Label %d (%s): best set IOU = %f (filter %d)' % (label_i,
ds.name(category=None,j=label_i), set_ious[best_filter], best_filter))
ind_ious = np.true_divide(iou_intersects, iou_unions + 1e-20)
set_ious_mmap[label_i] = set_ious
ind_ious_mmap[label_i, loader_idx] = ind_ious
train_idx = [i for i in range(len(loader_idx)) if ds.split(loader_idx[i]) == 'train']
val_idx = [i for i in range(len(loader_idx)) if ds.split(loader_idx[i]) == 'val']
set_ious_train_mmap[label_i] = np.true_divide(np.sum(iou_intersects[train_idx], axis=0),
np.sum(iou_unions[train_idx], axis=0) + 1e-20)
set_ious_val_mmap[label_i] = np.true_divide(np.sum(iou_intersects[val_idx], axis=0),
np.sum(iou_unions[val_idx], axis=0) + 1e-20)
#set_ious_mmap.flush()
#ind_ious_mmap.flush()
ed.finish_mmap(set_ious_train_mmap)
ed.finish_mmap(set_ious_val_mmap)
ed.finish_mmap(set_ious_mmap)
ed.finish_mmap(ind_ious_mmap)