def compute_alpha(directory): ed = expdir.ExperimentDirectory(directory) info = ed.load_info() ds = loadseg.SegmentationData(info.dataset) L = ds.label_size() if not has_image_to_label(directory): create_image_to_label(directory) image_to_label = load_image_to_label(directory) label_names = np.array([ds.label[i]['name'] for i in range(L)]) alphas = np.zeros((L, )) for label_i in range(1, L): label_categories = ds.label[label_i]['category'].keys() label_idx = np.where(image_to_label[:, label_i])[0] train_loader = loadseg.SegmentationPrefetcher( ds, categories=label_categories, split='train', indexes=label_idx, once=True, batch_size=64, ahead=4, thread=True) train_idx = np.array(train_loader.indexes) #sw = 0 #sh = 0 perc_label = [] #train_label_categories = [] for batch in train_loader.batches(): for rec in batch: sw, sh = [rec[k] for k in ['sw', 'sh']] #sw_r, sh_r = [rec[k] for k in ['sw', 'sh']] #if sw == 0 and sh == 0: # sw = sw_r # sh = sh_r #else: # assert(sw == sw_r and sh == sh_r) for cat in label_categories: if rec[cat] != []: #train_label_categories.append(cat) if type(rec[cat]) is np.ndarray: perc_label.append( np.sum(rec[cat] == label_i) / float(sw * sh)) else: perc_label.append(1.) break assert (len(perc_label) == len(train_idx)) alphas[label_i] = float(1. - np.mean(perc_label)) print label_i, label_names[label_i], alphas[label_i] train_loader.close() alphas_mmap = ed.open_mmap(part='train_alphas', mode='w+', dtype='float32', shape=alphas.shape) alphas_mmap[:] = alphas[:] ed.finish_mmap(alphas_mmap)
def create_image_dataset_label_index(directory, batch_size=64, ahead=16): ed = expdir.ExperimentDirectory(directory) info = ed.load_info() ds = loadseg.SegmentationData(info.dataset) categories = ds.category_names() shape = (ds.size(), len(DATASETS), len(ds.label)) index = np.zeros(shape) pf = loadseg.SegmentationPrefetcher(ds, categories=categories, once=True, batch_size=batch_size, ahead=ahead, thread=True) batch_count = 0 for batch in pf.batches(): if batch_count % 100 == 0: print('Processing batch %d ...' % batch_count) for rec in batch: dataset_index = get_dataset_index(rec['fn']) image_index = rec['i'] for cat in categories: if ((type(rec[cat]) is np.ndarray and rec[cat].size > 0) or type(rec[cat]) is list and len(rec[cat]) > 0): index[image_index][dataset_index][np.unique( rec[cat])] = True batch_count += 1 mmap = ed.open_mmap(part='image_dataset_label', mode='w+', dtype=bool, shape=shape) mmap[:] = index[:] ed.finish_mmap(mmap) print('Finished and saved at %s' % ed.mmap_filename(part='image_dataset_label'))
def load_image_dataset_label_index(directory): ed = expdir.ExperimentDirectory(directory) info = ed.load_info() ds = loadseg.SegmentationData(info.dataset) shape = (ds.size(), len(DATASETS), len(ds.label)) return ed.open_mmap(part='image_dataset_label', mode='r', dtype=bool, shape=shape)
def __init__(self, directory, blobs=None): self.ed = expdir.ExperimentDirectory(directory) # Load probe metadata info = self.ed.load_info() self.ih, self.iw = info.input_dim self.layers = info.blobs if blobs is None else blobs self.ds = loadseg.SegmentationData(info.dataset) self.layer = {} for blob in self.layers: self.layer[blob] = LayerProbe(self.ed, blob, self.ds)
def create_image_to_label(directory, batch_size=16, ahead=4): ed = expdir.ExperimentDirectory(directory) info = ed.load_info() print info.dataset if 'broden' in info.dataset: ds = loadseg.SegmentationData(info.dataset) categories = ds.category_names() shape = (ds.size(), len(ds.label)) pf = loadseg.SegmentationPrefetcher(ds, categories=categories, once=True, batch_size=batch_size, ahead=ahead, thread=False) image_to_label = np.zeros(shape, dtype='int32') batch_count = 0 for batch in pf.batches(): if batch_count % 100 == 0: print('Processing batch %d ...' % batch_count) for rec in batch: image_index = rec['i'] for cat in categories: if ((type(rec[cat]) is np.ndarray and rec[cat].size > 0) or type(rec[cat]) is list and len(rec[cat]) > 0): image_to_label[image_index][np.unique(rec[cat])] = True batch_count += 1 elif 'imagenet' in info.dataset or 'ILSVRC' in info.dataset: classes, class_to_idx = find_classes(info.dataset) imgs = make_dataset(info.dataset, class_to_idx) _, labels = zip(*imgs) labels = np.array(labels) L = 1000 shape = (len(labels), L) image_to_label = np.zeros(shape) for i in range(L): image_to_label[labels == i, i] = 1 else: assert (False) mmap = ed.open_mmap(part='image_to_label', mode='w+', dtype=bool, shape=shape) mmap[:] = image_to_label[:] ed.finish_mmap(mmap) f = ed.mmap_filename(part='image_to_label') print('Finished and saved index_to_label at %s' % f)
def extract_concept_data(directory, batch_size=64, ahead=16, verbose=True): ed = expdir.ExperimentDirectory(directory) if ed.has_mmap(part='concept_data'): print('%s already has %s, so skipping' % (directory, ed.mmap_filename(part='concept_data'))) return info = ed.load_info() (sh, sw) = get_seg_size(info.input_dim) ds = loadseg.SegmentationData(info.dataset) categories = np.array(ds.category_names()) L = ds.label_size() N = ds.size() pf = loadseg.SegmentationPrefetcher(ds, categories=categories, once=True, batch_size=batch_size, ahead=ahead, thread=True) if verbose: print 'Creating new mmap at %s' % ed.mmap_filename(part='concept_data') data = ed.open_mmap(part='concept_data', mode='w+', shape=(N, L, sh, sw)) start_time = time.time() last_batch_time = start_time index = 0 for batch in pf.batches(): batch_time = time.time() rate = index / (batch_time - start_time + 1e-15) batch_rate = batch_size / (batch_time - last_batch_time + 1e-15) last_batch_time = batch_time if verbose: print 'extract_concept_data index %d/%d (%.2f)\titems per sec %.2f\t%.2f' % ( index, N, index / float(N), batch_rate, rate) for rec in batch: for cat in categories: if len(rec[cat]) == 0: continue if cat == 'texture' or cat == 'scene': for i in range(len(rec[cat])): data[index][rec[cat][i] - 1, :, :] = 1 else: for i in range(len(rec[cat])): ys, xs = np.where(rec[cat][i]) for j in range(len(xs)): data[index][rec[cat][i][ys[j]][xs[j]] - 1][ys[j]][xs[j]] = 1 index += 1 assert index == N, ("Data source should return every item once %d %d." % (index, N)) if verbose: print 'Renaming mmap.' ed.finish_mmap(data)
def BrodenDataGenerator(): split = "train" batch_size = 8 randomize = True while True: datasource = loadseg.SegmentationData(TEST_DIR, categories=categories) prefetcher = loadseg.SegmentationPrefetcher(datasource, split=split, categories=['image'] + categories, segmentation_shape=None, batch_size=batch_size, randomize=randomize, ahead=12) batch = prefetcher.fetch_batch() yield batch, None
def load_image_to_label(directory, blob=None): ed = expdir.ExperimentDirectory(directory) info = ed.load_info() if 'broden' in info.dataset: ds = loadseg.SegmentationData(info.dataset) shape = (ds.size(), len(ds.label)) elif 'imagenet' in info.dataset or 'ILSVRC' in info.dataset: assert (blob is not None) blob_info = ed.load_info(blob=blob) shape = (blob_info.shape[0], 1000) else: assert (False) return ed.open_mmap(part='image_to_label', mode='r', dtype=bool, shape=shape)
def open_dataset(ed): return loadseg.SegmentationData(ed.load_info().dataset)
'features.11': 'relu5', 'features': 'pool5' } directory='/home/ruthfong/NetDissect/probes/pytorch_alexnet_imagenet' if not os.path.exists(directory): directory='/scratch/shared/slow/ruthfong/pytorch_alexnet_imagenet' assert(os.path.exists(directory)) blob='features.11' ed = expdir.ExperimentDirectory(directory) info = ed.load_info() blob_info = ed.load_info(blob=blob) shape = blob_info.shape ds = loadseg.SegmentationData(info.dataset) categories = ds.category_names() K = shape[1] L = ds.label_size() N = ds.size() label_names = np.array([ds.label[i]['name'] for i in range(L)]) blobs = ['features.1', 'features.4', 'features.7', 'features.9', 'features.11'] suffix='' linear_ind_ious = {} single_ind_ious = {} linear_set_ious = {}
def compute_correlation(directory, blob, num_samples=None, num_components=1, out_file=None, verbose=False): ed = expdir.ExperimentDirectory(directory) info = ed.load_info() ds = loadseg.SegmentationData(info.dataset) L = ds.label_size() N = ds.size() blob_info = ed.load_info(blob=blob) shape = blob_info.shape K = shape[1] categories = np.array(ds.category_names()) label_names = np.array([ds.label[i]['name'] for i in range(L)]) (Hs, Ws) = get_seg_size(info.input_dim) if verbose: start = time.time() print 'Loading data...' upsampled_data = ed.open_mmap(blob=blob, part='upsampled', mode='r', shape=(N,K,Hs,Ws)) concept_data = ed.open_mmap(part='concept_data', mode='r', shape=(N,L,Hs,Ws)) if verbose: print 'Finished loading data in %d secs.' % (time.time() - start) if verbose: start = time.time() print 'Selecting data...' if num_samples is not None: rand_idx = np.random.choice(N, num_samples, replace=False) X = upsampled_data[rand_idx,:,Hs/2,Ws/2] Y = concept_data[rand_idx,:,Hs/2,Ws/2] else: X = upsampled_data[:,:,Hs/2,Ws/2] Y = concept_data[:,:,Hs/2,Ws/2] if verbose: print 'Finished selecting data in %d secs.' % (time.time() - start) cca = CCA(n_components=num_components) if verbose: start = time.time() if num_samples is None: num_samples = N print 'Fitting %d-component CCA with N = %d samples...' % (num_components, num_samples) cca.fit(X,Y) if verbose: print 'Fitted %d-component CCA with N = %d samples in %d secs.' % (num_components, num_samples, time.time() - start) X_c, Y_c = cca.transform(X,Y) score = cca.score(X,Y) results = {} if out_file is not None: if verbose: start = time.time() print 'Saving results...' results['model'] = cca try: results['idx'] = rand_idx except: results['idx'] = None results['directory'] = directory results['blob'] = blob results['num_samples'] = num_samples results['num_components'] = num_components results['score'] = score pkl.dump(results, open(out_file, 'wb')) if verbose: print 'Saved results at %s in %d secs.' % (out_file, time.time() - start) return results
def max_probe(directory, blob, batch_size=None, quantile=0.005, results=None, num_components=None, use_y_weights=False, suffix='', new_suffix='', normalize=True, pool='max_pool', should_thresh=False, disc=False): # Make sure we have a directory to work in ed = expdir.ExperimentDirectory(directory) # If it's already computed, then skip it!!! if disc: suffix = '_disc%s' % suffix print "Checking", ed.mmap_filename(blob=blob, part='linear_imgmax%s%s' % (suffix, new_suffix)) if ed.has_mmap(blob=blob, part='linear_imgmax%s%s' % (suffix, new_suffix)): print "Already have %s-imgmax.mmap, skipping." % (blob) return info = ed.load_info() ds = loadseg.SegmentationData(info.dataset) # Read about the blob shape blob_info = ed.load_info(blob=blob) shape = blob_info.shape N = shape[0] K = shape[1] L = ds.label_size() if should_thresh: if quantile == 1: thresh = np.zeros((1, 1, K, 1, 1)) else: quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(K, -1)) threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))] thresh = threshold[numpy.newaxis, numpy.newaxis, :, numpy.newaxis, numpy.newaxis] if use_y_weights: shape = ( N, L, 113, 113 ) # TODO: don't hardcode segmentation size, this only works for alexnet data = ed.open_mmap(part='concept_data', mode='r', shape=shape) else: data = ed.open_mmap(blob=blob, shape=shape) print 'Computing imgmax for %s shape %r' % (blob, shape) if results is not None: imgmax = ed.open_mmap(blob=blob, part='linear_imgmax%s%s' % (suffix, new_suffix), mode='w+', shape=(N, num_components)) if use_y_weights: all_weights = pkl.load(open(results, 'rb'))['model'].y_weights_.T else: all_weights = pkl.load(open(results, 'rb'))['model'].x_weights_.T else: imgmax = ed.open_mmap(blob=blob, part='linear_imgmax%s%s' % (suffix, new_suffix), mode='w+', shape=(N, L)) if disc: assert (ed.has_mmap(blob=blob, part='linear_weights%s' % suffix)) all_weights = ed.open_mmap(blob=blob, part='linear_weights%s' % suffix, mode='r', dtype='float32', shape=(L, 2, K)) all_weights = all_weights[:, -1, :] else: assert (ed.has_mmap(blob=blob, part='linear_weights%s' % suffix)) all_weights = ed.open_mmap(blob=blob, part='linear_weights%s' % suffix, mode='r', dtype='float32', shape=(L, K)) if normalize: all_weights = numpy.array([ numpy.true_divide(all_weights[i], numpy.linalg.norm(all_weights[i])) for i in range(L) ]) # Automatic batch size selection: 64mb batches if batch_size is None: batch_size = max(1, int(128 * 1024 * 1024 / numpy.prod(shape[1:]))) # Algorithm: one pass over the data start_time = time.time() last_batch_time = start_time for i in range(0, data.shape[0], batch_size): batch_time = time.time() rate = i / (batch_time - start_time + 1e-15) batch_rate = batch_size / (batch_time - last_batch_time + 1e-15) last_batch_time = batch_time print 'Imgmax %s index %d: %f %f' % (blob, i, rate, batch_rate) sys.stdout.flush() batch = data[i:i + batch_size][:, numpy. newaxis, :, :, :] # (batch_size, L, K, S, S) if should_thresh: batch = (batch > thresh).astype(float) if pool == 'max_pool': imgmax[i:i + batch_size, :] = ( batch * all_weights[numpy.newaxis, :, :, numpy.newaxis, numpy.newaxis]).sum(axis=2).max(axis=(2, 3)) elif pool == 'avg_pool': imgmax[i:i + batch_size, :] = ( batch * all_weights[numpy.newaxis, :, :, numpy.newaxis, numpy.newaxis]).sum(axis=2).mean(axis=(2, 3)) print 'Writing imgmax' sys.stdout.flush() # Save as mat file filename = ed.filename('linear_imgmax%s.mat' % suffix, blob=blob) savemat(filename, {'linear_imgmax': imgmax}) # And as mmap ed.finish_mmap(imgmax)
def consolidate_probe(directory, blob, bias=False, num_filters=None, suffix='', delete=False): ed = expdir.ExperimentDirectory(directory) info = ed.load_info() blob_info = ed.load_info(blob=blob) shape = blob_info.shape ds = loadseg.SegmentationData(info.dataset) L = ds.label_size() # number of labels (including background at index 0) N = ds.size() # total number of images in the dataset K = shape[1] # number of units for the given blob F = 1 if num_filters is None else len(num_filters) + 1 if num_filters is None: suffixes = [suffix] else: suffixes = ['%s_num_filters_%d' % (suffix, n) for n in num_filters] suffixes.append(suffix) suffix = '%s_num_filters_%d' % (suffix, F) if (ed.has_mmap(blob=blob, part='linear_weights%s' % suffix) and (not bias or ed.has_mmap(blob=blob, part='linear_bias%s' % suffix))): print('Linear weights (and bias) have already been consolidated') else: weights_mmap = ed.open_mmap(blob=blob, part='linear_weights%s' % suffix, mode='w+', dtype='float32', shape=(L, F, K)) if bias: bias_mmap = ed.open_mmap(blob=blob, part='linear_bias%s' % suffix, mode='w+', dtype='float32', shape=(L, F)) missing_idx = [] for l in range(L): if not ed.has_mmap(blob=blob, part='label_i_%d_weights%s' % (l, suffixes[0])): missing_idx.append(l) continue for i in range(F): suffix = suffixes[i] if ed.has_mmap(blob=blob, part='label_i_%d_weights%s' % (l, suffix)): try: label_weights_mmap = ed.open_mmap( blob=blob, part='label_i_%d_weights%s' % (l, suffix), mode='r', dtype='float32', shape=(K, )) except ValueError: print('here') # SUPPORT LEGACY CODE, TODO: remove eventually label_weights_mmap = ed.open_mmap( blob=blob, part='label_i_%d_weights%s' % (l, suffix), mode='r', dtype=float, shape=(K, )) else: all_weights_mmap = ed.open_mmap(blob=blob, part='linear_weights%s' % suffix, mode='r', dtype='float32', shape=(L, K)) label_weights_mmap = all_weights_mmap[l] weights_mmap[l, i, :] = label_weights_mmap[:] if bias: if ed.has_mmap(blob=blob, part='label_i_%d_bias' % (l, suffix)): label_bias_mmap = ed.open_mmap( blob=blob, part='label_i_%d_bias%s' % (l, suffix), mode='r', dtype='float32', shape=(1, )) else: all_bias_mmap = ed.open_mmap(blob=blob, part='linear_bias%s' % suffix, mode='r', dtype='float32', shape=(K, )) label_bias_mmap = all_bias_mmap[l] bias_mmap[l, i] = label_bias_mmap[:] ed.finish_mmap(weights_mmap) if bias: ed.finish_mmap(bias_mmap) print('Finished consolidating existing weights files ' + '(Files for %d labels were missing).' % len(missing_idx)) print(missing_idx) if delete: c_w = 0 c_b = 0 print('Deleting all unnecessary label weights (and bias) files...') for l in range(L): for i in range(F): suffix = suffixes[i] if ed.has_mmap(blob=blob, part='label_i_%d_weights%s' % (l, suffix)): fn = ed.mmap_filename(blob=blob, part='label_i_%d_weights%s' % (l, suffix)) os.remove(fn) c_w += 1 if bias and ed.has_mmap( blob=blob, part='label_i_%d_bias%s' % (l, suffix)): fn = ed.mmap_filename(blob=blob, part='label_i_%d_bias%s' % (l, suffix)) os.remove(fn) c_b += 1 print('Removed %d weights and %d bias files.' % (c_w, c_b))
def linear_probe_discriminative(directory, blob, label_i, suffix='', batch_size=16, quantile=0.005, bias=False, pool='avg_pool', positive=False, num_filters=None, init_suffix='', num_epochs=30, epoch_iter=10, lr=1e-4, momentum=0.9, l1_weight_decay=0, l2_weight_decay=0, nesterov=False, lower_bound=None, optimizer_type='sgd', cuda=False, fig_path=None, show_fig=True): ed = expdir.ExperimentDirectory(directory) try: info = ed.load_info() except: ed = expdir.ExperimentDirectory(os.path.join(directory, 'train')) ed_val = expdir.ExperimentDirectory(os.path.join(directory, 'val')) info = ed.load_info() assert('imagenet' in info.dataset or 'ILSVRC' in info.dataset) # Check if linear weights have already been learned if ed.has_mmap(blob=blob, part='label_i_%d_weights_disc%s' % (label_i, suffix)): print('%s already has %s, so skipping.' % (directory, ed.mmap_filename(blob=blob, part='label_i_%d_weights_disc%s' % ( label_i, suffix)))) return # Load probe metadata info = ed.load_info() ih, iw = info.input_dim # Load blob metadata blob_info = ed.load_info(blob=blob) shape = blob_info.shape unit_size = shape[1] fieldmap = blob_info.fieldmap # Load the blob quantile data and grab thresholds if quantile == 1: if len(shape) == 4: thresh = np.zeros((unit_size, 1, 1)) elif len(shape) == 2: thresh = np.zeros(unit_size) else: assert(False) else: quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(unit_size, -1)) threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))] if len(shape) == 4: thresh = threshold[:, np.newaxis, np.newaxis] elif len(shape) == 2: thresh = threshold else: assert(False) # print np.max(thresh), thresh.shape, type(thresh) # Map the blob activation data for reading fn_read = ed.mmap_filename(blob=blob) blobdata = cached_memmap(fn_read, mode='r', dtype='float32', shape=shape) if 'broden' in info.dataset: # Load the dataset ds = loadseg.SegmentationData(info.dataset) # Get all the categories the label is a part of label_categories = ds.label[label_i]['category'].keys() num_categories = len(label_categories) # Get label name label_name = ds.name(category=None, j=label_i) # Get indices of images containing the given label if not has_image_to_label(directory): print('image_to_label does not exist in %s; creating it now...' % directory) create_image_to_label(directory, batch_size=batch_size) image_to_label = load_image_to_label(directory) label_idx = np.where(image_to_label[:, label_i])[0] non_label_idx = np.where(image_to_label[:, label_i] == 0)[0] print('Number of positive and negative examples of label %d (%s): %d %d' % ( label_i, label_name, len(label_idx), len(non_label_idx))) val_blobdata = blobdata elif 'imagenet' in info.dataset or 'ILSVRC' in info.dataset: label_desc = np.loadtxt('synset_words.txt', str, delimiter='\t') label_name = ' '.join(label_desc[label_i].split(',')[0].split()[1:]) image_to_label_train = load_image_to_label(os.path.join(directory, 'train'), blob=blob) train_label_idx = np.where(image_to_label_train[:, label_i])[0] train_non_label_idx = np.where(1-image_to_label_train[:, label_i])[0] image_to_label_val = load_image_to_label(os.path.join(directory, 'val'), blob=blob) val_label_idx = np.where(image_to_label_val[:, label_i])[0] val_non_label_idx = np.where(1-image_to_label_val[:, label_i])[0] print('Number of positive and negative examples of label %d (%s): %d %d %d %d' % (label_i, label_name, len(train_label_idx), len(train_non_label_idx), len(val_label_idx), len(val_non_label_idx))) val_blob_info = ed_val.load_info(blob=blob) val_shape = val_blob_info.shape val_fn_read = ed_val.mmap_filename(blob=blob) val_blobdata = cached_memmap(val_fn_read, mode='r', dtype='float32', shape=val_shape) else: assert(False) criterion = torch.nn.BCEWithLogitsLoss() if num_filters is not None: init_weights_mmap = ed.open_mmap(blob=blob, part='label_i_%d_weights_disc%s' % (label_i, init_suffix), mode='r', dtype='float32', shape=unit_size) sorted_idx = np.argsort(np.abs(init_weights_mmap))[::-1] mask_idx = np.zeros(unit_size, dtype=int) mask_idx[sorted_idx[:num_filters]] = 1 layer = CustomDiscLayer(unit_size, act=False, pool=pool, bias=bias, positive=positive, mask_idx=torch.ByteTensor(mask_idx), cuda=cuda) else: layer = CustomDiscLayer(unit_size, act=False, pool=pool, bias=bias, positive=positive, cuda=cuda) if cuda: criterion.cuda() layer.cuda() if optimizer_type == 'sgd': optimizer = Custom_SGD(layer.parameters(), lr, momentum, l1_weight_decay=l1_weight_decay, l2_weight_decay=l2_weight_decay, nesterov=nesterov, lower_bound=lower_bound) elif optimizer_type == 'adam': optimizer = Custom_Adam(layer.parameters(), lr, l1_weight_decay=l1_weight_decay, l2_weight_decay=l2_weight_decay, lower_bound=lower_bound) if 'broden' in info.dataset: train_label_idx = [] val_label_idx = [] for ind in label_idx: if ds.split(ind) == 'train': train_label_idx.append(ind) elif ds.split(ind) == 'val': val_label_idx.append(ind) train_non_label_idx = [] val_non_label_idx = [] for ind in non_label_idx: if ds.split(ind) == 'train': train_non_label_idx.append(ind) elif ds.split(ind) == 'val': val_non_label_idx.append(ind) train_label_idx = np.array(train_label_idx) val_label_idx = np.array(val_label_idx) train_non_label_idx = np.array(train_non_label_idx) val_non_label_idx = np.array(val_non_label_idx) num_train_labels = len(train_label_idx) num_train_non_labels = len(train_non_label_idx) train_neg_greater = num_train_labels < num_train_non_labels if len(val_label_idx) < len(val_non_label_idx): pos_val_idx = np.arange(len(val_label_idx)) neg_val_idx = np.random.choice(len(val_non_label_idx), len(val_label_idx), replace=False) else: pos_val_idx = np.random.choice(len(val_label_idx), len(val_non_label_idx), replace=False) neg_val_idx = np.arange(len(val_non_label_idx)) val_indexes = np.concatenate((val_label_idx[pos_val_idx], val_non_label_idx[neg_val_idx])) val_targets = np.concatenate((np.ones(len(pos_val_idx)), np.zeros(len(neg_val_idx)))) print 'Number of train and val examples: %d %d' % (min(2*num_train_non_labels, 2*num_train_labels), len(val_targets)) train_losses = [] train_accs = [] val_losses = [] val_accs = [] if 'imagenet' in info.dataset or 'ILSVRC' in info.dataset: if train_neg_greater: pos_train_idx = np.arange(num_train_labels) neg_train_idx = np.random.choice(num_train_non_labels, num_train_labels, replace=False) else: pos_train_idx = np.random.choice(num_train_labels, num_train_non_labels, replace=False) neg_train_idx = np.arange(num_train_non_labels) train_indexes = np.concatenate((train_label_idx[pos_train_idx], train_non_label_idx[neg_train_idx])) train_targets = np.concatenate((np.ones(len(pos_train_idx)), np.zeros(len(neg_train_idx)))) rand_idx = np.random.permutation(len(train_targets)) for t in range(num_epochs): if epoch_iter is not None: adjust_learning_rate(lr, optimizer, t, epoch_iter=epoch_iter) if 'broden' in info.dataset: if train_neg_greater: pos_train_idx = np.arange(num_train_labels) neg_train_idx = np.random.choice(num_train_non_labels, num_train_labels, replace=False) else: pos_train_idx = np.random.choice(num_train_labels, num_train_non_labels, replace=False) neg_train_idx = np.arange(num_train_non_labels) train_indexes = np.concatenate((train_label_idx[pos_train_idx], train_non_label_idx[neg_train_idx])) train_targets = np.concatenate((np.ones(len(pos_train_idx)), np.zeros(len(neg_train_idx)))) rand_idx = np.random.permutation(len(train_targets)) (train_loss, train_acc) = run_epoch(blobdata, train_targets[rand_idx], train_indexes[rand_idx], thresh, layer, criterion, optimizer, t, batch_size=batch_size, train=True, cuda=cuda) (val_loss, val_acc) = run_epoch(val_blobdata, val_targets, val_indexes, thresh, layer, criterion, optimizer, t, batch_size=batch_size, train=False, cuda=cuda) train_losses.append(train_loss) train_accs.append(train_acc) val_losses.append(val_loss) val_accs.append(val_acc) if fig_path is not None or show_fig: f, ax = plt.subplots(1,2, figsize=(8,4)) ax[0].plot(range(num_epochs), train_losses, label='train') ax[0].plot(range(num_epochs), val_losses, label='val') ax[0].set_title('BCE Loss (train=%d, val=%d)' % (len(train_targets), len(val_targets))) ax[1].plot(range(num_epochs), train_accs, label='train') ax[1].plot(range(num_epochs), val_accs, label='val') ax[1].set_title('Accuracy (%s, %s)' % (label_name, blob)) plt.legend() if fig_path is not None: plt.savefig(fig_path) if show_fig: plt.show() plt.close() # Save weights weights = (layer.mask * layer.weight).data.cpu().numpy() print np.sum(weights != 0) weights_mmap = ed.open_mmap(blob=blob, part='label_i_%d_weights_disc%s' % (label_i, suffix), mode='w+', dtype='float32', shape=weights.shape) weights_mmap[:] = weights[:] ed.finish_mmap(weights_mmap) if bias: bias_v = layer.bias.data.cpu().numpy() bias_mmap = ed.open_mmap(blob=blob, part='label_i_%d_bias_disc%s' % (label_i, suffix), mode='w+', dtype='float32', shape=(1,)) bias_mmap[:] = bias_v[:] ed.finish_mmap(bias_mmap) results_mmap = ed.open_mmap(blob=blob, part='label_i_%d_results_disc%s' % (label_i, suffix), mode='w+', dtype='float32', shape=(4,num_epochs)) results_mmap[0] = train_losses results_mmap[1] = val_losses results_mmap[2] = train_accs results_mmap[3] = val_accs ed.finish_mmap(results_mmap)
def label_probe(directory, blob, batch_size=5, ahead=4, tally_depth=2048, quantile=0.01, categories=None, verbose=False, parallel=0): ''' Evaluates the given blob from the probe stored in the given directory. Note that as part of the evaluation, the dataset is loaded again. ''' # Make sure we have a directory to work in qcode = ('%f' % quantile).replace('0.', '').rstrip('0') ed = expdir.ExperimentDirectory(directory) # If the tally file already exists, return. if ed.has_mmap(blob=blob, part='tally-%s' % qcode): if verbose: print 'Have', (ed.mmap_filename(blob=blob, part='tally-%s' % qcode)), 'already. Skipping.' return # Load probe metadata info = ed.load_info() ih, iw = info.input_dim # Load blob metadata blob_info = ed.load_info(blob=blob) shape = blob_info.shape unit_size = shape[1] fieldmap = blob_info.fieldmap # Load the blob quantile data and grab thresholds quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(shape[1], -1)) threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))] thresh = threshold[:, numpy.newaxis, numpy.newaxis] # Map the blob activation data for reading fn_read = ed.mmap_filename(blob=blob) # Load the dataset ds = loadseg.SegmentationData(info.dataset) if categories is None: categories = ds.category_names() labelcat = onehot(primary_categories_per_index(ds, categories)) # Be sure to zero out the background label - it belongs to no category. labelcat[0, :] = 0 # Set up output data # G = ground truth counts # A = activation counts # I = intersection counts # TODO: consider activation_quantile and intersect_quantile # fn_s = os.path.join(directory, '%s-%s-scores.txt' % (blob, qcode)) # T = tally, a compressed per-sample record of intersection counts. fn_t = ed.mmap_filename(blob=blob, part='tally-%s' % qcode, inc=True) # Create the file so that it can be mapped 'r+' ed.open_mmap(blob=blob, part='tally-%s' % qcode, mode='w+', dtype='int32', shape=(ds.size(), tally_depth, 3)) if parallel > 1: fast_process(fn_t, fn_read, shape, tally_depth, ds, iw, ih, categories, fieldmap, thresh, labelcat, batch_size, ahead, verbose, parallel) else: process_data(fn_t, fn_read, shape, tally_depth, ds, iw, ih, categories, fieldmap, thresh, labelcat, batch_size, ahead, verbose, False, 0, ds.size()) fn_final = re.sub('-inc$', '', fn_t) if verbose: print 'Renaming', fn_t, 'to', fn_final os.rename(fn_t, fn_final)
def probe_linear(directory, blob, suffix='', start=None, end=None, batch_size=16, ahead=4, quantile=0.005, bias=False, positive=False, cuda=False, force=False): qcode = ('%f' % quantile).replace('0.', '.').rstrip('0') ed = expdir.ExperimentDirectory(directory) if (ed.has_mmap(blob=blob, part='linear_ind_ious%s' % suffix) and ed.has_mmap(blob=blob, part='linear_set_ious%s' % suffix)): print('Linear weights have already been probed.') print ed.mmap_filename(blob=blob, part='linear_set_val_ious%s' % suffix) if not force: return else: print('Forcefully continuing...') info = ed.load_info() seg_size = get_seg_size(info.input_dim) blob_info = ed.load_info(blob=blob) ds = loadseg.SegmentationData(info.dataset) shape = blob_info.shape N = shape[0] # number of total images K = shape[1] # number of units in given blob L = ds.label_size() # number of labels if quantile == 1: thresh = np.zeros((K, 1, 1)) else: quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(K, -1)) threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))] thresh = threshold[:, np.newaxis, np.newaxis] fn_read = ed.mmap_filename(blob=blob) blobdata = cached_memmap(fn_read, mode='r', dtype='float32', shape=shape) image_to_label = load_image_to_label(directory) if ed.has_mmap(blob=blob, part='linear_ind_ious%s' % suffix, inc=True) and not force: assert (ed.has_mmap(blob=blob, part='linear_set_ious%s' % suffix, inc=True)) ind_ious = ed.open_mmap(blob=blob, part='linear_ind_ious%s' % suffix, mode='r+', inc=True, dtype='float32', shape=(L, N)) set_ious = ed.open_mmap(blob=blob, part='linear_set_ious%s' % suffix, mode='r+', inc=True, dtype='float32', shape=(L, )) set_ious_train = ed.open_mmap(blob=blob, part='linear_set_train_ious%s' % suffix, mode='r+', inc=True, dtype='float32', shape=(L, )) try: set_ious_val = ed.open_mmap(blob=blob, part='linear_set_val_ious%s' % suffix, mode='r+', inc=True, dtype='float32', shape=(L, )) except: set_ious_val = ed.open_mmap(blob=blob, part='linear_set_val_ious%s' % suffix, mode='r+', dtype='float32', shape=(L, )) else: ind_ious = ed.open_mmap(blob=blob, part='linear_ind_ious%s' % suffix, mode='w+', dtype='float32', shape=(L, N)) set_ious = ed.open_mmap(blob=blob, part='linear_set_ious%s' % suffix, mode='w+', dtype='float32', shape=(L, )) set_ious_train = ed.open_mmap(blob=blob, part='linear_set_train_ious%s' % suffix, mode='w+', dtype='float32', shape=(L, )) set_ious_val = ed.open_mmap(blob=blob, part='linear_set_val_ious%s' % suffix, mode='w+', dtype='float32', shape=(L, )) if start is None: start = 1 if end is None: end = L for label_i in range(start, end): if ed.has_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix)): try: weights = ed.open_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix), mode='r', dtype='float32', shape=(K, )) except ValueError: # SUPPORTING LEGACY CODE (TODO: Remove) weights = ed.open_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix), mode='r', dtype=float, shape=(K, )) elif ed.has_mmap(blob=blob, part='linear_weights%s' % suffix): all_weights = ed.open_mmap(blob=blob, part='linear_weights%s' % suffix, mode='r', dtype='float32', shape=(L, K)) weights = all_weights[label_i] if not np.any(weights): print( 'Label %d does not have associated weights to it, so skipping.' % label_i) continue else: print( 'Label %d does not have associated weights to it, so skipping.' % label_i) continue if bias: if ed.has_mmap(blob=blob, part='label_i_%d_bias%s' % (label_i, suffix)): bias_v = ed.open_mmap(blob=blob, part='label_i_%d_bias%s' % (label_i, suffix), mode='r', dtype='float32', shape=(1, )) else: assert (ed.has_mmap(blob=blob, part='linear_bias%s' % suffix)) all_bias_v = ed.open_mmap(blob=blob, part='linear_bias%s' % suffix, mode='r', dtype='float32', shape=(L, )) bias_v = np.array([all_bias_v[label_i]]) label_categories = ds.label[label_i]['category'].keys() label_name = ds.name(category=None, j=label_i) label_idx = np.where(image_to_label[:, label_i])[0] loader = loadseg.SegmentationPrefetcher(ds, categories=label_categories, indexes=label_idx, once=True, batch_size=batch_size, ahead=ahead, thread=True) loader_idx = loader.indexes num_imgs = len(loader.indexes) print( 'Probing with learned weights for label %d (%s) with %d images...' % (label_i, label_name, num_imgs)) model = CustomLayer(K, upsample=True, up_size=seg_size, act=True, bias=bias, positive=positive, cuda=cuda) model.weight.data[...] = torch.Tensor(weights) if bias: model.bias.data[...] = torch.Tensor(bias_v) if cuda: model.cuda() model.eval() iou_intersects = np.zeros(num_imgs) iou_unions = np.zeros(num_imgs) i = 0 for batch in loader.batches(): start_t = time.time() if (i + 1) * batch_size < num_imgs: idx = range(i * batch_size, (i + 1) * batch_size) else: idx = range(i * batch_size, num_imgs) i += 1 input = torch.Tensor( (blobdata[loader_idx[idx]] > thresh).astype(float)) input_var = (Variable(input.cuda(), volatile=True) if cuda else Variable(input, volatile=True)) target_ = [] for rec in batch: for cat in label_categories: if rec[cat] != []: if type(rec[cat]) is np.ndarray: target_.append( np.max((rec[cat] == label_i).astype(float), axis=0)) else: target_.append(np.ones(seg_size)) break target = torch.Tensor(target_) target_var = (Variable(target.cuda(), volatile=True) if cuda else Variable(target, volatile=True)) #target_var = Variable(target.unsqueeze(1).expand_as( # input_var).cuda() if cuda # else target.unsqueeze(1).expand_as(input_var)) output_var = model(input_var) iou_intersects[idx] = np.squeeze( iou_intersect_d(output_var, target_var).data.cpu().numpy()) iou_unions[idx] = np.squeeze( iou_union_d(output_var, target_var).data.cpu().numpy()) print('Batch: %d/%d\tTime: %f secs\tAvg Ind IOU: %f' % (i, num_imgs / batch_size, time.time() - start_t, np.mean( np.true_divide(iou_intersects[idx], iou_unions[idx] + 1e-20)))) loader.close() label_ind_ious = np.true_divide(iou_intersects, iou_unions + 1e-20) label_set_iou = np.true_divide(np.sum(iou_intersects), np.sum(iou_unions) + 1e-20) ind_ious[label_i, loader_idx] = label_ind_ious set_ious[label_i] = label_set_iou train_idx = [ i for i in range(len(loader_idx)) if ds.split(loader_idx[i]) == 'train' ] val_idx = [ i for i in range(len(loader_idx)) if ds.split(loader_idx[i]) == 'val' ] set_ious_train[label_i] = np.true_divide( np.sum(iou_intersects[train_idx]), np.sum(iou_unions[train_idx]) + 1e-20) set_ious_val[label_i] = np.true_divide( np.sum(iou_intersects[val_idx]), np.sum(iou_unions[val_idx]) + 1e-20) print( 'Label %d (%s) Set IOU: %f, Train Set IOU: %f, Val Set IOU: %f, Max Ind IOU: %f' % (label_i, label_name, label_set_iou, set_ious_train[label_i], set_ious_val[label_i], np.max(label_ind_ious))) ed.finish_mmap(ind_ious) ed.finish_mmap(set_ious) ed.finish_mmap(set_ious_train) ed.finish_mmap(set_ious_val)
def save_neural_statistics(directory, blob, split=None): ed = expdir.ExperimentDirectory(directory) info = ed.load_info() dataset = info.dataset blob_info = ed.load_info(blob=blob) if 'broden' in dataset: assert (split is not None) suffix = '_%s' % split else: suffix = '' if ed.has_mmap(blob=blob, part='act_probs%s' % suffix): print('%s already exists' % ed.mmap_filename(blob=blob, part='act_probs%s' % suffix)) return acts = ed.open_mmap(blob=blob, mode='r', dtype='float32', shape=blob_info.shape) if 'broden' in dataset: ds = loadseg.SegmentationData(dataset) split_ind = np.array([ True if ds.split(i) == split else False for i in range(ds.size()) ]) L = ds.label_size() elif 'imagenet' in dataset or 'ILSVRC' in dataset: assert (split is None) L = 1000 split_ind = True image_to_label = load_image_to_label(directory, blob=blob) K = blob_info.shape[1] probs = np.zeros((L, K)) mus = np.zeros((L, K)) sigmas = np.zeros((L, K)) for class_i in range(L): class_idx = np.where(split_ind & image_to_label[:, class_i])[0] if len(class_idx) == 0: probs[class_i, :] = np.nan mus[class_i, :] = np.nan sigmas[class_i, :] = np.nan print class_i, 'no examples' continue print class_i max_acts = np.amax(acts[class_idx], axis=(2, 3)) for filter_i in range(blob_info.shape[1]): nz_idx = np.where(max_acts[:, filter_i] > 0)[0] probs[class_i][filter_i] = len(nz_idx) / float( len(max_acts[:, filter_i])) try: mus[class_i][filter_i] = np.mean(max_acts[nz_idx, filter_i]) sigmas[class_i][filter_i] = np.std(max_acts[nz_idx, filter_i]) except: mus[class_i][filter_i] = np.nan sigmas[class_i][filter_i] = np.nan print class_i, filter_i, 'no nonzero idx' probs_mmap = ed.open_mmap(blob=blob, part='act_probs%s' % suffix, mode='w+', dtype='float32', shape=probs.shape) mus_mmap = ed.open_mmap(blob=blob, part='act_mus%s' % suffix, mode='w+', dtype='float32', shape=mus.shape) sigmas_mmap = ed.open_mmap(blob=blob, part='act_sigmas%s' % suffix, mode='w+', dtype='float32', shape=sigmas.shape) probs_mmap[:] = probs[:] mus_mmap[:] = mus[:] sigmas_mmap[:] = sigmas[:] ed.finish_mmap(probs_mmap) ed.finish_mmap(mus_mmap) ed.finish_mmap(sigmas_mmap) print('Finished saving neural statistics for %s' % blob)
def create_probe( directory, dataset, weights, mean, blobs): data = loadseg.SegmentationData(dataset) ed = expdir.ExperimentDirectory(directory)
def reverse_linear_probe(directory, blob, filter_i, suffix='', prev_suffix=None, batch_size=64, quantile=0.005, bias=False, positive=False, use_svm=True, num_examples=1000, num_epochs=30, lr=1e-4, momentum=0.9, l1_weight_decay=0, l2_weight_decay=0, validation=False, nesterov=False, lower_bound=None, cuda=False): ed = expdir.ExperimentDirectory(directory) info = ed.load_info() ds = loadseg.SegmentationData(info.dataset) L = ds.label_size() N = ds.size() (Hs, Ws) = get_seg_size(info.input_dim) blob_info = ed.load_info(blob=blob) shape = blob_info.shape K = shape[1] if quantile == 1: thresh = np.zeros((1, 1)) else: quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(K, -1)) #threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))] threshold = quantdata[filter_i, int(round(quantdata.shape[1] * quantile))] #thresh = threshold[:, np.newaxis, np.newaxis] thresh = threshold blobdata = ed.open_mmap(blob=blob, mode='r', shape=shape) #upsampled_data = ed.open_mmap(blob=blob, part='upsampled', mode='r') conceptdata = ed.open_mmap(part='concept_data', mode='r', shape=(N, L, Hs, Ws)) train_idx = np.array([i for i in range(N) if ds.split(i) == 'train']) val_idx = np.array([i for i in range(N) if ds.split(i) == 'val']) # calculate mean non-negative filter activation perc_label = [] thresh_idx = [] for i in train_idx: num_thresh = np.sum((blobdata[i, filter_i] > thresh).astype(float)) perc_label.append(num_thresh / float(np.prod(shape[2:]))) if num_thresh > 0: thresh_idx.append(i) #perc_label.append(num_nz / float(np.prod(shape[2:]))) perc_label = np.array(perc_label) alpha_unnorm = float(1. - np.mean(perc_label)) alpha = float(1. - np.mean(perc_label[thresh_idx])) train_idx = train_idx[thresh_idx] print('Alpha for filter %d: %f (%f unnorm)' % (filter_i, alpha, alpha_unnorm)) print('# above thresh train examples for filter %d: %d' % (filter_i, len(thresh_idx))) if use_svm: train_subset_idx = range(num_examples) val_subset_idx = range(num_examples) start = time.time() print 'Selecting training data...' X_train = np.mean(np.mean(conceptdata[train_idx[train_subset_idx], :, 52:62, 52:62], axis=-1), axis=-1) Y_train = (blobdata[train_idx[train_subset_idx], filter_i, 5, 5] > thresh).astype(int) print 'Finished selecting training data in %d secs. Fitting SVM...' % ( time.time() - start) start = time.time() clf = svm.SVC(kernel="linear", class_weight="balanced") clf.fit(X_train, Y_train) print 'Finished fitting SVM in %d secs. Selecting validation data...' % ( time.time() - start) start = time.time() X_val = np.mean(np.mean(conceptdata[train_idx[val_subset_idx], :, 52:62, 52:62], axis=-1), axis=-1) Y_val = (blobdata[val_idx[val_subset_idx], filter_i, 5, 5] > thresh).astype(float) print 'Finished selecting validation data in %d secs. Predicting for validation data...' % ( time.time() - start) start = time.time() Y_pred = clf.predict(X_val) print 'Finished predicting validation in %d secs.' % (time.time() - start) print 'Val MSE:', np.mean((Y_pred - Y_val)**2) joblib.dump( clf, os.path.join( directory, "%s-filter_i_%d_num_examples_%d.pkl" % (blob, filter_i, num_examples))) else: layer = CustomLayer(num_features=L - 1, upsample=False, act=False, positive=positive, bias=bias, cuda=cuda) if prev_suffix is not None: prev_weights_mmap = ed.open_mmap(blob=blob, part='filter_i_%d_weights%s' % (filter_i, prev_suffix), mode='r', shape=layer.weights.shape) layer.weights.data[:] = torch.Tensor(prev_weights_mmap[:]) if cuda: layer = layer.cuda() #if quantile == 1: # criterion = nn.MSELoss() #else: # criterion = nn.BCEWithLogitsLoss() #if cuda: # criterion = criterion.cuda() criterion = lambda x, y: BCELoss2d(x, y, alpha) optimizer = Custom_SGD(layer.parameters(), lr, momentum, l1_weight_decay=l1_weight_decay, l2_weight_decay=l2_weight_decay, nesterov=nesterov, lower_bound=lower_bound) results = np.zeros((8, num_epochs)) for t in range(num_epochs): (trn_loss, trn_set_iou, trn_ind_ious, trn_c_given_f, trn_f_given_c) = run_epoch(blobdata, conceptdata, train_idx, filter_i, thresh, layer, batch_size, criterion, optimizer, t + 1, train=True, cuda=cuda, iou_threshold=0) (val_loss, val_set_iou, val_ind_ious, val_c_given_f, val_f_given_c) = run_epoch(blobdata, conceptdata, val_idx, filter_i, thresh, layer, batch_size, criterion, optimizer, t + 1, train=False, cuda=cuda, iou_threshold=0) results[0][t] = trn_loss results[1][t] = val_loss results[2][t] = trn_set_iou results[3][t] = val_set_iou results[4][t] = trn_c_given_f results[5][t] = val_c_given_f results[6][t] = trn_f_given_c results[7][t] = val_f_given_c ind_ious = np.zeros(N) ind_ious[train_idx] = trn_ind_ious ind_ious[val_idx] = val_ind_ious ind_ious_mmap = ed.open_mmap(blob=blob, part='filter_i_%d_ind_ious%s' % (filter_i, suffix), mode='w+', shape=ind_ious.shape) ind_ious_mmap[:] = ind_ious[:] ed.finish_mmap(ind_ious_mmap) results_mmap = ed.open_mmap(blob=blob, part='filter_i_%d_results%s' % (filter_i, suffix), mode='w+', shape=results.shape) results_mmap[:] = results[:] ed.finish_mmap(results_mmap) # save learned weights (and bias) weights = layer.weight.data.cpu().numpy() weights_mmap = ed.open_mmap(blob=blob, part='filter_i_%d_weights%s' % (filter_i, suffix), mode='w+', shape=weights.shape) weights_mmap[:] = weights[:] ed.finish_mmap(weights_mmap) if bias: bias_v = layer.bias.data.cpu().numpy() bias_mmap = ed.open_mmap(blob=blob, part='filter_i_%d_bias%s' % (filter_i, suffix), mode='w+', shape=(1, )) bias_mmap[:] = bias_v[:] ed.finish_mmap(bias_mmap)
def compute_distance_correlation(directory, blob, out_file=None, learning_rate=1e-4, momentum=0.9, nesterov=False, num_epochs=10, batch_size=128, randomize=True, cuda=False, verbose=False): ed = expdir.ExperimentDirectory(directory) info = ed.load_info() ds = loadseg.SegmentationData(info.dataset) L = ds.label_size() N = ds.size() blob_info = ed.load_info(blob=blob) K = blob_info.shape[1] categories = np.array(ds.category_names()) label_names = np.array([ds.label[i]['name'] for i in range(L)]) (Hs, Ws) = get_seg_size(info.input_dim) #concept_data = ed.open_mmap(part='concept_data', mode='r', # shape=(N,L,Hs,Ws)) #upsampled_data = ed.open_mmap(blob=blob, part='upsampled', mode='r', # shape=(N,K,Hs,Ws)) concept_fn = ed.mmap_filename(part='concept_data') upsampled_fn = ed.mmap_filename(blob=blob, part='upsampled') concept_data = cached_memmap(concept_fn, mode='r', dtype='float32', shape=(N, L, Hs, Ws)) upsampled_data = cached_memmap(upsampled_fn, mode='r', dtype='float32', shape=(N, K, Hs, Ws)) concept_weights = WeightedData(L) act_weights = WeightedData(K) if cuda: concept_weights = concept_weights.cuda() act_weights = act_weights.cuda() criterion = lambda x, y: -1 * distcorr(x, y, use_quad=False) optimizer = torch.optim.SGD([{ 'params': concept_weights.parameters() }, { 'params': act_weights.parameters() }], lr=args.learning_rate, momentum=args.momentum, nesterov=args.nesterov) train_idx = np.array([i for i in range(N) if ds.split(i) == 'train']) val_idx = np.array([i for i in range(N) if ds.split(i) == 'val']) train_losses = np.zeros(num_epochs) val_losses = np.zeros(num_epochs) for t in range(num_epochs): train_loss = run_epoch(concept_data[:, :, Hs / 2, Ws / 2], upsampled_data[:, :, Hs / 2, Ws / 2], concept_weights, act_weights, train_idx, criterion, optimizer, t + 1, batch_size=batch_size, train=True, randomize=randomize, cuda=cuda, verbose=verbose) val_loss = run_epoch(concept_data[:, :, Hs / 2, Ws / 2], upsampled_data[:, :, Hs / 2, Ws / 2], concept_weights, act_weights, val_idx, criterion, optimizer, t + 1, batch_size=batch_size, train=False, randomize=randomize, cuda=cuda, verbose=verbose) train_losses[t] = train_loss val_losses[t] = val_loss #print(distcorr(x_weights(Variable(torch.Tensor(x).cuda())), # y_weights(Variable(torch.Tensor(y).cuda())).data.cpu().numpy())) print(concept_weights.weight) print(act_weights.weight) results = {} if out_file is not None: if verbose: start = time.time() print('Saving results...') results['directory'] = directory results['blob'] = blob results['learning_rate'] = learning_rate results['momentum'] = momentum results['nesterov'] = nesterov results['batch_size'] = batch_size results['train_losses'] = train_losses results['val_losses'] = val_losses results['num_epochs'] = num_epochs results['randomize'] = randomize results['concept_weights'] = concept_weights.weight.data.cpu().numpy() results['act_weights'] = act_weights.weight.data.cpu().numpy() pkl.dump(results, open(out_file, 'wb')) if verbose: print('Saved results at %s in %d secs.' % (out_file, time.time() - start))
def __init__(self): """ Setup data layer according to parameters: - mean: tuple of mean values to subtract - randomize: load in random order (default: True) - seed: seed for randomization (default: None / current time) """ # config # params = eval(self.param_str) self.directory = TEST_DIR # Really should be from param_str self.split = params['split'] # I have not implemented splits yet self.mean = numpy.array(params['mean']) self.random = params.get('randomize', True) self.random_flip = True #params.get('randomize', True) self.seed = params.get('seed', None) self.batch_size = params.get('batch_size', 1) self.disp = 0 self.categories = ['object', 'part', 'texture', 'material', 'color'] # self.categories = ['object'] self.categories_num_class = [584, 234, 47, 32, 11] self.segmentation_shape = params.get('segmentation_shape', None) self.splitmap = {'train': 1, 'val': 2} # Convert to 2-dimensional shape. if self.segmentation_shape and len(numpy.shape( self.segmentation_shape)) == 0: self.segmentation_shape = (self.segmentation_shape, ) * 2 # Specific object classes to ignore. self.blacklist = { #'object': [1,2] # wall, floor, ceiling, sky: in uniseg: 4 become tree!! } # Thresholds to ignore: these classes and any ones rarer (higher). self.outliers = { 'object': 537, # brick occurs only 9 times in test_384. #'part': 155, # porch occurs only 9 times in test_384 # if switching to uniseg, switch 561->544. # because there are fewer object classes. # part classes remain the same. } # make eval deterministic if 'train' not in self.split: self.random = False self.random_flip = False # Load up metadata for images and labels self.datasource = loadseg.SegmentationData(self.directory, categories=self.categories) self.prefetcher = loadseg.SegmentationPrefetcher( self.datasource, split=self.split, categories=['image'] + self.categories, segmentation_shape=self.segmentation_shape, batch_size=self.batch_size, randomize=self.random) # ahead=12) # Now make a blacklist map for blacklisted types self.zeromap = {} for k, z in self.blacklist.items(): self.zeromap[k] = numpy.arange(self.datasource.label_size(k)) self.zeromap[k][z] = 0 for k, z in self.outliers.items(): if k not in self.zeromap: self.zeromap[k] = numpy.arange(self.datasource.label_size(k)) self.zeromap[k][numpy.arange(z, self.datasource.label_size(k))] = 0 # Build our category map which merges the category map and the zeromap self.categorymap = {} for cat in self.categories: catmap = self.datasource.category_index_map(cat) if cat in self.zeromap: catmap = self.zeromap[cat][catmap] self.categorymap[cat] = catmap
def consolidate_disc_probe(directory, blob, bias=False, num_filters=None, suffix='', epochs=30, delete=False): ed = expdir.ExperimentDirectory(directory) try: info = ed.load_info() except: ed = expdir.ExperimentDirectory(os.path.join(directory, 'train')) #ed_val = expdir.ExperimentDirectory(os.path.join(directory, 'val')) info = ed.load_info() assert ('imagenet' in info.dataset or 'ILSVRC' in info.dataset) blob_info = ed.load_info(blob=blob) shape = blob_info.shape if 'broden' in info.dataset: ds = loadseg.SegmentationData(info.dataset) L = ds.label_size() elif 'imagenet' in info.dataset or 'ILSVRC' in info.dataset: L = 1000 N = shape[0] # total number of images in the dataset K = shape[1] # number of units for the given blob F = 1 if num_filters is None else len(num_filters) + 1 if num_filters is None: suffixes = [suffix] else: suffixes = ['%s_num_filters_%d' % (suffix, n) for n in num_filters] suffixes.append(suffix) suffix = '%s_num_filters_%d' % (suffix, F) if (ed.has_mmap(blob=blob, part='linear_weights_disc%s' % suffix) and (not bias or ed.has_mmap(blob=blob, part='linear_bias_disc%s' % suffix))): print('Linear weights (and bias) have already been consolidated') print ed.mmap_filename(blob=blob, part='linear_weights_disc%s' % suffix) else: weights_mmap = ed.open_mmap(blob=blob, part='linear_weights_disc%s' % suffix, mode='w+', dtype='float32', shape=(L, F, K)) if bias: bias_mmap = ed.open_mmap(blob=blob, part='linear_bias_disc%s' % suffix, mode='w+', dtype='float32', shape=(L, F)) results_mmap = ed.open_mmap(blob=blob, part='linear_results_disc%s' % suffix, mode='w+', dtype='float32', shape=(L, F, 4, epochs)) missing_idx = [] for l in range(L): if not ed.has_mmap(blob=blob, part='label_i_%d_weights_disc%s' % (l, suffixes[0])): missing_idx.append(l) continue for i in range(len(suffixes)): suffix = suffixes[i] label_weights_mmap = ed.open_mmap( blob=blob, part='label_i_%d_weights_disc%s' % (l, suffix), mode='r', dtype='float32', shape=(K, )) weights_mmap[l, i, :] = label_weights_mmap[:] if bias: label_bias_mmap = ed.open_mmap( blob=blob, part='label_i_%d_bias_disc%s' % (l, suffix), mode='r', dtype='float32', shape=(1, )) bias_mmap[l, i] = label_bias_mmap[:] label_results_mmap = ed.open_mmap( blob=blob, part='label_i_%d_results_disc%s' % (l, suffix), mode='r', dtype='float32', shape=(4, epochs)) results_mmap[l, i, :, :] = label_results_mmap[:, :] print l ed.finish_mmap(weights_mmap) if bias: ed.finish_mmap(bias_mmap) ed.finish_mmap(results_mmap) print('Finished consolidating existing weights files ' + '(Files for %d labels were missing).' % len(missing_idx)) print(missing_idx) if delete: c_w = 0 c_b = 0 c_r = 0 print('Deleting all unnecessary label weights (and bias) files...') for l in range(L): for i in range(F): suffix = suffixes[i] if ed.has_mmap(blob=blob, part='label_i_%d_weights_disc%s' % (l, suffix)): fn = ed.mmap_filename(blob=blob, part='label_i_%d_weights_disc%s' % (l, suffix)) os.remove(fn) c_w += 1 if bias and ed.has_mmap(blob=blob, part='label_i_%d_bias_disc%s' % (l, suffix)): fn = ed.mmap_filename(blob=blob, part='label_i_%d_bias_disc%s' % (l, suffix)) os.remove(fn) c_b += 1 if ed.has_mmap(blob=blob, part='label_i_%d_results_disc%s' % (l, suffix)): fn = ed.mmap_filename(blob=blob, part='label_i_%d_results_disc%s' % (l, suffix)) os.remove(fn) c_r += 1 print('Removed %d weights, %d bias, and %d results files.' % (c_w, c_b, c_r))
def linear_probe(directory, blob, label_i, suffix='', init_suffix='', num_filters=None, batch_size=16, ahead=4, quantile=0.005, bias=False, positive=False, num_epochs=30, lr=1e-4, momentum=0.9, l1_weight_decay=0, l2_weight_decay=0, validation=False, nesterov=False, lower_bound=None, min_train=None, max_train=None, max_val=None, cuda=False): # Make sure we have a directory to work in #qcode = ('%f' % quantile).replace('0.','').rstrip('0') ed = expdir.ExperimentDirectory(directory) # Check if linear weights have already been learned print ed.mmap_filename(blob=blob, part='label_i_%s_weights%s' % (label_i, suffix)) if ed.has_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix)): print('%s already has %s, so skipping.' % (directory, ed.mmap_filename(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix)))) return # Load probe metadata info = ed.load_info() ih, iw = info.input_dim # Load blob metadata blob_info = ed.load_info(blob=blob) shape = blob_info.shape unit_size = shape[1] fieldmap = blob_info.fieldmap # Load the blob quantile data and grab thresholds if quantile == 1: thresh = np.zeros((unit_size,1,1)) else: quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(unit_size, -1)) threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))] thresh = threshold[:, np.newaxis, np.newaxis] #print np.max(thresh), thresh.shape, type(thresh) # Map the blob activation data for reading fn_read = ed.mmap_filename(blob=blob) # Load the dataset ds = loadseg.SegmentationData(info.dataset) # Get all the categories the label is a part of label_categories = ds.label[label_i]['category'].keys() num_categories = len(label_categories) # Get label name label_name = ds.name(category=None, j=label_i) blobdata = cached_memmap(fn_read, mode='r', dtype='float32', shape=shape) # Get indices of images containing the given label if not has_image_to_label(directory): print('image_to_label does not exist in %s; creating it now...' % directory) create_image_to_label(directory, batch_size=batch_size, ahead=ahead) image_to_label = load_image_to_label(directory) label_idx = np.where(image_to_label[:, label_i])[0] train_idx = np.array([i for i in label_idx if ds.split(i) == 'train']) val_idx = np.array([i for i in label_idx if ds.split(i) == 'val']) if min_train is not None and len(train_idx) < min_train: print('Number of training examples for label %d (%s) is %d, which is less than the minimum of %d so skipping.' % (label_i, label_name, len(train_idx), min_train)) if max_train is not None and len(train_idx) > max_train: train_idx = train_idx[:max_train] if max_val is not None and len(val_idx) > max_val: val_idx = val_idx[:max_val] print('Total number of images containing label %d (%s): %d' % ( label_i, label_name, len(label_idx))) try: train_loader = loadseg.SegmentationPrefetcher(ds, categories=label_categories, indexes=train_idx, once=False, batch_size=batch_size, ahead=ahead, thread=True) except IndexError as err: print(err.args) return sw = 0 sh = 0 perc_label = [] train_label_categories = [] for batch in train_loader.batches(): for rec in batch: # Check that the same segmentation dimensions are used for all # examples sw_r, sh_r = [rec[k] for k in ['sw', 'sh']] if sw == 0 and sh == 0: sw = sw_r sh = sh_r else: assert(sw == sw_r and sh == sh_r) for cat in label_categories: if rec[cat] != []: train_label_categories.append(cat) if type(rec[cat]) is np.ndarray: perc_label.append(np.sum(rec[cat] == label_i) / float(sw * sh)) else: perc_label.append(1.) break assert(len(perc_label) == len(train_idx)) # Compute reduction from segmentation dimensions to image dimensions reduction = int(round(iw / float(sw))) # Calculate class-weighting alpha parameter for segmentation loss # (Note: float typecast is necessary) alpha = float(1. - np.mean(perc_label)) if alpha == 0: alpha = None print('Not using class-weighting because no pixel-level annotations') else: print('Alpha for label %d (%s): %f' % (label_i, label_name, alpha)) # Prepare segmentation loss function using class-weight alpha criterion = lambda x,y: BCELoss2d(x,y,alpha) # Prepare to learn linear weights with a sigmoid activation after # the linear layer #layer = CustomLayer(unit_size, upsample=False, act=True, positive=False) if num_filters is not None: if ed.has_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, init_suffix)): init_weights_mmap = ed.open_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, init_suffix), mode='r', dtype='float32', shape=unit_size) elif ed.has_mmap(blob=blob, part='linear_weights%s' % (init_suffix)): all_weights_mmap = ed.open_mmap(blob=blob, part='linear_weights%s' % init_suffix, mode='r', dtype='float32', shape=(ds.label_size(),unit_size)) init_weights_mmap = all_weights_mmap[label_i] else: assert(False) sorted_idx = np.argsort(np.abs(init_weights_mmap))[::-1] mask_idx = np.zeros(unit_size, dtype=int) mask_idx[sorted_idx[:num_filters]] = 1 layer = CustomLayer(unit_size, upsample=True, up_size=(sh,sw), act=True, bias=bias, positive=positive, mask_idx=torch.ByteTensor(mask_idx), cuda=cuda) else: layer = CustomLayer(unit_size, upsample=True, up_size=(sh,sw), act=True, bias=bias, positive=positive, cuda=cuda) if cuda: layer.cuda() optimizer = Custom_SGD(layer.parameters(), lr, momentum, l1_weight_decay=l1_weight_decay, l2_weight_decay=l2_weight_decay, nesterov=nesterov, lower_bound=lower_bound) if not validation: try: val_loader = loadseg.SegmentationPrefetcher(ds, categories=label_categories, indexes=val_idx, once=False, batch_size=batch_size, ahead=ahead, thread=True) except IndexError as err: print(err.args) train_loader.close() return val_label_categories = [] for batch in val_loader.batches(): for rec in batch: for cat in label_categories: if rec[cat] != []: val_label_categories.append(cat) break assert(len(val_label_categories) == len(val_idx)) for t in range(num_epochs): (_, iou) = run_epoch(blobdata, train_idx, train_label_categories, label_i, fieldmap, thresh, sh, sw, reduction, train_loader, layer, criterion, optimizer, t+1, train=True, cuda=cuda, iou_threshold=0.5) if not validation: (_, iou) = run_epoch(blobdata, val_idx, val_label_categories, label_i, fieldmap, thresh, sh, sw, reduction, val_loader, layer, criterion, optimizer, t+1, train=False, cuda=cuda, iou_threshold=0.5) # Close segmentation prefetcher (i.e. close pools) train_loader.close() if not validation: val_loader.close() # Save weights weights = (layer.mask * layer.weight).data.cpu().numpy() weights_mmap = ed.open_mmap(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix), mode='w+', dtype='float32', shape=weights.shape) weights_mmap[:] = weights[:] ed.finish_mmap(weights_mmap) if bias: bias_v = layer.bias.data.cpu().numpy() bias_mmap = ed.open_mmap(blob=blob, part='label_i_%d_bias%s' % (label_i, suffix), mode='w+', dtype='float32', shape=(1,)) bias_mmap[:] = bias_v[:] ed.finish_mmap(bias_mmap) print '%s finished' % ed.mmap_filename(blob=blob, part='label_i_%d_weights%s' % (label_i, suffix))
def label_probe(directory, blob, quantile=0.005, batch_size=16, ahead=4, start=None, end=None, suffix='', cuda=False): # Make sure we have a directory to work in qcode = ('%f' % quantile).replace('0.','').rstrip('0') ed = expdir.ExperimentDirectory(directory) # Check if label probe has already been created if (ed.has_mmap(blob=blob, part='single_set_ious%s' % suffix) and ed.has_mmap(blob=blob, part='single_ind_ious%s' % suffix)): print('label_probe_pytorch.py has already been run.') return # Load probe metadata info = ed.load_info() seg_size = get_seg_size(info.input_dim) # Load blob metadata blob_info = ed.load_info(blob=blob) shape = blob_info.shape tot_imgs = shape[0] unit_size = shape[1] # Load the blob quantile data and grab thresholds quantdata = ed.open_mmap(blob=blob, part='quant-*', shape=(unit_size, -1)) threshold = quantdata[:, int(round(quantdata.shape[1] * quantile))] thresh = threshold[:, np.newaxis, np.newaxis] # Load the dataset ds = loadseg.SegmentationData(info.dataset) # Map the blob activation data for reading #fn_read = ed.mmap_filename(blob=blob) #blobdata = cached_memmap(fn_read, mode='r', dtype='float32', shape=shape) blobdata = ed.open_mmap(blob=blob, mode='r', shape=shape) # Get image-to-labels mapping if not has_image_to_label(directory): print('image_to_label does not exist in %s; creating it now...' % directory) create_image_to_label(directory, batch_size=batch_size, ahead=ahead) image_to_label = load_image_to_label(directory) num_labels = ds.label_size() upsample = nn.Upsample(size=seg_size, mode='bilinear') set_ious_train_mmap = ed.open_mmap(blob=blob, part='single_set_train_ious%s' % suffix, mode='w+', dtype='float32', shape=(num_labels, unit_size)) set_ious_val_mmap = ed.open_mmap(blob=blob, part='single_set_val_ious%s' % suffix, mode='w+', dtype='float32', shape=(num_labels, unit_size)) set_ious_mmap = ed.open_mmap(blob=blob, part='single_set_ious%s' % suffix, mode='w+', dtype='float32', shape=(num_labels, unit_size)) ind_ious_mmap = ed.open_mmap(blob=blob, part='single_ind_ious%s' % suffix, mode='w+', dtype='float32', shape=(num_labels, tot_imgs, unit_size)) if start is None: start = 1 if end is None: end = num_labels #for label_i in range(1, num_labels): for label_i in range(start, end): print('Starting for label %d (%s)' % (label_i, ds.name(category=None, j=label_i))) label_categories = ds.label[label_i]['category'].keys() num_cats = len(label_categories) label_idx = np.where(image_to_label[:, label_i])[0] loader = loadseg.SegmentationPrefetcher(ds, categories=label_categories, indexes=label_idx, once=False, batch_size=batch_size, ahead=ahead, thread=True) loader_idx = loader.indexes N = len(loader_idx) iou_intersects = np.zeros((N, unit_size)) iou_unions = np.zeros((N, unit_size)) if num_cats > 1: rec_labcat = [] for batch in loader.batches(): for rec in batch: for cat in label_categories: if rec[cat] != []: rec_labcat.append(cat) break else: rec_labcat = [label_categories[0] for i in range(N)] i = 0 for batch in loader.batches(): start_t = time.time() if (i+1)*batch_size < N: idx = range(i*batch_size, (i+1)*batch_size) else: idx = range(i*batch_size, N) i += 1 input = torch.Tensor((blobdata[loader_idx[idx]] > thresh).astype(float)) input_var = upsample(Variable(input.cuda()) if cuda else Variable(input)) target = torch.Tensor([np.max((rec[rec_labcat[j]] == label_i).astype(float), axis=0) if type(rec[rec_labcat[j]]) is np.ndarray else np.ones(seg_size) for j, rec in enumerate(batch)]) target_var = Variable(target.unsqueeze(1).expand_as( input_var).cuda() if cuda else target.unsqueeze(1).expand_as(input_var)) iou_intersects[idx] = np.squeeze(iou_intersect_d(input_var, target_var).data.cpu().numpy()) iou_unions[idx] = np.squeeze(iou_union_d(input_var, target_var).data.cpu().numpy()) print('Batch %d/%d\tTime %f secs\tAvg Ind IOU %f\t' % (i, N/batch_size, time.time()-start_t, np.mean(np.true_divide(iou_intersects[idx], iou_unions[idx] + 1e-20)))) set_ious = np.true_divide(np.sum(iou_intersects, axis=0), np.sum(iou_unions, axis=0) + 1e-20) loader.close() best_filter = np.argmax(set_ious) print('Label %d (%s): best set IOU = %f (filter %d)' % (label_i, ds.name(category=None,j=label_i), set_ious[best_filter], best_filter)) ind_ious = np.true_divide(iou_intersects, iou_unions + 1e-20) set_ious_mmap[label_i] = set_ious ind_ious_mmap[label_i, loader_idx] = ind_ious train_idx = [i for i in range(len(loader_idx)) if ds.split(loader_idx[i]) == 'train'] val_idx = [i for i in range(len(loader_idx)) if ds.split(loader_idx[i]) == 'val'] set_ious_train_mmap[label_i] = np.true_divide(np.sum(iou_intersects[train_idx], axis=0), np.sum(iou_unions[train_idx], axis=0) + 1e-20) set_ious_val_mmap[label_i] = np.true_divide(np.sum(iou_intersects[val_idx], axis=0), np.sum(iou_unions[val_idx], axis=0) + 1e-20) #set_ious_mmap.flush() #ind_ious_mmap.flush() ed.finish_mmap(set_ious_train_mmap) ed.finish_mmap(set_ious_val_mmap) ed.finish_mmap(set_ious_mmap) ed.finish_mmap(ind_ious_mmap)