def train_jumping_windows(d, codebook, use_scale=True, trun=False, diff=False, feature='sift'): tocer = ut.TicToc() llc_dir = '../../research/jumping_windows/llc/' featdir = '../../research/jumping_windows/sift/' if feature == 'sift': trainfile = join(config.VOC_dir, 'ImageSets','Main','trainval.txt') trainfiles = open(trainfile,'r').readlines() elif feature == 'llc': trainfiles = os.listdir(featdir) grids = 4 if feature == 'sift': numcenters = codebook.shape[0] elif feature == 'llc': a = sio.loadmat(join(llc_dir,trainfiles[0]))['codes'] numcenters = a.shape[0] ccmat = np.zeros((numcenters, len(d.classes)*grids*grids+1)) e = Extractor() #first_visit = True print 'Read all features to create weights' tocer.tic() for filename in trainfiles: print filename # Load feature positions if feature == 'sift': filename = filename[:-1] + '.jpg' assignment = e.get_assignments(np.asarray([0,0,100000,1000000]), 'sift', codebook, d.get_image_by_filename(filename)) pts = assignment[:,0:2] codes = assignment[:,2] elif feature == 'llc': feaSet = sio.loadmat(join(featdir,filename))['feaSet'] x = feaSet['x'][0][0] y = feaSet['y'][0][0] pts = np.hstack((x,y)) codes = sio.loadmat(join(llc_dir,filename))['codes'] bg = np.ones((pts.shape[0], 1)) image = d.get_image_by_filename(filename[:-4]+'.jpg') im_ind = d.get_img_ind(image) gt = d.get_ground_truth_for_img_inds([im_ind]) for row in gt.arr: #for row in gt.arr[0,:]: cls = row[gt.cols.index('cls_ind')] bbox = row[0:4] inds = get_indices_for_pos(pts, bbox[0], bbox[0]+bbox[2], bbox[1], bbox[1]+bbox[3]) bg[inds] = 0 selbins = get_selbins(grids, inds, pts, bbox) binidx = np.transpose(sub2ind([grids, grids], selbins[:,0], selbins[:,1]) \ + cls*grids*grids); if feature == 'sift': binidx -= np.tile(grids, binidx.shape) idx = get_idx(inds, codes, ccmat.shape, feature, binidx) for i in idx: [x, y] = ind2sub(ccmat.shape[0], i) #print x, y ccmat[x, y] = ccmat[x, y] + 1 # Now record background features cls = len(d.classes)*grids*grids inds = np.where(bg > 0)[0] if feature == 'llc': ind = np.where(codes[:,inds].data > 0)[0] words = codes[:,inds].nonzero()[0][ind] words = np.unique(words) elif feature == 'sift': words = codes[inds] for w in words: ccmat[w, cls] = ccmat[w, cls] + 1 #sio.savemat('ccmat', {'ccmat2': ccmat}) #break print 'features counted' tocer.toc() tocer.tic() # counted all words for all images&object, now compute weights div = np.sum(ccmat,1) for didx in range(len(div)): ta = div[didx] if ta == 0: ccmat[didx, :] = 2.5 continue ccmat[didx, :] /= ta print 'computed weights' tocer.toc() tocer.tic() numwords = 500 [sortedprob, discwords] = sort_cols(ccmat, numwords) #return # Lookup for every class for cls_idx in range(len(d.classes)): #for cls_idx in [14]: cls = d.classes[cls_idx] print cls clswords = discwords[:, cls_idx*grids*grids:(cls_idx+1)*grids*grids] binidx,_ = np.meshgrid(range(grids*grids), np.zeros((clswords.shape[0],1))) clswords = sub2ind([numcenters, grids*grids], line_up_cols(clswords), line_up_cols(binidx)) # === GOOD! === wordprobs = sortedprob[:, cls_idx*grids*grids:(cls_idx+1)*grids*grids]; wordprobs = line_up_cols(wordprobs); [wordprobs, idx] = sort_cols(wordprobs); clswords = mat_from_col_idx(clswords, idx); bbinfo = LookupTable(grids, numwords, clswords, wordprobs, numcenters, cls) fileids = d.get_ground_truth_for_class(cls) last_filename = '000000' for row_idx in range(fileids.shape[0]): row = fileids.arr[row_idx, :] filename = d.images[row[fileids.cols.index('img_ind')].astype('int32')].name[:-4] if not os.path.isfile(join(llc_dir, filename+'.mat')): continue print filename if not last_filename == filename: # This is a new filename. Load codes and pts. feaSet = sio.loadmat(join(featdir,filename))['feaSet'] x = feaSet['x'][0][0] y = feaSet['y'][0][0] pts = np.hstack((x,y)) bg = np.ones((pts.shape[0], 1)) codes = sio.loadmat(join(llc_dir,filename))['codes'] last_filename = filename bbox = row[0:4] inds = get_indices_for_pos(pts, bbox[0], bbox[0]+bbox[2], bbox[1], bbox[1]+bbox[3]) # Compute grid that each point falls into selbins = get_selbins(grids, inds, pts, bbox) binidx = np.transpose(sub2ind([grids, grids], selbins[:,0]-1, selbins[:,1]-1)); selbins = get_selbins(grids, inds, pts, bbox) binidx = np.transpose(sub2ind([grids, grids], selbins[:,0], selbins[:,1]) \ + cls*grids*grids); idx = get_idx(inds, codes, ccmat.shape, feature, binidx) if feature == 'llc': ind = np.where(codes[:,inds].data > 0)[0] ind = codes.nonzero()[1][ind] elif feature == 'sift': ind = inds #intersect idx and clswords clswords = np.unique(np.asarray(clswords)) I = np.sort(idx) # J = np.sort(ind) idx = np.unique(np.asarray(idx)) imgwords = np.unique(np.intersect1d(np.asarray(idx), np.asarray(clswords), assume_unique=True)) for c in imgwords: idc = np.where(I == c)[0] featidx = inds[ind[idc]] featpts = pts[featidx] featpts = np.hstack((featpts, featpts)) ins = np.tile(bbox, (featpts.shape[0],1)) - featpts + np.matrix([0,0,bbox[0]-1,bbox[1]-1]) bbinfo.insert(c, ins) bbinfo.perform_mean_shifts() bbinfo.top_words = np.asarray(bbinfo.wordprobs.argsort(axis=0))[::-1] bbinfo.save() print 'computed all lookup tables' tocer.toc()
def main(): parser = argparse.ArgumentParser(description='Execute different functions of our system') parser.add_argument('mode', choices=[ 'window_stats', 'evaluate_metaparams', 'evaluate_jw', 'evaluate_get_pos_windows', 'train_svm', 'extract_sift','extract_assignments','extract_codebook', 'evaluate_jw_grid', 'final_metaparams', 'assemble_dpm_dets','ctfdet','assemble_ctf_dets' ]) parser.add_argument('--test_dataset', choices=['val','test','train'], default='test', help='dataset to use for testing. the training dataset \ is automatically inferred (val->train and test->trainval).') parser.add_argument('--first_n', type=int, help='only take the first N images in the datasets') parser.add_argument('--bounds', type=str, help='the start_time and deadline_time for the ImagePolicy and corresponding evaluation. ex: (1,5)') parser.add_argument('--name', help='name for this run') parser.add_argument('--priors', default='random', help= \ "list of choice for the policy for selecting the next action. choose from random, oracle,fixed_order, no_smooth, backoff. ex: --priors=random,oracle,no_smooth") parser.add_argument('--compare_evals', action='store_true', default=False, help='plot all the priors modes given on same plot'), parser.add_argument('--detector', choices=['perfect','perfect_with_noise', 'dpm','ctf'], default='perfect', help='detector type') parser.add_argument('--force', action='store_true', default=False, help='force overwrite') parser.add_argument('--gist', action='store_true', default=False, help='use GIST as one of the actions') parser.add_argument('--clear_tmp', action='store_true', default=False, help='clear the cached windows folder before running'), parser.add_argument('--feature_type', choices=['sift','dsift'], default='dsift', help='use this feature type'), parser.add_argument('--kernel', choices=['chi2','rbf'], default='chi2', help='kernel to train svm on'), args = parser.parse_args() if args.priors: args.priors = args.priors.split(',') if args.bounds: args.bounds = [float(x) for x in re.findall(r'\d+', args.bounds)] assert(len(args.bounds)==2) print(args) # Load the dataset dataset = Dataset('full_pascal_'+args.test_dataset) if args.first_n: dataset.images = dataset.images[:args.first_n] # Infer train_dataset if args.test_dataset=='test': train_dataset = Dataset('full_pascal_trainval') elif args.test_dataset=='val': train_dataset = Dataset('full_pascal_train') else: print("Impossible, setting train_dataset to dataset") train_dataset = dataset # Create window generator sw = SlidingWindows(dataset,train_dataset) if args.clear_tmp: dirname = config.get_sliding_windows_cached_dir(train_dataset.get_name()) shutil.rmtree(dirname) dirname = config.get_sliding_windows_cached_dir(dataset.get_name()) shutil.rmtree(dirname) if args.mode=='assemble_dpm_dets': policy = DatasetPolicy(dataset,train_dataset,sw) dets = policy.load_ext_detections(dataset,suffix='dpm_may25') if args.mode=='assemble_ctf_dets': policy = DatasetPolicy(dataset,train_dataset,sw) dets = policy.load_ext_detections(dataset,'ctf','ctf_default') dets = policy.load_ext_detections(dataset,'ctf','ctf_nohal') dets = policy.load_ext_detections(dataset,'ctf', 'ctf_halfsize') if args.mode=='evaluate_get_pos_windows': evaluate_get_pos_windows(train_dataset) return if args.mode=='window_stats': "Compute and plot the statistics of ground truth window parameters." results = SlidingWindows.get_dataset_window_stats(train_dataset,plot=True) if args.mode=='ctfdet': """Run Pedersoli's detector on the dataset and assemble into one Table.""" run_pedersoli(dataset) if args.mode=='evaluate_jw': """ Evaluate the jumping window approach by producing plots of recall vs. #windows. """ # TODO hack: both sw and jw should subclass something like WindowGenerator jw = JumpingWindowsDetector(use_scale=True) sw.jw = jw #classes = dataset.classes classes = ['car'] # classes = ['bicycle' ,'car','horse', 'sofa',\ # 'bird', 'chair', 'motorbike', 'train',\ # 'boat', 'cow', 'person', 'tvmonitor',\ # 'bottle','diningtable', 'pottedplant',\ # 'bus','dog' ,'sheep'] for cls_idx in range(comm_rank, len(classes), comm_size): #for cls in dataset.classes: cls = classes[cls_idx] dirname = config.get_jumping_windows_dir(dataset.get_name()) filename = os.path.join(dirname,'%s'%cls) sw.evaluate_recall(cls, filename, metaparams=None, mode='jw', plot=True) if args.mode=='evaluate_jw_grid': """ Evaluate the jumping window approach by producing plots of recall vs. #windows. """ sw = SlidingWindows(dataset,train_dataset) jw = JumpingWindowsDetectorGrid() sw.jw = jw for cls in dataset.classes: dirname = config.get_jumping_windows_dir(dataset.get_name()) filename = os.path.join(dirname,'%s'%cls) if os.path.isfile(config.data_dir + 'JumpingWindows/'+cls): sw.evaluate_recall(cls, filename, metaparams=None, mode='jw', plot=True) if args.mode=='train_svm': randomize = not os.path.exists('/home/tobibaum') d = Dataset('full_pascal_train') dtest = Dataset('full_pascal_val') e = Extractor() classes = config.pascal_classes num_words = 3000 iters = 5 feature_type = 'dsift' codebook_samples = 15 num_pos = 'max' testsize = 'max' if args.first_n: num_pos = args.first_n testsize = 1.5*num_pos kernel = args.kernel if comm_rank == 0: ut.makedirs(config.data_dir + 'features/' + feature_type + '/times/') ut.makedirs(config.data_dir + 'features/' + feature_type + '/codebooks/times/') ut.makedirs(config.data_dir + 'features/' + feature_type + '/svms/train_times/') for cls_idx in range(comm_rank, len(classes), comm_size): #for cls in classes: cls = classes[cls_idx] codebook = e.get_codebook(d, feature_type) pos_arr = d.get_pos_windows(cls) neg_arr = d.get_neg_windows(pos_arr.shape[0], cls, max_overlap=0) if not num_pos == 'max': if not randomize: pos_arr = pos_arr[:num_pos] neg_arr = pos_arr[:num_pos] else: rand = np.random.random_integers(0, pos_arr.shape[0] - 1, size=num_pos) pos_arr = pos_arr[rand] rand = np.random.random_integers(0, neg_arr.shape[0] - 1, size=num_pos) neg_arr = neg_arr[rand] pos_table = Table(pos_arr, ['x','y','w','h','img_ind']) neg_table = Table(neg_arr, pos_table.cols) train_with_hard_negatives(d, dtest, num_words,codebook_samples,codebook,\ cls, pos_table, neg_table,feature_type, \ iterations=iters, kernel=kernel, L=2, \ testsize=testsize,randomize=randomize) if args.mode=='evaluate_metaparams': """ Grid search over metaparams values for get_windows_new, with the AUC of recall vs. # windows evaluation. """ sw.grid_search_over_metaparams() return if args.mode=='final_metaparams': dirname = config.get_sliding_windows_metaparams_dir(train_dataset.get_name()) # currently these are the best auc/complexity params best_params_for_classes = [ (62,15,12,'importance',0), #aeroplane (83,15,12,'importance',0), #bicycle (62,15,12,'importance',0), #bird (62,15,12,'importance',0), #boat (125,12,12,'importance',0), #bottle (83,12,9,'importance',0), #bus (125,15,9,'importance',0), #car (125,12,12,'linear',0), #cat (125,15,9,'importance',0), #chair (125,9,6,'importance',0), #cow (125,15,6,'linear',0), #diningtable (62,15,12,'importance',0), #dog (83,15,6,'importance',0), #horse (83,12,6,'importance',0), #motorbike (83,15,12,'importance',0), #person (83,15,6,'importance',0), #pottedplant (83,15,12,'importance',0), #sheep (83,9,6,'importance',0), #sofa (62,12,6,'importance',0), #train (62,12,12,'importance',0), #tvmonitor (125,9,12,'importance',0) #all ] # ACTUALLY THEY ARE ALL THE SAME! cheap_params = (62, 9, 6, 'importance', 0) for i in range(comm_rank,dataset.num_classes(),comm_size): cls = dataset.classes[i] best_params = best_params_for_classes[i] #samples,num_scales,num_ratios,mode,priority,cls = cheap_params metaparams = { 'samples_per_500px': samples, 'num_scales': num_scales, 'num_ratios': num_ratios, 'mode': mode, 'priority': 0 } filename = '%s_%d_%d_%d_%s_%d'%( cls, metaparams['samples_per_500px'], metaparams['num_scales'], metaparams['num_ratios'], metaparams['mode'], metaparams['priority']) filename = os.path.join(dirname,filename) tables = sw.evaluate_recall(cls,filename,metaparams,'sw',plot=True,force=False) metaparams = { 'samples_per_500px': samples, 'num_scales': num_scales, 'num_ratios': num_ratios, 'mode': mode, 'priority': 1 } filename = '%s_%d_%d_%d_%s_%d'%( cls, metaparams['samples_per_500px'], metaparams['num_scales'], metaparams['num_ratios'], metaparams['mode'], metaparams['priority']) filename = os.path.join(dirname,filename) tables = sw.evaluate_recall(cls,filename,metaparams,'sw',plot=True,force=False) return if args.mode=='extract_sift': e=Extractor() e.extract_all(['sift'], ['full_pascal_trainval','full_pascal_test'], 0, 0) if args.mode=='extract_assignments': e=Extractor() feature_type = 'sift' for image_set in ['full_pascal_trainval','full_pascal_test']: d = Dataset(image_set) codebook = e.get_codebook(d, feature_type) print 'codebook loaded' for img_ind in range(comm_rank,len(d.images),comm_size): img = d.images[img_ind] #for img in d.images: e.get_assignments(np.array([0,0,img.size[0],img.size[1]]), feature_type, \ codebook, img) if args.mode=='extract_codebook': d = Dataset('full_pascal_trainval') e = Extractor() codebook = e.get_codebook(d, args.feature_type)
codebook = e.get_codebook(dtest, 'sift') filename_lookup = join(foldname_lookup,cls) store_file = open(filename_lookup, 'r') bbinfo = cPickle.load(store_file) test_gt = gt_t.arr npos = test_gt.shape[0] test_imgs = test_gt[:,gt_t.cols.index('img_ind')] #test_imgs = np.unique(test_imgs) test_imgs = np.unique(test_imgs)[:1] for i in range(comm_rank, len(test_imgs), comm_size): img_ind = int(test_imgs[i]) image = dtest.images[img_ind] codes = e.get_assignments(np.array([0,0,100000,100000]), 'sift', codebook, image) dets = generate_jwin(bbinfo, image, cls, codes, codes[:,0:2]) bbox_curr = np.zeros((dets.shape[0], 7)) ent_idx = 0 num_rows = dets.shape[0] print dets for i, det in enumerate(dets): bbox_curr[i,:] = np.hstack((det, 1 - i/float((num_rows - 1)), cls_idx, img_ind)) # save this detections filename = join(foldname_det, image.name + "_" + cls) np.savetxt(filename, bbox_curr)