def __init__(self, dataset, L, numfolds=4): self.d = Dataset(dataset) self.e = Extractor() self.dense_codebook = self.e.get_codebook(self.d, 'dsift') self.sparse_codebook = self.e.get_codebook(self.d, 'sift') self.L = L self.numfolds = numfolds
class ClassifierConfig(): def __init__(self, dataset, L, numfolds=4): self.d = Dataset(dataset) self.e = Extractor() self.dense_codebook = self.e.get_codebook(self.d, 'dsift') self.sparse_codebook = self.e.get_codebook(self.d, 'sift') self.L = L self.numfolds = numfolds def kfold(self): train_idx, val_idx = KFold(len(len(self.d.images), self.numfolds)) self.d.create_folds(self.numfolds)
from common_imports import * from common_mpi import * import synthetic.config as config from synthetic.dataset import Dataset from synthetic.extractor import Extractor if __name__ == "__main__": d = Dataset("full_pascal_trainval") feature_type = "dsift" numpos = 15 num_words = 3000 iterations = 8 e = Extractor() all_classes = config.pascal_classes # for cls_idx in range(comm_rank, len(all_classes), comm_size): # PARALLEL # #for cls in all_classes: # cls = all_classes[cls_idx] # print cls # d, feature_type, num_words=3000,iterations=10, force_new=False, kmeansBatch=True e.get_codebook(d, feature_type, numpos, iterations, force_new=False, kmeansBatch=True)
def train_with_hard_negatives(d, dtest,cbwords, cbsamps, codebook, cls, pos_table, neg_table,feature_type,\ iterations, kernel='chi2', L=2, \ testsize = 'max',randomize=False): """ An iterative training with hard negatives -input: d - training Dataset dtest - test Dataset codebook - dsift codebook for pyramid (recommended size 3000) cls - the class to be trained pos_table - Table with cols [x,y,w,h,img_ind] neg_table - same kernel - 'linear', 'rbf', 'chi2' iterations - number of rounds of training L - levels for pyramids testsize - size of initial test """ # Featurize and pyramidize the input e = Extractor() M = codebook.shape[0] pyr_size = M*1./3.*(4**(L+1)-1) print 'get pos train pyrs' pos_pyrs = get_pyr(d,e,pos_table,pyr_size,L,codebook,feature_type,cls) print 'get neg train pyrs' neg_pyrs = get_pyr(d,e,neg_table,pyr_size,L,codebook,feature_type,cls) print 'built all positive pyramids' classification = np.asarray([1]*pos_table.arr.shape[0] + [-1]*neg_table.arr.shape[0]) filename = config.data_dir + 'features/' + feature_type + '/svms/' + kernel + \ '/'+ cls ut.makedirs(config.data_dir + 'features/' + feature_type + '/svms/' + kernel) # with that we now determined our whole dataset D #D = np.concatenate((pos_pyrs, neg_pyrs, pos_test_pyr, neg_test_pyr)) #Y = np.concatenate((classification, test_classification)) #idx_train = np.arange(pos_pyrs.shape[0] + neg_pyrs.shape[0]) #idx_test = np.arange(pos_test_pyr.shape[0] + neg_test_pyr.shape[0]) + idx_train.size train_pyrs = np.concatenate((pos_pyrs,neg_pyrs)) for i in range(iterations): # train new model - according to hard mining algorithm by Felszenswalb et al. # "Object Detection with Discriminatively Trained Part Based Modles" [test_pyrs, test_classification] = get_test_windows(testsize,dtest,e,\ pyr_size,L,codebook,feature_type,cls,\ pos_table.cols, randomize=randomize) print 'node',comm_rank,'training in round', i, 'with', np.sum(classification==1),'pos and',\ np.sum(classification == -1),'negs' print 'testing', test_pyrs.shape[0], 'new samples' print time.strftime('%m-%d %H:%M') # get new test samples # ----------1------------- model = train_svm(train_pyrs, classification, kernel) testY = svm_predict(np.concatenate((train_pyrs,test_pyrs)), model) result = testY print result res_train = testY[:train_pyrs.shape[0]] res_test = testY[train_pyrs.shape[0]:] # ----------3------------- # remove easy samples from train-set idx_tr_list = [] for s_ind in range(res_train.shape[0]): if res_train[s_ind]*classification[s_ind] <= 1: idx_tr_list.append(s_ind) indices = np.matrix(idx_tr_list).reshape(1,len(idx_tr_list)) indices = np.array(indices.astype(Int))[0] train_pyrs = train_pyrs[indices] classification = classification[indices] # ----------4------------- idx_hn_list = [] new_hards = False for s_ind in range(res_test.shape[0]): if res_test[s_ind]*test_classification[s_ind] < 1: new_hards = True idx_hn_list.append(s_ind) nu_train_idc = np.matrix(idx_hn_list).reshape(1,len(idx_hn_list)) nu_train_idc = np.array(nu_train_idc.astype(Int))[0] train_pyrs = np.vstack((train_pyrs, test_pyrs[nu_train_idc])) classification = np.concatenate((classification, test_classification[nu_train_idc])) test_result = result[-test_pyrs.shape[0]:] fp = np.sum(np.multiply(test_result < 0, np.transpose(np.matrix(test_classification == 1)))) tp = np.sum(np.multiply(test_result > 0, np.transpose(np.matrix(test_classification == 1)))) fn = np.sum(np.multiply(test_result > 0, np.transpose(np.matrix(test_classification == -1)))) # save these to the training file prec = tp/float(tp+fp) rec = tp/float(tp+fn) print 'tp, fp:',tp,fp print 'prec, rec:', prec,rec with open(filename + '_train', "a") as myfile: myfile.write(str(prec) + ' ' + str(rec)+'\n') # ----------2------------- if not new_hards: # no new_hards from test set,we want to quit. break
# model = train_svm(x, y) # print 'result:' # result = svm_predict(x0, model) # print result # d = Dataset('full_pascal_trainval') # d.evaluate_get_pos_windows(0.5) #if False: randomize = not os.path.exists('/home/tobibaum') d = Dataset('full_pascal_train') dtest = Dataset('full_pascal_val') e = Extractor() classes = config.pascal_classes num_words = 3000 iters = 10 feature_type = 'dsift' codebook_samples = 15 num_pos = 'max' testsize = 'max' kernel = 'chi2' # num_pos = 3 # testsize = 4 # For my local testings classes = ['dog']
from common_imports import * from common_mpi import * import synthetic.config as config from synthetic.dataset import Dataset from synthetic.extractor import Extractor if __name__ == '__main__': d = Dataset('full_pascal_trainval') feature_type = 'dsift' numpos = 15 num_words = 3000 iterations = 8 e = Extractor() all_classes = config.pascal_classes # for cls_idx in range(comm_rank, len(all_classes), comm_size): # PARALLEL # #for cls in all_classes: # cls = all_classes[cls_idx] # print cls #d, feature_type, num_words=3000,iterations=10, force_new=False, kmeansBatch=True e.get_codebook(d, feature_type, numpos, iterations, force_new=False, kmeansBatch=True)
def main(): parser = argparse.ArgumentParser(description='Execute different functions of our system') parser.add_argument('mode', choices=[ 'window_stats', 'evaluate_metaparams', 'evaluate_jw', 'evaluate_get_pos_windows', 'train_svm', 'extract_sift','extract_assignments','extract_codebook', 'evaluate_jw_grid', 'final_metaparams', 'assemble_dpm_dets','ctfdet','assemble_ctf_dets' ]) parser.add_argument('--test_dataset', choices=['val','test','train'], default='test', help='dataset to use for testing. the training dataset \ is automatically inferred (val->train and test->trainval).') parser.add_argument('--first_n', type=int, help='only take the first N images in the datasets') parser.add_argument('--bounds', type=str, help='the start_time and deadline_time for the ImagePolicy and corresponding evaluation. ex: (1,5)') parser.add_argument('--name', help='name for this run') parser.add_argument('--priors', default='random', help= \ "list of choice for the policy for selecting the next action. choose from random, oracle,fixed_order, no_smooth, backoff. ex: --priors=random,oracle,no_smooth") parser.add_argument('--compare_evals', action='store_true', default=False, help='plot all the priors modes given on same plot'), parser.add_argument('--detector', choices=['perfect','perfect_with_noise', 'dpm','ctf'], default='perfect', help='detector type') parser.add_argument('--force', action='store_true', default=False, help='force overwrite') parser.add_argument('--gist', action='store_true', default=False, help='use GIST as one of the actions') parser.add_argument('--clear_tmp', action='store_true', default=False, help='clear the cached windows folder before running'), parser.add_argument('--feature_type', choices=['sift','dsift'], default='dsift', help='use this feature type'), parser.add_argument('--kernel', choices=['chi2','rbf'], default='chi2', help='kernel to train svm on'), args = parser.parse_args() if args.priors: args.priors = args.priors.split(',') if args.bounds: args.bounds = [float(x) for x in re.findall(r'\d+', args.bounds)] assert(len(args.bounds)==2) print(args) # Load the dataset dataset = Dataset('full_pascal_'+args.test_dataset) if args.first_n: dataset.images = dataset.images[:args.first_n] # Infer train_dataset if args.test_dataset=='test': train_dataset = Dataset('full_pascal_trainval') elif args.test_dataset=='val': train_dataset = Dataset('full_pascal_train') else: print("Impossible, setting train_dataset to dataset") train_dataset = dataset # Create window generator sw = SlidingWindows(dataset,train_dataset) if args.clear_tmp: dirname = config.get_sliding_windows_cached_dir(train_dataset.get_name()) shutil.rmtree(dirname) dirname = config.get_sliding_windows_cached_dir(dataset.get_name()) shutil.rmtree(dirname) if args.mode=='assemble_dpm_dets': policy = DatasetPolicy(dataset,train_dataset,sw) dets = policy.load_ext_detections(dataset,suffix='dpm_may25') if args.mode=='assemble_ctf_dets': policy = DatasetPolicy(dataset,train_dataset,sw) dets = policy.load_ext_detections(dataset,'ctf','ctf_default') dets = policy.load_ext_detections(dataset,'ctf','ctf_nohal') dets = policy.load_ext_detections(dataset,'ctf', 'ctf_halfsize') if args.mode=='evaluate_get_pos_windows': evaluate_get_pos_windows(train_dataset) return if args.mode=='window_stats': "Compute and plot the statistics of ground truth window parameters." results = SlidingWindows.get_dataset_window_stats(train_dataset,plot=True) if args.mode=='ctfdet': """Run Pedersoli's detector on the dataset and assemble into one Table.""" run_pedersoli(dataset) if args.mode=='evaluate_jw': """ Evaluate the jumping window approach by producing plots of recall vs. #windows. """ # TODO hack: both sw and jw should subclass something like WindowGenerator jw = JumpingWindowsDetector(use_scale=True) sw.jw = jw #classes = dataset.classes classes = ['car'] # classes = ['bicycle' ,'car','horse', 'sofa',\ # 'bird', 'chair', 'motorbike', 'train',\ # 'boat', 'cow', 'person', 'tvmonitor',\ # 'bottle','diningtable', 'pottedplant',\ # 'bus','dog' ,'sheep'] for cls_idx in range(comm_rank, len(classes), comm_size): #for cls in dataset.classes: cls = classes[cls_idx] dirname = config.get_jumping_windows_dir(dataset.get_name()) filename = os.path.join(dirname,'%s'%cls) sw.evaluate_recall(cls, filename, metaparams=None, mode='jw', plot=True) if args.mode=='evaluate_jw_grid': """ Evaluate the jumping window approach by producing plots of recall vs. #windows. """ sw = SlidingWindows(dataset,train_dataset) jw = JumpingWindowsDetectorGrid() sw.jw = jw for cls in dataset.classes: dirname = config.get_jumping_windows_dir(dataset.get_name()) filename = os.path.join(dirname,'%s'%cls) if os.path.isfile(config.data_dir + 'JumpingWindows/'+cls): sw.evaluate_recall(cls, filename, metaparams=None, mode='jw', plot=True) if args.mode=='train_svm': randomize = not os.path.exists('/home/tobibaum') d = Dataset('full_pascal_train') dtest = Dataset('full_pascal_val') e = Extractor() classes = config.pascal_classes num_words = 3000 iters = 5 feature_type = 'dsift' codebook_samples = 15 num_pos = 'max' testsize = 'max' if args.first_n: num_pos = args.first_n testsize = 1.5*num_pos kernel = args.kernel if comm_rank == 0: ut.makedirs(config.data_dir + 'features/' + feature_type + '/times/') ut.makedirs(config.data_dir + 'features/' + feature_type + '/codebooks/times/') ut.makedirs(config.data_dir + 'features/' + feature_type + '/svms/train_times/') for cls_idx in range(comm_rank, len(classes), comm_size): #for cls in classes: cls = classes[cls_idx] codebook = e.get_codebook(d, feature_type) pos_arr = d.get_pos_windows(cls) neg_arr = d.get_neg_windows(pos_arr.shape[0], cls, max_overlap=0) if not num_pos == 'max': if not randomize: pos_arr = pos_arr[:num_pos] neg_arr = pos_arr[:num_pos] else: rand = np.random.random_integers(0, pos_arr.shape[0] - 1, size=num_pos) pos_arr = pos_arr[rand] rand = np.random.random_integers(0, neg_arr.shape[0] - 1, size=num_pos) neg_arr = neg_arr[rand] pos_table = Table(pos_arr, ['x','y','w','h','img_ind']) neg_table = Table(neg_arr, pos_table.cols) train_with_hard_negatives(d, dtest, num_words,codebook_samples,codebook,\ cls, pos_table, neg_table,feature_type, \ iterations=iters, kernel=kernel, L=2, \ testsize=testsize,randomize=randomize) if args.mode=='evaluate_metaparams': """ Grid search over metaparams values for get_windows_new, with the AUC of recall vs. # windows evaluation. """ sw.grid_search_over_metaparams() return if args.mode=='final_metaparams': dirname = config.get_sliding_windows_metaparams_dir(train_dataset.get_name()) # currently these are the best auc/complexity params best_params_for_classes = [ (62,15,12,'importance',0), #aeroplane (83,15,12,'importance',0), #bicycle (62,15,12,'importance',0), #bird (62,15,12,'importance',0), #boat (125,12,12,'importance',0), #bottle (83,12,9,'importance',0), #bus (125,15,9,'importance',0), #car (125,12,12,'linear',0), #cat (125,15,9,'importance',0), #chair (125,9,6,'importance',0), #cow (125,15,6,'linear',0), #diningtable (62,15,12,'importance',0), #dog (83,15,6,'importance',0), #horse (83,12,6,'importance',0), #motorbike (83,15,12,'importance',0), #person (83,15,6,'importance',0), #pottedplant (83,15,12,'importance',0), #sheep (83,9,6,'importance',0), #sofa (62,12,6,'importance',0), #train (62,12,12,'importance',0), #tvmonitor (125,9,12,'importance',0) #all ] # ACTUALLY THEY ARE ALL THE SAME! cheap_params = (62, 9, 6, 'importance', 0) for i in range(comm_rank,dataset.num_classes(),comm_size): cls = dataset.classes[i] best_params = best_params_for_classes[i] #samples,num_scales,num_ratios,mode,priority,cls = cheap_params metaparams = { 'samples_per_500px': samples, 'num_scales': num_scales, 'num_ratios': num_ratios, 'mode': mode, 'priority': 0 } filename = '%s_%d_%d_%d_%s_%d'%( cls, metaparams['samples_per_500px'], metaparams['num_scales'], metaparams['num_ratios'], metaparams['mode'], metaparams['priority']) filename = os.path.join(dirname,filename) tables = sw.evaluate_recall(cls,filename,metaparams,'sw',plot=True,force=False) metaparams = { 'samples_per_500px': samples, 'num_scales': num_scales, 'num_ratios': num_ratios, 'mode': mode, 'priority': 1 } filename = '%s_%d_%d_%d_%s_%d'%( cls, metaparams['samples_per_500px'], metaparams['num_scales'], metaparams['num_ratios'], metaparams['mode'], metaparams['priority']) filename = os.path.join(dirname,filename) tables = sw.evaluate_recall(cls,filename,metaparams,'sw',plot=True,force=False) return if args.mode=='extract_sift': e=Extractor() e.extract_all(['sift'], ['full_pascal_trainval','full_pascal_test'], 0, 0) if args.mode=='extract_assignments': e=Extractor() feature_type = 'sift' for image_set in ['full_pascal_trainval','full_pascal_test']: d = Dataset(image_set) codebook = e.get_codebook(d, feature_type) print 'codebook loaded' for img_ind in range(comm_rank,len(d.images),comm_size): img = d.images[img_ind] #for img in d.images: e.get_assignments(np.array([0,0,img.size[0],img.size[1]]), feature_type, \ codebook, img) if args.mode=='extract_codebook': d = Dataset('full_pascal_trainval') e = Extractor() codebook = e.get_codebook(d, args.feature_type)
def train_jumping_windows(d, codebook, use_scale=True, trun=False, diff=False, feature='sift'): tocer = ut.TicToc() llc_dir = '../../research/jumping_windows/llc/' featdir = '../../research/jumping_windows/sift/' if feature == 'sift': trainfile = join(config.VOC_dir, 'ImageSets','Main','trainval.txt') trainfiles = open(trainfile,'r').readlines() elif feature == 'llc': trainfiles = os.listdir(featdir) grids = 4 if feature == 'sift': numcenters = codebook.shape[0] elif feature == 'llc': a = sio.loadmat(join(llc_dir,trainfiles[0]))['codes'] numcenters = a.shape[0] ccmat = np.zeros((numcenters, len(d.classes)*grids*grids+1)) e = Extractor() #first_visit = True print 'Read all features to create weights' tocer.tic() for filename in trainfiles: print filename # Load feature positions if feature == 'sift': filename = filename[:-1] + '.jpg' assignment = e.get_assignments(np.asarray([0,0,100000,1000000]), 'sift', codebook, d.get_image_by_filename(filename)) pts = assignment[:,0:2] codes = assignment[:,2] elif feature == 'llc': feaSet = sio.loadmat(join(featdir,filename))['feaSet'] x = feaSet['x'][0][0] y = feaSet['y'][0][0] pts = np.hstack((x,y)) codes = sio.loadmat(join(llc_dir,filename))['codes'] bg = np.ones((pts.shape[0], 1)) image = d.get_image_by_filename(filename[:-4]+'.jpg') im_ind = d.get_img_ind(image) gt = d.get_ground_truth_for_img_inds([im_ind]) for row in gt.arr: #for row in gt.arr[0,:]: cls = row[gt.cols.index('cls_ind')] bbox = row[0:4] inds = get_indices_for_pos(pts, bbox[0], bbox[0]+bbox[2], bbox[1], bbox[1]+bbox[3]) bg[inds] = 0 selbins = get_selbins(grids, inds, pts, bbox) binidx = np.transpose(sub2ind([grids, grids], selbins[:,0], selbins[:,1]) \ + cls*grids*grids); if feature == 'sift': binidx -= np.tile(grids, binidx.shape) idx = get_idx(inds, codes, ccmat.shape, feature, binidx) for i in idx: [x, y] = ind2sub(ccmat.shape[0], i) #print x, y ccmat[x, y] = ccmat[x, y] + 1 # Now record background features cls = len(d.classes)*grids*grids inds = np.where(bg > 0)[0] if feature == 'llc': ind = np.where(codes[:,inds].data > 0)[0] words = codes[:,inds].nonzero()[0][ind] words = np.unique(words) elif feature == 'sift': words = codes[inds] for w in words: ccmat[w, cls] = ccmat[w, cls] + 1 #sio.savemat('ccmat', {'ccmat2': ccmat}) #break print 'features counted' tocer.toc() tocer.tic() # counted all words for all images&object, now compute weights div = np.sum(ccmat,1) for didx in range(len(div)): ta = div[didx] if ta == 0: ccmat[didx, :] = 2.5 continue ccmat[didx, :] /= ta print 'computed weights' tocer.toc() tocer.tic() numwords = 500 [sortedprob, discwords] = sort_cols(ccmat, numwords) #return # Lookup for every class for cls_idx in range(len(d.classes)): #for cls_idx in [14]: cls = d.classes[cls_idx] print cls clswords = discwords[:, cls_idx*grids*grids:(cls_idx+1)*grids*grids] binidx,_ = np.meshgrid(range(grids*grids), np.zeros((clswords.shape[0],1))) clswords = sub2ind([numcenters, grids*grids], line_up_cols(clswords), line_up_cols(binidx)) # === GOOD! === wordprobs = sortedprob[:, cls_idx*grids*grids:(cls_idx+1)*grids*grids]; wordprobs = line_up_cols(wordprobs); [wordprobs, idx] = sort_cols(wordprobs); clswords = mat_from_col_idx(clswords, idx); bbinfo = LookupTable(grids, numwords, clswords, wordprobs, numcenters, cls) fileids = d.get_ground_truth_for_class(cls) last_filename = '000000' for row_idx in range(fileids.shape[0]): row = fileids.arr[row_idx, :] filename = d.images[row[fileids.cols.index('img_ind')].astype('int32')].name[:-4] if not os.path.isfile(join(llc_dir, filename+'.mat')): continue print filename if not last_filename == filename: # This is a new filename. Load codes and pts. feaSet = sio.loadmat(join(featdir,filename))['feaSet'] x = feaSet['x'][0][0] y = feaSet['y'][0][0] pts = np.hstack((x,y)) bg = np.ones((pts.shape[0], 1)) codes = sio.loadmat(join(llc_dir,filename))['codes'] last_filename = filename bbox = row[0:4] inds = get_indices_for_pos(pts, bbox[0], bbox[0]+bbox[2], bbox[1], bbox[1]+bbox[3]) # Compute grid that each point falls into selbins = get_selbins(grids, inds, pts, bbox) binidx = np.transpose(sub2ind([grids, grids], selbins[:,0]-1, selbins[:,1]-1)); selbins = get_selbins(grids, inds, pts, bbox) binidx = np.transpose(sub2ind([grids, grids], selbins[:,0], selbins[:,1]) \ + cls*grids*grids); idx = get_idx(inds, codes, ccmat.shape, feature, binidx) if feature == 'llc': ind = np.where(codes[:,inds].data > 0)[0] ind = codes.nonzero()[1][ind] elif feature == 'sift': ind = inds #intersect idx and clswords clswords = np.unique(np.asarray(clswords)) I = np.sort(idx) # J = np.sort(ind) idx = np.unique(np.asarray(idx)) imgwords = np.unique(np.intersect1d(np.asarray(idx), np.asarray(clswords), assume_unique=True)) for c in imgwords: idc = np.where(I == c)[0] featidx = inds[ind[idc]] featpts = pts[featidx] featpts = np.hstack((featpts, featpts)) ins = np.tile(bbox, (featpts.shape[0],1)) - featpts + np.matrix([0,0,bbox[0]-1,bbox[1]-1]) bbinfo.insert(c, ins) bbinfo.perform_mean_shifts() bbinfo.top_words = np.asarray(bbinfo.wordprobs.argsort(axis=0))[::-1] bbinfo.save() print 'computed all lookup tables' tocer.toc()
missing_items = all_classes[(-missing_size):(len(all_classes))] if rank < missing_size: all_nu_classes.append(missing_items[rank]) return all_nu_classes if __name__=='__main__': all_classes = config.pascal_classes val_set = 'full_pascal_test' train_set = 'full_pascal_trainval' K = 3000 num_pos = 'max' use_scale = False e = Extractor() d = Dataset(train_set) train = False if train: # this is the codebook size # This is just for that it broke down during the night # MPI this feature = 'sift' codebook = e.get_codebook(d, 'sift') ut.makedirs(join(config.data_dir, 'jumping_window','lookup')) train_jumping_windows(d, codebook, use_scale=use_scale,trun=True,diff=False, feature=feature) debug = True just_eval = True