def main(argv): path_to_db = argv[1] out_dir = argv[2] hash_table = int(argv[3]) hash_val = int(argv[4]) mani_hash = TubeHash_Manipulator(path_to_db) mani_hash.openSession() out_file = os.path.join(out_dir, str(hash_table) + '_' + str(hash_val) + '.npz') print out_file if not os.path.exists(out_file): t = time.time() toSelect = (TubeHash.idx, Tube.class_idx_pascal, Tube.video_id, Tube.shot_id, Tube.frame_id) criterion = (TubeHash.hash_table == hash_table, TubeHash.hash_val == hash_val) vals = mani_hash.selectMix(toSelect, criterion) vals = np.array(vals) print vals.shape np.savez(out_file, vals) print time.time() - t mani_hash.closeSession()
def getScoreForIdx(table_idx, path_to_db, class_idx_pascal=None, npz_path=None, n_jobs=1, total_counts=None): mani = Tube_Manipulator(path_to_db) mani.openSession() mani_hash = TubeHash_Manipulator(path_to_db) mani_hash.openSession() toSelect = (Tube.class_idx_pascal, Tube.video_id, Tube.img_path) criterion = (Tube.idx == table_idx, ) [(class_idx_gt, video_idx, frame_path)] = mani.select(toSelect, criterion) if class_idx_pascal is not None: class_idx = class_idx_pascal else: class_idx = class_idx_gt toSelect = (TubeHash.hash_table, TubeHash.hash_val) criterion = (TubeHash.idx == table_idx, ) hash_table_info = mani_hash.select(toSelect, criterion) print len(hash_table_info) mani_hash.closeSession() mani.closeSession() args = [] for hash_table_no in range(len(hash_table_info)): hash_table = hash_table_info[hash_table_no][0] hash_val = hash_table_info[hash_table_no][1] if npz_path is not None: args.append((npz_path, hash_table, hash_val, class_idx, video_idx, class_idx_gt, total_counts)) else: args.append((path_to_db, hash_table, hash_val, class_idx, video_idx, class_idx_gt)) if n_jobs > 1: p = multiprocessing.Pool(min(multiprocessing.cpu_count(), n_jobs)) if npz_path is not None: scores = p.map(getScoreForHashValFromNpz, args) else: scores = p.map(getScoreForHashVal, args) else: scores = [] for arg in args: if npz_path is not None: scores.append(getScoreForHashValFromNpz(arg)) else: scores.append(getScoreForHashVal(arg)) return scores, class_idx_gt, frame_path
def getHashValsForFrameFromDeepFeaturesIdx(path_to_db, class_idx, video_id, shot_id, tube_id, deep_features_idx): mani = TubeHash_Manipulator(path_to_db) mani.openSession() toSelect = (TubeHash.idx, TubeHash.hash_table, TubeHash.hash_val) criterion = (Tube.class_idx_pascal == class_idx, Tube.video_id == video_id, Tube.shot_id == shot_id, Tube.tube_id == tube_id, Tube.deep_features_idx == deep_features_idx) vals = mani.selectMix(toSelect, criterion) mani.closeSession() return vals
def main(): return path_to_db = 'sqlite://///disk2/novemberExperiments/experiments_youtube/patches_nn_hash.db' mani = Tube_Manipulator(path_to_db) mani_hash = TubeHash_Manipulator(path_to_db) mani.openSession() deep_features_path_all = mani.select((Tube.deep_features_path, ), distinct=True) deep_features_path_all = [x[0] for x in deep_features_path_all] print len(deep_features_path_all) # print deep_features_path_all[:10] mani_hash.openSession() for idx_deep_features_path, deep_features_path in enumerate( deep_features_path_all[11:]): t = time.time() hash_file = deep_features_path[:-4] + '_hash.npy' print hash_file idx_info = mani.select( (Tube.idx, Tube.deep_features_idx), (Tube.deep_features_path == deep_features_path, )) # idx_all,deep_features_idx_all=zip(*idx_info); hash_vals = np.load(hash_file) # print len(idx_all),hash_vals.shape for idx_foreign, row in idx_info: # hash_vals_curr=hash_vals[row]; for hash_table, hash_val in enumerate(hash_vals[row]): # pass; # print type(idx_foreign),type(hash_table),type(int(hash_val)) mani_hash.insert(idx=idx_foreign, hash_table=hash_table, hash_val=int(hash_val), commit=False) if idx_deep_features_path % 10 == 0: mani_hash.session.commit() # print time.time()-t; mani_hash.closeSession() mani.closeSession()
def getHashInfoForImg(path_to_db, img_path): mani = Tube_Manipulator(path_to_db) mani.openSession() #get patch id patch_id = mani.select((Tube.idx, ), (Tube.img_path == img_path, )) assert len(patch_id) == 1 patch_id = patch_id[0][0] mani.closeSession() #get hash vals mani_hash = TubeHash_Manipulator(path_to_db) mani_hash.openSession() toSelect = (TubeHash.hash_table, TubeHash.hash_val) criterion = (TubeHash.idx == patch_id, ) hash_info_patch = mani_hash.select(toSelect, criterion) mani_hash.closeSession() return hash_info_patch
def script_saveHashTableDensities(hash_table, path_to_db, out_file): mani_hash = TubeHash_Manipulator(path_to_db) mani_hash.openSession() toSelect = (TubeHash.hash_val, ) criterion = (TubeHash.hash_table == hash_table, ) hash_vals = mani_hash.select(toSelect, criterion) mani_hash.closeSession() hash_vals = [hash_val[0] for hash_val in hash_vals] hash_vals = np.array(hash_vals, dtype=np.uint8) hash_dict = {} for val_curr in np.unique(hash_vals): hash_dict[val_curr] = (hash_vals == val_curr).sum() pickle.dump(hash_dict, open(out_file, 'wb'))
def meta_script_saveNpzScorePerShot_normalized(params, out_dir_scores, out_dir_actual): mani = TubeHash_Manipulator(params['path_to_db']) mani.openSession() vals = mani.select((Tube.class_idx_pascal, Tube.video_id, Tube.shot_id), distinct=True) mani.closeSession() print len(vals) args = [] for idx, (class_idx, video_id, shot_id) in enumerate(vals): params_curr = copy.deepcopy(params) params_curr['class_idx'] = class_idx params_curr['video_id'] = video_id params_curr['shot_id'] = shot_id file_curr = str(params_curr['class_idx']) + '_' + str( params_curr['video_id']) + '_' + str(params_curr['shot_id']) + '.p' params_curr['out_file_scores'] = os.path.join(out_dir_scores, file_curr) params_curr['idx'] = idx if os.path.exists(params_curr['out_file_scores']): # continue; # else: params_curr['out_file_scores'] = os.path.join( out_dir_actual, file_curr) args.append(params_curr) print len(args) # args=args[3000:]; print len(args) n_jobs = 12 p = multiprocessing.Pool(min(multiprocessing.cpu_count(), n_jobs)) p.map(script_saveNpzScorePerShot_normalized, args)
def script_testNpzScoreAccuracy(params): path_to_db = params.path_to_db path_to_hash = params.path_to_hash total_class_counts = params.total_class_counts class_idx = params.class_idx video_id = params.video_id shot_id = params.shot_id num_hash_tables = params.num_hash_tables num_hash_vals = params.num_hash_vals tube_id = params.tube_id deep_features_idx = params.deep_features_idx tube_file = params.tube_file tube_scores_all = pickle.load(open(tube_file, 'rb')) #get hash_vals for frame vals = getHashValsForFrameFromDeepFeaturesIdx(path_to_db, class_idx, video_id, shot_id, tube_id, deep_features_idx) frame_count = getShotFrameCount(path_to_db, class_idx, video_id, shot_id) vals = np.array(vals) # print vals.shape,np.unique(vals[:,0]),np.unique(vals[:,0]).size; assert np.unique(vals[:, 0]).size == 1 assert vals.shape == (num_hash_tables, 3) total_class_count = total_class_counts[class_idx] deno = total_class_count - frame_count scores_all = {} #count total number of that shot in that bin mani = TubeHash_Manipulator(path_to_db) mani.openSession() for table_no in range(vals.shape[0]): print table_no hash_table_curr = vals[table_no, 1] hash_val_curr = vals[table_no, 2] toSelect = (TubeHash.idx, ) criterion = (Tube.class_idx_pascal == class_idx, Tube.video_id == video_id, Tube.shot_id == shot_id, TubeHash.hash_table == hash_table_curr, TubeHash.hash_val == hash_val_curr) count_shot_in_bin = mani.count(toSelect=toSelect, criterion=criterion, mix=True) criterion = (Tube.class_idx_pascal == class_idx, TubeHash.hash_table == hash_table_curr, TubeHash.hash_val == hash_val_curr) count_class_in_bin = mani.count(toSelect, criterion, mix=True) numo_curr = count_class_in_bin - count_shot_in_bin scores_all[hash_table_curr] = numo_curr / float(deno) mani.closeSession() #check for the accuracy of frame for hash_table_curr in scores_all: print hash_table_curr, print 'from tube_scores', tube_scores_all[tube_id][deep_features_idx, hash_table_curr], print 'calculated', scores_all[hash_table_curr] assert tube_scores_all[tube_id][ deep_features_idx, hash_table_curr] == scores_all[hash_table_curr]