def main(args): # Parameters left as default args.S = 800 args.L = 2 args.multires = False args.gpu = 0 print('--> Building dataset') dataset_obj = evaluate_lib.PanoRetrievalDataset(args.dataset, [], args.view_poses_file) # Set query / db info dataset_obj.set_retrieval_info('', '') # Set ground truth info dataset_obj.set_ground_truth_info(0.0) # Load and reshape the means to subtract to the inputs args.means = np.array([103.93900299, 116.77899933, 123.68000031], dtype=np.float32)[None, :, None, None] # Configure caffe and load the network caffe.set_device(args.gpu) caffe.set_mode_gpu() net = caffe.Net(args.proto, args.weights, caffe.TEST) # Load the image helper image_helper = evaluate_lib.ImageHelper(args.S, args.L, args.means) # Extract features if not os.path.exists(args.temp_dir): os.makedirs(args.temp_dir) feature_extractor = evaluate_lib.FeatureExtractor(args.temp_dir, args.multires, args.S, args.L) feature_extractor.extract_features(dataset_obj, image_helper, net)
def main(args): # Parameters left as default args.S = 800 args.L = 2 args.multires = False args.proto = '/scratch/PI/bgirod/localization/deep-retrieval-files/deep_retrieval/deploy_resnet101_normpython.prototxt' args.weights = '/scratch/PI/bgirod/localization/deep-retrieval-files/deep_retrieval/model.caffemodel' args.ignore_rooms = False print('--> Building dataset') dataset_obj = evaluate_lib.PanoRetrievalDataset(args.dataset, [], args.view_poses_file) # Set query / db info dataset_obj.set_retrieval_info(args.query_file, args.db_file) # Set ground truth info dataset_obj.set_ground_truth_info(args.pano_dist_threshold, args.ignore_rooms) # Load the image helper image_helper = evaluate_lib.ImageHelper(args.S, args.L, None) # Note: We assume that all necessary features were already extracted and are saved in a file feature_extractor = evaluate_lib.FeatureExtractor(args.temp_dir, args.multires, args.S, args.L) feature_extractor.extract_features(dataset_obj, image_helper, None) if args.dataset_distractor: print('--> Building distractor dataset') dataset_distractor = evaluate_lib.PanoRetrievalDataset(args.dataset_distractor, [], args.view_poses_file_distractor) # Set db info dataset_distractor.set_retrieval_info('', args.db_file_distractor) dataset_distractor.query_indices = np.empty(0, dtype=int) # Note: We assume that all necessary features were already extracted and are saved in a file feature_extractor_distractor = evaluate_lib.FeatureExtractor(args.temp_dir_distractor, args.multires, args.S, args.L) feature_extractor_distractor.extract_features(dataset_distractor, evaluate_lib.ImageHelper(args.S, args.L, None), None) # Get mean of full dataset if args.dataset_distractor: full_dataset = np.concatenate((feature_extractor.features_dataset[0], feature_extractor_distractor.features_dataset[0])) dataset_mean = full_dataset.mean(axis=0) else: dataset_mean = feature_extractor.features_dataset[0].mean(axis=0) # Center + normalize dataset if args.center_norm_dataset: features = feature_extractor.features_dataset[0] features -= dataset_mean features /= np.sqrt((features * features).sum(axis=1))[:, None] if args.dataset_distractor: features_distractor = feature_extractor_distractor.features_dataset[0] features_distractor -= dataset_mean features_distractor /= np.sqrt((features_distractor * features_distractor).sum(axis=1))[:, None] print('--> Performing feature aggregation on dataset') feature_extractor.features_dataset = dataset_obj.aggregate_features(feature_extractor.features_dataset[0], args.feat_aggregation, args.pooling, []) queries = feature_extractor.features_queries if args.center_norm_dataset: queries -= dataset_mean query_id_to_positive_classes_int = dataset_obj.query_id_to_positive_classes query_id_to_positive_classes = {} for x in query_id_to_positive_classes_int: query_id_to_positive_classes[str(x)] = list(query_id_to_positive_classes_int[x]) view_info = dataset_obj.cluster_info[-1] dataset = feature_extractor.features_dataset[-1] dataset = dataset.astype(np.float32) if args.dataset_distractor: print('--> Performing feature aggregation on distractor dataset') index_offset = len(dataset_obj.image_data) pano_index_offset = len(set([image['source_pano_filename'] for image in dataset_obj.image_data])) feature_extractor_distractor.features_dataset = dataset_distractor.aggregate_features(feature_extractor_distractor.features_dataset[0], args.feat_aggregation, args.pooling, []) # Merge datasets for view in dataset_distractor.cluster_info[-1]: view['image_ids'] = [x + index_offset for x in view['image_ids']] if view['pano_id'] is not None: view['pano_id'] += pano_index_offset view_info.extend(dataset_distractor.cluster_info[-1]) dataset = np.concatenate((dataset, feature_extractor_distractor.features_dataset[-1])) print('--> Building FLANN tree') flann = pyflann.FLANN() params = flann.build_index(dataset, algorithm="autotuned", target_precision=0.9, log_level="info", random_seed=1) print(params)
def main(args): # Parameters left as default args.S = 800 args.L = 2 args.multires = False args.proto = '/scratch/PI/bgirod/localization/deep-retrieval-files/deep_retrieval/deploy_resnet101_normpython.prototxt' args.weights = '/scratch/PI/bgirod/localization/deep-retrieval-files/deep_retrieval/model.caffemodel' args.ignore_rooms = False print('--> Building dataset') dataset_obj = evaluate_lib.PanoRetrievalDataset(args.dataset, [], args.view_poses_file) # Set query / db info dataset_obj.set_retrieval_info(args.query_file, args.db_file) # Set ground truth info dataset_obj.set_ground_truth_info(args.pano_dist_threshold, args.ignore_rooms) # Load the image helper image_helper = evaluate_lib.ImageHelper(args.S, args.L, None) # Note: We assume that all necessary features were already extracted and are saved in a file feature_extractor = evaluate_lib.FeatureExtractor(args.temp_dir, args.multires, args.S, args.L) feature_extractor.extract_features(dataset_obj, image_helper, None) if args.dataset_distractor: print('--> Building distractor dataset') dataset_distractor = evaluate_lib.PanoRetrievalDataset(args.dataset_distractor, [], args.view_poses_file_distractor) # Set db info dataset_distractor.set_retrieval_info('', args.db_file_distractor) dataset_distractor.query_indices = np.empty(0, dtype=int) # Note: We assume that all necessary features were already extracted and are saved in a file feature_extractor_distractor = evaluate_lib.FeatureExtractor(args.temp_dir_distractor, args.multires, args.S, args.L) feature_extractor_distractor.extract_features(dataset_distractor, evaluate_lib.ImageHelper(args.S, args.L, None), None) # Get mean of full dataset if args.dataset_distractor: full_dataset = np.concatenate((feature_extractor.features_dataset[0], feature_extractor_distractor.features_dataset[0])) dataset_mean = full_dataset.mean(axis=0) else: dataset_mean = feature_extractor.features_dataset[0].mean(axis=0) # Center + normalize dataset if args.center_norm_dataset: features = feature_extractor.features_dataset[0] features -= dataset_mean features /= np.sqrt((features * features).sum(axis=1))[:, None] if args.dataset_distractor: features_distractor = feature_extractor_distractor.features_dataset[0] features_distractor -= dataset_mean features_distractor /= np.sqrt((features_distractor * features_distractor).sum(axis=1))[:, None] print('--> Performing feature aggregation on dataset') feature_extractor.features_dataset = dataset_obj.aggregate_features(feature_extractor.features_dataset[0], args.feat_aggregation, args.pooling, []) queries = feature_extractor.features_queries if args.center_norm_dataset: queries -= dataset_mean query_indices = dataset_obj.query_indices query_id_to_positive_classes_int = dataset_obj.query_id_to_positive_classes query_id_to_positive_classes = {} for x in query_id_to_positive_classes_int: query_id_to_positive_classes[str(x)] = list(query_id_to_positive_classes_int[x]) view_info = dataset_obj.cluster_info[-1] dataset = feature_extractor.features_dataset[-1] dataset = dataset.astype(np.float32) if args.dataset_distractor: print('--> Performing feature aggregation on distractor dataset') index_offset = len(dataset_obj.image_data) pano_index_offset = len(set([image['source_pano_filename'] for image in dataset_obj.image_data])) feature_extractor_distractor.features_dataset = dataset_distractor.aggregate_features(feature_extractor_distractor.features_dataset[0], args.feat_aggregation, args.pooling, []) # Merge datasets dataset_distractor.db_indices += index_offset for view in dataset_distractor.cluster_info[-1]: view['image_ids'] = [x + index_offset for x in view['image_ids']] if view['pano_id'] is not None: view['pano_id'] += pano_index_offset view_info.extend(dataset_distractor.cluster_info[-1]) dataset = np.concatenate((dataset, feature_extractor_distractor.features_dataset[-1])) print('--> Building FLANN tree') flann = pyflann.FLANN() flann.build_index(dataset, algorithm="kmeans", branching=args.bf, iterations=15, cb_index=0.0, random_seed=1) # Save .flann index to temporary file then load that file to read it as a string index_flann_file = tempfile.NamedTemporaryFile(delete=True).name flann.save_index(index_flann_file) serialized_tree = tree_lib.serialize_flann_index(index_flann_file, dataset) root = tree_lib.deserialize_tree(serialized_tree, dataset, view_info, gmp=args.tree_pooling == 'gmp', normalize=args.tree_normalized_desc) # Exploration parameters complete_list = True all_leaf_exploration_mode = True pull_node_at_every_step = False use_internal_nodes = False if args.tree_normalized_desc: dist_fun = tree_lib.cosine_sim else: dist_fun = tree_lib.l2_sq_dist for k in np.exp(np.arange(0, 12, .1)): tree_lib.flann_type_evaluation(dist_fun, None, query_indices, query_id_to_positive_classes, queries, root, k, complete_list, all_leaf_exploration_mode, pull_node_at_every_step, use_internal_nodes)
def main(args): # Parameters left as default args.S = 800 args.L = 2 args.multires = False args.proto = '/scratch/PI/bgirod/localization/deep-retrieval-files/deep_retrieval/deploy_resnet101_normpython.prototxt' args.weights = '/scratch/PI/bgirod/localization/deep-retrieval-files/deep_retrieval/model.caffemodel' args.ignore_rooms = False print('--> Building dataset') dataset_obj = evaluate_lib.PanoRetrievalDataset(args.dataset, [], args.view_poses_file) # Set query / db info dataset_obj.set_retrieval_info(args.query_file, args.db_file) # Set ground truth info dataset_obj.set_ground_truth_info(args.pano_dist_threshold, args.ignore_rooms) # Note: We assume that all necessary features were already extracted and are saved in a file image_helper = evaluate_lib.ImageHelper(args.S, args.L, None) feature_extractor = evaluate_lib.FeatureExtractor(args.temp_dir, args.multires, args.S, args.L) feature_extractor.extract_features(dataset_obj, image_helper, None) if args.dataset_distractor: print('--> Building distractor dataset') dataset_distractor = evaluate_lib.PanoRetrievalDataset( args.dataset_distractor, [], args.view_poses_file_distractor) # Set db info dataset_distractor.set_retrieval_info('', args.db_file_distractor) dataset_distractor.query_indices = np.empty(0, dtype=int) # Note: We assume that all necessary features were already extracted and are saved in a file feature_extractor_distractor = evaluate_lib.FeatureExtractor( args.temp_dir_distractor, args.multires, args.S, args.L) feature_extractor_distractor.extract_features( dataset_distractor, evaluate_lib.ImageHelper(args.S, args.L, None), None) # Get mean of full dataset if args.dataset_distractor: full_dataset = np.concatenate( (feature_extractor.features_dataset[0], feature_extractor_distractor.features_dataset[0])) dataset_mean = full_dataset.mean(axis=0) else: dataset_mean = feature_extractor.features_dataset[0].mean(axis=0) # Center + normalize dataset if args.center_norm_dataset: features = feature_extractor.features_dataset[0] features -= dataset_mean features /= np.sqrt((features * features).sum(axis=1))[:, None] if args.dataset_distractor: features_distractor = feature_extractor_distractor.features_dataset[ 0] features_distractor -= dataset_mean features_distractor /= np.sqrt( (features_distractor * features_distractor).sum(axis=1))[:, None] # Note: we add an extra "no aggregation" stage in order to keep track of view statistics (will be removed later) print('--> Performing feature aggregation on dataset') feature_extractor.features_dataset = dataset_obj.aggregate_features( feature_extractor.features_dataset[0], args.feat_aggregation + ['none'], args.pooling, []) queries = feature_extractor.features_queries if args.center_norm_dataset: queries -= dataset_mean db_indices = dataset_obj.db_indices query_indices = dataset_obj.query_indices query_id_to_positive_classes_int = dataset_obj.query_id_to_positive_classes query_id_to_positive_classes = {} for x in query_id_to_positive_classes_int: query_id_to_positive_classes[str(x)] = list( query_id_to_positive_classes_int[x]) cluster_info = dataset_obj.cluster_info view_info = dataset_obj.cluster_info[-1] dataset = feature_extractor.features_dataset[-1] dataset = dataset.astype(np.float32) if args.num_rooms_dataset > 0 and 'room' in args.feat_aggregation: assert args.feat_aggregation.count('room') == 1 # Get panorama information (location, db ids) pano_info = dataset_obj.get_pano_info() # Remove non-db panoramas pano_info = [p for p in pano_info if len(p['db_ids']) > 0] # Cluster panorama locations using the desired number of clusters pano_locations = np.array([pano['location'] for pano in pano_info]) kmeans = KMeans(n_clusters=args.num_rooms_dataset, random_state=0).fit(pano_locations) # Define clusters for descriptor aggregation stage_idx = args.feat_aggregation.index('room') cluster_info[stage_idx] = [] for room_label in range(args.num_rooms_dataset): image_ids = [] for pano_id in np.where(kmeans.labels_ == room_label)[0]: image_ids += pano_info[pano_id]['db_ids'] cluster_info[stage_idx].append({ 'image_id': None, 'room_id': None, 'pano_id': None, 'image_ids': image_ids }) if args.dataset_distractor: print('--> Performing feature aggregation on distractor dataset') index_offset = len(dataset_obj.image_data) pano_index_offset = len( set([ image['source_pano_filename'] for image in dataset_obj.image_data ])) # Note: we add an extra "no aggregation" stage in order to keep track of view statistics (will be removed later) feature_extractor_distractor.features_dataset = dataset_distractor.aggregate_features( feature_extractor_distractor.features_dataset[0], args.feat_aggregation + ['none'], args.pooling, []) # Merge datasets dataset_distractor.db_indices += index_offset db_indices = np.concatenate( (db_indices, dataset_distractor.db_indices)) assert len(db_indices) == len( set(db_indices)), 'DB indices should be unique' for cluster_stage_idx, cluster_stage in enumerate( dataset_distractor.cluster_info): for cluster in cluster_stage: cluster['image_ids'] = [ x + index_offset for x in cluster['image_ids'] ] if cluster['pano_id'] is not None: cluster['pano_id'] += pano_index_offset cluster_info[cluster_stage_idx].extend(cluster_stage) dataset = np.concatenate( (dataset, feature_extractor_distractor.features_dataset[-1])) print('--> Building tree') # Note: we remove the last "no aggregation" stage root = tree_lib.build_tree(db_indices, cluster_info[:-1], view_info, dataset, gmp=args.pooling == 'gmp', normalize=args.normalized_desc) if args.bf > 0: flann = pyflann.FLANN() aggr_dataset = np.array([c.descriptor for c in root.children]) flann.build_index(aggr_dataset, algorithm="kmeans", branching=args.bf, iterations=15, cb_index=0.0, random_seed=1) # Save .flann index to temporary file then load that file to read it as a string index_flann_file = tempfile.NamedTemporaryFile(delete=True).name flann.save_index(index_flann_file) serialized_tree = tree_lib.serialize_flann_index( index_flann_file, aggr_dataset) root = tree_lib.deserialize_tree_head(serialized_tree, dataset, view_info, root.children, gmp=args.pooling == 'gmp', normalize=args.normalized_desc) tree_lib.simplify_tree(root) # Exploration parameters complete_list = True all_leaf_exploration_mode = True pull_node_at_every_step = False use_internal_nodes = False if args.normalized_desc: dist_fun = tree_lib.cosine_sim else: dist_fun = tree_lib.l2_sq_dist if args.evaluation_mode == 'flann': for k in np.exp(np.arange(0, 12, .1)): tree_lib.flann_type_evaluation(dist_fun, None, query_indices, query_id_to_positive_classes, queries, root, k, complete_list, all_leaf_exploration_mode, pull_node_at_every_step, use_internal_nodes) elif args.evaluation_mode == 'flann_single': tree_lib.flann_type_evaluation(dist_fun, None, query_indices, query_id_to_positive_classes, queries, root, 1, complete_list, all_leaf_exploration_mode, pull_node_at_every_step, use_internal_nodes) elif args.evaluation_mode == 'multistage_rank': for num_clusters in range(100): tree_lib.multi_stage_evaluation_extended_rank_based( None, query_indices, query_id_to_positive_classes, queries, root, num_clusters) elif args.evaluation_mode == 'multistage_dist': for dist_threshold in np.arange(2.0, -1, -.05): tree_lib.multi_stage_evaluation_extended_dist_based( None, query_indices, query_id_to_positive_classes, queries, root, dist_threshold) else: raise NotImplementedError('Unsupported evaluation mode')