def main(args):
# Parameters left as default
args.S = 800
args.L = 2
args.multires = False
args.gpu = 0
print('--> Building dataset')
dataset_obj = evaluate_lib.PanoRetrievalDataset(args.dataset, [], args.view_poses_file)
# Set query / db info
dataset_obj.set_retrieval_info('', '')
# Set ground truth info
dataset_obj.set_ground_truth_info(0.0)
# Load and reshape the means to subtract to the inputs
args.means = np.array([103.93900299, 116.77899933, 123.68000031], dtype=np.float32)[None, :, None, None]
# Configure caffe and load the network
caffe.set_device(args.gpu)
caffe.set_mode_gpu()
net = caffe.Net(args.proto, args.weights, caffe.TEST)
# Load the image helper
image_helper = evaluate_lib.ImageHelper(args.S, args.L, args.means)
# Extract features
if not os.path.exists(args.temp_dir):
os.makedirs(args.temp_dir)
feature_extractor = evaluate_lib.FeatureExtractor(args.temp_dir, args.multires, args.S, args.L)
feature_extractor.extract_features(dataset_obj, image_helper, net)
import config
import evaluate_lib
import os
import numpy as np
from sklearn.cluster import KMeans
from collections import defaultdict
dataset_distractor_path = os.path.join(config.MATTERPORT_ROOT, 'distractor_dataset')
view_poses_file_distractor = 'view_poses.json'
dataset_names = ['DUC1', 'DUC2']
# Load distractor dataset
dataset_distractor = evaluate_lib.PanoRetrievalDataset(dataset_distractor_path, [], view_poses_file_distractor)
dataset_distractor.set_retrieval_info('', '')
# Get panorama locations in each room
pano_info_distractor = [p for p in dataset_distractor.get_pano_info() if len(p['db_ids']) > 0]
room_to_locations = defaultdict(list)
for p in pano_info_distractor:
room_to_locations[p['room_id']].append(p['location'])
# Get mean distance to room centroid
dists = []
for room, locations in room_to_locations.items():
res = locations - np.mean(locations, 0)
for x in np.linalg.norm(res, axis=1):
dists.append(x)
mean_dist_distractor = np.mean(dists)
def main(args):
# Parameters left as default
args.S = 800
args.L = 2
args.multires = False
args.proto = '/scratch/PI/bgirod/localization/deep-retrieval-files/deep_retrieval/deploy_resnet101_normpython.prototxt'
args.weights = '/scratch/PI/bgirod/localization/deep-retrieval-files/deep_retrieval/model.caffemodel'
args.ignore_rooms = False
print('--> Building dataset')
dataset_obj = evaluate_lib.PanoRetrievalDataset(args.dataset, [], args.view_poses_file)
# Set query / db info
dataset_obj.set_retrieval_info(args.query_file, args.db_file)
# Set ground truth info
dataset_obj.set_ground_truth_info(args.pano_dist_threshold, args.ignore_rooms)
# Load the image helper
image_helper = evaluate_lib.ImageHelper(args.S, args.L, None)
# Note: We assume that all necessary features were already extracted and are saved in a file
feature_extractor = evaluate_lib.FeatureExtractor(args.temp_dir, args.multires, args.S, args.L)
feature_extractor.extract_features(dataset_obj, image_helper, None)
if args.dataset_distractor:
print('--> Building distractor dataset')
dataset_distractor = evaluate_lib.PanoRetrievalDataset(args.dataset_distractor, [], args.view_poses_file_distractor)
# Set db info
dataset_distractor.set_retrieval_info('', args.db_file_distractor)
dataset_distractor.query_indices = np.empty(0, dtype=int)
# Note: We assume that all necessary features were already extracted and are saved in a file
feature_extractor_distractor = evaluate_lib.FeatureExtractor(args.temp_dir_distractor, args.multires, args.S, args.L)
feature_extractor_distractor.extract_features(dataset_distractor, evaluate_lib.ImageHelper(args.S, args.L, None), None)
# Get mean of full dataset
if args.dataset_distractor:
full_dataset = np.concatenate((feature_extractor.features_dataset[0], feature_extractor_distractor.features_dataset[0]))
dataset_mean = full_dataset.mean(axis=0)
else:
dataset_mean = feature_extractor.features_dataset[0].mean(axis=0)
# Center + normalize dataset
if args.center_norm_dataset:
features = feature_extractor.features_dataset[0]
features -= dataset_mean
features /= np.sqrt((features * features).sum(axis=1))[:, None]
if args.dataset_distractor:
features_distractor = feature_extractor_distractor.features_dataset[0]
features_distractor -= dataset_mean
features_distractor /= np.sqrt((features_distractor * features_distractor).sum(axis=1))[:, None]
print('--> Performing feature aggregation on dataset')
feature_extractor.features_dataset = dataset_obj.aggregate_features(feature_extractor.features_dataset[0],
args.feat_aggregation,
args.pooling,
[])
queries = feature_extractor.features_queries
if args.center_norm_dataset:
queries -= dataset_mean
query_id_to_positive_classes_int = dataset_obj.query_id_to_positive_classes
query_id_to_positive_classes = {}
for x in query_id_to_positive_classes_int:
query_id_to_positive_classes[str(x)] = list(query_id_to_positive_classes_int[x])
view_info = dataset_obj.cluster_info[-1]
dataset = feature_extractor.features_dataset[-1]
dataset = dataset.astype(np.float32)
if args.dataset_distractor:
print('--> Performing feature aggregation on distractor dataset')
index_offset = len(dataset_obj.image_data)
pano_index_offset = len(set([image['source_pano_filename'] for image in dataset_obj.image_data]))
feature_extractor_distractor.features_dataset = dataset_distractor.aggregate_features(feature_extractor_distractor.features_dataset[0],
args.feat_aggregation,
args.pooling,
[])
# Merge datasets
for view in dataset_distractor.cluster_info[-1]:
view['image_ids'] = [x + index_offset for x in view['image_ids']]
if view['pano_id'] is not None:
view['pano_id'] += pano_index_offset
view_info.extend(dataset_distractor.cluster_info[-1])
dataset = np.concatenate((dataset, feature_extractor_distractor.features_dataset[-1]))
print('--> Building FLANN tree')
flann = pyflann.FLANN()
params = flann.build_index(dataset, algorithm="autotuned", target_precision=0.9, log_level="info", random_seed=1)
print(params)
def main(args):
# Parameters left as default
args.S = 800
args.L = 2
args.multires = False
args.proto = '/scratch/PI/bgirod/localization/deep-retrieval-files/deep_retrieval/deploy_resnet101_normpython.prototxt'
args.weights = '/scratch/PI/bgirod/localization/deep-retrieval-files/deep_retrieval/model.caffemodel'
args.ignore_rooms = False
print('--> Building dataset')
dataset_obj = evaluate_lib.PanoRetrievalDataset(args.dataset, [], args.view_poses_file)
# Set query / db info
dataset_obj.set_retrieval_info(args.query_file, args.db_file)
# Set ground truth info
dataset_obj.set_ground_truth_info(args.pano_dist_threshold, args.ignore_rooms)
# Load the image helper
image_helper = evaluate_lib.ImageHelper(args.S, args.L, None)
# Note: We assume that all necessary features were already extracted and are saved in a file
feature_extractor = evaluate_lib.FeatureExtractor(args.temp_dir, args.multires, args.S, args.L)
feature_extractor.extract_features(dataset_obj, image_helper, None)
if args.dataset_distractor:
print('--> Building distractor dataset')
dataset_distractor = evaluate_lib.PanoRetrievalDataset(args.dataset_distractor, [], args.view_poses_file_distractor)
# Set db info
dataset_distractor.set_retrieval_info('', args.db_file_distractor)
dataset_distractor.query_indices = np.empty(0, dtype=int)
# Note: We assume that all necessary features were already extracted and are saved in a file
feature_extractor_distractor = evaluate_lib.FeatureExtractor(args.temp_dir_distractor, args.multires, args.S, args.L)
feature_extractor_distractor.extract_features(dataset_distractor, evaluate_lib.ImageHelper(args.S, args.L, None), None)
# Get mean of full dataset
if args.dataset_distractor:
full_dataset = np.concatenate((feature_extractor.features_dataset[0], feature_extractor_distractor.features_dataset[0]))
dataset_mean = full_dataset.mean(axis=0)
else:
dataset_mean = feature_extractor.features_dataset[0].mean(axis=0)
# Center + normalize dataset
if args.center_norm_dataset:
features = feature_extractor.features_dataset[0]
features -= dataset_mean
features /= np.sqrt((features * features).sum(axis=1))[:, None]
if args.dataset_distractor:
features_distractor = feature_extractor_distractor.features_dataset[0]
features_distractor -= dataset_mean
features_distractor /= np.sqrt((features_distractor * features_distractor).sum(axis=1))[:, None]
print('--> Performing feature aggregation on dataset')
feature_extractor.features_dataset = dataset_obj.aggregate_features(feature_extractor.features_dataset[0],
args.feat_aggregation,
args.pooling,
[])
queries = feature_extractor.features_queries
if args.center_norm_dataset:
queries -= dataset_mean
query_indices = dataset_obj.query_indices
query_id_to_positive_classes_int = dataset_obj.query_id_to_positive_classes
query_id_to_positive_classes = {}
for x in query_id_to_positive_classes_int:
query_id_to_positive_classes[str(x)] = list(query_id_to_positive_classes_int[x])
view_info = dataset_obj.cluster_info[-1]
dataset = feature_extractor.features_dataset[-1]
dataset = dataset.astype(np.float32)
if args.dataset_distractor:
print('--> Performing feature aggregation on distractor dataset')
index_offset = len(dataset_obj.image_data)
pano_index_offset = len(set([image['source_pano_filename'] for image in dataset_obj.image_data]))
feature_extractor_distractor.features_dataset = dataset_distractor.aggregate_features(feature_extractor_distractor.features_dataset[0],
args.feat_aggregation,
args.pooling,
[])
# Merge datasets
dataset_distractor.db_indices += index_offset
for view in dataset_distractor.cluster_info[-1]:
view['image_ids'] = [x + index_offset for x in view['image_ids']]
if view['pano_id'] is not None:
view['pano_id'] += pano_index_offset
view_info.extend(dataset_distractor.cluster_info[-1])
dataset = np.concatenate((dataset, feature_extractor_distractor.features_dataset[-1]))
print('--> Building FLANN tree')
flann = pyflann.FLANN()
flann.build_index(dataset, algorithm="kmeans", branching=args.bf, iterations=15, cb_index=0.0, random_seed=1)
# Save .flann index to temporary file then load that file to read it as a string
index_flann_file = tempfile.NamedTemporaryFile(delete=True).name
flann.save_index(index_flann_file)
serialized_tree = tree_lib.serialize_flann_index(index_flann_file, dataset)
root = tree_lib.deserialize_tree(serialized_tree, dataset, view_info, gmp=args.tree_pooling == 'gmp', normalize=args.tree_normalized_desc)
# Exploration parameters
complete_list = True
all_leaf_exploration_mode = True
pull_node_at_every_step = False
use_internal_nodes = False
if args.tree_normalized_desc:
dist_fun = tree_lib.cosine_sim
else:
dist_fun = tree_lib.l2_sq_dist
for k in np.exp(np.arange(0, 12, .1)):
tree_lib.flann_type_evaluation(dist_fun, None, query_indices, query_id_to_positive_classes, queries, root, k,
complete_list, all_leaf_exploration_mode, pull_node_at_every_step, use_internal_nodes)
def main(args):
# Parameters left as default
args.S = 800
args.L = 2
args.multires = False
args.proto = '/scratch/PI/bgirod/localization/deep-retrieval-files/deep_retrieval/deploy_resnet101_normpython.prototxt'
args.weights = '/scratch/PI/bgirod/localization/deep-retrieval-files/deep_retrieval/model.caffemodel'
args.ignore_rooms = False
print('--> Building dataset')
dataset_obj = evaluate_lib.PanoRetrievalDataset(args.dataset, [],
args.view_poses_file)
# Set query / db info
dataset_obj.set_retrieval_info(args.query_file, args.db_file)
# Set ground truth info
dataset_obj.set_ground_truth_info(args.pano_dist_threshold,
args.ignore_rooms)
# Note: We assume that all necessary features were already extracted and are saved in a file
image_helper = evaluate_lib.ImageHelper(args.S, args.L, None)
feature_extractor = evaluate_lib.FeatureExtractor(args.temp_dir,
args.multires, args.S,
args.L)
feature_extractor.extract_features(dataset_obj, image_helper, None)
if args.dataset_distractor:
print('--> Building distractor dataset')
dataset_distractor = evaluate_lib.PanoRetrievalDataset(
args.dataset_distractor, [], args.view_poses_file_distractor)
# Set db info
dataset_distractor.set_retrieval_info('', args.db_file_distractor)
dataset_distractor.query_indices = np.empty(0, dtype=int)
# Note: We assume that all necessary features were already extracted and are saved in a file
feature_extractor_distractor = evaluate_lib.FeatureExtractor(
args.temp_dir_distractor, args.multires, args.S, args.L)
feature_extractor_distractor.extract_features(
dataset_distractor, evaluate_lib.ImageHelper(args.S, args.L, None),
None)
# Get mean of full dataset
if args.dataset_distractor:
full_dataset = np.concatenate(
(feature_extractor.features_dataset[0],
feature_extractor_distractor.features_dataset[0]))
dataset_mean = full_dataset.mean(axis=0)
else:
dataset_mean = feature_extractor.features_dataset[0].mean(axis=0)
# Center + normalize dataset
if args.center_norm_dataset:
features = feature_extractor.features_dataset[0]
features -= dataset_mean
features /= np.sqrt((features * features).sum(axis=1))[:, None]
if args.dataset_distractor:
features_distractor = feature_extractor_distractor.features_dataset[
0]
features_distractor -= dataset_mean
features_distractor /= np.sqrt(
(features_distractor * features_distractor).sum(axis=1))[:,
None]
# Note: we add an extra "no aggregation" stage in order to keep track of view statistics (will be removed later)
print('--> Performing feature aggregation on dataset')
feature_extractor.features_dataset = dataset_obj.aggregate_features(
feature_extractor.features_dataset[0],
args.feat_aggregation + ['none'], args.pooling, [])
queries = feature_extractor.features_queries
if args.center_norm_dataset:
queries -= dataset_mean
db_indices = dataset_obj.db_indices
query_indices = dataset_obj.query_indices
query_id_to_positive_classes_int = dataset_obj.query_id_to_positive_classes
query_id_to_positive_classes = {}
for x in query_id_to_positive_classes_int:
query_id_to_positive_classes[str(x)] = list(
query_id_to_positive_classes_int[x])
cluster_info = dataset_obj.cluster_info
view_info = dataset_obj.cluster_info[-1]
dataset = feature_extractor.features_dataset[-1]
dataset = dataset.astype(np.float32)
if args.num_rooms_dataset > 0 and 'room' in args.feat_aggregation:
assert args.feat_aggregation.count('room') == 1
# Get panorama information (location, db ids)
pano_info = dataset_obj.get_pano_info()
# Remove non-db panoramas
pano_info = [p for p in pano_info if len(p['db_ids']) > 0]
# Cluster panorama locations using the desired number of clusters
pano_locations = np.array([pano['location'] for pano in pano_info])
kmeans = KMeans(n_clusters=args.num_rooms_dataset,
random_state=0).fit(pano_locations)
# Define clusters for descriptor aggregation
stage_idx = args.feat_aggregation.index('room')
cluster_info[stage_idx] = []
for room_label in range(args.num_rooms_dataset):
image_ids = []
for pano_id in np.where(kmeans.labels_ == room_label)[0]:
image_ids += pano_info[pano_id]['db_ids']
cluster_info[stage_idx].append({
'image_id': None,
'room_id': None,
'pano_id': None,
'image_ids': image_ids
})
if args.dataset_distractor:
print('--> Performing feature aggregation on distractor dataset')
index_offset = len(dataset_obj.image_data)
pano_index_offset = len(
set([
image['source_pano_filename']
for image in dataset_obj.image_data
]))
# Note: we add an extra "no aggregation" stage in order to keep track of view statistics (will be removed later)
feature_extractor_distractor.features_dataset = dataset_distractor.aggregate_features(
feature_extractor_distractor.features_dataset[0],
args.feat_aggregation + ['none'], args.pooling, [])
# Merge datasets
dataset_distractor.db_indices += index_offset
db_indices = np.concatenate(
(db_indices, dataset_distractor.db_indices))
assert len(db_indices) == len(
set(db_indices)), 'DB indices should be unique'
for cluster_stage_idx, cluster_stage in enumerate(
dataset_distractor.cluster_info):
for cluster in cluster_stage:
cluster['image_ids'] = [
x + index_offset for x in cluster['image_ids']
]
if cluster['pano_id'] is not None:
cluster['pano_id'] += pano_index_offset
cluster_info[cluster_stage_idx].extend(cluster_stage)
dataset = np.concatenate(
(dataset, feature_extractor_distractor.features_dataset[-1]))
print('--> Building tree')
# Note: we remove the last "no aggregation" stage
root = tree_lib.build_tree(db_indices,
cluster_info[:-1],
view_info,
dataset,
gmp=args.pooling == 'gmp',
normalize=args.normalized_desc)
if args.bf > 0:
flann = pyflann.FLANN()
aggr_dataset = np.array([c.descriptor for c in root.children])
flann.build_index(aggr_dataset,
algorithm="kmeans",
branching=args.bf,
iterations=15,
cb_index=0.0,
random_seed=1)
# Save .flann index to temporary file then load that file to read it as a string
index_flann_file = tempfile.NamedTemporaryFile(delete=True).name
flann.save_index(index_flann_file)
serialized_tree = tree_lib.serialize_flann_index(
index_flann_file, aggr_dataset)
root = tree_lib.deserialize_tree_head(serialized_tree,
dataset,
view_info,
root.children,
gmp=args.pooling == 'gmp',
normalize=args.normalized_desc)
tree_lib.simplify_tree(root)
# Exploration parameters
complete_list = True
all_leaf_exploration_mode = True
pull_node_at_every_step = False
use_internal_nodes = False
if args.normalized_desc:
dist_fun = tree_lib.cosine_sim
else:
dist_fun = tree_lib.l2_sq_dist
if args.evaluation_mode == 'flann':
for k in np.exp(np.arange(0, 12, .1)):
tree_lib.flann_type_evaluation(dist_fun, None, query_indices,
query_id_to_positive_classes,
queries, root, k, complete_list,
all_leaf_exploration_mode,
pull_node_at_every_step,
use_internal_nodes)
elif args.evaluation_mode == 'flann_single':
tree_lib.flann_type_evaluation(dist_fun, None, query_indices,
query_id_to_positive_classes, queries,
root, 1, complete_list,
all_leaf_exploration_mode,
pull_node_at_every_step,
use_internal_nodes)
elif args.evaluation_mode == 'multistage_rank':
for num_clusters in range(100):
tree_lib.multi_stage_evaluation_extended_rank_based(
None, query_indices, query_id_to_positive_classes, queries,
root, num_clusters)
elif args.evaluation_mode == 'multistage_dist':
for dist_threshold in np.arange(2.0, -1, -.05):
tree_lib.multi_stage_evaluation_extended_dist_based(
None, query_indices, query_id_to_positive_classes, queries,
root, dist_threshold)
else:
raise NotImplementedError('Unsupported evaluation mode')