def setUp(self):
self._tempdir = tempfile.TemporaryDirectory()
self._samples_folder = path.abspath(path.join(path.dirname(__file__), '..', 'samples', 'maupertuis'))
self._kapture_dirpath = path.join(self._samples_folder, 'kapture')
self._kapture_data = csv.kapture_from_dir(self._kapture_dirpath)
def test_import_single_model(self):
nvm_path = path.join(self.aachen_models_folder,
'aachen_cvpr2018_db.nvm')
with tempfile.TemporaryDirectory() as tmpdirname:
import_nvm(nvm_path,
self.images_folder,
tmpdirname,
filter_list_path="",
ignore_trajectories=False,
add_reconstruction=True,
force_overwrite_existing=True)
expected_kdata = kapture_from_dir(
path.join(self.aachen_folder, 'kapture/training'))
imported_aachen_data = kapture_from_dir(tmpdirname)
self.assertTrue(equal_kapture(imported_aachen_data,
expected_kdata))
# test without trajectories
with tempfile.TemporaryDirectory() as tmpdirname:
import_nvm(nvm_path,
self.images_folder,
tmpdirname,
filter_list_path="",
ignore_trajectories=True,
add_reconstruction=True,
force_overwrite_existing=True)
expected_kdata = kapture_from_dir(
path.join(self.aachen_folder, 'kapture/training'))
expected_kdata._trajectories = None
imported_aachen_data = kapture_from_dir(tmpdirname)
self.assertTrue(equal_kapture(imported_aachen_data,
expected_kdata))
# test without points3d
with tempfile.TemporaryDirectory() as tmpdirname:
import_nvm(nvm_path,
self.images_folder,
tmpdirname,
filter_list_path="",
ignore_trajectories=False,
add_reconstruction=False,
force_overwrite_existing=True)
expected_kdata = kapture_from_dir(
path.join(self.aachen_folder, 'kapture/training'))
expected_kdata._points3d = None
expected_kdata._keypoints = None
expected_kdata._observations = None
imported_aachen_data = kapture_from_dir(tmpdirname)
self.assertTrue(equal_kapture(imported_aachen_data,
expected_kdata))
# test with filter list
with tempfile.TemporaryDirectory() as tmpdirname:
import_nvm(nvm_path,
self.images_folder,
tmpdirname,
filter_list_path=self.filter,
ignore_trajectories=True,
add_reconstruction=False,
force_overwrite_existing=True)
imported_aachen_data = kapture_from_dir(tmpdirname)
self.assertTrue(
len(imported_aachen_data.records_camera.key_pairs()) == 2)
images_set = {
image_path
for _, _, image_path in kapture.flatten(
imported_aachen_data.records_camera)
}
self.assertTrue('db/1045.jpg' in images_set)
self.assertFalse('db/4446.jpg' in images_set)
self.assertTrue('db/1135.jpg' in images_set)
self.assertIsNone(imported_aachen_data.trajectories)
self.assertIsNone(imported_aachen_data.points3d)
def test_kapture_format_version_memory(self):
kapture_data = kapture.Kapture()
kapture_data.sensors = kapture.Sensors()
csv.kapture_to_dir(self._tempdir.name, kapture_data)
kapture_read = csv.kapture_from_dir(self._tempdir.name)
self.assertEqual(csv.current_format_version(), kapture_read.format_version, "We have the current version")
def test_read_write(self):
kapture_data = csv.kapture_from_dir(self._samples_folder)
csv.kapture_to_dir(self._tempdir.name, kapture_data)
def extract_kapture_keypoints(args):
"""
Extract r2d2 keypoints and descritors to the kapture format directly
"""
print('extract_kapture_keypoints...')
kdata = kapture_from_dir(args.kapture_root,
matches_pairs_file_path=None,
skip_list=[
kapture.GlobalFeatures, kapture.Matches,
kapture.Points3d, kapture.Observations
])
assert kdata.records_camera is not None
image_list = [
filename for _, _, filename in kapture.flatten(kdata.records_camera)
]
if kdata.keypoints is not None and kdata.descriptors is not None:
image_list = [
name for name in image_list
if name not in kdata.keypoints or name not in kdata.descriptors
]
if len(image_list) == 0:
print('All features were already extracted')
return
else:
print(f'Extracting r2d2 features for {len(image_list)} images')
iscuda = common.torch_set_gpu(args.gpu)
# load the network...
net = load_network(args.model)
if iscuda: net = net.cuda()
# create the non-maxima detector
detector = NonMaxSuppression(rel_thr=args.reliability_thr,
rep_thr=args.repeatability_thr)
keypoints_dtype = None if kdata.keypoints is None else kdata.keypoints.dtype
descriptors_dtype = None if kdata.descriptors is None else kdata.descriptors.dtype
keypoints_dsize = None if kdata.keypoints is None else kdata.keypoints.dsize
descriptors_dsize = None if kdata.descriptors is None else kdata.descriptors.dsize
for image_name in image_list:
img_path = get_image_fullpath(args.kapture_root, image_name)
print(f"\nExtracting features for {img_path}")
img = Image.open(img_path).convert('RGB')
W, H = img.size
img = norm_RGB(img)[None]
if iscuda: img = img.cuda()
# extract keypoints/descriptors for a single image
xys, desc, scores = extract_multiscale(net,
img,
detector,
scale_f=args.scale_f,
min_scale=args.min_scale,
max_scale=args.max_scale,
min_size=args.min_size,
max_size=args.max_size,
verbose=True)
xys = xys.cpu().numpy()
desc = desc.cpu().numpy()
scores = scores.cpu().numpy()
idxs = scores.argsort()[-args.top_k or None:]
xys = xys[idxs]
desc = desc[idxs]
if keypoints_dtype is None or descriptors_dtype is None:
keypoints_dtype = xys.dtype
descriptors_dtype = desc.dtype
keypoints_dsize = xys.shape[1]
descriptors_dsize = desc.shape[1]
kdata.keypoints = kapture.Keypoints('r2d2', keypoints_dtype,
keypoints_dsize)
kdata.descriptors = kapture.Descriptors('r2d2', descriptors_dtype,
descriptors_dsize)
keypoints_config_absolute_path = get_csv_fullpath(
kapture.Keypoints, args.kapture_root)
descriptors_config_absolute_path = get_csv_fullpath(
kapture.Descriptors, args.kapture_root)
keypoints_to_file(keypoints_config_absolute_path, kdata.keypoints)
descriptors_to_file(descriptors_config_absolute_path,
kdata.descriptors)
else:
assert kdata.keypoints.type_name == 'r2d2'
assert kdata.descriptors.type_name == 'r2d2'
assert kdata.keypoints.dtype == xys.dtype
assert kdata.descriptors.dtype == desc.dtype
assert kdata.keypoints.dsize == xys.shape[1]
assert kdata.descriptors.dsize == desc.shape[1]
keypoints_fullpath = get_keypoints_fullpath(args.kapture_root,
image_name)
print(f"Saving {xys.shape[0]} keypoints to {keypoints_fullpath}")
image_keypoints_to_file(keypoints_fullpath, xys)
kdata.keypoints.add(image_name)
descriptors_fullpath = get_descriptors_fullpath(
args.kapture_root, image_name)
print(f"Saving {desc.shape[0]} descriptors to {descriptors_fullpath}")
image_descriptors_to_file(descriptors_fullpath, desc)
kdata.descriptors.add(image_name)
if not keypoints_check_dir(kdata.keypoints, args.kapture_root) or \
not descriptors_check_dir(kdata.descriptors, args.kapture_root):
print(
'local feature extraction ended successfully but not all files were saved'
)
def compute_image_pairs(mapping_path: str, query_path: str, output_path: str,
topk: int):
"""
compute image pairs between query -> mapping from global features, and write the result in a text file
:param mapping_path: input path to kapture input root directory
:type mapping_path: str
:param query_path: input path to a kapture root directory
:type query_path: str
:param output_path: output path to pairsfile
:type output_path: str
:param topk: the max number of top retained images
:type topk: int
"""
logger.info(f'compute_image_pairs. loading mapping: {mapping_path}')
kdata_mapping = kapture_from_dir(mapping_path)
assert kdata_mapping.sensors is not None
assert kdata_mapping.records_camera is not None
assert kdata_mapping.global_features is not None
if mapping_path == query_path:
kdata_query = kdata_mapping
else:
logger.info(f'compute_image_pairs. loading query: {query_path}')
kdata_query = kapture_from_dir(query_path)
assert kdata_query.sensors is not None
assert kdata_query.records_camera is not None
assert kdata_query.global_features is not None
assert kdata_mapping.global_features is not None
assert kdata_query.global_features is not None
assert kdata_mapping.global_features.type_name == kdata_query.global_features.type_name
assert kdata_mapping.global_features.dtype == kdata_query.global_features.dtype
assert kdata_mapping.global_features.dsize == kdata_query.global_features.dsize
global_features_config = ImageFeatureConfig(
kdata_mapping.global_features.type_name,
kdata_mapping.global_features.dtype,
kdata_mapping.global_features.dsize)
logger.info(
f'computing pairs from with {kdata_mapping.global_features.type_name}...'
)
mapping_global_features_to_filepaths = global_features_to_filepaths(
kdata_mapping.global_features, mapping_path)
mapping_list = list(
kapture.flatten(mapping_global_features_to_filepaths, is_sorted=True))
mapping_indexes, mapping_features = stack_global_features(
global_features_config, mapping_list)
if mapping_path == query_path:
query_indexes, query_features = mapping_indexes, mapping_features
else:
query_global_features_to_filepaths = global_features_to_filepaths(
kdata_query.global_features, query_path)
query_list = list(
kapture.flatten(query_global_features_to_filepaths,
is_sorted=True))
query_indexes, query_features = stack_global_features(
global_features_config, query_list)
# compute similarity matrix
similarity_matrix = query_features.dot(mapping_features.T)
# convert similarity matrix to dictionary query_name -> sorted (high score first) list [(mapping_name, score), ...]
similarity_dict = {}
for i, line in enumerate(similarity_matrix):
scores = line
indexes = np.argsort(-scores)
query_name = query_indexes[i]
similarity_dict[query_name] = list(
zip(mapping_indexes[indexes], scores[indexes]))
# get list of image pairs
image_pairs = []
for query_image_name, images_to_match in sorted(similarity_dict.items()):
for mapping_image_name, score in images_to_match[:topk]:
image_pairs.append([query_image_name, mapping_image_name, score])
logger.info('saving to file ...')
p = pathlib.Path(output_path)
os.makedirs(str(p.parent.resolve()), exist_ok=True)
with open(output_path, 'w') as fid:
table_to_file(fid,
image_pairs,
header='# query_image, map_image, score')
logger.info('all done')
def extract_kapture_global_features(kapture_root_path: str,
net,
global_features_type: str,
trfs,
pooling='mean',
gemp=3,
whiten=None,
threads=8,
batch_size=16):
""" Extract features from trained model (network) on a given dataset.
"""
print(f'loading {kapture_root_path}')
with get_all_tar_handlers(kapture_root_path,
mode={
kapture.Keypoints: 'r',
kapture.Descriptors: 'r',
kapture.GlobalFeatures: 'a',
kapture.Matches: 'r'
}) as tar_handlers:
kdata = kapture_from_dir(kapture_root_path,
None,
skip_list=[
kapture.Keypoints, kapture.Descriptors,
kapture.Matches, kapture.Points3d,
kapture.Observations
],
tar_handlers=tar_handlers)
root = get_image_fullpath(kapture_root_path, image_filename=None)
assert kdata.records_camera is not None
imgs = [
image_name
for _, _, image_name in kapture.flatten(kdata.records_camera)
]
if kdata.global_features is None:
kdata.global_features = {}
if global_features_type in kdata.global_features:
imgs = [
image_name for image_name in imgs if image_name not in
kdata.global_features[global_features_type]
]
if len(imgs) == 0:
print('All global features are already extracted')
return
dataset = ImageList(img_list_path=None, root=root, imgs=imgs)
print(f'\nEvaluation on {dataset}')
# extract DB feats
bdescs = []
trfs_list = [trfs] if isinstance(trfs, str) else trfs
for trfs in trfs_list:
kw = dict(iscuda=net.iscuda,
threads=threads,
batch_size=batch_size,
same_size='Pad' in trfs or 'Crop' in trfs)
bdescs.append(
extract_image_features(dataset, trfs, net, desc="DB", **kw))
# pool from multiple transforms (scales)
bdescs = tonumpy(F.normalize(pool(bdescs, pooling, gemp), p=2, dim=1))
if whiten is not None:
bdescs = common.whiten_features(bdescs, net.pca, **whiten)
print('writing extracted global features')
os.umask(0o002)
gfeat_dtype = bdescs.dtype
gfeat_dsize = bdescs.shape[1]
if global_features_type not in kdata.global_features:
kdata.global_features[
global_features_type] = kapture.GlobalFeatures(
'dirtorch', gfeat_dtype, gfeat_dsize, 'L2')
global_features_config_absolute_path = get_feature_csv_fullpath(
kapture.GlobalFeatures, global_features_type,
kapture_root_path)
global_features_to_file(
global_features_config_absolute_path,
kdata.global_features[global_features_type])
else:
assert kdata.global_features[
global_features_type].dtype == gfeat_dtype
assert kdata.global_features[
global_features_type].dsize == gfeat_dsize
assert kdata.global_features[
global_features_type].metric_type == 'L2'
for i in tqdm.tqdm(range(dataset.nimg)):
image_name = dataset.get_key(i)
global_feature_fullpath = get_global_features_fullpath(
global_features_type, kapture_root_path, image_name,
tar_handlers)
gfeat_i = bdescs[i, :]
assert gfeat_i.shape == (gfeat_dsize, )
image_global_features_to_file(global_feature_fullpath, gfeat_i)
kdata.global_features[global_features_type].add(image_name)
del gfeat_i
del bdescs
if not global_features_check_dir(
kdata.global_features[global_features_type],
global_features_type, kapture_root_path, tar_handlers):
print(
'global feature extraction ended successfully but not all files were saved'
)
else:
print('Features extracted.')
def plot_ply(kapture_path: str, ply_path: str, axis_length: float, only: [],
skip: []) -> None:
"""
Plot the kapture data in a PLY file.
:param kapture_path: top directory of the kapture
:param ply_path: path to the ply file to create
:param axis_length: length of axis representation (in world unit)
:param only: list of the only kapture objects to plot (optional)
:param skip: list of the kapture objects to skip
"""
try:
os.makedirs(ply_path, exist_ok=True)
logger.info('loading data ...')
kapture_data = csv.kapture_from_dir(kapture_path)
def _should_do(choice: str) -> bool:
return (not only or choice in only) and choice not in skip
logger.info('plotting ...')
if _should_do('rigs') and kapture_data.rigs:
logger.info(f'creating {len(kapture_data.rigs)} rigs.')
for rig_id, rig in kapture_data.rigs.items():
rig_ply_filepath = path.join(ply_path, f'rig_{rig_id}.ply')
logger.info(f'creating rig file : {rig_ply_filepath}.')
logger.debug(rig_ply_filepath)
ply.rig_to_ply(rig_ply_filepath, rig, axis_length)
if _should_do('trajectories') and kapture_data.trajectories:
trajectories_ply_filepath = path.join(ply_path, 'trajectories.ply')
logger.info(
f'creating trajectories file : {trajectories_ply_filepath}')
ply.trajectories_to_ply(trajectories_ply_filepath,
kapture_data.trajectories, axis_length)
if _should_do('points3d') and kapture_data.points3d:
points3d_ply_filepath = path.join(ply_path, 'points3d.ply')
logger.info(f'creating 3D points file : {points3d_ply_filepath}')
ply.points3d_to_ply(points3d_ply_filepath, kapture_data.points3d)
if _should_do('keypoints') and kapture_data.keypoints:
logger.info(
f'creating keypoints in 3D : {kapture.io.features.get_keypoints_fullpath(ply_path)}'
)
keypoints_dsize = kapture_data.keypoints.dsize
keypoints_dtype = kapture_data.keypoints.dtype
keypoints_filepaths = kapture.io.features.keypoints_to_filepaths(
kapture_data.keypoints, kapture_path)
for image_filename, keypoints_filepath in tqdm(
keypoints_filepaths.items(),
disable=logger.level >= logging.CRITICAL):
image_filepath = kapture.io.records.get_image_fullpath(
kapture_path, image_filename)
image_keypoints_filepath = kapture.io.features.get_keypoints_fullpath(
ply_path, image_filename) + '.jpg'
image.image_keypoints_to_image_file(image_keypoints_filepath,
image_filepath,
keypoints_filepath,
keypoints_dtype,
keypoints_dsize)
logger.info('done.')
except Exception as e:
logging.critical(e)
raise
def evaluate_command_line() -> None:
"""
Do the evaluation using the parameters given on the command line.
"""
parser = argparse.ArgumentParser(
description='Evaluation script for kapture data.')
parser_verbosity = parser.add_mutually_exclusive_group()
parser_verbosity.add_argument(
'-v',
'--verbose',
nargs='?',
default=logging.WARNING,
const=logging.INFO,
action=kapture.utils.logging.VerbosityParser,
help=
'verbosity level (debug, info, warning, critical, ... or int value) [warning]'
)
parser_verbosity.add_argument('-q',
'--silent',
'--quiet',
action='store_const',
dest='verbose',
const=logging.CRITICAL)
parser.add_argument(
'-i',
'--inputs',
nargs='+',
help=
'input path to kapture data root directory. You can compare multiple models'
)
parser.add_argument(
'--labels',
nargs='+',
default=[],
help='labels for inputs. must be of same length as inputs')
parser.add_argument(
'-gt',
'--ground-truth',
required=True,
help='input path to data ground truth root directory in kapture format'
)
parser.add_argument('-o',
'--output',
help='output directory.',
required=True)
parser.add_argument(
'-l',
'--image-list',
default="",
help=
'optional, path to a text file containing the list of images to consider'
' (1 line per image or a pairsfile). if not present, all gt images are used'
)
parser.add_argument(
'--bins',
nargs='+',
default=["0.25 2", "0.5 5", "5 10"],
help='the desired positions/rotations thresholds for bins'
'format is string : position_threshold_in_m space rotation_threshold_in_degree'
)
parser.add_argument(
'-p',
'--plot-rotation-threshold',
default=-1,
type=float,
help=
'rotation threshold for position error threshold plot. negative values -> ignore rotation'
)
parser.add_argument('--plot-max',
default=100,
type=int,
help='maximum distance in cm shown in plot')
parser.add_argument('--plot-title',
default="",
help='title for position error threshold plot')
parser.add_argument(
'--plot-loc',
default="best",
choices=[
'best', 'upper right', 'upper left', 'lower left', 'lower right',
'right', 'center left', 'center right', 'lower center',
'upper center', 'center'
],
help='position of plot legend. loc param for plt.legend.')
parser.add_argument('--plot-font-size',
default=15,
type=int,
help='value for plt.rcParams[\'font.size\']')
parser.add_argument('--plot-legend-font-size',
default=8,
type=int,
help='value for plt.rcParams[\'legend.fontsize\']')
parser.add_argument('-f',
'-y',
'--force',
action='store_true',
default=False,
help='Force delete output directory if already exists')
args = parser.parse_args()
logger.setLevel(args.verbose)
if args.verbose <= logging.DEBUG:
# also let kapture express its logs
kapture.utils.logging.getLogger().setLevel(args.verbose)
kapture_localization.utils.logging.getLogger().setLevel(args.verbose)
assert (len(args.inputs) > 0)
if len(args.labels) == 0:
args.labels = [f'input{i}' for i in range(1, len(args.inputs) + 1)]
assert (len(args.labels) == len(args.inputs))
try:
logger.debug(''.join(
['\n\t{:13} = {}'.format(k, v) for k, v in vars(args).items()]))
os.makedirs(args.output, exist_ok=True)
logger.debug('loading: {}'.format(args.inputs))
all_kapture_to_eval = [
csv.kapture_from_dir(folder) for folder in args.inputs
]
logger.info('loading ground truth data')
gt_kapture = csv.kapture_from_dir(args.ground_truth)
assert gt_kapture.records_camera is not None
assert gt_kapture.trajectories is not None
if args.image_list:
with open(args.image_list, 'r') as fid:
table = table_from_file(fid)
image_set = {line[0] for line in table}
else:
if gt_kapture.rigs is not None:
gt_trajectories = kapture.rigs_remove(gt_kapture.trajectories,
gt_kapture.rigs)
else:
gt_trajectories = gt_kapture.trajectories
image_set = set(image_name for ts, sensor_id, image_name in
kapture.flatten(gt_kapture.records_camera)
if (ts, sensor_id) in gt_trajectories)
if len(image_set) == 0:
logger.info(
'image_set is empty, for some reason, I could not find images to evaluate'
)
exit(0)
results = [
evaluate(kapture_to_eval, gt_kapture, image_set)
for kapture_to_eval in all_kapture_to_eval
]
save_evaluation(results, args.output, args.labels, args.bins,
args.plot_rotation_threshold, args.plot_max,
args.plot_title, args.plot_loc, args.plot_font_size,
args.plot_legend_font_size, args.force)
except Exception as e:
logger.critical(e)
if args.verbose > 1:
raise
def pose_approximation_from_pairsfile(input_path: str, pairsfile_path: str,
output_path: str,
query_path: Optional[str],
topk: Optional[int], method: str,
additional_parameters: dict,
force: bool):
"""
localize from pairsfile
"""
os.makedirs(output_path, exist_ok=True)
delete_existing_kapture_files(output_path, force_erase=force)
logger.info(f'pose_approximation. loading mapping: {input_path}')
kdata = kapture_from_dir(input_path,
None,
skip_list=[
kapture.Keypoints, kapture.Descriptors,
kapture.GlobalFeatures, kapture.Matches,
kapture.Points3d, kapture.Observations
])
if query_path is not None:
logger.info(f'pose_approximation. loading query: {query_path}')
kdata_query = kapture_from_dir(query_path,
skip_list=[
kapture.Keypoints,
kapture.Descriptors,
kapture.GlobalFeatures,
kapture.Matches, kapture.Points3d,
kapture.Observations
])
else:
kdata_query = kdata
logger.info(f'pose_approximation. loading pairs: {pairsfile_path}')
similarity_dict = get_ordered_pairs_from_file(pairsfile_path,
kdata_query.records_camera,
kdata.records_camera, topk)
query_images = set(similarity_dict.keys())
kdata_result = kapture.Kapture(sensors=kapture.Sensors(),
records_camera=kapture.RecordsCamera(),
trajectories=kapture.Trajectories())
for timestamp, cam_id, image_name in kapture.flatten(
kdata_query.records_camera):
if image_name not in query_images:
continue
if cam_id not in kdata_result.sensors:
kdata_result.sensors[cam_id] = kdata_query.sensors[cam_id]
kdata_result.records_camera[(timestamp, cam_id)] = image_name
if kdata.rigs is None:
map_trajectories = kdata.trajectories
else:
map_trajectories = kapture.rigs_remove(kdata.trajectories, kdata.rigs)
training_trajectories_reversed = {
image_name: map_trajectories[(timestamp, cam_id)]
for timestamp, cam_id, image_name in kapture.flatten(
kdata.records_camera) if (timestamp, cam_id) in map_trajectories
}
records_camera_reversed = {
image_name: (timestamp, cam_id)
for timestamp, cam_id, image_name in kapture.flatten(
kdata_result.records_camera)
}
for image_name, similar_images in similarity_dict.items():
pose_inv_list = [
training_trajectories_reversed[k].inverse()
for k, _ in similar_images
]
timestamp = records_camera_reversed[image_name][0]
cam_id = records_camera_reversed[image_name][1]
if method == 'equal_weighted_barycenter':
weight_list = [
1.0 / len(pose_inv_list) for _ in range(len(pose_inv_list))
]
else:
assert 'alpha' in additional_parameters
alpha = additional_parameters['alpha']
weights = np.zeros((len(pose_inv_list), ))
for i, (_, score) in enumerate(similar_images):
weights[i] = score
weights[:] = weights[:]**(alpha)
weights[:] = weights[:] / np.sum(weights[:])
weight_list = weights.tolist()
final_pose = average_pose_transform_weighted(pose_inv_list,
weight_list).inverse()
kdata_result.trajectories[(timestamp, cam_id)] = final_pose
kapture_to_dir(output_path, kdata_result)
logger.info('all done')
def local_sfm(map_plus_query_path: str, map_plus_query_gv_path: str,
query_path: str, pairsfile_path: str, output_path_root: str,
colmap_binary: str, force: bool):
"""
Localize query images in a COLMAP model built from topk retrieved images.
:param map_plus_query_path: path to the kapture data consisting of mapping and query data (sensors and reconstruction)
:param map_plus_query_gv_path: path to the kapture data consisting of mapping and query data after geometric verification (sensors and reconstruction)
:param query_path: path to the query kapture data (sensors)
:param pairsfile_path: path to the pairsfile that contains the topk retrieved mapping images for each query image
:param output_path_root: root path where outputs should be stored
:param colmap_binary: path to the COLMAP binary
:param force: silently overwrite already existing results
"""
# load query kapture (we use query kapture to reuse sensor_ids etc.)
kdata_query = kapture_from_dir(query_path)
if kdata_query.trajectories:
logger.warning(
"Query data contains trajectories: they will be ignored")
kdata_query.trajectories.clear()
else:
kdata_query.trajectories = kapture.Trajectories()
# load output kapture
output_path = os.path.join(output_path_root, 'localized')
if os.path.exists(os.path.join(output_path, 'sensors/trajectories.txt')):
kdata_output = kapture_from_dir(output_path)
if kdata_query.records_camera == kdata_output.records_camera and len(
kdata_output.trajectories) != 0 and not force:
kdata_query.trajectories = kdata_output.trajectories
# load kapture maps
kdata_map = kapture_from_dir(map_plus_query_path)
if kdata_map.rigs != None:
rigs_remove_inplace(kdata_map.trajectories, kdata_map.rigs)
kdata_map_gv = kapture_from_dir(map_plus_query_gv_path)
if kdata_map_gv.rigs != None:
rigs_remove_inplace(kdata_map_gv.trajectories, kdata_map_gv.rigs)
# load pairsfile
pairs = {}
with open(pairsfile_path, 'r') as fid:
table = table_from_file(fid)
for img_query, img_map, score in table:
if not img_query in pairs:
pairs[img_query] = []
pairs[img_query].append(img_map)
kdata_sub_colmap_path = os.path.join(output_path_root, 'colmap')
kdata_reg_query_path = os.path.join(output_path_root, 'query_registered')
sub_kapture_pairsfile_path = os.path.join(output_path_root,
'tmp_pairs_map.txt')
query_img_kapture_pairsfile_path = os.path.join(output_path_root,
'tmp_pairs_query.txt')
# loop over query images
for img_query, img_list_map in pairs.items():
if pose_found(kdata_query, img_query):
logger.info(f'{img_query} already processed, skipping...')
continue
else:
logger.info(f'processing {img_query}')
# write pairsfile for sub-kapture
map_pairs = write_pairfile_from_img_list(img_list_map,
sub_kapture_pairsfile_path)
# write pairsfile for query_img_kapture
query_pairs = write_pairfile_img_vs_img_list(
img_query, img_list_map, query_img_kapture_pairsfile_path)
# create sub-kapture
kdata_sub = sub_kapture_from_img_list(kdata_map, map_plus_query_path,
img_list_map, map_pairs)
kdata_sub_gv = sub_kapture_from_img_list(kdata_map_gv,
map_plus_query_gv_path,
img_list_map, map_pairs)
# match missing pairs for mapping
compute_matches_from_loaded_data(map_plus_query_path, kdata_sub,
map_pairs)
# kdata_sub needs to be re-created to add the new matches
kdata_sub = sub_kapture_from_img_list(kdata_map, map_plus_query_path,
img_list_map, map_pairs)
# run colmap gv on missing pairs
if len(kdata_sub.matches) != len(kdata_sub_gv.matches):
run_colmap_gv_from_loaded_data(kdata_sub, kdata_sub_gv,
map_plus_query_path,
map_plus_query_gv_path,
colmap_binary, [], True)
# kdata_sub_gv needs to be re-created to add the new matches
kdata_sub_gv = sub_kapture_from_img_list(kdata_map_gv,
map_plus_query_gv_path,
img_list_map, map_pairs)
# sanity check
if len(map_pairs) != len(kdata_sub_gv.matches):
logger.info(f'not all mapping matches available')
# build COLMAP map
try:
colmap_build_map_from_loaded_data(kdata_sub_gv,
map_plus_query_gv_path,
kdata_sub_colmap_path,
colmap_binary, False, [],
['model_converter'], True)
except ValueError:
logger.info(f'{img_query} was not localized')
continue
if not os.path.exists(
os.path.join(kdata_sub_colmap_path,
'reconstruction/images.bin')):
logger.info(
f'colmap mapping for {img_query} did not work, image was not localized'
)
continue
# create single image kapture (kdata_sub needs to be recreated because descriptors are deleted in build_colmap_model)
kdata_sub = sub_kapture_from_img_list(kdata_map, map_plus_query_path,
img_list_map, map_pairs)
kdata_sub_gv = sub_kapture_from_img_list(kdata_map_gv,
map_plus_query_gv_path,
img_list_map, map_pairs)
query_img_kapture = add_image_to_kapture(kdata_map,
map_plus_query_path,
kdata_sub, img_query,
query_pairs)
query_img_kapture_gv = add_image_to_kapture(kdata_map_gv,
map_plus_query_gv_path,
kdata_sub_gv, img_query,
query_pairs)
# match missing pairs for localization
compute_matches_from_loaded_data(map_plus_query_path,
query_img_kapture, query_pairs)
# query_img_kapture needs to be re-created to add the new matches
query_img_kapture = add_image_to_kapture(kdata_map,
map_plus_query_path,
kdata_sub, img_query,
query_pairs)
# run colmap gv on missing pairs
if len(query_img_kapture.matches) != len(query_img_kapture_gv.matches):
run_colmap_gv_from_loaded_data(query_img_kapture,
query_img_kapture_gv,
map_plus_query_path,
map_plus_query_gv_path,
colmap_binary, [], True)
# query_img_kapture_gv needs to be re-created to add the new matches
query_img_kapture_gv = add_image_to_kapture(
kdata_map_gv, map_plus_query_gv_path, kdata_sub_gv, img_query,
query_pairs)
# sanity check
if len(query_pairs) != len(query_img_kapture_gv.matches):
logger.info(f'not all query matches available')
# localize in COLMAP map
try:
colmap_localize_from_loaded_data(
query_img_kapture_gv, map_plus_query_gv_path,
os.path.join(kdata_sub_colmap_path, 'registered'),
os.path.join(kdata_sub_colmap_path, 'colmap.db'),
os.path.join(kdata_sub_colmap_path, 'reconstruction'),
colmap_binary, False, [
'--Mapper.ba_refine_focal_length', '0',
'--Mapper.ba_refine_principal_point', '0',
'--Mapper.ba_refine_extra_params', '0',
'--Mapper.min_num_matches', '4',
'--Mapper.init_min_num_inliers', '4',
'--Mapper.abs_pose_min_num_inliers', '4',
'--Mapper.abs_pose_min_inlier_ratio', '0.05',
'--Mapper.ba_local_max_num_iterations', '50',
'--Mapper.abs_pose_max_error', '20',
'--Mapper.filter_max_reproj_error', '12'
], [], True)
except ValueError:
logger.info(f'{img_query} was not localized')
continue
if not os.path.exists(
os.path.join(os.path.join(kdata_sub_colmap_path, 'registered'),
'reconstruction/images.txt')):
logger.info(
f'colmap localization of {img_query} did not work, image was not localized'
)
continue
# add to results kapture
kdata_reg_query = import_colmap(
kdata_reg_query_path,
os.path.join(os.path.join(kdata_sub_colmap_path, 'registered'),
'colmap.db'),
os.path.join(os.path.join(kdata_sub_colmap_path, 'registered'),
'reconstruction'), None, None, True, True, True,
TransferAction.skip)
if add_pose_to_query_kapture(kdata_reg_query, kdata_query, img_query):
logger.info('successfully localized')
# write results (after each image to see the progress)
kapture_to_dir(output_path, kdata_query)
# clean up (e.g. remove temporal files and folders)
safe_remove_any_path(kdata_sub_colmap_path, True)
safe_remove_any_path(kdata_reg_query_path, True)
safe_remove_file(sub_kapture_pairsfile_path, True)
safe_remove_file(query_img_kapture_pairsfile_path, True)
logger.info('all done')
def compute_image_pairs(mapping_path: str, query_path: str, output_path: str,
topk: int):
"""
compute image pairs between query -> mapping from global features, and write the result in a text file
:param mapping_path: input path to kapture input root directory
:type mapping_path: str
:param query_path: input path to a kapture root directory
:type query_path: str
:param output_path: output path to pairsfile
:type output_path: str
:param topk: the max number of top retained images
:type topk: int
"""
logger.info(f'compute_image_pairs. loading mapping: {mapping_path}')
kdata_mapping = kapture_from_dir(mapping_path,
None,
skip_list=[
kapture.Keypoints,
kapture.Descriptors, kapture.Matches,
kapture.Observations, kapture.Points3d
])
assert kdata_mapping.sensors is not None
assert kdata_mapping.records_camera is not None
assert kdata_mapping.global_features is not None
if mapping_path == query_path:
kdata_query = kdata_mapping
else:
logger.info(f'compute_image_pairs. loading query: {query_path}')
kdata_query = kapture_from_dir(query_path,
None,
skip_list=[
kapture.Keypoints,
kapture.Descriptors,
kapture.Matches,
kapture.Observations,
kapture.Points3d
])
assert kdata_query.sensors is not None
assert kdata_query.records_camera is not None
assert kdata_query.global_features is not None
assert kdata_mapping.global_features is not None
assert kdata_query.global_features is not None
assert kdata_mapping.global_features.type_name == kdata_query.global_features.type_name
assert kdata_mapping.global_features.dtype == kdata_query.global_features.dtype
assert kdata_mapping.global_features.dsize == kdata_query.global_features.dsize
global_features_config = ImageFeatureConfig(
kdata_mapping.global_features.type_name,
kdata_mapping.global_features.dtype,
kdata_mapping.global_features.dsize)
logger.info(
f'computing pairs with {kdata_mapping.global_features.type_name}...')
mapping_global_features_to_filepaths = global_features_to_filepaths(
kdata_mapping.global_features, mapping_path)
mapping_list = list(
kapture.flatten(mapping_global_features_to_filepaths, is_sorted=True))
mapping_stacked_features = stack_global_features(global_features_config,
mapping_list)
if mapping_path == query_path:
query_stacked_features = mapping_stacked_features
else:
query_global_features_to_filepaths = global_features_to_filepaths(
kdata_query.global_features, query_path)
query_list = list(
kapture.flatten(query_global_features_to_filepaths,
is_sorted=True))
query_stacked_features = stack_global_features(global_features_config,
query_list)
similarity = get_similarity(query_stacked_features,
mapping_stacked_features)
# get list of image pairs
image_pairs = get_image_pairs(similarity, topk)
logger.info('saving to file ...')
p = pathlib.Path(output_path)
os.makedirs(str(p.parent.resolve()), exist_ok=True)
with open(output_path, 'w') as fid:
table_to_file(fid,
image_pairs,
header='# query_image, map_image, score')
logger.info('all done')
def local_sfm_from_loaded_data(kdata_map: kapture.Kapture,
kdata_map_gv: kapture.Kapture,
kdata_query: kapture.Kapture,
map_plus_query_path: str,
map_plus_query_gv_path: str,
tar_handlers_map: Optional[TarCollection],
tar_handlers_map_gv: Optional[TarCollection],
descriptors_type: Optional[str],
pairsfile_path: str,
output_path_root: str,
colmap_binary: str,
force: bool):
"""
Localize query images in a COLMAP model built from topk retrieved images.
:param map_plus_query_path: path to the kapture data consisting of mapping and query data (sensors and reconstruction)
:param map_plus_query_gv_path: path to the kapture data consisting of mapping and query data after geometric verification (sensors and reconstruction)
:param query_path: path to the query kapture data (sensors)
:param descriptors_type: type of descriptors, name of the descriptors subfolder
:param pairsfile_path: path to the pairsfile that contains the topk retrieved mapping images for each query image
:param output_path_root: root path where outputs should be stored
:param colmap_binary: path to the COLMAP binary
:param force: silently overwrite already existing results
"""
# load query kapture (we use query kapture to reuse sensor_ids etc.)
if kdata_query.trajectories:
logger.warning("Query data contains trajectories: they will be ignored")
kdata_query.trajectories.clear()
else:
kdata_query.trajectories = kapture.Trajectories()
# clear query trajectories in map_plus_query
kdata_map_cleared_trajectories = kapture.Trajectories()
query_image_list = set(kdata_query.records_camera.data_list())
for timestamp, subdict in kdata_map.records_camera.items():
for sensor_id, image_name in subdict.items():
if image_name in query_image_list:
continue
if (timestamp, sensor_id) in kdata_map.trajectories:
pose = kdata_map.trajectories.get(timestamp)[sensor_id]
kdata_map_cleared_trajectories.setdefault(timestamp, {})[sensor_id] = pose
kdata_map.trajectories = kdata_map_cleared_trajectories
# load output kapture
output_path = os.path.join(output_path_root, 'localized')
if os.path.exists(os.path.join(output_path, 'sensors/trajectories.txt')):
kdata_output = kapture_from_dir(output_path)
if kdata_query.records_camera == kdata_output.records_camera and len(
kdata_output.trajectories) != 0 and not force:
kdata_query.trajectories = kdata_output.trajectories
if kdata_map.rigs is not None:
rigs_remove_inplace(kdata_map.trajectories, kdata_map.rigs)
if kdata_map_gv.rigs is not None:
rigs_remove_inplace(kdata_map_gv.trajectories, kdata_map_gv.rigs)
# load pairsfile
pairs = {}
with open(pairsfile_path, 'r') as fid:
table = table_from_file(fid)
for img_query, img_map, _ in table:
if img_query not in pairs:
pairs[img_query] = []
pairs[img_query].append(img_map)
kdata_sub_colmap_path = os.path.join(output_path_root, 'colmap')
kdata_reg_query_path = os.path.join(output_path_root, 'query_registered')
sub_kapture_pairsfile_path = os.path.join(output_path_root, 'tmp_pairs.txt')
if descriptors_type is None:
descriptors_type = try_get_only_key_from_collection(kdata_map.descriptors)
assert descriptors_type is not None
assert descriptors_type in kdata_map.descriptors
keypoints_type = kdata_map.descriptors[descriptors_type].keypoints_type
# init matches for kdata_map and kdata_map_gv
if kdata_map.matches is None:
kdata_map.matches = {}
if keypoints_type not in kdata_map.matches:
kdata_map.matches[keypoints_type] = kapture.Matches()
if kdata_map_gv.matches is None:
kdata_map_gv.matches = {}
if keypoints_type not in kdata_map_gv.matches:
kdata_map_gv.matches[keypoints_type] = kapture.Matches()
# run all matching
# loop over query images
img_skip_list = set()
for img_query, img_list_map in pairs.items():
if pose_found(kdata_query, img_query):
logger.info(f'{img_query} already processed, skipping...')
img_skip_list.add(img_query)
continue
else:
map_pairs = get_pairfile_from_img_list(img_list_map)
query_pairs = get_pairfile_img_vs_img_list(img_query, img_list_map)
with open(sub_kapture_pairsfile_path, 'w') as fid:
logger.info(f'matching for {img_query}')
table_to_file(fid, map_pairs)
table_to_file(fid, query_pairs)
pairs_all = map_pairs + query_pairs
pairs_all = [(i, j) for i, j, _ in pairs_all]
# match missing pairs
# kdata_map.matches is being updated by compute_matches_from_loaded_data
compute_matches_from_loaded_data(map_plus_query_path,
tar_handlers_map,
kdata_map,
descriptors_type,
pairs_all)
# if kdata_map have matches in tar, they need to be switched to read mode
matches_handler = retrieve_tar_handler_from_collection(kapture.Matches, keypoints_type, tar_handlers_map)
if matches_handler is not None:
matches_handler.close()
tarfile_path = get_feature_tar_fullpath(kapture.Matches, keypoints_type, map_plus_query_path)
tar_handlers_map.matches[keypoints_type] = TarHandler(tarfile_path, 'r')
# run all gv
# loop over query images
for img_query, img_list_map in pairs.items():
if img_query in img_skip_list:
continue
else:
# recompute the pairs
map_pairs = get_pairfile_from_img_list(img_list_map)
query_pairs = get_pairfile_img_vs_img_list(img_query, img_list_map)
with open(sub_kapture_pairsfile_path, 'w') as fid:
logger.info(f'geometric verification of {img_query}')
table_to_file(fid, map_pairs)
table_to_file(fid, query_pairs)
pairs_all = map_pairs + query_pairs
pairs_all = [(i, j) for i, j, _ in pairs_all]
if all(pair in kdata_map_gv.matches[keypoints_type] for pair in pairs_all):
continue
# create a sub kapture in order to minimize the amount of data exported to colmap
# kdata_sub needs to be re-created to add the new matches
kdata_sub = sub_kapture_from_img_list(kdata_map, img_list_map + [img_query], pairs_all,
keypoints_type, descriptors_type)
kdata_sub_gv = sub_kapture_from_img_list(kdata_map_gv, img_list_map + [img_query], pairs_all,
keypoints_type, descriptors_type)
# run colmap gv on missing pairs
run_colmap_gv_from_loaded_data(kdata_sub,
kdata_sub_gv,
map_plus_query_path,
map_plus_query_gv_path,
tar_handlers_map,
tar_handlers_map_gv,
colmap_binary,
keypoints_type,
[],
True)
# update kdata_map_gv.matches
kdata_map_gv.matches[keypoints_type].update(kdata_sub_gv.matches[keypoints_type])
# if kdata_map_gv have matches in tar, they need to be switched to read mode
matches_gv_handler = retrieve_tar_handler_from_collection(kapture.Matches, keypoints_type, tar_handlers_map_gv)
if matches_gv_handler is not None:
print(matches_gv_handler)
matches_gv_handler.close()
tarfile_path = get_feature_tar_fullpath(kapture.Matches, keypoints_type, map_plus_query_gv_path)
tar_handlers_map_gv.matches[keypoints_type] = TarHandler(tarfile_path, 'r')
# loop over query images
for img_query, img_list_map in pairs.items():
if img_query in img_skip_list:
continue
else:
map_pairs = get_pairfile_from_img_list(img_list_map)
with open(sub_kapture_pairsfile_path, 'w') as fid:
logger.info(f'mapping and localization for {img_query}')
table_to_file(fid, map_pairs)
map_pairs = [(i, j) for i, j, _ in map_pairs]
kdata_sub_gv = sub_kapture_from_img_list(kdata_map_gv, img_list_map, map_pairs,
keypoints_type, descriptors_type)
# sanity check
if len(map_pairs) != len(kdata_sub_gv.matches[keypoints_type]):
logger.info(f'not all mapping matches available')
# build COLMAP map
try:
colmap_build_map_from_loaded_data(
kdata_sub_gv,
map_plus_query_gv_path,
tar_handlers_map_gv,
kdata_sub_colmap_path,
colmap_binary,
keypoints_type,
False,
[],
['model_converter'],
True)
except ValueError:
logger.info(f'{img_query} was not localized')
continue
if not os.path.exists(os.path.join(kdata_sub_colmap_path, 'reconstruction/images.bin')):
logger.info(f'colmap mapping for {img_query} did not work, image was not localized')
continue
query_pairs = get_pairfile_img_vs_img_list(img_query, img_list_map)
with open(sub_kapture_pairsfile_path, 'w') as fid:
table_to_file(fid, query_pairs)
query_pairs = [(i, j) for i, j, _ in query_pairs]
query_img_kapture_gv = add_image_to_kapture(kdata_map_gv,
kdata_sub_gv, img_query, query_pairs,
keypoints_type, descriptors_type)
# sanity check
if len(query_pairs) != len(query_img_kapture_gv.matches[keypoints_type]):
logger.info(f'not all query matches available')
# localize in COLMAP map
try:
colmap_localize_from_loaded_data(
query_img_kapture_gv,
map_plus_query_gv_path,
tar_handlers_map_gv,
os.path.join(kdata_sub_colmap_path, 'registered'),
os.path.join(kdata_sub_colmap_path, 'colmap.db'),
os.path.join(kdata_sub_colmap_path, 'reconstruction'),
colmap_binary,
keypoints_type,
False,
['--Mapper.ba_refine_focal_length', '0',
'--Mapper.ba_refine_principal_point', '0',
'--Mapper.ba_refine_extra_params', '0',
'--Mapper.min_num_matches', '4',
'--Mapper.init_min_num_inliers', '4',
'--Mapper.abs_pose_min_num_inliers', '4',
'--Mapper.abs_pose_min_inlier_ratio', '0.05',
'--Mapper.ba_local_max_num_iterations', '50',
'--Mapper.abs_pose_max_error', '20',
'--Mapper.filter_max_reproj_error', '12'],
[],
True)
except ValueError:
logger.info(f'{img_query} was not localized')
continue
if not os.path.exists(os.path.join(os.path.join(kdata_sub_colmap_path, 'registered'),
'reconstruction/images.txt')):
logger.info(f'colmap localization of {img_query} did not work, image was not localized')
continue
# add to results kapture
kdata_reg_query = import_colmap(
kdata_reg_query_path,
os.path.join(os.path.join(kdata_sub_colmap_path, 'registered'), 'colmap.db'),
os.path.join(os.path.join(kdata_sub_colmap_path, 'registered'),
'reconstruction'),
None,
None,
True,
True,
True,
TransferAction.skip)
if add_pose_to_query_kapture(kdata_reg_query, kdata_query, img_query):
logger.info('successfully localized')
# write results (after each image to see the progress)
kapture_to_dir(output_path, kdata_query)
# clean up (e.g. remove temporal files and folders)
safe_remove_any_path(kdata_sub_colmap_path, True)
safe_remove_any_path(kdata_reg_query_path, True)
safe_remove_file(sub_kapture_pairsfile_path, True)
logger.info('all done')
default=float("+inf"),
help='max number of keypoints save to disk')
args = parser.parse_args()
print(args)
with get_all_tar_handlers(args.kapture_root,
mode={
kapture.Keypoints: 'a',
kapture.Descriptors: 'a',
kapture.GlobalFeatures: 'r',
kapture.Matches: 'r'
}) as tar_handlers:
kdata = kapture_from_dir(args.kapture_root,
skip_list=[
kapture.GlobalFeatures, kapture.Matches,
kapture.Points3d, kapture.Observations
],
tar_handlers=tar_handlers)
if kdata.keypoints is None:
kdata.keypoints = {}
if kdata.descriptors is None:
kdata.descriptors = {}
assert kdata.records_camera is not None
image_list = [
filename for _, _, filename in kapture.flatten(kdata.records_camera)
]
if args.keypoints_type is None:
args.keypoints_type = path.splitext(path.basename(args.model_file))[0]
print(f'keypoints_type set to {args.keypoints_type}')
if args.descriptors_type is None:
def compute_image_pairs(mapping_path: str, query_path: str, output_path: str,
global_features_type: Optional[str], topk: int):
"""
compute image pairs between query -> mapping from global features, and write the result in a text file
:param mapping_path: input path to kapture input root directory
:type mapping_path: str
:param query_path: input path to a kapture root directory
:type query_path: str
:param output_path: output path to pairsfile
:type output_path: str
:param global_features_type: type of global_features, name of the global_features subfolder
:param topk: the max number of top retained images
:type topk: int
"""
logger.info(f'compute_image_pairs. loading mapping: {mapping_path}')
with get_all_tar_handlers(mapping_path) as mapping_tar_handlers:
kdata_mapping = kapture_from_dir(mapping_path,
None,
skip_list=[
kapture.Keypoints,
kapture.Descriptors,
kapture.Matches,
kapture.Observations,
kapture.Points3d
],
tar_handlers=mapping_tar_handlers)
assert kdata_mapping.sensors is not None
assert kdata_mapping.records_camera is not None
assert kdata_mapping.global_features is not None
if global_features_type is None:
global_features_type = try_get_only_key_from_collection(
kdata_mapping.global_features)
assert global_features_type is not None
assert global_features_type in kdata_mapping.global_features
global_features_config = GlobalFeaturesConfig(
kdata_mapping.global_features[global_features_type].type_name,
kdata_mapping.global_features[global_features_type].dtype,
kdata_mapping.global_features[global_features_type].dsize,
kdata_mapping.global_features[global_features_type].metric_type)
logger.info(f'computing pairs with {global_features_type}...')
mapping_global_features_to_filepaths = global_features_to_filepaths(
kdata_mapping.global_features[global_features_type],
global_features_type, mapping_path, mapping_tar_handlers)
mapping_list = list(
sorted(mapping_global_features_to_filepaths.items()))
mapping_stacked_features = stack_global_features(
global_features_config, mapping_list)
if mapping_path == query_path:
kdata_query = kdata_mapping
query_stacked_features = mapping_stacked_features
else:
logger.info(f'compute_image_pairs. loading query: {query_path}')
with get_all_tar_handlers(query_path) as query_tar_handlers:
kdata_query = kapture_from_dir(query_path,
None,
skip_list=[
kapture.Keypoints,
kapture.Descriptors,
kapture.Matches,
kapture.Observations,
kapture.Points3d
],
tar_handlers=query_tar_handlers)
assert kdata_query.sensors is not None
assert kdata_query.records_camera is not None
assert kdata_query.global_features is not None
assert global_features_type in kdata_query.global_features
kdata_mapping_gfeat = kdata_mapping.global_features[
global_features_type]
kdata_query_gfeat = kdata_query.global_features[
global_features_type]
assert kdata_mapping_gfeat.type_name == kdata_query_gfeat.type_name
assert kdata_mapping_gfeat.dtype == kdata_query_gfeat.dtype
assert kdata_mapping_gfeat.dsize == kdata_query_gfeat.dsize
query_global_features_to_filepaths = global_features_to_filepaths(
kdata_query_gfeat, global_features_type, query_path,
query_tar_handlers)
query_list = list(
sorted(query_global_features_to_filepaths.items()))
query_stacked_features = stack_global_features(
global_features_config, query_list)
similarity = get_similarity(query_stacked_features,
mapping_stacked_features)
# get list of image pairs
image_pairs = get_image_pairs(similarity, topk)
logger.info('saving to file ...')
p = pathlib.Path(output_path)
os.makedirs(str(p.parent.resolve()), exist_ok=True)
with open(output_path, 'w') as fid:
table_to_file(fid,
image_pairs,
header='# query_image, map_image, score')
logger.info('all done')
def export_colmap(kapture_dirpath: str,
colmap_database_filepath: str,
colmap_reconstruction_dirpath: Optional[str],
colmap_rig_filepath: str = None,
force_overwrite_existing: bool = False) -> None:
"""
Exports kapture data to colmap database and or reconstruction text files.
:param kapture_dirpath: kapture top directory
:param colmap_database_filepath: path to colmap database file
:param colmap_reconstruction_dirpath: path to colmap reconstruction directory
:param colmap_rig_filepath: path to colmap rig file
:param force_overwrite_existing: Silently overwrite colmap files if already exists.
"""
os.makedirs(path.dirname(colmap_database_filepath), exist_ok=True)
if colmap_reconstruction_dirpath:
os.makedirs(colmap_reconstruction_dirpath, exist_ok=True)
assert colmap_database_filepath
if path.isfile(colmap_database_filepath):
to_delete = force_overwrite_existing or (input(
'database file already exist, would you like to delete it ? [y/N]'
).lower() == 'y')
if to_delete:
logger.info('deleting already existing {}'.format(
colmap_database_filepath))
os.remove(colmap_database_filepath)
logger.info(
'creating colmap database in {}'.format(colmap_database_filepath))
db = COLMAPDatabase.connect(colmap_database_filepath)
if not is_colmap_db_empty(db):
raise ValueError(
'the existing colmap database is not empty : {}'.format(
colmap_database_filepath))
logger.info('loading kapture files...')
kapture_data = csv.kapture_from_dir(kapture_dirpath)
assert isinstance(kapture_data, kapture.Kapture)
# COLMAP does not fully support rigs.
if kapture_data.rigs is not None:
# make sure, rigs are not used in trajectories.
logger.info('remove rigs notation.')
rigs_remove_inplace(kapture_data.trajectories, kapture_data.rigs)
# write colmap database
kapture_to_colmap(kapture_data, kapture_dirpath, db)
if colmap_reconstruction_dirpath:
# create text files
colmap_camera_ids = get_colmap_camera_ids_from_db(
db, kapture_data.records_camera)
colmap_image_ids = get_colmap_image_ids_from_db(db)
export_to_colmap_txt(
colmap_reconstruction_dirpath=colmap_reconstruction_dirpath,
kapture_data=kapture_data,
kapture_dirpath=kapture_dirpath,
colmap_camera_ids=colmap_camera_ids,
colmap_image_ids=colmap_image_ids)
if colmap_rig_filepath:
try:
if kapture_data.rigs is None:
raise ValueError('No rig to export.')
export_colmap_rigs(colmap_rig_filepath, kapture_data.rigs,
kapture_data.records_camera,
colmap_camera_ids)
except Exception as e:
warnings.warn(e)
logger.warning(e)
db.close()
def extract_kapture_global(kapture_root,
config,
output_dir='',
overwrite=False):
logging.info('Extracting NetVLAD features with configuration:\n', config)
# use kapture io to identify image paths and loop
kdata = kapture_from_dir(kapture_root, matches_pairsfile_path=None,
skip_list= [kapture.Matches,
kapture.Points3d,
kapture.Observations,
kapture.Keypoints,
kapture.Descriptors])
assert kdata.records_camera is not None
export_dir = output_dir if output_dir else kapture_root # root of output directory for features
os.makedirs(export_dir, exist_ok=True)
image_list = [filename for _, _, filename in kapture.flatten(kdata.records_camera)]
# resume extraction if some features exist
try:
# load features if there are any
kdata.global_features = global_features_from_dir(export_dir, None)
if kdata.global_features is not None and not overwrite:
image_list = [name for name in image_list if name not in kdata.global_features]
except FileNotFoundError:
pass
except:
logging.exception("Error with importing existing global features.")
# clear features first if overwriting
if overwrite: delete_existing_kapture_files(export_dir, True, only=[kapture.GlobalFeatures])
if len(image_list) == 0:
print('All features were already extracted')
return
else:
print(f'Extracting NetVLAD features for {len(image_list)} images')
# for the global descriptor type specification
global_dtype = None if kdata.global_features is None else kdata.global_features.dtype
global_dsize = None if kdata.global_features is None else kdata.global_features.dsize
# setup network
tf.reset_default_graph()
if config['grayscale']:
tf_batch = tf.placeholder(
dtype=tf.float32, shape=[None, None, None, 1])
else:
tf_batch = tf.placeholder(
dtype=tf.float32, shape=[None, None, None, 3])
# load network and checkpoint
net = nets.vgg16NetvladPca(tf_batch)
saver = tf.train.Saver()
sess = tf.Session()
checkpoint = chkpt_path + '/' + config['checkpoint']
saver.restore(sess, checkpoint)
for image_name in image_list:
img_path = get_image_fullpath(kapture_root, image_name)
if img_path.endswith('.txt'):
args.images = open(img_path).read().splitlines() + args.images
continue
print(f"\nExtracting features for {img_path}")
if config['grayscale']:
image = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
image = np.expand_dims(
np.expand_dims(image, axis=0), axis=-1)
else:
image = cv2.imread(img_path, cv2.COLOR_BGR2RGB)
image = np.expand_dims(image, axis=0)
descriptor = sess.run(net, feed_dict={tf_batch: image})[:, :config['pca_dim']]
descriptor = np.squeeze(descriptor)
# write global descriptor type specification
if global_dtype is None:
global_dtype = descriptor.dtype
global_dsize = len(descriptor)
kdata.global_features = kapture.GlobalFeatures('netvlad', global_dtype, global_dsize)
global_descriptors_config_abs_path = get_csv_fullpath(kapture.GlobalFeatures, export_dir)
descriptors_to_file(global_descriptors_config_abs_path, kdata.global_features)
else:
assert kdata.global_features.type_name == "netvlad"
assert kdata.global_features.dtype == descriptor.dtype
assert kdata.global_features.dsize == len(descriptor)
# get output paths
global_descriptors_abs_path = get_global_features_fullpath(export_dir, image_name)
image_global_features_to_file(global_descriptors_abs_path, descriptor)
kdata.global_features.add(image_name)
# sess.close() # close session before initializing again for next submap
if not global_features_check_dir(kdata.global_features, export_dir):
print('global feature extraction ended successfully but not all files were saved')
