def extract_kapture_keypoints(kapture_root,
config,
output_dir='',
overwrite=False):
"""
Extract r2d2 keypoints and descritors to the kapture format directly
"""
print('extract_kapture_keypoints...')
kdata = kapture_from_dir(kapture_root, matches_pairsfile_path=None,
skip_list= [kapture.GlobalFeatures,
kapture.Matches,
kapture.Points3d,
kapture.Observations])
export_dir = output_dir if output_dir else kapture_root # root of output directory for features
os.makedirs(export_dir, exist_ok=True)
assert kdata.records_camera is not None
image_list = [filename for _, _, filename in kapture.flatten(kdata.records_camera)]
# resume extraction if some features exist
try:
# load existing features, if any
kdata.keypoints = keypoints_from_dir(export_dir, None)
kdata.descriptors = descriptors_from_dir(export_dir, None)
if kdata.keypoints is not None and kdata.descriptors is not None and not overwrite:
image_list = [name for name in image_list if name not in kdata.keypoints or name not in kdata.descriptors]
except FileNotFoundError:
pass
except:
logging.exception("Error with importing existing local features.")
# clear features first if overwriting
if overwrite: delete_existing_kapture_files(export_dir, True, only=[kapture.Descriptors, kapture.Keypoints])
if len(image_list) == 0:
print('All features were already extracted')
return
else:
print(f'Extracting r2d2 features for {len(image_list)} images')
iscuda = common.torch_set_gpu([torch.cuda.is_available()])
# load the network...
net = load_network(config['checkpoint'])
if iscuda: net = net.cuda()
# create the non-maxima detector
detector = NonMaxSuppression(
rel_thr = config['reliability_thr'],
rep_thr = config['repeatability_thr'])
keypoints_dtype = None if kdata.keypoints is None else kdata.keypoints.dtype
descriptors_dtype = None if kdata.descriptors is None else kdata.descriptors.dtype
keypoints_dsize = None if kdata.keypoints is None else kdata.keypoints.dsize
descriptors_dsize = None if kdata.descriptors is None else kdata.descriptors.dsize
for image_name in image_list:
img_path = get_image_fullpath(kapture_root, image_name)
if img_path.endswith('.txt'):
images = open(img_path).read().splitlines() + images
continue
print(f"\nExtracting features for {img_path}")
img = Image.open(img_path).convert('RGB')
W, H = img.size
img = norm_RGB(img)[None]
if iscuda: img = img.cuda()
# extract keypoints/descriptors for a single image
xys, desc, scores = extract_multiscale(net, img, detector,
scale_f = config['scale_f'],
min_scale = config['min_scale'],
max_scale = config['max_scale'],
min_size = config['min_size'],
max_size = config['max_size'],
verbose = True)
xys = xys.cpu().numpy()
desc = desc.cpu().numpy()
scores = scores.cpu().numpy()
idxs = scores.argsort()[-config['top_k'] or None:]
xys = xys[idxs]
desc = desc[idxs]
if keypoints_dtype is None or descriptors_dtype is None:
keypoints_dtype = xys.dtype
descriptors_dtype = desc.dtype
keypoints_dsize = xys.shape[1]
descriptors_dsize = desc.shape[1]
kdata.keypoints = kapture.Keypoints('r2d2', keypoints_dtype, keypoints_dsize)
kdata.descriptors = kapture.Descriptors('r2d2', descriptors_dtype, descriptors_dsize)
keypoints_config_absolute_path = get_csv_fullpath(kapture.Keypoints, export_dir)
descriptors_config_absolute_path = get_csv_fullpath(kapture.Descriptors, export_dir)
keypoints_to_file(keypoints_config_absolute_path, kdata.keypoints)
descriptors_to_file(descriptors_config_absolute_path, kdata.descriptors)
else:
assert kdata.keypoints.type_name == 'r2d2'
assert kdata.descriptors.type_name == 'r2d2'
assert kdata.keypoints.dtype == xys.dtype
assert kdata.descriptors.dtype == desc.dtype
assert kdata.keypoints.dsize == xys.shape[1]
assert kdata.descriptors.dsize == desc.shape[1]
keypoints_fullpath = get_keypoints_fullpath(export_dir, image_name)
print(f"Saving {xys.shape[0]} keypoints to {keypoints_fullpath}")
image_keypoints_to_file(keypoints_fullpath, xys)
kdata.keypoints.add(image_name)
descriptors_fullpath = get_descriptors_fullpath(export_dir, image_name)
print(f"Saving {desc.shape[0]} descriptors to {descriptors_fullpath}")
image_descriptors_to_file(descriptors_fullpath, desc)
kdata.descriptors.add(image_name)
if not keypoints_check_dir(kdata.keypoints, export_dir) or \
not descriptors_check_dir(kdata.descriptors, export_dir):
print('local feature extraction ended successfully but not all files were saved')
if keypoints_dtype is None or descriptors_dtype is None:
keypoints_dtype = keypoints.dtype
descriptors_dtype = descriptors.dtype
keypoints_dsize = keypoints.shape[1]
descriptors_dsize = descriptors.shape[1]
kdata.keypoints = kapture.Keypoints('d2net', keypoints_dtype, keypoints_dsize)
kdata.descriptors = kapture.Descriptors('d2net', descriptors_dtype, descriptors_dsize)
keypoints_config_absolute_path = get_csv_fullpath(kapture.Keypoints, args.kapture_root)
descriptors_config_absolute_path = get_csv_fullpath(kapture.Descriptors, args.kapture_root)
keypoints_to_file(keypoints_config_absolute_path, kdata.keypoints)
descriptors_to_file(descriptors_config_absolute_path, kdata.descriptors)
else:
assert kdata.keypoints.type_name == 'd2net'
assert kdata.descriptors.type_name == 'd2net'
assert kdata.keypoints.dtype == keypoints.dtype
assert kdata.descriptors.dtype == descriptors.dtype
assert kdata.keypoints.dsize == keypoints.shape[1]
assert kdata.descriptors.dsize == descriptors.shape[1]
keypoints_fullpath = get_keypoints_fullpath(args.kapture_root, image_name)
print(f"Saving {keypoints.shape[0]} keypoints to {keypoints_fullpath}")
image_keypoints_to_file(keypoints_fullpath, keypoints)
kdata.keypoints.add(image_name)
descriptors_fullpath = get_descriptors_fullpath(args.kapture_root, image_name)
descriptors_dsize = descriptors.shape[1]
kdata.keypoints[args.keypoints_type] = kapture.Keypoints(
'd2net', keypoints_dtype, keypoints_dsize)
kdata.descriptors[args.descriptors_type] = kapture.Descriptors(
'd2net', descriptors_dtype, descriptors_dsize,
args.keypoints_type, 'L2')
keypoints_config_absolute_path = get_feature_csv_fullpath(
kapture.Keypoints, args.keypoints_type, args.kapture_root)
descriptors_config_absolute_path = get_feature_csv_fullpath(
kapture.Descriptors, args.descriptors_type, args.kapture_root)
keypoints_to_file(keypoints_config_absolute_path,
kdata.keypoints[args.keypoints_type])
descriptors_to_file(descriptors_config_absolute_path,
kdata.descriptors[args.descriptors_type])
else:
assert kdata.keypoints[
args.keypoints_type].dtype == keypoints.dtype
assert kdata.descriptors[
args.descriptors_type].dtype == descriptors.dtype
assert kdata.keypoints[
args.keypoints_type].dsize == keypoints.shape[1]
assert kdata.descriptors[
args.descriptors_type].dsize == descriptors.shape[1]
assert kdata.descriptors[
args.descriptors_type].keypoints_type == args.keypoints_type
assert kdata.descriptors[args.descriptors_type].metric_type == 'L2'
keypoints_fullpath = get_keypoints_fullpath(args.keypoints_type,
args.kapture_root,
def extract_kapture_keypoints(args):
"""
Extract r2d2 keypoints and descritors to the kapture format directly
"""
print('extract_kapture_keypoints...')
kdata = kapture_from_dir(args.kapture_root,
matches_pairsfile_path=None,
skip_list=[
kapture.GlobalFeatures, kapture.Matches,
kapture.Points3d, kapture.Observations
])
assert kdata.records_camera is not None
image_list = [
filename for _, _, filename in kapture.flatten(kdata.records_camera)
]
if kdata.keypoints is not None and kdata.descriptors is not None:
image_list = [
name for name in image_list
if name not in kdata.keypoints or name not in kdata.descriptors
]
if len(image_list) == 0:
print('All features were already extracted')
return
else:
print(f'Extracting r2d2 features for {len(image_list)} images')
iscuda = common.torch_set_gpu(args.gpu)
# load the network...
net = load_network(args.model)
if iscuda: net = net.cuda()
# create the non-maxima detector
detector = NonMaxSuppression(rel_thr=args.reliability_thr,
rep_thr=args.repeatability_thr)
keypoints_dtype = None if kdata.keypoints is None else kdata.keypoints.dtype
descriptors_dtype = None if kdata.descriptors is None else kdata.descriptors.dtype
keypoints_dsize = None if kdata.keypoints is None else kdata.keypoints.dsize
descriptors_dsize = None if kdata.descriptors is None else kdata.descriptors.dsize
for image_name in image_list:
img_path = get_image_fullpath(args.kapture_root, image_name)
print(f"\nExtracting features for {img_path}")
img = Image.open(img_path).convert('RGB')
W, H = img.size
img = norm_RGB(img)[None]
if iscuda: img = img.cuda()
# extract keypoints/descriptors for a single image
xys, desc, scores = extract_multiscale(net,
img,
detector,
scale_f=args.scale_f,
min_scale=args.min_scale,
max_scale=args.max_scale,
min_size=args.min_size,
max_size=args.max_size,
verbose=True)
xys = xys.cpu().numpy()
desc = desc.cpu().numpy()
scores = scores.cpu().numpy()
idxs = scores.argsort()[-args.top_k or None:]
xys = xys[idxs]
desc = desc[idxs]
if keypoints_dtype is None or descriptors_dtype is None:
keypoints_dtype = xys.dtype
descriptors_dtype = desc.dtype
keypoints_dsize = xys.shape[1]
descriptors_dsize = desc.shape[1]
kdata.keypoints = kapture.Keypoints('r2d2', keypoints_dtype,
keypoints_dsize)
kdata.descriptors = kapture.Descriptors('r2d2', descriptors_dtype,
descriptors_dsize)
keypoints_config_absolute_path = get_csv_fullpath(
kapture.Keypoints, args.kapture_root)
descriptors_config_absolute_path = get_csv_fullpath(
kapture.Descriptors, args.kapture_root)
keypoints_to_file(keypoints_config_absolute_path, kdata.keypoints)
descriptors_to_file(descriptors_config_absolute_path,
kdata.descriptors)
else:
assert kdata.keypoints.type_name == 'r2d2'
assert kdata.descriptors.type_name == 'r2d2'
assert kdata.keypoints.dtype == xys.dtype
assert kdata.descriptors.dtype == desc.dtype
assert kdata.keypoints.dsize == xys.shape[1]
assert kdata.descriptors.dsize == desc.shape[1]
keypoints_fullpath = get_keypoints_fullpath(args.kapture_root,
image_name)
print(f"Saving {xys.shape[0]} keypoints to {keypoints_fullpath}")
image_keypoints_to_file(keypoints_fullpath, xys)
kdata.keypoints.add(image_name)
descriptors_fullpath = get_descriptors_fullpath(
args.kapture_root, image_name)
print(f"Saving {desc.shape[0]} descriptors to {descriptors_fullpath}")
image_descriptors_to_file(descriptors_fullpath, desc)
kdata.descriptors.add(image_name)
if not keypoints_check_dir(kdata.keypoints, args.kapture_root) or \
not descriptors_check_dir(kdata.descriptors, args.kapture_root):
print(
'local feature extraction ended successfully but not all files were saved'
)
def extract_kapture_global(kapture_root,
config,
output_dir='',
overwrite=False):
logging.info('Extracting NetVLAD features with configuration:\n', config)
# use kapture io to identify image paths and loop
kdata = kapture_from_dir(kapture_root, matches_pairsfile_path=None,
skip_list= [kapture.Matches,
kapture.Points3d,
kapture.Observations,
kapture.Keypoints,
kapture.Descriptors])
assert kdata.records_camera is not None
export_dir = output_dir if output_dir else kapture_root # root of output directory for features
os.makedirs(export_dir, exist_ok=True)
image_list = [filename for _, _, filename in kapture.flatten(kdata.records_camera)]
# resume extraction if some features exist
try:
# load features if there are any
kdata.global_features = global_features_from_dir(export_dir, None)
if kdata.global_features is not None and not overwrite:
image_list = [name for name in image_list if name not in kdata.global_features]
except FileNotFoundError:
pass
except:
logging.exception("Error with importing existing global features.")
# clear features first if overwriting
if overwrite: delete_existing_kapture_files(export_dir, True, only=[kapture.GlobalFeatures])
if len(image_list) == 0:
print('All features were already extracted')
return
else:
print(f'Extracting NetVLAD features for {len(image_list)} images')
# for the global descriptor type specification
global_dtype = None if kdata.global_features is None else kdata.global_features.dtype
global_dsize = None if kdata.global_features is None else kdata.global_features.dsize
# setup network
tf.reset_default_graph()
if config['grayscale']:
tf_batch = tf.placeholder(
dtype=tf.float32, shape=[None, None, None, 1])
else:
tf_batch = tf.placeholder(
dtype=tf.float32, shape=[None, None, None, 3])
# load network and checkpoint
net = nets.vgg16NetvladPca(tf_batch)
saver = tf.train.Saver()
sess = tf.Session()
checkpoint = chkpt_path + '/' + config['checkpoint']
saver.restore(sess, checkpoint)
for image_name in image_list:
img_path = get_image_fullpath(kapture_root, image_name)
if img_path.endswith('.txt'):
args.images = open(img_path).read().splitlines() + args.images
continue
print(f"\nExtracting features for {img_path}")
if config['grayscale']:
image = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
image = np.expand_dims(
np.expand_dims(image, axis=0), axis=-1)
else:
image = cv2.imread(img_path, cv2.COLOR_BGR2RGB)
image = np.expand_dims(image, axis=0)
descriptor = sess.run(net, feed_dict={tf_batch: image})[:, :config['pca_dim']]
descriptor = np.squeeze(descriptor)
# write global descriptor type specification
if global_dtype is None:
global_dtype = descriptor.dtype
global_dsize = len(descriptor)
kdata.global_features = kapture.GlobalFeatures('netvlad', global_dtype, global_dsize)
global_descriptors_config_abs_path = get_csv_fullpath(kapture.GlobalFeatures, export_dir)
descriptors_to_file(global_descriptors_config_abs_path, kdata.global_features)
else:
assert kdata.global_features.type_name == "netvlad"
assert kdata.global_features.dtype == descriptor.dtype
assert kdata.global_features.dsize == len(descriptor)
# get output paths
global_descriptors_abs_path = get_global_features_fullpath(export_dir, image_name)
image_global_features_to_file(global_descriptors_abs_path, descriptor)
kdata.global_features.add(image_name)
# sess.close() # close session before initializing again for next submap
if not global_features_check_dir(kdata.global_features, export_dir):
print('global feature extraction ended successfully but not all files were saved')