def test_type_checking(self):
self.assertRaises(TypeError, kapture.Points3d, (2, 3, 4))
points = kapture.Points3d()
self.assertEqual(points.dtype.name, 'float64')
data = list()
data.append(list(range(1, 7)))
points = kapture.Points3d(data)
self.assertEqual(points.dtype.name, 'float64')
rows = 18
points = np.array(range(0, rows * kapture.Points3d.XYZRGB))
points = np.reshape(points, (rows, kapture.Points3d.XYZRGB))
points = kapture.Points3d(points)
self.assertEqual(points.dtype.name, 'float64')
# test slicing
val1 = points[:, 0:3]
val2 = points[:]
val3 = points[:, 0]
val4 = points[0, :]
val5 = points[0]
val6 = points[0:2, :]
val7 = points[0, 0]
self.assertNotIsInstance(val1, kapture.Points3d)
self.assertIsInstance(val2, kapture.Points3d)
self.assertNotIsInstance(val3, kapture.Points3d)
self.assertIsInstance(val4, kapture.Points3d)
self.assertIsInstance(val5, kapture.Points3d)
self.assertIsInstance(val6, kapture.Points3d)
self.assertIsInstance(val7, float)
def test_from_numpy_array(self):
data = np.array([[0] * 6])
points3d = kapture.Points3d(data)
self.assertIsInstance(points3d, kapture.Points3d)
self.assertEqual(points3d.shape, (1, 6))
self.assertTrue(points3d)
data = np.arange(12).reshape((2, 6))
points3d = kapture.Points3d(data)
self.assertIsInstance(points3d, kapture.Points3d)
self.assertEqual(points3d.shape, (2, 6))
self.assertTrue(points3d)
data = np.arange(16).reshape((2, 8))
points3d = kapture.Points3d(data[:, 0:6])
self.assertIsInstance(points3d, kapture.Points3d)
self.assertEqual(points3d.shape, (2, 6))
self.assertTrue(points3d)
data = list()
data.append(list(range(1, 7)))
points3d = kapture.Points3d(data)
self.assertIsInstance(points3d, kapture.Points3d)
self.assertEqual(points3d.shape, (1, 6))
self.assertTrue(np.all(np.isclose(points3d.as_array(), data)))
data = np.vstack([data, data])
points3d = kapture.Points3d(data)
self.assertIsInstance(points3d, kapture.Points3d)
self.assertEqual(points3d.shape, (2, 6))
self.assertTrue(np.all(np.isclose(points3d.as_array(), data)))
def merge_points3d_and_observations(
pts3d_obs: List[Tuple[Optional[kapture.Points3d],
Optional[kapture.Observations]]]
) -> Tuple[kapture.Points3d, kapture.Observations]:
"""
Merge a list of points3d with their observations.
:param pts3d_obs: list of points3d with observations to merge
:return: merged points3d associated to observations
"""
assert len(pts3d_obs) > 0
merged_points3d = kapture.Points3d()
merged_observations = kapture.Observations()
point3d_offset = 0
for points3d, observations in pts3d_obs:
if points3d is None:
continue
merged_points3d = kapture.Points3d(
np.vstack([merged_points3d, points3d]))
if observations is not None:
for point3d_idx, (image_path,
keypoint_idx) in kapture.flatten(observations):
merged_observations.add(point3d_idx + point3d_offset,
image_path, keypoint_idx)
point3d_offset += merged_points3d.shape[0]
return merged_points3d, merged_observations
def test_from_numpy_array_invalid(self):
data = np.arange(16).reshape((8, 2))
with self.assertRaises(ValueError):
kapture.Points3d(data)
data = np.arange(16)
with self.assertRaises(ValueError):
kapture.Points3d(data)
def merge_points3d(points3d_list: List[Optional[kapture.Points3d]]) -> kapture.Points3d:
"""
Merge several points3d lists in one.
:param points3d_list: list of points3d to merge
:return: merged points3d
"""
assert len(points3d_list) > 0
merged_points3d = kapture.Points3d()
for points3d in points3d_list:
if points3d is None:
continue
merged_points3d = kapture.Points3d(np.vstack([merged_points3d, points3d]))
return merged_points3d
def test_slices(self):
rows = 18
np_points = np.array(range(0, rows * kapture.Points3d.XYZRGB))
np_points = np.reshape(np_points, (rows * 2, 3))
self.assertRaises(ValueError, kapture.Points3d, np_points)
np_points = np.reshape(np_points, (rows, kapture.Points3d.XYZRGB))
# construct from numpy array
points = kapture.Points3d(np_points)
# construct from a points3d
points = kapture.Points3d(points)
self.assertEqual(points[0, 0], 0.0)
self.assertEqual(points[17, 5], 107.0)
np.testing.assert_array_equal(points.as_array(), np_points)
np.testing.assert_array_equal(points[0, 0:3], np.array([0.0, 1.0, 2.0]))
np.testing.assert_array_equal(points[0, 3:], np.array([3.0, 4.0, 5.0]))
np.testing.assert_array_equal(points[0, :].as_array(), np.array([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]))
def points3d_from_file(filepath: str) -> kapture.Points3d:
"""
Reads 3d points from CSV file.
:param filepath: path to CSV file
:return: the 3d points
"""
data = np.loadtxt(filepath, dtype=np.float, delimiter=',', comments='#')
data = data.reshape((-1, 6)) # make sure of the shape, even if single line file.
return kapture.Points3d(data)
def import_from_colmap_points3d_txt(colmap_points3d_filepath: str,
image_names: Dict[int, str] = None,
skip_observations: bool = False
) -> Tuple[kapture.Points3d, Optional[kapture.Observations]]:
"""
Imports the colmap file named "points3d.txt" containing both points 3D and observations.
:param colmap_points3d_filepath: path to the colmap file named "points3d.txt"
:param image_names: dict of image names matching the colmap image id to the kapture image name
colmap_image_idx -> kapture_image_filename
:param skip_observations: skip import of observations id true.
:return: kapture 3D points and observations
"""
assert path.basename(colmap_points3d_filepath) == 'points3D.txt'
points3d = []
observations = kapture.Observations()
# colmap points3D.txt contains both points 3D and observations
with open(colmap_points3d_filepath, 'r') as file:
# in the reconstruction, spaces and commas can be used as separators
lines = file.readlines()
# eliminate comments
lines = (line for line in lines if not line.startswith('#'))
# split by space and or comma
lines = ([float(value) for value in re.findall(colmap_reconstruction_split_pattern, values)]
for values in lines) # split into an array of floats
for index, values in enumerate(lines):
points3d.append(values[1:4] + values[4:7])
if not skip_observations and len(values) > 8 and len(image_names) > 0:
for image_idx, point_2d_idx in zip(values[8::2], values[9::2]):
filename = image_names.get(int(image_idx), 'unknown')
observations.add(index, filename, int(point_2d_idx))
points3d = kapture.Points3d(np.array(points3d)) if points3d else kapture.Points3d()
observations = None if len(observations) == 0 else observations
return points3d, observations
def test_points_to_files(self):
data = np.array([
[0, 1, 0, 0, 0, 0],
[1, 0, 0, 255, 0, 0],
[0, 0, 1, 255, 0, 255],
])
points3d = kapture.Points3d(data)
filepath = path.join(self._tempdir.name, 'points3d.txt')
csv.points3d_to_file(filepath, points3d)
with open(filepath, 'rt') as file:
lines = file.readlines()
self.assertEqual(4, len(lines))
self.assertTrue(lines[0].startswith('#'))
first_line = [float(f) for f in lines[1].split(',')]
self.assertEqual(6, len(first_line))
self.assertAlmostEqual([0.0, 1.0, 0.0, 0.0, 0.0, 0.0], first_line)
def test_empty(self):
points = kapture.Points3d()
self.assertIsInstance(points, kapture.Points3d)
self.assertEqual(points.shape, (0, kapture.Points3d.XYZRGB))
self.assertRaises(IndexError, points.__getitem__, (0, 0))
self.assertFalse(points)
def import_bundler(
bundler_path: str,
image_list_path: str,
image_dir_path: str,
kapture_dir_path: str,
ignore_trajectories: bool,
add_reconstruction: bool,
force_overwrite_existing: bool = False,
images_import_method: TransferAction = TransferAction.skip) -> None:
"""
Imports bundler data and save them as kapture.
:param bundler_path: path to the bundler model file
:param image_list_path: path to the file containing the list of image names
:param image_dir_path: input path to bundler image directory.
:param kapture_dir_path: path to kapture top directory
:param ignore_trajectories: if True, will not import the trajectories
:param add_reconstruction: if True, will create 3D points and observations
:param force_overwrite_existing: Silently overwrite kapture files if already exists.
:param images_import_method: choose how to import actual image files.
"""
os.makedirs(kapture_dir_path, exist_ok=True)
delete_existing_kapture_files(kapture_dir_path,
force_erase=force_overwrite_existing)
logger.info('loading all content...')
# if there is a filter list, parse it
with open(image_list_path) as file:
file_content = file.readlines()
# remove end line char and empty lines
image_list = [line.rstrip() for line in file_content if line != '\n']
with open(bundler_path) as file:
bundler_content = file.readlines()
# remove end line char and empty lines
bundler_content = [
line.rstrip() for line in bundler_content if line != '\n'
]
assert bundler_content[0] == "# Bundle file v0.3"
# <num_cameras> <num_points>
line_1 = bundler_content[1].split()
number_of_cameras = int(line_1[0])
number_of_points = int(line_1[1])
offset = 2
number_of_lines_per_camera = 5 # 1 camera + 3 rotation + 1 translation
cameras = kapture.Sensors()
images = kapture.RecordsCamera()
trajectories = kapture.Trajectories() if not ignore_trajectories else None
points3d = [] if add_reconstruction else None
keypoints = kapture.Keypoints('sift', np.float32,
2) if add_reconstruction else None
observations = kapture.Observations() if add_reconstruction else None
image_mapping = [] # bundler camera_id -> (name, width, height)
for i in range(0, number_of_cameras):
start_index = i * number_of_lines_per_camera + offset
file_name = image_list[i]
# process camera info
line_camera = bundler_content[start_index].split()
focal_length = float(line_camera[0])
k1 = float(line_camera[1])
k2 = float(line_camera[2])
# lazy open
with Image.open(path.join(image_dir_path, file_name)) as im:
width, height = im.size
image_mapping.append((file_name, width, height))
camera = kapture.Camera(
MODEL,
[width, height, focal_length, width / 2, height / 2, k1, k2])
camera_id = f'sensor{i}'
cameras[camera_id] = camera
# process extrinsics
rotation_matrix = [[float(v) for v in line.split()]
for line in bundler_content[start_index +
1:start_index + 4]]
quaternion_wxyz = quaternion.from_rotation_matrix(rotation_matrix)
translation = np.array(
[float(v) for v in bundler_content[start_index + 4].split()])
pose = kapture.PoseTransform(quaternion_wxyz, translation)
# The Bundler model uses a coordinate system that differs from the *computer vision camera
# coordinate system*. More specifically, they use the camera coordinate system typically used
# in *computer graphics*. In this camera coordinate system, the camera is looking down the
# `-z`-axis, with the `x`-axis pointing to the right and the `y`-axis pointing upwards.
# rotation Pi around the x axis to get the *computer vision camera
# coordinate system*
rotation_around_x = quaternion.quaternion(0.0, 1.0, 0.0, 0.0)
transformation = kapture.PoseTransform(rotation_around_x,
np.array([0, 0, 0]))
images[(i, camera_id)] = file_name
if trajectories is not None:
# transformation.inverse() is equal to transformation (rotation around -Pi or Pi around X is the same)
trajectories[(i, camera_id)] = kapture.PoseTransform.compose(
[transformation, pose, transformation])
if points3d is not None and number_of_points > 0:
assert keypoints is not None
assert observations is not None
offset += number_of_cameras * number_of_lines_per_camera
number_of_lines_per_point = 3 # position color viewlist
# (image_name, bundler_keypoint_id ) -> keypoint_id
known_keypoints = {}
local_keypoints = {}
for i in range(0, number_of_points):
start_index = i * number_of_lines_per_point + offset
position = [float(v) for v in bundler_content[start_index].split()]
# apply transformation
position = [position[0], -position[1], -position[2]]
color = [
float(v) for v in bundler_content[start_index + 1].split()
]
# <view list>: length of the list + [<camera> <key> <x> <y>]
# x, y origin is the center of the image
view_list = bundler_content[start_index + 2].split()
number_of_observations = int(view_list[0])
for j in range(number_of_observations):
camera_id = int(view_list[1 + 4 * j + 0])
keypoint_id = int(view_list[1 + 4 * j + 1])
x = float(view_list[1 + 4 * j + 2])
y = float(view_list[1 + 4 * j + 3])
file_name, width, height = image_mapping[camera_id]
# put (0,0) in upper left corner
x += (width / 2)
y += (height / 2)
# init local_keypoints if needed
if file_name not in local_keypoints:
local_keypoints[file_name] = []
# do not add the same keypoint twice
if (file_name, keypoint_id) not in known_keypoints:
# in the kapture format, keypoint id is different. Note that it starts from 0
known_keypoints[(file_name, keypoint_id)] = len(
local_keypoints[file_name])
local_keypoints[file_name].append([x, y])
keypoint_idx = known_keypoints[(file_name, keypoint_id)]
observations.add(i, file_name, keypoint_idx)
points3d.append(position + color)
points3d = np.array(points3d)
# finally, convert local_keypoints to np.ndarray and add them to the global keypoints variable
keypoints = kapture.Keypoints('sift', np.float32, 2)
for image_filename, keypoints_array in local_keypoints.items():
keypoints_np_array = np.array(keypoints_array).astype(np.float32)
keypoints_out_path = kapture.io.features.get_keypoints_fullpath(
kapture_dir_path, image_filename)
kapture.io.features.image_keypoints_to_file(
keypoints_out_path, keypoints_np_array)
keypoints.add(image_filename)
if points3d is not None:
points3d = kapture.Points3d(points3d)
# import (copy) image files.
logger.info('import image files ...')
filename_list = [f for _, _, f in kapture.flatten(images)]
import_record_data_from_dir_auto(image_dir_path, kapture_dir_path,
filename_list, images_import_method)
# pack into kapture format
imported_kapture = kapture.Kapture(sensors=cameras,
records_camera=images,
trajectories=trajectories,
points3d=points3d,
keypoints=keypoints,
observations=observations)
logger.info('writing imported data...')
kapture_to_dir(kapture_dir_path, imported_kapture)
def add_frames(self, frames: List[Frame], points3d: List[Keypoint]):
k = self.kapture
if k.records_camera is None:
k.records_camera = kt.RecordsCamera()
if k.trajectories is None:
k.trajectories = kt.Trajectories()
if k.keypoints is None:
k.keypoints = {
self.default_kp_type:
kt.Keypoints(self.default_kp_type, np.float32, 2)
}
if k.points3d is None:
k.points3d = kt.Points3d()
if k.observations is None:
k.observations = kt.Observations()
def check_kp(kp):
return not kp.bad_qlt and kp.inlier_count > self.min_pt3d_obs and kp.inlier_count / kp.total_count > self.min_pt3d_ratio
kp_ids, pts3d = zip(*[(kp.id, kp.pt3d) for kp in points3d
if check_kp(kp)])
I = np.argsort(kp_ids)
pt3d_ids = dict(zip(np.array(kp_ids)[I], np.arange(len(I))))
pt3d_arr = np.array(pts3d)[I, :]
k.points3d = kt.Points3d(
np.concatenate((pt3d_arr, np.ones_like(pt3d_arr) * 128), axis=1))
for f in frames:
if not f.pose.post:
continue
id = f.frame_num
img = f.orig_image
img_file = os.path.join(self.default_cam[1],
'frame%06d.%s' % (id, self.img_format))
img_fullpath = get_record_fullpath(self.path, img_file)
os.makedirs(os.path.dirname(img_fullpath), exist_ok=True)
if not np.isclose(self.scale, 1.0):
img = cv2.resize(img,
None,
fx=self.scale,
fy=self.scale,
interpolation=cv2.INTER_AREA)
if self.img_format == self.IMG_FORMAT_PNG:
cv2.imwrite(img_fullpath, img,
(cv2.IMWRITE_PNG_COMPRESSION, 9))
elif self.img_format == self.IMG_FORMAT_JPG:
cv2.imwrite(img_fullpath, img,
(cv2.IMWRITE_JPEG_QUALITY, self.jpg_qlt))
else:
assert False, 'Invalid image format: %s' % (self.img_format, )
record_id = (id, self.default_cam[0])
k.records_camera[record_id] = img_file
pose = f.pose.post if 1 else (-f.pose.post)
k.trajectories[record_id] = kt.PoseTransform(
r=pose.quat.components, t=pose.loc)
k.keypoints[self.default_kp_type].add(img_file)
uvs = np.zeros((len(f.kps_uv), 2), np.float32)
i = 0
for kp_id, uv in f.kps_uv.items():
if kp_id in pt3d_ids:
k.observations.add(int(pt3d_ids[kp_id]),
self.default_kp_type, img_file, i)
uvs[i, :] = uv / f.img_sc * self.scale
i += 1
image_keypoints_to_file(
get_keypoints_fullpath(self.default_kp_type, self.path,
img_file), uvs[:i, :])
def create_3D_model_from_depth_from_loaded_data(
kdata: kapture.Kapture, input_path: str, tar_handlers: TarCollection,
output_path: str, keypoints_type: Optional[str], depth_sensor_id: str,
topk: int, method: Method, cellsizes: List[str], force: bool):
"""
Create 3D model from a kapture dataset that has registered depth data
Assumes the kapture data is already loaded
"""
logger.info(f'create 3D model using depth data')
if os.path.exists(output_path) and not force:
print(f'outpath already exists, use --force to overwrite')
return -1
if kdata.rigs is not None:
assert kdata.trajectories is not None
kapture.rigs_remove_inplace(kdata.trajectories, kdata.rigs)
if keypoints_type is None:
keypoints_type = try_get_only_key_from_collection(kdata.keypoints)
assert keypoints_type is not None
assert kdata.keypoints is not None
assert keypoints_type in kdata.keypoints
if method == Method.voxelgrid:
vg = VoxelGrid(cellsizes)
# add all 3D points to map that correspond to a keypoint
logger.info('adding points from scan to kapture')
points3d = []
observations = kapture.Observations()
progress_bar = tqdm(total=len(
list(kapture.flatten(kdata.records_camera, is_sorted=True))),
disable=logger.level >= logging.CRITICAL)
for timestamp, sensor_id, sensing_filepath in kapture.flatten(
kdata.records_camera, is_sorted=True):
logger.info(
f'total 3d points: {len(points3d)}, processing {sensing_filepath}')
# check if images have a pose
if timestamp not in kdata.trajectories:
logger.info('{} does not have a pose. skipping ...'.format(
sensing_filepath))
continue
# check if depth map exists
depth_map_record = ''
if timestamp in kdata.records_depth:
if depth_sensor_id is None:
depth_id = sensor_id + '_depth'
else:
depth_id = depth_sensor_id
if depth_id in kdata.records_depth[timestamp]:
depth_map_record = kdata.records_depth[timestamp][depth_id]
depth_map_size = tuple(
[int(x) for x in kdata.sensors[depth_id].camera_params[0:2]])
depth_path = get_depth_map_fullpath(input_path, depth_map_record)
if not os.path.exists(depth_path):
logger.info('no 3D data found for {}. skipping ...'.format(
sensing_filepath))
continue
depth_map = depth_map_from_file(depth_path, depth_map_size)
img = Image.open(get_image_fullpath(input_path,
sensing_filepath)).convert('RGB')
assert img.size[0] == depth_map_size[0]
assert img.size[1] == depth_map_size[1]
kps_raw = load_keypoints(keypoints_type, input_path, sensing_filepath,
kdata.keypoints[keypoints_type].dtype,
kdata.keypoints[keypoints_type].dsize,
tar_handlers)
_, camera_sensor_C, camera_dist = get_camera_matrix_from_kapture(
np.zeros((1, 0, 2), dtype=np.float64), kdata.sensors[sensor_id])
cv2_keypoints, depth_sensor_C, depth_dist = get_camera_matrix_from_kapture(
kps_raw, kdata.sensors[depth_id])
assert np.isclose(depth_sensor_C, camera_sensor_C).all()
assert np.isclose(depth_dist, camera_dist).all()
if np.count_nonzero(camera_dist) > 0:
epsilon = np.finfo(np.float64).eps
stop_criteria = (cv2.TERM_CRITERIA_MAX_ITER +
cv2.TERM_CRITERIA_EPS, 500, epsilon)
undistorted_cv2_keypoints = cv2.undistortPointsIter(
cv2_keypoints,
camera_sensor_C,
camera_dist,
R=None,
P=camera_sensor_C,
criteria=stop_criteria)
else:
undistorted_cv2_keypoints = cv2_keypoints
cv2_keypoints = cv2_keypoints.reshape((kps_raw.shape[0], 2))
undistorted_cv2_keypoints = undistorted_cv2_keypoints.reshape(
(kps_raw.shape[0], 2))
points3d_img = []
rgb_img = []
kp_idxs = []
for idx_kp, kp in enumerate(cv2_keypoints[0:topk]):
u = round(kp[0])
v = round(kp[1])
undist_kp = undistorted_cv2_keypoints[idx_kp]
undist_u = round(undist_kp[0])
undist_v = round(undist_kp[1])
if u >= 0 and u < depth_map_size[
0] and v >= 0 and v < depth_map_size[1]:
if depth_map[v, u] == 0:
continue
pt3d = project_kp_to_3D(undist_u, undist_v, depth_map[v, u],
depth_sensor_C[0,
2], depth_sensor_C[1,
2],
depth_sensor_C[0,
0], depth_sensor_C[1,
1])
points3d_img.append(pt3d)
rgb_img.append(img.getpixel((u, v)))
kp_idxs.append(idx_kp)
# transform to world coordinates (pt3d from a depth map is in camera coordinates)
# we use sensor_id here because we assume that the image and the corresponding depthmap have the same pose
# and sometimes, the pose might only be provided for the images
cam_to_world = kdata.trajectories[timestamp][sensor_id].inverse()
if len(points3d_img) == 0:
continue
points3d_img = cam_to_world.transform_points(np.array(points3d_img))
for idx_kp, pt3d, rgb in zip(kp_idxs, points3d_img, rgb_img):
if not np.isnan(pt3d).any():
# apply transform (alignment)
if method == Method.voxelgrid:
assert vg is not None
if not vg.exists(pt3d):
# add 3D point
points3d.append(list(pt3d) + list(rgb))
# add observation
observations.add(
len(points3d) - 1, keypoints_type,
sensing_filepath, idx_kp)
vg.add(pt3d, len(points3d) - 1, sensing_filepath)
else:
ret = vg.append(pt3d, sensing_filepath)
if ret is not None:
observations.add(ret[0], keypoints_type,
sensing_filepath, idx_kp)
elif method == Method.all:
# add 3D point
points3d.append(list(pt3d) + list(rgb))
# add observation
observations.add(
len(points3d) - 1, keypoints_type, sensing_filepath,
idx_kp)
# save_3Dpts_to_ply(points3d, os.path.join(output_path, 'map.ply'))
progress_bar.update(1)
progress_bar.close()
kdata.points3d = kapture.Points3d(np.array(points3d))
kdata.observations = observations
logger.info('saving ...')
kapture_to_dir(output_path, kdata)
# save_3Dpts_to_ply(points3d, os.path.join(output_path, 'map.ply'))
logger.info('all done')
def import_opensfm(
opensfm_root_dir: str,
kapture_root_dir: str,
force_overwrite_existing: bool = False,
images_import_method: TransferAction = TransferAction.copy) -> None:
"""
Convert an openSfM structure to a kapture on disk. Also copy, move or link the images files if necessary.
:param opensfm_root_dir: the openSfM top directory
:param kapture_root_dir: top directory of kapture created
:param force_overwrite_existing: if true, will remove existing kapture data without prompting the user
:param images_import_method: action to apply on images: link, copy, move or do nothing.
:return: the constructed kapture object
"""
disable_tqdm = logger.getEffectiveLevel() != logging.INFO
# load reconstruction
opensfm_reconstruction_filepath = path.join(opensfm_root_dir,
'reconstruction.json')
with open(opensfm_reconstruction_filepath, 'rt') as f:
opensfm_reconstruction = json.load(f)
# remove the single list @ root
opensfm_reconstruction = opensfm_reconstruction[0]
# prepare space for output
os.makedirs(kapture_root_dir, exist_ok=True)
delete_existing_kapture_files(kapture_root_dir,
force_erase=force_overwrite_existing)
# import cameras
kapture_sensors = kapture.Sensors()
assert 'cameras' in opensfm_reconstruction
# import cameras
for osfm_camera_id, osfm_camera in opensfm_reconstruction['cameras'].items(
):
camera = import_camera(osfm_camera, name=osfm_camera_id)
kapture_sensors[osfm_camera_id] = camera
# import shots
logger.info('importing images and trajectories ...')
kapture_images = kapture.RecordsCamera()
kapture_trajectories = kapture.Trajectories()
opensfm_image_dir_path = path.join(opensfm_root_dir, 'images')
assert 'shots' in opensfm_reconstruction
image_timestamps, image_sensors = {}, {
} # used later to retrieve the timestamp of an image.
for timestamp, (image_filename, shot) in enumerate(
opensfm_reconstruction['shots'].items()):
sensor_id = shot['camera']
image_timestamps[image_filename] = timestamp
image_sensors[image_filename] = sensor_id
# in OpenSfm, (sensor, timestamp) is not unique.
rotation_vector = shot['rotation']
q = quaternion.from_rotation_vector(rotation_vector)
translation = shot['translation']
# capture_time = shot['capture_time'] # may be invalid
# gps_position = shot['gps_position']
kapture_images[timestamp, sensor_id] = image_filename
kapture_trajectories[timestamp,
sensor_id] = kapture.PoseTransform(r=q,
t=translation)
# copy image files
filename_list = [f for _, _, f in kapture.flatten(kapture_images)]
import_record_data_from_dir_auto(
source_record_dirpath=opensfm_image_dir_path,
destination_kapture_dirpath=kapture_root_dir,
filename_list=filename_list,
copy_strategy=images_import_method)
# Imports Gnss
kapture_gnss = _import_gnss(opensfm_root_dir, kapture_sensors,
image_sensors, image_timestamps, disable_tqdm)
# Imports descriptors, keypoints and matches
kapture_descriptors, kapture_keypoints, kapture_matches = _import_features_and_matches(
opensfm_root_dir, kapture_root_dir, disable_tqdm)
# import 3-D points
if 'points' in opensfm_reconstruction:
logger.info('importing points 3-D')
opensfm_points = opensfm_reconstruction['points']
points_data = []
for point_id in sorted(opensfm_points):
point_data = opensfm_points[point_id]
point_data = point_data['coordinates'] + point_data['color']
points_data.append(point_data)
kapture_points = kapture.Points3d(points_data)
else:
kapture_points = None
# saving kapture csv files
logger.info('saving kapture files')
kapture_data = kapture.Kapture(sensors=kapture_sensors,
records_camera=kapture_images,
records_gnss=kapture_gnss,
trajectories=kapture_trajectories,
keypoints=kapture_keypoints,
descriptors=kapture_descriptors,
matches=kapture_matches,
points3d=kapture_points)
kapture_to_dir(kapture_root_dir, kapture_data)
def import_nvm(nvm_file_path: str,
nvm_images_path: str,
kapture_path: str,
filter_list_path: Optional[str],
ignore_trajectories: bool,
add_reconstruction: bool,
force_overwrite_existing: bool = False,
images_import_method: TransferAction = TransferAction.skip) -> None:
"""
Imports nvm data to kapture format.
:param nvm_file_path: path to nvm file
:param nvm_images_path: path to NVM images directory.
:param kapture_path: path to kapture root directory.
:param filter_list_path: path to the optional file containing a list of images to process
:param ignore_trajectories: if True, will not create trajectories
:param add_reconstruction: if True, will add observations, keypoints and 3D points.
:param force_overwrite_existing: Silently overwrite kapture files if already exists.
:param images_import_method: choose how to import actual image files.
"""
# TODO implement [optional calibration]
# doc : http://ccwu.me/vsfm/doc.html#nvm
os.makedirs(kapture_path, exist_ok=True)
delete_existing_kapture_files(kapture_path, force_erase=force_overwrite_existing)
logger.info('loading all content...')
# if there is a filter list, parse it
# keep it as Set[str] to easily find images
if filter_list_path:
with open(filter_list_path) as file:
file_content = file.readlines()
# remove end line char and empty lines
filter_list = {line.rstrip() for line in file_content if line != '\n'}
else:
filter_list = None
# now do the nvm
with open(nvm_file_path) as file:
nvm_content = file.readlines()
# remove end line char and empty lines
nvm_content = [line.rstrip() for line in nvm_content if line != '\n']
# only NVM_V3 is supported
assert nvm_content[0] == "NVM_V3"
# offset represents the line pointer
offset = 1
# camera_id_offset keeps tracks of used camera_id in case of multiple reconstructed models
camera_id_offset = 0
# point_id_offset keeps tracks of used point_id in case of multiple reconstructed models
point_id_offset = 0
cameras = kapture.Sensors()
images = kapture.RecordsCamera()
trajectories = kapture.Trajectories() if not ignore_trajectories else None
observations = kapture.Observations() if add_reconstruction else None if add_reconstruction else None
keypoints = kapture.Keypoints('sift', np.float32, 2) if add_reconstruction else None
points3d = [] if add_reconstruction else None
# break if number of cameras == 0 or reached end of file
while True:
# <Model1> <Model2> ...
# Each reconstructed <model> contains the following
# <Number of cameras> <List of cameras>
# <Number of 3D points> <List of points>
# In practice,
# <Number of cameras>
# <List of cameras>, one per line
# <Number of 3D points>
# <List of points>, one per line
number_of_cameras = int(nvm_content[offset])
offset += 1
if number_of_cameras == 0: # a line with <0> signify the end of models
break
logger.debug('importing model cameras...')
# parse all cameras for current model
image_idx_to_image_name = parse_cameras(number_of_cameras,
nvm_content,
offset,
camera_id_offset,
filter_list,
nvm_images_path,
cameras,
images,
trajectories)
offset += number_of_cameras
camera_id_offset += number_of_cameras
# parse all points3d
number_of_points = int(nvm_content[offset])
offset += 1
if points3d is not None and number_of_points > 0:
assert keypoints is not None
assert observations is not None
logger.debug('importing model points...')
parse_points3d(kapture_path,
number_of_points,
nvm_content,
offset,
point_id_offset,
image_idx_to_image_name,
filter_list,
points3d,
keypoints,
observations)
point_id_offset += number_of_points
offset += number_of_points
# reached end of file?
if offset >= len(nvm_content):
break
# do not export values if none were found.
if points3d is not None:
points3d = kapture.Points3d(points3d)
# import (copy) image files.
logger.info('import image files ...')
images_filenames = [f for _, _, f in kapture.flatten(images)]
import_record_data_from_dir_auto(nvm_images_path, kapture_path, images_filenames, images_import_method)
# pack into kapture format
imported_kapture = kapture.Kapture(sensors=cameras, records_camera=images, trajectories=trajectories,
points3d=points3d, keypoints=keypoints, observations=observations)
logger.info('writing imported data...')
kapture_to_dir(kapture_path, imported_kapture)
def import_opensfm(
opensfm_rootdir: str,
kapture_rootdir: str,
force_overwrite_existing: bool = False,
images_import_method: TransferAction = TransferAction.copy) -> None:
disable_tqdm = logger.getEffectiveLevel() != logging.INFO
# load reconstruction
opensfm_reconstruction_filepath = path.join(opensfm_rootdir,
'reconstruction.json')
with open(opensfm_reconstruction_filepath, 'rt') as f:
opensfm_reconstruction = json.load(f)
# remove the single list @ root
opensfm_reconstruction = opensfm_reconstruction[0]
# prepare space for output
os.makedirs(kapture_rootdir, exist_ok=True)
delete_existing_kapture_files(kapture_rootdir,
force_erase=force_overwrite_existing)
# import cameras
kapture_sensors = kapture.Sensors()
assert 'cameras' in opensfm_reconstruction
# import cameras
for osfm_camera_id, osfm_camera in opensfm_reconstruction['cameras'].items(
):
camera = import_camera(osfm_camera, name=osfm_camera_id)
kapture_sensors[osfm_camera_id] = camera
# import shots
logger.info('importing images and trajectories ...')
kapture_images = kapture.RecordsCamera()
kapture_trajectories = kapture.Trajectories()
opensfm_image_dirpath = path.join(opensfm_rootdir, 'images')
assert 'shots' in opensfm_reconstruction
image_timestamps, image_sensors = {}, {
} # used later to retrieve the timestamp of an image.
for timestamp, (image_filename, shot) in enumerate(
opensfm_reconstruction['shots'].items()):
sensor_id = shot['camera']
image_timestamps[image_filename] = timestamp
image_sensors[image_filename] = sensor_id
# in OpenSfm, (sensor, timestamp) is not unique.
rotation_vector = shot['rotation']
q = quaternion.from_rotation_vector(rotation_vector)
translation = shot['translation']
# capture_time = shot['capture_time'] # may be invalid
# gps_position = shot['gps_position']
kapture_images[timestamp, sensor_id] = image_filename
kapture_trajectories[timestamp,
sensor_id] = kapture.PoseTransform(r=q,
t=translation)
# copy image files
filename_list = [f for _, _, f in kapture.flatten(kapture_images)]
import_record_data_from_dir_auto(
source_record_dirpath=opensfm_image_dirpath,
destination_kapture_dirpath=kapture_rootdir,
filename_list=filename_list,
copy_strategy=images_import_method)
# gps from pre-extracted exif, in exif/image_name.jpg.exif
kapture_gnss = None
opensfm_exif_dirpath = path.join(opensfm_rootdir, 'exif')
opensfm_exif_suffix = '.exif'
if path.isdir(opensfm_exif_dirpath):
logger.info('importing GNSS from exif ...')
camera_ids = set(image_sensors.values())
# add a gps sensor for each camera
map_cam_to_gnss_sensor = {
cam_id: 'GPS_' + cam_id
for cam_id in camera_ids
}
for gnss_id in map_cam_to_gnss_sensor.values():
kapture_sensors[gnss_id] = kapture.Sensor(
sensor_type='gnss', sensor_params=['EPSG:4326'])
# build epsg_code for all cameras
kapture_gnss = kapture.RecordsGnss()
opensfm_exif_filepath_list = (
path.join(dirpath, filename)
for dirpath, _, filename_list in os.walk(opensfm_exif_dirpath)
for filename in filename_list
if filename.endswith(opensfm_exif_suffix))
for opensfm_exif_filepath in tqdm(opensfm_exif_filepath_list,
disable=disable_tqdm):
image_filename = path.relpath(
opensfm_exif_filepath,
opensfm_exif_dirpath)[:-len(opensfm_exif_suffix)]
image_timestamp = image_timestamps[image_filename]
image_sensor_id = image_sensors[image_filename]
gnss_timestamp = image_timestamp
gnss_sensor_id = map_cam_to_gnss_sensor[image_sensor_id]
with open(opensfm_exif_filepath, 'rt') as f:
js_root = json.load(f)
if 'gps' not in js_root:
logger.warning(f'NO GPS data in "{opensfm_exif_filepath}"')
continue
gps_coords = {
'x': js_root['gps']['longitude'],
'y': js_root['gps']['latitude'],
'z': js_root['gps'].get('altitude', 0.0),
'dop': js_root['gps'].get('dop', 0),
'utc': 0,
}
logger.debug(
f'found GPS data for ({gnss_timestamp}, {gnss_sensor_id}) in "{opensfm_exif_filepath}"'
)
kapture_gnss[gnss_timestamp,
gnss_sensor_id] = kapture.RecordGnss(**gps_coords)
# import features (keypoints + descriptors)
kapture_keypoints = None # kapture.Keypoints(type_name='opensfm', dsize=4, dtype=np.float64)
kapture_descriptors = None # kapture.Descriptors(type_name='opensfm', dsize=128, dtype=np.uint8)
opensfm_features_dirpath = path.join(opensfm_rootdir, 'features')
opensfm_features_suffix = '.features.npz'
if path.isdir(opensfm_features_dirpath):
logger.info('importing keypoints and descriptors ...')
opensfm_features_file_list = (path.join(
dp, fn) for dp, _, fs in os.walk(opensfm_features_dirpath)
for fn in fs)
opensfm_features_file_list = (
filepath for filepath in opensfm_features_file_list
if filepath.endswith(opensfm_features_suffix))
for opensfm_feature_filename in tqdm(opensfm_features_file_list,
disable=disable_tqdm):
image_filename = path.relpath(
opensfm_feature_filename,
opensfm_features_dirpath)[:-len(opensfm_features_suffix)]
opensfm_image_features = np.load(opensfm_feature_filename)
opensfm_image_keypoints = opensfm_image_features['points']
opensfm_image_descriptors = opensfm_image_features['descriptors']
logger.debug(
f'parsing keypoints and descriptors in {opensfm_feature_filename}'
)
if kapture_keypoints is None:
# print(type(opensfm_image_keypoints.dtype))
# HAHOG = Hessian Affine feature point detector + HOG descriptor
kapture_keypoints = kapture.Keypoints(
type_name='HessianAffine',
dsize=opensfm_image_keypoints.shape[1],
dtype=opensfm_image_keypoints.dtype)
if kapture_descriptors is None:
kapture_descriptors = kapture.Descriptors(
type_name='HOG',
dsize=opensfm_image_descriptors.shape[1],
dtype=opensfm_image_descriptors.dtype)
# convert keypoints file
keypoint_filpath = kapture.io.features.get_features_fullpath(
data_type=kapture.Keypoints,
kapture_dirpath=kapture_rootdir,
image_filename=image_filename)
kapture.io.features.image_keypoints_to_file(
filepath=keypoint_filpath,
image_keypoints=opensfm_image_keypoints)
# register the file
kapture_keypoints.add(image_filename)
# convert descriptors file
descriptor_filpath = kapture.io.features.get_features_fullpath(
data_type=kapture.Descriptors,
kapture_dirpath=kapture_rootdir,
image_filename=image_filename)
kapture.io.features.image_descriptors_to_file(
filepath=descriptor_filpath,
image_descriptors=opensfm_image_descriptors)
# register the file
kapture_descriptors.add(image_filename)
# import matches
kapture_matches = kapture.Matches()
opensfm_matches_suffix = '_matches.pkl.gz'
opensfm_matches_dirpath = path.join(opensfm_rootdir, 'matches')
if path.isdir(opensfm_matches_dirpath):
logger.info('importing matches ...')
opensfm_matches_file_list = (path.join(
dp, fn) for dp, _, fs in os.walk(opensfm_matches_dirpath)
for fn in fs)
opensfm_matches_file_list = (
filepath for filepath in opensfm_matches_file_list
if filepath.endswith(opensfm_matches_suffix))
for opensfm_matches_filename in tqdm(opensfm_matches_file_list,
disable=disable_tqdm):
image_filename_1 = path.relpath(
opensfm_matches_filename,
opensfm_matches_dirpath)[:-len(opensfm_matches_suffix)]
logger.debug(f'parsing mathes in {image_filename_1}')
with gzip.open(opensfm_matches_filename, 'rb') as f:
opensfm_matches = pickle.load(f)
for image_filename_2, opensfm_image_matches in opensfm_matches.items(
):
image_pair = (image_filename_1, image_filename_2)
# register the pair to kapture
kapture_matches.add(*image_pair)
# convert the bin file to kapture
kapture_matches_filepath = kapture.io.features.get_matches_fullpath(
image_filename_pair=image_pair,
kapture_dirpath=kapture_rootdir)
kapture_image_matches = np.hstack([
opensfm_image_matches.astype(np.float64),
# no macthes scoring = assume all to one
np.ones(shape=(opensfm_image_matches.shape[0], 1),
dtype=np.float64)
])
kapture.io.features.image_matches_to_file(
kapture_matches_filepath, kapture_image_matches)
# import 3-D points
if 'points' in opensfm_reconstruction:
logger.info('importing points 3-D')
opensfm_points = opensfm_reconstruction['points']
points_data = []
for point_id in sorted(opensfm_points):
point_data = opensfm_points[point_id]
point_data = point_data['coordinates'] + point_data['color']
points_data.append(point_data)
kapture_points = kapture.Points3d(points_data)
else:
kapture_points = None
# saving kapture csv files
logger.info('saving kapture files')
kapture_data = kapture.Kapture(sensors=kapture_sensors,
records_camera=kapture_images,
records_gnss=kapture_gnss,
trajectories=kapture_trajectories,
keypoints=kapture_keypoints,
descriptors=kapture_descriptors,
matches=kapture_matches,
points3d=kapture_points)
kapture.io.csv.kapture_to_dir(dirpath=kapture_rootdir,
kapture_data=kapture_data)
