def test_type_checking(self): self.assertRaises(TypeError, kapture.Points3d, (2, 3, 4)) points = kapture.Points3d() self.assertEqual(points.dtype.name, 'float64') data = list() data.append(list(range(1, 7))) points = kapture.Points3d(data) self.assertEqual(points.dtype.name, 'float64') rows = 18 points = np.array(range(0, rows * kapture.Points3d.XYZRGB)) points = np.reshape(points, (rows, kapture.Points3d.XYZRGB)) points = kapture.Points3d(points) self.assertEqual(points.dtype.name, 'float64') # test slicing val1 = points[:, 0:3] val2 = points[:] val3 = points[:, 0] val4 = points[0, :] val5 = points[0] val6 = points[0:2, :] val7 = points[0, 0] self.assertNotIsInstance(val1, kapture.Points3d) self.assertIsInstance(val2, kapture.Points3d) self.assertNotIsInstance(val3, kapture.Points3d) self.assertIsInstance(val4, kapture.Points3d) self.assertIsInstance(val5, kapture.Points3d) self.assertIsInstance(val6, kapture.Points3d) self.assertIsInstance(val7, float)
def test_from_numpy_array(self): data = np.array([[0] * 6]) points3d = kapture.Points3d(data) self.assertIsInstance(points3d, kapture.Points3d) self.assertEqual(points3d.shape, (1, 6)) self.assertTrue(points3d) data = np.arange(12).reshape((2, 6)) points3d = kapture.Points3d(data) self.assertIsInstance(points3d, kapture.Points3d) self.assertEqual(points3d.shape, (2, 6)) self.assertTrue(points3d) data = np.arange(16).reshape((2, 8)) points3d = kapture.Points3d(data[:, 0:6]) self.assertIsInstance(points3d, kapture.Points3d) self.assertEqual(points3d.shape, (2, 6)) self.assertTrue(points3d) data = list() data.append(list(range(1, 7))) points3d = kapture.Points3d(data) self.assertIsInstance(points3d, kapture.Points3d) self.assertEqual(points3d.shape, (1, 6)) self.assertTrue(np.all(np.isclose(points3d.as_array(), data))) data = np.vstack([data, data]) points3d = kapture.Points3d(data) self.assertIsInstance(points3d, kapture.Points3d) self.assertEqual(points3d.shape, (2, 6)) self.assertTrue(np.all(np.isclose(points3d.as_array(), data)))
def merge_points3d_and_observations( pts3d_obs: List[Tuple[Optional[kapture.Points3d], Optional[kapture.Observations]]] ) -> Tuple[kapture.Points3d, kapture.Observations]: """ Merge a list of points3d with their observations. :param pts3d_obs: list of points3d with observations to merge :return: merged points3d associated to observations """ assert len(pts3d_obs) > 0 merged_points3d = kapture.Points3d() merged_observations = kapture.Observations() point3d_offset = 0 for points3d, observations in pts3d_obs: if points3d is None: continue merged_points3d = kapture.Points3d( np.vstack([merged_points3d, points3d])) if observations is not None: for point3d_idx, (image_path, keypoint_idx) in kapture.flatten(observations): merged_observations.add(point3d_idx + point3d_offset, image_path, keypoint_idx) point3d_offset += merged_points3d.shape[0] return merged_points3d, merged_observations
def test_from_numpy_array_invalid(self): data = np.arange(16).reshape((8, 2)) with self.assertRaises(ValueError): kapture.Points3d(data) data = np.arange(16) with self.assertRaises(ValueError): kapture.Points3d(data)
def merge_points3d(points3d_list: List[Optional[kapture.Points3d]]) -> kapture.Points3d: """ Merge several points3d lists in one. :param points3d_list: list of points3d to merge :return: merged points3d """ assert len(points3d_list) > 0 merged_points3d = kapture.Points3d() for points3d in points3d_list: if points3d is None: continue merged_points3d = kapture.Points3d(np.vstack([merged_points3d, points3d])) return merged_points3d
def test_slices(self): rows = 18 np_points = np.array(range(0, rows * kapture.Points3d.XYZRGB)) np_points = np.reshape(np_points, (rows * 2, 3)) self.assertRaises(ValueError, kapture.Points3d, np_points) np_points = np.reshape(np_points, (rows, kapture.Points3d.XYZRGB)) # construct from numpy array points = kapture.Points3d(np_points) # construct from a points3d points = kapture.Points3d(points) self.assertEqual(points[0, 0], 0.0) self.assertEqual(points[17, 5], 107.0) np.testing.assert_array_equal(points.as_array(), np_points) np.testing.assert_array_equal(points[0, 0:3], np.array([0.0, 1.0, 2.0])) np.testing.assert_array_equal(points[0, 3:], np.array([3.0, 4.0, 5.0])) np.testing.assert_array_equal(points[0, :].as_array(), np.array([0.0, 1.0, 2.0, 3.0, 4.0, 5.0]))
def points3d_from_file(filepath: str) -> kapture.Points3d: """ Reads 3d points from CSV file. :param filepath: path to CSV file :return: the 3d points """ data = np.loadtxt(filepath, dtype=np.float, delimiter=',', comments='#') data = data.reshape((-1, 6)) # make sure of the shape, even if single line file. return kapture.Points3d(data)
def import_from_colmap_points3d_txt(colmap_points3d_filepath: str, image_names: Dict[int, str] = None, skip_observations: bool = False ) -> Tuple[kapture.Points3d, Optional[kapture.Observations]]: """ Imports the colmap file named "points3d.txt" containing both points 3D and observations. :param colmap_points3d_filepath: path to the colmap file named "points3d.txt" :param image_names: dict of image names matching the colmap image id to the kapture image name colmap_image_idx -> kapture_image_filename :param skip_observations: skip import of observations id true. :return: kapture 3D points and observations """ assert path.basename(colmap_points3d_filepath) == 'points3D.txt' points3d = [] observations = kapture.Observations() # colmap points3D.txt contains both points 3D and observations with open(colmap_points3d_filepath, 'r') as file: # in the reconstruction, spaces and commas can be used as separators lines = file.readlines() # eliminate comments lines = (line for line in lines if not line.startswith('#')) # split by space and or comma lines = ([float(value) for value in re.findall(colmap_reconstruction_split_pattern, values)] for values in lines) # split into an array of floats for index, values in enumerate(lines): points3d.append(values[1:4] + values[4:7]) if not skip_observations and len(values) > 8 and len(image_names) > 0: for image_idx, point_2d_idx in zip(values[8::2], values[9::2]): filename = image_names.get(int(image_idx), 'unknown') observations.add(index, filename, int(point_2d_idx)) points3d = kapture.Points3d(np.array(points3d)) if points3d else kapture.Points3d() observations = None if len(observations) == 0 else observations return points3d, observations
def test_points_to_files(self): data = np.array([ [0, 1, 0, 0, 0, 0], [1, 0, 0, 255, 0, 0], [0, 0, 1, 255, 0, 255], ]) points3d = kapture.Points3d(data) filepath = path.join(self._tempdir.name, 'points3d.txt') csv.points3d_to_file(filepath, points3d) with open(filepath, 'rt') as file: lines = file.readlines() self.assertEqual(4, len(lines)) self.assertTrue(lines[0].startswith('#')) first_line = [float(f) for f in lines[1].split(',')] self.assertEqual(6, len(first_line)) self.assertAlmostEqual([0.0, 1.0, 0.0, 0.0, 0.0, 0.0], first_line)
def test_empty(self): points = kapture.Points3d() self.assertIsInstance(points, kapture.Points3d) self.assertEqual(points.shape, (0, kapture.Points3d.XYZRGB)) self.assertRaises(IndexError, points.__getitem__, (0, 0)) self.assertFalse(points)
def import_bundler( bundler_path: str, image_list_path: str, image_dir_path: str, kapture_dir_path: str, ignore_trajectories: bool, add_reconstruction: bool, force_overwrite_existing: bool = False, images_import_method: TransferAction = TransferAction.skip) -> None: """ Imports bundler data and save them as kapture. :param bundler_path: path to the bundler model file :param image_list_path: path to the file containing the list of image names :param image_dir_path: input path to bundler image directory. :param kapture_dir_path: path to kapture top directory :param ignore_trajectories: if True, will not import the trajectories :param add_reconstruction: if True, will create 3D points and observations :param force_overwrite_existing: Silently overwrite kapture files if already exists. :param images_import_method: choose how to import actual image files. """ os.makedirs(kapture_dir_path, exist_ok=True) delete_existing_kapture_files(kapture_dir_path, force_erase=force_overwrite_existing) logger.info('loading all content...') # if there is a filter list, parse it with open(image_list_path) as file: file_content = file.readlines() # remove end line char and empty lines image_list = [line.rstrip() for line in file_content if line != '\n'] with open(bundler_path) as file: bundler_content = file.readlines() # remove end line char and empty lines bundler_content = [ line.rstrip() for line in bundler_content if line != '\n' ] assert bundler_content[0] == "# Bundle file v0.3" # <num_cameras> <num_points> line_1 = bundler_content[1].split() number_of_cameras = int(line_1[0]) number_of_points = int(line_1[1]) offset = 2 number_of_lines_per_camera = 5 # 1 camera + 3 rotation + 1 translation cameras = kapture.Sensors() images = kapture.RecordsCamera() trajectories = kapture.Trajectories() if not ignore_trajectories else None points3d = [] if add_reconstruction else None keypoints = kapture.Keypoints('sift', np.float32, 2) if add_reconstruction else None observations = kapture.Observations() if add_reconstruction else None image_mapping = [] # bundler camera_id -> (name, width, height) for i in range(0, number_of_cameras): start_index = i * number_of_lines_per_camera + offset file_name = image_list[i] # process camera info line_camera = bundler_content[start_index].split() focal_length = float(line_camera[0]) k1 = float(line_camera[1]) k2 = float(line_camera[2]) # lazy open with Image.open(path.join(image_dir_path, file_name)) as im: width, height = im.size image_mapping.append((file_name, width, height)) camera = kapture.Camera( MODEL, [width, height, focal_length, width / 2, height / 2, k1, k2]) camera_id = f'sensor{i}' cameras[camera_id] = camera # process extrinsics rotation_matrix = [[float(v) for v in line.split()] for line in bundler_content[start_index + 1:start_index + 4]] quaternion_wxyz = quaternion.from_rotation_matrix(rotation_matrix) translation = np.array( [float(v) for v in bundler_content[start_index + 4].split()]) pose = kapture.PoseTransform(quaternion_wxyz, translation) # The Bundler model uses a coordinate system that differs from the *computer vision camera # coordinate system*. More specifically, they use the camera coordinate system typically used # in *computer graphics*. In this camera coordinate system, the camera is looking down the # `-z`-axis, with the `x`-axis pointing to the right and the `y`-axis pointing upwards. # rotation Pi around the x axis to get the *computer vision camera # coordinate system* rotation_around_x = quaternion.quaternion(0.0, 1.0, 0.0, 0.0) transformation = kapture.PoseTransform(rotation_around_x, np.array([0, 0, 0])) images[(i, camera_id)] = file_name if trajectories is not None: # transformation.inverse() is equal to transformation (rotation around -Pi or Pi around X is the same) trajectories[(i, camera_id)] = kapture.PoseTransform.compose( [transformation, pose, transformation]) if points3d is not None and number_of_points > 0: assert keypoints is not None assert observations is not None offset += number_of_cameras * number_of_lines_per_camera number_of_lines_per_point = 3 # position color viewlist # (image_name, bundler_keypoint_id ) -> keypoint_id known_keypoints = {} local_keypoints = {} for i in range(0, number_of_points): start_index = i * number_of_lines_per_point + offset position = [float(v) for v in bundler_content[start_index].split()] # apply transformation position = [position[0], -position[1], -position[2]] color = [ float(v) for v in bundler_content[start_index + 1].split() ] # <view list>: length of the list + [<camera> <key> <x> <y>] # x, y origin is the center of the image view_list = bundler_content[start_index + 2].split() number_of_observations = int(view_list[0]) for j in range(number_of_observations): camera_id = int(view_list[1 + 4 * j + 0]) keypoint_id = int(view_list[1 + 4 * j + 1]) x = float(view_list[1 + 4 * j + 2]) y = float(view_list[1 + 4 * j + 3]) file_name, width, height = image_mapping[camera_id] # put (0,0) in upper left corner x += (width / 2) y += (height / 2) # init local_keypoints if needed if file_name not in local_keypoints: local_keypoints[file_name] = [] # do not add the same keypoint twice if (file_name, keypoint_id) not in known_keypoints: # in the kapture format, keypoint id is different. Note that it starts from 0 known_keypoints[(file_name, keypoint_id)] = len( local_keypoints[file_name]) local_keypoints[file_name].append([x, y]) keypoint_idx = known_keypoints[(file_name, keypoint_id)] observations.add(i, file_name, keypoint_idx) points3d.append(position + color) points3d = np.array(points3d) # finally, convert local_keypoints to np.ndarray and add them to the global keypoints variable keypoints = kapture.Keypoints('sift', np.float32, 2) for image_filename, keypoints_array in local_keypoints.items(): keypoints_np_array = np.array(keypoints_array).astype(np.float32) keypoints_out_path = kapture.io.features.get_keypoints_fullpath( kapture_dir_path, image_filename) kapture.io.features.image_keypoints_to_file( keypoints_out_path, keypoints_np_array) keypoints.add(image_filename) if points3d is not None: points3d = kapture.Points3d(points3d) # import (copy) image files. logger.info('import image files ...') filename_list = [f for _, _, f in kapture.flatten(images)] import_record_data_from_dir_auto(image_dir_path, kapture_dir_path, filename_list, images_import_method) # pack into kapture format imported_kapture = kapture.Kapture(sensors=cameras, records_camera=images, trajectories=trajectories, points3d=points3d, keypoints=keypoints, observations=observations) logger.info('writing imported data...') kapture_to_dir(kapture_dir_path, imported_kapture)
def add_frames(self, frames: List[Frame], points3d: List[Keypoint]): k = self.kapture if k.records_camera is None: k.records_camera = kt.RecordsCamera() if k.trajectories is None: k.trajectories = kt.Trajectories() if k.keypoints is None: k.keypoints = { self.default_kp_type: kt.Keypoints(self.default_kp_type, np.float32, 2) } if k.points3d is None: k.points3d = kt.Points3d() if k.observations is None: k.observations = kt.Observations() def check_kp(kp): return not kp.bad_qlt and kp.inlier_count > self.min_pt3d_obs and kp.inlier_count / kp.total_count > self.min_pt3d_ratio kp_ids, pts3d = zip(*[(kp.id, kp.pt3d) for kp in points3d if check_kp(kp)]) I = np.argsort(kp_ids) pt3d_ids = dict(zip(np.array(kp_ids)[I], np.arange(len(I)))) pt3d_arr = np.array(pts3d)[I, :] k.points3d = kt.Points3d( np.concatenate((pt3d_arr, np.ones_like(pt3d_arr) * 128), axis=1)) for f in frames: if not f.pose.post: continue id = f.frame_num img = f.orig_image img_file = os.path.join(self.default_cam[1], 'frame%06d.%s' % (id, self.img_format)) img_fullpath = get_record_fullpath(self.path, img_file) os.makedirs(os.path.dirname(img_fullpath), exist_ok=True) if not np.isclose(self.scale, 1.0): img = cv2.resize(img, None, fx=self.scale, fy=self.scale, interpolation=cv2.INTER_AREA) if self.img_format == self.IMG_FORMAT_PNG: cv2.imwrite(img_fullpath, img, (cv2.IMWRITE_PNG_COMPRESSION, 9)) elif self.img_format == self.IMG_FORMAT_JPG: cv2.imwrite(img_fullpath, img, (cv2.IMWRITE_JPEG_QUALITY, self.jpg_qlt)) else: assert False, 'Invalid image format: %s' % (self.img_format, ) record_id = (id, self.default_cam[0]) k.records_camera[record_id] = img_file pose = f.pose.post if 1 else (-f.pose.post) k.trajectories[record_id] = kt.PoseTransform( r=pose.quat.components, t=pose.loc) k.keypoints[self.default_kp_type].add(img_file) uvs = np.zeros((len(f.kps_uv), 2), np.float32) i = 0 for kp_id, uv in f.kps_uv.items(): if kp_id in pt3d_ids: k.observations.add(int(pt3d_ids[kp_id]), self.default_kp_type, img_file, i) uvs[i, :] = uv / f.img_sc * self.scale i += 1 image_keypoints_to_file( get_keypoints_fullpath(self.default_kp_type, self.path, img_file), uvs[:i, :])
def create_3D_model_from_depth_from_loaded_data( kdata: kapture.Kapture, input_path: str, tar_handlers: TarCollection, output_path: str, keypoints_type: Optional[str], depth_sensor_id: str, topk: int, method: Method, cellsizes: List[str], force: bool): """ Create 3D model from a kapture dataset that has registered depth data Assumes the kapture data is already loaded """ logger.info(f'create 3D model using depth data') if os.path.exists(output_path) and not force: print(f'outpath already exists, use --force to overwrite') return -1 if kdata.rigs is not None: assert kdata.trajectories is not None kapture.rigs_remove_inplace(kdata.trajectories, kdata.rigs) if keypoints_type is None: keypoints_type = try_get_only_key_from_collection(kdata.keypoints) assert keypoints_type is not None assert kdata.keypoints is not None assert keypoints_type in kdata.keypoints if method == Method.voxelgrid: vg = VoxelGrid(cellsizes) # add all 3D points to map that correspond to a keypoint logger.info('adding points from scan to kapture') points3d = [] observations = kapture.Observations() progress_bar = tqdm(total=len( list(kapture.flatten(kdata.records_camera, is_sorted=True))), disable=logger.level >= logging.CRITICAL) for timestamp, sensor_id, sensing_filepath in kapture.flatten( kdata.records_camera, is_sorted=True): logger.info( f'total 3d points: {len(points3d)}, processing {sensing_filepath}') # check if images have a pose if timestamp not in kdata.trajectories: logger.info('{} does not have a pose. skipping ...'.format( sensing_filepath)) continue # check if depth map exists depth_map_record = '' if timestamp in kdata.records_depth: if depth_sensor_id is None: depth_id = sensor_id + '_depth' else: depth_id = depth_sensor_id if depth_id in kdata.records_depth[timestamp]: depth_map_record = kdata.records_depth[timestamp][depth_id] depth_map_size = tuple( [int(x) for x in kdata.sensors[depth_id].camera_params[0:2]]) depth_path = get_depth_map_fullpath(input_path, depth_map_record) if not os.path.exists(depth_path): logger.info('no 3D data found for {}. skipping ...'.format( sensing_filepath)) continue depth_map = depth_map_from_file(depth_path, depth_map_size) img = Image.open(get_image_fullpath(input_path, sensing_filepath)).convert('RGB') assert img.size[0] == depth_map_size[0] assert img.size[1] == depth_map_size[1] kps_raw = load_keypoints(keypoints_type, input_path, sensing_filepath, kdata.keypoints[keypoints_type].dtype, kdata.keypoints[keypoints_type].dsize, tar_handlers) _, camera_sensor_C, camera_dist = get_camera_matrix_from_kapture( np.zeros((1, 0, 2), dtype=np.float64), kdata.sensors[sensor_id]) cv2_keypoints, depth_sensor_C, depth_dist = get_camera_matrix_from_kapture( kps_raw, kdata.sensors[depth_id]) assert np.isclose(depth_sensor_C, camera_sensor_C).all() assert np.isclose(depth_dist, camera_dist).all() if np.count_nonzero(camera_dist) > 0: epsilon = np.finfo(np.float64).eps stop_criteria = (cv2.TERM_CRITERIA_MAX_ITER + cv2.TERM_CRITERIA_EPS, 500, epsilon) undistorted_cv2_keypoints = cv2.undistortPointsIter( cv2_keypoints, camera_sensor_C, camera_dist, R=None, P=camera_sensor_C, criteria=stop_criteria) else: undistorted_cv2_keypoints = cv2_keypoints cv2_keypoints = cv2_keypoints.reshape((kps_raw.shape[0], 2)) undistorted_cv2_keypoints = undistorted_cv2_keypoints.reshape( (kps_raw.shape[0], 2)) points3d_img = [] rgb_img = [] kp_idxs = [] for idx_kp, kp in enumerate(cv2_keypoints[0:topk]): u = round(kp[0]) v = round(kp[1]) undist_kp = undistorted_cv2_keypoints[idx_kp] undist_u = round(undist_kp[0]) undist_v = round(undist_kp[1]) if u >= 0 and u < depth_map_size[ 0] and v >= 0 and v < depth_map_size[1]: if depth_map[v, u] == 0: continue pt3d = project_kp_to_3D(undist_u, undist_v, depth_map[v, u], depth_sensor_C[0, 2], depth_sensor_C[1, 2], depth_sensor_C[0, 0], depth_sensor_C[1, 1]) points3d_img.append(pt3d) rgb_img.append(img.getpixel((u, v))) kp_idxs.append(idx_kp) # transform to world coordinates (pt3d from a depth map is in camera coordinates) # we use sensor_id here because we assume that the image and the corresponding depthmap have the same pose # and sometimes, the pose might only be provided for the images cam_to_world = kdata.trajectories[timestamp][sensor_id].inverse() if len(points3d_img) == 0: continue points3d_img = cam_to_world.transform_points(np.array(points3d_img)) for idx_kp, pt3d, rgb in zip(kp_idxs, points3d_img, rgb_img): if not np.isnan(pt3d).any(): # apply transform (alignment) if method == Method.voxelgrid: assert vg is not None if not vg.exists(pt3d): # add 3D point points3d.append(list(pt3d) + list(rgb)) # add observation observations.add( len(points3d) - 1, keypoints_type, sensing_filepath, idx_kp) vg.add(pt3d, len(points3d) - 1, sensing_filepath) else: ret = vg.append(pt3d, sensing_filepath) if ret is not None: observations.add(ret[0], keypoints_type, sensing_filepath, idx_kp) elif method == Method.all: # add 3D point points3d.append(list(pt3d) + list(rgb)) # add observation observations.add( len(points3d) - 1, keypoints_type, sensing_filepath, idx_kp) # save_3Dpts_to_ply(points3d, os.path.join(output_path, 'map.ply')) progress_bar.update(1) progress_bar.close() kdata.points3d = kapture.Points3d(np.array(points3d)) kdata.observations = observations logger.info('saving ...') kapture_to_dir(output_path, kdata) # save_3Dpts_to_ply(points3d, os.path.join(output_path, 'map.ply')) logger.info('all done')
def import_opensfm( opensfm_root_dir: str, kapture_root_dir: str, force_overwrite_existing: bool = False, images_import_method: TransferAction = TransferAction.copy) -> None: """ Convert an openSfM structure to a kapture on disk. Also copy, move or link the images files if necessary. :param opensfm_root_dir: the openSfM top directory :param kapture_root_dir: top directory of kapture created :param force_overwrite_existing: if true, will remove existing kapture data without prompting the user :param images_import_method: action to apply on images: link, copy, move or do nothing. :return: the constructed kapture object """ disable_tqdm = logger.getEffectiveLevel() != logging.INFO # load reconstruction opensfm_reconstruction_filepath = path.join(opensfm_root_dir, 'reconstruction.json') with open(opensfm_reconstruction_filepath, 'rt') as f: opensfm_reconstruction = json.load(f) # remove the single list @ root opensfm_reconstruction = opensfm_reconstruction[0] # prepare space for output os.makedirs(kapture_root_dir, exist_ok=True) delete_existing_kapture_files(kapture_root_dir, force_erase=force_overwrite_existing) # import cameras kapture_sensors = kapture.Sensors() assert 'cameras' in opensfm_reconstruction # import cameras for osfm_camera_id, osfm_camera in opensfm_reconstruction['cameras'].items( ): camera = import_camera(osfm_camera, name=osfm_camera_id) kapture_sensors[osfm_camera_id] = camera # import shots logger.info('importing images and trajectories ...') kapture_images = kapture.RecordsCamera() kapture_trajectories = kapture.Trajectories() opensfm_image_dir_path = path.join(opensfm_root_dir, 'images') assert 'shots' in opensfm_reconstruction image_timestamps, image_sensors = {}, { } # used later to retrieve the timestamp of an image. for timestamp, (image_filename, shot) in enumerate( opensfm_reconstruction['shots'].items()): sensor_id = shot['camera'] image_timestamps[image_filename] = timestamp image_sensors[image_filename] = sensor_id # in OpenSfm, (sensor, timestamp) is not unique. rotation_vector = shot['rotation'] q = quaternion.from_rotation_vector(rotation_vector) translation = shot['translation'] # capture_time = shot['capture_time'] # may be invalid # gps_position = shot['gps_position'] kapture_images[timestamp, sensor_id] = image_filename kapture_trajectories[timestamp, sensor_id] = kapture.PoseTransform(r=q, t=translation) # copy image files filename_list = [f for _, _, f in kapture.flatten(kapture_images)] import_record_data_from_dir_auto( source_record_dirpath=opensfm_image_dir_path, destination_kapture_dirpath=kapture_root_dir, filename_list=filename_list, copy_strategy=images_import_method) # Imports Gnss kapture_gnss = _import_gnss(opensfm_root_dir, kapture_sensors, image_sensors, image_timestamps, disable_tqdm) # Imports descriptors, keypoints and matches kapture_descriptors, kapture_keypoints, kapture_matches = _import_features_and_matches( opensfm_root_dir, kapture_root_dir, disable_tqdm) # import 3-D points if 'points' in opensfm_reconstruction: logger.info('importing points 3-D') opensfm_points = opensfm_reconstruction['points'] points_data = [] for point_id in sorted(opensfm_points): point_data = opensfm_points[point_id] point_data = point_data['coordinates'] + point_data['color'] points_data.append(point_data) kapture_points = kapture.Points3d(points_data) else: kapture_points = None # saving kapture csv files logger.info('saving kapture files') kapture_data = kapture.Kapture(sensors=kapture_sensors, records_camera=kapture_images, records_gnss=kapture_gnss, trajectories=kapture_trajectories, keypoints=kapture_keypoints, descriptors=kapture_descriptors, matches=kapture_matches, points3d=kapture_points) kapture_to_dir(kapture_root_dir, kapture_data)
def import_nvm(nvm_file_path: str, nvm_images_path: str, kapture_path: str, filter_list_path: Optional[str], ignore_trajectories: bool, add_reconstruction: bool, force_overwrite_existing: bool = False, images_import_method: TransferAction = TransferAction.skip) -> None: """ Imports nvm data to kapture format. :param nvm_file_path: path to nvm file :param nvm_images_path: path to NVM images directory. :param kapture_path: path to kapture root directory. :param filter_list_path: path to the optional file containing a list of images to process :param ignore_trajectories: if True, will not create trajectories :param add_reconstruction: if True, will add observations, keypoints and 3D points. :param force_overwrite_existing: Silently overwrite kapture files if already exists. :param images_import_method: choose how to import actual image files. """ # TODO implement [optional calibration] # doc : http://ccwu.me/vsfm/doc.html#nvm os.makedirs(kapture_path, exist_ok=True) delete_existing_kapture_files(kapture_path, force_erase=force_overwrite_existing) logger.info('loading all content...') # if there is a filter list, parse it # keep it as Set[str] to easily find images if filter_list_path: with open(filter_list_path) as file: file_content = file.readlines() # remove end line char and empty lines filter_list = {line.rstrip() for line in file_content if line != '\n'} else: filter_list = None # now do the nvm with open(nvm_file_path) as file: nvm_content = file.readlines() # remove end line char and empty lines nvm_content = [line.rstrip() for line in nvm_content if line != '\n'] # only NVM_V3 is supported assert nvm_content[0] == "NVM_V3" # offset represents the line pointer offset = 1 # camera_id_offset keeps tracks of used camera_id in case of multiple reconstructed models camera_id_offset = 0 # point_id_offset keeps tracks of used point_id in case of multiple reconstructed models point_id_offset = 0 cameras = kapture.Sensors() images = kapture.RecordsCamera() trajectories = kapture.Trajectories() if not ignore_trajectories else None observations = kapture.Observations() if add_reconstruction else None if add_reconstruction else None keypoints = kapture.Keypoints('sift', np.float32, 2) if add_reconstruction else None points3d = [] if add_reconstruction else None # break if number of cameras == 0 or reached end of file while True: # <Model1> <Model2> ... # Each reconstructed <model> contains the following # <Number of cameras> <List of cameras> # <Number of 3D points> <List of points> # In practice, # <Number of cameras> # <List of cameras>, one per line # <Number of 3D points> # <List of points>, one per line number_of_cameras = int(nvm_content[offset]) offset += 1 if number_of_cameras == 0: # a line with <0> signify the end of models break logger.debug('importing model cameras...') # parse all cameras for current model image_idx_to_image_name = parse_cameras(number_of_cameras, nvm_content, offset, camera_id_offset, filter_list, nvm_images_path, cameras, images, trajectories) offset += number_of_cameras camera_id_offset += number_of_cameras # parse all points3d number_of_points = int(nvm_content[offset]) offset += 1 if points3d is not None and number_of_points > 0: assert keypoints is not None assert observations is not None logger.debug('importing model points...') parse_points3d(kapture_path, number_of_points, nvm_content, offset, point_id_offset, image_idx_to_image_name, filter_list, points3d, keypoints, observations) point_id_offset += number_of_points offset += number_of_points # reached end of file? if offset >= len(nvm_content): break # do not export values if none were found. if points3d is not None: points3d = kapture.Points3d(points3d) # import (copy) image files. logger.info('import image files ...') images_filenames = [f for _, _, f in kapture.flatten(images)] import_record_data_from_dir_auto(nvm_images_path, kapture_path, images_filenames, images_import_method) # pack into kapture format imported_kapture = kapture.Kapture(sensors=cameras, records_camera=images, trajectories=trajectories, points3d=points3d, keypoints=keypoints, observations=observations) logger.info('writing imported data...') kapture_to_dir(kapture_path, imported_kapture)
def import_opensfm( opensfm_rootdir: str, kapture_rootdir: str, force_overwrite_existing: bool = False, images_import_method: TransferAction = TransferAction.copy) -> None: disable_tqdm = logger.getEffectiveLevel() != logging.INFO # load reconstruction opensfm_reconstruction_filepath = path.join(opensfm_rootdir, 'reconstruction.json') with open(opensfm_reconstruction_filepath, 'rt') as f: opensfm_reconstruction = json.load(f) # remove the single list @ root opensfm_reconstruction = opensfm_reconstruction[0] # prepare space for output os.makedirs(kapture_rootdir, exist_ok=True) delete_existing_kapture_files(kapture_rootdir, force_erase=force_overwrite_existing) # import cameras kapture_sensors = kapture.Sensors() assert 'cameras' in opensfm_reconstruction # import cameras for osfm_camera_id, osfm_camera in opensfm_reconstruction['cameras'].items( ): camera = import_camera(osfm_camera, name=osfm_camera_id) kapture_sensors[osfm_camera_id] = camera # import shots logger.info('importing images and trajectories ...') kapture_images = kapture.RecordsCamera() kapture_trajectories = kapture.Trajectories() opensfm_image_dirpath = path.join(opensfm_rootdir, 'images') assert 'shots' in opensfm_reconstruction image_timestamps, image_sensors = {}, { } # used later to retrieve the timestamp of an image. for timestamp, (image_filename, shot) in enumerate( opensfm_reconstruction['shots'].items()): sensor_id = shot['camera'] image_timestamps[image_filename] = timestamp image_sensors[image_filename] = sensor_id # in OpenSfm, (sensor, timestamp) is not unique. rotation_vector = shot['rotation'] q = quaternion.from_rotation_vector(rotation_vector) translation = shot['translation'] # capture_time = shot['capture_time'] # may be invalid # gps_position = shot['gps_position'] kapture_images[timestamp, sensor_id] = image_filename kapture_trajectories[timestamp, sensor_id] = kapture.PoseTransform(r=q, t=translation) # copy image files filename_list = [f for _, _, f in kapture.flatten(kapture_images)] import_record_data_from_dir_auto( source_record_dirpath=opensfm_image_dirpath, destination_kapture_dirpath=kapture_rootdir, filename_list=filename_list, copy_strategy=images_import_method) # gps from pre-extracted exif, in exif/image_name.jpg.exif kapture_gnss = None opensfm_exif_dirpath = path.join(opensfm_rootdir, 'exif') opensfm_exif_suffix = '.exif' if path.isdir(opensfm_exif_dirpath): logger.info('importing GNSS from exif ...') camera_ids = set(image_sensors.values()) # add a gps sensor for each camera map_cam_to_gnss_sensor = { cam_id: 'GPS_' + cam_id for cam_id in camera_ids } for gnss_id in map_cam_to_gnss_sensor.values(): kapture_sensors[gnss_id] = kapture.Sensor( sensor_type='gnss', sensor_params=['EPSG:4326']) # build epsg_code for all cameras kapture_gnss = kapture.RecordsGnss() opensfm_exif_filepath_list = ( path.join(dirpath, filename) for dirpath, _, filename_list in os.walk(opensfm_exif_dirpath) for filename in filename_list if filename.endswith(opensfm_exif_suffix)) for opensfm_exif_filepath in tqdm(opensfm_exif_filepath_list, disable=disable_tqdm): image_filename = path.relpath( opensfm_exif_filepath, opensfm_exif_dirpath)[:-len(opensfm_exif_suffix)] image_timestamp = image_timestamps[image_filename] image_sensor_id = image_sensors[image_filename] gnss_timestamp = image_timestamp gnss_sensor_id = map_cam_to_gnss_sensor[image_sensor_id] with open(opensfm_exif_filepath, 'rt') as f: js_root = json.load(f) if 'gps' not in js_root: logger.warning(f'NO GPS data in "{opensfm_exif_filepath}"') continue gps_coords = { 'x': js_root['gps']['longitude'], 'y': js_root['gps']['latitude'], 'z': js_root['gps'].get('altitude', 0.0), 'dop': js_root['gps'].get('dop', 0), 'utc': 0, } logger.debug( f'found GPS data for ({gnss_timestamp}, {gnss_sensor_id}) in "{opensfm_exif_filepath}"' ) kapture_gnss[gnss_timestamp, gnss_sensor_id] = kapture.RecordGnss(**gps_coords) # import features (keypoints + descriptors) kapture_keypoints = None # kapture.Keypoints(type_name='opensfm', dsize=4, dtype=np.float64) kapture_descriptors = None # kapture.Descriptors(type_name='opensfm', dsize=128, dtype=np.uint8) opensfm_features_dirpath = path.join(opensfm_rootdir, 'features') opensfm_features_suffix = '.features.npz' if path.isdir(opensfm_features_dirpath): logger.info('importing keypoints and descriptors ...') opensfm_features_file_list = (path.join( dp, fn) for dp, _, fs in os.walk(opensfm_features_dirpath) for fn in fs) opensfm_features_file_list = ( filepath for filepath in opensfm_features_file_list if filepath.endswith(opensfm_features_suffix)) for opensfm_feature_filename in tqdm(opensfm_features_file_list, disable=disable_tqdm): image_filename = path.relpath( opensfm_feature_filename, opensfm_features_dirpath)[:-len(opensfm_features_suffix)] opensfm_image_features = np.load(opensfm_feature_filename) opensfm_image_keypoints = opensfm_image_features['points'] opensfm_image_descriptors = opensfm_image_features['descriptors'] logger.debug( f'parsing keypoints and descriptors in {opensfm_feature_filename}' ) if kapture_keypoints is None: # print(type(opensfm_image_keypoints.dtype)) # HAHOG = Hessian Affine feature point detector + HOG descriptor kapture_keypoints = kapture.Keypoints( type_name='HessianAffine', dsize=opensfm_image_keypoints.shape[1], dtype=opensfm_image_keypoints.dtype) if kapture_descriptors is None: kapture_descriptors = kapture.Descriptors( type_name='HOG', dsize=opensfm_image_descriptors.shape[1], dtype=opensfm_image_descriptors.dtype) # convert keypoints file keypoint_filpath = kapture.io.features.get_features_fullpath( data_type=kapture.Keypoints, kapture_dirpath=kapture_rootdir, image_filename=image_filename) kapture.io.features.image_keypoints_to_file( filepath=keypoint_filpath, image_keypoints=opensfm_image_keypoints) # register the file kapture_keypoints.add(image_filename) # convert descriptors file descriptor_filpath = kapture.io.features.get_features_fullpath( data_type=kapture.Descriptors, kapture_dirpath=kapture_rootdir, image_filename=image_filename) kapture.io.features.image_descriptors_to_file( filepath=descriptor_filpath, image_descriptors=opensfm_image_descriptors) # register the file kapture_descriptors.add(image_filename) # import matches kapture_matches = kapture.Matches() opensfm_matches_suffix = '_matches.pkl.gz' opensfm_matches_dirpath = path.join(opensfm_rootdir, 'matches') if path.isdir(opensfm_matches_dirpath): logger.info('importing matches ...') opensfm_matches_file_list = (path.join( dp, fn) for dp, _, fs in os.walk(opensfm_matches_dirpath) for fn in fs) opensfm_matches_file_list = ( filepath for filepath in opensfm_matches_file_list if filepath.endswith(opensfm_matches_suffix)) for opensfm_matches_filename in tqdm(opensfm_matches_file_list, disable=disable_tqdm): image_filename_1 = path.relpath( opensfm_matches_filename, opensfm_matches_dirpath)[:-len(opensfm_matches_suffix)] logger.debug(f'parsing mathes in {image_filename_1}') with gzip.open(opensfm_matches_filename, 'rb') as f: opensfm_matches = pickle.load(f) for image_filename_2, opensfm_image_matches in opensfm_matches.items( ): image_pair = (image_filename_1, image_filename_2) # register the pair to kapture kapture_matches.add(*image_pair) # convert the bin file to kapture kapture_matches_filepath = kapture.io.features.get_matches_fullpath( image_filename_pair=image_pair, kapture_dirpath=kapture_rootdir) kapture_image_matches = np.hstack([ opensfm_image_matches.astype(np.float64), # no macthes scoring = assume all to one np.ones(shape=(opensfm_image_matches.shape[0], 1), dtype=np.float64) ]) kapture.io.features.image_matches_to_file( kapture_matches_filepath, kapture_image_matches) # import 3-D points if 'points' in opensfm_reconstruction: logger.info('importing points 3-D') opensfm_points = opensfm_reconstruction['points'] points_data = [] for point_id in sorted(opensfm_points): point_data = opensfm_points[point_id] point_data = point_data['coordinates'] + point_data['color'] points_data.append(point_data) kapture_points = kapture.Points3d(points_data) else: kapture_points = None # saving kapture csv files logger.info('saving kapture files') kapture_data = kapture.Kapture(sensors=kapture_sensors, records_camera=kapture_images, records_gnss=kapture_gnss, trajectories=kapture_trajectories, keypoints=kapture_keypoints, descriptors=kapture_descriptors, matches=kapture_matches, points3d=kapture_points) kapture.io.csv.kapture_to_dir(dirpath=kapture_rootdir, kapture_data=kapture_data)