def __init__(self, rosbag_path: str, rigs: Optional[Rigs], sensors: Sensors, kapture_path: str, force_overwrite_existing: bool = False) -> None: """ :param rosbag_path: full path to the rosbag file :param rigs: rigs of the sensors :param sensors: sensors definition used for the capture :param kapture_path: full path to the top kapture directory to save :param force_overwrite_existing: silently overwrite kapture files if already exists """ if not path.isfile(rosbag_path): raise ValueError(f'Rosbag file {rosbag_path} does not exist') self._rosbag_path = rosbag_path self._rigs = rigs self._sensors = sensors self._kapture_path = kapture_path self.logger = logging.getLogger('rosbag') self.logger.info(f'Reading rosbag file {rosbag_path} and exporting as Kapture format into {kapture_path}') os.makedirs(kapture_path, exist_ok=True) kapture.io.structure.delete_existing_kapture_files(kapture_path, force_overwrite_existing) self._images_full_path = get_image_fullpath(kapture_path) # Keys = timestamp, values odometer poses self._last_poses = LimitedDictionary(20) self.images_info = list() # Of ImageInfo: type annotation is not supported in 3.6 self.poses_info = list() # Of PositionInfo
def test_export(self): temp_kapture_dirpath = path.join(self._tempdir.name, 'kapture') shutil.copytree(self._kapture_dirpath, temp_kapture_dirpath) kapture_data = kapture.io.csv.kapture_from_dir(temp_kapture_dirpath) images_filepaths = images_to_filepaths(kapture_data.records_camera, temp_kapture_dirpath) # make sure there is no EXIF in images for image_filepath in images_filepaths.values(): clear_exif(image_filepath) # insert gps to exif export_gps_to_exif(kapture_data=kapture_data, kapture_dirpath=temp_kapture_dirpath) rebuilt_records = kapture.RecordsGnss() for timestamp, cam_id, image_name in kapture.flatten( kapture_data.records_camera): image_filepath = get_image_fullpath(temp_kapture_dirpath, image_name) exif_data = read_exif(image_filepath) rebuilt_records[timestamp, 'GPS_' + cam_id] = convert_gps_to_kapture_record(exif_data) self.assertTrue( equal_records_gnss(kapture_data.records_gnss, rebuilt_records))
def test_import_openmvg(self) -> None: """ Test the import_openmvg function on a small JSON file while linking the images """ self.assertTrue(path.isdir(self._openmvg_sample_path)) self.assertTrue(path.exists(self._kapture_path), "Kapture directory exists") sfm_file = path.join(self._openmvg_sample_path, 'sfm_data_small.json') # on windows, without admin rights, fails with OSError: symbolic link privilege not held # see https://docs.python.org/3.6/library/os.html#os.symlink logger.info(f'Running on "{sys.platform}" which is {"" if self.isWindows else "not"} a Windows platform') file_operation = TransferAction.skip if self.isWindows else TransferAction.link_relative import_openmvg(sfm_file, self._kapture_path, file_operation, True) # test presence or absence of kapture files cameras_file_path = path.join(self._kapture_path, kcsv.CSV_FILENAMES[kapture.Sensors]) self.assertTrue(path.isfile(cameras_file_path), "Camera file written") rigs_file_path = path.join(self._kapture_path, kcsv.CSV_FILENAMES[kapture.Rigs]) self.assertFalse(path.isfile(rigs_file_path), "Rigs file should be missing") records_file_path = path.join(self._kapture_path, kcsv.CSV_FILENAMES[kapture.RecordsCamera]) self.assertTrue(path.isfile(records_file_path), "Camera Records file written") lidars_file_path = path.join(self._kapture_path, kcsv.CSV_FILENAMES[kapture.RecordsLidar]) self.assertFalse(path.isfile(lidars_file_path), "Lidar Records file should be missing") trajectories_file_path = path.join(self._kapture_path, kcsv.CSV_FILENAMES[kapture.Trajectories]) self.assertTrue(path.isfile(trajectories_file_path), "Trajectories file written") # Reload data and verify kapture_data = kcsv.kapture_from_dir(self._kapture_path) self._verify_data(kapture_data) if not self.isWindows: # Test images path all_records_camera = list(kapture.flatten(kapture_data.records_camera)) for _, _, name in all_records_camera: img_path = get_image_fullpath(self._kapture_path, name) self.assertTrue(path.islink(img_path), f"image link {img_path}")
def export_gps_to_exif(kapture_data: kapture.Kapture, kapture_dirpath: str, gps_id_to_cam_id: Optional[Dict[str, str]] = None): """ Export GPS from GNSS data in kapture, to image exifs. If the mapping from gnns_id to camera_id is not given, gnss_id is assumed to be to be gps_<cam_id>. """ # sanity check if None in [ kapture_data.sensors, kapture_data.records_camera, kapture_data.records_gnss ]: logger.warning('cannot export exif: missing data.') return # auto build GPS/camera map, based on the prefix rule. if gps_id_to_cam_id is None: camera_ids = { cam_id for cam_id, sensor in kapture_data.sensors.items() if sensor.sensor_type == 'camera' } gps_ids = { gps_id for gps_id, sensor in kapture_data.sensors.items() if sensor.sensor_type == 'gnss' and gps_id.startswith('GPS_') } gps_id_to_cam_id = { 'GPS_' + cam_id: cam_id for cam_id in camera_ids if 'GPS_' + cam_id in gps_ids } if len(gps_id_to_cam_id) != len(gps_ids): logger.warning('unable to map some GPS to a camera.') gps_records = ((timestamp, gps_id, gps_id_to_cam_id[gps_id], gnss_record) for timestamp, gps_id, gnss_record in kapture.flatten( kapture_data.records_gnss) if gps_id in gps_id_to_cam_id) for timestamp, gps_id, cam_id, gps_record in gps_records: if (timestamp, cam_id) not in kapture_data.records_camera: logger.warning( f'no image found corresponding to GPS record ({timestamp}, {cam_id})' ) else: image_name = kapture_data.records_camera[timestamp, cam_id] image_filepath = get_image_fullpath( kapture_dir_path=kapture_dirpath, image_filename=image_name) exif_data = gps_record_to_exif_dict(gps_record) update_exif(image_filepath, exif_data)
def test_maupertuis_import(self): kapture_data = import_colmap( self._temp_dirpath, self._database_filepath, self._reconstruction_path, self._images_filepath, force_overwrite_existing=True, images_import_strategy=TransferAction.copy, no_geometric_filtering=True) # check the numbers self.assertEqual(1, len(kapture_data.sensors)) self.assertEqual(4, len(kapture_data.trajectories)) self.assertEqual(4, len(kapture_data.records_camera)) self.assertIs(kapture_data.records_lidar, None) self.assertIs(kapture_data.records_wifi, None) self.assertIs(kapture_data.records_gnss, None) self.assertEqual(4, len(kapture_data.keypoints)) self.assertEqual(4, len(kapture_data.descriptors)) self.assertEqual(6, len(kapture_data.matches)) self.assertEqual(1039, len(kapture_data.points3d)) self.assertEqual(1039, len(kapture_data.observations)) # compare against golden kapture kapture_data_golden = kapture_from_dir(self._kapture_dirpath) # drop GPS, Wifi, Lidar kapture_data.records_lidar = None kapture_data.records_wifi = None kapture_data_golden.records_gnss = None kapture_data_golden.sensors = kapture.Sensors({ sensor_id: sensor for sensor_id, sensor in kapture_data_golden.sensors.items() if sensor.sensor_type == 'camera' }) # compare equivalence = kapture.algo.compare.equal_kapture( kapture_data, kapture_data_golden) self.assertTrue(equivalence) # Check images copy all_records_cameras = list(kapture.flatten( kapture_data.records_camera)) for _, _, name in all_records_cameras: image_path = get_image_fullpath(self._temp_dirpath, name) self.assertTrue(path.isfile(image_path), f"image link {image_path}")
def _import_images(input_json, image_action, kapture_images_path, openmvg_images_dir, root_path, device_identifiers, timestamp_for_pose): records_camera = kapture.RecordsCamera() if input_json.get(VIEWS): views = input_json[VIEWS] if image_action == TransferAction.root_link: # Do a unique images directory link # kapture/<records_dir>/openmvg_top_images_directory -> openmvg_root_path kapture_records_path = get_image_fullpath(kapture_images_path) os.makedirs(kapture_records_path, exist_ok=True) os.symlink(root_path, path.join(kapture_records_path, openmvg_images_dir)) logger.info(f'Importing {len(views)} images') # Progress bar only in debug or info level if image_action != TransferAction.skip and image_action != TransferAction.root_link \ and logger.getEffectiveLevel() <= logging.INFO: progress_bar = tqdm(total=len(views)) else: progress_bar = None for view in views: input_data = view[VALUE][PTR_WRAPPER][DATA] pose_id = input_data[ID_POSE] # All two values should be the same (?) if input_data[ID_VIEW]: timestamp = input_data[ID_VIEW] else: timestamp = view[KEY] device_id = str(input_data[ID_INTRINSIC] ) # device_id must be a string for kapture device_identifiers[pose_id] = device_id timestamp_for_pose[pose_id] = timestamp kapture_filename = _import_image_file(input_data, openmvg_images_dir, root_path, kapture_images_path, image_action) progress_bar and progress_bar.update(1) key = (timestamp, device_id) # tuple of int,str records_camera[key] = path_secure(kapture_filename) progress_bar and progress_bar.close() return records_camera
def import_openmvg_image_file(input_data, openmvg_images_dir, root_path, kapture_images_path, image_action) -> str: # Add the common openmvg images directory in front of the filename filename: str if input_data.get(JSON_KEY.LOCAL_PATH): filename = path.join(input_data[JSON_KEY.LOCAL_PATH], input_data[JSON_KEY.FILENAME]) else: filename = input_data[JSON_KEY.FILENAME] kapture_filename = path.join(openmvg_images_dir, filename) if image_action != TransferAction.skip and image_action != TransferAction.root_link: src_path: str if root_path: src_path = path.join(root_path, filename) else: src_path = filename dst_path = get_image_fullpath(kapture_images_path, kapture_filename) # Create destination directory if necessary dst_dir = path.dirname(dst_path) if not path.isdir(dst_dir): os.makedirs(dst_dir, exist_ok=True) # Check if already exist if path.exists(dst_path): os.unlink(dst_path) # Create file or link if image_action == TransferAction.copy: shutil.copy2(src_path, dst_path) elif image_action == TransferAction.move: shutil.move(src_path, dst_path) else: # Individual link if image_action == TransferAction.link_relative: # Compute relative path src_path = path.relpath(src_path, dst_dir) os.symlink(src_path, dst_path) # Symlink might crash on Windows if the user executing this code has no admin privilege return kapture_filename
# Creating CNN model model = D2Net( model_file=args.model_file, use_relu=args.use_relu, use_cuda=use_cuda ) keypoints_dtype = None if kdata.keypoints is None else kdata.keypoints.dtype descriptors_dtype = None if kdata.descriptors is None else kdata.descriptors.dtype keypoints_dsize = None if kdata.keypoints is None else kdata.keypoints.dsize descriptors_dsize = None if kdata.descriptors is None else kdata.descriptors.dsize # Process the files for image_name in tqdm(image_list, total=len(image_list)): img_path = get_image_fullpath(args.kapture_root, image_name) image = Image.open(img_path).convert('RGB') width, height = image.size resized_image = image resized_width = width resized_height = height max_edge = args.max_edge max_sum_edges = args.max_sum_edges if max(resized_width, resized_height) > max_edge: scale_multiplier = max_edge / max(resized_width, resized_height) resized_width = math.floor(resized_width * scale_multiplier) resized_height = math.floor(resized_height * scale_multiplier) resized_image = image.resize((resized_width, resized_height))
def extract_kapture_keypoints(kapture_root, config, output_dir='', overwrite=False): """ Extract r2d2 keypoints and descritors to the kapture format directly """ print('extract_kapture_keypoints...') kdata = kapture_from_dir(kapture_root, matches_pairsfile_path=None, skip_list= [kapture.GlobalFeatures, kapture.Matches, kapture.Points3d, kapture.Observations]) export_dir = output_dir if output_dir else kapture_root # root of output directory for features os.makedirs(export_dir, exist_ok=True) assert kdata.records_camera is not None image_list = [filename for _, _, filename in kapture.flatten(kdata.records_camera)] # resume extraction if some features exist try: # load existing features, if any kdata.keypoints = keypoints_from_dir(export_dir, None) kdata.descriptors = descriptors_from_dir(export_dir, None) if kdata.keypoints is not None and kdata.descriptors is not None and not overwrite: image_list = [name for name in image_list if name not in kdata.keypoints or name not in kdata.descriptors] except FileNotFoundError: pass except: logging.exception("Error with importing existing local features.") # clear features first if overwriting if overwrite: delete_existing_kapture_files(export_dir, True, only=[kapture.Descriptors, kapture.Keypoints]) if len(image_list) == 0: print('All features were already extracted') return else: print(f'Extracting r2d2 features for {len(image_list)} images') iscuda = common.torch_set_gpu([torch.cuda.is_available()]) # load the network... net = load_network(config['checkpoint']) if iscuda: net = net.cuda() # create the non-maxima detector detector = NonMaxSuppression( rel_thr = config['reliability_thr'], rep_thr = config['repeatability_thr']) keypoints_dtype = None if kdata.keypoints is None else kdata.keypoints.dtype descriptors_dtype = None if kdata.descriptors is None else kdata.descriptors.dtype keypoints_dsize = None if kdata.keypoints is None else kdata.keypoints.dsize descriptors_dsize = None if kdata.descriptors is None else kdata.descriptors.dsize for image_name in image_list: img_path = get_image_fullpath(kapture_root, image_name) if img_path.endswith('.txt'): images = open(img_path).read().splitlines() + images continue print(f"\nExtracting features for {img_path}") img = Image.open(img_path).convert('RGB') W, H = img.size img = norm_RGB(img)[None] if iscuda: img = img.cuda() # extract keypoints/descriptors for a single image xys, desc, scores = extract_multiscale(net, img, detector, scale_f = config['scale_f'], min_scale = config['min_scale'], max_scale = config['max_scale'], min_size = config['min_size'], max_size = config['max_size'], verbose = True) xys = xys.cpu().numpy() desc = desc.cpu().numpy() scores = scores.cpu().numpy() idxs = scores.argsort()[-config['top_k'] or None:] xys = xys[idxs] desc = desc[idxs] if keypoints_dtype is None or descriptors_dtype is None: keypoints_dtype = xys.dtype descriptors_dtype = desc.dtype keypoints_dsize = xys.shape[1] descriptors_dsize = desc.shape[1] kdata.keypoints = kapture.Keypoints('r2d2', keypoints_dtype, keypoints_dsize) kdata.descriptors = kapture.Descriptors('r2d2', descriptors_dtype, descriptors_dsize) keypoints_config_absolute_path = get_csv_fullpath(kapture.Keypoints, export_dir) descriptors_config_absolute_path = get_csv_fullpath(kapture.Descriptors, export_dir) keypoints_to_file(keypoints_config_absolute_path, kdata.keypoints) descriptors_to_file(descriptors_config_absolute_path, kdata.descriptors) else: assert kdata.keypoints.type_name == 'r2d2' assert kdata.descriptors.type_name == 'r2d2' assert kdata.keypoints.dtype == xys.dtype assert kdata.descriptors.dtype == desc.dtype assert kdata.keypoints.dsize == xys.shape[1] assert kdata.descriptors.dsize == desc.shape[1] keypoints_fullpath = get_keypoints_fullpath(export_dir, image_name) print(f"Saving {xys.shape[0]} keypoints to {keypoints_fullpath}") image_keypoints_to_file(keypoints_fullpath, xys) kdata.keypoints.add(image_name) descriptors_fullpath = get_descriptors_fullpath(export_dir, image_name) print(f"Saving {desc.shape[0]} descriptors to {descriptors_fullpath}") image_descriptors_to_file(descriptors_fullpath, desc) kdata.descriptors.add(image_name) if not keypoints_check_dir(kdata.keypoints, export_dir) or \ not descriptors_check_dir(kdata.descriptors, export_dir): print('local feature extraction ended successfully but not all files were saved')
def merge_remap(kapture_list: List[kapture.Kapture], skip_list: List[Type], data_paths: List[str], kapture_path: str, images_import_method: TransferAction) -> kapture.Kapture: """ Merge multiple kapture while keeping ids (sensor_id) identical in merged and inputs. :param kapture_list: list of kapture to merge. :param skip_list: input optional types to not merge. sensors and rigs are unskippable :param data_paths: list of path to root path directory in same order as mentioned in kapture_list. :param kapture_path: directory root path to the merged kapture. :return: """ merged_kapture = kapture.Kapture() # find new sensor ids / rig ids sensor_offset = 0 rigs_offset = 0 sensors_mapping = [] rigs_mapping = [] for every_kapture in kapture_list: if every_kapture.sensors is not None: sensors_mapping.append( get_sensors_mapping(every_kapture.sensors, sensor_offset)) sensor_offset += len(every_kapture.sensors) else: sensors_mapping.append({}) if every_kapture.rigs is not None: rigs_mapping.append( get_rigs_mapping(every_kapture.rigs, rigs_offset)) rigs_offset += len(every_kapture.rigs) else: rigs_mapping.append({}) # concatenate all sensors with the remapped ids new_sensors = merge_sensors( [a_kapture.sensors for a_kapture in kapture_list], sensors_mapping) if new_sensors: # if merge_sensors returned an empty object, keep merged_kapture.sensors to None merged_kapture.sensors = new_sensors # concatenate all rigs with the remapped ids new_rigs = merge_rigs([a_kapture.rigs for a_kapture in kapture_list], rigs_mapping, sensors_mapping) if new_rigs: # if merge_rigs returned an empty object, keep merged_kapture.rigs to None merged_kapture.rigs = new_rigs # all fields below can be skipped with skip_list # we do not assign the properties when the merge evaluate to false, we keep it as None if kapture.Trajectories not in skip_list: new_trajectories = merge_trajectories( [a_kapture.trajectories for a_kapture in kapture_list], rigs_mapping, sensors_mapping) if new_trajectories: merged_kapture.trajectories = new_trajectories if kapture.RecordsCamera not in skip_list: new_records_camera = merge_records_camera( [a_kapture.records_camera for a_kapture in kapture_list], sensors_mapping) if new_records_camera: merged_kapture.records_camera = new_records_camera merge_records_data( [[ image_name for _, _, image_name in kapture.flatten( every_kapture.records_camera) ] if every_kapture.records_camera is not None else [] for every_kapture in kapture_list], [ get_image_fullpath(data_path, image_filename=None) for data_path in data_paths ], kapture_path, images_import_method) if kapture.RecordsLidar not in skip_list: new_records_lidar = merge_records_lidar( [a_kapture.records_lidar for a_kapture in kapture_list], sensors_mapping) if new_records_lidar: merged_kapture.records_lidar = new_records_lidar if kapture.RecordsWifi not in skip_list: new_records_wifi = merge_records_wifi( [a_kapture.records_wifi for a_kapture in kapture_list], sensors_mapping) if new_records_wifi: merged_kapture.records_wifi = new_records_wifi if kapture.RecordsGnss not in skip_list: new_records_gnss = merge_records_gnss( [a_kapture.records_gnss for a_kapture in kapture_list], sensors_mapping) if new_records_gnss: merged_kapture.records_gnss = new_records_gnss # for the reconstruction, except points and observations, the files are copied with shutil.copy # if kapture_path evaluates to False, all copies will be skipped (but classes will be filled normally) if kapture.Keypoints not in skip_list: keypoints = [a_kapture.keypoints for a_kapture in kapture_list] keypoints_not_none = [k for k in keypoints if k is not None] if len(keypoints_not_none) > 0: new_keypoints = merge_keypoints(keypoints, data_paths, kapture_path) if new_keypoints: merged_kapture.keypoints = new_keypoints if kapture.Descriptors not in skip_list: descriptors = [a_kapture.descriptors for a_kapture in kapture_list] descriptors_not_none = [k for k in descriptors if k is not None] if len(descriptors_not_none) > 0: new_descriptors = merge_descriptors(descriptors, data_paths, kapture_path) if new_descriptors: merged_kapture.descriptors = new_descriptors if kapture.GlobalFeatures not in skip_list: global_features = [ a_kapture.global_features for a_kapture in kapture_list ] global_features_not_none = [ k for k in global_features if k is not None ] if len(global_features_not_none) > 0: new_global_features = merge_global_features( global_features, data_paths, kapture_path) if new_global_features: merged_kapture.global_features = new_global_features if kapture.Matches not in skip_list: matches = [a_kapture.matches for a_kapture in kapture_list] new_matches = merge_matches(matches, data_paths, kapture_path) if new_matches: merged_kapture.matches = new_matches if kapture.Points3d not in skip_list and kapture.Observations not in skip_list: points_and_obs = [(a_kapture.points3d, a_kapture.observations) for a_kapture in kapture_list] new_points, new_observations = merge_points3d_and_observations( points_and_obs) if new_points: merged_kapture.points3d = new_points if new_observations: merged_kapture.observations = new_observations elif kapture.Points3d not in skip_list: points = [a_kapture.points3d for a_kapture in kapture_list] new_points = merge_points3d(points) if new_points: merged_kapture.points3d = new_points return merged_kapture
def colmap_localize_sift(kapture_path: str, colmap_path: str, input_database_path: str, input_reconstruction_path: str, colmap_binary: str, colmap_use_cpu: bool, colmap_gpu_index: str, vocab_tree_path: str, image_registrator_options: List[str], skip_list: List[str], force: bool) -> None: """ Localize images on a colmap model using default SIFT features with the kapture data. :param kapture_path: path to the kapture to use :param colmap_path: path to the colmap build :param input_database_path: path to the map colmap.db :param input_database_path: path to the map colmap.db :param input_reconstruction_path: path to the map reconstruction folder :param colmap_use_cpu: to use cpu only (and ignore gpu) or to use also gpu :param colmap_gpu_index: gpu index for sift extractor and mapper :param vocab_tree_path: path to the colmap vocabulary tree file :param image_registrator_options: options for the image registrator :param skip_list: list of steps to skip :param force: Silently overwrite kapture files if already exists. """ os.makedirs(colmap_path, exist_ok=True) # Set fixed name for COLMAP database # Load input files first to make sure it is OK logger.info('loading kapture files...') kapture_data = kapture.io.csv.kapture_from_dir(kapture_path) if not (kapture_data.records_camera and kapture_data.sensors): raise ValueError('records_camera, sensors are mandatory') if kapture_data.trajectories: logger.warning("Input data contains trajectories: they will be ignored") kapture_data.trajectories.clear() else: kapture_data.trajectories = kapture.Trajectories() if not os.path.isfile(vocab_tree_path): raise ValueError(f'Vocabulary Tree file does not exist: {vocab_tree_path}') # COLMAP does not fully support rigs. if kapture_data.rigs is not None and kapture_data.trajectories is not None: # make sure, rigs are not used in trajectories. logger.info('remove rigs notation.') rigs_remove_inplace(kapture_data.trajectories, kapture_data.rigs) kapture_data.rigs.clear() # Prepare output # Set fixed name for COLMAP database colmap_db_path = path.join(colmap_path, 'colmap.db') image_list_path = path.join(colmap_path, 'images.list') reconstruction_path = path.join(colmap_path, "reconstruction") if 'delete_existing' not in skip_list: safe_remove_file(colmap_db_path, force) safe_remove_file(image_list_path, force) safe_remove_any_path(reconstruction_path, force) os.makedirs(reconstruction_path, exist_ok=True) # Copy colmap db to output if not os.path.exists(colmap_db_path): shutil.copy(input_database_path, colmap_db_path) # find correspondences between the colmap db and the kapture data images_all = {image_path: (ts, cam_id) for ts, shot in kapture_data.records_camera.items() for cam_id, image_path in shot.items()} colmap_db = COLMAPDatabase.connect(colmap_db_path) colmap_image_ids = database_extra.get_colmap_image_ids_from_db(colmap_db) colmap_cameras = database_extra.get_camera_ids_from_database(colmap_db) colmap_images = database_extra.get_images_from_database(colmap_db) colmap_db.close() # dict ( kapture_camera -> colmap_camera_id ) colmap_camera_ids = {images_all[image_path][1]: colmap_cam_id for image_path, colmap_cam_id in colmap_images if image_path in images_all} images_to_add = {image_path: value for image_path, value in images_all.items() if image_path not in colmap_image_ids} flatten_images_to_add = [(ts, kapture_cam_id, image_path) for image_path, (ts, kapture_cam_id) in images_to_add.items()] if 'feature_extract' not in skip_list: logger.info("Step 1: Feature extraction using colmap") with open(image_list_path, 'w') as fid: for image in images_to_add.keys(): fid.write(image + "\n") colmap_lib.run_feature_extractor( colmap_binary, colmap_use_cpu, colmap_gpu_index, colmap_db_path, get_image_fullpath(kapture_path), image_list_path ) if 'matches' not in skip_list: logger.info("Step 2: Compute matches with colmap") colmap_lib.run_vocab_tree_matcher( colmap_binary, colmap_use_cpu, colmap_gpu_index, colmap_db_path, vocab_tree_path, image_list_path ) if 'fix_db_cameras' not in skip_list: logger.info("Step 3: Replace colmap generated cameras with kapture cameras") colmap_db = COLMAPDatabase.connect(colmap_db_path) database_extra.foreign_keys_off(colmap_db) # remove colmap generated cameras after_feature_extraction_colmap_cameras = database_extra.get_camera_ids_from_database(colmap_db) colmap_cameras_to_remove = [cam_id for cam_id in after_feature_extraction_colmap_cameras if cam_id not in colmap_cameras] for cam_id in colmap_cameras_to_remove: database_extra.remove_camera(colmap_db, cam_id) # put the correct cameras and image extrinsic back into the database cameras_to_add = kapture.Sensors() for image_path, (ts, kapture_cam_id) in images_to_add.items(): if kapture_cam_id not in colmap_camera_ids: kapture_cam = kapture_data.sensors[kapture_cam_id] cameras_to_add[kapture_cam_id] = kapture_cam colmap_added_camera_ids = database_extra.add_cameras_to_database(cameras_to_add, colmap_db) colmap_camera_ids.update(colmap_added_camera_ids) database_extra.update_images_in_database_from_flatten( colmap_db, flatten_images_to_add, kapture_data.trajectories, colmap_camera_ids ) database_extra.foreign_keys_on(colmap_db) colmap_db.commit() colmap_db.close() if 'image_registrator' not in skip_list: logger.info("Step 4: Run image_registrator") # run image_registrator colmap_lib.run_image_registrator( colmap_binary, colmap_db_path, input_reconstruction_path, reconstruction_path, image_registrator_options ) # run model_converter if 'model_converter' not in skip_list: logger.info("Step 5: Export reconstruction results to txt") colmap_lib.run_model_converter( colmap_binary, reconstruction_path, reconstruction_path )
def colmap_build_sift_map(kapture_path: str, colmap_path: str, colmap_binary: str, colmap_use_cpu: bool, colmap_gpu_index: str, vocab_tree_path: str, point_triangulator_options: List[str], skip_list: List[str], force: bool) -> None: """ Build a colmap model using default SIFT features with the kapture data. :param kapture_path: path to the kapture to use :param colmap_path: path to the colmap build :param colmap_binary: path to the colmap executable :param colmap_use_cpu: to use cpu only (and ignore gpu) or to use also gpu :param colmap_gpu_index: gpu index for sift extractor and mapper :param vocab_tree_path: path to the colmap vocabulary tree file :param point_triangulator_options: options for the point triangulator :param skip_list: list of steps to skip :param force: Silently overwrite kapture files if already exists. """ os.makedirs(colmap_path, exist_ok=True) # Load input files first to make sure it is OK logger.info('loading kapture files...') kapture_data = kapture.io.csv.kapture_from_dir(kapture_path) if not (kapture_data.records_camera and kapture_data.sensors): raise ValueError('records_camera, sensors are mandatory') if not kapture_data.trajectories: logger.info('there are no trajectories, running mapper instead of point_triangulator') if not os.path.isfile(vocab_tree_path): raise ValueError(f'Vocabulary Tree file does not exist: {vocab_tree_path}') # COLMAP does not fully support rigs. if kapture_data.rigs is not None and kapture_data.trajectories is not None: # make sure, rigs are not used in trajectories. logger.info('remove rigs notation.') rigs_remove_inplace(kapture_data.trajectories, kapture_data.rigs) kapture_data.rigs.clear() # Set fixed name for COLMAP database colmap_db_path = path.join(colmap_path, 'colmap.db') image_list_path = path.join(colmap_path, 'images.list') reconstruction_path = path.join(colmap_path, "reconstruction") if 'delete_existing' not in skip_list: safe_remove_file(colmap_db_path, force) safe_remove_file(image_list_path, force) safe_remove_any_path(reconstruction_path, force) os.makedirs(reconstruction_path, exist_ok=True) if 'feature_extract' not in skip_list: logger.info("Step 1: Feature extraction using colmap") with open(image_list_path, 'w') as fid: for timestamp, sensor_id in sorted(kapture_data.records_camera.key_pairs()): fid.write(kapture_data.records_camera[timestamp][sensor_id] + "\n") colmap_lib.run_feature_extractor( colmap_binary, colmap_use_cpu, colmap_gpu_index, colmap_db_path, get_image_fullpath(kapture_path), image_list_path ) # Update cameras in COLMAP: # - use only one camera for all images taken with the same camera (update all camera IDs) # - import camera intrinsics # - import camera pose if 'update_db_cameras' not in skip_list: logger.info("Step 2: Populate COLMAP DB with cameras and poses") colmap_db = COLMAPDatabase.connect(colmap_db_path) database_extra.update_DB_cameras_and_poses(colmap_db, kapture_data) # close db before running colmap processes in order to avoid locks colmap_db.close() # Extract matches with COLMAP if 'matches' not in skip_list: logger.info("Step 3: Compute matches with colmap") colmap_lib.run_vocab_tree_matcher( colmap_binary, colmap_use_cpu, colmap_gpu_index, colmap_db_path, vocab_tree_path) if kapture_data.trajectories is not None: # Generate priors for reconstruction txt_path = path.join(colmap_path, "priors_for_reconstruction") os.makedirs(txt_path, exist_ok=True) if 'priors_for_reconstruction' not in skip_list: logger.info('Step 4: Exporting priors for reconstruction.') colmap_db = COLMAPDatabase.connect(colmap_db_path) database_extra.generate_priors_for_reconstruction(kapture_data, colmap_db, txt_path) colmap_db.close() # Point triangulator reconstruction_path = path.join(colmap_path, "reconstruction") os.makedirs(reconstruction_path, exist_ok=True) if 'triangulation' not in skip_list: logger.info("Step 5: Triangulation") colmap_lib.run_point_triangulator( colmap_binary, colmap_db_path, get_image_fullpath(kapture_path), txt_path, reconstruction_path, point_triangulator_options ) else: # mapper reconstruction_path = path.join(colmap_path, "reconstruction") os.makedirs(reconstruction_path, exist_ok=True) if 'triangulation' not in skip_list: logger.info("Step 5: Triangulation") colmap_lib.run_mapper( colmap_binary, colmap_db_path, get_image_fullpath(kapture_path), None, reconstruction_path, point_triangulator_options ) # use reconstruction 0 as main first_reconstruction = os.path.join(reconstruction_path, '0') files = os.listdir(first_reconstruction) for f in files: shutil.move(os.path.join(first_reconstruction, f), os.path.join(reconstruction_path, f)) shutil.rmtree(first_reconstruction) # run model_converter if 'model_converter' not in skip_list: logger.info("Step 6: Export reconstruction results to txt") colmap_lib.run_model_converter( colmap_binary, reconstruction_path, reconstruction_path )
def kapture_to_openmvg(kapture_data: kapture.Kapture, kapture_path: str, image_action: TransferAction, openmvg_path: str) -> Dict: """ Convert the kapture data into an openMVG dataset stored as a dictionary. The format is defined here: https://openmvg.readthedocs.io/en/latest/software/SfM/SfM_OutputFormat/ :param kapture_data: the kapture data :param kapture_path: top directory of the kapture data and the images :param image_action: action to apply on images: link, copy, move or do nothing. :param openmvg_path: top directory of the openmvg data and images :return: an SfM_data, the openmvg structure, stored as a dictionary ready to be serialized """ assert kapture_data.cameras is not None assert kapture_data.records_camera is not None cameras = kapture_data.cameras # Check we don't have other sensors defined extra_sensor_number = len(kapture_data.sensors)-len(cameras) if extra_sensor_number > 0: logger.warning(f'We will ignore {extra_sensor_number} sensors that are not camera') records_camera = kapture_data.records_camera all_records_camera = list(kapture.flatten(records_camera)) trajectories = kapture_data.trajectories # openmvg does not support rigs if kapture_data.rigs: logger.info('remove rigs notation.') rigs_remove_inplace(kapture_data.trajectories, kapture_data.rigs) kapture_data.rigs.clear() # Compute root path and camera used in records sub_root_path: str = '' image_dirs = {} used_cameras = {} for _, cam_id, name in all_records_camera: used_cameras[cam_id] = cam_id img_dir = path.dirname(name) image_dirs[img_dir] = img_dir if len(image_dirs) > 1: # Find if they share a top path image_dirs_list = list(image_dirs.keys()) sub_root_path = path.commonpath(image_dirs_list) elif len(image_dirs) == 1: sub_root_path = next(iter(image_dirs.keys())) if image_action == TransferAction.skip: root_path = kapture_path else: # We will create a new hierarchy of images root_path = openmvg_path root_path = os.path.abspath(path.join(root_path, sub_root_path)) if image_action == TransferAction.root_link: if not sub_root_path: # We can not link directly to the top destination openmvg directory # We need an additional level root_path = path.join(root_path, 'images') kapture_records_path = get_image_fullpath(kapture_path, sub_root_path) # Do a unique images directory link # openmvg_root_path -> kapture/<records_dir>/openmvg_top_images_directory # beware that the paths are reverted in the symlink call os.symlink(kapture_records_path, root_path) sfm_data = {SFM_DATA_VERSION: SFM_DATA_VERSION_NUMBER, ROOT_PATH: root_path} polymorphic_id_current = 1 ptr_wrapper_id_current = 1 polymorphic_id_types = {} intrinsics, polymorphic_id_current, ptr_wrapper_id_current = _export_cameras(cameras, used_cameras, polymorphic_id_types, polymorphic_id_current, ptr_wrapper_id_current) views, extrinsics = _export_images_and_poses(all_records_camera, cameras, trajectories, image_action, kapture_path, root_path, sub_root_path, polymorphic_id_types, polymorphic_id_current, ptr_wrapper_id_current) sfm_data[VIEWS] = views sfm_data[INTRINSICS] = intrinsics sfm_data[EXTRINSICS] = extrinsics sfm_data[STRUCTURE] = [] sfm_data[CONTROL_POINTS] = [] return sfm_data
def import_silda( silda_dir_path: str, destination_kapture_dir_path: str, fallback_cam_model: str = 'FOV', do_split_cams: bool = False, corpus: Optional[str] = None, replace_pose_rig: bool = False, force_overwrite_existing: bool = False, images_import_strategy: TransferAction = TransferAction.link_absolute ) -> None: """ Imports data from silda dataset. :param silda_dir_path: path to the silda top directory :param destination_kapture_dir_path: input path to kapture directory. :param fallback_cam_model: camera model to fallback when necessary :param do_split_cams: If true, re-organises and renames the image files to split apart cameras. :param corpus: the list of corpus to be imported, among 'mapping', 'query'. :param replace_pose_rig: if True, replaces poses of individual cameras with poses of the rig. :param force_overwrite_existing: if true, Silently overwrite kapture files if already exists. :param images_import_strategy: how to copy image files. """ # sanity check silda_dir_path = path_secure(path.abspath(silda_dir_path)) destination_kapture_dir_path = path_secure( path.abspath(destination_kapture_dir_path)) if TransferAction.root_link == images_import_strategy and do_split_cams: raise ValueError( 'impossible to only link images directory and applying split cam.') hide_progress_bars = logger.getEffectiveLevel() >= logging.INFO # prepare output directory kapture.io.structure.delete_existing_kapture_files( destination_kapture_dir_path, force_overwrite_existing) os.makedirs(destination_kapture_dir_path, exist_ok=True) # images ########################################################################################################### logger.info('Processing images ...') # silda-images # ... # ├── 1445_0.png # ├── 1445_1.png # ... silda_images_root_path = path.join(silda_dir_path, 'silda-images') # list all png files (its PNG in silda) using a generator. if corpus is not None: assert corpus in SILDA_CORPUS_SPLIT_FILENAMES # if corpus specified, filter by those which directory name match corpus. logger.debug(f'only importing {corpus} part.') corpus_file_path = path.join(silda_dir_path, SILDA_CORPUS_SPLIT_FILENAMES[corpus]) with open(corpus_file_path, 'rt') as corpus_file: corpus_filenames = corpus_file.readlines() image_filenames_original = sorted(filename.strip() for filename in corpus_filenames) else: image_filenames_original = sorted( filename for dir_path, sd, fs in os.walk(silda_images_root_path) for filename in fs if filename.endswith('.png')) image_filenames_kapture = [] snapshots = kapture.RecordsCamera() image_name_to_ids = {} # '1445_0.png' -> (1445, 0) for image_filename_original in tqdm(image_filenames_original, disable=hide_progress_bars): # retrieve info from image filename name_parts_match = SILDA_IMAGE_NAME_PATTERN.match( image_filename_original) assert name_parts_match is not None shot_info: Dict[str, Any] shot_info = name_parts_match.groupdict() shot_info['timestamp'] = int( shot_info['timestamp'] ) # To avoid warnings about type of the value # eg. file_info = {'filename': '1445_0.png', 'timestamp': 1445, 'cam_id': '0'} # create a path of the image into NLE dir if do_split_cams: # re-organise images with subfolders per corpus/camera/timestamp.png kapture_image_filename = path.join( shot_info['cam_id'], '{:04d}.png'.format(shot_info['timestamp'])) else: # keep the original file hierarchy kapture_image_filename = image_filename_original image_filenames_kapture.append(kapture_image_filename) snapshots[shot_info['timestamp'], shot_info['cam_id']] = kapture_image_filename image_name_to_ids[shot_info['filename']] = (shot_info['timestamp'], shot_info['cam_id']) assert len(image_filenames_kapture) == len(image_filenames_original) # intrinsics ####################################################################################################### cameras = _import_cameras(silda_dir_path, snapshots, fallback_cam_model) # extrinsics ####################################################################################################### trajectories = _import_trajectories(silda_dir_path, image_name_to_ids, hide_progress_bars) # rigs rigs = _make_rigs(replace_pose_rig, trajectories) # pack it all together kapture_data = kapture.Kapture(sensors=cameras, records_camera=snapshots, trajectories=trajectories, rigs=rigs) logger.info('saving to Kapture ...') kapture.io.csv.kapture_to_dir(destination_kapture_dir_path, kapture_data) # finally import images if images_import_strategy != TransferAction.skip: # importing image files logger.info(f'importing {len(image_filenames_original)} images ...') assert len(image_filenames_original) == len(image_filenames_kapture) image_file_paths_original = [ path.join(silda_images_root_path, image_filename_kapture) for image_filename_kapture in image_filenames_original ] image_file_paths_kapture = [ get_image_fullpath(destination_kapture_dir_path, image_filename_kapture) for image_filename_kapture in image_filenames_kapture ] transfer_files_from_dir(image_file_paths_original, image_file_paths_kapture, images_import_strategy) logger.info('done.')
def openmvg_to_kapture(input_json: Dict[str, Union[str, Dict]], kapture_images_path: str, image_action=TransferAction.skip) -> kapture.Kapture: """ Convert an openMVG structure to a kapture object. Also copy, move or link the images files if necessary. :param input_json: the openmvg JSON parsed as a dictionary :param kapture_images_path: top directory to create the kapture images tree :param image_action: action to apply on images: link, copy, move or do nothing. :return: the constructed kapture object """ polymorphic_id_to_value = {} root_path: str = '' if input_json[ROOT_PATH]: root_path = input_json[ROOT_PATH] elif image_action == TransferAction.skip: logger.warning("No root_path in input file") else: # It is needed to execute an action with the image file raise ValueError( f"Missing root_path to do image action '{image_action.name}'") openmvg_images_dir = path.basename(root_path) kapture_cameras = kapture.Sensors() if input_json.get(INTRINSICS): logger.info(f'Importing intrinsics') for sensor in input_json[INTRINSICS]: value = sensor[VALUE] if POLYMORPHIC_NAME in value: # new type name: store it for next instances polymorphic_id = value[POLYMORPHIC_ID] & GET_ID_MASK polymorphic_id_to_value[polymorphic_id] = value[ POLYMORPHIC_NAME] logger.debug("New camera_type: " + polymorphic_id_to_value[polymorphic_id]) else: if POLYMORPHIC_ID not in value: raise ValueError( f'{POLYMORPHIC_ID} is missing (intrinsics)') polymorphic_id = value[POLYMORPHIC_ID] if polymorphic_id not in polymorphic_id_to_value: raise ValueError(f'Unknown polymorphic_id {polymorphic_id}') camera_model = CameraModel(polymorphic_id_to_value[polymorphic_id]) camera_data = value[PTR_WRAPPER][DATA] if camera_model == CameraModel.pinhole: # w, h, f, cx, cy camera = kapture.Camera(kapture.CameraType.SIMPLE_PINHOLE, [ int(camera_data[WIDTH]), int(camera_data[HEIGHT]), camera_data[FOCAL_LENGTH], camera_data[PRINCIPAL_POINT][0], camera_data[PRINCIPAL_POINT][1], ]) elif camera_model == CameraModel.pinhole_radial_k1: # w, h, f, cx, cy, k camera = kapture.Camera(kapture.CameraType.SIMPLE_RADIAL, [ int(camera_data[WIDTH]), int(camera_data[HEIGHT]), camera_data[FOCAL_LENGTH], camera_data[PRINCIPAL_POINT][0], camera_data[PRINCIPAL_POINT][1], camera_data[DISTO_K1][0] ]) elif camera_model == CameraModel.pinhole_radial_k3: # w, h, f, cx, cy, k1, k2, k3 camera = kapture.Camera(kapture.CameraType.RADIAL, [ int(camera_data[WIDTH]), int(camera_data[HEIGHT]), camera_data[FOCAL_LENGTH], camera_data[PRINCIPAL_POINT][0], camera_data[PRINCIPAL_POINT][1], camera_data[DISTO_K3][0], camera_data[DISTO_K3][1] ]) # camera_data["disto_k3"][2] ignored: radial model has two distortion param, while openMVG's has three elif camera_model == CameraModel.pinhole_brown_t2: # w, h, f, cx, cy, k1, k2, k3, t1, t2 if float(camera_data[DISTO_T2][2]) != 0: # if k3 not null, use FULL_OPENCV, otherwise OPENCV # w, h, fx, fy, cx, cy, k1, k2, p1, p2, k3, k4, k5, k6 value0 = camera_data[VALUE0] disto_t2 = camera_data[DISTO_T2] camera = kapture.Camera(kapture.CameraType.FULL_OPENCV, [ int(value0[WIDTH]), int(value0[HEIGHT]), value0[FOCAL_LENGTH], value0[FOCAL_LENGTH], value0[PRINCIPAL_POINT][0], value0[PRINCIPAL_POINT][1], disto_t2[0], disto_t2[1], disto_t2[3], disto_t2[4], disto_t2[2], 0, 0, 0 ]) else: # w, h, fx, fy, cx, cy, k1, k2, p1, p2 value0 = camera_data[VALUE0] disto_t2 = camera_data[DISTO_T2] camera = kapture.Camera(kapture.CameraType.OPENCV, [ int(value0[WIDTH]), int(value0[HEIGHT]), value0[FOCAL_LENGTH], value0[FOCAL_LENGTH], value0[PRINCIPAL_POINT][0], value0[PRINCIPAL_POINT][1], disto_t2[0], disto_t2[1], disto_t2[3], disto_t2[4] ]) elif camera_model == CameraModel.fisheye: logger.warning( "OpenMVG fisheye models are not compatible with OpenCV." " Using SIMPLE_RADIAL_FISHEYE and forcing distortion to 0") # w, h, f, cx, cy, k value0 = camera_data[VALUE0] camera = kapture.Camera( kapture.CameraType.SIMPLE_RADIAL_FISHEYE, [ int(value0[WIDTH]), int(value0[HEIGHT]), value0[FOCAL_LENGTH], value0[PRINCIPAL_POINT][0], value0[PRINCIPAL_POINT][1], 0 ]) else: raise ValueError(f'Camera model {camera_model} not supported') kapture_cameras[str(sensor[KEY])] = camera device_identifiers = {int: str} # Pose id -> device id timestamp_for_pose = {int: int} # Pose id -> timestamp records_camera = kapture.RecordsCamera() if input_json.get(VIEWS): views = input_json[VIEWS] if image_action == TransferAction.root_link: # Do a unique images directory link # kapture/<records_dir>/openmvg_top_images_directory -> openmvg_root_path kapture_records_path = get_image_fullpath(kapture_images_path) os.makedirs(kapture_records_path, exist_ok=True) os.symlink(root_path, path.join(kapture_records_path, openmvg_images_dir)) logger.info(f'Importing {len(views)} images') # Progress bar only in debug or info level if image_action != TransferAction.skip and image_action != TransferAction.root_link\ and logger.getEffectiveLevel() <= logging.INFO: progress_bar = tqdm(total=len(views)) else: progress_bar = None for view in views: input_data = view[VALUE][PTR_WRAPPER][DATA] pose_id = input_data[ID_POSE] # All two values should be the same (?) if input_data[ID_VIEW]: timestamp = input_data[ID_VIEW] else: timestamp = view[KEY] device_id = str(input_data[ID_INTRINSIC] ) # device_id must be a string for kapture device_identifiers[pose_id] = device_id timestamp_for_pose[pose_id] = timestamp filename: str if input_data.get(LOCAL_PATH): filename = path.join(input_data[LOCAL_PATH], input_data[FILENAME]) else: filename = input_data[FILENAME] if root_path: src_path = path.join(root_path, filename) else: src_path = filename # Add the common openmvg images directory in front of the filename kapture_filename = path.join(openmvg_images_dir, filename) if image_action != TransferAction.skip and image_action != TransferAction.root_link: dst_path = get_image_fullpath(kapture_images_path, kapture_filename) # Create destination directory if necessary dst_dir = path.dirname(dst_path) if not path.isdir(dst_dir): os.makedirs(dst_dir, exist_ok=True) # Check if already exist if path.exists(dst_path): os.unlink(dst_path) # Create file or link if image_action == TransferAction.copy: shutil.copy2(src_path, dst_path) elif image_action == TransferAction.move: shutil.move(src_path, dst_path) else: # Individual link if image_action == TransferAction.link_relative: # Compute relative path src_path = path.relpath(src_path, dst_dir) os.symlink(src_path, dst_path) # This might crash on Windows if the user executing this code has no admin privilege progress_bar and progress_bar.update(1) key = (timestamp, device_id) # tuple of int,str records_camera[key] = path_secure(kapture_filename) progress_bar and progress_bar.close() trajectories = kapture.Trajectories() if input_json.get(EXTRINSICS): extrinsics = input_json[EXTRINSICS] logger.info(f'Importing {len(extrinsics)} extrinsics -> trajectories') for pose in extrinsics: pose_id = pose[KEY] center = pose[VALUE][CENTER] rotation = pose[VALUE][ROTATION] kap_translation = -1 * np.matmul(rotation, center) kap_pose = kapture.PoseTransform( quaternion.from_rotation_matrix(rotation), kap_translation) timestamp = timestamp_for_pose.get(pose_id) if timestamp is None: logger.warning(f'Missing timestamp for extrinsic {pose_id}') continue device_id = device_identifiers.get(pose_id) if device_id is None: logger.warning(f'Missing device for extrinsic {pose_id}') continue trajectories[(timestamp, device_id)] = kap_pose # tuple of int,str -> 6D pose kapture_data = kapture.Kapture(sensors=kapture_cameras, records_camera=records_camera, trajectories=trajectories) return kapture_data
def _export_images_and_poses(all_records_camera, cameras, trajectories, image_action, kapture_path, root_path, sub_root_path, polymorphic_id_types, polymorphic_id_current, ptr_wrapper_id_current) -> Tuple[List, List]: views = [] extrinsics = [] global_timestamp = 0 # process all images for timestamp, cam_id, kapture_name in all_records_camera: local_path = path.dirname(path.relpath(kapture_name, sub_root_path)) filename = path.basename(kapture_name) if image_action != TransferAction.skip and image_action != TransferAction.root_link: # Process the image action src_path = get_image_fullpath(kapture_path, kapture_name) dst_path = path.join(root_path, local_path, filename) dst_dir = path.dirname(dst_path) if not path.isdir(dst_dir): os.makedirs(dst_dir, exist_ok=True) # Check if already exist if path.exists(dst_path): os.unlink(dst_path) # Create file or link if image_action == TransferAction.copy: shutil.copy2(src_path, dst_path) elif image_action == TransferAction.move: shutil.move(src_path, dst_path) else: # Link if image_action == TransferAction.link_relative: # Compute relative path src_path = path.relpath(src_path, dst_dir) os.symlink(src_path, dst_path) camera_params = cameras[cam_id].camera_params view_data = {LOCAL_PATH: local_path, FILENAME: filename, WIDTH: int(camera_params[0]), HEIGHT: int(camera_params[1]), ID_VIEW: global_timestamp, ID_INTRINSIC: cam_id, ID_POSE: global_timestamp} view = {} # retrieve image pose from trajectories if timestamp not in trajectories: view[POLYMORPHIC_ID] = VIEW_SPECIAL_POLYMORPHIC_ID else: # there is a pose for that timestamp # The poses are stored both as priors (in the 'views' table) and as known poses (in the 'extrinsics' table) assert cam_id in trajectories[timestamp] if VIEW_PRIORS not in polymorphic_id_types: # if this is the first time view_priors is encountered # set the first bit of polymorphic_id_current to 1 view[POLYMORPHIC_ID] = polymorphic_id_current | NEW_ID_MASK view[POLYMORPHIC_NAME] = VIEW_PRIORS polymorphic_id_types[VIEW_PRIORS] = polymorphic_id_current polymorphic_id_current += 1 else: view[POLYMORPHIC_ID] = polymorphic_id_types[VIEW_PRIORS] pose_tr = trajectories[timestamp].get(cam_id) prior_q = pose_tr.r prior_t = pose_tr.inverse().t_raw pose_data = {CENTER: prior_t, ROTATION: quaternion.as_rotation_matrix(prior_q).tolist()} view_data[USE_POSE_CENTER_PRIOR] = True view_data[CENTER_WEIGHT] = [1.0, 1.0, 1.0] view_data[CENTER] = prior_t view_data[USE_POSE_ROTATION_PRIOR] = True view_data[ROTATION_WEIGHT] = 1.0 view_data[ROTATION] = pose_data[ROTATION] extrinsics.append({KEY: global_timestamp, VALUE: pose_data}) # it is assumed that this view is only encountered once # set the first bit of ptr_wrapper_id_current to 1 view_wrapper = {ID: ptr_wrapper_id_current | NEW_ID_MASK, DATA: view_data} ptr_wrapper_id_current += 1 view[PTR_WRAPPER] = view_wrapper views.append({KEY: global_timestamp, VALUE: view}) global_timestamp += 1 return views, extrinsics
def kapture_to_openmvg(kapture_data: kapture.Kapture, kapture_path: str, image_action: TransferAction, openmvg_path: str) -> Dict: """ Convert the kapture data into an openMVG dataset stored as a dictionary. The format is defined here: https://openmvg.readthedocs.io/en/latest/software/SfM/SfM_OutputFormat/ :param kapture_data: the kapture data :param kapture_path: top directory of the kapture data and the images :param image_action: action to apply on images: link, copy, move or do nothing. :param openmvg_path: top directory of the openmvg data and images :return: an SfM_data, the openmvg structure, stored as a dictionary ready to be serialized """ assert kapture_data.cameras is not None assert kapture_data.records_camera is not None cameras = kapture_data.cameras # Check we don't have other sensors defined extra_sensor_number = len(kapture_data.sensors)-len(cameras) if extra_sensor_number > 0: logger.warning(f'We will ignore {extra_sensor_number} sensors that are not camera') records_camera = kapture_data.records_camera all_records_camera = list(kapture.flatten(records_camera)) trajectories = kapture_data.trajectories # openmvg does not support rigs if kapture_data.rigs: logger.info('remove rigs notation.') rigs_remove_inplace(kapture_data.trajectories, kapture_data.rigs) kapture_data.rigs.clear() # Compute root path and camera used in records sub_root_path: str = '' image_dirs = {} used_cameras = {} for _, cam_id, name in all_records_camera: used_cameras[cam_id] = cam_id img_dir = path.dirname(name) image_dirs[img_dir] = img_dir if len(image_dirs) > 1: # Find if they share a top path image_dirs_list = list(image_dirs.keys()) sub_root_path = path.commonpath(image_dirs_list) elif len(image_dirs) == 1: sub_root_path = next(iter(image_dirs.keys())) if image_action == TransferAction.skip: root_path = kapture_path else: # We will create a new hierarchy of images root_path = openmvg_path root_path = os.path.abspath(path.join(root_path, sub_root_path)) if image_action == TransferAction.root_link: if not sub_root_path: # We can not link directly to the top destination openmvg directory # We need an additional level root_path = path.join(root_path, 'images') kapture_records_path = get_image_fullpath(kapture_path, sub_root_path) # Do a unique images directory link # openmvg_root_path -> kapture/<records_dir>/openmvg_top_images_directory # beware that the paths are reverted in the symlink call os.symlink(kapture_records_path, root_path) sfm_data = {SFM_DATA_VERSION: SFM_DATA_VERSION_NUMBER, ROOT_PATH: root_path} views = [] intrinsics = [] extrinsics = [] polymorphic_id_current = 1 ptr_wrapper_id_current = 1 polymorphic_id_types = {} # process all cameras for cam_id, camera in cameras.items(): # Ignore not used cameras if not used_cameras.get(cam_id): logger.warning(f'Skipping camera definition {cam_id} {camera.name} without recorded images.') continue cam_type = camera.camera_type camera_params = camera.camera_params if cam_type == kapture.CameraType.SIMPLE_PINHOLE: # w, h, f, cx, cy model_used = CameraModel.pinhole data = _get_intrinsic_pinhole(camera_params) elif cam_type == kapture.CameraType.PINHOLE: # w, h, f, cx, cy model_used = CameraModel.pinhole faked_params = [camera_params[0], camera_params[1], # width height (camera_params[2] + camera_params[3])/2, # fx+fy/2 as f camera_params[4], camera_params[5]] # cx cy data = _get_intrinsic_pinhole(faked_params) elif cam_type == kapture.CameraType.SIMPLE_RADIAL: # w, h, f, cx, cy, k model_used = CameraModel.pinhole_radial_k1 data = _get_intrinsic_pinhole_radial_k1(camera_params) elif cam_type == kapture.CameraType.RADIAL: # w, h, f, cx, cy, k1, k2, k3 model_used = CameraModel.pinhole_radial_k3 faked_params = [camera_params[0], camera_params[1], # width height camera_params[2], # f camera_params[3], camera_params[4], # cx cy camera_params[5], camera_params[6], 0 # k1, k2, k3 ] data = _get_intrinsic_pinhole_radial_k3(faked_params) elif cam_type == kapture.CameraType.FULL_OPENCV or cam_type == kapture.CameraType.OPENCV: # w, h, f, cx, cy, k1, k2, k3, t1, t2 model_used = CameraModel.pinhole_brown_t2 k3 = camera_params[10] if len(camera_params) > 10 else 0 faked_params = [camera_params[0], camera_params[1], # width height (camera_params[2] + camera_params[3])/2, # fx+fy/2 as f camera_params[4], camera_params[5], # cx cy camera_params[6], camera_params[7], k3, # k1, k2, k3 camera_params[8], camera_params[9] # p1, p2 (=t1, t2) ] data = _get_intrinsic_pinhole_brown_t2(faked_params) elif cam_type == kapture.CameraType.OPENCV_FISHEYE: logger.warning('OpenCV fisheye model is not compatible with OpenMVG. Forcing distortion to 0') # w, h, f, cx, cy, k1, k2, k3, k4 model_used = CameraModel.fisheye faked_params = [camera_params[0], camera_params[1], # width height (camera_params[2] + camera_params[3]) / 2, # fx+fy/2 as f camera_params[4], camera_params[5], # cx cy 0, 0, # k1, k2 0, 0 # k3, k4 ] data = _get_intrinsic_fisheye(faked_params) elif cam_type == kapture.CameraType.RADIAL_FISHEYE or cam_type == kapture.CameraType.SIMPLE_RADIAL_FISHEYE: logger.warning('OpenCV fisheye model is not compatible with OpenMVG. Forcing distortion to 0') # w, h, f, cx, cy, k1, k2, k3, k4 model_used = CameraModel.fisheye faked_params = [camera_params[0], camera_params[1], # width height camera_params[2], # f camera_params[3], camera_params[4], # cx cy 0, 0, # k1, k2 0, 0 # k3, k4 ] data = _get_intrinsic_fisheye(faked_params) elif cam_type == kapture.CameraType.UNKNOWN_CAMERA: logger.info(f'Camera {cam_id}: Unknown camera model, using simple radial') # Choose simple radial model, to allow openMVG to determine distortion param # w, h, f, cx, cy, k model_used = CameraModel.pinhole_radial_k1 faked_params = [camera_params[0], camera_params[1], # width height max(camera_params[0], camera_params[1])*DEFAULT_FOCAL_LENGTH_FACTOR, # max(w,h)*1.2 as f int(camera_params[0]/2), int(camera_params[1]/2), # cx cy 0.0] # k1 data = _get_intrinsic_pinhole_radial_k1(faked_params) else: raise ValueError(f'Camera model {cam_type.value} not supported') intrinsic = {} if model_used not in polymorphic_id_types: # if this is the first time model_used is encountered # set the first bit of polymorphic_id_current to 1 intrinsic[POLYMORPHIC_ID] = polymorphic_id_current | NEW_ID_MASK intrinsic[POLYMORPHIC_NAME] = model_used.name polymorphic_id_types[model_used] = polymorphic_id_current polymorphic_id_current += 1 else: intrinsic[POLYMORPHIC_ID] = polymorphic_id_types[model_used] # it is assumed that this camera is only encountered once # set the first bit of ptr_wrapper_id_current to 1 data_wrapper = {ID: ptr_wrapper_id_current | NEW_ID_MASK, DATA: data} ptr_wrapper_id_current += 1 intrinsic[PTR_WRAPPER] = data_wrapper intrinsics.append({KEY: cam_id, VALUE: intrinsic}) global_timestamp = 0 # process all images for timestamp, cam_id, kapture_name in all_records_camera: local_path = path.dirname(path.relpath(kapture_name, sub_root_path)) filename = path.basename(kapture_name) if image_action != TransferAction.skip and image_action != TransferAction.root_link: # Process the image action src_path = get_image_fullpath(kapture_path, kapture_name) dst_path = path.join(root_path, local_path, filename) dst_dir = path.dirname(dst_path) if not path.isdir(dst_dir): os.makedirs(dst_dir, exist_ok=True) # Check if already exist if path.exists(dst_path): os.unlink(dst_path) # Create file or link if image_action == TransferAction.copy: shutil.copy2(src_path, dst_path) elif image_action == TransferAction.move: shutil.move(src_path, dst_path) else: # Link if image_action == TransferAction.link_relative: # Compute relative path src_path = path.relpath(src_path, dst_dir) os.symlink(src_path, dst_path) camera_params = cameras[cam_id].camera_params view_data = {LOCAL_PATH: local_path, FILENAME: filename, WIDTH: int(camera_params[0]), HEIGHT: int(camera_params[1]), ID_VIEW: global_timestamp, ID_INTRINSIC: cam_id, ID_POSE: global_timestamp} view = {} # retrieve image pose from trajectories if timestamp not in trajectories: view[POLYMORPHIC_ID] = VIEW_SPECIAL_POLYMORPHIC_ID else: # there is a pose for that timestamp # The poses are stored both as priors (in the 'views' table) and as known poses (in the 'extrinsics' table) assert cam_id in trajectories[timestamp] if VIEW_PRIORS not in polymorphic_id_types: # if this is the first time view_priors is encountered # set the first bit of polymorphic_id_current to 1 view[POLYMORPHIC_ID] = polymorphic_id_current | NEW_ID_MASK view[POLYMORPHIC_NAME] = VIEW_PRIORS polymorphic_id_types[VIEW_PRIORS] = polymorphic_id_current polymorphic_id_current += 1 else: view[POLYMORPHIC_ID] = polymorphic_id_types[VIEW_PRIORS] pose_tr = trajectories[timestamp].get(cam_id) prior_q = pose_tr.r prior_t = pose_tr.inverse().t_raw pose_data = {CENTER: prior_t, ROTATION: quaternion.as_rotation_matrix(prior_q).tolist()} view_data[USE_POSE_CENTER_PRIOR] = True view_data[CENTER_WEIGHT] = [1.0, 1.0, 1.0] view_data[CENTER] = prior_t view_data[USE_POSE_ROTATION_PRIOR] = True view_data[ROTATION_WEIGHT] = 1.0 view_data[ROTATION] = pose_data[ROTATION] extrinsics.append({KEY: global_timestamp, VALUE: pose_data}) # it is assumed that this view is only encountered once # set the first bit of ptr_wrapper_id_current to 1 view_wrapper = {ID: ptr_wrapper_id_current | NEW_ID_MASK, DATA: view_data} ptr_wrapper_id_current += 1 view[PTR_WRAPPER] = view_wrapper views.append({KEY: global_timestamp, VALUE: view}) global_timestamp += 1 sfm_data[VIEWS] = views sfm_data[INTRINSICS] = intrinsics sfm_data[EXTRINSICS] = extrinsics sfm_data[STRUCTURE] = [] sfm_data[CONTROL_POINTS] = [] return sfm_data
def extract_kapture_global_features(kapture_root_path: str, net, global_features_type: str, trfs, pooling='mean', gemp=3, whiten=None, threads=8, batch_size=16): """ Extract features from trained model (network) on a given dataset. """ print(f'loading {kapture_root_path}') with get_all_tar_handlers(kapture_root_path, mode={ kapture.Keypoints: 'r', kapture.Descriptors: 'r', kapture.GlobalFeatures: 'a', kapture.Matches: 'r' }) as tar_handlers: kdata = kapture_from_dir(kapture_root_path, None, skip_list=[ kapture.Keypoints, kapture.Descriptors, kapture.Matches, kapture.Points3d, kapture.Observations ], tar_handlers=tar_handlers) root = get_image_fullpath(kapture_root_path, image_filename=None) assert kdata.records_camera is not None imgs = [ image_name for _, _, image_name in kapture.flatten(kdata.records_camera) ] if kdata.global_features is None: kdata.global_features = {} if global_features_type in kdata.global_features: imgs = [ image_name for image_name in imgs if image_name not in kdata.global_features[global_features_type] ] if len(imgs) == 0: print('All global features are already extracted') return dataset = ImageList(img_list_path=None, root=root, imgs=imgs) print(f'\nEvaluation on {dataset}') # extract DB feats bdescs = [] trfs_list = [trfs] if isinstance(trfs, str) else trfs for trfs in trfs_list: kw = dict(iscuda=net.iscuda, threads=threads, batch_size=batch_size, same_size='Pad' in trfs or 'Crop' in trfs) bdescs.append( extract_image_features(dataset, trfs, net, desc="DB", **kw)) # pool from multiple transforms (scales) bdescs = tonumpy(F.normalize(pool(bdescs, pooling, gemp), p=2, dim=1)) if whiten is not None: bdescs = common.whiten_features(bdescs, net.pca, **whiten) print('writing extracted global features') os.umask(0o002) gfeat_dtype = bdescs.dtype gfeat_dsize = bdescs.shape[1] if global_features_type not in kdata.global_features: kdata.global_features[ global_features_type] = kapture.GlobalFeatures( 'dirtorch', gfeat_dtype, gfeat_dsize, 'L2') global_features_config_absolute_path = get_feature_csv_fullpath( kapture.GlobalFeatures, global_features_type, kapture_root_path) global_features_to_file( global_features_config_absolute_path, kdata.global_features[global_features_type]) else: assert kdata.global_features[ global_features_type].dtype == gfeat_dtype assert kdata.global_features[ global_features_type].dsize == gfeat_dsize assert kdata.global_features[ global_features_type].metric_type == 'L2' for i in tqdm.tqdm(range(dataset.nimg)): image_name = dataset.get_key(i) global_feature_fullpath = get_global_features_fullpath( global_features_type, kapture_root_path, image_name, tar_handlers) gfeat_i = bdescs[i, :] assert gfeat_i.shape == (gfeat_dsize, ) image_global_features_to_file(global_feature_fullpath, gfeat_i) kdata.global_features[global_features_type].add(image_name) del gfeat_i del bdescs if not global_features_check_dir( kdata.global_features[global_features_type], global_features_type, kapture_root_path, tar_handlers): print( 'global feature extraction ended successfully but not all files were saved' ) else: print('Features extracted.')
def merge_keep_ids( kapture_list: List[ kapture. Kapture], # noqa: C901: function a bit long but not too complex skip_list: List[Type], data_paths: List[str], kapture_path: str, images_import_method: TransferAction) -> kapture.Kapture: """ Merge multiple kapture while keeping ids (sensor_id) identical in merged and inputs. :param kapture_list: list of kapture to merge. :param skip_list: optional types not to merge. sensors and rigs are unskippable :param data_paths: list of path to root path directory in same order as mentioned in kapture_list. :param kapture_path: directory root path to the merged kapture. :param images_import_method: method to transfer image files :return: merged kapture """ merged_kapture = kapture.Kapture() # get the union of all sensors new_sensors = merge_sensors( [every_kapture.sensors for every_kapture in kapture_list]) # if merge_sensors returned an empty object, keep merged_kapture.sensors to None merged_kapture.sensors = get_new_if_not_empty(new_sensors, merged_kapture.sensors) # get the union of all rigs new_rigs = merge_rigs( [every_kapture.rigs for every_kapture in kapture_list]) # if merge_rigs returned an empty object, keep merged_kapture.rigs to None merged_kapture.rigs = get_new_if_not_empty(new_rigs, merged_kapture.rigs) # all fields below can be skipped with skip_list # we do not assign the properties when the merge evaluate to false, we keep it as None if kapture.Trajectories not in skip_list: new_trajectories = merge_trajectories( [every_kapture.trajectories for every_kapture in kapture_list]) merged_kapture.trajectories = get_new_if_not_empty( new_trajectories, merged_kapture.trajectories) if kapture.RecordsCamera not in skip_list: new_records_camera = merge_records_camera( [every_kapture.records_camera for every_kapture in kapture_list]) merged_kapture.records_camera = get_new_if_not_empty( new_records_camera, merged_kapture.records_camera) merge_records_data( [[ image_name for _, _, image_name in kapture.flatten( every_kapture.records_camera) ] if every_kapture.records_camera is not None else [] for every_kapture in kapture_list], [ get_image_fullpath(data_path, image_filename=None) for data_path in data_paths ], kapture_path, images_import_method) if kapture.RecordsLidar not in skip_list: new_records_lidar = merge_records_lidar( [every_kapture.records_lidar for every_kapture in kapture_list]) merged_kapture.records_lidar = get_new_if_not_empty( new_records_lidar, merged_kapture.records_lidar) if kapture.RecordsWifi not in skip_list: new_records_wifi = merge_records_wifi( [every_kapture.records_wifi for every_kapture in kapture_list]) merged_kapture.records_wifi = get_new_if_not_empty( new_records_wifi, merged_kapture.records_wifi) if kapture.RecordsBluetooth not in skip_list: new_records_bluetooth = merge_records_bluetooth([ every_kapture.records_bluetooth for every_kapture in kapture_list ]) merged_kapture.records_bluetooth = get_new_if_not_empty( new_records_bluetooth, merged_kapture.records_bluetooth) if kapture.RecordsGnss not in skip_list: new_records_gnss = merge_records_gnss( [every_kapture.records_gnss for every_kapture in kapture_list]) merged_kapture.records_gnss = get_new_if_not_empty( new_records_gnss, merged_kapture.records_gnss) if kapture.RecordsAccelerometer not in skip_list: new_records_accelerometer = merge_records_accelerometer([ every_kapture.records_accelerometer for every_kapture in kapture_list ]) merged_kapture.records_accelerometer = get_new_if_not_empty( new_records_accelerometer, merged_kapture.records_accelerometer) if kapture.RecordsGyroscope not in skip_list: new_records_gyroscope = merge_records_gyroscope([ every_kapture.records_gyroscope for every_kapture in kapture_list ]) merged_kapture.records_gyroscope = get_new_if_not_empty( new_records_gyroscope, merged_kapture.records_gyroscope) if kapture.RecordsMagnetic not in skip_list: new_records_magnetic = merge_records_magnetic( [every_kapture.records_magnetic for every_kapture in kapture_list]) merged_kapture.records_magnetic = get_new_if_not_empty( new_records_magnetic, merged_kapture.records_magnetic) # for the reconstruction, except points and observations, the files are copied with shutil.copy # if kapture_path evaluates to False, all copies will be skipped (but classes will be filled normally) if kapture.Keypoints not in skip_list: keypoints = [every_kapture.keypoints for every_kapture in kapture_list] keypoints_not_none = [k for k in keypoints if k is not None] if len(keypoints_not_none) > 0: new_keypoints = merge_keypoints(keypoints, data_paths, kapture_path) merged_kapture.keypoints = get_new_if_not_empty( new_keypoints, merged_kapture.keypoints) if kapture.Descriptors not in skip_list: descriptors = [ every_kapture.descriptors for every_kapture in kapture_list ] descriptors_not_none = [k for k in descriptors if k is not None] if len(descriptors_not_none) > 0: new_descriptors = merge_descriptors(descriptors, data_paths, kapture_path) merged_kapture.descriptors = get_new_if_not_empty( new_descriptors, merged_kapture.descriptors) if kapture.GlobalFeatures not in skip_list: global_features = [ every_kapture.global_features for every_kapture in kapture_list ] global_features_not_none = [ k for k in global_features if k is not None ] if len(global_features_not_none) > 0: new_global_features = merge_global_features( global_features, data_paths, kapture_path) merged_kapture.global_features = get_new_if_not_empty( new_global_features, merged_kapture.global_features) if kapture.Matches not in skip_list: matches = [every_kapture.matches for every_kapture in kapture_list] new_matches = merge_matches(matches, data_paths, kapture_path) merged_kapture.matches = get_new_if_not_empty(new_matches, merged_kapture.matches) if kapture.Points3d not in skip_list and kapture.Observations not in skip_list: points_and_obs = [(every_kapture.points3d, every_kapture.observations) for every_kapture in kapture_list] new_points, new_observations = merge_points3d_and_observations( points_and_obs) merged_kapture.points3d = get_new_if_not_empty(new_points, merged_kapture.points3d) merged_kapture.observations = get_new_if_not_empty( new_observations, merged_kapture.observations) elif kapture.Points3d not in skip_list: points = [every_kapture.points3d for every_kapture in kapture_list] new_points = merge_points3d(points) merged_kapture.points3d = get_new_if_not_empty(new_points, merged_kapture.points3d) return merged_kapture
def extract_kapture_keypoints(args): """ Extract r2d2 keypoints and descritors to the kapture format directly """ print('extract_kapture_keypoints...') kdata = kapture_from_dir(args.kapture_root, matches_pairsfile_path=None, skip_list=[ kapture.GlobalFeatures, kapture.Matches, kapture.Points3d, kapture.Observations ]) assert kdata.records_camera is not None image_list = [ filename for _, _, filename in kapture.flatten(kdata.records_camera) ] if kdata.keypoints is not None and kdata.descriptors is not None: image_list = [ name for name in image_list if name not in kdata.keypoints or name not in kdata.descriptors ] if len(image_list) == 0: print('All features were already extracted') return else: print(f'Extracting r2d2 features for {len(image_list)} images') iscuda = common.torch_set_gpu(args.gpu) # load the network... net = load_network(args.model) if iscuda: net = net.cuda() # create the non-maxima detector detector = NonMaxSuppression(rel_thr=args.reliability_thr, rep_thr=args.repeatability_thr) keypoints_dtype = None if kdata.keypoints is None else kdata.keypoints.dtype descriptors_dtype = None if kdata.descriptors is None else kdata.descriptors.dtype keypoints_dsize = None if kdata.keypoints is None else kdata.keypoints.dsize descriptors_dsize = None if kdata.descriptors is None else kdata.descriptors.dsize for image_name in image_list: img_path = get_image_fullpath(args.kapture_root, image_name) print(f"\nExtracting features for {img_path}") img = Image.open(img_path).convert('RGB') W, H = img.size img = norm_RGB(img)[None] if iscuda: img = img.cuda() # extract keypoints/descriptors for a single image xys, desc, scores = extract_multiscale(net, img, detector, scale_f=args.scale_f, min_scale=args.min_scale, max_scale=args.max_scale, min_size=args.min_size, max_size=args.max_size, verbose=True) xys = xys.cpu().numpy() desc = desc.cpu().numpy() scores = scores.cpu().numpy() idxs = scores.argsort()[-args.top_k or None:] xys = xys[idxs] desc = desc[idxs] if keypoints_dtype is None or descriptors_dtype is None: keypoints_dtype = xys.dtype descriptors_dtype = desc.dtype keypoints_dsize = xys.shape[1] descriptors_dsize = desc.shape[1] kdata.keypoints = kapture.Keypoints('r2d2', keypoints_dtype, keypoints_dsize) kdata.descriptors = kapture.Descriptors('r2d2', descriptors_dtype, descriptors_dsize) keypoints_config_absolute_path = get_csv_fullpath( kapture.Keypoints, args.kapture_root) descriptors_config_absolute_path = get_csv_fullpath( kapture.Descriptors, args.kapture_root) keypoints_to_file(keypoints_config_absolute_path, kdata.keypoints) descriptors_to_file(descriptors_config_absolute_path, kdata.descriptors) else: assert kdata.keypoints.type_name == 'r2d2' assert kdata.descriptors.type_name == 'r2d2' assert kdata.keypoints.dtype == xys.dtype assert kdata.descriptors.dtype == desc.dtype assert kdata.keypoints.dsize == xys.shape[1] assert kdata.descriptors.dsize == desc.shape[1] keypoints_fullpath = get_keypoints_fullpath(args.kapture_root, image_name) print(f"Saving {xys.shape[0]} keypoints to {keypoints_fullpath}") image_keypoints_to_file(keypoints_fullpath, xys) kdata.keypoints.add(image_name) descriptors_fullpath = get_descriptors_fullpath( args.kapture_root, image_name) print(f"Saving {desc.shape[0]} descriptors to {descriptors_fullpath}") image_descriptors_to_file(descriptors_fullpath, desc) kdata.descriptors.add(image_name) if not keypoints_check_dir(kdata.keypoints, args.kapture_root) or \ not descriptors_check_dir(kdata.descriptors, args.kapture_root): print( 'local feature extraction ended successfully but not all files were saved' )
def create_3D_model_from_depth_from_loaded_data( kdata: kapture.Kapture, input_path: str, tar_handlers: TarCollection, output_path: str, keypoints_type: Optional[str], depth_sensor_id: str, topk: int, method: Method, cellsizes: List[str], force: bool): """ Create 3D model from a kapture dataset that has registered depth data Assumes the kapture data is already loaded """ logger.info(f'create 3D model using depth data') if os.path.exists(output_path) and not force: print(f'outpath already exists, use --force to overwrite') return -1 if kdata.rigs is not None: assert kdata.trajectories is not None kapture.rigs_remove_inplace(kdata.trajectories, kdata.rigs) if keypoints_type is None: keypoints_type = try_get_only_key_from_collection(kdata.keypoints) assert keypoints_type is not None assert kdata.keypoints is not None assert keypoints_type in kdata.keypoints if method == Method.voxelgrid: vg = VoxelGrid(cellsizes) # add all 3D points to map that correspond to a keypoint logger.info('adding points from scan to kapture') points3d = [] observations = kapture.Observations() progress_bar = tqdm(total=len( list(kapture.flatten(kdata.records_camera, is_sorted=True))), disable=logger.level >= logging.CRITICAL) for timestamp, sensor_id, sensing_filepath in kapture.flatten( kdata.records_camera, is_sorted=True): logger.info( f'total 3d points: {len(points3d)}, processing {sensing_filepath}') # check if images have a pose if timestamp not in kdata.trajectories: logger.info('{} does not have a pose. skipping ...'.format( sensing_filepath)) continue # check if depth map exists depth_map_record = '' if timestamp in kdata.records_depth: if depth_sensor_id is None: depth_id = sensor_id + '_depth' else: depth_id = depth_sensor_id if depth_id in kdata.records_depth[timestamp]: depth_map_record = kdata.records_depth[timestamp][depth_id] depth_map_size = tuple( [int(x) for x in kdata.sensors[depth_id].camera_params[0:2]]) depth_path = get_depth_map_fullpath(input_path, depth_map_record) if not os.path.exists(depth_path): logger.info('no 3D data found for {}. skipping ...'.format( sensing_filepath)) continue depth_map = depth_map_from_file(depth_path, depth_map_size) img = Image.open(get_image_fullpath(input_path, sensing_filepath)).convert('RGB') assert img.size[0] == depth_map_size[0] assert img.size[1] == depth_map_size[1] kps_raw = load_keypoints(keypoints_type, input_path, sensing_filepath, kdata.keypoints[keypoints_type].dtype, kdata.keypoints[keypoints_type].dsize, tar_handlers) _, camera_sensor_C, camera_dist = get_camera_matrix_from_kapture( np.zeros((1, 0, 2), dtype=np.float64), kdata.sensors[sensor_id]) cv2_keypoints, depth_sensor_C, depth_dist = get_camera_matrix_from_kapture( kps_raw, kdata.sensors[depth_id]) assert np.isclose(depth_sensor_C, camera_sensor_C).all() assert np.isclose(depth_dist, camera_dist).all() if np.count_nonzero(camera_dist) > 0: epsilon = np.finfo(np.float64).eps stop_criteria = (cv2.TERM_CRITERIA_MAX_ITER + cv2.TERM_CRITERIA_EPS, 500, epsilon) undistorted_cv2_keypoints = cv2.undistortPointsIter( cv2_keypoints, camera_sensor_C, camera_dist, R=None, P=camera_sensor_C, criteria=stop_criteria) else: undistorted_cv2_keypoints = cv2_keypoints cv2_keypoints = cv2_keypoints.reshape((kps_raw.shape[0], 2)) undistorted_cv2_keypoints = undistorted_cv2_keypoints.reshape( (kps_raw.shape[0], 2)) points3d_img = [] rgb_img = [] kp_idxs = [] for idx_kp, kp in enumerate(cv2_keypoints[0:topk]): u = round(kp[0]) v = round(kp[1]) undist_kp = undistorted_cv2_keypoints[idx_kp] undist_u = round(undist_kp[0]) undist_v = round(undist_kp[1]) if u >= 0 and u < depth_map_size[ 0] and v >= 0 and v < depth_map_size[1]: if depth_map[v, u] == 0: continue pt3d = project_kp_to_3D(undist_u, undist_v, depth_map[v, u], depth_sensor_C[0, 2], depth_sensor_C[1, 2], depth_sensor_C[0, 0], depth_sensor_C[1, 1]) points3d_img.append(pt3d) rgb_img.append(img.getpixel((u, v))) kp_idxs.append(idx_kp) # transform to world coordinates (pt3d from a depth map is in camera coordinates) # we use sensor_id here because we assume that the image and the corresponding depthmap have the same pose # and sometimes, the pose might only be provided for the images cam_to_world = kdata.trajectories[timestamp][sensor_id].inverse() if len(points3d_img) == 0: continue points3d_img = cam_to_world.transform_points(np.array(points3d_img)) for idx_kp, pt3d, rgb in zip(kp_idxs, points3d_img, rgb_img): if not np.isnan(pt3d).any(): # apply transform (alignment) if method == Method.voxelgrid: assert vg is not None if not vg.exists(pt3d): # add 3D point points3d.append(list(pt3d) + list(rgb)) # add observation observations.add( len(points3d) - 1, keypoints_type, sensing_filepath, idx_kp) vg.add(pt3d, len(points3d) - 1, sensing_filepath) else: ret = vg.append(pt3d, sensing_filepath) if ret is not None: observations.add(ret[0], keypoints_type, sensing_filepath, idx_kp) elif method == Method.all: # add 3D point points3d.append(list(pt3d) + list(rgb)) # add observation observations.add( len(points3d) - 1, keypoints_type, sensing_filepath, idx_kp) # save_3Dpts_to_ply(points3d, os.path.join(output_path, 'map.ply')) progress_bar.update(1) progress_bar.close() kdata.points3d = kapture.Points3d(np.array(points3d)) kdata.observations = observations logger.info('saving ...') kapture_to_dir(output_path, kdata) # save_3Dpts_to_ply(points3d, os.path.join(output_path, 'map.ply')) logger.info('all done')
def import_silda( silda_dirpath: str, destination_kapture_dirpath: str, fallback_cam_model: str = 'FOV', do_split_cams: bool = False, corpus: Optional[str] = None, replace_pose_rig: bool = False, force_overwrite_existing: bool = False, images_import_strategy: TransferAction = TransferAction.link_absolute ) -> None: """ Imports data from silda dataset. :param silda_dirpath: path to the silda top directory :param destination_kapture_dirpath: input path to kapture directory. :param fallback_cam_model: camera model to fallback when necessary :param do_split_cams: If true, re-organises and renames the image files to split apart cameras. :param corpus: the list of corpus to be imported, among 'mapping', 'query'. :param replace_pose_rig: if True, replaces poses of individual cameras with poses of the rig. :param force_overwrite_existing: if true, Silently overwrite kapture files if already exists. :param images_import_strategy: how to copy image files. """ # sanity check silda_dirpath = path_secure(path.abspath(silda_dirpath)) destination_kapture_dirpath = path_secure( path.abspath(destination_kapture_dirpath)) if TransferAction.root_link == images_import_strategy and do_split_cams: raise ValueError( 'impossible to only link images directory and applying split cam.') hide_progress_bars = logger.getEffectiveLevel() >= logging.INFO # prepare output directory kapture.io.structure.delete_existing_kapture_files( destination_kapture_dirpath, force_overwrite_existing) os.makedirs(destination_kapture_dirpath, exist_ok=True) # images ########################################################################################################### logger.info('Processing images ...') # silda-images # ... # ├── 1445_0.png # ├── 1445_1.png # ... silda_images_root_path = path.join(silda_dirpath, 'silda-images') # list all png files (its PNG in silda) using a generator. if corpus is not None: assert corpus in SILDA_CORPUS_SPLIT_FILENAMES # if corpus specified, filter by those which directory name match corpus. logger.debug(f'only importing {corpus} part.') coprus_filepath = path.join(silda_dirpath, SILDA_CORPUS_SPLIT_FILENAMES[corpus]) with open(coprus_filepath, 'rt') as corpus_file: corpus_filenames = corpus_file.readlines() image_filenames_original = sorted(filename.strip() for filename in corpus_filenames) else: image_filenames_original = sorted( filename for dirpath, sd, fs in os.walk(silda_images_root_path) for filename in fs if filename.endswith('.png')) image_filenames_kapture = [] snapshots = kapture.RecordsCamera() image_name_to_ids = {} # '1445_0.png' -> (1445, 0) for image_filename_original in tqdm(image_filenames_original, disable=hide_progress_bars): # retrieve info from image filename shot_info = SILDA_IMAGE_NAME_PATTERN.match(image_filename_original) assert shot_info is not None shot_info = shot_info.groupdict() shot_info['timestamp'] = int( shot_info['timestamp'] ) # To avoid warnings about type of the value # eg. file_info = {'filename': '1445_0.png', 'timestamp': 1445, 'cam_id': '0'} # create a path of the image into NLE dir if do_split_cams: # re-organise images with subfolders per corpus/camera/timestamp.png kapture_image_filename = path.join( shot_info['cam_id'], '{:04d}.png'.format(shot_info['timestamp'])) else: # keep the original file hierarchy kapture_image_filename = image_filename_original image_filenames_kapture.append(kapture_image_filename) snapshots[shot_info['timestamp'], shot_info['cam_id']] = kapture_image_filename image_name_to_ids[shot_info['filename']] = (shot_info['timestamp'], shot_info['cam_id']) assert len(image_filenames_kapture) == len(image_filenames_original) # intrinsics ####################################################################################################### logger.info('Processing sensors ...') cameras = kapture.Sensors() # use hard coded intrinsics # evaluated using colmap # 1 OPENCV_FISHEYE 1024 1024 393.299 394.815 512 512 -0.223483 0.117325 -0.0326138 0.00361082 # fx, fy, cx, cy, omega # 1 FOV 1024 1024 300 300 512 512 0.899632 cam_id_list = sorted( set(cam_id for _, cam_id, _ in kapture.flatten(snapshots))) for cam_id in cam_id_list: # pick a image for that cam id random_image_intrinsic = next( f'{timestamp}_{cam_id}.intrinsics' # keep only filename (thats what silda expect) for timestamp, cid, filename in kapture.flatten(snapshots) if cid == cam_id) logger.debug( f'camera {cam_id} intrinsics : picking at random: ("{random_image_intrinsic}")' ) intrinsic_filepath = path.join(silda_dirpath, 'camera-intrinsics', random_image_intrinsic) logger.debug(f'loading file: "{intrinsic_filepath}"') silda_proj_params = np.loadtxt(intrinsic_filepath) # only retrieve principal point from intrinsics, # because the rest correspond to a fisheye model not available in colmap. principal_point = (silda_proj_params[0:2] * SILDA_IMAGE_SIZE).flatten().tolist() projection = fallback_cam_model if 'OPENCV_FISHEYE' == projection: focal_length = [393.299, 394.815] fisheye_coefficients = [ -0.223483, 0.117325, -0.0326138, 0.00361082 ] # // fx, fy, cx, cy, k1, k2, k3, k4 proj_params = focal_length + principal_point + fisheye_coefficients elif 'FOV' == projection: # use hard coded intrinsics from Torsten reconstruction, ie. : # 217.294036, 217.214703, 512.000000, 507.897400, -0.769113 focal_length = [217.294036, 217.214703] # principal_point = [512.000000, 507.897400] omega = [-0.769113] # fx, fy, cx, cy, omega proj_params = focal_length + principal_point + omega else: raise ValueError( 'Only accepts OPENCV_FISHEYE, or FOV as projection model.') camera = kapture.Camera(projection, SILDA_IMAGE_SIZE.tolist() + proj_params) cameras[cam_id] = camera # extrinsics ####################################################################################################### logger.info('Processing trajectories ...') trajectories = kapture.Trajectories() with open(path.join(silda_dirpath, 'silda-train-poses.txt')) as file: lines = file.readlines() lines = (line.rstrip().split() for line in lines) extrinsics = { line[0]: np.array(line[1:8], dtype=np.float) for line in lines } for silda_image_name, pose_params in tqdm(extrinsics.items(), disable=hide_progress_bars): # Silda poses are 7-dim vectors with the rotation quaternion, # and the translation vector. The order needs to be: # qw,qx,qy,qz,tx,ty,tz # The parameters should be described in terms of camera to world transformations if silda_image_name not in image_name_to_ids: # if this is not referenced: means its part of the corpus to be ignored. continue pose = kapture.PoseTransform(pose_params[0:4], pose_params[4:7]).inverse() timestamp, cam_id = image_name_to_ids[silda_image_name] trajectories[timestamp, cam_id] = pose # rigs logger.info('Making up a rig ...') rigs = kapture.Rigs() pose_babord = kapture.PoseTransform(t=[0, 0, 0], r=quaternion.from_rotation_vector( [0, -np.pi / 2, 0])) pose_tribord = kapture.PoseTransform(t=[0, 0, 0], r=quaternion.from_rotation_vector( [0, np.pi / 2, 0])) rigs['silda_rig', '0'] = pose_babord rigs['silda_rig', '1'] = pose_tribord if replace_pose_rig: logger.info('replacing camera poses with rig poses.') kapture.rigs_recover_inplace(trajectories, rigs) # pack it all together kapture_data = kapture.Kapture(sensors=cameras, records_camera=snapshots, trajectories=trajectories, rigs=rigs) logger.info('saving to Kapture ...') kapture.io.csv.kapture_to_dir(destination_kapture_dirpath, kapture_data) # finally import images if images_import_strategy != TransferAction.skip: # importing image files logger.info(f'importing {len(image_filenames_original)} images ...') assert len(image_filenames_original) == len(image_filenames_kapture) image_filepaths_original = [ path.join(silda_images_root_path, image_filename_kapture) for image_filename_kapture in image_filenames_original ] image_filepaths_kapture = [ get_image_fullpath(destination_kapture_dirpath, image_filename_kapture) for image_filename_kapture in image_filenames_kapture ] transfer_files_from_dir(image_filepaths_original, image_filepaths_kapture, images_import_strategy) logger.info('done.')
def export_openmvg_sfm_data(kapture_path: str, kapture_data: kapture.Kapture, openmvg_sfm_data_file_path: str, openmvg_image_root_path: str, image_action: TransferAction, image_path_flatten: bool, force: bool, kapture_to_openmvg_view_ids: dict = {}) -> Dict: """ Convert the kapture data into an openMVG dataset stored as a dictionary. The format is defined here: https://openmvg.readthedocs.io/en/latest/software/SfM/SfM_OutputFormat/ :param kapture_data: the kapture data :param kapture_path: top directory of the kapture data and the images :param openmvg_sfm_data_file_path: input path to the SfM data file to be written. :param openmvg_image_root_path: input path to openMVG image directory to be created. :param image_action: action to apply on images: link, copy, move or do nothing. :param image_path_flatten: flatten image path (eg. to avoid image name collision in openMVG regions). :param force: if true, will remove existing openMVG data without prompting the user. :param kapture_to_openmvg_view_ids: input/output mapping of kapture image name to corresponding openmvg view id. :return: an SfM_data, the openmvg structure, stored as a dictionary ready to be serialized """ if kapture_data.cameras is None or kapture_data.records_camera is None: raise ValueError( 'export_openmvg_sfm_data needs kapture camera and records_camera.') if image_action == TransferAction.root_link: raise NotImplementedError( 'root link is not implemented, use skip instead.') # refer to the original image dir when skipping image transfer. if image_action == TransferAction.skip: openmvg_image_root_path = get_image_fullpath(kapture_path) if openmvg_image_root_path is None: raise ValueError( f'openmvg_image_root_path must be defined to be able to perform {image_action}.' ) # make sure directory is ready to contain openmvg_sfm_data_file_path os.makedirs(path.dirname(openmvg_sfm_data_file_path), exist_ok=True) # Check we don't have other sensors defined if len(kapture_data.sensors) != len(kapture_data.cameras): extra_sensor_number = len(kapture_data.sensors) - len( kapture_data.cameras) logger.warning( f'We will ignore {extra_sensor_number} sensors that are not camera' ) # openmvg does not support rigs if kapture_data.rigs: logger.info('remove rigs notation.') rigs_remove_inplace(kapture_data.trajectories, kapture_data.rigs) kapture_data.rigs.clear() # Compute root path and camera used in records kapture_to_openmvg_cam_ids = {} # kapture_cam_id -> openmvg_cam_id for i, (_, _, kapture_image_name) in enumerate( kapture.flatten(kapture_data.records_camera)): if kapture_image_name not in kapture_to_openmvg_view_ids: kapture_to_openmvg_view_ids[kapture_image_name] = i # polymorphic_status = PolymorphicStatus({}, 1, 1) polymorphic_registry = CerealPointerRegistry( id_key=JSON_KEY.POLYMORPHIC_ID, value_key=JSON_KEY.POLYMORPHIC_NAME) ptr_wrapper_registry = CerealPointerRegistry(id_key=JSON_KEY.ID, value_key=JSON_KEY.DATA) logger.debug(f'exporting intrinsics ...') openmvg_sfm_data_intrinsics = export_openmvg_intrinsics( kapture_cameras=kapture_data.cameras, kapture_to_openmvg_cam_ids=kapture_to_openmvg_cam_ids, polymorphic_registry=polymorphic_registry, ptr_wrapper_registry=ptr_wrapper_registry, ) logger.debug(f'exporting views ...') openmvg_sfm_data_views = export_openmvg_views( kapture_cameras=kapture_data.cameras, kapture_images=kapture_data.records_camera, kapture_trajectories=kapture_data.trajectories, kapture_to_openmvg_cam_ids=kapture_to_openmvg_cam_ids, kapture_to_openmvg_view_ids=kapture_to_openmvg_view_ids, polymorphic_registry=polymorphic_registry, ptr_wrapper_registry=ptr_wrapper_registry, image_path_flatten=image_path_flatten, ) logger.debug(f'exporting poses ...') openmvg_sfm_data_poses = export_openmvg_poses( kapture_images=kapture_data.records_camera, kapture_trajectories=kapture_data.trajectories, kapture_to_openmvg_view_ids=kapture_to_openmvg_view_ids) # structure : correspond to kapture observations + 3D points logger.debug(f'exporting structure ...') openmvg_sfm_data_structure = export_openmvg_structure( kapture_points_3d=kapture_data.points3d, kapture_to_openmvg_view_ids=kapture_to_openmvg_view_ids, kapture_observations=kapture_data.observations, kapture_keypoints=kapture_data.keypoints, kapture_path=kapture_path) openmvg_sfm_data = { JSON_KEY.SFM_DATA_VERSION: OPENMVG_SFM_DATA_VERSION_NUMBER, JSON_KEY.ROOT_PATH: path.abspath(openmvg_image_root_path), JSON_KEY.INTRINSICS: openmvg_sfm_data_intrinsics, JSON_KEY.VIEWS: openmvg_sfm_data_views, JSON_KEY.EXTRINSICS: openmvg_sfm_data_poses, JSON_KEY.STRUCTURE: openmvg_sfm_data_structure, JSON_KEY.CONTROL_POINTS: [], } logger.debug(f'Saving to openmvg {openmvg_sfm_data_file_path}...') with open(openmvg_sfm_data_file_path, "w") as fid: json.dump(openmvg_sfm_data, fid, indent=4) # do the actual image transfer if not image_action == TransferAction.skip: job_copy = ( ( # source path -> dest path get_image_fullpath(kapture_path, kapture_image_name), path.join( openmvg_image_root_path, get_openmvg_image_path(kapture_image_name, image_path_flatten))) for _, _, kapture_image_name in kapture.flatten(kapture_data.records_camera)) source_filepath_list, destination_filepath_list = zip(*job_copy) transfer_files_from_dir( source_filepath_list=source_filepath_list, destination_filepath_list=destination_filepath_list, copy_strategy=image_action, force_overwrite=force)
def colmap_build_map_from_loaded_data(kapture_data: kapture.Kapture, kapture_path: str, colmap_path: str, colmap_binary: str, pairsfile_path: Optional[str], use_colmap_matches_importer: bool, point_triangulator_options: List[str], skip_list: List[str], force: bool) -> None: """ Build a colmap model using custom features with the kapture data. :param kapture_data: kapture data to use :param kapture_path: path to the kapture to use :param colmap_path: path to the colmap build :param colmap_binary: path to the colmap executable :param pairsfile_path: Optional[str], :param use_colmap_matches_importer: bool, :param point_triangulator_options: options for the point triangulator :param skip_list: list of steps to skip :param force: Silently overwrite kapture files if already exists. """ os.makedirs(colmap_path, exist_ok=True) if not (kapture_data.records_camera and kapture_data.sensors and kapture_data.keypoints and kapture_data.matches): raise ValueError('records_camera, sensors, keypoints, matches are mandatory') if not kapture_data.trajectories: logger.info('there are no trajectories, running mapper instead of point_triangulator') # COLMAP does not fully support rigs. if kapture_data.rigs is not None and kapture_data.trajectories is not None: # make sure, rigs are not used in trajectories. logger.info('remove rigs notation.') rigs_remove_inplace(kapture_data.trajectories, kapture_data.rigs) kapture_data.rigs.clear() # Set fixed name for COLMAP database colmap_db_path = path.join(colmap_path, 'colmap.db') reconstruction_path = path.join(colmap_path, "reconstruction") priors_txt_path = path.join(colmap_path, "priors_for_reconstruction") if 'delete_existing' not in skip_list: safe_remove_file(colmap_db_path, force) safe_remove_any_path(reconstruction_path, force) safe_remove_any_path(priors_txt_path, force) os.makedirs(reconstruction_path, exist_ok=True) if 'colmap_db' not in skip_list: logger.info('Using precomputed keypoints and matches') logger.info('Step 1: Export kapture format to colmap') colmap_db = COLMAPDatabase.connect(colmap_db_path) if kapture_data.descriptors is not None: kapture_data.descriptors.clear() database_extra.kapture_to_colmap(kapture_data, kapture_path, colmap_db, export_two_view_geometry=not use_colmap_matches_importer) # close db before running colmap processes in order to avoid locks colmap_db.close() if use_colmap_matches_importer: logger.info('Step 2: Run geometric verification') logger.debug('running colmap matches_importer...') colmap_lib.run_matches_importer_from_kapture( colmap_binary, colmap_use_cpu=True, colmap_gpu_index=None, colmap_db_path=colmap_db_path, kapture_data=kapture_data, force=force ) else: logger.info('Step 2: Run geometric verification - skipped') if kapture_data.trajectories is not None: # Generate priors for reconstruction os.makedirs(priors_txt_path, exist_ok=True) if 'priors_for_reconstruction' not in skip_list: logger.info('Step 3: Exporting priors for reconstruction.') colmap_db = COLMAPDatabase.connect(colmap_db_path) database_extra.generate_priors_for_reconstruction(kapture_data, colmap_db, priors_txt_path) colmap_db.close() # Point triangulator reconstruction_path = path.join(colmap_path, "reconstruction") os.makedirs(reconstruction_path, exist_ok=True) if 'triangulation' not in skip_list: logger.info("Step 4: Triangulation") colmap_lib.run_point_triangulator( colmap_binary, colmap_db_path, get_image_fullpath(kapture_path), priors_txt_path, reconstruction_path, point_triangulator_options ) else: # mapper reconstruction_path = path.join(colmap_path, "reconstruction") os.makedirs(reconstruction_path, exist_ok=True) if 'triangulation' not in skip_list: logger.info("Step 4: Triangulation") colmap_lib.run_mapper( colmap_binary, colmap_db_path, get_image_fullpath(kapture_path), None, reconstruction_path, point_triangulator_options ) # use reconstruction 0 as main first_reconstruction = os.path.join(reconstruction_path, '0') files = os.listdir(first_reconstruction) for f in files: shutil.move(os.path.join(first_reconstruction, f), os.path.join(reconstruction_path, f)) shutil.rmtree(first_reconstruction) # run model_converter if 'model_converter' not in skip_list: logger.info("Step 5: Export reconstruction results to txt") colmap_lib.run_model_converter( colmap_binary, reconstruction_path, reconstruction_path )
def extract_kapture_global(kapture_root, config, output_dir='', overwrite=False): logging.info('Extracting NetVLAD features with configuration:\n', config) # use kapture io to identify image paths and loop kdata = kapture_from_dir(kapture_root, matches_pairsfile_path=None, skip_list= [kapture.Matches, kapture.Points3d, kapture.Observations, kapture.Keypoints, kapture.Descriptors]) assert kdata.records_camera is not None export_dir = output_dir if output_dir else kapture_root # root of output directory for features os.makedirs(export_dir, exist_ok=True) image_list = [filename for _, _, filename in kapture.flatten(kdata.records_camera)] # resume extraction if some features exist try: # load features if there are any kdata.global_features = global_features_from_dir(export_dir, None) if kdata.global_features is not None and not overwrite: image_list = [name for name in image_list if name not in kdata.global_features] except FileNotFoundError: pass except: logging.exception("Error with importing existing global features.") # clear features first if overwriting if overwrite: delete_existing_kapture_files(export_dir, True, only=[kapture.GlobalFeatures]) if len(image_list) == 0: print('All features were already extracted') return else: print(f'Extracting NetVLAD features for {len(image_list)} images') # for the global descriptor type specification global_dtype = None if kdata.global_features is None else kdata.global_features.dtype global_dsize = None if kdata.global_features is None else kdata.global_features.dsize # setup network tf.reset_default_graph() if config['grayscale']: tf_batch = tf.placeholder( dtype=tf.float32, shape=[None, None, None, 1]) else: tf_batch = tf.placeholder( dtype=tf.float32, shape=[None, None, None, 3]) # load network and checkpoint net = nets.vgg16NetvladPca(tf_batch) saver = tf.train.Saver() sess = tf.Session() checkpoint = chkpt_path + '/' + config['checkpoint'] saver.restore(sess, checkpoint) for image_name in image_list: img_path = get_image_fullpath(kapture_root, image_name) if img_path.endswith('.txt'): args.images = open(img_path).read().splitlines() + args.images continue print(f"\nExtracting features for {img_path}") if config['grayscale']: image = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE) image = np.expand_dims( np.expand_dims(image, axis=0), axis=-1) else: image = cv2.imread(img_path, cv2.COLOR_BGR2RGB) image = np.expand_dims(image, axis=0) descriptor = sess.run(net, feed_dict={tf_batch: image})[:, :config['pca_dim']] descriptor = np.squeeze(descriptor) # write global descriptor type specification if global_dtype is None: global_dtype = descriptor.dtype global_dsize = len(descriptor) kdata.global_features = kapture.GlobalFeatures('netvlad', global_dtype, global_dsize) global_descriptors_config_abs_path = get_csv_fullpath(kapture.GlobalFeatures, export_dir) descriptors_to_file(global_descriptors_config_abs_path, kdata.global_features) else: assert kdata.global_features.type_name == "netvlad" assert kdata.global_features.dtype == descriptor.dtype assert kdata.global_features.dsize == len(descriptor) # get output paths global_descriptors_abs_path = get_global_features_fullpath(export_dir, image_name) image_global_features_to_file(global_descriptors_abs_path, descriptor) kdata.global_features.add(image_name) # sess.close() # close session before initializing again for next submap if not global_features_check_dir(kdata.global_features, export_dir): print('global feature extraction ended successfully but not all files were saved')