def _reconstruction_from_rigs_and_assignments(data: DataSetBase): assignments = data.load_rig_assignments() models = data.load_rig_models() if not data.reference_lla_exists(): data.invent_reference_lla() base_rotation = np.zeros(3) reconstructions = [] for rig_id, instances in assignments.items(): rig_cameras = models[rig_id]["rig_cameras"] reconstruction = types.Reconstruction() reconstruction.cameras = data.load_camera_models() for instance in instances: for image, camera_id in instance: rig_camera = rig_cameras[camera_id] rig_pose = pygeometry.Pose(base_rotation) rig_pose.set_origin( orec.get_image_metadata(data, image).gps_position.value) rig_camera_pose = pygeometry.Pose(rig_camera["rotation"], rig_camera["translation"]) d = data.load_exif(image) shot = reconstruction.create_shot(image, d["camera"]) shot.pose = rig_camera_pose.compose(rig_pose) shot.metadata = orec.get_image_metadata(data, image) reconstructions.append(reconstruction) return reconstructions
def _reconstruction_from_rigs_and_assignments(data: DataSetBase): assignments = data.load_rig_assignments() rig_cameras = data.load_rig_cameras() data.init_reference() reconstruction = types.Reconstruction() reconstruction.cameras = data.load_camera_models() for rig_instance_id, instance in assignments.items(): for image, rig_camera_id in instance: rig_camera = rig_cameras[rig_camera_id] reconstruction.add_rig_camera( pymap.RigCamera(rig_camera.pose, rig_camera_id)) instance_obj = reconstruction.add_rig_instance( pymap.RigInstance(rig_instance_id)) instance_obj.pose.set_origin( helpers.get_image_metadata(data, image).gps_position.value) d = data.load_exif(image) shot = reconstruction.create_shot( image, camera_id=d["camera"], rig_camera_id=rig_camera_id, rig_instance_id=rig_instance_id, ) shot.metadata = helpers.get_image_metadata(data, image) return [reconstruction]
def match_images_with_pairs(data: DataSetBase, config_override, exifs, pairs): """ Perform pair matchings given pairs. """ cameras = data.load_camera_models() args = list(match_arguments(pairs, data, config_override, cameras, exifs)) # Perform all pair matchings in parallel start = timer() logger.info("Matching {} image pairs".format(len(pairs))) mem_per_process = 512 jobs_per_process = 2 processes = context.processes_that_fit_in_memory(data.config["processes"], mem_per_process) logger.info("Computing pair matching with %d processes" % processes) matches = context.parallel_map(match_unwrap_args, args, processes, jobs_per_process) logger.info("Matched {} pairs {} in {} seconds ({} seconds/pair).".format( len(pairs), log_projection_types(pairs, exifs, cameras), timer() - start, (timer() - start) / len(pairs) if pairs else 0, )) # Index results per pair resulting_pairs = {} for im1, im2, m in matches: resulting_pairs[im1, im2] = m return resulting_pairs
def cameras_statistics(data: DataSetBase, reconstructions): stats = {} permutation = np.argsort([-len(r.shots) for r in reconstructions]) for camera_id, camera_model in data.load_camera_models().items(): stats[camera_id] = {"initial_values": _cameras_statistics(camera_model)} for idx in permutation: rec = reconstructions[idx] for camera in rec.cameras.values(): if "optimized_values" in stats[camera.id]: continue stats[camera.id]["optimized_values"] = _cameras_statistics(camera) for camera_id in data.load_camera_models(): if "optimized_values" not in stats[camera_id]: del stats[camera_id] return stats
def compute_image_pairs(track_dict, data: DataSetBase): """All matched image pairs sorted by reconstructability.""" cameras = data.load_camera_models() args = _pair_reconstructability_arguments(track_dict, cameras, data) processes = data.config["processes"] result = parallel_map(_compute_pair_reconstructability, args, processes) result = list(result) pairs = [(im1, im2) for im1, im2, r in result if r > 0] score = [r for im1, im2, r in result if r > 0] order = np.argsort(-np.array(score)) return [pairs[o] for o in order]
def incremental_reconstruction(data: DataSetBase, tracks_manager): """Run the entire incremental reconstruction pipeline.""" logger.info("Starting incremental reconstruction") report = {} chrono = Chronometer() images = tracks_manager.get_shot_ids() if not data.reference_lla_exists(): data.invent_reference_lla(images) remaining_images = set(images) camera_priors = data.load_camera_models() gcp = data.load_ground_control_points() common_tracks = tracking.all_common_tracks(tracks_manager) reconstructions = [] pairs = compute_image_pairs(common_tracks, camera_priors, data) chrono.lap("compute_image_pairs") report["num_candidate_image_pairs"] = len(pairs) report["reconstructions"] = [] for im1, im2 in pairs: if im1 in remaining_images and im2 in remaining_images: rec_report = {} report["reconstructions"].append(rec_report) _, p1, p2 = common_tracks[im1, im2] reconstruction, rec_report["bootstrap"] = bootstrap_reconstruction( data, tracks_manager, camera_priors, im1, im2, p1, p2) if reconstruction: remaining_images.remove(im1) remaining_images.remove(im2) reconstruction, rec_report["grow"] = grow_reconstruction( data, tracks_manager, reconstruction, remaining_images, camera_priors, gcp, ) reconstructions.append(reconstruction) reconstructions = sorted(reconstructions, key=lambda x: -len(x.shots)) for k, r in enumerate(reconstructions): logger.info("Reconstruction {}: {} images, {} points".format( k, len(r.shots), len(r.points))) logger.info("{} partial reconstructions in total.".format( len(reconstructions))) chrono.lap("compute_reconstructions") report["wall_times"] = dict(chrono.lap_times()) report["not_reconstructed_images"] = list(remaining_images) return report, reconstructions
def is_high_res_panorama(data: DataSetBase, image_key, image_array): """Detect if image is a panorama.""" exif = data.load_exif(image_key) if exif: camera = data.load_camera_models()[exif["camera"]] w, h = int(exif["width"]), int(exif["height"]) exif_pano = pygeometry.Camera.is_panorama(camera.projection_type) elif image_array is not None: h, w = image_array.shape[:2] exif_pano = False else: return False return w == 2 * h or exif_pano
def run_dataset(dataset: DataSetBase, input, output): """Bundle a reconstructions. Args: input: input reconstruction JSON in the dataset output: input reconstruction JSON in the dataset """ reconstructions = dataset.load_reconstruction(input) camera_priors = dataset.load_camera_models() gcp = dataset.load_ground_control_points() tracks_manager = dataset.load_tracks_manager() # load the tracks manager and add its observations to the reconstruction # go through all the points and add their shots for reconstruction in reconstructions: reconstruction.add_correspondences_from_tracks_manager(tracks_manager) orec.bundle(reconstruction, camera_priors, gcp, dataset.config) dataset.save_reconstruction(reconstructions, output)
def rig_statistics( data: DataSetBase, reconstructions: List[types.Reconstruction]) -> Dict[str, Any]: stats = {} permutation = np.argsort([-len(r.shots) for r in reconstructions]) rig_cameras = data.load_rig_cameras() cameras = data.load_camera_models() for rig_camera_id, rig_camera in rig_cameras.items(): # we skip per-camera rig camera for now if rig_camera_id in cameras: continue stats[rig_camera_id] = { "initial_values": { "rotation": list(rig_camera.pose.rotation), "translation": list(rig_camera.pose.translation), } } for idx in permutation: rec = reconstructions[idx] for rig_camera in rec.rig_cameras.values(): if rig_camera.id not in stats: continue if "optimized_values" in stats[rig_camera.id]: continue stats[rig_camera.id]["optimized_values"] = { "rotation": list(rig_camera.pose.rotation), "translation": list(rig_camera.pose.translation), } for rig_camera_id in rig_cameras: if rig_camera.id not in stats: continue if "optimized_values" not in stats[rig_camera_id]: del stats[rig_camera_id] return stats
def bootstrap_reconstruction( data: DataSetBase, tracks_manager: pymap.TracksManager, im1: str, im2: str, p1: np.ndarray, p2: np.ndarray, ) -> Tuple[Optional[types.Reconstruction], Dict[str, Any]]: """Start a reconstruction using two shots.""" logger.info("Starting reconstruction with {} and {}".format(im1, im2)) report: Dict[str, Any] = { "image_pair": (im1, im2), "common_tracks": len(p1), } camera_priors = data.load_camera_models() camera1 = camera_priors[data.load_exif(im1)["camera"]] camera2 = camera_priors[data.load_exif(im2)["camera"]] threshold = data.config["five_point_algo_threshold"] min_inliers = data.config["five_point_algo_min_inliers"] iterations = data.config["five_point_refine_rec_iterations"] R, t, inliers, report[ "two_view_reconstruction"] = two_view_reconstruction_general( p1, p2, camera1, camera2, threshold, iterations) logger.info("Two-view reconstruction inliers: {} / {}".format( len(inliers), len(p1))) if len(inliers) <= 5: report["decision"] = "Could not find initial motion" logger.info(report["decision"]) return None, report rig_camera_priors = data.load_rig_cameras() rig_assignments = data.load_rig_assignments_per_image() reconstruction = types.Reconstruction() reconstruction.reference = data.load_reference() reconstruction.cameras = camera_priors reconstruction.rig_cameras = rig_camera_priors new_shots = add_shot(data, reconstruction, rig_assignments, im1, pygeometry.Pose()) if im2 not in new_shots: new_shots |= add_shot(data, reconstruction, rig_assignments, im2, pygeometry.Pose(R, t)) align_reconstruction(reconstruction, None, data.config) triangulate_shot_features(tracks_manager, reconstruction, new_shots, data.config) logger.info("Triangulated: {}".format(len(reconstruction.points))) report["triangulated_points"] = len(reconstruction.points) if len(reconstruction.points) < min_inliers: report["decision"] = "Initial motion did not generate enough points" logger.info(report["decision"]) return None, report to_adjust = {s for s in new_shots if s != im1} bundle_shot_poses(reconstruction, to_adjust, camera_priors, rig_camera_priors, data.config) retriangulate(tracks_manager, reconstruction, data.config) if len(reconstruction.points) < min_inliers: report[ "decision"] = "Re-triangulation after initial motion did not generate enough points" logger.info(report["decision"]) return None, report bundle_shot_poses(reconstruction, to_adjust, camera_priors, rig_camera_priors, data.config) report["decision"] = "Success" report["memory_usage"] = current_memory_usage() return reconstruction, report
def grow_reconstruction( data: DataSetBase, tracks_manager: pymap.TracksManager, reconstruction: types.Reconstruction, images: Set[str], gcp: List[pymap.GroundControlPoint], ) -> Tuple[types.Reconstruction, Dict[str, Any]]: """Incrementally add shots to an initial reconstruction.""" config = data.config report = {"steps": []} camera_priors = data.load_camera_models() rig_camera_priors = data.load_rig_cameras() paint_reconstruction(data, tracks_manager, reconstruction) align_reconstruction(reconstruction, gcp, config) bundle(reconstruction, camera_priors, rig_camera_priors, None, config) remove_outliers(reconstruction, config) paint_reconstruction(data, tracks_manager, reconstruction) should_bundle = ShouldBundle(data, reconstruction) should_retriangulate = ShouldRetriangulate(data, reconstruction) while True: if config["save_partial_reconstructions"]: paint_reconstruction(data, tracks_manager, reconstruction) data.save_reconstruction( [reconstruction], "reconstruction.{}.json".format( datetime.datetime.now().isoformat().replace(":", "_")), ) candidates = reconstructed_points_for_images(tracks_manager, reconstruction, images) if not candidates: break logger.info("-------------------------------------------------------") threshold = data.config["resection_threshold"] min_inliers = data.config["resection_min_inliers"] for image, _ in candidates: ok, new_shots, resrep = resect( data, tracks_manager, reconstruction, image, threshold, min_inliers, ) if not ok: continue images -= new_shots bundle_shot_poses( reconstruction, new_shots, camera_priors, rig_camera_priors, data.config, ) logger.info( f"Adding {' and '.join(new_shots)} to the reconstruction") step = { "images": list(new_shots), "resection": resrep, "memory_usage": current_memory_usage(), } report["steps"].append(step) np_before = len(reconstruction.points) triangulate_shot_features(tracks_manager, reconstruction, new_shots, config) np_after = len(reconstruction.points) step["triangulated_points"] = np_after - np_before if should_retriangulate.should(): logger.info("Re-triangulating") align_reconstruction(reconstruction, gcp, config) b1rep = bundle(reconstruction, camera_priors, rig_camera_priors, None, config) rrep = retriangulate(tracks_manager, reconstruction, config) b2rep = bundle(reconstruction, camera_priors, rig_camera_priors, None, config) remove_outliers(reconstruction, config) step["bundle"] = b1rep step["retriangulation"] = rrep step["bundle_after_retriangulation"] = b2rep should_retriangulate.done() should_bundle.done() elif should_bundle.should(): align_reconstruction(reconstruction, gcp, config) brep = bundle(reconstruction, camera_priors, rig_camera_priors, None, config) remove_outliers(reconstruction, config) step["bundle"] = brep should_bundle.done() elif config["local_bundle_radius"] > 0: bundled_points, brep = bundle_local( reconstruction, camera_priors, rig_camera_priors, None, image, config, ) remove_outliers(reconstruction, config, bundled_points) step["local_bundle"] = brep break else: logger.info("Some images can not be added") break logger.info("-------------------------------------------------------") align_reconstruction(reconstruction, gcp, config) bundle(reconstruction, camera_priors, rig_camera_priors, gcp, config) remove_outliers(reconstruction, config) paint_reconstruction(data, tracks_manager, reconstruction) return reconstruction, report
def average_image_size(data: DataSetBase) -> float: average_size_mb = 0 for camera in data.load_camera_models().values(): average_size_mb += camera.width * camera.height * 4 / 1024 / 1024 return average_size_mb / len(data.load_camera_models())