def gcp_errors(data: DataSetBase, reconstructions: List[types.Reconstruction]) -> Dict[str, Any]: all_errors = [] reference = data.load_reference() gcps = data.load_ground_control_points() if not gcps: return {} all_errors = [] for gcp in gcps: if not gcp.lla: continue triangulated = None for rec in reconstructions: triangulated = multiview.triangulate_gcp(gcp, rec.shots, 1.0, 0.1) if triangulated is None: continue else: break if triangulated is None: continue gcp_enu = reference.to_topocentric(*gcp.lla_vec) all_errors.append(triangulated - gcp_enu) return _gps_gcp_errors_stats(np.array(all_errors))
def rig_statistics(data: DataSetBase, reconstructions): stats = {} permutation = np.argsort([-len(r.shots) for r in reconstructions]) for rig_camera_id, rig_camera in data.load_rig_cameras().items(): stats[rig_camera_id] = { "initial_values": { "rotation": list(rig_camera.pose.rotation), "translation": list(rig_camera.pose.translation), } } for idx in permutation: rec = reconstructions[idx] for rig_camera in rec.rig_cameras.values(): if "optimized_values" in stats[rig_camera.id]: continue stats[rig_camera.id]["optimized_values"] = { "rotation": list(rig_camera.pose.rotation), "translation": list(rig_camera.pose.translation), } for rig_camera_id in data.load_rig_cameras(): if "optimized_values" not in stats[rig_camera_id]: del stats[rig_camera_id] return stats
def run_dataset(data: DataSetBase): """ Link matches pair-wise matches into tracks. """ start = timer() features, colors, segmentations, instances = tracking.load_features( data, data.images()) features_end = timer() matches = tracking.load_matches(data, data.images()) matches_end = timer() tracks_manager = tracking.create_tracks_manager( features, colors, segmentations, instances, matches, data.config, ) tracks_end = timer() data.save_tracks_manager(tracks_manager) write_report( data, tracks_manager, features_end - start, matches_end - features_end, tracks_end - matches_end, )
def bootstrap_reconstruction(data: DataSetBase, tracks_manager, camera_priors, im1, im2, p1, p2): """Start a reconstruction using two shots.""" logger.info("Starting reconstruction with {} and {}".format(im1, im2)) report: Dict[str, Any] = { "image_pair": (im1, im2), "common_tracks": len(p1), } camera_id1 = data.load_exif(im1)["camera"] camera_id2 = data.load_exif(im2)["camera"] camera1 = camera_priors[camera_id1] camera2 = camera_priors[camera_id2] threshold = data.config["five_point_algo_threshold"] min_inliers = data.config["five_point_algo_min_inliers"] iterations = data.config["five_point_refine_rec_iterations"] R, t, inliers, report[ "two_view_reconstruction"] = two_view_reconstruction_general( p1, p2, camera1, camera2, threshold, iterations) logger.info("Two-view reconstruction inliers: {} / {}".format( len(inliers), len(p1))) if len(inliers) <= 5: report["decision"] = "Could not find initial motion" logger.info(report["decision"]) return None, report reconstruction = types.Reconstruction() reconstruction.reference = data.load_reference() reconstruction.cameras = camera_priors shot1 = reconstruction.create_shot(im1, camera_id1, pygeometry.Pose()) shot1.metadata = get_image_metadata(data, im1) shot2 = reconstruction.create_shot(im2, camera_id2, pygeometry.Pose(R, t)) shot2.metadata = get_image_metadata(data, im2) triangulate_shot_features(tracks_manager, reconstruction, im1, data.config) logger.info("Triangulated: {}".format(len(reconstruction.points))) report["triangulated_points"] = len(reconstruction.points) if len(reconstruction.points) < min_inliers: report["decision"] = "Initial motion did not generate enough points" logger.info(report["decision"]) return None, report bundle_single_view(reconstruction, im2, camera_priors, data.config) retriangulate(tracks_manager, reconstruction, data.config) if len(reconstruction.points) < min_inliers: report[ "decision"] = "Re-triangulation after initial motion did not generate enough points" logger.info(report["decision"]) return None, report bundle_single_view(reconstruction, im2, camera_priors, data.config) report["decision"] = "Success" report["memory_usage"] = current_memory_usage() return reconstruction, report
def write_report(data: DataSetBase, preport, pairs, wall_time): report = { "wall_time": wall_time, "num_pairs": len(pairs), "pairs": pairs, } report.update(preport) data.save_report(io.json_dumps(report), "matches.json")
def run_dataset(data: DataSetBase): """ Compute the SfM reconstruction. """ tracks_manager = data.load_tracks_manager() report, reconstructions = reconstruction.incremental_reconstruction( data, tracks_manager) data.save_reconstruction(reconstructions) data.save_report(io.json_dumps(report), "reconstruction.json")
def save_matches(data: DataSetBase, images_ref, matched_pairs): """Given pairwise matches (image 1, image 2) - > matches, save them such as only {image E images_ref} will store the matches. """ matches_per_im1 = {im: {} for im in images_ref} for (im1, im2), m in matched_pairs.items(): matches_per_im1[im1][im2] = m for im1, im1_matches in matches_per_im1.items(): data.save_matches(im1, im1_matches)
def write_report(data: DataSetBase, wall_time): image_reports = [] for image in data.images(): try: txt = data.load_report("features/{}.json".format(image)) image_reports.append(io.json_loads(txt)) except IOError: logger.warning("No feature report image {}".format(image)) report = {"wall_time": wall_time, "image_reports": image_reports} data.save_report(io.json_dumps(report), "features.json")
def incremental_reconstruction( data: DataSetBase, tracks_manager: pysfm.TracksManager ) -> Tuple[Dict[str, Any], List[types.Reconstruction]]: """Run the entire incremental reconstruction pipeline.""" logger.info("Starting incremental reconstruction") report = {} chrono = Chronometer() images = tracks_manager.get_shot_ids() if not data.reference_lla_exists(): data.invent_reference_lla(images) remaining_images = set(images) gcp = data.load_ground_control_points() common_tracks = tracking.all_common_tracks(tracks_manager) reconstructions = [] pairs = compute_image_pairs(common_tracks, data) chrono.lap("compute_image_pairs") report["num_candidate_image_pairs"] = len(pairs) report["reconstructions"] = [] for im1, im2 in pairs: if im1 in remaining_images and im2 in remaining_images: rec_report = {} report["reconstructions"].append(rec_report) _, p1, p2 = common_tracks[im1, im2] reconstruction, rec_report["bootstrap"] = bootstrap_reconstruction( data, tracks_manager, im1, im2, p1, p2 ) if reconstruction: remaining_images -= set(reconstruction.shots) reconstruction, rec_report["grow"] = grow_reconstruction( data, tracks_manager, reconstruction, remaining_images, gcp, ) reconstructions.append(reconstruction) reconstructions = sorted(reconstructions, key=lambda x: -len(x.shots)) for k, r in enumerate(reconstructions): logger.info( "Reconstruction {}: {} images, {} points".format( k, len(r.shots), len(r.points) ) ) logger.info("{} partial reconstructions in total.".format(len(reconstructions))) chrono.lap("compute_reconstructions") report["wall_times"] = dict(chrono.lap_times()) report["not_reconstructed_images"] = list(remaining_images) return report, reconstructions
def is_high_res_panorama(data: DataSetBase, image_key, image_array): """Detect if image is a panorama.""" exif = data.load_exif(image_key) if exif: camera = data.load_camera_models()[exif["camera"]] w, h = int(exif["width"]), int(exif["height"]) exif_pano = pygeometry.Camera.is_panorama(camera.projection_type) elif image_array is not None: h, w = image_array.shape[:2] exif_pano = False else: return False return w == 2 * h or exif_pano
def run_dataset(data: DataSetBase, input, output): recs_base = data.load_reconstruction(input) if len(recs_base) == 0: return rec_base = recs_base[0] tracks_manager = data.load_tracks_manager() rec_base.add_correspondences_from_tracks_manager(tracks_manager) images = data.images() remaining_images = set(images) - set(rec_base.shots) gcp = data.load_ground_control_points() report = {} rec_report = {} report["extend_reconstruction"] = [rec_report] rec, rec_report["grow"] = reconstruction.grow_reconstruction( data, tracks_manager, rec_base, remaining_images, gcp, ) rec_report["num_remaining_images"] = len(remaining_images) report["not_reconstructed_images"] = list(remaining_images) data.save_reconstruction([rec], output) data.save_report(io.json_dumps(report), "reconstruction.json")
def load_mask(self, data: DataSetBase, image): points, _, _, segmentations, _ = self._load_all_data_unmasked( data, image) if data.config[ "features_bake_segmentation"] and segmentations is not None: ignore_values = set(data.segmentation_ignore_values(image)) return [ False if segmentations[i] in ignore_values else True for i in range(len(segmentations)) ] else: if points is None: return None return data.load_features_mask(image, points[:, :2])
def match_images_with_pairs(data: DataSetBase, config_override, exifs, pairs): """ Perform pair matchings given pairs. """ cameras = data.load_camera_models() args = list(match_arguments(pairs, data, config_override, cameras, exifs)) # Perform all pair matchings in parallel start = timer() logger.info("Matching {} image pairs".format(len(pairs))) mem_per_process = 512 jobs_per_process = 2 processes = context.processes_that_fit_in_memory(data.config["processes"], mem_per_process) logger.info("Computing pair matching with %d processes" % processes) matches = context.parallel_map(match_unwrap_args, args, processes, jobs_per_process) logger.info("Matched {} pairs {} in {} seconds ({} seconds/pair).".format( len(pairs), log_projection_types(pairs, exifs, cameras), timer() - start, (timer() - start) / len(pairs) if pairs else 0, )) # Index results per pair resulting_pairs = {} for im1, im2, m in matches: resulting_pairs[im1, im2] = m return resulting_pairs
def load_words(self, data: DataSetBase, image: str, masked: bool) -> np.ndarray: words = data.load_words(image) if masked: mask = self.load_mask(data, image) if mask is not None: words = words[mask] return words
def load_words(self, data: DataSetBase, image, masked): words = data.load_words(image) if masked: mask = self.load_mask(data, image) if mask is not None: words = words[mask] return words
def gcp_errors(data: DataSetBase, reconstructions): all_errors = [] gcp = data.load_ground_control_points() if not gcp: return {} all_errors = [] for gcp in gcp: if not gcp.coordinates.has_value: continue for rec in reconstructions: triangulated = multiview.triangulate_gcp(gcp, rec.shots, 1.0, 0.1) if triangulated is None: continue else: break # pyre-fixme[61]: `triangulated` may not be initialized here. if triangulated is None: continue all_errors.append(triangulated - gcp.coordinates.value) return _gps_gcp_errors_stats(all_errors)
def _load_features_nocache(self, data: DataSetBase, image): points, features, colors, segmentation_data = data.load_features(image) if points is None: logger.error("Could not load features for image {}".format(image)) else: points = np.array(points[:, :3], dtype=float) return points, features, colors, segmentation_data
def load_mask(self, data: DataSetBase, image: str) -> Optional[np.ndarray]: all_features_data = self._load_all_data_unmasked(data, image) if not all_features_data: return None if (data.config["features_bake_segmentation"] and all_features_data.semantic is not None): # pyre-fixme [16]: `Optional` has no attribute `segmentation` segmentations = all_features_data.semantic.segmentation ignore_values = set(data.segmentation_ignore_values(image)) return np.array([ False if segmentations[i] in ignore_values else True for i in range(len(segmentations)) ]) else: return data.load_features_mask(image, all_features_data.points[:, :2])
def match_images(data: DataSetBase, config_override, ref_images, cand_images): """Perform pair matchings between two sets of images. It will do matching for each pair (i, j), i being in ref_images and j in cand_images, taking assumption that matching(i, j) == matching(j ,i). This does not hold for non-symmetric matching options like WORDS. Data will be stored in i matching only. """ # Get EXIFs data all_images = list(set(ref_images + cand_images)) exifs = {im: data.load_exif(im) for im in all_images} # Generate pairs for matching pairs, preport = pairs_selection.match_candidates_from_metadata( ref_images, cand_images, exifs, data, config_override, ) # Match them ! return ( match_images_with_pairs(data, config_override, exifs, ref_images, pairs), preport, )
def _pair_reconstructability_arguments(track_dict, cameras, data: DataSetBase): threshold = 4 * data.config["five_point_algo_threshold"] args = [] for (im1, im2), (_, p1, p2) in track_dict.items(): camera1 = cameras[data.load_exif(im1)["camera"]] camera2 = cameras[data.load_exif(im2)["camera"]] args.append((im1, im2, p1, p2, camera1, camera2, threshold)) return args
def _reconstruction_from_rigs_and_assignments(data: DataSetBase): assignments = data.load_rig_assignments() models = data.load_rig_models() if not data.reference_lla_exists(): data.invent_reference_lla() base_rotation = np.zeros(3) reconstructions = [] for rig_id, instances in assignments.items(): rig_cameras = models[rig_id]["rig_cameras"] reconstruction = types.Reconstruction() reconstruction.cameras = data.load_camera_models() for instance in instances: for image, camera_id in instance: rig_camera = rig_cameras[camera_id] rig_pose = pygeometry.Pose(base_rotation) rig_pose.set_origin( orec.get_image_metadata(data, image).gps_position.value) rig_camera_pose = pygeometry.Pose(rig_camera["rotation"], rig_camera["translation"]) d = data.load_exif(image) shot = reconstruction.create_shot(image, d["camera"]) shot.pose = rig_camera_pose.compose(rig_pose) shot.metadata = orec.get_image_metadata(data, image) reconstructions.append(reconstruction) return reconstructions
def run_dataset(data: DataSetBase): """ Add delaunay meshes to the reconstruction. """ tracks_manager = data.load_tracks_manager() reconstructions = data.load_reconstruction() all_shot_ids = set(tracks_manager.get_shot_ids()) for r in reconstructions: for shot in r.shots.values(): if shot.id in all_shot_ids: vertices, faces = mesh.triangle_mesh(shot.id, r, tracks_manager) shot.mesh.vertices = vertices shot.mesh.faces = faces data.save_reconstruction(reconstructions, filename="reconstruction.meshed.json", minify=True)
def cameras_statistics(data: DataSetBase, reconstructions): stats = {} permutation = np.argsort([-len(r.shots) for r in reconstructions]) for camera_id, camera_model in data.load_camera_models().items(): stats[camera_id] = {"initial_values": _cameras_statistics(camera_model)} for idx in permutation: rec = reconstructions[idx] for camera in rec.cameras.values(): if "optimized_values" in stats[camera.id]: continue stats[camera.id]["optimized_values"] = _cameras_statistics(camera) for camera_id in data.load_camera_models(): if "optimized_values" not in stats[camera_id]: del stats[camera_id] return stats
def save_matches(data: DataSetBase, images_ref, matched_pairs): """Given pairwise matches (image 1, image 2) - > matches, save them such as only {image E images_ref} will store the matches. """ images_ref_set = set(images_ref) matches_per_im1 = {im: {} for im in images_ref} for (im1, im2), m in matched_pairs.items(): if im1 in images_ref_set: matches_per_im1[im1][im2] = m elif im2 in images_ref_set: matches_per_im1[im2][im1] = m else: raise RuntimeError( "Couldn't save matches for {}. No image found in images_ref.". format((im1, im2))) for im1, im1_matches in matches_per_im1.items(): data.save_matches(im1, im1_matches)
def write_report(data: DataSetBase, tracks_manager, features_time, matches_time, tracks_time): view_graph = [ (k[0], k[1], v) for k, v in tracks_manager.get_all_pairs_connectivity().items() ] report = { "wall_times": { "load_features": features_time, "load_matches": matches_time, "compute_tracks": tracks_time, }, "wall_time": features_time + matches_time + tracks_time, "num_images": tracks_manager.num_shots(), "num_tracks": tracks_manager.num_tracks(), "view_graph": view_graph, } data.save_report(io.json_dumps(report), "tracks.json")
def _load_features_nocache(self, data: DataSetBase, image: str) -> Optional[ft.FeaturesData]: features_data = data.load_features(image) if features_data is None: logger.error("Could not load features for image {}".format(image)) return None else: features_data.points = np.array(features_data.points[:, :3], dtype=float) return features_data
def _not_on_blackvue_watermark(p1, p2, matches, im1, im2, data: DataSetBase): """Filter Blackvue's watermark.""" meta1 = data.load_exif(im1) meta2 = data.load_exif(im2) if meta1["make"].lower() == "blackvue": matches = [m for m in matches if _blackvue_valid_mask(p1[m[0]])] if meta2["make"].lower() == "blackvue": matches = [m for m in matches if _blackvue_valid_mask(p2[m[1]])] return matches
def _not_on_vermont_watermark(p1, p2, matches, im1, im2, data: DataSetBase): """Filter Vermont images watermark.""" meta1 = data.load_exif(im1) meta2 = data.load_exif(im2) if meta1["make"] == "VTrans_Camera" and meta1["model"] == "VTrans_Camera": matches = [m for m in matches if _vermont_valid_mask(p1[m[0]])] if meta2["make"] == "VTrans_Camera" and meta2["model"] == "VTrans_Camera": matches = [m for m in matches if _vermont_valid_mask(p2[m[1]])] return matches
def features_statistics(data: DataSetBase, tracks_manager, reconstructions): stats = {} detected = [] for im in data.images(): features_data = data.load_features(im) if not features_data: continue detected.append(len(features_data.points)) if len(detected) > 0: stats["detected_features"] = { "min": min(detected), "max": max(detected), "mean": int(np.mean(detected)), "median": int(np.median(detected)), } else: stats["detected_features"] = { "min": -1, "max": -1, "mean": -1, "median": -1 } per_shots = defaultdict(int) for rec in reconstructions: all_points_keys = set(rec.points.keys()) for shot_id in rec.shots: if shot_id not in tracks_manager.get_shot_ids(): continue for point_id in tracks_manager.get_shot_observations(shot_id): if point_id not in all_points_keys: continue per_shots[shot_id] += 1 per_shots = list(per_shots.values()) stats["reconstructed_features"] = { "min": int(min(per_shots)) if len(per_shots) > 0 else -1, "max": int(max(per_shots)) if len(per_shots) > 0 else -1, "mean": int(np.mean(per_shots)) if len(per_shots) > 0 else -1, "median": int(np.median(per_shots)) if len(per_shots) > 0 else -1, } return stats
def load_matches(dataset: DataSetBase, images): matches = {} for im1 in images: try: im1_matches = dataset.load_matches(im1) except IOError: continue for im2 in im1_matches: if im2 in images: matches[im1, im2] = im1_matches[im2] return matches