def triangle_mesh_spherical( shot_id: str, r: types.Reconstruction, tracks_manager: pymap.TracksManager) -> Tuple[List[Any], List[Any]]: shot = r.shots[shot_id] bearings = [] vertices = [] # Add vertices to ensure that the camera is inside the convex hull # of the points for point in itertools.product([-1, 1], repeat=3): # vertices of a cube bearing = 0.3 * np.array(point) / np.linalg.norm(point) bearings.append(bearing) point = shot.pose.transform_inverse(bearing) vertices.append(point.tolist()) for track_id in tracks_manager.get_shot_observations(shot_id): if track_id in r.points: point = r.points[track_id].coordinates direction = shot.pose.transform(point) pixel = direction / np.linalg.norm(direction) if not np.isnan(pixel).any(): vertices.append(point) bearings.append(pixel.tolist()) tri = scipy.spatial.ConvexHull(bearings) faces = tri.simplices.tolist() return vertices, faces
def retriangulate( tracks_manager: pymap.TracksManager, reconstruction: types.Reconstruction, config: Dict[str, Any], ) -> Dict[str, Any]: """Retrianguate all points""" chrono = Chronometer() report = {} report["num_points_before"] = len(reconstruction.points) threshold = config["triangulation_threshold"] min_ray_angle = config["triangulation_min_ray_angle"] reconstruction.points = {} all_shots_ids = set(tracks_manager.get_shot_ids()) triangulator = TrackTriangulator(tracks_manager, reconstruction) tracks = set() for image in reconstruction.shots.keys(): if image in all_shots_ids: tracks.update(tracks_manager.get_shot_observations(image).keys()) for track in tracks: if config["triangulation_type"] == "ROBUST": triangulator.triangulate_robust(track, threshold, min_ray_angle) elif config["triangulation_type"] == "FULL": triangulator.triangulate(track, threshold, min_ray_angle) report["num_points_after"] = len(reconstruction.points) chrono.lap("retriangulate") report["wall_time"] = chrono.total_time() return report
def add_pano_subshot_tracks( tracks_manager: pymap.TracksManager, utracks_manager: pymap.TracksManager, panoshot: pymap.Shot, perspectiveshot: pymap.Shot, ) -> None: """Add edges between subshots and visible tracks.""" for track_id, obs in tracks_manager.get_shot_observations(panoshot.id).items(): bearing = panoshot.camera.pixel_bearing(obs.point) rotation = np.dot( perspectiveshot.pose.get_rotation_matrix(), panoshot.pose.get_rotation_matrix().T, ) rotated_bearing = np.dot(bearing, rotation.T) if rotated_bearing[2] <= 0: continue perspective_feature = perspectiveshot.camera.project(rotated_bearing) if ( perspective_feature[0] < -0.5 or perspective_feature[0] > 0.5 or perspective_feature[1] < -0.5 or perspective_feature[1] > 0.5 ): continue obs.point = perspective_feature utracks_manager.add_observation(perspectiveshot.id, track_id, obs)
def compute_common_tracks( reconstruction1: types.Reconstruction, reconstruction2: types.Reconstruction, tracks_manager1: pymap.TracksManager, tracks_manager2: pymap.TracksManager, ) -> List[Tuple[str, str]]: common_tracks = set() common_images = set(reconstruction1.shots.keys()).intersection( reconstruction2.shots.keys()) all_shot_ids1 = set(tracks_manager1.get_shot_ids()) all_shot_ids2 = set(tracks_manager2.get_shot_ids()) for image in common_images: if image not in all_shot_ids1 or image not in all_shot_ids2: continue at_shot1 = tracks_manager1.get_shot_observations(image) at_shot2 = tracks_manager2.get_shot_observations(image) for t1, t2 in corresponding_tracks(at_shot1, at_shot2): if t1 in reconstruction1.points and t2 in reconstruction2.points: common_tracks.add((t1, t2)) return list(common_tracks)
def add_subshot_tracks( tracks_manager: pymap.TracksManager, utracks_manager: pymap.TracksManager, shot: pymap.Shot, subshot: pymap.Shot, ) -> None: """Add shot tracks to the undistorted tracks_manager.""" if shot.id not in tracks_manager.get_shot_ids(): return if pygeometry.Camera.is_panorama(shot.camera.projection_type): add_pano_subshot_tracks(tracks_manager, utracks_manager, shot, subshot) else: for track_id, obs in tracks_manager.get_shot_observations(shot.id).items(): utracks_manager.add_observation(subshot.id, track_id, obs)
def features_statistics( data: DataSetBase, tracks_manager: pymap.TracksManager, reconstructions: List[types.Reconstruction], ) -> Dict[str, Any]: stats = {} detected = [] images = {s for r in reconstructions for s in r.shots} for im in images: features_data = feature_loader.instance.load_all_data( data, im, False, False) if not features_data: continue detected.append(len(features_data.points)) if len(detected) > 0: stats["detected_features"] = { "min": min(detected), "max": max(detected), "mean": int(np.mean(detected)), "median": int(np.median(detected)), } else: stats["detected_features"] = { "min": -1, "max": -1, "mean": -1, "median": -1 } per_shots = defaultdict(int) for rec in reconstructions: all_points_keys = set(rec.points.keys()) for shot_id in rec.shots: if shot_id not in tracks_manager.get_shot_ids(): continue for point_id in tracks_manager.get_shot_observations(shot_id): if point_id not in all_points_keys: continue per_shots[shot_id] += 1 per_shots = list(per_shots.values()) stats["reconstructed_features"] = { "min": int(min(per_shots)) if len(per_shots) > 0 else -1, "max": int(max(per_shots)) if len(per_shots) > 0 else -1, "mean": int(np.mean(per_shots)) if len(per_shots) > 0 else -1, "median": int(np.median(per_shots)) if len(per_shots) > 0 else -1, } return stats
def triangle_mesh_fisheye( shot_id: str, r: types.Reconstruction, tracks_manager: pymap.TracksManager) -> Tuple[List[Any], List[Any]]: shot = r.shots[shot_id] bearings = [] vertices = [] # Add boundary vertices num_circle_points = 20 for i in range(num_circle_points): a = 2 * np.pi * float(i) / num_circle_points point = 30 * np.array([np.cos(a), np.sin(a), 0]) bearing = point / np.linalg.norm(point) point = shot.pose.transform_inverse(point) vertices.append(point.tolist()) bearings.append(bearing) # Add a single vertex in front of the camera point = 30 * np.array([0, 0, 1]) bearing = 0.3 * point / np.linalg.norm(point) point = shot.pose.transform_inverse(point) vertices.append(point.tolist()) bearings.append(bearing) # Add reconstructed points for track_id in tracks_manager.get_shot_observations(shot_id): if track_id in r.points: point = r.points[track_id].coordinates direction = shot.pose.transform(point) pixel = direction / np.linalg.norm(direction) if not np.isnan(pixel).any(): vertices.append(point) bearings.append(pixel.tolist()) # Triangulate tri = scipy.spatial.ConvexHull(bearings) faces = tri.simplices.tolist() # Remove faces having only boundary vertices def good_face(face): return (face[0] >= num_circle_points or face[1] >= num_circle_points or face[2] >= num_circle_points) faces = list(filter(good_face, faces)) return vertices, faces
def triangle_mesh_perspective( shot_id: str, r: types.Reconstruction, tracks_manager: pymap.TracksManager) -> Tuple[List[Any], List[Any]]: shot = r.shots[shot_id] cam = shot.camera dx = float(cam.width) / 2 / max(cam.width, cam.height) dy = float(cam.height) / 2 / max(cam.width, cam.height) pixels = [[-dx, -dy], [-dx, dy], [dx, dy], [dx, -dy]] vertices = [None for i in range(4)] for track_id in tracks_manager.get_shot_observations(shot_id): if track_id in r.points: point = r.points[track_id] pixel = shot.project(point.coordinates) nonans = not np.isnan(pixel).any() if nonans and -dx <= pixel[0] <= dx and -dy <= pixel[1] <= dy: vertices.append(point.coordinates) pixels.append(pixel.tolist()) try: tri = scipy.spatial.Delaunay(pixels) except Exception as e: logger.error("Delaunay triangulation failed for input: {}".format( repr(pixels))) raise e sums = [0.0, 0.0, 0.0, 0.0] depths = [0.0, 0.0, 0.0, 0.0] for t in tri.simplices: for i in range(4): if i in t: for j in t: if j >= 4: depths[i] += shot.pose.transform(vertices[j])[2] sums[i] += 1 for i in range(4): if sums[i] > 0: d = depths[i] / sums[i] else: d = 50.0 vertices[i] = back_project_no_distortion(shot, pixels[i], d).tolist() faces = tri.simplices.tolist() return vertices, faces
def triangulate_shot_features( tracks_manager: pymap.TracksManager, reconstruction: types.Reconstruction, shot_ids: Set[str], config: Dict[str, Any], ) -> None: """Reconstruct as many tracks seen in shot_id as possible.""" reproj_threshold = config["triangulation_threshold"] min_ray_angle = config["triangulation_min_ray_angle"] triangulator = TrackTriangulator(tracks_manager, reconstruction) all_shots_ids = set(tracks_manager.get_shot_ids()) tracks_ids = { t for s in shot_ids if s in all_shots_ids for t in tracks_manager.get_shot_observations(s) } for track in tracks_ids: if track not in reconstruction.points: triangulator.triangulate(track, reproj_threshold, min_ray_angle)
def common_tracks( tracks_manager: pymap.TracksManager, im1: str, im2: str ) -> t.Tuple[t.List[str], np.ndarray, np.ndarray]: """List of tracks observed in both images. Args: tracks_manager: tracks manager im1: name of the first image im2: name of the second image Returns: tuple: tracks, feature from first image, feature from second image """ t1 = tracks_manager.get_shot_observations(im1) t2 = tracks_manager.get_shot_observations(im2) tracks, p1, p2 = [], [], [] for track, obs in t1.items(): if track in t2: p1.append(obs.point) p2.append(t2[track].point) tracks.append(track) p1 = np.array(p1) p2 = np.array(p2) return tracks, p1, p2
def resect( data: DataSetBase, tracks_manager: pymap.TracksManager, reconstruction: types.Reconstruction, shot_id: str, threshold: float, min_inliers: int, ) -> Tuple[bool, Set[str], Dict[str, Any]]: """Try resecting and adding a shot to the reconstruction. Return: True on success. """ rig_assignments = data.load_rig_assignments_per_image() camera = reconstruction.cameras[data.load_exif(shot_id)["camera"]] bs, Xs, ids = [], [], [] for track, obs in tracks_manager.get_shot_observations(shot_id).items(): if track in reconstruction.points: b = camera.pixel_bearing(obs.point) bs.append(b) Xs.append(reconstruction.points[track].coordinates) ids.append(track) bs = np.array(bs) Xs = np.array(Xs) if len(bs) < 5: return False, set(), {"num_common_points": len(bs)} T = multiview.absolute_pose_ransac(bs, Xs, threshold, 1000, 0.999) R = T[:, :3] t = T[:, 3] reprojected_bs = R.T.dot((Xs - t).T).T reprojected_bs /= np.linalg.norm(reprojected_bs, axis=1)[:, np.newaxis] inliers = np.linalg.norm(reprojected_bs - bs, axis=1) < threshold ninliers = int(sum(inliers)) logger.info("{} resection inliers: {} / {}".format(shot_id, ninliers, len(bs))) report = { "num_common_points": len(bs), "num_inliers": ninliers, } if ninliers >= min_inliers: R = T[:, :3].T t = -R.dot(T[:, 3]) assert shot_id not in reconstruction.shots new_shots = add_shot(data, reconstruction, rig_assignments, shot_id, pygeometry.Pose(R, t)) if shot_id in rig_assignments: triangulate_shot_features(tracks_manager, reconstruction, new_shots, data.config) for i, succeed in enumerate(inliers): if succeed: add_observation_to_reconstruction(tracks_manager, reconstruction, shot_id, ids[i]) # pyre-fixme [6]: Expected `int` for 2nd positional report["shots"] = list(new_shots) return True, new_shots, report else: return False, set(), report