def _reconstruction_from_rigs_and_assignments(data: DataSetBase): assignments = data.load_rig_assignments() rig_cameras = data.load_rig_cameras() data.init_reference() reconstruction = types.Reconstruction() reconstruction.cameras = data.load_camera_models() for rig_instance_id, instance in assignments.items(): for image, rig_camera_id in instance: rig_camera = rig_cameras[rig_camera_id] reconstruction.add_rig_camera( pymap.RigCamera(rig_camera.pose, rig_camera_id)) instance_obj = reconstruction.add_rig_instance( pymap.RigInstance(rig_instance_id)) instance_obj.pose.set_origin( helpers.get_image_metadata(data, image).gps_position.value) d = data.load_exif(image) shot = reconstruction.create_shot( image, camera_id=d["camera"], rig_camera_id=rig_camera_id, rig_instance_id=rig_instance_id, ) shot.metadata = helpers.get_image_metadata(data, image) return [reconstruction]
def create_shot( self, shot_id: str, camera_id: str, pose: Optional[pygeometry.Pose] = None, rig_camera_id: Optional[str] = None, rig_instance_id: Optional[str] = None, ) -> pymap.Shot: passed_rig_camera_id = rig_camera_id if rig_camera_id else camera_id passed_rig_instance_id = rig_instance_id if rig_instance_id else shot_id if (not rig_camera_id) and (camera_id not in self.rig_cameras): self.add_rig_camera(pymap.RigCamera(pygeometry.Pose(), camera_id)) if passed_rig_camera_id not in self.rig_cameras: raise RuntimeError( f"Rig Camera {passed_rig_camera_id} doesn't exist in reconstruction" ) if (not rig_instance_id) and (shot_id not in self.rig_instances): self.add_rig_instance(pymap.RigInstance(shot_id)) if passed_rig_instance_id not in self.rig_instances: raise RuntimeError( f"Rig Instance {passed_rig_instance_id} doesn't exist in reconstruction" ) if pose is None: created_shot = self.map.create_shot(shot_id, camera_id, passed_rig_camera_id, passed_rig_instance_id) else: created_shot = self.map.create_shot(shot_id, camera_id, passed_rig_camera_id, passed_rig_instance_id, pose) return created_shot
def default_rig_cameras( camera_ids: Iterable[str]) -> Dict[str, pymap.RigCamera]: """Return per-camera models default rig cameras (identity pose).""" default_rig_cameras = {} for camera_id in camera_ids: default_rig_cameras[camera_id] = pymap.RigCamera( pygeometry.Pose(), camera_id) return default_rig_cameras
def rig_camera_from_json(key, obj): """ Read a rig cameras from a json object """ pose = pygeometry.Pose() pose.rotation = obj["rotation"] pose.translation = obj["translation"] rig_camera = pymap.RigCamera(pose, key) return rig_camera
def compute_relative_pose( pose_instances: List[List[Tuple[pymap.Shot, str]]], ) -> Dict[str, pymap.RigCamera]: """ Compute a rig model relatives poses given poses grouped by rig instance. """ # Put all poses instances into some canonical frame taken as the mean of their R|t centered_pose_instances = [] for instance in pose_instances: origin_center = np.zeros(3) rotation_center = np.zeros(3) for shot, _ in instance: rotation_center += shot.pose.rotation origin_center += shot.pose.get_origin() origin_center /= len(instance) rotation_center /= len(instance) centered_pose_instance = [] for shot, rig_camera_id in instance: instance_pose = pygeometry.Pose(rotation_center) instance_pose.set_origin(origin_center) instance_pose_camera = shot.pose.relative_to(instance_pose) centered_pose_instance.append( ( instance_pose_camera, rig_camera_id, shot.camera.id, ) ) centered_pose_instances.append(centered_pose_instance) # Average canonical poses per RigCamera ID average_origin, average_rotation, count_poses, camera_ids = {}, {}, {}, {} for centered_pose_instance in centered_pose_instances: for pose, rig_camera_id, camera_id in centered_pose_instance: if rig_camera_id not in average_origin: average_origin[rig_camera_id] = np.zeros(3) average_rotation[rig_camera_id] = np.zeros(3) count_poses[rig_camera_id] = 0 average_origin[rig_camera_id] += pose.get_origin() average_rotation[rig_camera_id] += pose.rotation camera_ids[rig_camera_id] = camera_id count_poses[rig_camera_id] += 1 # Construct final RigCamera results rig_cameras: Dict[str, pymap.RigCamera] = {} for rig_camera_id, count in count_poses.items(): o = average_origin[rig_camera_id] / count r = average_rotation[rig_camera_id] / count pose = pygeometry.Pose(r) pose.set_origin(o) rig_cameras[rig_camera_id] = pymap.RigCamera(pose, rig_camera_id) return rig_cameras
def perspective_views_of_a_panorama( spherical_shot: pymap.Shot, width: int, reconstruction: types.Reconstruction, image_format: str, rig_instance_count: Iterator[int], ) -> List[pymap.Shot]: """Create 6 perspective views of a panorama.""" camera = pygeometry.Camera.create_perspective(0.5, 0.0, 0.0) camera.id = "perspective_panorama_camera" camera.width = width camera.height = width reconstruction.add_camera(camera) names = ["front", "left", "back", "right", "top", "bottom"] rotations = [ tf.rotation_matrix(-0 * np.pi / 2, np.array([0, 1, 0])), tf.rotation_matrix(-1 * np.pi / 2, np.array([0, 1, 0])), tf.rotation_matrix(-2 * np.pi / 2, np.array([0, 1, 0])), tf.rotation_matrix(-3 * np.pi / 2, np.array([0, 1, 0])), tf.rotation_matrix(-np.pi / 2, np.array([1, 0, 0])), tf.rotation_matrix(+np.pi / 2, np.array([1, 0, 0])), ] rig_instance = reconstruction.add_rig_instance( pymap.RigInstance(str(next(rig_instance_count))) ) shots = [] for name, rotation in zip(names, rotations): if name not in reconstruction.rig_cameras: rig_camera_pose = pygeometry.Pose() rig_camera_pose.set_rotation_matrix(rotation[:3, :3]) rig_camera = pymap.RigCamera(rig_camera_pose, name) reconstruction.add_rig_camera(rig_camera) rig_camera = reconstruction.rig_cameras[name] shot_id = add_image_format_extension( f"{spherical_shot.id}_perspective_view_{name}", image_format ) shot = reconstruction.create_shot( shot_id, camera.id, pygeometry.Pose(), rig_camera.id, rig_instance.id ) shot.metadata = spherical_shot.metadata shots.append(shot) rig_instance.pose = spherical_shot.pose return shots
def add_camera_sequence( self, camera: pygeometry.Camera, length: float, height: float, interval: float, position_noise: List[float], rotation_noise: float, positions_shift: Optional[List[float]] = None, end: Optional[float] = None, ) -> "SyntheticStreetScene": default_noise_interval = 0.25 * interval actual_end = length if end is None else end generator = self.generator assert generator positions, rotations = sg.generate_cameras( sg.samples_generator_interval(length, actual_end, interval, default_noise_interval), generator, height, ) sg.perturb_points(positions, position_noise) sg.perturb_rotations(rotations, rotation_noise) if positions_shift: positions += np.array(positions_shift) shift = 0 if len(self.shot_ids) == 0 else sum( len(s) for s in self.shot_ids) new_shot_ids = [f"Shot {shift+i:04d}" for i in range(len(positions))] self.shot_ids.append(new_shot_ids) self.cameras.append([camera]) rig_camera = pymap.RigCamera(pygeometry.Pose(), camera.id) self.rig_cameras.append([rig_camera]) rig_instances = [] for shot_id in new_shot_ids: rig_instances.append([(shot_id, camera.id)]) self.rig_instances.append(rig_instances) self.instances_positions.append(positions) self.instances_rotations.append(rotations) return self
def create_pano_shot(self, shot_id: str, camera_id: str, pose: Optional[pygeometry.Pose] = None) -> pymap.Shot: if pose is None: pose = pygeometry.Pose() rig_camera_id = f"{PANOSHOT_RIG_PREFIX}{camera_id}" if rig_camera_id not in self.rig_cameras: self.add_rig_camera( pymap.RigCamera(pygeometry.Pose(), rig_camera_id)) rig_instance_id = f"{PANOSHOT_RIG_PREFIX}{shot_id}" if rig_instance_id not in self.rig_instances: self.add_rig_instance(pymap.RigInstance(rig_instance_id)) created_shot = self.map.create_pano_shot(shot_id, camera_id, rig_camera_id, rig_instance_id, pose) return created_shot
def perspective_views_of_a_panorama(spherical_shot, width, reconstruction, image_format, rig_instance_count): """Create 6 perspective views of a panorama.""" camera = pygeometry.Camera.create_perspective(0.5, 0.0, 0.0) camera.id = "perspective_panorama_camera" camera.width = width camera.height = width reconstruction.add_camera(camera) names = ["front", "left", "back", "right", "top", "bottom"] rotations = [ tf.rotation_matrix(-0 * np.pi / 2, (0, 1, 0)), tf.rotation_matrix(-1 * np.pi / 2, (0, 1, 0)), tf.rotation_matrix(-2 * np.pi / 2, (0, 1, 0)), tf.rotation_matrix(-3 * np.pi / 2, (0, 1, 0)), tf.rotation_matrix(-np.pi / 2, (1, 0, 0)), tf.rotation_matrix(+np.pi / 2, (1, 0, 0)), ] rig_instance = pymap.RigInstance(next(rig_instance_count)) rig_instance.pose = spherical_shot.pose shots = [] for name, rotation in zip(names, rotations): if name not in reconstruction.rig_cameras: rig_camera_pose = pygeometry.Pose() rig_camera_pose.set_rotation_matrix(rotation[:3, :3]) rig_camera = pymap.RigCamera(rig_camera_pose, name) reconstruction.add_rig_camera(rig_camera) rig_camera = reconstruction.rig_cameras[name] shot_id = add_image_format_extension( f"{spherical_shot.id}_perspective_view_{name}", image_format) shot = reconstruction.create_shot(shot_id, camera.id) rig_instance.add_shot(rig_camera, shot) shots.append(shot) reconstruction.add_rig_instance(rig_instance) return shots
def test_many_observations_delete() -> None: # Given a map with 10 shots, 1000 landmarks ... m = pymap.Map() n_cams = 2 n_shots = 10 n_landmarks = 1000 for cam_id in range(n_cams): cam = pygeometry.Camera.create_perspective(0.5, 0, 0) cam.id = "cam" + str(cam_id) m.create_camera(cam) m.create_rig_camera(pymap.RigCamera(pygeometry.Pose(), cam.id)) for shot_id in range(n_shots): cam_id = "cam" + str(int(np.random.rand(1) * 10 % n_cams)) shot_id = str(shot_id) m.create_rig_instance(shot_id) m.create_shot(shot_id, cam_id, cam_id, shot_id, pygeometry.Pose()) for point_id in range(n_landmarks): m.create_landmark(str(point_id), np.random.rand(3)) # ... and random connections (observations) between shots and points n_total_obs = 0 for lm in m.get_landmarks().values(): n_obs = 0 for shot in m.get_shots().values(): # create a new observation obs = pymap.Observation(100, 200, 0.5, 255, 0, 0, int(lm.id)) m.add_observation(shot, lm, obs) n_obs += 1 n_total_obs += 1 # (we expect it to be created correctly) for lm in m.get_landmarks().values(): n_total_obs -= lm.number_of_observations() assert n_total_obs == 0 # and when we clear all the observations m.clear_observations_and_landmarks()
def test_clean_landmarks_with_min_observations() -> None: m = pymap.Map() n_cams = 2 n_shots = 2 n_landmarks = 10 for cam_id in range(n_cams): cam = pygeometry.Camera.create_perspective(0.5, 0, 0) cam.id = "cam" + str(cam_id) m.create_camera(cam) m.create_rig_camera(pymap.RigCamera(pygeometry.Pose(), cam.id)) for shot_id in range(n_shots): cam_id = "cam" + str(int(np.random.rand(1) * 10 % n_cams)) m.create_rig_instance(str(shot_id)) m.create_shot(str(shot_id), cam_id, cam_id, str(shot_id), pygeometry.Pose()) for point_id in range(n_landmarks): m.create_landmark(str(point_id), np.random.rand(3)) for point_id in range(int(n_landmarks / 2)): for shot in m.get_shots().values(): # create a new observation obs = pymap.Observation(100, 200, 0.5, 255, 0, 0, point_id) m.add_observation(shot, m.get_landmark(str(point_id)), obs) for point_id in range(int(n_landmarks / 2), n_landmarks): shot = m.get_shot("0") # create a new observation obs = pymap.Observation(100, 200, 0.5, 255, 0, 0, point_id) m.add_observation(shot, m.get_landmark(str(point_id)), obs) m.clean_landmarks_below_min_observations(n_shots) assert len(m.get_landmarks()) == int(n_landmarks / 2) m.clean_landmarks_below_min_observations(n_shots + 1) assert len(m.get_landmarks()) == 0
def reconstruction_from_metadata(data: DataSetBase, images: Iterable[str]) -> types.Reconstruction: """Initialize a reconstruction by using EXIF data for constructing shot poses and cameras.""" data.init_reference() rig_assignments = rig.rig_assignments_per_image(data.load_rig_assignments()) reconstruction = types.Reconstruction() reconstruction.reference = data.load_reference() reconstruction.cameras = data.load_camera_models() for image in images: camera_id = data.load_exif(image)["camera"] if image in rig_assignments: rig_instance_id, rig_camera_id, _ = rig_assignments[image] else: rig_instance_id = image rig_camera_id = camera_id reconstruction.add_rig_camera(pymap.RigCamera(pygeometry.Pose(), rig_camera_id)) reconstruction.add_rig_instance(pymap.RigInstance(rig_instance_id)) shot = reconstruction.create_shot( shot_id=image, camera_id=camera_id, rig_camera_id=rig_camera_id, rig_instance_id=rig_instance_id, ) shot.metadata = get_image_metadata(data, image) if not shot.metadata.gps_position.has_value: reconstruction.remove_shot(image) continue gps_pos = shot.metadata.gps_position.value shot.pose.set_rotation_matrix(rotation_from_shot_metadata(shot)) shot.pose.set_origin(gps_pos) shot.scale = 1.0 return reconstruction
def add_rig_camera_sequence( self, cameras: List[pygeometry.Camera], relative_positions: List[List[float]], relative_rotations: List[List[float]], start: float, length: float, height: float, interval: float, position_noise: List[float], rotation_noise: float, ): default_noise_interval = 0.25 * interval instances_positions, instances_rotations = sg.generate_cameras( sg.samples_generator_interval(start, length, interval, default_noise_interval), self.generator, # pyre-fixme [6] height, ) sg.perturb_points(instances_positions, position_noise) sg.perturb_rotations(instances_rotations, rotation_noise) shots_ids_per_camera = [] for rig_camera_p, rig_camera_r, camera in zip(relative_positions, relative_rotations, cameras): pose_rig_camera = pygeometry.Pose(rig_camera_r) pose_rig_camera.set_origin(rig_camera_p) rotations = [] positions = [] for instance_p, instance_r in zip(instances_positions, instances_rotations): pose_instance = pygeometry.Pose(instance_r) pose_instance.set_origin(instance_p) composed = pose_rig_camera.compose(pose_instance) rotations.append(composed.rotation) positions.append(composed.get_origin()) self.shot_rotations.append(np.array(rotations)) self.shot_positions.append(np.array(positions)) shift = sum(len(s) for s in shots_ids_per_camera) shots_ids_per_camera.append( [f"Shot {shift+i:04d}" for i in range(len(positions))]) self.cameras.append(camera) self.shot_ids += shots_ids_per_camera rig_camera_ids = [] rig_cameras = [] rig_camera_id_shift = sum(len(s) for s in self.rig_cameras) for i, (rig_camera_p, rig_camera_r) in enumerate( zip(relative_positions, relative_rotations)): pose_rig_camera = pygeometry.Pose(rig_camera_r) pose_rig_camera.set_origin(rig_camera_p) rig_camera_id = f"RigCamera {rig_camera_id_shift + i}" rig_camera = pymap.RigCamera(pose_rig_camera, rig_camera_id) rig_camera_ids.append(rig_camera_id) rig_cameras.append(rig_camera) self.rig_cameras.append(rig_cameras) rig_instances = [] for i in range(len(instances_positions)): instance = [] for j in range(len(shots_ids_per_camera)): instance.append( (shots_ids_per_camera[j][i], rig_camera_ids[j])) rig_instances.append(instance) self.rig_instances.append(rig_instances) self.instances_positions.append(instances_positions) self.instances_rotations.append(instances_rotations) return self
def add_rig_camera_sequence( self, cameras, relative_positions, relative_rotations, start, length, height, interval, position_noise=None, rotation_noise=None, gps_noise=None, ): default_noise_interval = 0.25 * interval instances_positions, instances_rotations = sg.generate_cameras( sg.samples_generator_interval(start, length, interval, default_noise_interval), self.generator, height, ) sg.perturb_points(instances_positions, position_noise) sg.perturb_rotations(instances_rotations, rotation_noise) shots_ids_per_camera = [] for rig_camera_p, rig_camera_r, camera in zip(relative_positions, relative_rotations, cameras): pose_rig_camera = pygeometry.Pose(rig_camera_r) pose_rig_camera.set_origin(rig_camera_p) rotations = [] positions = [] for instance_p, instance_r in zip(instances_positions, instances_rotations): pose_instance = pygeometry.Pose(instance_r) pose_instance.set_origin(instance_p) composed = pose_rig_camera.compose(pose_instance) rotations.append(composed.rotation) positions.append(composed.get_origin()) self.shot_rotations.append(rotations) self.shot_positions.append(positions) shift = sum(len(s) for s in shots_ids_per_camera) shots_ids_per_camera.append( [f"Shot {shift+i:04d}" for i in range(len(positions))]) self.cameras.append(camera) self.shot_ids += shots_ids_per_camera rig_camera_ids = [] rig_model = pymap.RigModel(f"RigModel {len(self.rig_models)}") for i, (rig_camera_p, rig_camera_r) in enumerate( zip(relative_positions, relative_rotations)): pose_rig_camera = pygeometry.Pose(rig_camera_r) pose_rig_camera.set_origin(rig_camera_p) rig_camera_id = f"RigCamera {i}" rig_camera = pymap.RigCamera(pose_rig_camera, rig_camera_id) rig_model.add_rig_camera(rig_camera) rig_camera_ids.append(rig_camera_id) self.rig_models.append(rig_model) rig_instances = [] for i in range(len(instances_positions)): instance = [] for j in range(len(shots_ids_per_camera)): instance.append( (shots_ids_per_camera[j][i], rig_camera_ids[j])) rig_instances.append(instance) self.rig_instances.append(rig_instances) self.instances_positions.append(instances_positions) self.instances_rotations.append(instances_rotations) return self
def _create_rig_camera(): rig_camera = pymap.RigCamera() rig_camera.id = "rig_camera" return rig_camera
def _create_rig_camera(): rig_camera = pymap.RigCamera() rig_camera.id = "rig_camera" rig_camera.pose = pygeometry.Pose(np.array([0.1, 0.2, 0.3]), np.array([0.1, 0.2, 0.3])) return rig_camera
def add_rig_camera_sequence( self, cameras: List[pygeometry.Camera], relative_positions: List[List[float]], relative_rotations: List[List[float]], length: float, height: float, interval: float, position_noise: List[float], rotation_noise: float, end: Optional[float] = None, ) -> "SyntheticStreetScene": default_noise_interval = 0.25 * interval actual_end = length if end is None else end generator = self.generator assert generator instances_positions, instances_rotations = sg.generate_cameras( sg.samples_generator_interval(length, actual_end, interval, default_noise_interval), generator, height, ) sg.perturb_points(instances_positions, position_noise) sg.perturb_rotations(instances_rotations, rotation_noise) shots_ids_per_camera = [] for rig_camera_p, rig_camera_r in zip(relative_positions, relative_rotations): pose_rig_camera = pygeometry.Pose(rig_camera_r) pose_rig_camera.set_origin(rig_camera_p) rotations = [] positions = [] for instance_p, instance_r in zip(instances_positions, instances_rotations): pose_instance = pygeometry.Pose(instance_r) pose_instance.set_origin(instance_p) composed = pose_rig_camera.compose(pose_instance) rotations.append(composed.rotation) positions.append(composed.get_origin()) shift = sum(len(s) for s in shots_ids_per_camera) shots_ids_per_camera.append( [f"Shot {shift+i:04d}" for i in range(len(positions))]) self.cameras.append(cameras) self.shot_ids += shots_ids_per_camera rig_camera_ids = [] rig_cameras = [] rig_camera_id_shift = sum(len(s) for s in self.rig_cameras) for i, (rig_camera_p, rig_camera_r) in enumerate( zip(relative_positions, relative_rotations)): pose_rig_camera = pygeometry.Pose(rig_camera_r) pose_rig_camera.set_origin(rig_camera_p) rig_camera_id = f"RigCamera {rig_camera_id_shift + i}" rig_camera = pymap.RigCamera(pose_rig_camera, rig_camera_id) rig_camera_ids.append(rig_camera_id) rig_cameras.append(rig_camera) self.rig_cameras.append(rig_cameras) rig_instances: List[List[Tuple[str, str]]] = [] for i in range(len(instances_positions)): instance = [] for j in range(len(shots_ids_per_camera)): instance.append( (shots_ids_per_camera[j][i], rig_camera_ids[j])) rig_instances.append(instance) self.rig_instances.append(rig_instances) self.instances_positions.append(instances_positions) self.instances_rotations.append(instances_rotations) return self
def _create_rig_model(): rig_model = pymap.RigModel("rig_model") rig_camera = pymap.RigCamera() rig_camera.id = "rig_camera" rig_model.add_rig_camera(rig_camera) return rig_model