def _reconstruction_from_rigs_and_assignments(data: DataSetBase):
assignments = data.load_rig_assignments()
rig_cameras = data.load_rig_cameras()
data.init_reference()
reconstruction = types.Reconstruction()
reconstruction.cameras = data.load_camera_models()
for rig_instance_id, instance in assignments.items():
for image, rig_camera_id in instance:
rig_camera = rig_cameras[rig_camera_id]
reconstruction.add_rig_camera(
pymap.RigCamera(rig_camera.pose, rig_camera_id))
instance_obj = reconstruction.add_rig_instance(
pymap.RigInstance(rig_instance_id))
instance_obj.pose.set_origin(
helpers.get_image_metadata(data, image).gps_position.value)
d = data.load_exif(image)
shot = reconstruction.create_shot(
image,
camera_id=d["camera"],
rig_camera_id=rig_camera_id,
rig_instance_id=rig_instance_id,
)
shot.metadata = helpers.get_image_metadata(data, image)
return [reconstruction]
def create_shot(
self,
shot_id: str,
camera_id: str,
pose: Optional[pygeometry.Pose] = None,
rig_camera_id: Optional[str] = None,
rig_instance_id: Optional[str] = None,
) -> pymap.Shot:
passed_rig_camera_id = rig_camera_id if rig_camera_id else camera_id
passed_rig_instance_id = rig_instance_id if rig_instance_id else shot_id
if (not rig_camera_id) and (camera_id not in self.rig_cameras):
self.add_rig_camera(pymap.RigCamera(pygeometry.Pose(), camera_id))
if passed_rig_camera_id not in self.rig_cameras:
raise RuntimeError(
f"Rig Camera {passed_rig_camera_id} doesn't exist in reconstruction"
)
if (not rig_instance_id) and (shot_id not in self.rig_instances):
self.add_rig_instance(pymap.RigInstance(shot_id))
if passed_rig_instance_id not in self.rig_instances:
raise RuntimeError(
f"Rig Instance {passed_rig_instance_id} doesn't exist in reconstruction"
)
if pose is None:
created_shot = self.map.create_shot(shot_id, camera_id,
passed_rig_camera_id,
passed_rig_instance_id)
else:
created_shot = self.map.create_shot(shot_id, camera_id,
passed_rig_camera_id,
passed_rig_instance_id, pose)
return created_shot
def default_rig_cameras(
camera_ids: Iterable[str]) -> Dict[str, pymap.RigCamera]:
"""Return per-camera models default rig cameras (identity pose)."""
default_rig_cameras = {}
for camera_id in camera_ids:
default_rig_cameras[camera_id] = pymap.RigCamera(
pygeometry.Pose(), camera_id)
return default_rig_cameras
def rig_camera_from_json(key, obj):
"""
Read a rig cameras from a json object
"""
pose = pygeometry.Pose()
pose.rotation = obj["rotation"]
pose.translation = obj["translation"]
rig_camera = pymap.RigCamera(pose, key)
return rig_camera
def compute_relative_pose(
pose_instances: List[List[Tuple[pymap.Shot, str]]],
) -> Dict[str, pymap.RigCamera]:
""" Compute a rig model relatives poses given poses grouped by rig instance. """
# Put all poses instances into some canonical frame taken as the mean of their R|t
centered_pose_instances = []
for instance in pose_instances:
origin_center = np.zeros(3)
rotation_center = np.zeros(3)
for shot, _ in instance:
rotation_center += shot.pose.rotation
origin_center += shot.pose.get_origin()
origin_center /= len(instance)
rotation_center /= len(instance)
centered_pose_instance = []
for shot, rig_camera_id in instance:
instance_pose = pygeometry.Pose(rotation_center)
instance_pose.set_origin(origin_center)
instance_pose_camera = shot.pose.relative_to(instance_pose)
centered_pose_instance.append(
(
instance_pose_camera,
rig_camera_id,
shot.camera.id,
)
)
centered_pose_instances.append(centered_pose_instance)
# Average canonical poses per RigCamera ID
average_origin, average_rotation, count_poses, camera_ids = {}, {}, {}, {}
for centered_pose_instance in centered_pose_instances:
for pose, rig_camera_id, camera_id in centered_pose_instance:
if rig_camera_id not in average_origin:
average_origin[rig_camera_id] = np.zeros(3)
average_rotation[rig_camera_id] = np.zeros(3)
count_poses[rig_camera_id] = 0
average_origin[rig_camera_id] += pose.get_origin()
average_rotation[rig_camera_id] += pose.rotation
camera_ids[rig_camera_id] = camera_id
count_poses[rig_camera_id] += 1
# Construct final RigCamera results
rig_cameras: Dict[str, pymap.RigCamera] = {}
for rig_camera_id, count in count_poses.items():
o = average_origin[rig_camera_id] / count
r = average_rotation[rig_camera_id] / count
pose = pygeometry.Pose(r)
pose.set_origin(o)
rig_cameras[rig_camera_id] = pymap.RigCamera(pose, rig_camera_id)
return rig_cameras
def perspective_views_of_a_panorama(
spherical_shot: pymap.Shot,
width: int,
reconstruction: types.Reconstruction,
image_format: str,
rig_instance_count: Iterator[int],
) -> List[pymap.Shot]:
"""Create 6 perspective views of a panorama."""
camera = pygeometry.Camera.create_perspective(0.5, 0.0, 0.0)
camera.id = "perspective_panorama_camera"
camera.width = width
camera.height = width
reconstruction.add_camera(camera)
names = ["front", "left", "back", "right", "top", "bottom"]
rotations = [
tf.rotation_matrix(-0 * np.pi / 2, np.array([0, 1, 0])),
tf.rotation_matrix(-1 * np.pi / 2, np.array([0, 1, 0])),
tf.rotation_matrix(-2 * np.pi / 2, np.array([0, 1, 0])),
tf.rotation_matrix(-3 * np.pi / 2, np.array([0, 1, 0])),
tf.rotation_matrix(-np.pi / 2, np.array([1, 0, 0])),
tf.rotation_matrix(+np.pi / 2, np.array([1, 0, 0])),
]
rig_instance = reconstruction.add_rig_instance(
pymap.RigInstance(str(next(rig_instance_count)))
)
shots = []
for name, rotation in zip(names, rotations):
if name not in reconstruction.rig_cameras:
rig_camera_pose = pygeometry.Pose()
rig_camera_pose.set_rotation_matrix(rotation[:3, :3])
rig_camera = pymap.RigCamera(rig_camera_pose, name)
reconstruction.add_rig_camera(rig_camera)
rig_camera = reconstruction.rig_cameras[name]
shot_id = add_image_format_extension(
f"{spherical_shot.id}_perspective_view_{name}", image_format
)
shot = reconstruction.create_shot(
shot_id, camera.id, pygeometry.Pose(), rig_camera.id, rig_instance.id
)
shot.metadata = spherical_shot.metadata
shots.append(shot)
rig_instance.pose = spherical_shot.pose
return shots
def add_camera_sequence(
self,
camera: pygeometry.Camera,
length: float,
height: float,
interval: float,
position_noise: List[float],
rotation_noise: float,
positions_shift: Optional[List[float]] = None,
end: Optional[float] = None,
) -> "SyntheticStreetScene":
default_noise_interval = 0.25 * interval
actual_end = length if end is None else end
generator = self.generator
assert generator
positions, rotations = sg.generate_cameras(
sg.samples_generator_interval(length, actual_end, interval,
default_noise_interval),
generator,
height,
)
sg.perturb_points(positions, position_noise)
sg.perturb_rotations(rotations, rotation_noise)
if positions_shift:
positions += np.array(positions_shift)
shift = 0 if len(self.shot_ids) == 0 else sum(
len(s) for s in self.shot_ids)
new_shot_ids = [f"Shot {shift+i:04d}" for i in range(len(positions))]
self.shot_ids.append(new_shot_ids)
self.cameras.append([camera])
rig_camera = pymap.RigCamera(pygeometry.Pose(), camera.id)
self.rig_cameras.append([rig_camera])
rig_instances = []
for shot_id in new_shot_ids:
rig_instances.append([(shot_id, camera.id)])
self.rig_instances.append(rig_instances)
self.instances_positions.append(positions)
self.instances_rotations.append(rotations)
return self
def create_pano_shot(self,
shot_id: str,
camera_id: str,
pose: Optional[pygeometry.Pose] = None) -> pymap.Shot:
if pose is None:
pose = pygeometry.Pose()
rig_camera_id = f"{PANOSHOT_RIG_PREFIX}{camera_id}"
if rig_camera_id not in self.rig_cameras:
self.add_rig_camera(
pymap.RigCamera(pygeometry.Pose(), rig_camera_id))
rig_instance_id = f"{PANOSHOT_RIG_PREFIX}{shot_id}"
if rig_instance_id not in self.rig_instances:
self.add_rig_instance(pymap.RigInstance(rig_instance_id))
created_shot = self.map.create_pano_shot(shot_id, camera_id,
rig_camera_id,
rig_instance_id, pose)
return created_shot
def perspective_views_of_a_panorama(spherical_shot, width, reconstruction,
image_format, rig_instance_count):
"""Create 6 perspective views of a panorama."""
camera = pygeometry.Camera.create_perspective(0.5, 0.0, 0.0)
camera.id = "perspective_panorama_camera"
camera.width = width
camera.height = width
reconstruction.add_camera(camera)
names = ["front", "left", "back", "right", "top", "bottom"]
rotations = [
tf.rotation_matrix(-0 * np.pi / 2, (0, 1, 0)),
tf.rotation_matrix(-1 * np.pi / 2, (0, 1, 0)),
tf.rotation_matrix(-2 * np.pi / 2, (0, 1, 0)),
tf.rotation_matrix(-3 * np.pi / 2, (0, 1, 0)),
tf.rotation_matrix(-np.pi / 2, (1, 0, 0)),
tf.rotation_matrix(+np.pi / 2, (1, 0, 0)),
]
rig_instance = pymap.RigInstance(next(rig_instance_count))
rig_instance.pose = spherical_shot.pose
shots = []
for name, rotation in zip(names, rotations):
if name not in reconstruction.rig_cameras:
rig_camera_pose = pygeometry.Pose()
rig_camera_pose.set_rotation_matrix(rotation[:3, :3])
rig_camera = pymap.RigCamera(rig_camera_pose, name)
reconstruction.add_rig_camera(rig_camera)
rig_camera = reconstruction.rig_cameras[name]
shot_id = add_image_format_extension(
f"{spherical_shot.id}_perspective_view_{name}", image_format)
shot = reconstruction.create_shot(shot_id, camera.id)
rig_instance.add_shot(rig_camera, shot)
shots.append(shot)
reconstruction.add_rig_instance(rig_instance)
return shots
def test_many_observations_delete() -> None:
# Given a map with 10 shots, 1000 landmarks ...
m = pymap.Map()
n_cams = 2
n_shots = 10
n_landmarks = 1000
for cam_id in range(n_cams):
cam = pygeometry.Camera.create_perspective(0.5, 0, 0)
cam.id = "cam" + str(cam_id)
m.create_camera(cam)
m.create_rig_camera(pymap.RigCamera(pygeometry.Pose(), cam.id))
for shot_id in range(n_shots):
cam_id = "cam" + str(int(np.random.rand(1) * 10 % n_cams))
shot_id = str(shot_id)
m.create_rig_instance(shot_id)
m.create_shot(shot_id, cam_id, cam_id, shot_id, pygeometry.Pose())
for point_id in range(n_landmarks):
m.create_landmark(str(point_id), np.random.rand(3))
# ... and random connections (observations) between shots and points
n_total_obs = 0
for lm in m.get_landmarks().values():
n_obs = 0
for shot in m.get_shots().values():
# create a new observation
obs = pymap.Observation(100, 200, 0.5, 255, 0, 0, int(lm.id))
m.add_observation(shot, lm, obs)
n_obs += 1
n_total_obs += 1
# (we expect it to be created correctly)
for lm in m.get_landmarks().values():
n_total_obs -= lm.number_of_observations()
assert n_total_obs == 0
# and when we clear all the observations
m.clear_observations_and_landmarks()
def test_clean_landmarks_with_min_observations() -> None:
m = pymap.Map()
n_cams = 2
n_shots = 2
n_landmarks = 10
for cam_id in range(n_cams):
cam = pygeometry.Camera.create_perspective(0.5, 0, 0)
cam.id = "cam" + str(cam_id)
m.create_camera(cam)
m.create_rig_camera(pymap.RigCamera(pygeometry.Pose(), cam.id))
for shot_id in range(n_shots):
cam_id = "cam" + str(int(np.random.rand(1) * 10 % n_cams))
m.create_rig_instance(str(shot_id))
m.create_shot(str(shot_id), cam_id, cam_id, str(shot_id),
pygeometry.Pose())
for point_id in range(n_landmarks):
m.create_landmark(str(point_id), np.random.rand(3))
for point_id in range(int(n_landmarks / 2)):
for shot in m.get_shots().values():
# create a new observation
obs = pymap.Observation(100, 200, 0.5, 255, 0, 0, point_id)
m.add_observation(shot, m.get_landmark(str(point_id)), obs)
for point_id in range(int(n_landmarks / 2), n_landmarks):
shot = m.get_shot("0")
# create a new observation
obs = pymap.Observation(100, 200, 0.5, 255, 0, 0, point_id)
m.add_observation(shot, m.get_landmark(str(point_id)), obs)
m.clean_landmarks_below_min_observations(n_shots)
assert len(m.get_landmarks()) == int(n_landmarks / 2)
m.clean_landmarks_below_min_observations(n_shots + 1)
assert len(m.get_landmarks()) == 0
def reconstruction_from_metadata(data: DataSetBase, images: Iterable[str]) -> types.Reconstruction:
"""Initialize a reconstruction by using EXIF data for constructing shot poses and cameras."""
data.init_reference()
rig_assignments = rig.rig_assignments_per_image(data.load_rig_assignments())
reconstruction = types.Reconstruction()
reconstruction.reference = data.load_reference()
reconstruction.cameras = data.load_camera_models()
for image in images:
camera_id = data.load_exif(image)["camera"]
if image in rig_assignments:
rig_instance_id, rig_camera_id, _ = rig_assignments[image]
else:
rig_instance_id = image
rig_camera_id = camera_id
reconstruction.add_rig_camera(pymap.RigCamera(pygeometry.Pose(), rig_camera_id))
reconstruction.add_rig_instance(pymap.RigInstance(rig_instance_id))
shot = reconstruction.create_shot(
shot_id=image,
camera_id=camera_id,
rig_camera_id=rig_camera_id,
rig_instance_id=rig_instance_id,
)
shot.metadata = get_image_metadata(data, image)
if not shot.metadata.gps_position.has_value:
reconstruction.remove_shot(image)
continue
gps_pos = shot.metadata.gps_position.value
shot.pose.set_rotation_matrix(rotation_from_shot_metadata(shot))
shot.pose.set_origin(gps_pos)
shot.scale = 1.0
return reconstruction
def add_rig_camera_sequence(
self,
cameras: List[pygeometry.Camera],
relative_positions: List[List[float]],
relative_rotations: List[List[float]],
start: float,
length: float,
height: float,
interval: float,
position_noise: List[float],
rotation_noise: float,
):
default_noise_interval = 0.25 * interval
instances_positions, instances_rotations = sg.generate_cameras(
sg.samples_generator_interval(start, length, interval,
default_noise_interval),
self.generator, # pyre-fixme [6]
height,
)
sg.perturb_points(instances_positions, position_noise)
sg.perturb_rotations(instances_rotations, rotation_noise)
shots_ids_per_camera = []
for rig_camera_p, rig_camera_r, camera in zip(relative_positions,
relative_rotations,
cameras):
pose_rig_camera = pygeometry.Pose(rig_camera_r)
pose_rig_camera.set_origin(rig_camera_p)
rotations = []
positions = []
for instance_p, instance_r in zip(instances_positions,
instances_rotations):
pose_instance = pygeometry.Pose(instance_r)
pose_instance.set_origin(instance_p)
composed = pose_rig_camera.compose(pose_instance)
rotations.append(composed.rotation)
positions.append(composed.get_origin())
self.shot_rotations.append(np.array(rotations))
self.shot_positions.append(np.array(positions))
shift = sum(len(s) for s in shots_ids_per_camera)
shots_ids_per_camera.append(
[f"Shot {shift+i:04d}" for i in range(len(positions))])
self.cameras.append(camera)
self.shot_ids += shots_ids_per_camera
rig_camera_ids = []
rig_cameras = []
rig_camera_id_shift = sum(len(s) for s in self.rig_cameras)
for i, (rig_camera_p, rig_camera_r) in enumerate(
zip(relative_positions, relative_rotations)):
pose_rig_camera = pygeometry.Pose(rig_camera_r)
pose_rig_camera.set_origin(rig_camera_p)
rig_camera_id = f"RigCamera {rig_camera_id_shift + i}"
rig_camera = pymap.RigCamera(pose_rig_camera, rig_camera_id)
rig_camera_ids.append(rig_camera_id)
rig_cameras.append(rig_camera)
self.rig_cameras.append(rig_cameras)
rig_instances = []
for i in range(len(instances_positions)):
instance = []
for j in range(len(shots_ids_per_camera)):
instance.append(
(shots_ids_per_camera[j][i], rig_camera_ids[j]))
rig_instances.append(instance)
self.rig_instances.append(rig_instances)
self.instances_positions.append(instances_positions)
self.instances_rotations.append(instances_rotations)
return self
def add_rig_camera_sequence(
self,
cameras,
relative_positions,
relative_rotations,
start,
length,
height,
interval,
position_noise=None,
rotation_noise=None,
gps_noise=None,
):
default_noise_interval = 0.25 * interval
instances_positions, instances_rotations = sg.generate_cameras(
sg.samples_generator_interval(start, length, interval,
default_noise_interval),
self.generator,
height,
)
sg.perturb_points(instances_positions, position_noise)
sg.perturb_rotations(instances_rotations, rotation_noise)
shots_ids_per_camera = []
for rig_camera_p, rig_camera_r, camera in zip(relative_positions,
relative_rotations,
cameras):
pose_rig_camera = pygeometry.Pose(rig_camera_r)
pose_rig_camera.set_origin(rig_camera_p)
rotations = []
positions = []
for instance_p, instance_r in zip(instances_positions,
instances_rotations):
pose_instance = pygeometry.Pose(instance_r)
pose_instance.set_origin(instance_p)
composed = pose_rig_camera.compose(pose_instance)
rotations.append(composed.rotation)
positions.append(composed.get_origin())
self.shot_rotations.append(rotations)
self.shot_positions.append(positions)
shift = sum(len(s) for s in shots_ids_per_camera)
shots_ids_per_camera.append(
[f"Shot {shift+i:04d}" for i in range(len(positions))])
self.cameras.append(camera)
self.shot_ids += shots_ids_per_camera
rig_camera_ids = []
rig_model = pymap.RigModel(f"RigModel {len(self.rig_models)}")
for i, (rig_camera_p, rig_camera_r) in enumerate(
zip(relative_positions, relative_rotations)):
pose_rig_camera = pygeometry.Pose(rig_camera_r)
pose_rig_camera.set_origin(rig_camera_p)
rig_camera_id = f"RigCamera {i}"
rig_camera = pymap.RigCamera(pose_rig_camera, rig_camera_id)
rig_model.add_rig_camera(rig_camera)
rig_camera_ids.append(rig_camera_id)
self.rig_models.append(rig_model)
rig_instances = []
for i in range(len(instances_positions)):
instance = []
for j in range(len(shots_ids_per_camera)):
instance.append(
(shots_ids_per_camera[j][i], rig_camera_ids[j]))
rig_instances.append(instance)
self.rig_instances.append(rig_instances)
self.instances_positions.append(instances_positions)
self.instances_rotations.append(instances_rotations)
return self
def _create_rig_camera():
rig_camera = pymap.RigCamera()
rig_camera.id = "rig_camera"
return rig_camera
def _create_rig_camera():
rig_camera = pymap.RigCamera()
rig_camera.id = "rig_camera"
rig_camera.pose = pygeometry.Pose(np.array([0.1, 0.2, 0.3]),
np.array([0.1, 0.2, 0.3]))
return rig_camera
def add_rig_camera_sequence(
self,
cameras: List[pygeometry.Camera],
relative_positions: List[List[float]],
relative_rotations: List[List[float]],
length: float,
height: float,
interval: float,
position_noise: List[float],
rotation_noise: float,
end: Optional[float] = None,
) -> "SyntheticStreetScene":
default_noise_interval = 0.25 * interval
actual_end = length if end is None else end
generator = self.generator
assert generator
instances_positions, instances_rotations = sg.generate_cameras(
sg.samples_generator_interval(length, actual_end, interval,
default_noise_interval),
generator,
height,
)
sg.perturb_points(instances_positions, position_noise)
sg.perturb_rotations(instances_rotations, rotation_noise)
shots_ids_per_camera = []
for rig_camera_p, rig_camera_r in zip(relative_positions,
relative_rotations):
pose_rig_camera = pygeometry.Pose(rig_camera_r)
pose_rig_camera.set_origin(rig_camera_p)
rotations = []
positions = []
for instance_p, instance_r in zip(instances_positions,
instances_rotations):
pose_instance = pygeometry.Pose(instance_r)
pose_instance.set_origin(instance_p)
composed = pose_rig_camera.compose(pose_instance)
rotations.append(composed.rotation)
positions.append(composed.get_origin())
shift = sum(len(s) for s in shots_ids_per_camera)
shots_ids_per_camera.append(
[f"Shot {shift+i:04d}" for i in range(len(positions))])
self.cameras.append(cameras)
self.shot_ids += shots_ids_per_camera
rig_camera_ids = []
rig_cameras = []
rig_camera_id_shift = sum(len(s) for s in self.rig_cameras)
for i, (rig_camera_p, rig_camera_r) in enumerate(
zip(relative_positions, relative_rotations)):
pose_rig_camera = pygeometry.Pose(rig_camera_r)
pose_rig_camera.set_origin(rig_camera_p)
rig_camera_id = f"RigCamera {rig_camera_id_shift + i}"
rig_camera = pymap.RigCamera(pose_rig_camera, rig_camera_id)
rig_camera_ids.append(rig_camera_id)
rig_cameras.append(rig_camera)
self.rig_cameras.append(rig_cameras)
rig_instances: List[List[Tuple[str, str]]] = []
for i in range(len(instances_positions)):
instance = []
for j in range(len(shots_ids_per_camera)):
instance.append(
(shots_ids_per_camera[j][i], rig_camera_ids[j]))
rig_instances.append(instance)
self.rig_instances.append(rig_instances)
self.instances_positions.append(instances_positions)
self.instances_rotations.append(instances_rotations)
return self
def _create_rig_model():
rig_model = pymap.RigModel("rig_model")
rig_camera = pymap.RigCamera()
rig_camera.id = "rig_camera"
rig_model.add_rig_camera(rig_camera)
return rig_model
