def triangle_mesh_spherical(
shot_id: str, r: types.Reconstruction,
tracks_manager: pymap.TracksManager) -> Tuple[List[Any], List[Any]]:
shot = r.shots[shot_id]
bearings = []
vertices = []
# Add vertices to ensure that the camera is inside the convex hull
# of the points
for point in itertools.product([-1, 1], repeat=3): # vertices of a cube
bearing = 0.3 * np.array(point) / np.linalg.norm(point)
bearings.append(bearing)
point = shot.pose.transform_inverse(bearing)
vertices.append(point.tolist())
for track_id in tracks_manager.get_shot_observations(shot_id):
if track_id in r.points:
point = r.points[track_id].coordinates
direction = shot.pose.transform(point)
pixel = direction / np.linalg.norm(direction)
if not np.isnan(pixel).any():
vertices.append(point)
bearings.append(pixel.tolist())
tri = scipy.spatial.ConvexHull(bearings)
faces = tri.simplices.tolist()
return vertices, faces
def retriangulate(
tracks_manager: pymap.TracksManager,
reconstruction: types.Reconstruction,
config: Dict[str, Any],
) -> Dict[str, Any]:
"""Retrianguate all points"""
chrono = Chronometer()
report = {}
report["num_points_before"] = len(reconstruction.points)
threshold = config["triangulation_threshold"]
min_ray_angle = config["triangulation_min_ray_angle"]
reconstruction.points = {}
all_shots_ids = set(tracks_manager.get_shot_ids())
triangulator = TrackTriangulator(tracks_manager, reconstruction)
tracks = set()
for image in reconstruction.shots.keys():
if image in all_shots_ids:
tracks.update(tracks_manager.get_shot_observations(image).keys())
for track in tracks:
if config["triangulation_type"] == "ROBUST":
triangulator.triangulate_robust(track, threshold, min_ray_angle)
elif config["triangulation_type"] == "FULL":
triangulator.triangulate(track, threshold, min_ray_angle)
report["num_points_after"] = len(reconstruction.points)
chrono.lap("retriangulate")
report["wall_time"] = chrono.total_time()
return report
def add_pano_subshot_tracks(
tracks_manager: pymap.TracksManager,
utracks_manager: pymap.TracksManager,
panoshot: pymap.Shot,
perspectiveshot: pymap.Shot,
) -> None:
"""Add edges between subshots and visible tracks."""
for track_id, obs in tracks_manager.get_shot_observations(panoshot.id).items():
bearing = panoshot.camera.pixel_bearing(obs.point)
rotation = np.dot(
perspectiveshot.pose.get_rotation_matrix(),
panoshot.pose.get_rotation_matrix().T,
)
rotated_bearing = np.dot(bearing, rotation.T)
if rotated_bearing[2] <= 0:
continue
perspective_feature = perspectiveshot.camera.project(rotated_bearing)
if (
perspective_feature[0] < -0.5
or perspective_feature[0] > 0.5
or perspective_feature[1] < -0.5
or perspective_feature[1] > 0.5
):
continue
obs.point = perspective_feature
utracks_manager.add_observation(perspectiveshot.id, track_id, obs)
def compute_common_tracks(
reconstruction1: types.Reconstruction,
reconstruction2: types.Reconstruction,
tracks_manager1: pymap.TracksManager,
tracks_manager2: pymap.TracksManager,
) -> List[Tuple[str, str]]:
common_tracks = set()
common_images = set(reconstruction1.shots.keys()).intersection(
reconstruction2.shots.keys())
all_shot_ids1 = set(tracks_manager1.get_shot_ids())
all_shot_ids2 = set(tracks_manager2.get_shot_ids())
for image in common_images:
if image not in all_shot_ids1 or image not in all_shot_ids2:
continue
at_shot1 = tracks_manager1.get_shot_observations(image)
at_shot2 = tracks_manager2.get_shot_observations(image)
for t1, t2 in corresponding_tracks(at_shot1, at_shot2):
if t1 in reconstruction1.points and t2 in reconstruction2.points:
common_tracks.add((t1, t2))
return list(common_tracks)
def add_subshot_tracks(
tracks_manager: pymap.TracksManager,
utracks_manager: pymap.TracksManager,
shot: pymap.Shot,
subshot: pymap.Shot,
) -> None:
"""Add shot tracks to the undistorted tracks_manager."""
if shot.id not in tracks_manager.get_shot_ids():
return
if pygeometry.Camera.is_panorama(shot.camera.projection_type):
add_pano_subshot_tracks(tracks_manager, utracks_manager, shot, subshot)
else:
for track_id, obs in tracks_manager.get_shot_observations(shot.id).items():
utracks_manager.add_observation(subshot.id, track_id, obs)
def features_statistics(
data: DataSetBase,
tracks_manager: pymap.TracksManager,
reconstructions: List[types.Reconstruction],
) -> Dict[str, Any]:
stats = {}
detected = []
images = {s for r in reconstructions for s in r.shots}
for im in images:
features_data = feature_loader.instance.load_all_data(
data, im, False, False)
if not features_data:
continue
detected.append(len(features_data.points))
if len(detected) > 0:
stats["detected_features"] = {
"min": min(detected),
"max": max(detected),
"mean": int(np.mean(detected)),
"median": int(np.median(detected)),
}
else:
stats["detected_features"] = {
"min": -1,
"max": -1,
"mean": -1,
"median": -1
}
per_shots = defaultdict(int)
for rec in reconstructions:
all_points_keys = set(rec.points.keys())
for shot_id in rec.shots:
if shot_id not in tracks_manager.get_shot_ids():
continue
for point_id in tracks_manager.get_shot_observations(shot_id):
if point_id not in all_points_keys:
continue
per_shots[shot_id] += 1
per_shots = list(per_shots.values())
stats["reconstructed_features"] = {
"min": int(min(per_shots)) if len(per_shots) > 0 else -1,
"max": int(max(per_shots)) if len(per_shots) > 0 else -1,
"mean": int(np.mean(per_shots)) if len(per_shots) > 0 else -1,
"median": int(np.median(per_shots)) if len(per_shots) > 0 else -1,
}
return stats
def triangle_mesh_fisheye(
shot_id: str, r: types.Reconstruction,
tracks_manager: pymap.TracksManager) -> Tuple[List[Any], List[Any]]:
shot = r.shots[shot_id]
bearings = []
vertices = []
# Add boundary vertices
num_circle_points = 20
for i in range(num_circle_points):
a = 2 * np.pi * float(i) / num_circle_points
point = 30 * np.array([np.cos(a), np.sin(a), 0])
bearing = point / np.linalg.norm(point)
point = shot.pose.transform_inverse(point)
vertices.append(point.tolist())
bearings.append(bearing)
# Add a single vertex in front of the camera
point = 30 * np.array([0, 0, 1])
bearing = 0.3 * point / np.linalg.norm(point)
point = shot.pose.transform_inverse(point)
vertices.append(point.tolist())
bearings.append(bearing)
# Add reconstructed points
for track_id in tracks_manager.get_shot_observations(shot_id):
if track_id in r.points:
point = r.points[track_id].coordinates
direction = shot.pose.transform(point)
pixel = direction / np.linalg.norm(direction)
if not np.isnan(pixel).any():
vertices.append(point)
bearings.append(pixel.tolist())
# Triangulate
tri = scipy.spatial.ConvexHull(bearings)
faces = tri.simplices.tolist()
# Remove faces having only boundary vertices
def good_face(face):
return (face[0] >= num_circle_points or face[1] >= num_circle_points
or face[2] >= num_circle_points)
faces = list(filter(good_face, faces))
return vertices, faces
def triangle_mesh_perspective(
shot_id: str, r: types.Reconstruction,
tracks_manager: pymap.TracksManager) -> Tuple[List[Any], List[Any]]:
shot = r.shots[shot_id]
cam = shot.camera
dx = float(cam.width) / 2 / max(cam.width, cam.height)
dy = float(cam.height) / 2 / max(cam.width, cam.height)
pixels = [[-dx, -dy], [-dx, dy], [dx, dy], [dx, -dy]]
vertices = [None for i in range(4)]
for track_id in tracks_manager.get_shot_observations(shot_id):
if track_id in r.points:
point = r.points[track_id]
pixel = shot.project(point.coordinates)
nonans = not np.isnan(pixel).any()
if nonans and -dx <= pixel[0] <= dx and -dy <= pixel[1] <= dy:
vertices.append(point.coordinates)
pixels.append(pixel.tolist())
try:
tri = scipy.spatial.Delaunay(pixels)
except Exception as e:
logger.error("Delaunay triangulation failed for input: {}".format(
repr(pixels)))
raise e
sums = [0.0, 0.0, 0.0, 0.0]
depths = [0.0, 0.0, 0.0, 0.0]
for t in tri.simplices:
for i in range(4):
if i in t:
for j in t:
if j >= 4:
depths[i] += shot.pose.transform(vertices[j])[2]
sums[i] += 1
for i in range(4):
if sums[i] > 0:
d = depths[i] / sums[i]
else:
d = 50.0
vertices[i] = back_project_no_distortion(shot, pixels[i], d).tolist()
faces = tri.simplices.tolist()
return vertices, faces
def triangulate_shot_features(
tracks_manager: pymap.TracksManager,
reconstruction: types.Reconstruction,
shot_ids: Set[str],
config: Dict[str, Any],
) -> None:
"""Reconstruct as many tracks seen in shot_id as possible."""
reproj_threshold = config["triangulation_threshold"]
min_ray_angle = config["triangulation_min_ray_angle"]
triangulator = TrackTriangulator(tracks_manager, reconstruction)
all_shots_ids = set(tracks_manager.get_shot_ids())
tracks_ids = {
t
for s in shot_ids if s in all_shots_ids
for t in tracks_manager.get_shot_observations(s)
}
for track in tracks_ids:
if track not in reconstruction.points:
triangulator.triangulate(track, reproj_threshold, min_ray_angle)
def common_tracks(
tracks_manager: pymap.TracksManager, im1: str, im2: str
) -> t.Tuple[t.List[str], np.ndarray, np.ndarray]:
"""List of tracks observed in both images.
Args:
tracks_manager: tracks manager
im1: name of the first image
im2: name of the second image
Returns:
tuple: tracks, feature from first image, feature from second image
"""
t1 = tracks_manager.get_shot_observations(im1)
t2 = tracks_manager.get_shot_observations(im2)
tracks, p1, p2 = [], [], []
for track, obs in t1.items():
if track in t2:
p1.append(obs.point)
p2.append(t2[track].point)
tracks.append(track)
p1 = np.array(p1)
p2 = np.array(p2)
return tracks, p1, p2
def resect(
data: DataSetBase,
tracks_manager: pymap.TracksManager,
reconstruction: types.Reconstruction,
shot_id: str,
threshold: float,
min_inliers: int,
) -> Tuple[bool, Set[str], Dict[str, Any]]:
"""Try resecting and adding a shot to the reconstruction.
Return:
True on success.
"""
rig_assignments = data.load_rig_assignments_per_image()
camera = reconstruction.cameras[data.load_exif(shot_id)["camera"]]
bs, Xs, ids = [], [], []
for track, obs in tracks_manager.get_shot_observations(shot_id).items():
if track in reconstruction.points:
b = camera.pixel_bearing(obs.point)
bs.append(b)
Xs.append(reconstruction.points[track].coordinates)
ids.append(track)
bs = np.array(bs)
Xs = np.array(Xs)
if len(bs) < 5:
return False, set(), {"num_common_points": len(bs)}
T = multiview.absolute_pose_ransac(bs, Xs, threshold, 1000, 0.999)
R = T[:, :3]
t = T[:, 3]
reprojected_bs = R.T.dot((Xs - t).T).T
reprojected_bs /= np.linalg.norm(reprojected_bs, axis=1)[:, np.newaxis]
inliers = np.linalg.norm(reprojected_bs - bs, axis=1) < threshold
ninliers = int(sum(inliers))
logger.info("{} resection inliers: {} / {}".format(shot_id, ninliers,
len(bs)))
report = {
"num_common_points": len(bs),
"num_inliers": ninliers,
}
if ninliers >= min_inliers:
R = T[:, :3].T
t = -R.dot(T[:, 3])
assert shot_id not in reconstruction.shots
new_shots = add_shot(data, reconstruction, rig_assignments, shot_id,
pygeometry.Pose(R, t))
if shot_id in rig_assignments:
triangulate_shot_features(tracks_manager, reconstruction,
new_shots, data.config)
for i, succeed in enumerate(inliers):
if succeed:
add_observation_to_reconstruction(tracks_manager,
reconstruction, shot_id,
ids[i])
# pyre-fixme [6]: Expected `int` for 2nd positional
report["shots"] = list(new_shots)
return True, new_shots, report
else:
return False, set(), report