def undistort_reconstruction(tracks_manager, reconstruction, data, udata):
urec = types.Reconstruction()
urec.points = reconstruction.points
utracks_manager = pysfm.TracksManager()
logger.debug("Undistorting the reconstruction")
undistorted_shots = {}
for shot in reconstruction.shots.values():
if shot.camera.projection_type == "perspective":
camera = perspective_camera_from_perspective(shot.camera)
urec.add_camera(camera)
subshots = [get_shot_with_different_camera(urec, shot, camera)]
elif shot.camera.projection_type == "brown":
camera = perspective_camera_from_brown(shot.camera)
urec.add_camera(camera)
subshots = [get_shot_with_different_camera(urec, shot, camera)]
elif shot.camera.projection_type in ["fisheye", "fisheye_opencv"]:
camera = perspective_camera_from_fisheye(shot.camera)
urec.add_camera(camera)
subshots = [get_shot_with_different_camera(urec, shot, camera)]
elif pygeometry.Camera.is_panorama(shot.camera.projection_type):
subshot_width = int(data.config["depthmap_resolution"])
subshots = perspective_views_of_a_panorama(shot, subshot_width,
urec)
for subshot in subshots:
if tracks_manager:
add_subshot_tracks(tracks_manager, utracks_manager, shot,
subshot)
undistorted_shots[shot.id] = subshots
udata.save_undistorted_reconstruction([urec])
if tracks_manager:
udata.save_undistorted_tracks_manager(utracks_manager)
return undistorted_shots
def compute_and_save_undistorted_reconstruction(reconstruction, tracks_manager, data, udata):
urec = types.Reconstruction()
utracks_manager = pysfm.TracksManager()
undistorted_shots = []
for shot in reconstruction.shots.values():
if shot.camera.projection_type == 'perspective':
ucamera = osfm_u.perspective_camera_from_perspective(shot.camera)
elif shot.camera.projection_type == 'brown':
ucamera = osfm_u.perspective_camera_from_brown(shot.camera)
elif shot.camera.projection_type == 'fisheye':
ucamera = osfm_u.perspective_camera_from_fisheye(shot.camera)
else:
raise ValueError
urec.add_camera(ucamera)
ushot = osfm_u.get_shot_with_different_camera(urec, shot, ucamera)
if tracks_manager:
osfm_u.add_subshot_tracks(tracks_manager, utracks_manager, shot, ushot)
undistorted_shots.append(ushot)
image = data.load_image(shot.id, unchanged=True, anydepth=True)
if image is not None:
max_size = data.config['undistorted_image_max_size']
undistorted = osfm_u.undistort_image(shot, undistorted_shots, image,
cv2.INTER_AREA, max_size)
for k, v in undistorted.items():
udata.save_undistorted_image(k, v)
udata.save_undistorted_reconstruction([urec])
if tracks_manager:
udata.save_undistorted_tracks_manager(utracks_manager)
return urec
def create_tracks_manager(features, colors, matches, config):
"""Link matches into tracks."""
logger.debug('Merging features onto tracks')
uf = UnionFind()
for im1, im2 in matches:
for f1, f2 in matches[im1, im2]:
uf.union((im1, f1), (im2, f2))
sets = {}
for i in uf:
p = uf[i]
if p in sets:
sets[p].append(i)
else:
sets[p] = [i]
min_length = config['min_track_length']
tracks = [t for t in sets.values() if _good_track(t, min_length)]
logger.debug('Good tracks: {}'.format(len(tracks)))
tracks_manager = pysfm.TracksManager()
for track_id, track in enumerate(tracks):
for image, featureid in track:
if image not in features:
continue
x, y, s = features[image][featureid]
r, g, b = colors[image][featureid]
obs = pysfm.Observation(x, y, s, int(r), int(g), int(b), featureid)
tracks_manager.add_observation(image, str(track_id), obs)
return tracks_manager
def generate_track_data(reconstruction, maximum_depth, noise):
tracks_manager = pysfm.TracksManager()
feature_data_type = np.float32
desc_size = 128
non_zeroes = 5
track_descriptors = {}
for track_index in reconstruction.points:
descriptor = np.zeros(desc_size)
for i in range(non_zeroes):
index = np.random.randint(0, desc_size)
descriptor[index] = np.random.random()*255
track_descriptors[track_index] = descriptor.round().\
astype(feature_data_type)
colors = {}
features = {}
descriptors = {}
default_scale = 0.004
for shot_index, shot in reconstruction.shots.items():
# need to have these as we lost track of keys
all_keys = list(reconstruction.points.keys())
all_values = list(reconstruction.points.values())
# temporary work on numpy array
all_coordinates = [p.coordinates for p in all_values]
projections = shot.project_many(np.array(all_coordinates))
projections_inside = []
descriptors_inside = []
colors_inside = []
for i, projection in enumerate(projections):
if not _is_inside_camera(projection, shot.camera):
continue
original_key = all_keys[i]
original_point = all_values[i]
if not _is_in_front(original_point, shot):
continue
if not _check_depth(original_point, shot, maximum_depth):
continue
# add perturbation
perturbation = float(noise)/float(max(shot.camera.width,
shot.camera.height))
perturb_points([projection], np.array([perturbation, perturbation]))
projections_inside.append(np.hstack((projection, [default_scale])))
descriptors_inside.append(track_descriptors[original_key])
colors_inside.append(original_point.color)
obs = pysfm.Observation(
projection[0], projection[1], default_scale,
original_point.color[0], original_point.color[1],
original_point.color[2], len(projections_inside) - 1)
tracks_manager.add_observation(str(shot_index), str(original_key), obs)
features[shot_index] = np.array(projections_inside)
colors[shot_index] = np.array(colors_inside)
descriptors[shot_index] = np.array(descriptors_inside)
return features, descriptors, colors, tracks_manager
def import_images_reconstruction(path_images, keypoints, rec):
"""
Read images.bin, building shots and tracks graph
"""
logger.info("Importing images from {}".format(path_images))
tracks_manager = pysfm.TracksManager()
image_ix_to_shot_id = {}
with open(path_images, "rb") as f:
n_ims = unpack("<Q", f.read(8))[0]
for image_ix in range(n_ims):
image_id = unpack("<I", f.read(4))[0]
q0 = unpack("<d", f.read(8))[0]
q1 = unpack("<d", f.read(8))[0]
q2 = unpack("<d", f.read(8))[0]
q3 = unpack("<d", f.read(8))[0]
t0 = unpack("<d", f.read(8))[0]
t1 = unpack("<d", f.read(8))[0]
t2 = unpack("<d", f.read(8))[0]
camera_id = unpack("<I", f.read(4))[0]
filename = ""
while True:
c = f.read(1).decode()
if c == "\0":
break
filename += c
q = np.array([q0, q1, q2, q3])
q /= np.linalg.norm(q)
t = np.array([t0, t1, t2])
pose = pygeometry.Pose(rotation=quaternion_to_angle_axis(q),
translation=t)
shot = rec.create_shot(filename, str(camera_id), pose)
image_ix_to_shot_id[image_ix] = shot.id
n_points_2d = unpack("<Q", f.read(8))[0]
for point2d_ix in range(n_points_2d):
x = unpack("<d", f.read(8))[0]
y = unpack("<d", f.read(8))[0]
point3d_id = unpack("<Q", f.read(8))[0]
if point3d_id != np.iinfo(np.uint64).max:
kp = keypoints[image_id][point2d_ix]
r, g, b = rec.points[str(point3d_id)].color
obs = pysfm.Observation(
x,
y,
kp[2],
int(r),
int(g),
int(b),
point2d_ix,
)
tracks_manager.add_observation(shot.id, str(point3d_id),
obs)
return tracks_manager, image_ix_to_shot_id
def undistort_reconstruction(tracks_manager, reconstruction, data: DataSetBase,
udata: UndistortedDataSet):
image_format = data.config["undistorted_image_format"]
urec = types.Reconstruction()
urec.points = reconstruction.points
rig_instance_count = itertools.count()
utracks_manager = pysfm.TracksManager()
logger.debug("Undistorting the reconstruction")
undistorted_shots = {}
for shot in reconstruction.shots.values():
if shot.camera.projection_type == "perspective":
camera = perspective_camera_from_perspective(shot.camera)
urec.add_camera(camera)
subshots = [
get_shot_with_different_camera(urec, shot, camera,
image_format)
]
elif shot.camera.projection_type == "brown":
camera = perspective_camera_from_brown(shot.camera)
urec.add_camera(camera)
subshots = [
get_shot_with_different_camera(urec, shot, camera,
image_format)
]
elif shot.camera.projection_type in ["fisheye", "fisheye_opencv"]:
camera = perspective_camera_from_fisheye(shot.camera)
urec.add_camera(camera)
subshots = [
get_shot_with_different_camera(urec, shot, camera,
image_format)
]
elif pygeometry.Camera.is_panorama(shot.camera.projection_type):
subshot_width = int(data.config["depthmap_resolution"])
subshots = perspective_views_of_a_panorama(shot, subshot_width,
urec, image_format,
rig_instance_count)
for subshot in subshots:
if tracks_manager:
add_subshot_tracks(tracks_manager, utracks_manager, shot,
subshot)
undistorted_shots[shot.id] = subshots
udata.save_undistorted_reconstruction([urec])
if tracks_manager:
udata.save_undistorted_tracks_manager(utracks_manager)
udata.save_undistorted_shot_ids({
shot_id: [ushot.id for ushot in ushots]
for shot_id, ushots in undistorted_shots.items()
})
return undistorted_shots
def undistort_reconstruction(self, tracks_manager, reconstruction, data,
udata):
urec = types.Reconstruction()
urec.points = reconstruction.points
utracks_manager = pysfm.TracksManager()
logger.debug('Undistorting the reconstruction')
undistorted_shots = {}
for shot in reconstruction.shots.values():
if shot.camera.projection_type == 'perspective':
camera = perspective_camera_from_perspective(shot.camera)
urec.add_camera(camera)
subshots = [get_shot_with_different_camera(urec, shot, camera)]
elif shot.camera.projection_type == 'brown':
camera = perspective_camera_from_brown(shot.camera)
urec.add_camera(camera)
subshots = [get_shot_with_different_camera(urec, shot, camera)]
elif shot.camera.projection_type in ['fisheye', 'fisheye_opencv']:
camera = perspective_camera_from_fisheye(shot.camera)
urec.add_camera(camera)
subshots = [get_shot_with_different_camera(urec, shot, camera)]
elif shot.camera.projection_type in [
'equirectangular', 'spherical'
]:
subshot_width = int(data.config['depthmap_resolution'])
subshots = perspective_views_of_a_panorama(
shot, subshot_width, urec)
for subshot in subshots:
if tracks_manager:
add_subshot_tracks(tracks_manager, utracks_manager, shot,
subshot)
undistorted_shots[shot.id] = subshots
udata.save_undistorted_reconstruction([urec])
if tracks_manager:
udata.save_undistorted_tracks_manager(utracks_manager)
arguments = []
for shot in reconstruction.shots.values():
arguments.append((shot, undistorted_shots[shot.id], data, udata))
processes = data.config['processes']
parallel_map(undistort_image_and_masks, arguments, processes)
def __init__(
self,
reconstruction: types.Reconstruction,
projection_max_depth: float,
projection_noise: float,
gps_noise: Union[Dict[str, float], float],
causal_gps_noise: bool,
generate_projections: bool = True,
):
self.reconstruction = reconstruction
self.exifs = sg.generate_exifs(reconstruction,
gps_noise,
causal_gps_noise=causal_gps_noise)
if generate_projections:
(self.features, self.tracks_manager) = sg.generate_track_data(
reconstruction, projection_max_depth, projection_noise)
else:
self.features = {}
self.tracks_manager = pysfm.TracksManager()
def create_tracks_manager(features, colors, segmentations, instances, matches,
config):
"""Link matches into tracks."""
logger.debug("Merging features onto tracks")
uf = UnionFind()
for im1, im2 in matches:
for f1, f2 in matches[im1, im2]:
uf.union((im1, f1), (im2, f2))
sets = {}
for i in uf:
p = uf[i]
if p in sets:
sets[p].append(i)
else:
sets[p] = [i]
min_length = config["min_track_length"]
tracks = [t for t in sets.values() if _good_track(t, min_length)]
logger.debug("Good tracks: {}".format(len(tracks)))
NO_VALUE = pysfm.Observation.NO_SEMANTIC_VALUE
tracks_manager = pysfm.TracksManager()
for track_id, track in enumerate(tracks):
for image, featureid in track:
if image not in features:
continue
x, y, s = features[image][featureid]
r, g, b = colors[image][featureid]
segmentation, instance = (
segmentations[image][featureid]
if image in segmentations else NO_VALUE,
instances[image][featureid]
if image in instances else NO_VALUE,
)
obs = pysfm.Observation(x, y, s, int(r), int(g), int(b), featureid,
segmentation, instance)
tracks_manager.add_observation(image, str(track_id), obs)
return tracks_manager
def test_track_triangulator_equirectangular():
"""Test triangulating tracks of spherical images."""
tracks_manager = pysfm.TracksManager()
tracks_manager.add_observation('im1', '1',
pysfm.Observation(0, 0, 1.0, 0, 0, 0, 0))
tracks_manager.add_observation('im2', '1',
pysfm.Observation(-0.1, 0, 1.0, 0, 0, 0, 1))
rec = io.reconstruction_from_json({
"cameras": {
"theta": {
"projection_type": "equirectangular",
"width": 800,
"height": 400,
}
},
"shots": {
'im1': {
"camera": "theta",
"rotation": [0.0, 0.0, 0.0],
"translation": [0.0, 0.0, 0.0],
},
'im2': {
"camera": "theta",
"rotation": [0, 0, 0.0],
"translation": [-1, 0, 0.0],
},
},
"points": {},
})
graph_inliers = nx.Graph()
triangulator = reconstruction.TrackTriangulator(tracks_manager,
graph_inliers, rec)
triangulator.triangulate('1', 0.01, 2.0)
assert '1' in rec.points
p = rec.points['1'].coordinates
assert np.allclose(p, [0, 0, 1.3763819204711])
assert len(graph_inliers.edges()) == 2
def test_track_triangulator_spherical():
"""Test triangulating tracks of spherical images."""
tracks_manager = pysfm.TracksManager()
tracks_manager.add_observation("im1", "1", pysfm.Observation(0, 0, 1.0, 0, 0, 0, 0))
tracks_manager.add_observation(
"im2", "1", pysfm.Observation(-0.1, 0, 1.0, 0, 0, 0, 1)
)
rec = io.reconstruction_from_json(
{
"cameras": {
"theta": {
"projection_type": "spherical",
"width": 800,
"height": 400,
}
},
"shots": {
"im1": {
"camera": "theta",
"rotation": [0.0, 0.0, 0.0],
"translation": [0.0, 0.0, 0.0],
},
"im2": {
"camera": "theta",
"rotation": [0, 0, 0.0],
"translation": [-1, 0, 0.0],
},
},
"points": {},
}
)
triangulator = reconstruction.TrackTriangulator(tracks_manager, rec)
triangulator.triangulate("1", 0.01, 2.0)
assert "1" in rec.points
p = rec.points["1"].coordinates
assert np.allclose(p, [0, 0, 1.3763819204711])
assert len(rec.points["1"].get_observations()) == 2
def generate_track_data(
reconstruction: types.Reconstruction, maximum_depth: float, noise: float
) -> Tuple[Dict[str, np.ndarray], Dict[str, np.ndarray], Dict[str, np.ndarray],
pysfm.TracksManager, ]:
"""Generate projection data from a reconstruction, considering a maximum
viewing depth and gaussian noise added to the ideal projections.
Returns feature/descriptor/color data per shot and a tracks manager object.
"""
tracks_manager = pysfm.TracksManager()
feature_data_type = np.float32
desc_size = 128
non_zeroes = 5
points_ids = list(reconstruction.points)
points_coordinates = [
p.coordinates for p in reconstruction.points.values()
]
points_colors = [p.color for p in reconstruction.points.values()]
# generate random descriptors per point
track_descriptors = []
for _ in points_coordinates:
descriptor = np.zeros(desc_size)
for _ in range(non_zeroes):
index = np.random.randint(0, desc_size)
descriptor[index] = np.random.random() * 255
track_descriptors.append(descriptor.round().astype(feature_data_type))
# should speed-up projection queries
points_tree = spatial.cKDTree(points_coordinates)
colors = {}
features = {}
descriptors = {}
default_scale = 0.004
for shot_index, shot in reconstruction.shots.items():
# query all closest points
neighbors = list(
sorted(
points_tree.query_ball_point(shot.pose.get_origin(),
maximum_depth)))
# project them
projections = shot.project_many(
np.array([points_coordinates[c] for c in neighbors]))
# shot constants
center = shot.pose.get_origin()
z_axis = shot.pose.get_rotation_matrix()[2]
is_panorama = pygeometry.Camera.is_panorama(
shot.camera.projection_type)
perturbation = float(noise) / float(
max(shot.camera.width, shot.camera.height))
sigmas = np.array([perturbation, perturbation])
# pre-generate random perturbations
perturbations = np.random.normal(0.0, sigmas, (len(projections), 2))
# run and check valid projections
projections_inside = []
descriptors_inside = []
colors_inside = []
for i, (p_id, projection) in enumerate(zip(neighbors, projections)):
if not _is_inside_camera(projection, shot.camera):
continue
point = points_coordinates[p_id]
if not is_panorama and not _is_in_front(point, center, z_axis):
continue
# add perturbation
projection += perturbations[i]
# push data
color = points_colors[p_id]
original_id = points_ids[p_id]
projections_inside.append(
[projection[0], projection[1], default_scale])
descriptors_inside.append(track_descriptors[p_id])
colors_inside.append(color)
obs = pysfm.Observation(
projection[0],
projection[1],
default_scale,
color[0],
color[1],
color[2],
len(projections_inside) - 1,
)
tracks_manager.add_observation(str(shot_index), str(original_id),
obs)
features[shot_index] = np.array(projections_inside)
colors[shot_index] = np.array(colors_inside)
descriptors[shot_index] = np.array(descriptors_inside)
return features, descriptors, colors, tracks_manager
