def undistort_reconstruction(tracks_manager, reconstruction, data, udata): urec = types.Reconstruction() urec.points = reconstruction.points utracks_manager = pysfm.TracksManager() logger.debug("Undistorting the reconstruction") undistorted_shots = {} for shot in reconstruction.shots.values(): if shot.camera.projection_type == "perspective": camera = perspective_camera_from_perspective(shot.camera) urec.add_camera(camera) subshots = [get_shot_with_different_camera(urec, shot, camera)] elif shot.camera.projection_type == "brown": camera = perspective_camera_from_brown(shot.camera) urec.add_camera(camera) subshots = [get_shot_with_different_camera(urec, shot, camera)] elif shot.camera.projection_type in ["fisheye", "fisheye_opencv"]: camera = perspective_camera_from_fisheye(shot.camera) urec.add_camera(camera) subshots = [get_shot_with_different_camera(urec, shot, camera)] elif pygeometry.Camera.is_panorama(shot.camera.projection_type): subshot_width = int(data.config["depthmap_resolution"]) subshots = perspective_views_of_a_panorama(shot, subshot_width, urec) for subshot in subshots: if tracks_manager: add_subshot_tracks(tracks_manager, utracks_manager, shot, subshot) undistorted_shots[shot.id] = subshots udata.save_undistorted_reconstruction([urec]) if tracks_manager: udata.save_undistorted_tracks_manager(utracks_manager) return undistorted_shots
def compute_and_save_undistorted_reconstruction(reconstruction, tracks_manager, data, udata): urec = types.Reconstruction() utracks_manager = pysfm.TracksManager() undistorted_shots = [] for shot in reconstruction.shots.values(): if shot.camera.projection_type == 'perspective': ucamera = osfm_u.perspective_camera_from_perspective(shot.camera) elif shot.camera.projection_type == 'brown': ucamera = osfm_u.perspective_camera_from_brown(shot.camera) elif shot.camera.projection_type == 'fisheye': ucamera = osfm_u.perspective_camera_from_fisheye(shot.camera) else: raise ValueError urec.add_camera(ucamera) ushot = osfm_u.get_shot_with_different_camera(urec, shot, ucamera) if tracks_manager: osfm_u.add_subshot_tracks(tracks_manager, utracks_manager, shot, ushot) undistorted_shots.append(ushot) image = data.load_image(shot.id, unchanged=True, anydepth=True) if image is not None: max_size = data.config['undistorted_image_max_size'] undistorted = osfm_u.undistort_image(shot, undistorted_shots, image, cv2.INTER_AREA, max_size) for k, v in undistorted.items(): udata.save_undistorted_image(k, v) udata.save_undistorted_reconstruction([urec]) if tracks_manager: udata.save_undistorted_tracks_manager(utracks_manager) return urec
def create_tracks_manager(features, colors, matches, config): """Link matches into tracks.""" logger.debug('Merging features onto tracks') uf = UnionFind() for im1, im2 in matches: for f1, f2 in matches[im1, im2]: uf.union((im1, f1), (im2, f2)) sets = {} for i in uf: p = uf[i] if p in sets: sets[p].append(i) else: sets[p] = [i] min_length = config['min_track_length'] tracks = [t for t in sets.values() if _good_track(t, min_length)] logger.debug('Good tracks: {}'.format(len(tracks))) tracks_manager = pysfm.TracksManager() for track_id, track in enumerate(tracks): for image, featureid in track: if image not in features: continue x, y, s = features[image][featureid] r, g, b = colors[image][featureid] obs = pysfm.Observation(x, y, s, int(r), int(g), int(b), featureid) tracks_manager.add_observation(image, str(track_id), obs) return tracks_manager
def generate_track_data(reconstruction, maximum_depth, noise): tracks_manager = pysfm.TracksManager() feature_data_type = np.float32 desc_size = 128 non_zeroes = 5 track_descriptors = {} for track_index in reconstruction.points: descriptor = np.zeros(desc_size) for i in range(non_zeroes): index = np.random.randint(0, desc_size) descriptor[index] = np.random.random()*255 track_descriptors[track_index] = descriptor.round().\ astype(feature_data_type) colors = {} features = {} descriptors = {} default_scale = 0.004 for shot_index, shot in reconstruction.shots.items(): # need to have these as we lost track of keys all_keys = list(reconstruction.points.keys()) all_values = list(reconstruction.points.values()) # temporary work on numpy array all_coordinates = [p.coordinates for p in all_values] projections = shot.project_many(np.array(all_coordinates)) projections_inside = [] descriptors_inside = [] colors_inside = [] for i, projection in enumerate(projections): if not _is_inside_camera(projection, shot.camera): continue original_key = all_keys[i] original_point = all_values[i] if not _is_in_front(original_point, shot): continue if not _check_depth(original_point, shot, maximum_depth): continue # add perturbation perturbation = float(noise)/float(max(shot.camera.width, shot.camera.height)) perturb_points([projection], np.array([perturbation, perturbation])) projections_inside.append(np.hstack((projection, [default_scale]))) descriptors_inside.append(track_descriptors[original_key]) colors_inside.append(original_point.color) obs = pysfm.Observation( projection[0], projection[1], default_scale, original_point.color[0], original_point.color[1], original_point.color[2], len(projections_inside) - 1) tracks_manager.add_observation(str(shot_index), str(original_key), obs) features[shot_index] = np.array(projections_inside) colors[shot_index] = np.array(colors_inside) descriptors[shot_index] = np.array(descriptors_inside) return features, descriptors, colors, tracks_manager
def import_images_reconstruction(path_images, keypoints, rec): """ Read images.bin, building shots and tracks graph """ logger.info("Importing images from {}".format(path_images)) tracks_manager = pysfm.TracksManager() image_ix_to_shot_id = {} with open(path_images, "rb") as f: n_ims = unpack("<Q", f.read(8))[0] for image_ix in range(n_ims): image_id = unpack("<I", f.read(4))[0] q0 = unpack("<d", f.read(8))[0] q1 = unpack("<d", f.read(8))[0] q2 = unpack("<d", f.read(8))[0] q3 = unpack("<d", f.read(8))[0] t0 = unpack("<d", f.read(8))[0] t1 = unpack("<d", f.read(8))[0] t2 = unpack("<d", f.read(8))[0] camera_id = unpack("<I", f.read(4))[0] filename = "" while True: c = f.read(1).decode() if c == "\0": break filename += c q = np.array([q0, q1, q2, q3]) q /= np.linalg.norm(q) t = np.array([t0, t1, t2]) pose = pygeometry.Pose(rotation=quaternion_to_angle_axis(q), translation=t) shot = rec.create_shot(filename, str(camera_id), pose) image_ix_to_shot_id[image_ix] = shot.id n_points_2d = unpack("<Q", f.read(8))[0] for point2d_ix in range(n_points_2d): x = unpack("<d", f.read(8))[0] y = unpack("<d", f.read(8))[0] point3d_id = unpack("<Q", f.read(8))[0] if point3d_id != np.iinfo(np.uint64).max: kp = keypoints[image_id][point2d_ix] r, g, b = rec.points[str(point3d_id)].color obs = pysfm.Observation( x, y, kp[2], int(r), int(g), int(b), point2d_ix, ) tracks_manager.add_observation(shot.id, str(point3d_id), obs) return tracks_manager, image_ix_to_shot_id
def undistort_reconstruction(tracks_manager, reconstruction, data: DataSetBase, udata: UndistortedDataSet): image_format = data.config["undistorted_image_format"] urec = types.Reconstruction() urec.points = reconstruction.points rig_instance_count = itertools.count() utracks_manager = pysfm.TracksManager() logger.debug("Undistorting the reconstruction") undistorted_shots = {} for shot in reconstruction.shots.values(): if shot.camera.projection_type == "perspective": camera = perspective_camera_from_perspective(shot.camera) urec.add_camera(camera) subshots = [ get_shot_with_different_camera(urec, shot, camera, image_format) ] elif shot.camera.projection_type == "brown": camera = perspective_camera_from_brown(shot.camera) urec.add_camera(camera) subshots = [ get_shot_with_different_camera(urec, shot, camera, image_format) ] elif shot.camera.projection_type in ["fisheye", "fisheye_opencv"]: camera = perspective_camera_from_fisheye(shot.camera) urec.add_camera(camera) subshots = [ get_shot_with_different_camera(urec, shot, camera, image_format) ] elif pygeometry.Camera.is_panorama(shot.camera.projection_type): subshot_width = int(data.config["depthmap_resolution"]) subshots = perspective_views_of_a_panorama(shot, subshot_width, urec, image_format, rig_instance_count) for subshot in subshots: if tracks_manager: add_subshot_tracks(tracks_manager, utracks_manager, shot, subshot) undistorted_shots[shot.id] = subshots udata.save_undistorted_reconstruction([urec]) if tracks_manager: udata.save_undistorted_tracks_manager(utracks_manager) udata.save_undistorted_shot_ids({ shot_id: [ushot.id for ushot in ushots] for shot_id, ushots in undistorted_shots.items() }) return undistorted_shots
def undistort_reconstruction(self, tracks_manager, reconstruction, data, udata): urec = types.Reconstruction() urec.points = reconstruction.points utracks_manager = pysfm.TracksManager() logger.debug('Undistorting the reconstruction') undistorted_shots = {} for shot in reconstruction.shots.values(): if shot.camera.projection_type == 'perspective': camera = perspective_camera_from_perspective(shot.camera) urec.add_camera(camera) subshots = [get_shot_with_different_camera(urec, shot, camera)] elif shot.camera.projection_type == 'brown': camera = perspective_camera_from_brown(shot.camera) urec.add_camera(camera) subshots = [get_shot_with_different_camera(urec, shot, camera)] elif shot.camera.projection_type in ['fisheye', 'fisheye_opencv']: camera = perspective_camera_from_fisheye(shot.camera) urec.add_camera(camera) subshots = [get_shot_with_different_camera(urec, shot, camera)] elif shot.camera.projection_type in [ 'equirectangular', 'spherical' ]: subshot_width = int(data.config['depthmap_resolution']) subshots = perspective_views_of_a_panorama( shot, subshot_width, urec) for subshot in subshots: if tracks_manager: add_subshot_tracks(tracks_manager, utracks_manager, shot, subshot) undistorted_shots[shot.id] = subshots udata.save_undistorted_reconstruction([urec]) if tracks_manager: udata.save_undistorted_tracks_manager(utracks_manager) arguments = [] for shot in reconstruction.shots.values(): arguments.append((shot, undistorted_shots[shot.id], data, udata)) processes = data.config['processes'] parallel_map(undistort_image_and_masks, arguments, processes)
def __init__( self, reconstruction: types.Reconstruction, projection_max_depth: float, projection_noise: float, gps_noise: Union[Dict[str, float], float], causal_gps_noise: bool, generate_projections: bool = True, ): self.reconstruction = reconstruction self.exifs = sg.generate_exifs(reconstruction, gps_noise, causal_gps_noise=causal_gps_noise) if generate_projections: (self.features, self.tracks_manager) = sg.generate_track_data( reconstruction, projection_max_depth, projection_noise) else: self.features = {} self.tracks_manager = pysfm.TracksManager()
def create_tracks_manager(features, colors, segmentations, instances, matches, config): """Link matches into tracks.""" logger.debug("Merging features onto tracks") uf = UnionFind() for im1, im2 in matches: for f1, f2 in matches[im1, im2]: uf.union((im1, f1), (im2, f2)) sets = {} for i in uf: p = uf[i] if p in sets: sets[p].append(i) else: sets[p] = [i] min_length = config["min_track_length"] tracks = [t for t in sets.values() if _good_track(t, min_length)] logger.debug("Good tracks: {}".format(len(tracks))) NO_VALUE = pysfm.Observation.NO_SEMANTIC_VALUE tracks_manager = pysfm.TracksManager() for track_id, track in enumerate(tracks): for image, featureid in track: if image not in features: continue x, y, s = features[image][featureid] r, g, b = colors[image][featureid] segmentation, instance = ( segmentations[image][featureid] if image in segmentations else NO_VALUE, instances[image][featureid] if image in instances else NO_VALUE, ) obs = pysfm.Observation(x, y, s, int(r), int(g), int(b), featureid, segmentation, instance) tracks_manager.add_observation(image, str(track_id), obs) return tracks_manager
def test_track_triangulator_equirectangular(): """Test triangulating tracks of spherical images.""" tracks_manager = pysfm.TracksManager() tracks_manager.add_observation('im1', '1', pysfm.Observation(0, 0, 1.0, 0, 0, 0, 0)) tracks_manager.add_observation('im2', '1', pysfm.Observation(-0.1, 0, 1.0, 0, 0, 0, 1)) rec = io.reconstruction_from_json({ "cameras": { "theta": { "projection_type": "equirectangular", "width": 800, "height": 400, } }, "shots": { 'im1': { "camera": "theta", "rotation": [0.0, 0.0, 0.0], "translation": [0.0, 0.0, 0.0], }, 'im2': { "camera": "theta", "rotation": [0, 0, 0.0], "translation": [-1, 0, 0.0], }, }, "points": {}, }) graph_inliers = nx.Graph() triangulator = reconstruction.TrackTriangulator(tracks_manager, graph_inliers, rec) triangulator.triangulate('1', 0.01, 2.0) assert '1' in rec.points p = rec.points['1'].coordinates assert np.allclose(p, [0, 0, 1.3763819204711]) assert len(graph_inliers.edges()) == 2
def test_track_triangulator_spherical(): """Test triangulating tracks of spherical images.""" tracks_manager = pysfm.TracksManager() tracks_manager.add_observation("im1", "1", pysfm.Observation(0, 0, 1.0, 0, 0, 0, 0)) tracks_manager.add_observation( "im2", "1", pysfm.Observation(-0.1, 0, 1.0, 0, 0, 0, 1) ) rec = io.reconstruction_from_json( { "cameras": { "theta": { "projection_type": "spherical", "width": 800, "height": 400, } }, "shots": { "im1": { "camera": "theta", "rotation": [0.0, 0.0, 0.0], "translation": [0.0, 0.0, 0.0], }, "im2": { "camera": "theta", "rotation": [0, 0, 0.0], "translation": [-1, 0, 0.0], }, }, "points": {}, } ) triangulator = reconstruction.TrackTriangulator(tracks_manager, rec) triangulator.triangulate("1", 0.01, 2.0) assert "1" in rec.points p = rec.points["1"].coordinates assert np.allclose(p, [0, 0, 1.3763819204711]) assert len(rec.points["1"].get_observations()) == 2
def generate_track_data( reconstruction: types.Reconstruction, maximum_depth: float, noise: float ) -> Tuple[Dict[str, np.ndarray], Dict[str, np.ndarray], Dict[str, np.ndarray], pysfm.TracksManager, ]: """Generate projection data from a reconstruction, considering a maximum viewing depth and gaussian noise added to the ideal projections. Returns feature/descriptor/color data per shot and a tracks manager object. """ tracks_manager = pysfm.TracksManager() feature_data_type = np.float32 desc_size = 128 non_zeroes = 5 points_ids = list(reconstruction.points) points_coordinates = [ p.coordinates for p in reconstruction.points.values() ] points_colors = [p.color for p in reconstruction.points.values()] # generate random descriptors per point track_descriptors = [] for _ in points_coordinates: descriptor = np.zeros(desc_size) for _ in range(non_zeroes): index = np.random.randint(0, desc_size) descriptor[index] = np.random.random() * 255 track_descriptors.append(descriptor.round().astype(feature_data_type)) # should speed-up projection queries points_tree = spatial.cKDTree(points_coordinates) colors = {} features = {} descriptors = {} default_scale = 0.004 for shot_index, shot in reconstruction.shots.items(): # query all closest points neighbors = list( sorted( points_tree.query_ball_point(shot.pose.get_origin(), maximum_depth))) # project them projections = shot.project_many( np.array([points_coordinates[c] for c in neighbors])) # shot constants center = shot.pose.get_origin() z_axis = shot.pose.get_rotation_matrix()[2] is_panorama = pygeometry.Camera.is_panorama( shot.camera.projection_type) perturbation = float(noise) / float( max(shot.camera.width, shot.camera.height)) sigmas = np.array([perturbation, perturbation]) # pre-generate random perturbations perturbations = np.random.normal(0.0, sigmas, (len(projections), 2)) # run and check valid projections projections_inside = [] descriptors_inside = [] colors_inside = [] for i, (p_id, projection) in enumerate(zip(neighbors, projections)): if not _is_inside_camera(projection, shot.camera): continue point = points_coordinates[p_id] if not is_panorama and not _is_in_front(point, center, z_axis): continue # add perturbation projection += perturbations[i] # push data color = points_colors[p_id] original_id = points_ids[p_id] projections_inside.append( [projection[0], projection[1], default_scale]) descriptors_inside.append(track_descriptors[p_id]) colors_inside.append(color) obs = pysfm.Observation( projection[0], projection[1], default_scale, color[0], color[1], color[2], len(projections_inside) - 1, ) tracks_manager.add_observation(str(shot_index), str(original_id), obs) features[shot_index] = np.array(projections_inside) colors[shot_index] = np.array(colors_inside) descriptors[shot_index] = np.array(descriptors_inside) return features, descriptors, colors, tracks_manager