def load_segmentation_labels(data: UndistortedDataSet, shot): """Load the undistorted segmentation labels. If no segmentation exists return an array of zeros. """ if data.undistorted_segmentation_exists(shot.id): return data.load_undistorted_segmentation(shot.id) else: size = shot.camera.height, shot.camera.width return np.zeros(size, dtype=np.uint8)
def add_views_to_depth_cleaner(data: UndistortedDataSet, neighbors, dc): for shot in neighbors: if not data.raw_depthmap_exists(shot.id): continue depth, plane, score, nghbr, nghbrs = data.load_raw_depthmap(shot.id) height, width = depth.shape K = shot.camera.get_K_in_pixel_coordinates(width, height) R = shot.pose.get_rotation_matrix() t = shot.pose.translation dc.add_view(K, R, t, depth)
def compute_depthmaps( data: UndistortedDataSet, graph: pysfm.TracksManager, reconstruction: types.Reconstruction, ): """Compute and refine depthmaps for all shots. Args: data: an UndistortedDataset graph: the tracks graph reconstruction: the undistorted reconstruction """ logger.info("Computing neighbors") config = data.config processes = config["processes"] num_neighbors = config["depthmap_num_neighbors"] neighbors = {} common_tracks = common_tracks_double_dict(graph) for shot in reconstruction.shots.values(): neighbors[shot.id] = find_neighboring_images(shot, common_tracks, reconstruction, num_neighbors) arguments = [] for shot in reconstruction.shots.values(): if len(neighbors[shot.id]) <= 1: continue mind, maxd = compute_depth_range(graph, reconstruction, shot, config) arguments.append((data, neighbors[shot.id], mind, maxd, shot)) parallel_map(compute_depthmap_catched, arguments, processes) arguments = [] for shot in reconstruction.shots.values(): if len(neighbors[shot.id]) <= 1: continue arguments.append((data, neighbors[shot.id], shot)) parallel_map(clean_depthmap_catched, arguments, processes) arguments = [] for shot in reconstruction.shots.values(): if len(neighbors[shot.id]) <= 1: continue arguments.append((data, neighbors[shot.id], shot)) parallel_map(prune_depthmap_catched, arguments, processes) point_cloud = merge_depthmaps(data, reconstruction) data.save_point_cloud(*point_cloud, filename="merged.ply")
def add_views_to_depth_pruner(data: UndistortedDataSet, neighbors, dp): for shot in neighbors: if not data.clean_depthmap_exists(shot.id): continue depth, plane, score = data.load_clean_depthmap(shot.id) height, width = depth.shape color_image = data.load_undistorted_image(shot.id) labels = load_segmentation_labels(data, shot) height, width = depth.shape image = scale_down_image(color_image, width, height) labels = scale_image(labels, image.shape[1], image.shape[0], cv2.INTER_NEAREST) K = shot.camera.get_K_in_pixel_coordinates(width, height) R = shot.pose.get_rotation_matrix() t = shot.pose.translation dp.add_view(K, R, t, depth, plane, image, labels)
def merge_depthmaps(data: UndistortedDataSet, reconstruction): """Merge depthmaps into a single point cloud.""" logger.info("Merging depthmaps") shot_ids = [ s for s in reconstruction.shots if data.pruned_depthmap_exists(s) ] if not shot_ids: logger.warning("Depthmaps contain no points. Try using more images.") return def depthmap_provider(shot_id): return data.load_pruned_depthmap(shot_id) merge_depthmaps_from_provider(data, shot_ids, depthmap_provider, data.point_cloud_file())
def load_combined_mask(data: UndistortedDataSet, shot): """Load the undistorted mask. If no mask exists return an array of ones. """ mask = data.load_undistorted_combined_mask(shot.id) if mask is None: size = int(shot.camera.height), int(shot.camera.width) return np.ones(size, dtype=np.uint8) else: return mask
def merge_depthmaps( data: UndistortedDataSet, reconstruction: types.Reconstruction ) -> t.Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: """Merge depthmaps into a single point cloud.""" shot_ids = [ s for s in reconstruction.shots if data.pruned_depthmap_exists(s) ] def depthmap_provider(shot_id): return data.load_pruned_depthmap(shot_id) return merge_depthmaps_from_provider(data, shot_ids, depthmap_provider)
def undistort_reconstruction( tracks_manager: Optional[pymap.TracksManager], reconstruction: types.Reconstruction, data: DataSetBase, udata: UndistortedDataSet, ) -> Dict[pymap.Shot, List[pymap.Shot]]: all_images = set(data.images()) image_format = data.config["undistorted_image_format"] urec = types.Reconstruction() urec.points = reconstruction.points urec.reference = reconstruction.reference rig_instance_count = itertools.count() utracks_manager = pymap.TracksManager() logger.debug("Undistorting the reconstruction") undistorted_shots = {} for shot in reconstruction.shots.values(): if shot.id not in all_images: logger.warning( f"Not undistorting {shot.id} as it is missing from the dataset's input images." ) continue if shot.camera.projection_type == "perspective": urec.add_camera(perspective_camera_from_perspective(shot.camera)) subshots = [get_shot_with_different_camera(urec, shot, image_format)] elif shot.camera.projection_type == "brown": urec.add_camera(perspective_camera_from_brown(shot.camera)) subshots = [get_shot_with_different_camera(urec, shot, image_format)] elif shot.camera.projection_type in ["fisheye", "fisheye_opencv"]: urec.add_camera(perspective_camera_from_fisheye(shot.camera)) subshots = [get_shot_with_different_camera(urec, shot, image_format)] elif pygeometry.Camera.is_panorama(shot.camera.projection_type): subshot_width = int(data.config["depthmap_resolution"]) subshots = perspective_views_of_a_panorama( shot, subshot_width, urec, image_format, rig_instance_count ) else: logger.warning( f"Not undistorting {shot.id} with unknown camera type." ) continue for subshot in subshots: if tracks_manager: add_subshot_tracks(tracks_manager, utracks_manager, shot, subshot) undistorted_shots[shot.id] = subshots udata.save_undistorted_reconstruction([urec]) if tracks_manager: udata.save_undistorted_tracks_manager(utracks_manager) udata.save_undistorted_shot_ids( { shot_id: [ushot.id for ushot in ushots] for shot_id, ushots in undistorted_shots.items() } ) return undistorted_shots
def _transform_dense_point_cloud(udata: UndistortedDataSet, transformation, output_path): """Apply a transformation to the merged point cloud.""" A, b = transformation[:3, :3], transformation[:3, 3] input_path = udata.point_cloud_file() with io.open_rt(input_path) as fin: with io.open_wt(output_path) as fout: for i, line in enumerate(fin): if i < 13: fout.write(line) else: x, y, z, nx, ny, nz, red, green, blue = line.split() x, y, z = np.dot(A, map(float, [x, y, z])) + b nx, ny, nz = np.dot(A, map(float, [nx, ny, nz])) fout.write("{} {} {} {} {} {} {} {} {}\n".format( x, y, z, nx, ny, nz, red, green, blue))
def undistort_reconstruction( tracks_manager, reconstruction, data: DataSetBase, udata: UndistortedDataSet ): image_format = data.config["undistorted_image_format"] urec = types.Reconstruction() urec.points = reconstruction.points urec.reference = reconstruction.reference rig_instance_count = itertools.count() utracks_manager = pymap.TracksManager() logger.debug("Undistorting the reconstruction") undistorted_shots = {} for shot in reconstruction.shots.values(): if shot.camera.projection_type == "perspective": camera = perspective_camera_from_perspective(shot.camera) urec.add_camera(camera) subshots = [ get_shot_with_different_camera(urec, shot, camera, image_format) ] elif shot.camera.projection_type == "brown": camera = perspective_camera_from_brown(shot.camera) urec.add_camera(camera) subshots = [ get_shot_with_different_camera(urec, shot, camera, image_format) ] elif shot.camera.projection_type in ["fisheye", "fisheye_opencv"]: camera = perspective_camera_from_fisheye(shot.camera) urec.add_camera(camera) subshots = [ get_shot_with_different_camera(urec, shot, camera, image_format) ] elif pygeometry.Camera.is_panorama(shot.camera.projection_type): subshot_width = int(data.config["depthmap_resolution"]) subshots = perspective_views_of_a_panorama( shot, subshot_width, urec, image_format, rig_instance_count ) for subshot in subshots: if tracks_manager: add_subshot_tracks(tracks_manager, utracks_manager, shot, subshot) # pyre-fixme[61]: `subshots` may not be initialized here. undistorted_shots[shot.id] = subshots udata.save_undistorted_reconstruction([urec]) if tracks_manager: udata.save_undistorted_tracks_manager(utracks_manager) udata.save_undistorted_shot_ids( { shot_id: [ushot.id for ushot in ushots] for shot_id, ushots in undistorted_shots.items() } ) return undistorted_shots
def add_views_to_depth_estimator(data: UndistortedDataSet, neighbors, de): """Add neighboring views to the DepthmapEstimator.""" num_neighbors = data.config["depthmap_num_matching_views"] for shot in neighbors[:num_neighbors + 1]: assert shot.camera.projection_type == "perspective" color_image = data.load_undistorted_image(shot.id) mask = load_combined_mask(data, shot) gray_image = cv2.cvtColor(color_image, cv2.COLOR_RGB2GRAY) original_height, original_width = gray_image.shape width = min(original_width, int(data.config["depthmap_resolution"])) height = width * original_height // original_width image = scale_down_image(gray_image, width, height) mask = scale_down_image(mask, width, height, cv2.INTER_NEAREST) K = shot.camera.get_K_in_pixel_coordinates(width, height) R = shot.pose.get_rotation_matrix() t = shot.pose.translation de.add_view(K, R, t, image, mask)
def export(reconstruction, tracks_manager, udata: UndistortedDataSet, export_only): exporter = pydense.OpenMVSExporter() for camera in reconstruction.cameras.values(): if camera.projection_type == "perspective": w, h = camera.width, camera.height K = np.array([ [camera.focal * max(w, h), 0, (w - 1.0) / 2.0], [0, camera.focal * max(w, h), (h - 1.0) / 2.0], [0, 0, 1], ]) exporter.add_camera(str(camera.id), K, w, h) for shot in reconstruction.shots.values(): if export_only is not None and shot.id not in export_only: continue if shot.camera.projection_type == "perspective": image_path = udata._undistorted_image_file(shot.id) exporter.add_shot( str(os.path.abspath(image_path)), str(shot.id), str(shot.camera.id), shot.pose.get_rotation_matrix(), shot.pose.get_origin(), ) for point in reconstruction.points.values(): observations = tracks_manager.get_track_observations(point.id) if export_only is not None: shots = [k for k in observations if k in export_only] else: shots = list(observations) if shots: coordinates = np.array(point.coordinates, dtype=np.float64) exporter.add_point(coordinates, shots) io.mkdir_p(udata.data_path + "/openmvs") exporter.export(udata.data_path + "/openmvs/scene.mvs")
def export( reconstruction, index, image_graph, tracks_manager, base_output_path, data: DataSet, undistorted, udata: UndistortedDataSet, with_points, export_only, ): logger.info("Reconstruction %d" % index) output_path = os.path.join(base_output_path, "recon%d" % index) io.mkdir_p(output_path) io.mkdir_p(os.path.join(output_path, "visualize")) io.mkdir_p(os.path.join(output_path, "txt")) io.mkdir_p(os.path.join(output_path, "models")) shot_index = {image: i for i, image in enumerate(reconstruction.shots)} fvis = open(os.path.join(output_path, "vis.dat"), "w") fvis.write("VISDATA\n") fvis.write("%d\n" % len(shot_index)) for image, i in shot_index.items(): shot = reconstruction.shots[image] base = "%08d" % i logger.info("Image: %s %s" % (image, base)) # vis.dat for this image if image_graph: adj_indices = [] for adj_image in image_graph[image]: weight = image_graph[image][adj_image]["weight"] if weight > 0 and adj_image in shot_index: adj_indices.append(shot_index[adj_image]) num_covisible = len(adj_indices) fvis.write("%d " % i) fvis.write("%d " % num_covisible) for ai in adj_indices: fvis.write("%d " % ai) fvis.write("\n") # radially undistort the original image camera = shot.camera if undistorted: undistorted_image = udata.load_undistorted_image(image) else: original_image = data.load_image(image)[:, :, ::-1] original_h, original_w = original_image.shape[:2] K = camera.get_K_in_pixel_coordinates(original_w, original_h) distortion = np.array([camera.k1, camera.k2, 0, 0]) undistorted_image = cv2.undistort(original_image, K, distortion) # resize and save the undistorted to visualize/%08d.jpg resized_image = features.resized_image( undistorted_image, data.config["feature_process_size"]) new_image_path = os.path.join(output_path, "visualize", base + ".jpg") cv2.imwrite(new_image_path, resized_image) # write camera projection matrix to txt/%08d.txt resized_h, resized_w = resized_image.shape[:2] resized_K = camera.get_K_in_pixel_coordinates(resized_w, resized_h) P = resized_K.dot(shot.pose.get_world_to_cam()[:3]) new_txt = os.path.join(output_path, "txt", base + ".txt") with open(new_txt, "wb") as f: np.savetxt(f, P, str("%f"), header="CONTOUR") fvis.close() # txt with open(os.path.join(output_path, "pmvs_txt"), "w") as f: f.write("level 1\n") f.write("csize 2\n") f.write("threshold 0.7\n") f.write("wsize 7\n") f.write("minImageNum 3\n") f.write("CPU 8\n") f.write("setEdge 0\n") f.write("useBound 0\n") f.write("useVisData {}\n".format(int(image_graph is not None))) f.write("sequence -1\n") f.write("timages -1 0 %d\n" % len(shot_index)) f.write("oimages 0\n")
def export(reconstruction, tracks_manager, udata: UndistortedDataSet, with_points, export_only): lines = ["NVM_V3", "", str(len(reconstruction.shots))] shot_size_cache = {} shot_index = {} i = 0 skipped_shots = 0 for shot in reconstruction.shots.values(): if export_only is not None and shot.id not in export_only: skipped_shots += 1 continue q = tf.quaternion_from_matrix(shot.pose.get_rotation_matrix()) o = shot.pose.get_origin() shot_size_cache[shot.id] = udata.undistorted_image_size(shot.id) shot_index[shot.id] = i i += 1 if shot.camera.projection_type == "brown": # Will approximate Brown model, not optimal focal_normalized = (shot.camera.focal_x + shot.camera.focal_y) / 2.0 else: focal_normalized = shot.camera.focal words = [ image_path(shot.id, udata), focal_normalized * max(shot_size_cache[shot.id]), q[0], q[1], q[2], q[3], o[0], o[1], o[2], "0", "0", ] lines.append(" ".join(map(str, words))) # Adjust shots count lines[2] = str(int(lines[2]) - skipped_shots) if with_points: skipped_points = 0 lines.append("") points = reconstruction.points lines.append(str(len(points))) points_count_index = len(lines) - 1 for point_id, point in points.items(): shots = reconstruction.shots coord = point.coordinates color = list(map(int, point.color)) view_line = [] for shot_key, obs in tracks_manager.get_track_observations( point_id).items(): if export_only is not None and shot_key not in export_only: continue if shot_key in shots.keys(): v = obs.point x = (0.5 + v[0]) * shot_size_cache[shot_key][1] y = (0.5 + v[1]) * shot_size_cache[shot_key][0] view_line.append(" ".join( map(str, [shot_index[shot_key], obs.id, x, y]))) if len(view_line) > 1: lines.append(" ".join(map(str, coord)) + " " + " ".join(map(str, color)) + " " + str(len(view_line)) + " " + " ".join(view_line)) else: skipped_points += 1 # Adjust points count lines[points_count_index] = str( int(lines[points_count_index]) - skipped_points) else: lines += ["0", ""] lines += ["0", "", "0"] with io.open_wt(udata.data_path + "/reconstruction.nvm") as fout: fout.write("\n".join(lines))
def image_path(image, udata: UndistortedDataSet): """Path to the undistorted image relative to the dataset path.""" path = udata._undistorted_image_file(image) return os.path.relpath(path, udata.data_path)