def texture_mesh( self, openmvs_workspace_dp, openmvs_ifn, openmvs_ofn, export_type, lazy=False, ): # export_type: '.ply' or '.obj' logger.info("texture_mesh: ...") if not (export_type == "ply" or export_type == "obj"): logger.vinfo("export_type", export_type) assert False export_file = os.path.splitext(openmvs_ofn)[0] + "." + export_type if (not os.path.isfile(openmvs_ofn) or not os.path.isfile(export_file) or not lazy): texture_mesh_call = [ self.texture_mesh_fp, "-i", openmvs_ifn, "-w", openmvs_workspace_dp, "-o", openmvs_ofn, "--export-type", export_type, ] logger.info(str(texture_mesh_call)) pRecons = subprocess.Popen(texture_mesh_call) pRecons.wait() logger.info("texture_mesh: Done")
def _read_txt(self, stream): """ Load a PLY element from an ASCII-format PLY file. The element may contain list properties. """ self._data = _np.empty(self.count, dtype=self.dtype()) k = 0 for line in _islice(iter(stream.readline, b""), self.count): fields = iter(line.strip().split()) for prop in self.properties: try: self._data[prop.name][k] = prop._from_fields(fields) except StopIteration: raise PlyElementParseError("early end-of-line", self, k, prop) except ValueError: raise PlyElementParseError("malformed input", self, k, prop) try: next(fields) except StopIteration: pass else: from ssr.utility.logging_extension import logger logger.vinfo("fields", " ".join(fields)) logger.vinfo("k", " ".join(k)) raise PlyElementParseError("expected end-of-line", self, k) k += 1 if k < self.count: del self._data raise PlyElementParseError("early end-of-file", self, k)
def _check_decimation_backends(decimation_backends): valid_backends = [ DecimationBackends.meshlab, DecimationBackends.openmvs, ] for decimation_backend in decimation_backends: if not decimation_backend in valid_backends: logger.vinfo("decimation_backend", decimation_backend) logger.vinfo("decimation_backends", decimation_backends) assert False
def _check_texturing_backends(texturing_backends): valid_backends = [ TexturingBackends.openmvs, TexturingBackends.mve, ] for texturing_backend in texturing_backends: if not texturing_backend in valid_backends: logger.vinfo("texturing_backend", texturing_backend) logger.vinfo("texturing_backends", texturing_backends) assert False
def __init__(self): ssr_config = SSRConfig.get_instance() self.executable_fp = ssr_config.get_option_value( "meshlab_server_fp", str) self.meshlab_temp_dp = ssr_config.get_option_value_or_None( "meshlab_temp_dp", str) if self.meshlab_temp_dp is not None: if not os.path.isdir(self.meshlab_temp_dp): logger.vinfo("meshlab_temp_dp", self.meshlab_temp_dp) assert False, "Choose a valid path (in the config file) for Meshlab's temp directory"
def import_images_to_scene(self, image_idp): logger.info("import_images_to_scene: ...") make_scene_call = [ self.make_scene_fp, "--images-only", image_idp, self.workspace_folder, ] logger.vinfo("make_scene_call", make_scene_call) subprocess.call(make_scene_call)
def process_meshing_task(self, meshing_task, lazy): mesh_ply_ofn = meshing_task.mesh_ply_ofn plain_mesh_ply_ofn = meshing_task.plain_mesh_ply_ofn logger.vinfo("mesh_ply_ofn", mesh_ply_ofn) if meshing_task.meshing_backend == MeshingBackends.colmap_poisson: mkdir_safely(meshing_task.mesh_odp) MeshingStep.compute_mesh_with_colmap( meshing_task.colmap_idp, meshing_task.mesh_odp, mesh_ply_ofn, plain_mesh_ply_ofn, "poisson_mesher", poisson_trim_thresh=10, lazy=lazy, ) elif meshing_task.meshing_backend == MeshingBackends.colmap_delaunay: mkdir_safely(meshing_task.mesh_odp) MeshingStep.compute_mesh_with_colmap( meshing_task.colmap_idp, meshing_task.mesh_odp, mesh_ply_ofn, plain_mesh_ply_ofn, "delaunay_mesher", lazy=lazy, ) elif meshing_task.meshing_backend == MeshingBackends.openmvs: mkdir_safely(meshing_task.mesh_odp) MeshingStep.compute_mesh_with_openmvs( meshing_task.colmap_idp, meshing_task.mesh_odp, mesh_ply_ofn, plain_mesh_ply_ofn, lazy=lazy, ) elif meshing_task.meshing_backend == MeshingBackends.mve_fssr: MeshingStep.compute_mesh_with_mve( meshing_task.colmap_idp, meshing_task.mesh_odp, mesh_ply_ofn, plain_mesh_ply_ofn, meshing_algo="fssr", lazy=lazy, ) elif meshing_task.meshing_backend == MeshingBackends.mve_gdmr: MeshingStep.compute_mesh_with_mve( meshing_task.colmap_idp, meshing_task.mesh_odp, mesh_ply_ofn, plain_mesh_ply_ofn, meshing_algo="gdmr", lazy=lazy, ) else: assert False
def compute_mesh_with_openmvs(colmap_idp, odp, mesh_ply_ofn, plain_mesh_ply_ofn, lazy=False): logger.info("compute_mesh_with_openmvs: ...") # https://github.com/cdcseacave/openMVS/wiki/Usage # Exporting and Viewing Results # Each of the following commands also writes a PLY file that can be # used with many third-party tools. Alternatively, Viewer can be used # to export the MVS projects to PLY or OBJ formats. interface_mvs_fn = "interface_colmap.mvs" # This will create a "plain_mesh.ply" file mesh_mvs_fn = os.path.splitext(mesh_ply_ofn)[0] + ".mvs" openmvs_mvs_reconstructor = OpenMVSReconstructor() # This step already imports the dense point cloud computed by colmap # (Stored in fused.ply and fused.ply.vis) openmvs_mvs_reconstructor.convert_colmap_to_openMVS( colmap_dense_idp=colmap_idp, openmvs_workspace_dp=odp, openmvs_ofn=interface_mvs_fn, image_folder="images/", lazy=lazy, ) logger.vinfo("ofp", os.path.join(odp, mesh_ply_ofn)) # Adjust the default parameters (min_point_distance=2.5, # smoothing_iterations=2) to improve satellite reconstruction results min_point_distance = 0 smoothing_iterations = 0 openmvs_mvs_reconstructor.reconstruct_mesh( odp, interface_mvs_fn, mesh_mvs_fn, # OpenMVS Default Value: min_point_distance=2.5 min_point_distance=min_point_distance, # Clean options: # OpenMVS Default Value: decimate_value=1.0 decimate_value=1.0, # OpenMVS Default Value: smoothing_iterations=2 smoothing_iterations=smoothing_iterations, lazy=lazy, ) mesh_ply_ofp = os.path.join(odp, mesh_ply_ofn) plain_mesh_ply_ofp = os.path.join(odp, plain_mesh_ply_ofn) meshlab = Meshlab() meshlab.remove_color(mesh_ply_ofp, plain_mesh_ply_ofp) logger.info("compute_mesh_with_openmvs: Done")
def parse_ply_file(ifp): logger.info("Parse PLY File: ...") logger.vinfo("ifp", ifp) ply_data = PlyData.read(ifp) vertices, _, _ = PLYFileHandler.__ply_data_vertices_to_vetex_list( ply_data) faces, _, _ = PLYFileHandler.__ply_data_faces_to_face_list(ply_data) logger.info("Parse PLY File: Done") return vertices, faces
def compute_depth_maps(self, downscale_level=0): logger.info("compute_depth_maps: ...") downscale_s_str = "-s" + str(downscale_level) dm_recon_call = [ self.dm_recon_fp, downscale_s_str, self.workspace_folder, ] logger.vinfo("dm_recon_call", dm_recon_call) subprocess.call(dm_recon_call) logger.info("compute_depth_maps: ...")
def _check_meshing_backends(meshing_backends): valid_backends = [ MeshingBackends.colmap_poisson, MeshingBackends.colmap_delaunay, MeshingBackends.openmvs, MeshingBackends.mve_fssr, MeshingBackends.mve_gdmr, ] for meshing_backend in meshing_backends: if not meshing_backend in valid_backends: logger.vinfo("meshing_backend", meshing_backend) logger.vinfo("valid_backends", valid_backends) assert False
def compute_mesh_with_mve( colmap_idp, odp, mesh_ply_ofn, plain_mesh_ply_ofn, meshing_algo, lazy=False, ): logger.info("compute_mesh_with_mve: ...") mesh_ply_ofp = os.path.join(odp, mesh_ply_ofn) plain_mesh_ply_ofp = os.path.join(odp, plain_mesh_ply_ofn) downscale_level = 0 mve_mvs_reconstructor = MVEMVSReconstructor() mve_mvs_reconstructor.create_scene_from_sfm_result( colmap_idp, odp, downscale_level=downscale_level, lazy=lazy) mve_mvs_reconstructor.compute_dense_point_cloud_from_depth_maps( odp, downscale_level=downscale_level, view_ids=None, fssr_output=True, lazy=lazy, ) meshlab = Meshlab() if meshing_algo == "fssr": mve_mvs_reconstructor.compute_fssr_reconstruction( odp, mesh_ply_ofp=mesh_ply_ofp, lazy=lazy) stem, ext = os.path.splitext(mesh_ply_ofp) cleaned_mesh_ply_ofp = stem + "_cleaned" + ext mve_mvs_reconstructor.compute_clean_mesh( odp, mesh_ply_ifp=mesh_ply_ofp, mesh_cleaned_ply_ofp=cleaned_mesh_ply_ofp, delete_color=False, lazy=lazy, ) meshlab.remove_color(cleaned_mesh_ply_ofp, plain_mesh_ply_ofp) elif meshing_algo == "gdmr": mve_mvs_reconstructor.compute_gdmr_reconstruction( odp, mesh_ply_ofp=mesh_ply_ofp, lazy=lazy) meshlab.remove_color(mesh_ply_ofp, plain_mesh_ply_ofp) else: logger.vinfo("meshing_algo", meshing_algo) assert False logger.vinfo("ofp", mesh_ply_ofp) logger.info("compute_mesh_with_mve: Done")
def create_scene_from_sfm_result(self, sfm_fp_or_dp, downscale_level=None): # Input can be a nvm file or a colmap model folder assert self.workspace_folder is not None logger.info("Creating Scene: ...") logger.info("Input file/folder: " + sfm_fp_or_dp) logger.info("Output Path: " + self.workspace_folder) downscale_s_str = "-s" + str(downscale_level) make_scene_call = [ self.make_scene_fp, downscale_s_str, sfm_fp_or_dp, self.workspace_folder, ] logger.vinfo("make_scene_call", make_scene_call) subprocess.call(make_scene_call) logger.info("Creating Scene: Done")
def perform_pan_sharpening(pan_ifp, msi_ifp, ofp, resampling_algorithm="cubic"): # https://gdal.org/programs/gdal_pansharpen.html # https://gis.stackexchange.com/questions/270476/pansharpening-using-gdal-tools # GDAL pan sharpening algorithm = weighted Brovey algorithm ext = os.path.splitext(ofp)[1] of = ext[1:] call_params = ["gdal_pansharpen.py"] call_params += ["-of", of, "-r", resampling_algorithm] call_params += [pan_ifp, msi_ifp, ofp] logger.vinfo("call_params", call_params) sharp_process = subprocess.Popen(call_params) sharp_process.wait()
def __init__(self, config_fp, working_file_suffix=None): self.config_fp = config_fp self.config = configparser.RawConfigParser() if not os.path.isfile(self.config_fp): abs_path = os.path.abspath(os.path.dirname(self.config_fp)) if not os.path.isdir(abs_path): logger.vinfo("abs_path", abs_path) assert False # config folder missing open(self.config_fp, "a").close() else: self.config.read(self.config_fp) if working_file_suffix is not None: self.path_to_working_copy = self.config_fp + working_file_suffix else: self.path_to_working_copy = self.config_fp
def compute_intrinsic_skew_decomposition(intrinsic_mat): f_x, f_y, skew, p_x, p_y = Intrinsics.split_intrinsic_mat( intrinsic_mat ) intrinsic_mat_wo_skew = np.array( [[f_x, 0, p_x - skew * p_y / f_y], [0, f_y, p_y], [0, 0, 1]], dtype=float, ) skew_mat = np.array( [[1, skew / f_y, 0], [0, 1, 0], [0, 0, 1]], dtype=float ) if not np.allclose(intrinsic_mat, skew_mat @ intrinsic_mat_wo_skew): err_mat = intrinsic_mat - skew_mat @ intrinsic_mat_wo_skew logger.vinfo("err_mat\n", err_mat) assert False return skew_mat, intrinsic_mat_wo_skew
def write_ply_file( ofp, vertices, with_colors=True, with_normals=False, faces=None, plain_text_output=False, with_measurements=False, ): logger.info("write_ply_file: " + ofp) ply_data_vertex_data_dtype_list = PLYFileHandler.build_type_list( vertices, with_colors, with_normals, with_measurements) logger.vinfo("ply_data_vertex_data_dtype_list", ply_data_vertex_data_dtype_list) output_ply_data_vertex_element = ( PLYFileHandler.__vertices_to_ply_vertex_element( vertices, ply_data_vertex_data_dtype_list)) if faces is None or len(faces) == 0: logger.info("Write File With Vertices Only (no faces)") output_data = PlyData([output_ply_data_vertex_element], text=plain_text_output) else: logger.info("Write File With Faces") logger.info("Number faces" + str(len(faces))) ply_data_face_data_type = [("vertex_indices", "i4", (3, ))] # we do not define colors for faces, # since we use the vertex colors to colorize the face output_ply_data_face_element = ( PLYFileHandler.__faces_to_ply_face_element( faces, ply_data_face_data_type)) output_data = PlyData( [output_ply_data_vertex_element, output_ply_data_face_element], text=plain_text_output, ) output_data.write(ofp)
def reconstruct_mesh_with_delaunay( self, reconstruction_idp, mesh_ply_ofp, max_proj_dist=None, max_depth_dist=None, lazy=False, ): logger.info("reconstruct_mesh: ...") assert os.path.isdir(self.colmap_exe_dp) assert os.path.isfile(os.path.join(self.colmap_exe_dp, "colmap")) if not os.path.isfile(mesh_ply_ofp) or not lazy: os.environ["PATH"] += os.pathsep + self.colmap_exe_dp dense_mesher_call = [ "colmap", "delaunay_mesher", "--input_path", reconstruction_idp, "--output_path", mesh_ply_ofp, "--input_type", "dense", ] if max_proj_dist is not None: dense_mesher_call += [ "--DelaunayMeshing.max_proj_dist", str(max_proj_dist), ] if max_depth_dist is not None: dense_mesher_call += [ "--DelaunayMeshing.max_depth_dist", str(max_depth_dist), ] logger.vinfo("dense_mesher_call: ", dense_mesher_call) subprocess.call(dense_mesher_call) logger.info("reconstruct_mesh: Done")
def sample_mesh(self, mesh_ifp, point_cloud_ofp, num_vertices, sampling_method): # While meshlab provides a variety of sampling methods, # Cloudcompare provides only a single method that randomly samples # points. assert sampling_method in [ "poisson_disk", "stratified_triangle", "montecarlo", ], f"Received unsupported sampling method: {sampling_method}" # Poisson-disk Sampling # Requires the definition of # number of samples OR radius or percentage # Stratified Triangle Sampling # Requires the definition of number of samples # Results look better than Monte Carlo Sampling template_fp = self._create_lmx_template(sampling_method + "_sampling.mlx") logger.vinfo("template_fp", template_fp) assert os.path.isfile(template_fp) _MLXFileHandler.set_value(template_fp, "SampleNum", num_vertices) num_vertices_str = _MLXFileHandler.get_value_as_str( template_fp, "SampleNum") assertion_msg = f"{num_vertices_str} vs {num_vertices}" assert int(num_vertices_str) == num_vertices, assertion_msg options = [] options += ["-i", mesh_ifp] options += ["-s", template_fp] options += ["-o", point_cloud_ofp] call_list = [self.executable_fp] + options logger.vinfo("call_list", call_list) subprocess.call(call_list) # Remove the file from the file system os.unlink(template_fp)
def create_textured_mesh( self, mve_scene_idp, untextured_mesh_ifp, texture_odp, occlusion_handling=True, ): logger.vinfo("texture_odp", texture_odp) current_working_dir = os.getcwd() if not os.path.isdir(texture_odp): os.makedirs(texture_odp) # Change the directory, since texrecon creates the files in the current # directory os.chdir(texture_odp) logger.vinfo("texture_odp", texture_odp) options = [] options += ["--keep_unseen_faces"] if occlusion_handling: # Only the following options created reasonable results: options += ["--data_term=area", "--outlier_removal=gauss_damping"] texturing_call = ([self.texrecon_executable] + options + [ mve_scene_idp + "::undistorted", untextured_mesh_ifp, "textured", ]) logger.vinfo("texturing_call", texturing_call) subprocess.call(texturing_call) # reset the working directory os.chdir(current_working_dir)
def compute_mesh_with_colmap( colmap_idp, mesh_odp, mesh_ply_ofn, plain_mesh_ply_ofn, meshing_algo, poisson_trim_thresh=10, lazy=False, ): """Poisson meshing works with a single point-cloud-ply-file, whereas delaunay meshing requires a full workspace. """ logger.info("compute_mesh_with_colmap: ...") assert meshing_algo in ["poisson_mesher", "delaunay_mesher"] mesh_ply_ofp = os.path.join(mesh_odp, mesh_ply_ofn) logger.vinfo("ofp", mesh_ply_ofp) colmap_mvs_reconstructor = ColmapMVSReconstructor() if meshing_algo == "poisson_mesher": point_cloud_ply_ifp = os.path.join(colmap_idp, "fused.ply") colmap_mvs_reconstructor.reconstruct_mesh_with_poisson( point_cloud_ply_ifp, mesh_ply_ofp, poisson_trim_thresh, lazy=lazy, ) elif meshing_algo == "delaunay_mesher": colmap_mvs_reconstructor.reconstruct_mesh_with_delaunay( colmap_idp, mesh_ply_ofp, # https://colmap.github.io/faq.html max_proj_dist=0, # Colmap Default Value: 2.5 lazy=lazy, ) plain_mesh_ply_ofp = os.path.join(mesh_odp, plain_mesh_ply_ofn) meshlab = Meshlab() meshlab.remove_color(mesh_ply_ofp, plain_mesh_ply_ofp) logger.info("compute_mesh_with_colmap: Done")
def compute_dense_point_cloud_from_depth_maps( self, mve_point_cloud_ply_ofp, downscale_level=0, view_ids=None, fssr_output=True, ): logger.info("compute_dense_point_cloud_from_depth_maps: ...") assert os.path.splitext(mve_point_cloud_ply_ofp)[1] == ".ply" scene2pset_call = [self.scene2pset_fp] if fssr_output: downscale_F_str = "-F" + str(downscale_level) scene2pset_call.append(downscale_F_str) if view_ids is not None: view_id_str = "--views=" + ",".join(map(str, view_ids)) scene2pset_call.append(view_id_str) scene2pset_call.append(self.workspace_folder) scene2pset_call.append(mve_point_cloud_ply_ofp) logger.vinfo("scene2pset_call", scene2pset_call) subprocess.call(scene2pset_call) logger.info("compute_dense_point_cloud_from_depth_maps: Done")
def reconstruct_mesh_with_poisson(self, point_cloud_ply_ifp, mesh_ply_ofp, poisson_trim_thresh, lazy): logger.info("reconstruct_mesh: ...") logger.vinfo("mesh_ply_ofp", mesh_ply_ofp) assert os.path.isdir(self.colmap_exe_dp) assert os.path.isfile(os.path.join(self.colmap_exe_dp, "colmap")) if not os.path.isfile(mesh_ply_ofp) or not lazy: os.environ["PATH"] += os.pathsep + self.colmap_exe_dp trim_thresh_str = str(poisson_trim_thresh) dense_mesher_call = [ "colmap", "poisson_mesher", "--input_path", point_cloud_ply_ifp, "--output_path", mesh_ply_ofp, "--PoissonMeshing.trim", trim_thresh_str, ] logger.vinfo("dense_mesher_call: ", dense_mesher_call) subprocess.call(dense_mesher_call) logger.info("reconstruct_mesh: Done")
def compute_sfm(self): logger.info("Compute SfM: ...") sfm_recon_call = [self.sfm_recon_fp, self.workspace_folder] logger.vinfo("sfm_recon_call", sfm_recon_call) subprocess.call(sfm_recon_call) logger.info("Compute SfM: Done")
def convert_depth_map_to_cam_coords( self, depth_map, depth_map_semantic, shift_to_pixel_center, # False for Colmap, True for MVE depth_map_display_sparsity=100, ): assert 0 < depth_map_display_sparsity height, width = depth_map.shape logger.info("height " + str(height)) logger.info("width " + str(width)) if self.height == height and self.width == width: x_step_size = 1.0 y_step_size = 1.0 else: x_step_size = self.width / width y_step_size = self.height / height logger.info("x_step_size " + str(x_step_size)) logger.info("y_step_size " + str(y_step_size)) fx, fy, skew, cx, cy = self.split_intrinsic_mat( self.get_calibration_mat()) logger.vinfo("fx, fy, skew, cx, cy: ", str([fx, fy, skew, cx, cy])) indices = np.indices((height, width)) y_index_list = indices[0].flatten() x_index_list = indices[1].flatten() depth_values = depth_map.flatten() assert len(x_index_list) == len(y_index_list) == len(depth_values) if shift_to_pixel_center: # https://github.com/simonfuhrmann/mve/blob/master/libs/mve/depthmap.cc # math::Vec3f v = invproj * math::Vec3f( # (float)x + 0.5f, (float)y + 0.5f, 1.0f); u_index_coord_list = x_step_size * x_index_list + 0.5 v_index_coord_list = y_step_size * y_index_list + 0.5 else: # https://github.com/colmap/colmap/blob/dev/src/base/reconstruction.cc # COLMAP assumes that the upper left pixel center is (0.5, 0.5) # i.e. pixels are already shifted u_index_coord_list = x_step_size * x_index_list v_index_coord_list = y_step_size * y_index_list # The cannoncial vectors are defined according to p.155 of # "Multiple View Geometry" by Hartley and Zisserman using a canonical # focal length of 1 , i.e. vec = [(x - cx) / fx, (y - cy) / fy, 1] skew_correction = (cy - v_index_coord_list) * skew / (fx * fy) x_coords_canonical = (u_index_coord_list - cx) / fx + skew_correction y_coords_canonical = (v_index_coord_list - cy) / fy z_coords_canonical = np.ones(len(depth_values), dtype=float) # Determine non-background data # non_background_flags = np.logical_not(np.isnan(depth_values)) depth_values_not_nan = np.nan_to_num(depth_values) non_background_flags = depth_values_not_nan > 0 x_coords_canonical_filtered = x_coords_canonical[non_background_flags] y_coords_canonical_filtered = y_coords_canonical[non_background_flags] z_coords_canonical_filtered = z_coords_canonical[non_background_flags] depth_values_filtered = depth_values[non_background_flags] if depth_map_display_sparsity != 100: x_coords_canonical_filtered = x_coords_canonical_filtered[:: depth_map_display_sparsity] y_coords_canonical_filtered = y_coords_canonical_filtered[:: depth_map_display_sparsity] z_coords_canonical_filtered = z_coords_canonical_filtered[:: depth_map_display_sparsity] depth_values_filtered = depth_values_filtered[:: depth_map_display_sparsity] if depth_map_semantic == Camera.DEPTH_MAP_WRT_CANONICAL_VECTORS: # In this case, the depth values are defined w.r.t. the canonical # vectors. This kind of depth data is used by Colmap. x_coords_filtered = (x_coords_canonical_filtered * depth_values_filtered) y_coords_filtered = (y_coords_canonical_filtered * depth_values_filtered) z_coords_filtered = (z_coords_canonical_filtered * depth_values_filtered) elif depth_map_semantic == Camera.DEPTH_MAP_WRT_UNIT_VECTORS: # In this case the depth values are defined w.r.t. the normalized # canonical vectors. This kind of depth data is used by MVE. cannonical_norms_filtered = np.linalg.norm( np.array( [ x_coords_canonical_filtered, y_coords_canonical_filtered, z_coords_canonical_filtered, ], dtype=float, ), axis=0, ) # Instead of normalizing the x,y and z component, we divide the # depth values by the corresponding norm. normalized_depth_values_filtered = (depth_values_filtered / cannonical_norms_filtered) x_coords_filtered = (x_coords_canonical_filtered * normalized_depth_values_filtered) y_coords_filtered = (y_coords_canonical_filtered * normalized_depth_values_filtered) z_coords_filtered = (z_coords_canonical_filtered * normalized_depth_values_filtered) else: assert False cam_coords = np.dstack( (x_coords_filtered, y_coords_filtered, z_coords_filtered))[0] return cam_coords
if __name__ == "__main__": f_x = 2800 f_y = 2100 s = 0.3 p_x = 2355 p_y = 2500 f_x_ = 2200 f_y_ = 2400 s_ = 0.1 p_x_ = 1600 p_y_ = 800 intrinsic_1 = np.array( [[f_x, s, p_x], [0, f_y, p_y], [0, 0, 1]], dtype=float ) intrinsic_2 = np.array( [[f_x_, s_, p_x_], [0, f_y_, p_y_], [0, 0, 1]], dtype=float ) trans_mat_2_to_1 = Intrinsics.compute_intrinsic_transformation( intrinsic_1, intrinsic_2, check_result=True ) intrinsic_1_restored = trans_mat_2_to_1 @ intrinsic_2 logger.vinfo("trans_mat_2_to_1", trans_mat_2_to_1) logger.vinfo("intrinsic_1", intrinsic_1) logger.vinfo("intrinsic_1_restored", intrinsic_1_restored)
def run(self, reconstruct_sfm_mvs): dataset_dp = self.ssr_config.get_option_value( "satellite_image_pan_dp", str ) workspace_dp = self.ssr_config.get_option_value( "workspace_vissat_dp", str ) mkdir_safely(workspace_dp) create_vissat_config_from_ssr_config( vissat_config_ofp=self.pm.vissat_config_fp, dataset_dp=dataset_dp, workspace_dp=workspace_dp, ssr_config=self.ssr_config, clean_data=True, crop_image=True, derive_approx=True, choose_subset=True, colmap_sfm_perspective=True, inspect_sfm_perspective=True, reparam_depth=True, colmap_mvs=True, aggregate_2p5d=True, aggregate_3d=True, ) if reconstruct_sfm_mvs: logger.vinfo("self.pm.vissat_config_fp", self.pm.vissat_config_fp) assert os.path.isdir(self.colmap_vissat_exe_dp) os.environ["PATH"] += os.pathsep + self.colmap_vissat_exe_dp # see https://github.com/Kai-46/VisSatSatelliteStereo/blob/master/stereo_pipeline.py from stereo_pipeline import StereoPipeline as VisSatStereoPipeline # https://github.com/Kai-46/VisSatSatelliteStereo # Our pipeline is written in a module way; you can run it step by step # by choosing what steps to execute in the configuration file. # Steps to run # clean_data (see clean_data() in clean_data.py) # creates the folder "cleaned_data" # copies the NTF-files contained in the input PAN folder # extracts the xml- and the jpg-files contained in the tar-files in the input PAN folder # crop_image (see image_crop() and image_crop_worker() in image_crop.py) # creates the folder "images" # uses the bounding box specified in the config-json-file to crop the corresponding area # of the NTF-files contained in the "cleaned_data" folder # derive_approx # choose_subset # colmap_sfm_perspective # inspect_sfm_perspective # reparam_depth # colmap_mvs # aggregate_2p5d # creates the folder "mvs_results/aggregate_2p5d" # with a height-colorized point cloud and corresponding images # and a geo-registered geo-tiff file # aggregate_3d # creates the folder "mvs_results/aggregate_3d" # with a 3d point cloud and corresponding images # and a geo-registered geo-tiff file pipeline = VisSatStereoPipeline(self.pm.vissat_config_fp) # Logs are created in the working_directory/logs folder pipeline.run()
def perform_mvs( self, ifp, openmvs_workspace, openmvs_ofp, image_idp=None, # For NVM input lazy=False, ): # ifp can be an .mvs (OpenMVS) or a .nvm (VisualSfM) or .bin/.json (OpenMVG) file # openmvs_ofp can be an .obj or an.ply file logger.info("perform_mvs : ...") logger.vinfo("openmvs_ofp", openmvs_ofp) if not os.path.isdir(openmvs_workspace): os.mkdir(openmvs_workspace) openmvs_workspace_temp = os.path.join(openmvs_workspace, "temp") if not os.path.isdir(openmvs_workspace_temp): os.mkdir(openmvs_workspace_temp) if os.path.splitext(ifp)[1] == ".mvs": logger.info("OpenMVS file detected") openmvs_ifp = ifp elif os.path.splitext(ifp)[1] == ".nvm": logger.info("NVM file detected") assert image_idp is not None openmvs_ifp = os.path.splitext(ifp)[0] + ".mvs" # TODO: This folder is only created if the parameters actually show # a distortion coefficient. Otherwise the original image folder # is referenced in the mvs file # undistorted_image_fp = os.path.join( # os.path.dirname(openmvs_ifp), "undistorted_images/" # ) self.convert_visualsfm_to_openMVS( nvm_ifp=ifp, image_idp=image_idp, openmvs_workspace_temp=openmvs_workspace_temp, openmvs_ofp=openmvs_ifp, ) elif (os.path.splitext(ifp)[1] == ".bin" or os.path.splitext(ifp)[1] == ".json"): logger.info("OpenMVG file detected") assert image_idp is not None openmvs_ifp = os.path.splitext(ifp)[0] + ".mvs" from ssr.sfm_utility.openmvg.openmvg_sfm import ( OpenMVGReconstructor, ) OpenMVGReconstructor.export_to_openmvs(ifp=ifp, ofp=openmvs_ifp) else: assert False dense_mvs_fn = "dense.mvs" dense_mvs_fp = os.path.join(openmvs_workspace, dense_mvs_fn) self.densify_point_cloud(openmvs_ifp, openmvs_workspace_temp, dense_mvs_fp, lazy) mesh_mvs_fn = "dense_mesh.mvs" mesh_mvs_fp = os.path.join(openmvs_workspace, mesh_mvs_fn) self.reconstruct_mesh(dense_mvs_fp, openmvs_workspace_temp, mesh_mvs_fp, lazy) texture_mvs_fn = "dense_mesh_texture.mvs" texture_mvs_fp = os.path.join(openmvs_workspace, texture_mvs_fn) export_type = "obj" self.texture_mesh( mesh_mvs_fp, openmvs_workspace_temp, texture_mvs_fp, export_type, lazy, ) texture_ply_fp = (os.path.splitext(texture_mvs_fp)[0] + "." + export_type) shutil.copyfile(texture_ply_fp, openmvs_ofp) if export_type == "ply": shutil.copyfile( os.path.splitext(texture_ply_fp)[0] + ".png", os.path.splitext(openmvs_ofp)[0] + ".png", ) elif export_type == "obj": shutil.copyfile( os.path.splitext(texture_ply_fp)[0] + ".mtl", os.path.splitext(openmvs_ofp)[0] + ".mtl", ) shutil.copyfile( os.path.join( os.path.dirname(texture_ply_fp), "dense_mesh_texture_material_0_map_Kd.jpg", ), os.path.join( os.path.dirname(openmvs_ofp), "dense_mesh_texture_material_0_map_Kd.jpg", ), ) logger.info("perform_mvs : Done")
def compute_skew_free_camera_models( colmap_model_with_skew_idp, gray_image_with_skew_idp, color_image_with_skew_idp, depth_map_with_skew_idp, colmap_model_no_skew_odp, gray_image_no_skew_odp, color_image_no_skew_odp, depth_map_no_skew_odp, perform_warping_evaluation=False, interpolation_type=cv2.INTER_CUBIC, ): assert colmap_model_with_skew_idp != colmap_model_no_skew_odp assert gray_image_with_skew_idp != gray_image_no_skew_odp assert depth_map_with_skew_idp != depth_map_no_skew_odp if gray_image_no_skew_odp is not None: mkdir_safely(gray_image_no_skew_odp) if color_image_no_skew_odp is not None: mkdir_safely(color_image_no_skew_odp) if depth_map_no_skew_odp is not None: mkdir_safely(depth_map_no_skew_odp) logger.vinfo("image_without_skew_odp", gray_image_no_skew_odp) logger.vinfo("interpolation_type", interpolation_type) cameras = ColmapFileHandler.parse_colmap_cams( colmap_model_with_skew_idp, gray_image_with_skew_idp ) # cameras, _ = ColmapFileHandler.parse_colmap_model_folder( # colmap_model_idp, image_idp # ) num_cameras = len(cameras) skew_free_camera_list = [] pil_better_count = 0 mat_better_count = 0 patt_count = 0 for camera in cameras: logger.vinfo("camera.id", str(camera.id) + " of " + str(num_cameras)) logger.vinfo("camera.file_name", camera.file_name) intrinsic_mat = camera.get_calibration_mat() ( skew_mat, intrinsic_mat_wo_skew, ) = Camera.compute_intrinsic_skew_decomposition(intrinsic_mat) image_invert_skew_mat = np.linalg.inv(skew_mat) image_ifn = camera.file_name image_stem, ext = os.path.splitext(image_ifn) if gray_image_with_skew_idp is not None: image_ifp = os.path.join(gray_image_with_skew_idp, image_ifn) mat_image_ofp = os.path.join( gray_image_no_skew_odp, image_stem + ext ) mat_image = imageio.imread(image_ifp) mat_image_skew_free = remove_skew_from_matrix( mat_image, image_invert_skew_mat, interpolation_type=interpolation_type, ) imageio.imwrite(mat_image_ofp, mat_image_skew_free) if color_image_with_skew_idp is not None: image_ifp = os.path.join(color_image_with_skew_idp, image_ifn) mat_image_ofp = os.path.join( color_image_no_skew_odp, image_stem + ext ) mat_image = imageio.imread(image_ifp) mat_image_skew_free = remove_skew_from_matrix( mat_image, image_invert_skew_mat, interpolation_type=interpolation_type, ) imageio.imwrite(mat_image_ofp, mat_image_skew_free) if depth_map_with_skew_idp is not None: depth_map_fn = get_corresponding_depth_map_fn(image_ifn) depth_ifp = os.path.join(depth_map_with_skew_idp, depth_map_fn) depth_ofp = os.path.join(depth_map_no_skew_odp, depth_map_fn) depth_mat = parse_depth_map(depth_ifp) depth_mat_transformed = remove_skew_from_matrix( depth_mat, image_invert_skew_mat, interpolation_type=interpolation_type, ) write_depth_map(depth_mat_transformed, depth_ofp) # if perform_warping_evaluation: # # logger.info('--------------------- Diffs --------------------') # # mat_image_recovered_mat = np.array( # remove_skew_from_matrix(mat_image_skew_free, skew_mat) # ) # mat_image_diff_mat = np.abs(mat_image - mat_image_recovered_mat) # mat_image_error_ofp = os.path.join( # gray_image_no_skew_odp, image_stem + '_mat_error' + ext # ) # imageio.imwrite(mat_image_error_ofp, mat_image_diff_mat) # mat_diff = np.nansum(mat_image_diff_mat) # logger.vinfo('mat_diff', mat_diff) # # # pil_image = PILImage.open(image_ifp) # # pil_image_transformed = remove_skew_from_images_legazy( # pil_image, image_invert_skew_mat # ) # # pil_image_transformed.save(pil_image_ofp) # # # # pil_image_recovered_mat = np.array( # remove_skew_from_images_legazy( # pil_image_transformed, skew_mat # ) # ) # # pil_image_mat = np.array(pil_image) # # pil_image_diff_mat = np.abs( # pil_image_mat - pil_image_recovered_mat # ) # # pil_image_error_ofp = os.path.join( # image_without_skew_odp, image_stem + '_pil_error' + ext # ) # # imageio.imwrite(pil_image_error_ofp, pil_image_diff_mat) # # pil_diff = np.nansum(pil_image_diff_mat) # # logger.vinfo('pil_diff', pil_diff) # # # # if pil_diff > mat_diff: # # mat_better_count += 1 # # elif mat_diff > pil_diff: # # pil_better_count += 1 # # else: # # patt_count += 1 skew_free_camera = copy.copy(camera) skew_free_camera.set_calibration( intrinsic_mat_wo_skew, radial_distortion=False ) skew_free_camera_list.append(skew_free_camera) if perform_warping_evaluation: logger.vinfo("pil_better_count", pil_better_count) logger.vinfo("mat_better_count", mat_better_count) logger.vinfo("patt_count", patt_count) if colmap_model_no_skew_odp is not None: mkdir_safely(colmap_model_no_skew_odp) ColmapFileHandler.write_colmap_cameras( odp=colmap_model_no_skew_odp, cameras=skew_free_camera_list, colmap_camera_model_name="PINHOLE", )
def recover_depth_maps( model_with_skew_idp, last_rows_ifp, image_with_skew_idp, depth_map_reparam_with_skew_idp, depth_map_real_with_skew_odp, depth_map_type="geometric", # Optional parameters check_inv_proj_mat=False, inv_proj_mat_ifp=None, check_depth_mat_storing=False, create_depth_map_point_cloud=False, depth_map_point_cloud_odp=None, create_depth_map_point_cloud_reference=False, depth_map_point_cloud_reference_odp=None, ): mkdir_safely(depth_map_real_with_skew_odp) if create_depth_map_point_cloud: mkdir_safely(depth_map_point_cloud_odp) if create_depth_map_point_cloud_reference: mkdir_safely(depth_map_point_cloud_reference_odp) assert depth_map_type in ["geometric", "photometric"] inv_proj_mats = None if check_inv_proj_mat: inv_proj_mats = parse_inv_proj_mats(inv_proj_mat_ifp) last_rows = parse_last_rows(last_rows_ifp) depth_map_suffix = "." + depth_map_type + ".bin" depth_map_reparam_ifp_list = get_file_paths_in_dir( depth_map_reparam_with_skew_idp, ext=".bin", target_str_or_list=depth_map_suffix, ) cameras, points3D = ColmapFileHandler.parse_colmap_model_folder( model_with_skew_idp, image_with_skew_idp) # cache = PythonCache() # cameras, points3D = cache.get_cached_result( # callback=ColmapFileHandler.parse_colmap_model_folder, # params=[model_with_skew_idp, image_with_skew_idp], # unique_id_or_path=1, # ) depth_map_semantic = Camera.DEPTH_MAP_WRT_CANONICAL_VECTORS logger.vinfo("depth_map_real_with_skew_odp", depth_map_real_with_skew_odp) for camera in cameras[::-1]: img_name = camera.file_name logger.vinfo("img_name", img_name) depth_map_name = img_name + "." + depth_map_type + ".bin" depth_map_reparam_with_skew_ifp = os.path.join( depth_map_reparam_with_skew_idp, depth_map_name) depth_map_real_with_skew_ofp = os.path.join( depth_map_real_with_skew_odp, depth_map_name) depth_map_point_cloud_ofp = None if create_depth_map_point_cloud: depth_map_point_cloud_ofp = os.path.join(depth_map_point_cloud_odp, depth_map_name + ".ply") depth_map_point_cloud_reference_ofp = None if create_depth_map_point_cloud_reference: depth_map_point_cloud_reference_ofp = os.path.join( depth_map_point_cloud_reference_odp, depth_map_name + ".ply") if not depth_map_reparam_with_skew_ifp in depth_map_reparam_ifp_list: logger.vinfo( "depth_map_reparam_with_skew_ifp", depth_map_reparam_with_skew_ifp, ) assert False last_row = last_rows[img_name] ( _, extended_inv_proj_mat_4_x_4, _, inv_proj_scaling_factor, ) = compute_extended_proj_and_inv_proj_mat(camera, last_row) if check_inv_proj_mat: extended_inv_proj_mat_4_x_4_reference = inv_proj_mats[img_name] np.testing.assert_allclose( extended_inv_proj_mat_4_x_4, extended_inv_proj_mat_4_x_4_reference, ) # Positions with invalid depth values contain nan depth_map_reparam_with_skew = parse_depth_map( depth_map_reparam_with_skew_ifp) depth_map_real_with_skew = ( convert_reparameterized_depth_map_to_real_depth_map( depth_map_reparam_with_skew, extended_inv_proj_mat_4_x_4, inv_proj_scaling_factor, )) write_depth_map(depth_map_real_with_skew, depth_map_real_with_skew_ofp) depth_map_real_with_skew_loaded = parse_depth_map( depth_map_real_with_skew_ofp) if check_depth_mat_storing: np.testing.assert_allclose(depth_map_real_with_skew, depth_map_real_with_skew_loaded) depth_mean_val = np.nanmean(depth_map_real_with_skew_loaded) logger.vinfo("depth_mean_val", depth_mean_val) if create_depth_map_point_cloud: cam_coords = camera.convert_depth_map_to_cam_coords( depth_map_real_with_skew, depth_map_semantic, shift_to_pixel_center=False, # Must be false depth_map_display_sparsity=100, ) logger.vinfo("cam_to_world_mat: ", camera.get_4x4_cam_to_world_mat()) world_coords = camera.cam_to_world_coord_multiple_coords( cam_coords) points = Point.get_points_from_coords(world_coords) PLYFileHandler.write_ply_file(depth_map_point_cloud_ofp, points) if create_depth_map_point_cloud_reference: coords_reference = ( convert_reparameterized_depth_map_to_world_points( depth_map_reparam_with_skew, extended_inv_proj_mat_4_x_4)) points_reference = Point.get_points_from_coords(coords_reference) PLYFileHandler.write_ply_file(depth_map_point_cloud_reference_ofp, points_reference)