def texture_mesh(
self,
openmvs_workspace_dp,
openmvs_ifn,
openmvs_ofn,
export_type,
lazy=False,
):
# export_type: '.ply' or '.obj'
logger.info("texture_mesh: ...")
if not (export_type == "ply" or export_type == "obj"):
logger.vinfo("export_type", export_type)
assert False
export_file = os.path.splitext(openmvs_ofn)[0] + "." + export_type
if (not os.path.isfile(openmvs_ofn) or not os.path.isfile(export_file)
or not lazy):
texture_mesh_call = [
self.texture_mesh_fp,
"-i",
openmvs_ifn,
"-w",
openmvs_workspace_dp,
"-o",
openmvs_ofn,
"--export-type",
export_type,
]
logger.info(str(texture_mesh_call))
pRecons = subprocess.Popen(texture_mesh_call)
pRecons.wait()
logger.info("texture_mesh: Done")
def _read_txt(self, stream):
"""
Load a PLY element from an ASCII-format PLY file. The element
may contain list properties.
"""
self._data = _np.empty(self.count, dtype=self.dtype())
k = 0
for line in _islice(iter(stream.readline, b""), self.count):
fields = iter(line.strip().split())
for prop in self.properties:
try:
self._data[prop.name][k] = prop._from_fields(fields)
except StopIteration:
raise PlyElementParseError("early end-of-line", self, k,
prop)
except ValueError:
raise PlyElementParseError("malformed input", self, k,
prop)
try:
next(fields)
except StopIteration:
pass
else:
from ssr.utility.logging_extension import logger
logger.vinfo("fields", " ".join(fields))
logger.vinfo("k", " ".join(k))
raise PlyElementParseError("expected end-of-line", self, k)
k += 1
if k < self.count:
del self._data
raise PlyElementParseError("early end-of-file", self, k)
def _check_decimation_backends(decimation_backends):
valid_backends = [
DecimationBackends.meshlab,
DecimationBackends.openmvs,
]
for decimation_backend in decimation_backends:
if not decimation_backend in valid_backends:
logger.vinfo("decimation_backend", decimation_backend)
logger.vinfo("decimation_backends", decimation_backends)
assert False
def _check_texturing_backends(texturing_backends):
valid_backends = [
TexturingBackends.openmvs,
TexturingBackends.mve,
]
for texturing_backend in texturing_backends:
if not texturing_backend in valid_backends:
logger.vinfo("texturing_backend", texturing_backend)
logger.vinfo("texturing_backends", texturing_backends)
assert False
def __init__(self):
ssr_config = SSRConfig.get_instance()
self.executable_fp = ssr_config.get_option_value(
"meshlab_server_fp", str)
self.meshlab_temp_dp = ssr_config.get_option_value_or_None(
"meshlab_temp_dp", str)
if self.meshlab_temp_dp is not None:
if not os.path.isdir(self.meshlab_temp_dp):
logger.vinfo("meshlab_temp_dp", self.meshlab_temp_dp)
assert False, "Choose a valid path (in the config file) for Meshlab's temp directory"
def import_images_to_scene(self, image_idp):
logger.info("import_images_to_scene: ...")
make_scene_call = [
self.make_scene_fp,
"--images-only",
image_idp,
self.workspace_folder,
]
logger.vinfo("make_scene_call", make_scene_call)
subprocess.call(make_scene_call)
def process_meshing_task(self, meshing_task, lazy):
mesh_ply_ofn = meshing_task.mesh_ply_ofn
plain_mesh_ply_ofn = meshing_task.plain_mesh_ply_ofn
logger.vinfo("mesh_ply_ofn", mesh_ply_ofn)
if meshing_task.meshing_backend == MeshingBackends.colmap_poisson:
mkdir_safely(meshing_task.mesh_odp)
MeshingStep.compute_mesh_with_colmap(
meshing_task.colmap_idp,
meshing_task.mesh_odp,
mesh_ply_ofn,
plain_mesh_ply_ofn,
"poisson_mesher",
poisson_trim_thresh=10,
lazy=lazy,
)
elif meshing_task.meshing_backend == MeshingBackends.colmap_delaunay:
mkdir_safely(meshing_task.mesh_odp)
MeshingStep.compute_mesh_with_colmap(
meshing_task.colmap_idp,
meshing_task.mesh_odp,
mesh_ply_ofn,
plain_mesh_ply_ofn,
"delaunay_mesher",
lazy=lazy,
)
elif meshing_task.meshing_backend == MeshingBackends.openmvs:
mkdir_safely(meshing_task.mesh_odp)
MeshingStep.compute_mesh_with_openmvs(
meshing_task.colmap_idp,
meshing_task.mesh_odp,
mesh_ply_ofn,
plain_mesh_ply_ofn,
lazy=lazy,
)
elif meshing_task.meshing_backend == MeshingBackends.mve_fssr:
MeshingStep.compute_mesh_with_mve(
meshing_task.colmap_idp,
meshing_task.mesh_odp,
mesh_ply_ofn,
plain_mesh_ply_ofn,
meshing_algo="fssr",
lazy=lazy,
)
elif meshing_task.meshing_backend == MeshingBackends.mve_gdmr:
MeshingStep.compute_mesh_with_mve(
meshing_task.colmap_idp,
meshing_task.mesh_odp,
mesh_ply_ofn,
plain_mesh_ply_ofn,
meshing_algo="gdmr",
lazy=lazy,
)
else:
assert False
def compute_mesh_with_openmvs(colmap_idp,
odp,
mesh_ply_ofn,
plain_mesh_ply_ofn,
lazy=False):
logger.info("compute_mesh_with_openmvs: ...")
# https://github.com/cdcseacave/openMVS/wiki/Usage
# Exporting and Viewing Results
# Each of the following commands also writes a PLY file that can be
# used with many third-party tools. Alternatively, Viewer can be used
# to export the MVS projects to PLY or OBJ formats.
interface_mvs_fn = "interface_colmap.mvs"
# This will create a "plain_mesh.ply" file
mesh_mvs_fn = os.path.splitext(mesh_ply_ofn)[0] + ".mvs"
openmvs_mvs_reconstructor = OpenMVSReconstructor()
# This step already imports the dense point cloud computed by colmap
# (Stored in fused.ply and fused.ply.vis)
openmvs_mvs_reconstructor.convert_colmap_to_openMVS(
colmap_dense_idp=colmap_idp,
openmvs_workspace_dp=odp,
openmvs_ofn=interface_mvs_fn,
image_folder="images/",
lazy=lazy,
)
logger.vinfo("ofp", os.path.join(odp, mesh_ply_ofn))
# Adjust the default parameters (min_point_distance=2.5,
# smoothing_iterations=2) to improve satellite reconstruction results
min_point_distance = 0
smoothing_iterations = 0
openmvs_mvs_reconstructor.reconstruct_mesh(
odp,
interface_mvs_fn,
mesh_mvs_fn,
# OpenMVS Default Value: min_point_distance=2.5
min_point_distance=min_point_distance,
# Clean options:
# OpenMVS Default Value: decimate_value=1.0
decimate_value=1.0,
# OpenMVS Default Value: smoothing_iterations=2
smoothing_iterations=smoothing_iterations,
lazy=lazy,
)
mesh_ply_ofp = os.path.join(odp, mesh_ply_ofn)
plain_mesh_ply_ofp = os.path.join(odp, plain_mesh_ply_ofn)
meshlab = Meshlab()
meshlab.remove_color(mesh_ply_ofp, plain_mesh_ply_ofp)
logger.info("compute_mesh_with_openmvs: Done")
def parse_ply_file(ifp):
logger.info("Parse PLY File: ...")
logger.vinfo("ifp", ifp)
ply_data = PlyData.read(ifp)
vertices, _, _ = PLYFileHandler.__ply_data_vertices_to_vetex_list(
ply_data)
faces, _, _ = PLYFileHandler.__ply_data_faces_to_face_list(ply_data)
logger.info("Parse PLY File: Done")
return vertices, faces
def compute_depth_maps(self, downscale_level=0):
logger.info("compute_depth_maps: ...")
downscale_s_str = "-s" + str(downscale_level)
dm_recon_call = [
self.dm_recon_fp,
downscale_s_str,
self.workspace_folder,
]
logger.vinfo("dm_recon_call", dm_recon_call)
subprocess.call(dm_recon_call)
logger.info("compute_depth_maps: ...")
def _check_meshing_backends(meshing_backends):
valid_backends = [
MeshingBackends.colmap_poisson,
MeshingBackends.colmap_delaunay,
MeshingBackends.openmvs,
MeshingBackends.mve_fssr,
MeshingBackends.mve_gdmr,
]
for meshing_backend in meshing_backends:
if not meshing_backend in valid_backends:
logger.vinfo("meshing_backend", meshing_backend)
logger.vinfo("valid_backends", valid_backends)
assert False
def compute_mesh_with_mve(
colmap_idp,
odp,
mesh_ply_ofn,
plain_mesh_ply_ofn,
meshing_algo,
lazy=False,
):
logger.info("compute_mesh_with_mve: ...")
mesh_ply_ofp = os.path.join(odp, mesh_ply_ofn)
plain_mesh_ply_ofp = os.path.join(odp, plain_mesh_ply_ofn)
downscale_level = 0
mve_mvs_reconstructor = MVEMVSReconstructor()
mve_mvs_reconstructor.create_scene_from_sfm_result(
colmap_idp, odp, downscale_level=downscale_level, lazy=lazy)
mve_mvs_reconstructor.compute_dense_point_cloud_from_depth_maps(
odp,
downscale_level=downscale_level,
view_ids=None,
fssr_output=True,
lazy=lazy,
)
meshlab = Meshlab()
if meshing_algo == "fssr":
mve_mvs_reconstructor.compute_fssr_reconstruction(
odp, mesh_ply_ofp=mesh_ply_ofp, lazy=lazy)
stem, ext = os.path.splitext(mesh_ply_ofp)
cleaned_mesh_ply_ofp = stem + "_cleaned" + ext
mve_mvs_reconstructor.compute_clean_mesh(
odp,
mesh_ply_ifp=mesh_ply_ofp,
mesh_cleaned_ply_ofp=cleaned_mesh_ply_ofp,
delete_color=False,
lazy=lazy,
)
meshlab.remove_color(cleaned_mesh_ply_ofp, plain_mesh_ply_ofp)
elif meshing_algo == "gdmr":
mve_mvs_reconstructor.compute_gdmr_reconstruction(
odp, mesh_ply_ofp=mesh_ply_ofp, lazy=lazy)
meshlab.remove_color(mesh_ply_ofp, plain_mesh_ply_ofp)
else:
logger.vinfo("meshing_algo", meshing_algo)
assert False
logger.vinfo("ofp", mesh_ply_ofp)
logger.info("compute_mesh_with_mve: Done")
def create_scene_from_sfm_result(self, sfm_fp_or_dp, downscale_level=None):
# Input can be a nvm file or a colmap model folder
assert self.workspace_folder is not None
logger.info("Creating Scene: ...")
logger.info("Input file/folder: " + sfm_fp_or_dp)
logger.info("Output Path: " + self.workspace_folder)
downscale_s_str = "-s" + str(downscale_level)
make_scene_call = [
self.make_scene_fp,
downscale_s_str,
sfm_fp_or_dp,
self.workspace_folder,
]
logger.vinfo("make_scene_call", make_scene_call)
subprocess.call(make_scene_call)
logger.info("Creating Scene: Done")
def perform_pan_sharpening(pan_ifp,
msi_ifp,
ofp,
resampling_algorithm="cubic"):
# https://gdal.org/programs/gdal_pansharpen.html
# https://gis.stackexchange.com/questions/270476/pansharpening-using-gdal-tools
# GDAL pan sharpening algorithm = weighted Brovey algorithm
ext = os.path.splitext(ofp)[1]
of = ext[1:]
call_params = ["gdal_pansharpen.py"]
call_params += ["-of", of, "-r", resampling_algorithm]
call_params += [pan_ifp, msi_ifp, ofp]
logger.vinfo("call_params", call_params)
sharp_process = subprocess.Popen(call_params)
sharp_process.wait()
def __init__(self, config_fp, working_file_suffix=None):
self.config_fp = config_fp
self.config = configparser.RawConfigParser()
if not os.path.isfile(self.config_fp):
abs_path = os.path.abspath(os.path.dirname(self.config_fp))
if not os.path.isdir(abs_path):
logger.vinfo("abs_path", abs_path)
assert False # config folder missing
open(self.config_fp, "a").close()
else:
self.config.read(self.config_fp)
if working_file_suffix is not None:
self.path_to_working_copy = self.config_fp + working_file_suffix
else:
self.path_to_working_copy = self.config_fp
def compute_intrinsic_skew_decomposition(intrinsic_mat):
f_x, f_y, skew, p_x, p_y = Intrinsics.split_intrinsic_mat(
intrinsic_mat
)
intrinsic_mat_wo_skew = np.array(
[[f_x, 0, p_x - skew * p_y / f_y], [0, f_y, p_y], [0, 0, 1]],
dtype=float,
)
skew_mat = np.array(
[[1, skew / f_y, 0], [0, 1, 0], [0, 0, 1]], dtype=float
)
if not np.allclose(intrinsic_mat, skew_mat @ intrinsic_mat_wo_skew):
err_mat = intrinsic_mat - skew_mat @ intrinsic_mat_wo_skew
logger.vinfo("err_mat\n", err_mat)
assert False
return skew_mat, intrinsic_mat_wo_skew
def write_ply_file(
ofp,
vertices,
with_colors=True,
with_normals=False,
faces=None,
plain_text_output=False,
with_measurements=False,
):
logger.info("write_ply_file: " + ofp)
ply_data_vertex_data_dtype_list = PLYFileHandler.build_type_list(
vertices, with_colors, with_normals, with_measurements)
logger.vinfo("ply_data_vertex_data_dtype_list",
ply_data_vertex_data_dtype_list)
output_ply_data_vertex_element = (
PLYFileHandler.__vertices_to_ply_vertex_element(
vertices, ply_data_vertex_data_dtype_list))
if faces is None or len(faces) == 0:
logger.info("Write File With Vertices Only (no faces)")
output_data = PlyData([output_ply_data_vertex_element],
text=plain_text_output)
else:
logger.info("Write File With Faces")
logger.info("Number faces" + str(len(faces)))
ply_data_face_data_type = [("vertex_indices", "i4", (3, ))]
# we do not define colors for faces,
# since we use the vertex colors to colorize the face
output_ply_data_face_element = (
PLYFileHandler.__faces_to_ply_face_element(
faces, ply_data_face_data_type))
output_data = PlyData(
[output_ply_data_vertex_element, output_ply_data_face_element],
text=plain_text_output,
)
output_data.write(ofp)
def reconstruct_mesh_with_delaunay(
self,
reconstruction_idp,
mesh_ply_ofp,
max_proj_dist=None,
max_depth_dist=None,
lazy=False,
):
logger.info("reconstruct_mesh: ...")
assert os.path.isdir(self.colmap_exe_dp)
assert os.path.isfile(os.path.join(self.colmap_exe_dp, "colmap"))
if not os.path.isfile(mesh_ply_ofp) or not lazy:
os.environ["PATH"] += os.pathsep + self.colmap_exe_dp
dense_mesher_call = [
"colmap",
"delaunay_mesher",
"--input_path",
reconstruction_idp,
"--output_path",
mesh_ply_ofp,
"--input_type",
"dense",
]
if max_proj_dist is not None:
dense_mesher_call += [
"--DelaunayMeshing.max_proj_dist",
str(max_proj_dist),
]
if max_depth_dist is not None:
dense_mesher_call += [
"--DelaunayMeshing.max_depth_dist",
str(max_depth_dist),
]
logger.vinfo("dense_mesher_call: ", dense_mesher_call)
subprocess.call(dense_mesher_call)
logger.info("reconstruct_mesh: Done")
def sample_mesh(self, mesh_ifp, point_cloud_ofp, num_vertices,
sampling_method):
# While meshlab provides a variety of sampling methods,
# Cloudcompare provides only a single method that randomly samples
# points.
assert sampling_method in [
"poisson_disk",
"stratified_triangle",
"montecarlo",
], f"Received unsupported sampling method: {sampling_method}"
# Poisson-disk Sampling
# Requires the definition of
# number of samples OR radius or percentage
# Stratified Triangle Sampling
# Requires the definition of number of samples
# Results look better than Monte Carlo Sampling
template_fp = self._create_lmx_template(sampling_method +
"_sampling.mlx")
logger.vinfo("template_fp", template_fp)
assert os.path.isfile(template_fp)
_MLXFileHandler.set_value(template_fp, "SampleNum", num_vertices)
num_vertices_str = _MLXFileHandler.get_value_as_str(
template_fp, "SampleNum")
assertion_msg = f"{num_vertices_str} vs {num_vertices}"
assert int(num_vertices_str) == num_vertices, assertion_msg
options = []
options += ["-i", mesh_ifp]
options += ["-s", template_fp]
options += ["-o", point_cloud_ofp]
call_list = [self.executable_fp] + options
logger.vinfo("call_list", call_list)
subprocess.call(call_list)
# Remove the file from the file system
os.unlink(template_fp)
def create_textured_mesh(
self,
mve_scene_idp,
untextured_mesh_ifp,
texture_odp,
occlusion_handling=True,
):
logger.vinfo("texture_odp", texture_odp)
current_working_dir = os.getcwd()
if not os.path.isdir(texture_odp):
os.makedirs(texture_odp)
# Change the directory, since texrecon creates the files in the current
# directory
os.chdir(texture_odp)
logger.vinfo("texture_odp", texture_odp)
options = []
options += ["--keep_unseen_faces"]
if occlusion_handling:
# Only the following options created reasonable results:
options += ["--data_term=area", "--outlier_removal=gauss_damping"]
texturing_call = ([self.texrecon_executable] + options + [
mve_scene_idp + "::undistorted",
untextured_mesh_ifp,
"textured",
])
logger.vinfo("texturing_call", texturing_call)
subprocess.call(texturing_call)
# reset the working directory
os.chdir(current_working_dir)
def compute_mesh_with_colmap(
colmap_idp,
mesh_odp,
mesh_ply_ofn,
plain_mesh_ply_ofn,
meshing_algo,
poisson_trim_thresh=10,
lazy=False,
):
"""Poisson meshing works with a single point-cloud-ply-file, whereas
delaunay meshing requires a full workspace.
"""
logger.info("compute_mesh_with_colmap: ...")
assert meshing_algo in ["poisson_mesher", "delaunay_mesher"]
mesh_ply_ofp = os.path.join(mesh_odp, mesh_ply_ofn)
logger.vinfo("ofp", mesh_ply_ofp)
colmap_mvs_reconstructor = ColmapMVSReconstructor()
if meshing_algo == "poisson_mesher":
point_cloud_ply_ifp = os.path.join(colmap_idp, "fused.ply")
colmap_mvs_reconstructor.reconstruct_mesh_with_poisson(
point_cloud_ply_ifp,
mesh_ply_ofp,
poisson_trim_thresh,
lazy=lazy,
)
elif meshing_algo == "delaunay_mesher":
colmap_mvs_reconstructor.reconstruct_mesh_with_delaunay(
colmap_idp,
mesh_ply_ofp,
# https://colmap.github.io/faq.html
max_proj_dist=0, # Colmap Default Value: 2.5
lazy=lazy,
)
plain_mesh_ply_ofp = os.path.join(mesh_odp, plain_mesh_ply_ofn)
meshlab = Meshlab()
meshlab.remove_color(mesh_ply_ofp, plain_mesh_ply_ofp)
logger.info("compute_mesh_with_colmap: Done")
def compute_dense_point_cloud_from_depth_maps(
self,
mve_point_cloud_ply_ofp,
downscale_level=0,
view_ids=None,
fssr_output=True,
):
logger.info("compute_dense_point_cloud_from_depth_maps: ...")
assert os.path.splitext(mve_point_cloud_ply_ofp)[1] == ".ply"
scene2pset_call = [self.scene2pset_fp]
if fssr_output:
downscale_F_str = "-F" + str(downscale_level)
scene2pset_call.append(downscale_F_str)
if view_ids is not None:
view_id_str = "--views=" + ",".join(map(str, view_ids))
scene2pset_call.append(view_id_str)
scene2pset_call.append(self.workspace_folder)
scene2pset_call.append(mve_point_cloud_ply_ofp)
logger.vinfo("scene2pset_call", scene2pset_call)
subprocess.call(scene2pset_call)
logger.info("compute_dense_point_cloud_from_depth_maps: Done")
def reconstruct_mesh_with_poisson(self, point_cloud_ply_ifp, mesh_ply_ofp,
poisson_trim_thresh, lazy):
logger.info("reconstruct_mesh: ...")
logger.vinfo("mesh_ply_ofp", mesh_ply_ofp)
assert os.path.isdir(self.colmap_exe_dp)
assert os.path.isfile(os.path.join(self.colmap_exe_dp, "colmap"))
if not os.path.isfile(mesh_ply_ofp) or not lazy:
os.environ["PATH"] += os.pathsep + self.colmap_exe_dp
trim_thresh_str = str(poisson_trim_thresh)
dense_mesher_call = [
"colmap",
"poisson_mesher",
"--input_path",
point_cloud_ply_ifp,
"--output_path",
mesh_ply_ofp,
"--PoissonMeshing.trim",
trim_thresh_str,
]
logger.vinfo("dense_mesher_call: ", dense_mesher_call)
subprocess.call(dense_mesher_call)
logger.info("reconstruct_mesh: Done")
def compute_sfm(self):
logger.info("Compute SfM: ...")
sfm_recon_call = [self.sfm_recon_fp, self.workspace_folder]
logger.vinfo("sfm_recon_call", sfm_recon_call)
subprocess.call(sfm_recon_call)
logger.info("Compute SfM: Done")
def convert_depth_map_to_cam_coords(
self,
depth_map,
depth_map_semantic,
shift_to_pixel_center, # False for Colmap, True for MVE
depth_map_display_sparsity=100,
):
assert 0 < depth_map_display_sparsity
height, width = depth_map.shape
logger.info("height " + str(height))
logger.info("width " + str(width))
if self.height == height and self.width == width:
x_step_size = 1.0
y_step_size = 1.0
else:
x_step_size = self.width / width
y_step_size = self.height / height
logger.info("x_step_size " + str(x_step_size))
logger.info("y_step_size " + str(y_step_size))
fx, fy, skew, cx, cy = self.split_intrinsic_mat(
self.get_calibration_mat())
logger.vinfo("fx, fy, skew, cx, cy: ", str([fx, fy, skew, cx, cy]))
indices = np.indices((height, width))
y_index_list = indices[0].flatten()
x_index_list = indices[1].flatten()
depth_values = depth_map.flatten()
assert len(x_index_list) == len(y_index_list) == len(depth_values)
if shift_to_pixel_center:
# https://github.com/simonfuhrmann/mve/blob/master/libs/mve/depthmap.cc
# math::Vec3f v = invproj * math::Vec3f(
# (float)x + 0.5f, (float)y + 0.5f, 1.0f);
u_index_coord_list = x_step_size * x_index_list + 0.5
v_index_coord_list = y_step_size * y_index_list + 0.5
else:
# https://github.com/colmap/colmap/blob/dev/src/base/reconstruction.cc
# COLMAP assumes that the upper left pixel center is (0.5, 0.5)
# i.e. pixels are already shifted
u_index_coord_list = x_step_size * x_index_list
v_index_coord_list = y_step_size * y_index_list
# The cannoncial vectors are defined according to p.155 of
# "Multiple View Geometry" by Hartley and Zisserman using a canonical
# focal length of 1 , i.e. vec = [(x - cx) / fx, (y - cy) / fy, 1]
skew_correction = (cy - v_index_coord_list) * skew / (fx * fy)
x_coords_canonical = (u_index_coord_list - cx) / fx + skew_correction
y_coords_canonical = (v_index_coord_list - cy) / fy
z_coords_canonical = np.ones(len(depth_values), dtype=float)
# Determine non-background data
# non_background_flags = np.logical_not(np.isnan(depth_values))
depth_values_not_nan = np.nan_to_num(depth_values)
non_background_flags = depth_values_not_nan > 0
x_coords_canonical_filtered = x_coords_canonical[non_background_flags]
y_coords_canonical_filtered = y_coords_canonical[non_background_flags]
z_coords_canonical_filtered = z_coords_canonical[non_background_flags]
depth_values_filtered = depth_values[non_background_flags]
if depth_map_display_sparsity != 100:
x_coords_canonical_filtered = x_coords_canonical_filtered[::
depth_map_display_sparsity]
y_coords_canonical_filtered = y_coords_canonical_filtered[::
depth_map_display_sparsity]
z_coords_canonical_filtered = z_coords_canonical_filtered[::
depth_map_display_sparsity]
depth_values_filtered = depth_values_filtered[::
depth_map_display_sparsity]
if depth_map_semantic == Camera.DEPTH_MAP_WRT_CANONICAL_VECTORS:
# In this case, the depth values are defined w.r.t. the canonical
# vectors. This kind of depth data is used by Colmap.
x_coords_filtered = (x_coords_canonical_filtered *
depth_values_filtered)
y_coords_filtered = (y_coords_canonical_filtered *
depth_values_filtered)
z_coords_filtered = (z_coords_canonical_filtered *
depth_values_filtered)
elif depth_map_semantic == Camera.DEPTH_MAP_WRT_UNIT_VECTORS:
# In this case the depth values are defined w.r.t. the normalized
# canonical vectors. This kind of depth data is used by MVE.
cannonical_norms_filtered = np.linalg.norm(
np.array(
[
x_coords_canonical_filtered,
y_coords_canonical_filtered,
z_coords_canonical_filtered,
],
dtype=float,
),
axis=0,
)
# Instead of normalizing the x,y and z component, we divide the
# depth values by the corresponding norm.
normalized_depth_values_filtered = (depth_values_filtered /
cannonical_norms_filtered)
x_coords_filtered = (x_coords_canonical_filtered *
normalized_depth_values_filtered)
y_coords_filtered = (y_coords_canonical_filtered *
normalized_depth_values_filtered)
z_coords_filtered = (z_coords_canonical_filtered *
normalized_depth_values_filtered)
else:
assert False
cam_coords = np.dstack(
(x_coords_filtered, y_coords_filtered, z_coords_filtered))[0]
return cam_coords
if __name__ == "__main__":
f_x = 2800
f_y = 2100
s = 0.3
p_x = 2355
p_y = 2500
f_x_ = 2200
f_y_ = 2400
s_ = 0.1
p_x_ = 1600
p_y_ = 800
intrinsic_1 = np.array(
[[f_x, s, p_x], [0, f_y, p_y], [0, 0, 1]], dtype=float
)
intrinsic_2 = np.array(
[[f_x_, s_, p_x_], [0, f_y_, p_y_], [0, 0, 1]], dtype=float
)
trans_mat_2_to_1 = Intrinsics.compute_intrinsic_transformation(
intrinsic_1, intrinsic_2, check_result=True
)
intrinsic_1_restored = trans_mat_2_to_1 @ intrinsic_2
logger.vinfo("trans_mat_2_to_1", trans_mat_2_to_1)
logger.vinfo("intrinsic_1", intrinsic_1)
logger.vinfo("intrinsic_1_restored", intrinsic_1_restored)
def run(self, reconstruct_sfm_mvs):
dataset_dp = self.ssr_config.get_option_value(
"satellite_image_pan_dp", str
)
workspace_dp = self.ssr_config.get_option_value(
"workspace_vissat_dp", str
)
mkdir_safely(workspace_dp)
create_vissat_config_from_ssr_config(
vissat_config_ofp=self.pm.vissat_config_fp,
dataset_dp=dataset_dp,
workspace_dp=workspace_dp,
ssr_config=self.ssr_config,
clean_data=True,
crop_image=True,
derive_approx=True,
choose_subset=True,
colmap_sfm_perspective=True,
inspect_sfm_perspective=True,
reparam_depth=True,
colmap_mvs=True,
aggregate_2p5d=True,
aggregate_3d=True,
)
if reconstruct_sfm_mvs:
logger.vinfo("self.pm.vissat_config_fp", self.pm.vissat_config_fp)
assert os.path.isdir(self.colmap_vissat_exe_dp)
os.environ["PATH"] += os.pathsep + self.colmap_vissat_exe_dp
# see https://github.com/Kai-46/VisSatSatelliteStereo/blob/master/stereo_pipeline.py
from stereo_pipeline import StereoPipeline as VisSatStereoPipeline
# https://github.com/Kai-46/VisSatSatelliteStereo
# Our pipeline is written in a module way; you can run it step by step
# by choosing what steps to execute in the configuration file.
# Steps to run
# clean_data (see clean_data() in clean_data.py)
# creates the folder "cleaned_data"
# copies the NTF-files contained in the input PAN folder
# extracts the xml- and the jpg-files contained in the tar-files in the input PAN folder
# crop_image (see image_crop() and image_crop_worker() in image_crop.py)
# creates the folder "images"
# uses the bounding box specified in the config-json-file to crop the corresponding area
# of the NTF-files contained in the "cleaned_data" folder
# derive_approx
# choose_subset
# colmap_sfm_perspective
# inspect_sfm_perspective
# reparam_depth
# colmap_mvs
# aggregate_2p5d
# creates the folder "mvs_results/aggregate_2p5d"
# with a height-colorized point cloud and corresponding images
# and a geo-registered geo-tiff file
# aggregate_3d
# creates the folder "mvs_results/aggregate_3d"
# with a 3d point cloud and corresponding images
# and a geo-registered geo-tiff file
pipeline = VisSatStereoPipeline(self.pm.vissat_config_fp)
# Logs are created in the working_directory/logs folder
pipeline.run()
def perform_mvs(
self,
ifp,
openmvs_workspace,
openmvs_ofp,
image_idp=None, # For NVM input
lazy=False,
):
# ifp can be an .mvs (OpenMVS) or a .nvm (VisualSfM) or .bin/.json (OpenMVG) file
# openmvs_ofp can be an .obj or an.ply file
logger.info("perform_mvs : ...")
logger.vinfo("openmvs_ofp", openmvs_ofp)
if not os.path.isdir(openmvs_workspace):
os.mkdir(openmvs_workspace)
openmvs_workspace_temp = os.path.join(openmvs_workspace, "temp")
if not os.path.isdir(openmvs_workspace_temp):
os.mkdir(openmvs_workspace_temp)
if os.path.splitext(ifp)[1] == ".mvs":
logger.info("OpenMVS file detected")
openmvs_ifp = ifp
elif os.path.splitext(ifp)[1] == ".nvm":
logger.info("NVM file detected")
assert image_idp is not None
openmvs_ifp = os.path.splitext(ifp)[0] + ".mvs"
# TODO: This folder is only created if the parameters actually show
# a distortion coefficient. Otherwise the original image folder
# is referenced in the mvs file
# undistorted_image_fp = os.path.join(
# os.path.dirname(openmvs_ifp), "undistorted_images/"
# )
self.convert_visualsfm_to_openMVS(
nvm_ifp=ifp,
image_idp=image_idp,
openmvs_workspace_temp=openmvs_workspace_temp,
openmvs_ofp=openmvs_ifp,
)
elif (os.path.splitext(ifp)[1] == ".bin"
or os.path.splitext(ifp)[1] == ".json"):
logger.info("OpenMVG file detected")
assert image_idp is not None
openmvs_ifp = os.path.splitext(ifp)[0] + ".mvs"
from ssr.sfm_utility.openmvg.openmvg_sfm import (
OpenMVGReconstructor, )
OpenMVGReconstructor.export_to_openmvs(ifp=ifp, ofp=openmvs_ifp)
else:
assert False
dense_mvs_fn = "dense.mvs"
dense_mvs_fp = os.path.join(openmvs_workspace, dense_mvs_fn)
self.densify_point_cloud(openmvs_ifp, openmvs_workspace_temp,
dense_mvs_fp, lazy)
mesh_mvs_fn = "dense_mesh.mvs"
mesh_mvs_fp = os.path.join(openmvs_workspace, mesh_mvs_fn)
self.reconstruct_mesh(dense_mvs_fp, openmvs_workspace_temp,
mesh_mvs_fp, lazy)
texture_mvs_fn = "dense_mesh_texture.mvs"
texture_mvs_fp = os.path.join(openmvs_workspace, texture_mvs_fn)
export_type = "obj"
self.texture_mesh(
mesh_mvs_fp,
openmvs_workspace_temp,
texture_mvs_fp,
export_type,
lazy,
)
texture_ply_fp = (os.path.splitext(texture_mvs_fp)[0] + "." +
export_type)
shutil.copyfile(texture_ply_fp, openmvs_ofp)
if export_type == "ply":
shutil.copyfile(
os.path.splitext(texture_ply_fp)[0] + ".png",
os.path.splitext(openmvs_ofp)[0] + ".png",
)
elif export_type == "obj":
shutil.copyfile(
os.path.splitext(texture_ply_fp)[0] + ".mtl",
os.path.splitext(openmvs_ofp)[0] + ".mtl",
)
shutil.copyfile(
os.path.join(
os.path.dirname(texture_ply_fp),
"dense_mesh_texture_material_0_map_Kd.jpg",
),
os.path.join(
os.path.dirname(openmvs_ofp),
"dense_mesh_texture_material_0_map_Kd.jpg",
),
)
logger.info("perform_mvs : Done")
def compute_skew_free_camera_models(
colmap_model_with_skew_idp,
gray_image_with_skew_idp,
color_image_with_skew_idp,
depth_map_with_skew_idp,
colmap_model_no_skew_odp,
gray_image_no_skew_odp,
color_image_no_skew_odp,
depth_map_no_skew_odp,
perform_warping_evaluation=False,
interpolation_type=cv2.INTER_CUBIC,
):
assert colmap_model_with_skew_idp != colmap_model_no_skew_odp
assert gray_image_with_skew_idp != gray_image_no_skew_odp
assert depth_map_with_skew_idp != depth_map_no_skew_odp
if gray_image_no_skew_odp is not None:
mkdir_safely(gray_image_no_skew_odp)
if color_image_no_skew_odp is not None:
mkdir_safely(color_image_no_skew_odp)
if depth_map_no_skew_odp is not None:
mkdir_safely(depth_map_no_skew_odp)
logger.vinfo("image_without_skew_odp", gray_image_no_skew_odp)
logger.vinfo("interpolation_type", interpolation_type)
cameras = ColmapFileHandler.parse_colmap_cams(
colmap_model_with_skew_idp, gray_image_with_skew_idp
)
# cameras, _ = ColmapFileHandler.parse_colmap_model_folder(
# colmap_model_idp, image_idp
# )
num_cameras = len(cameras)
skew_free_camera_list = []
pil_better_count = 0
mat_better_count = 0
patt_count = 0
for camera in cameras:
logger.vinfo("camera.id", str(camera.id) + " of " + str(num_cameras))
logger.vinfo("camera.file_name", camera.file_name)
intrinsic_mat = camera.get_calibration_mat()
(
skew_mat,
intrinsic_mat_wo_skew,
) = Camera.compute_intrinsic_skew_decomposition(intrinsic_mat)
image_invert_skew_mat = np.linalg.inv(skew_mat)
image_ifn = camera.file_name
image_stem, ext = os.path.splitext(image_ifn)
if gray_image_with_skew_idp is not None:
image_ifp = os.path.join(gray_image_with_skew_idp, image_ifn)
mat_image_ofp = os.path.join(
gray_image_no_skew_odp, image_stem + ext
)
mat_image = imageio.imread(image_ifp)
mat_image_skew_free = remove_skew_from_matrix(
mat_image,
image_invert_skew_mat,
interpolation_type=interpolation_type,
)
imageio.imwrite(mat_image_ofp, mat_image_skew_free)
if color_image_with_skew_idp is not None:
image_ifp = os.path.join(color_image_with_skew_idp, image_ifn)
mat_image_ofp = os.path.join(
color_image_no_skew_odp, image_stem + ext
)
mat_image = imageio.imread(image_ifp)
mat_image_skew_free = remove_skew_from_matrix(
mat_image,
image_invert_skew_mat,
interpolation_type=interpolation_type,
)
imageio.imwrite(mat_image_ofp, mat_image_skew_free)
if depth_map_with_skew_idp is not None:
depth_map_fn = get_corresponding_depth_map_fn(image_ifn)
depth_ifp = os.path.join(depth_map_with_skew_idp, depth_map_fn)
depth_ofp = os.path.join(depth_map_no_skew_odp, depth_map_fn)
depth_mat = parse_depth_map(depth_ifp)
depth_mat_transformed = remove_skew_from_matrix(
depth_mat,
image_invert_skew_mat,
interpolation_type=interpolation_type,
)
write_depth_map(depth_mat_transformed, depth_ofp)
# if perform_warping_evaluation:
#
# logger.info('--------------------- Diffs --------------------')
#
# mat_image_recovered_mat = np.array(
# remove_skew_from_matrix(mat_image_skew_free, skew_mat)
# )
# mat_image_diff_mat = np.abs(mat_image - mat_image_recovered_mat)
# mat_image_error_ofp = os.path.join(
# gray_image_no_skew_odp, image_stem + '_mat_error' + ext
# )
# imageio.imwrite(mat_image_error_ofp, mat_image_diff_mat)
# mat_diff = np.nansum(mat_image_diff_mat)
# logger.vinfo('mat_diff', mat_diff)
#
# # pil_image = PILImage.open(image_ifp)
# # pil_image_transformed = remove_skew_from_images_legazy(
# pil_image, image_invert_skew_mat
# )
# # pil_image_transformed.save(pil_image_ofp)
# #
# # pil_image_recovered_mat = np.array(
# remove_skew_from_images_legazy(
# pil_image_transformed, skew_mat
# )
# )
# # pil_image_mat = np.array(pil_image)
# # pil_image_diff_mat = np.abs(
# pil_image_mat - pil_image_recovered_mat
# )
# # pil_image_error_ofp = os.path.join(
# image_without_skew_odp, image_stem + '_pil_error' + ext
# )
# # imageio.imwrite(pil_image_error_ofp, pil_image_diff_mat)
# # pil_diff = np.nansum(pil_image_diff_mat)
# # logger.vinfo('pil_diff', pil_diff)
# #
# # if pil_diff > mat_diff:
# # mat_better_count += 1
# # elif mat_diff > pil_diff:
# # pil_better_count += 1
# # else:
# # patt_count += 1
skew_free_camera = copy.copy(camera)
skew_free_camera.set_calibration(
intrinsic_mat_wo_skew, radial_distortion=False
)
skew_free_camera_list.append(skew_free_camera)
if perform_warping_evaluation:
logger.vinfo("pil_better_count", pil_better_count)
logger.vinfo("mat_better_count", mat_better_count)
logger.vinfo("patt_count", patt_count)
if colmap_model_no_skew_odp is not None:
mkdir_safely(colmap_model_no_skew_odp)
ColmapFileHandler.write_colmap_cameras(
odp=colmap_model_no_skew_odp,
cameras=skew_free_camera_list,
colmap_camera_model_name="PINHOLE",
)
def recover_depth_maps(
model_with_skew_idp,
last_rows_ifp,
image_with_skew_idp,
depth_map_reparam_with_skew_idp,
depth_map_real_with_skew_odp,
depth_map_type="geometric",
# Optional parameters
check_inv_proj_mat=False,
inv_proj_mat_ifp=None,
check_depth_mat_storing=False,
create_depth_map_point_cloud=False,
depth_map_point_cloud_odp=None,
create_depth_map_point_cloud_reference=False,
depth_map_point_cloud_reference_odp=None,
):
mkdir_safely(depth_map_real_with_skew_odp)
if create_depth_map_point_cloud:
mkdir_safely(depth_map_point_cloud_odp)
if create_depth_map_point_cloud_reference:
mkdir_safely(depth_map_point_cloud_reference_odp)
assert depth_map_type in ["geometric", "photometric"]
inv_proj_mats = None
if check_inv_proj_mat:
inv_proj_mats = parse_inv_proj_mats(inv_proj_mat_ifp)
last_rows = parse_last_rows(last_rows_ifp)
depth_map_suffix = "." + depth_map_type + ".bin"
depth_map_reparam_ifp_list = get_file_paths_in_dir(
depth_map_reparam_with_skew_idp,
ext=".bin",
target_str_or_list=depth_map_suffix,
)
cameras, points3D = ColmapFileHandler.parse_colmap_model_folder(
model_with_skew_idp, image_with_skew_idp)
# cache = PythonCache()
# cameras, points3D = cache.get_cached_result(
# callback=ColmapFileHandler.parse_colmap_model_folder,
# params=[model_with_skew_idp, image_with_skew_idp],
# unique_id_or_path=1,
# )
depth_map_semantic = Camera.DEPTH_MAP_WRT_CANONICAL_VECTORS
logger.vinfo("depth_map_real_with_skew_odp", depth_map_real_with_skew_odp)
for camera in cameras[::-1]:
img_name = camera.file_name
logger.vinfo("img_name", img_name)
depth_map_name = img_name + "." + depth_map_type + ".bin"
depth_map_reparam_with_skew_ifp = os.path.join(
depth_map_reparam_with_skew_idp, depth_map_name)
depth_map_real_with_skew_ofp = os.path.join(
depth_map_real_with_skew_odp, depth_map_name)
depth_map_point_cloud_ofp = None
if create_depth_map_point_cloud:
depth_map_point_cloud_ofp = os.path.join(depth_map_point_cloud_odp,
depth_map_name + ".ply")
depth_map_point_cloud_reference_ofp = None
if create_depth_map_point_cloud_reference:
depth_map_point_cloud_reference_ofp = os.path.join(
depth_map_point_cloud_reference_odp, depth_map_name + ".ply")
if not depth_map_reparam_with_skew_ifp in depth_map_reparam_ifp_list:
logger.vinfo(
"depth_map_reparam_with_skew_ifp",
depth_map_reparam_with_skew_ifp,
)
assert False
last_row = last_rows[img_name]
(
_,
extended_inv_proj_mat_4_x_4,
_,
inv_proj_scaling_factor,
) = compute_extended_proj_and_inv_proj_mat(camera, last_row)
if check_inv_proj_mat:
extended_inv_proj_mat_4_x_4_reference = inv_proj_mats[img_name]
np.testing.assert_allclose(
extended_inv_proj_mat_4_x_4,
extended_inv_proj_mat_4_x_4_reference,
)
# Positions with invalid depth values contain nan
depth_map_reparam_with_skew = parse_depth_map(
depth_map_reparam_with_skew_ifp)
depth_map_real_with_skew = (
convert_reparameterized_depth_map_to_real_depth_map(
depth_map_reparam_with_skew,
extended_inv_proj_mat_4_x_4,
inv_proj_scaling_factor,
))
write_depth_map(depth_map_real_with_skew, depth_map_real_with_skew_ofp)
depth_map_real_with_skew_loaded = parse_depth_map(
depth_map_real_with_skew_ofp)
if check_depth_mat_storing:
np.testing.assert_allclose(depth_map_real_with_skew,
depth_map_real_with_skew_loaded)
depth_mean_val = np.nanmean(depth_map_real_with_skew_loaded)
logger.vinfo("depth_mean_val", depth_mean_val)
if create_depth_map_point_cloud:
cam_coords = camera.convert_depth_map_to_cam_coords(
depth_map_real_with_skew,
depth_map_semantic,
shift_to_pixel_center=False, # Must be false
depth_map_display_sparsity=100,
)
logger.vinfo("cam_to_world_mat: ",
camera.get_4x4_cam_to_world_mat())
world_coords = camera.cam_to_world_coord_multiple_coords(
cam_coords)
points = Point.get_points_from_coords(world_coords)
PLYFileHandler.write_ply_file(depth_map_point_cloud_ofp, points)
if create_depth_map_point_cloud_reference:
coords_reference = (
convert_reparameterized_depth_map_to_world_points(
depth_map_reparam_with_skew, extended_inv_proj_mat_4_x_4))
points_reference = Point.get_points_from_coords(coords_reference)
PLYFileHandler.write_ply_file(depth_map_point_cloud_reference_ofp,
points_reference)
