def post_point_cloud_steps(args, tree):
# XYZ point cloud output
if args.pc_csv:
log.ODM_INFO("Creating CSV file (XYZ format)")
system.run("pdal translate -i \"{}\" "
"-o \"{}\" "
"--writers.text.format=csv "
"--writers.text.order=\"X,Y,Z\" "
"--writers.text.keep_unspecified=false ".format(
tree.odm_georeferencing_model_laz,
tree.odm_georeferencing_xyz_file))
# LAS point cloud output
if args.pc_las:
log.ODM_INFO("Creating LAS file")
system.run("pdal translate -i \"{}\" "
"-o \"{}\" ".format(
tree.odm_georeferencing_model_laz,
tree.odm_georeferencing_model_las))
# EPT point cloud output
if args.pc_ept:
log.ODM_INFO("Creating Entwine Point Tile output")
entwine.build([tree.odm_georeferencing_model_laz], tree.entwine_pointcloud, max_concurrency=args.max_concurrency, rerun=False)
def post_point_cloud_steps(args, tree, rerun=False):
# XYZ point cloud output
if args.pc_csv:
log.ODM_INFO("Creating CSV file (XYZ format)")
if not io.file_exists(tree.odm_georeferencing_xyz_file) or rerun:
system.run("pdal translate -i \"{}\" "
"-o \"{}\" "
"--writers.text.format=csv "
"--writers.text.order=\"X,Y,Z\" "
"--writers.text.keep_unspecified=false ".format(
tree.odm_georeferencing_model_laz,
tree.odm_georeferencing_xyz_file))
else:
log.ODM_WARNING("Found existing CSV file %s" %
tree.odm_georeferencing_xyz_file)
# LAS point cloud output
if args.pc_las:
log.ODM_INFO("Creating LAS file")
if not io.file_exists(tree.odm_georeferencing_model_las) or rerun:
system.run("pdal translate -i \"{}\" "
"-o \"{}\" ".format(tree.odm_georeferencing_model_laz,
tree.odm_georeferencing_model_las))
else:
log.ODM_WARNING("Found existing LAS file %s" %
tree.odm_georeferencing_xyz_file)
# EPT point cloud output
if args.pc_ept:
log.ODM_INFO("Creating Entwine Point Tile output")
entwine.build([tree.odm_georeferencing_model_laz],
tree.entwine_pointcloud,
max_concurrency=args.max_concurrency,
rerun=rerun)
# COPC point clouds
if args.pc_copc:
log.ODM_INFO("Creating Cloud Optimized Point Cloud (COPC)")
copc_output = io.related_file_path(tree.odm_georeferencing_model_laz,
postfix=".copc")
entwine.build_copc([tree.odm_georeferencing_model_laz], copc_output)
def process(self, args, outputs):
tree = outputs['tree']
reconstruction = outputs['reconstruction']
if outputs['large']:
if not os.path.exists(tree.submodels_path):
log.ODM_ERROR(
"We reached the merge stage, but %s folder does not exist. Something must have gone wrong at an earlier stage. Check the log and fix possible problem before restarting?"
% tree.submodels_path)
exit(1)
# Merge point clouds
if args.merge in ['all', 'pointcloud']:
if not io.file_exists(
tree.odm_georeferencing_model_laz) or self.rerun():
all_point_clouds = get_submodel_paths(
tree.submodels_path, "odm_georeferencing",
"odm_georeferenced_model.laz")
try:
# pdal.merge_point_clouds(all_point_clouds, tree.odm_georeferencing_model_laz, args.verbose)
entwine.build(all_point_clouds,
tree.entwine_pointcloud,
max_concurrency=args.max_concurrency,
rerun=self.rerun())
except Exception as e:
log.ODM_WARNING(
"Could not merge point cloud: %s (skipping)" %
str(e))
if io.dir_exists(tree.entwine_pointcloud):
try:
system.run('pdal translate "ept://{}" "{}"'.format(
tree.entwine_pointcloud,
tree.odm_georeferencing_model_laz))
except Exception as e:
log.ODM_WARNING(
"Cannot export EPT dataset to LAZ: %s" %
str(e))
else:
log.ODM_WARNING("Found merged point cloud in %s" %
tree.odm_georeferencing_model_laz)
self.update_progress(25)
# Merge crop bounds
merged_bounds_file = os.path.join(
tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg')
if not io.file_exists(merged_bounds_file) or self.rerun():
all_bounds = get_submodel_paths(
tree.submodels_path, 'odm_georeferencing',
'odm_georeferenced_model.bounds.gpkg')
log.ODM_INFO("Merging all crop bounds: %s" % all_bounds)
if len(all_bounds) > 0:
# Calculate a new crop area
# based on the convex hull of all crop areas of all submodels
# (without a buffer, otherwise we are double-cropping)
Cropper.merge_bounds(all_bounds, merged_bounds_file, 0)
else:
log.ODM_WARNING("No bounds found for any submodel.")
# Merge orthophotos
if args.merge in ['all', 'orthophoto']:
if not io.dir_exists(tree.odm_orthophoto):
system.mkdir_p(tree.odm_orthophoto)
if not io.file_exists(tree.odm_orthophoto_tif) or self.rerun():
all_orthos_and_cutlines = get_all_submodel_paths(
tree.submodels_path,
os.path.join("odm_orthophoto", "odm_orthophoto.tif"),
os.path.join("odm_orthophoto", "cutline.gpkg"),
)
if len(all_orthos_and_cutlines) > 1:
log.ODM_INFO(
"Found %s submodels with valid orthophotos and cutlines"
% len(all_orthos_and_cutlines))
# TODO: histogram matching via rasterio
# currently parts have different color tones
merged_geotiff = os.path.join(
tree.odm_orthophoto, "odm_orthophoto.merged.tif")
kwargs = {
'orthophoto_merged':
merged_geotiff,
'input_files':
' '.join(
map(lambda i: quote(i[0]),
all_orthos_and_cutlines)),
'max_memory':
get_max_memory(),
'threads':
args.max_concurrency,
}
# use bounds as cutlines (blending)
if io.file_exists(merged_geotiff):
os.remove(merged_geotiff)
system.run('gdal_merge.py -o {orthophoto_merged} '
#'-createonly '
'-co "BIGTIFF=YES" '
'-co "BLOCKXSIZE=512" '
'-co "BLOCKYSIZE=512" '
'--config GDAL_CACHEMAX {max_memory}% '
'{input_files} '.format(**kwargs))
for ortho_cutline in all_orthos_and_cutlines:
kwargs['input_file'], kwargs[
'cutline'] = ortho_cutline
# Note: cblend has a high performance penalty
system.run(
'gdalwarp -cutline {cutline} '
'-cblend 20 '
'-r bilinear -multi '
'-wo NUM_THREADS={threads} '
'--config GDAL_CACHEMAX {max_memory}% '
'{input_file} {orthophoto_merged}'.format(
**kwargs))
# Apply orthophoto settings (compression, tiling, etc.)
orthophoto_vars = orthophoto.get_orthophoto_vars(args)
if io.file_exists(tree.odm_orthophoto_tif):
os.remove(tree.odm_orthophoto_tif)
kwargs = {
'vars':
' '.join([
'-co %s=%s' % (k, orthophoto_vars[k])
for k in orthophoto_vars
]),
'max_memory':
get_max_memory(),
'merged':
merged_geotiff,
'log':
tree.odm_orthophoto_tif_log,
'orthophoto':
tree.odm_orthophoto_tif,
}
system.run(
'gdal_translate '
'{vars} '
'--config GDAL_CACHEMAX {max_memory}% '
'{merged} {orthophoto} > {log}'.format(**kwargs))
os.remove(merged_geotiff)
# Crop
if args.crop > 0:
Cropper.crop(merged_bounds_file,
tree.odm_orthophoto_tif,
orthophoto_vars)
# Overviews
if args.build_overviews:
orthophoto.build_overviews(tree.odm_orthophoto_tif)
elif len(all_orthos_and_cutlines) == 1:
# Simply copy
log.ODM_WARNING(
"A single orthophoto/cutline pair was found between all submodels."
)
shutil.copyfile(all_orthos_and_cutlines[0][0],
tree.odm_orthophoto_tif)
else:
log.ODM_WARNING(
"No orthophoto/cutline pairs were found in any of the submodels. No orthophoto will be generated."
)
else:
log.ODM_WARNING("Found merged orthophoto in %s" %
tree.odm_orthophoto_tif)
self.update_progress(75)
# Merge DEMs
def merge_dems(dem_filename, human_name):
if not io.dir_exists(tree.path('odm_dem')):
system.mkdir_p(tree.path('odm_dem'))
dem_file = tree.path("odm_dem", dem_filename)
if not io.file_exists(dem_file) or self.rerun():
all_dems = get_submodel_paths(tree.submodels_path,
"odm_dem", dem_filename)
log.ODM_INFO("Merging %ss" % human_name)
# Merge
dem_vars = utils.get_dem_vars(args)
euclidean_merge_dems(all_dems, dem_file, dem_vars)
if io.file_exists(dem_file):
# Crop
if args.crop > 0:
Cropper.crop(merged_bounds_file, dem_file,
dem_vars)
log.ODM_INFO("Created %s" % dem_file)
else:
log.ODM_WARNING("Cannot merge %s, %s was not created" %
(human_name, dem_file))
else:
log.ODM_WARNING("Found merged %s in %s" %
(human_name, dem_filename))
if args.merge in ['all', 'dem'] and args.dsm:
merge_dems("dsm.tif", "DSM")
if args.merge in ['all', 'dem'] and args.dtm:
merge_dems("dtm.tif", "DTM")
# Stop the pipeline short! We're done.
self.next_stage = None
else:
log.ODM_INFO("Normal dataset, nothing to merge.")
self.progress = 0.0
def process(self, args, outputs):
tree = outputs['tree']
reconstruction = outputs['reconstruction']
doPointCloudGeo = True
transformPointCloud = True
verbose = '-verbose' if self.params.get('verbose') else ''
runs = [{
'georeferencing_dir':
tree.odm_georeferencing,
'texturing_dir':
tree.odm_texturing,
'model':
os.path.join(tree.odm_texturing, tree.odm_textured_model_obj)
}]
if args.skip_3dmodel:
runs = []
if not args.use_3dmesh:
# Make sure 2.5D mesh is georeferenced before the 3D mesh
# Because it will be used to calculate a transform
# for the point cloud. If we use the 3D model transform,
# DEMs and orthophoto might not align!
runs.insert(
0, {
'georeferencing_dir':
tree.odm_25dgeoreferencing,
'texturing_dir':
tree.odm_25dtexturing,
'model':
os.path.join(tree.odm_25dtexturing,
tree.odm_textured_model_obj)
})
for r in runs:
odm_georeferencing_model_obj_geo = os.path.join(
r['texturing_dir'], tree.odm_georeferencing_model_obj_geo)
odm_georeferencing_log = os.path.join(r['georeferencing_dir'],
tree.odm_georeferencing_log)
odm_georeferencing_transform_file = os.path.join(
r['georeferencing_dir'],
tree.odm_georeferencing_transform_file)
odm_georeferencing_model_txt_geo_file = os.path.join(
r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo)
if not io.file_exists(odm_georeferencing_model_obj_geo) or \
not io.file_exists(tree.odm_georeferencing_model_laz) or self.rerun():
# odm_georeference definitions
kwargs = {
'bin': context.odm_modules_path,
'input_pc_file': tree.filtered_point_cloud,
'bundle': tree.opensfm_bundle,
'imgs': tree.dataset_raw,
'imgs_list': tree.opensfm_bundle_list,
'model': r['model'],
'log': odm_georeferencing_log,
'input_trans_file': tree.opensfm_transformation,
'transform_file': odm_georeferencing_transform_file,
'coords': tree.odm_georeferencing_coords,
'output_pc_file': tree.odm_georeferencing_model_laz,
'geo_sys': odm_georeferencing_model_txt_geo_file,
'model_geo': odm_georeferencing_model_obj_geo,
'verbose': verbose
}
if transformPointCloud:
kwargs[
'pc_params'] = '-inputPointCloudFile {input_pc_file} -outputPointCloudFile {output_pc_file}'.format(
**kwargs)
if reconstruction.is_georeferenced():
kwargs[
'pc_params'] += ' -outputPointCloudSrs %s' % pipes.quote(
reconstruction.georef.proj4())
else:
log.ODM_WARNING(
'NO SRS: The output point cloud will not have a SRS.'
)
else:
kwargs['pc_params'] = ''
if io.file_exists(
tree.opensfm_transformation) and io.file_exists(
tree.odm_georeferencing_coords):
log.ODM_INFO(
'Running georeferencing with OpenSfM transformation matrix'
)
system.run(
'{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'
.format(**kwargs))
elif io.file_exists(tree.odm_georeferencing_coords):
log.ODM_INFO(
'Running georeferencing with generated coords file.')
system.run(
'{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'
.format(**kwargs))
else:
log.ODM_WARNING(
'Georeferencing failed. Make sure your '
'photos have geotags in the EXIF or you have '
'provided a GCP file. ')
doPointCloudGeo = False # skip the rest of the georeferencing
if doPointCloudGeo:
reconstruction.georef.extract_offsets(
odm_georeferencing_model_txt_geo_file)
# XYZ point cloud output
if args.pc_csv:
log.ODM_INFO(
"Creating geo-referenced CSV file (XYZ format)")
system.run(
"pdal translate -i \"{}\" "
"-o \"{}\" "
"--writers.text.format=csv "
"--writers.text.order=\"X,Y,Z\" "
"--writers.text.keep_unspecified=false ".format(
tree.odm_georeferencing_model_laz,
tree.odm_georeferencing_xyz_file))
# LAS point cloud output
if args.pc_las:
log.ODM_INFO("Creating geo-referenced LAS file")
system.run("pdal translate -i \"{}\" "
"-o \"{}\" ".format(
tree.odm_georeferencing_model_laz,
tree.odm_georeferencing_model_las))
# EPT point cloud output
if args.pc_ept:
log.ODM_INFO(
"Creating geo-referenced Entwine Point Tile output"
)
entwine.build([tree.odm_georeferencing_model_laz],
tree.entwine_pointcloud,
max_concurrency=args.max_concurrency,
rerun=self.rerun())
if args.crop > 0:
log.ODM_INFO(
"Calculating cropping area and generating bounds shapefile from point cloud"
)
cropper = Cropper(tree.odm_georeferencing,
'odm_georeferenced_model')
decimation_step = 40 if args.fast_orthophoto or args.use_opensfm_dense else 90
# More aggressive decimation for large datasets
if not args.fast_orthophoto:
decimation_step *= int(
len(reconstruction.photos) / 1000) + 1
cropper.create_bounds_gpkg(
tree.odm_georeferencing_model_laz,
args.crop,
decimation_step=decimation_step)
# Do not execute a second time, since
# We might be doing georeferencing for
# multiple models (3D, 2.5D, ...)
doPointCloudGeo = False
transformPointCloud = False
else:
log.ODM_WARNING('Found a valid georeferenced model in: %s' %
tree.odm_georeferencing_model_laz)