def process(self, inputs, outputs):
# Benchmarking
start_time = system.now_raw()
log.ODM_INFO('Running ODM DEM Cell')
# get inputs
args = self.inputs.args
tree = self.inputs.tree
las_model_found = io.file_exists(tree.odm_georeferencing_model_las)
# check if we rerun cell or not
rerun_cell = (args.rerun is not None and
args.rerun == 'odm_dem') or \
(args.rerun_all) or \
(args.rerun_from is not None and
'odm_dem' in args.rerun_from)
log.ODM_INFO('Classify: ' + str(args.pc_classify != "none"))
log.ODM_INFO('Create DSM: ' + str(args.dsm))
log.ODM_INFO('Create DTM: ' + str(args.dtm))
log.ODM_INFO('DEM input file {0} found: {1}'.format(
tree.odm_georeferencing_model_las, str(las_model_found)))
# Setup terrain parameters
terrain_params_map = {
'flatnonforest': (1, 3),
'flatforest': (1, 2),
'complexnonforest': (5, 2),
'complexforest': (10, 2)
}
terrain_params = terrain_params_map[args.dem_terrain_type.lower()]
slope, cellsize = terrain_params
# define paths and create working directories
odm_dem_root = tree.path('odm_dem')
if not io.dir_exists(odm_dem_root):
system.mkdir_p(odm_dem_root)
if args.pc_classify != "none" and las_model_found:
pc_classify_marker = os.path.join(odm_dem_root,
'pc_classify_done.txt')
if not io.file_exists(pc_classify_marker) or rerun_cell:
log.ODM_INFO("Classifying {} using {}".format(
tree.odm_georeferencing_model_las, args.pc_classify))
commands.classify(tree.odm_georeferencing_model_las,
args.pc_classify == "smrf",
slope,
cellsize,
approximate=args.dem_approximate,
initialDistance=args.dem_initial_distance,
verbose=args.verbose)
with open(pc_classify_marker, 'w') as f:
f.write('Classify: {}\n'.format(args.pc_classify))
f.write('Slope: {}\n'.format(slope))
f.write('Cellsize: {}\n'.format(cellsize))
f.write('Approximate: {}\n'.format(args.dem_approximate))
f.write('InitialDistance: {}\n'.format(
args.dem_initial_distance))
# Do we need to process anything here?
if (args.dsm or args.dtm) and las_model_found:
dsm_output_filename = os.path.join(odm_dem_root, 'dsm.tif')
dtm_output_filename = os.path.join(odm_dem_root, 'dtm.tif')
if (args.dtm and not io.file_exists(dtm_output_filename)) or \
(args.dsm and not io.file_exists(dsm_output_filename)) or \
rerun_cell:
products = []
if args.dsm: products.append('dsm')
if args.dtm: products.append('dtm')
radius_steps = [args.dem_resolution]
for _ in range(args.dem_gapfill_steps - 1):
radius_steps.append(
radius_steps[-1] *
3) # 3 is arbitrary, maybe there's a better value?
for product in products:
commands.create_dems([tree.odm_georeferencing_model_las],
product,
radius=map(str, radius_steps),
gapfill=True,
outdir=odm_dem_root,
resolution=args.dem_resolution,
maxsd=args.dem_maxsd,
maxangle=args.dem_maxangle,
decimation=args.dem_decimation,
verbose=args.verbose)
if args.crop > 0:
bounds_shapefile_path = os.path.join(
tree.odm_georeferencing,
'odm_georeferenced_model.bounds.shp')
if os.path.exists(bounds_shapefile_path):
Cropper.crop(
bounds_shapefile_path,
os.path.join(odm_dem_root,
"{}.tif".format(product)), {
'TILED': 'YES',
'COMPRESS': 'LZW',
'BLOCKXSIZE': 512,
'BLOCKYSIZE': 512,
'NUM_THREADS': 'ALL_CPUS'
})
else:
log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root)
else:
log.ODM_WARNING('DEM will not be generated')
if args.time:
system.benchmark(start_time, tree.benchmarking, 'Dem')
log.ODM_INFO('Running ODM DEM Cell - Finished')
return ecto.OK if args.end_with != 'odm_dem' else ecto.QUIT
def process(self, args, outputs):
tree = outputs['tree']
reconstruction = outputs['reconstruction']
if outputs['large']:
if not os.path.exists(tree.submodels_path):
log.ODM_ERROR(
"We reached the merge stage, but %s folder does not exist. Something must have gone wrong at an earlier stage. Check the log and fix possible problem before restarting?"
% tree.submodels_path)
exit(1)
# Merge point clouds
if args.merge in ['all', 'pointcloud']:
if not io.dir_exists(tree.entwine_pointcloud) or self.rerun():
all_point_clouds = get_submodel_paths(
tree.submodels_path, "odm_georeferencing",
"odm_georeferenced_model.laz")
try:
entwine.build(all_point_clouds,
tree.entwine_pointcloud,
max_concurrency=args.max_concurrency,
rerun=self.rerun())
except Exception as e:
log.ODM_WARNING(
"Could not merge EPT point cloud: %s (skipping)" %
str(e))
else:
log.ODM_WARNING("Found merged EPT point cloud in %s" %
tree.entwine_pointcloud)
if not io.file_exists(
tree.odm_georeferencing_model_laz) or self.rerun():
if io.dir_exists(tree.entwine_pointcloud):
try:
system.run('pdal translate "ept://{}" "{}"'.format(
tree.entwine_pointcloud,
tree.odm_georeferencing_model_laz))
except Exception as e:
log.ODM_WARNING(
"Cannot export EPT dataset to LAZ: %s" %
str(e))
else:
log.ODM_WARNING(
"No EPT point cloud found (%s), skipping LAZ conversion)"
% tree.entwine_pointcloud)
else:
log.ODM_WARNING("Found merged point cloud in %s" %
tree.odm_georeferencing_model_laz)
self.update_progress(25)
# Merge crop bounds
merged_bounds_file = os.path.join(
tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg')
if not io.file_exists(merged_bounds_file) or self.rerun():
all_bounds = get_submodel_paths(
tree.submodels_path, 'odm_georeferencing',
'odm_georeferenced_model.bounds.gpkg')
log.ODM_INFO("Merging all crop bounds: %s" % all_bounds)
if len(all_bounds) > 0:
# Calculate a new crop area
# based on the convex hull of all crop areas of all submodels
# (without a buffer, otherwise we are double-cropping)
Cropper.merge_bounds(all_bounds, merged_bounds_file, 0)
else:
log.ODM_WARNING("No bounds found for any submodel.")
# Merge orthophotos
if args.merge in ['all', 'orthophoto']:
if not io.dir_exists(tree.odm_orthophoto):
system.mkdir_p(tree.odm_orthophoto)
if not io.file_exists(tree.odm_orthophoto_tif) or self.rerun():
all_orthos_and_cutlines = get_all_submodel_paths(
tree.submodels_path,
os.path.join("odm_orthophoto", "odm_orthophoto.tif"),
os.path.join("odm_orthophoto", "cutline.gpkg"),
)
if len(all_orthos_and_cutlines) > 1:
log.ODM_INFO(
"Found %s submodels with valid orthophotos and cutlines"
% len(all_orthos_and_cutlines))
# TODO: histogram matching via rasterio
# currently parts have different color tones
merged_geotiff = os.path.join(
tree.odm_orthophoto, "odm_orthophoto.merged.tif")
kwargs = {
'orthophoto_merged':
merged_geotiff,
'input_files':
' '.join(
map(lambda i: quote(i[0]),
all_orthos_and_cutlines)),
'max_memory':
get_max_memory(),
'threads':
args.max_concurrency,
}
# use bounds as cutlines (blending)
if io.file_exists(merged_geotiff):
os.remove(merged_geotiff)
system.run('gdal_merge.py -o {orthophoto_merged} '
#'-createonly '
'-co "BIGTIFF=YES" '
'-co "BLOCKXSIZE=512" '
'-co "BLOCKYSIZE=512" '
'--config GDAL_CACHEMAX {max_memory}% '
'{input_files} '.format(**kwargs))
for ortho_cutline in all_orthos_and_cutlines:
kwargs['input_file'], kwargs[
'cutline'] = ortho_cutline
# Note: cblend has a high performance penalty
system.run(
'gdalwarp -cutline {cutline} '
'-cblend 20 '
'-r bilinear -multi '
'-wo NUM_THREADS={threads} '
'--config GDAL_CACHEMAX {max_memory}% '
'{input_file} {orthophoto_merged}'.format(
**kwargs))
# Apply orthophoto settings (compression, tiling, etc.)
orthophoto_vars = orthophoto.get_orthophoto_vars(args)
if io.file_exists(tree.odm_orthophoto_tif):
os.remove(tree.odm_orthophoto_tif)
kwargs = {
'vars':
' '.join([
'-co %s=%s' % (k, orthophoto_vars[k])
for k in orthophoto_vars
]),
'max_memory':
get_max_memory(),
'merged':
merged_geotiff,
'log':
tree.odm_orthophoto_tif_log,
'orthophoto':
tree.odm_orthophoto_tif,
}
system.run(
'gdal_translate '
'{vars} '
'--config GDAL_CACHEMAX {max_memory}% '
'{merged} {orthophoto} > {log}'.format(**kwargs))
os.remove(merged_geotiff)
# Crop
if args.crop > 0:
Cropper.crop(merged_bounds_file,
tree.odm_orthophoto_tif,
orthophoto_vars)
# Overviews
if args.build_overviews:
orthophoto.build_overviews(tree.odm_orthophoto_tif)
elif len(all_orthos_and_cutlines) == 1:
# Simply copy
log.ODM_WARNING(
"A single orthophoto/cutline pair was found between all submodels."
)
shutil.copyfile(all_orthos_and_cutlines[0][0],
tree.odm_orthophoto_tif)
else:
log.ODM_WARNING(
"No orthophoto/cutline pairs were found in any of the submodels. No orthophoto will be generated."
)
else:
log.ODM_WARNING("Found merged orthophoto in %s" %
tree.odm_orthophoto_tif)
self.update_progress(75)
# Merge DEMs
def merge_dems(dem_filename, human_name):
if not io.dir_exists(tree.path('odm_dem')):
system.mkdir_p(tree.path('odm_dem'))
dem_file = tree.path("odm_dem", dem_filename)
if not io.file_exists(dem_file) or self.rerun():
all_dems = get_submodel_paths(tree.submodels_path,
"odm_dem", dem_filename)
log.ODM_INFO("Merging %ss" % human_name)
# Merge
dem_vars = utils.get_dem_vars(args)
eu_map_source = None # Default
# Use DSM's euclidean map for DTMs
# (requires the DSM to be computed)
if human_name == "DTM":
eu_map_source = "dsm"
euclidean_merge_dems(all_dems,
dem_file,
dem_vars,
euclidean_map_source=eu_map_source)
if io.file_exists(dem_file):
# Crop
if args.crop > 0:
Cropper.crop(merged_bounds_file, dem_file,
dem_vars)
log.ODM_INFO("Created %s" % dem_file)
else:
log.ODM_WARNING("Cannot merge %s, %s was not created" %
(human_name, dem_file))
else:
log.ODM_WARNING("Found merged %s in %s" %
(human_name, dem_filename))
if args.merge in ['all', 'dem'] and args.dsm:
merge_dems("dsm.tif", "DSM")
if args.merge in ['all', 'dem'] and args.dtm:
merge_dems("dtm.tif", "DTM")
# Stop the pipeline short! We're done.
self.next_stage = None
else:
log.ODM_INFO("Normal dataset, nothing to merge.")
self.progress = 0.0
def process(self, args, outputs):
tree = outputs['tree']
reconstruction = outputs['reconstruction']
if outputs['large']:
if not os.path.exists(tree.submodels_path):
log.ODM_ERROR(
"We reached the merge stage, but %s folder does not exist. Something must have gone wrong at an earlier stage. Check the log and fix possible problem before restarting?"
% tree.submodels_path)
exit(1)
# Merge point clouds
if args.merge in ['all', 'pointcloud']:
if not io.file_exists(
tree.odm_georeferencing_model_laz) or self.rerun():
all_point_clouds = get_submodel_paths(
tree.submodels_path, "odm_georeferencing",
"odm_georeferenced_model.laz")
try:
point_cloud.merge(all_point_clouds,
tree.odm_georeferencing_model_laz,
rerun=self.rerun())
point_cloud.post_point_cloud_steps(args, tree)
except Exception as e:
log.ODM_WARNING(
"Could not merge point cloud: %s (skipping)" %
str(e))
else:
log.ODM_WARNING("Found merged point cloud in %s" %
tree.odm_georeferencing_model_laz)
self.update_progress(25)
# Merge crop bounds
merged_bounds_file = os.path.join(
tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg')
if not io.file_exists(merged_bounds_file) or self.rerun():
all_bounds = get_submodel_paths(
tree.submodels_path, 'odm_georeferencing',
'odm_georeferenced_model.bounds.gpkg')
log.ODM_INFO("Merging all crop bounds: %s" % all_bounds)
if len(all_bounds) > 0:
# Calculate a new crop area
# based on the convex hull of all crop areas of all submodels
# (without a buffer, otherwise we are double-cropping)
Cropper.merge_bounds(all_bounds, merged_bounds_file, 0)
else:
log.ODM_WARNING("No bounds found for any submodel.")
# Merge orthophotos
if args.merge in ['all', 'orthophoto']:
if not io.dir_exists(tree.odm_orthophoto):
system.mkdir_p(tree.odm_orthophoto)
if not io.file_exists(tree.odm_orthophoto_tif) or self.rerun():
all_orthos_and_ortho_cuts = get_all_submodel_paths(
tree.submodels_path,
os.path.join("odm_orthophoto",
"odm_orthophoto_feathered.tif"),
os.path.join("odm_orthophoto",
"odm_orthophoto_cut.tif"),
)
if len(all_orthos_and_ortho_cuts) > 1:
log.ODM_INFO(
"Found %s submodels with valid orthophotos and cutlines"
% len(all_orthos_and_ortho_cuts))
# TODO: histogram matching via rasterio
# currently parts have different color tones
if io.file_exists(tree.odm_orthophoto_tif):
os.remove(tree.odm_orthophoto_tif)
orthophoto_vars = orthophoto.get_orthophoto_vars(args)
orthophoto.merge(all_orthos_and_ortho_cuts,
tree.odm_orthophoto_tif,
orthophoto_vars)
orthophoto.post_orthophoto_steps(
args, merged_bounds_file, tree.odm_orthophoto_tif)
elif len(all_orthos_and_ortho_cuts) == 1:
# Simply copy
log.ODM_WARNING(
"A single orthophoto/cutline pair was found between all submodels."
)
shutil.copyfile(all_orthos_and_ortho_cuts[0][0],
tree.odm_orthophoto_tif)
else:
log.ODM_WARNING(
"No orthophoto/cutline pairs were found in any of the submodels. No orthophoto will be generated."
)
else:
log.ODM_WARNING("Found merged orthophoto in %s" %
tree.odm_orthophoto_tif)
self.update_progress(75)
# Merge DEMs
def merge_dems(dem_filename, human_name):
if not io.dir_exists(tree.path('odm_dem')):
system.mkdir_p(tree.path('odm_dem'))
dem_file = tree.path("odm_dem", dem_filename)
if not io.file_exists(dem_file) or self.rerun():
all_dems = get_submodel_paths(tree.submodels_path,
"odm_dem", dem_filename)
log.ODM_INFO("Merging %ss" % human_name)
# Merge
dem_vars = utils.get_dem_vars(args)
eu_map_source = None # Default
# Use DSM's euclidean map for DTMs
# (requires the DSM to be computed)
if human_name == "DTM":
eu_map_source = "dsm"
euclidean_merge_dems(all_dems,
dem_file,
dem_vars,
euclidean_map_source=eu_map_source)
if io.file_exists(dem_file):
# Crop
if args.crop > 0:
Cropper.crop(
merged_bounds_file,
dem_file,
dem_vars,
keep_original=not args.optimize_disk_space)
log.ODM_INFO("Created %s" % dem_file)
else:
log.ODM_WARNING("Cannot merge %s, %s was not created" %
(human_name, dem_file))
else:
log.ODM_WARNING("Found merged %s in %s" %
(human_name, dem_filename))
if args.merge in ['all', 'dem'] and args.dsm:
merge_dems("dsm.tif", "DSM")
if args.merge in ['all', 'dem'] and args.dtm:
merge_dems("dtm.tif", "DTM")
# Stop the pipeline short! We're done.
self.next_stage = None
else:
log.ODM_INFO("Normal dataset, nothing to merge.")
self.progress = 0.0
def process(self, args, outputs):
tree = outputs['tree']
reconstruction = outputs['reconstruction']
dem_input = tree.odm_georeferencing_model_laz
pc_model_found = io.file_exists(dem_input)
ignore_resolution = False
pseudo_georeference = False
if not reconstruction.is_georeferenced():
log.ODM_WARNING("Not georeferenced, using ungeoreferenced point cloud...")
ignore_resolution = True
pseudo_georeference = True
resolution = gsd.cap_resolution(args.dem_resolution, tree.opensfm_reconstruction,
gsd_error_estimate=-3,
ignore_gsd=args.ignore_gsd,
ignore_resolution=ignore_resolution,
has_gcp=reconstruction.has_gcp())
log.ODM_INFO('Classify: ' + str(args.pc_classify))
log.ODM_INFO('Create DSM: ' + str(args.dsm))
log.ODM_INFO('Create DTM: ' + str(args.dtm))
log.ODM_INFO('DEM input file {0} found: {1}'.format(dem_input, str(pc_model_found)))
# define paths and create working directories
odm_dem_root = tree.path('odm_dem')
if not io.dir_exists(odm_dem_root):
system.mkdir_p(odm_dem_root)
if args.pc_classify and pc_model_found:
pc_classify_marker = os.path.join(odm_dem_root, 'pc_classify_done.txt')
if not io.file_exists(pc_classify_marker) or self.rerun():
log.ODM_INFO("Classifying {} using Simple Morphological Filter".format(dem_input))
commands.classify(dem_input,
args.smrf_scalar,
args.smrf_slope,
args.smrf_threshold,
args.smrf_window,
verbose=args.verbose
)
with open(pc_classify_marker, 'w') as f:
f.write('Classify: smrf\n')
f.write('Scalar: {}\n'.format(args.smrf_scalar))
f.write('Slope: {}\n'.format(args.smrf_slope))
f.write('Threshold: {}\n'.format(args.smrf_threshold))
f.write('Window: {}\n'.format(args.smrf_window))
progress = 20
self.update_progress(progress)
if args.pc_rectify:
commands.rectify(dem_input, args.debug)
# Do we need to process anything here?
if (args.dsm or args.dtm) and pc_model_found:
dsm_output_filename = os.path.join(odm_dem_root, 'dsm.tif')
dtm_output_filename = os.path.join(odm_dem_root, 'dtm.tif')
if (args.dtm and not io.file_exists(dtm_output_filename)) or \
(args.dsm and not io.file_exists(dsm_output_filename)) or \
self.rerun():
products = []
if args.dsm or (args.dtm and args.dem_euclidean_map): products.append('dsm')
if args.dtm: products.append('dtm')
radius_steps = [(resolution / 100.0) / 2.0]
for _ in range(args.dem_gapfill_steps - 1):
radius_steps.append(radius_steps[-1] * 2) # 2 is arbitrary, maybe there's a better value?
for product in products:
commands.create_dem(
dem_input,
product,
output_type='idw' if product == 'dtm' else 'max',
radiuses=list(map(str, radius_steps)),
gapfill=args.dem_gapfill_steps > 0,
outdir=odm_dem_root,
resolution=resolution / 100.0,
decimation=args.dem_decimation,
verbose=args.verbose,
max_workers=args.max_concurrency,
keep_unfilled_copy=args.dem_euclidean_map
)
dem_geotiff_path = os.path.join(odm_dem_root, "{}.tif".format(product))
bounds_file_path = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg')
if args.crop > 0:
# Crop DEM
Cropper.crop(bounds_file_path, dem_geotiff_path, utils.get_dem_vars(args), keep_original=not args.optimize_disk_space)
if args.dem_euclidean_map:
unfilled_dem_path = io.related_file_path(dem_geotiff_path, postfix=".unfilled")
if args.crop > 0:
# Crop unfilled DEM
Cropper.crop(bounds_file_path, unfilled_dem_path, utils.get_dem_vars(args), keep_original=not args.optimize_disk_space)
commands.compute_euclidean_map(unfilled_dem_path,
io.related_file_path(dem_geotiff_path, postfix=".euclideand"),
overwrite=True)
if pseudo_georeference:
# 0.1 is arbitrary
pseudogeo.add_pseudo_georeferencing(dem_geotiff_path, 0.1)
if pseudo_georeference:
pseudogeo.add_pseudo_georeferencing(dem_geotiff_path)
if args.tiles:
generate_dem_tiles(dem_geotiff_path, tree.path("%s_tiles" % product), args.max_concurrency)
progress += 30
self.update_progress(progress)
else:
log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root)
else:
log.ODM_WARNING('DEM will not be generated')
def process(self, args, outputs):
tree = outputs['tree']
reconstruction = outputs['reconstruction']
verbose = '-verbose' if args.verbose else ''
# define paths and create working directories
system.mkdir_p(tree.odm_orthophoto)
if not io.file_exists(tree.odm_orthophoto_file) or self.rerun():
# odm_orthophoto definitions
kwargs = {
'bin':
context.odm_modules_path,
'log':
tree.odm_orthophoto_log,
'ortho':
tree.odm_orthophoto_file,
'corners':
tree.odm_orthophoto_corners,
'res':
1.0 / (gsd.cap_resolution(args.orthophoto_resolution,
tree.opensfm_reconstruction,
ignore_gsd=args.ignore_gsd) / 100.0),
'verbose':
verbose
}
# Have geo coordinates?
georef = reconstruction.georef
# Check if the georef object is initialized
# (during a --rerun this might not be)
# TODO: we should move this to a more central
# location (perhaps during the dataset initialization)
if georef and not georef.utm_east_offset:
georeferencing_dir = tree.odm_georeferencing if args.use_3dmesh and not args.skip_3dmodel else tree.odm_25dgeoreferencing
odm_georeferencing_model_txt_geo_file = os.path.join(
georeferencing_dir, tree.odm_georeferencing_model_txt_geo)
if io.file_exists(odm_georeferencing_model_txt_geo_file):
georef.extract_offsets(
odm_georeferencing_model_txt_geo_file)
else:
log.ODM_WARNING(
'Cannot read UTM offset from {}. An orthophoto will not be generated.'
.format(odm_georeferencing_model_txt_geo_file))
if georef:
if args.use_3dmesh:
kwargs['model_geo'] = os.path.join(
tree.odm_texturing,
tree.odm_georeferencing_model_obj_geo)
else:
kwargs['model_geo'] = os.path.join(
tree.odm_25dtexturing,
tree.odm_georeferencing_model_obj_geo)
else:
if args.use_3dmesh:
kwargs['model_geo'] = os.path.join(
tree.odm_texturing, tree.odm_textured_model_obj)
else:
kwargs['model_geo'] = os.path.join(
tree.odm_25dtexturing, tree.odm_textured_model_obj)
# run odm_orthophoto
system.run(
'{bin}/odm_orthophoto -inputFile {model_geo} '
'-logFile {log} -outputFile {ortho} -resolution {res} {verbose} '
'-outputCornerFile {corners}'.format(**kwargs))
# Create georeferenced GeoTiff
geotiffcreated = False
if georef and georef.projection and georef.utm_east_offset and georef.utm_north_offset:
ulx = uly = lrx = lry = 0.0
with open(tree.odm_orthophoto_corners) as f:
for lineNumber, line in enumerate(f):
if lineNumber == 0:
tokens = line.split(' ')
if len(tokens) == 4:
ulx = float(tokens[0]) + \
float(georef.utm_east_offset)
lry = float(tokens[1]) + \
float(georef.utm_north_offset)
lrx = float(tokens[2]) + \
float(georef.utm_east_offset)
uly = float(tokens[3]) + \
float(georef.utm_north_offset)
log.ODM_INFO('Creating GeoTIFF')
orthophoto_vars = orthophoto.get_orthophoto_vars(args)
kwargs = {
'ulx':
ulx,
'uly':
uly,
'lrx':
lrx,
'lry':
lry,
'vars':
' '.join([
'-co %s=%s' % (k, orthophoto_vars[k])
for k in orthophoto_vars
]),
'proj':
georef.projection.srs,
'png':
tree.odm_orthophoto_file,
'tiff':
tree.odm_orthophoto_tif,
'log':
tree.odm_orthophoto_tif_log,
'max_memory':
get_max_memory(),
}
system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
'{vars} '
'-a_srs \"{proj}\" '
'--config GDAL_CACHEMAX {max_memory}% '
'{png} {tiff} > {log}'.format(**kwargs))
bounds_file_path = os.path.join(
tree.odm_georeferencing,
'odm_georeferenced_model.bounds.gpkg')
# Cutline computation, before cropping
# We want to use the full orthophoto, not the cropped one.
if args.orthophoto_cutline:
compute_cutline(tree.odm_orthophoto_tif,
bounds_file_path,
os.path.join(tree.odm_orthophoto,
"cutline.gpkg"),
args.max_concurrency,
tmpdir=os.path.join(
tree.odm_orthophoto,
"grass_cutline_tmpdir"))
if args.crop > 0:
Cropper.crop(bounds_file_path, tree.odm_orthophoto_tif,
orthophoto_vars)
if args.build_overviews:
orthophoto.build_overviews(tree.odm_orthophoto_tif)
geotiffcreated = True
if not geotiffcreated:
log.ODM_WARNING(
'No geo-referenced orthophoto created due '
'to missing geo-referencing or corner coordinates.')
else:
log.ODM_WARNING('Found a valid orthophoto in: %s' %
tree.odm_orthophoto_file)
def process(self, args, outputs):
tree = outputs['tree']
reconstruction = outputs['reconstruction']
if not os.path.exists(tree.odm_report): system.mkdir_p(tree.odm_report)
log.ODM_INFO("Exporting shots.geojson")
shots_geojson = os.path.join(tree.odm_report, "shots.geojson")
if not io.file_exists(shots_geojson) or self.rerun():
# Extract geographical camera shots
if reconstruction.is_georeferenced():
shots = get_geojson_shots_from_opensfm(
tree.opensfm_reconstruction, tree.opensfm_transformation,
reconstruction.get_proj_srs())
else:
# Pseudo geo
shots = get_geojson_shots_from_opensfm(
tree.opensfm_reconstruction,
pseudo_geotiff=tree.odm_orthophoto_tif)
if shots:
with open(shots_geojson, "w") as fout:
fout.write(json.dumps(shots))
log.ODM_INFO("Wrote %s" % shots_geojson)
else:
log.ODM_WARNING("Cannot extract shots")
else:
log.ODM_WARNING('Found a valid shots file in: %s' % shots_geojson)
if args.skip_report:
# Stop right here
log.ODM_WARNING("Skipping report generation as requested")
return
# Augment OpenSfM stats file with our own stats
odm_stats_json = os.path.join(tree.odm_report, "stats.json")
octx = OSFMContext(tree.opensfm)
osfm_stats_json = octx.path("stats", "stats.json")
odm_stats = None
point_cloud_file = None
views_dimension = None
if not os.path.exists(odm_stats_json) or self.rerun():
if os.path.exists(osfm_stats_json):
with open(osfm_stats_json, 'r') as f:
odm_stats = json.loads(f.read())
# Add point cloud stats
if os.path.exists(tree.odm_georeferencing_model_laz):
point_cloud_file = tree.odm_georeferencing_model_laz
views_dimension = "UserData"
# pc_info_file should have been generated by cropper
pc_info_file = os.path.join(
tree.odm_georeferencing,
"odm_georeferenced_model.info.json")
odm_stats[
'point_cloud_statistics'] = generate_point_cloud_stats(
tree.odm_georeferencing_model_laz, pc_info_file)
else:
ply_pc = os.path.join(tree.odm_filterpoints,
"point_cloud.ply")
if os.path.exists(ply_pc):
point_cloud_file = ply_pc
views_dimension = "views"
pc_info_file = os.path.join(tree.odm_filterpoints,
"point_cloud.info.json")
odm_stats[
'point_cloud_statistics'] = generate_point_cloud_stats(
ply_pc, pc_info_file)
else:
log.ODM_WARNING("No point cloud found")
odm_stats['point_cloud_statistics'][
'dense'] = not args.fast_orthophoto
# Add runtime stats
total_time = (system.now_raw() -
outputs['start_time']).total_seconds()
odm_stats['odm_processing_statistics'] = {
'total_time':
total_time,
'total_time_human':
hms(total_time),
'average_gsd':
gsd.opensfm_reconstruction_average_gsd(
octx.recon_file(),
use_all_shots=reconstruction.has_gcp()),
}
with open(odm_stats_json, 'w') as f:
f.write(json.dumps(odm_stats))
else:
log.ODM_WARNING(
"Cannot generate report, OpenSfM stats are missing")
else:
log.ODM_WARNING("Reading existing stats %s" % odm_stats_json)
with open(odm_stats_json, 'r') as f:
odm_stats = json.loads(f.read())
# Generate overlap diagram
if odm_stats.get('point_cloud_statistics'
) and point_cloud_file and views_dimension:
bounds = odm_stats['point_cloud_statistics'].get('stats', {}).get(
'bbox', {}).get('native', {}).get('bbox')
if bounds:
image_target_size = 1400 # pixels
osfm_stats_dir = os.path.join(tree.opensfm, "stats")
diagram_tiff = os.path.join(osfm_stats_dir, "overlap.tif")
diagram_png = os.path.join(osfm_stats_dir, "overlap.png")
width = bounds.get('maxx') - bounds.get('minx')
height = bounds.get('maxy') - bounds.get('miny')
max_dim = max(width, height)
resolution = float(max_dim) / float(image_target_size)
radius = resolution * math.sqrt(2)
# Larger radius for sparse point cloud diagram
if not odm_stats['point_cloud_statistics']['dense']:
radius *= 10
system.run("pdal translate -i \"{}\" "
"-o \"{}\" "
"--writer gdal "
"--writers.gdal.resolution={} "
"--writers.gdal.data_type=uint8_t "
"--writers.gdal.dimension={} "
"--writers.gdal.output_type=max "
"--writers.gdal.radius={} ".format(
point_cloud_file, diagram_tiff, resolution,
views_dimension, radius))
report_assets = os.path.abspath(
os.path.join(os.path.dirname(__file__),
"../opendm/report"))
overlap_color_map = os.path.join(report_assets,
"overlap_color_map.txt")
bounds_file_path = os.path.join(
tree.odm_georeferencing,
'odm_georeferenced_model.bounds.gpkg')
if args.crop > 0 and os.path.isfile(bounds_file_path):
Cropper.crop(bounds_file_path,
diagram_tiff,
get_orthophoto_vars(args),
keep_original=False)
system.run(
"gdaldem color-relief \"{}\" \"{}\" \"{}\" -of PNG -alpha".
format(diagram_tiff, overlap_color_map, diagram_png))
# Copy assets
for asset in [
"overlap_diagram_legend.png", "dsm_gradient.png"
]:
shutil.copy(os.path.join(report_assets, asset),
os.path.join(osfm_stats_dir, asset))
# Generate previews of ortho/dsm
if os.path.isfile(tree.odm_orthophoto_tif):
osfm_ortho = os.path.join(osfm_stats_dir, "ortho.png")
generate_png(tree.odm_orthophoto_tif, osfm_ortho,
image_target_size)
dems = []
if args.dsm:
dems.append("dsm")
if args.dtm:
dems.append("dtm")
for dem in dems:
dem_file = tree.path("odm_dem", "%s.tif" % dem)
if os.path.isfile(dem_file):
# Resize first (faster)
resized_dem_file = io.related_file_path(
dem_file, postfix=".preview")
system.run(
"gdal_translate -outsize {} 0 \"{}\" \"{}\" --config GDAL_CACHEMAX {}%"
.format(image_target_size, dem_file,
resized_dem_file, get_max_memory()))
log.ODM_INFO("Computing raster stats for %s" %
resized_dem_file)
dem_stats = get_raster_stats(resized_dem_file)
if len(dem_stats) > 0:
odm_stats[dem + '_statistics'] = dem_stats[0]
osfm_dem = os.path.join(osfm_stats_dir, "%s.png" % dem)
colored_dem, hillshade_dem, colored_hillshade_dem = generate_colored_hillshade(
resized_dem_file)
system.run(
"gdal_translate -outsize {} 0 -of png \"{}\" \"{}\" --config GDAL_CACHEMAX {}%"
.format(image_target_size, colored_hillshade_dem,
osfm_dem, get_max_memory()))
for f in [
resized_dem_file, colored_dem, hillshade_dem,
colored_hillshade_dem
]:
if os.path.isfile(f):
os.remove(f)
else:
log.ODM_WARNING(
"Cannot generate overlap diagram, cannot compute point cloud bounds"
)
else:
log.ODM_WARNING(
"Cannot generate overlap diagram, point cloud stats missing")
octx.export_report(os.path.join(tree.odm_report, "report.pdf"),
odm_stats, self.rerun())
def process(self, inputs, outputs):
# Benchmarking
start_time = system.now_raw()
log.ODM_INFO('Running ODM Orthophoto Cell')
# get inputs
args = self.inputs.args
tree = self.inputs.tree
reconstruction = inputs.reconstruction
verbose = '-verbose' if self.params.verbose else ''
# define paths and create working directories
system.mkdir_p(tree.odm_orthophoto)
# check if we rerun cell or not
rerun_cell = (args.rerun is not None and
args.rerun == 'odm_orthophoto') or \
(args.rerun_all) or \
(args.rerun_from is not None and
'odm_orthophoto' in args.rerun_from)
if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell:
# odm_orthophoto definitions
kwargs = {
'bin':
context.odm_modules_path,
'log':
tree.odm_orthophoto_log,
'ortho':
tree.odm_orthophoto_file,
'corners':
tree.odm_orthophoto_corners,
'res':
1.0 / (gsd.cap_resolution(self.params.resolution,
tree.opensfm_reconstruction,
ignore_gsd=args.ignore_gsd) / 100.0),
'verbose':
verbose
}
# Have geo coordinates?
georef = reconstruction.georef
# Check if the georef object is initialized
# (during a --rerun this might not be)
# TODO: we should move this to a more central
# location (perhaps during the dataset initialization)
if georef and not georef.utm_east_offset:
odm_georeferencing_model_txt_geo_file = os.path.join(
tree.odm_georeferencing,
tree.odm_georeferencing_model_txt_geo)
if io.file_exists(odm_georeferencing_model_txt_geo_file):
georef.extract_offsets(
odm_georeferencing_model_txt_geo_file)
else:
log.ODM_WARNING(
'Cannot read UTM offset from {}. An orthophoto will not be generated.'
.format(odm_georeferencing_model_txt_geo_file))
if georef:
if args.use_3dmesh:
kwargs['model_geo'] = os.path.join(
tree.odm_texturing,
tree.odm_georeferencing_model_obj_geo)
else:
kwargs['model_geo'] = os.path.join(
tree.odm_25dtexturing,
tree.odm_georeferencing_model_obj_geo)
else:
if args.use_3dmesh:
kwargs['model_geo'] = os.path.join(
tree.odm_texturing, tree.odm_textured_model_obj)
else:
kwargs['model_geo'] = os.path.join(
tree.odm_25dtexturing, tree.odm_textured_model_obj)
# run odm_orthophoto
system.run(
'{bin}/odm_orthophoto -inputFile {model_geo} '
'-logFile {log} -outputFile {ortho} -resolution {res} {verbose} '
'-outputCornerFile {corners}'.format(**kwargs))
# Create georeferenced GeoTiff
geotiffcreated = False
if georef and georef.projection and georef.utm_east_offset and georef.utm_north_offset:
ulx = uly = lrx = lry = 0.0
with open(tree.odm_orthophoto_corners) as f:
for lineNumber, line in enumerate(f):
if lineNumber == 0:
tokens = line.split(' ')
if len(tokens) == 4:
ulx = float(tokens[0]) + \
float(georef.utm_east_offset)
lry = float(tokens[1]) + \
float(georef.utm_north_offset)
lrx = float(tokens[2]) + \
float(georef.utm_east_offset)
uly = float(tokens[3]) + \
float(georef.utm_north_offset)
log.ODM_INFO('Creating GeoTIFF')
kwargs = {
'ulx':
ulx,
'uly':
uly,
'lrx':
lrx,
'lry':
lry,
'tiled':
'' if self.params.no_tiled else '-co TILED=yes ',
'compress':
self.params.compress,
'predictor':
'-co PREDICTOR=2 '
if self.params.compress in ['LZW', 'DEFLATE'] else '',
'proj':
georef.projection.srs,
'bigtiff':
self.params.bigtiff,
'png':
tree.odm_orthophoto_file,
'tiff':
tree.odm_orthophoto_tif,
'log':
tree.odm_orthophoto_tif_log,
'max_memory':
get_max_memory(),
'threads':
self.params.max_concurrency
}
system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
'{tiled} '
'-co BIGTIFF={bigtiff} '
'-co COMPRESS={compress} '
'{predictor} '
'-co BLOCKXSIZE=512 '
'-co BLOCKYSIZE=512 '
'-co NUM_THREADS={threads} '
'-a_srs \"{proj}\" '
'--config GDAL_CACHEMAX {max_memory}% '
'{png} {tiff} > {log}'.format(**kwargs))
if args.crop > 0:
shapefile_path = os.path.join(
tree.odm_georeferencing,
'odm_georeferenced_model.bounds.shp')
Cropper.crop(
shapefile_path, tree.odm_orthophoto_tif, {
'TILED':
'NO' if self.params.no_tiled else 'YES',
'COMPRESS':
self.params.compress,
'PREDICTOR':
'2' if self.params.compress in ['LZW', 'DEFLATE']
else '1',
'BIGTIFF':
self.params.bigtiff,
'BLOCKXSIZE':
512,
'BLOCKYSIZE':
512,
'NUM_THREADS':
self.params.max_concurrency
})
if self.params.build_overviews:
log.ODM_DEBUG("Building Overviews")
kwargs = {
'orthophoto': tree.odm_orthophoto_tif,
'log': tree.odm_orthophoto_gdaladdo_log
}
# Run gdaladdo
system.run(
'gdaladdo -ro -r average '
'--config BIGTIFF_OVERVIEW IF_SAFER '
'--config COMPRESS_OVERVIEW JPEG '
'{orthophoto} 2 4 8 16 > {log}'.format(**kwargs))
geotiffcreated = True
if not geotiffcreated:
log.ODM_WARNING(
'No geo-referenced orthophoto created due '
'to missing geo-referencing or corner coordinates.')
else:
log.ODM_WARNING('Found a valid orthophoto in: %s' %
tree.odm_orthophoto_file)
if args.time:
system.benchmark(start_time, tree.benchmarking, 'Orthophoto')
log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished')
return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT
def process(self, inputs, outputs):
# Benchmarking
start_time = system.now_raw()
log.ODM_INFO('Running ODM Georeferencing Cell')
# get inputs
args = inputs.args
tree = inputs.tree
reconstruction = inputs.reconstruction
gcpfile = tree.odm_georeferencing_gcp
doPointCloudGeo = True
transformPointCloud = True
verbose = '-verbose' if self.params.verbose else ''
geo_ref = reconstruction.georef
# check if we rerun cell or not
rerun_cell = (args.rerun is not None and
args.rerun == 'odm_georeferencing') or \
(args.rerun_all) or \
(args.rerun_from is not None and
'odm_georeferencing' in args.rerun_from)
runs = [{
'georeferencing_dir':
tree.odm_georeferencing,
'texturing_dir':
tree.odm_texturing,
'model':
os.path.join(tree.odm_texturing, tree.odm_textured_model_obj)
}]
if args.skip_3dmodel:
runs = []
if not args.use_3dmesh:
# Make sure 2.5D mesh is georeferenced before the 3D mesh
# Because it will be used to calculate a transform
# for the point cloud. If we use the 3D model transform,
# DEMs and orthophoto might not align!
runs.insert(
0, {
'georeferencing_dir':
tree.odm_25dgeoreferencing,
'texturing_dir':
tree.odm_25dtexturing,
'model':
os.path.join(tree.odm_25dtexturing,
tree.odm_textured_model_obj)
})
for r in runs:
odm_georeferencing_model_obj_geo = os.path.join(
r['texturing_dir'], tree.odm_georeferencing_model_obj_geo)
odm_georeferencing_log = os.path.join(r['georeferencing_dir'],
tree.odm_georeferencing_log)
odm_georeferencing_transform_file = os.path.join(
r['georeferencing_dir'],
tree.odm_georeferencing_transform_file)
odm_georeferencing_model_txt_geo_file = os.path.join(
r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo)
if not io.file_exists(odm_georeferencing_model_obj_geo) or \
not io.file_exists(tree.odm_georeferencing_model_laz) or rerun_cell:
# odm_georeference definitions
kwargs = {
'bin': context.odm_modules_path,
'bundle': tree.opensfm_bundle,
'imgs': tree.dataset_raw,
'imgs_list': tree.opensfm_bundle_list,
'model': r['model'],
'log': odm_georeferencing_log,
'input_trans_file': tree.opensfm_transformation,
'transform_file': odm_georeferencing_transform_file,
'coords': tree.odm_georeferencing_coords,
'output_pc_file': tree.odm_georeferencing_model_laz,
'geo_sys': odm_georeferencing_model_txt_geo_file,
'model_geo': odm_georeferencing_model_obj_geo,
'gcp': gcpfile,
'verbose': verbose
}
if args.fast_orthophoto:
kwargs['input_pc_file'] = os.path.join(
tree.opensfm, 'reconstruction.ply')
elif args.use_opensfm_dense:
kwargs['input_pc_file'] = tree.opensfm_model
else:
kwargs['input_pc_file'] = tree.smvs_model
if transformPointCloud:
kwargs[
'pc_params'] = '-inputPointCloudFile {input_pc_file} -outputPointCloudFile {output_pc_file}'.format(
**kwargs)
if geo_ref and geo_ref.projection and geo_ref.projection.srs:
kwargs[
'pc_params'] += ' -outputPointCloudSrs %s' % pipes.quote(
geo_ref.projection.srs)
else:
log.ODM_WARNING(
'NO SRS: The output point cloud will not have a SRS.'
)
else:
kwargs['pc_params'] = ''
# Check to see if the GCP file exists
if not self.params.use_exif and (self.params.gcp_file or
tree.odm_georeferencing_gcp):
log.ODM_INFO('Found %s' % gcpfile)
try:
system.run(
'{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys} -gcpFile {gcp} '
'-outputCoordFile {coords}'.format(**kwargs))
except Exception:
log.ODM_EXCEPTION('Georeferencing failed. ')
return ecto.QUIT
elif io.file_exists(
tree.opensfm_transformation) and io.file_exists(
tree.odm_georeferencing_coords):
log.ODM_INFO(
'Running georeferencing with OpenSfM transformation matrix'
)
system.run(
'{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'
.format(**kwargs))
elif io.file_exists(tree.odm_georeferencing_coords):
log.ODM_INFO(
'Running georeferencing with generated coords file.')
system.run(
'{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'
.format(**kwargs))
else:
log.ODM_WARNING(
'Georeferencing failed. Make sure your '
'photos have geotags in the EXIF or you have '
'provided a GCP file. ')
doPointCloudGeo = False # skip the rest of the georeferencing
if doPointCloudGeo:
# update images metadata
geo_ref.extract_offsets(
odm_georeferencing_model_txt_geo_file)
reconstruction.georef = geo_ref
# XYZ point cloud output
if args.pc_csv:
log.ODM_INFO(
"Creating geo-referenced CSV file (XYZ format)")
system.run(
"pdal translate -i \"{}\" "
"-o \"{}\" "
"--writers.text.format=csv "
"--writers.text.order=\"X,Y,Z\" "
"--writers.text.keep_unspecified=false ".format(
tree.odm_georeferencing_model_laz,
tree.odm_georeferencing_xyz_file))
if args.crop > 0:
log.ODM_INFO(
"Calculating cropping area and generating bounds shapefile from point cloud"
)
cropper = Cropper(tree.odm_georeferencing,
'odm_georeferenced_model')
decimation_step = 40 if args.fast_orthophoto or args.use_opensfm_dense else 90
# More aggressive decimation for large datasets
if not args.fast_orthophoto:
decimation_step *= int(
len(reconstruction.photos) / 1000) + 1
cropper.create_bounds_shapefile(
tree.odm_georeferencing_model_laz,
args.crop,
decimation_step=decimation_step,
outlier_radius=20 if args.fast_orthophoto else 2)
# Do not execute a second time, since
# We might be doing georeferencing for
# multiple models (3D, 2.5D, ...)
doPointCloudGeo = False
transformPointCloud = False
else:
log.ODM_WARNING('Found a valid georeferenced model in: %s' %
tree.odm_georeferencing_model_laz)
outputs.reconstruction = reconstruction
if args.time:
system.benchmark(start_time, tree.benchmarking, 'Georeferencing')
log.ODM_INFO('Running ODM Georeferencing Cell - Finished')
return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
def process(self, inputs, outputs):
# find a file in the root directory
def find(file, dir):
for root, dirs, files in os.walk(dir):
return '/'.join((root, file)) if file in files else None
# Benchmarking
start_time = system.now_raw()
log.ODM_INFO('Running ODM Georeferencing Cell')
# get inputs
args = self.inputs.args
tree = self.inputs.tree
gcpfile = io.join_paths(tree.root_path, self.params.gcp_file) \
if self.params.gcp_file else find('gcp_list.txt', tree.root_path)
doPointCloudGeo = True
verbose = '-verbose' if self.params.verbose else ''
# define paths and create working directories
system.mkdir_p(tree.odm_georeferencing)
if args.use_25dmesh: system.mkdir_p(tree.odm_25dgeoreferencing)
# in case a gcp file it's not provided, let's try to generate it using
# images metadata. Internally calls jhead.
log.ODM_DEBUG(self.params.gcp_file)
if not self.params.gcp_file: # and \
# not io.file_exists(tree.odm_georeferencing_coords):
log.ODM_WARNING('No coordinates file. '
'Generating coordinates file: %s' %
tree.odm_georeferencing_coords)
# odm_georeference definitions
kwargs = {
'bin': context.odm_modules_path,
'imgs': tree.dataset_raw,
'imgs_list': tree.opensfm_bundle_list,
'coords': tree.odm_georeferencing_coords,
'log': tree.odm_georeferencing_utm_log,
'verbose': verbose
}
# run UTM extraction binary
extract_utm = system.run_and_return(
'{bin}/odm_extract_utm -imagesPath {imgs}/ '
'-imageListFile {imgs_list} -outputCoordFile {coords} {verbose} '
'-logFile {log}'.format(**kwargs))
if extract_utm != '':
log.ODM_WARNING('Could not generate coordinates file. '
'Ignore if there is a GCP file. Error: %s' %
extract_utm)
# check if we rerun cell or not
rerun_cell = (args.rerun is not None and
args.rerun == 'odm_georeferencing') or \
(args.rerun_all) or \
(args.rerun_from is not None and
'odm_georeferencing' in args.rerun_from)
runs = [{
'georeferencing_dir':
tree.odm_georeferencing,
'texturing_dir':
tree.odm_texturing,
'model':
os.path.join(tree.odm_texturing, tree.odm_textured_model_obj)
}]
if args.fast_orthophoto:
runs = []
if args.use_25dmesh:
runs += [{
'georeferencing_dir':
tree.odm_25dgeoreferencing,
'texturing_dir':
tree.odm_25dtexturing,
'model':
os.path.join(tree.odm_25dtexturing,
tree.odm_textured_model_obj)
}]
for r in runs:
odm_georeferencing_model_obj_geo = os.path.join(
r['texturing_dir'], tree.odm_georeferencing_model_obj_geo)
odm_georeferencing_model_ply_geo = os.path.join(
r['georeferencing_dir'], tree.odm_georeferencing_model_ply_geo)
odm_georeferencing_log = os.path.join(r['georeferencing_dir'],
tree.odm_georeferencing_log)
odm_georeferencing_transform_file = os.path.join(
r['georeferencing_dir'],
tree.odm_georeferencing_transform_file)
if not io.file_exists(odm_georeferencing_model_obj_geo) or \
not io.file_exists(odm_georeferencing_model_ply_geo) or rerun_cell:
# odm_georeference definitions
kwargs = {
'bin':
context.odm_modules_path,
'bundle':
tree.opensfm_bundle,
'imgs':
tree.dataset_raw,
'imgs_list':
tree.opensfm_bundle_list,
'model':
r['model'],
'log':
odm_georeferencing_log,
'transform_file':
odm_georeferencing_transform_file,
'coords':
tree.odm_georeferencing_coords,
'pc_geo':
odm_georeferencing_model_ply_geo,
'geo_sys':
os.path.join(r['georeferencing_dir'],
tree.odm_georeferencing_model_txt_geo),
'model_geo':
odm_georeferencing_model_obj_geo,
'gcp':
gcpfile,
'verbose':
verbose
}
if not args.use_pmvs:
if args.fast_orthophoto:
kwargs['pc'] = os.path.join(tree.opensfm,
'reconstruction.ply')
else:
kwargs['pc'] = tree.opensfm_model
else:
kwargs['pc'] = tree.pmvs_model
# Check to see if the GCP file exists
if not self.params.use_exif and (self.params.gcp_file or find(
'gcp_list.txt', tree.root_path)):
log.ODM_INFO('Found %s' % gcpfile)
try:
system.run(
'{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} '
'-inputFile {model} -outputFile {model_geo} '
'-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys} -gcpFile {gcp} '
'-outputCoordFile {coords}'.format(**kwargs))
except Exception:
log.ODM_EXCEPTION('Georeferencing failed. ')
return ecto.QUIT
elif io.file_exists(tree.odm_georeferencing_coords):
log.ODM_INFO(
'Running georeferencing with generated coords file.')
system.run(
'{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} '
'-inputFile {model} -outputFile {model_geo} '
'-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'
.format(**kwargs))
else:
log.ODM_WARNING(
'Georeferencing failed. Make sure your '
'photos have geotags in the EXIF or you have '
'provided a GCP file. ')
doPointCloudGeo = False # skip the rest of the georeferencing
if doPointCloudGeo:
geo_ref = types.ODM_GeoRef()
geo_ref.parse_coordinate_system(
tree.odm_georeferencing_coords)
# convert ply model to LAS reference system
geo_ref.convert_to_las(odm_georeferencing_model_ply_geo,
tree.odm_georeferencing_model_las,
tree.odm_georeferencing_las_json)
# XYZ point cloud output
log.ODM_INFO(
"Creating geo-referenced CSV file (XYZ format)")
with open(tree.odm_georeferencing_xyz_file,
"wb") as csvfile:
csvfile_writer = csv.writer(csvfile, delimiter=",")
reachedpoints = False
with open(odm_georeferencing_model_ply_geo) as f:
for lineNumber, line in enumerate(f):
if reachedpoints:
tokens = line.split(" ")
csv_line = [
float(tokens[0]) +
geo_ref.utm_east_offset,
float(tokens[1]) +
geo_ref.utm_north_offset, tokens[2]
]
csvfile_writer.writerow(csv_line)
if line.startswith("end_header"):
reachedpoints = True
csvfile.close()
if args.crop > 0:
log.ODM_INFO(
"Calculating cropping area and generating bounds shapefile from point cloud"
)
cropper = Cropper(tree.odm_georeferencing,
'odm_georeferenced_model')
cropper.create_bounds_shapefile(
tree.odm_georeferencing_model_las, args.crop)
# Do not execute a second time, since
# We might be doing georeferencing for
# multiple models (3D, 2.5D, ...)
doPointCloudGeo = False
else:
log.ODM_WARNING('Found a valid georeferenced model in: %s' %
odm_georeferencing_model_ply_geo)
if args.time:
system.benchmark(start_time, tree.benchmarking, 'Georeferencing')
log.ODM_INFO('Running ODM Georeferencing Cell - Finished')
return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
def process(self, inputs, outputs):
# Benchmarking
start_time = system.now_raw()
log.ODM_INFO('Running ODM Orthophoto Cell')
# get inputs
args = self.inputs.args
tree = self.inputs.tree
reconstruction = inputs.reconstruction
verbose = '-verbose' if self.params.verbose else ''
# define paths and create working directories
system.mkdir_p(tree.odm_orthophoto)
# check if we rerun cell or not
rerun_cell = (args.rerun is not None and
args.rerun == 'odm_orthophoto') or \
(args.rerun_all) or \
(args.rerun_from is not None and
'odm_orthophoto' in args.rerun_from)
if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell:
# odm_orthophoto definitions
kwargs = {
'bin': context.odm_modules_path,
'log': tree.odm_orthophoto_log,
'ortho': tree.odm_orthophoto_file,
'corners': tree.odm_orthophoto_corners,
'res': 1.0 / (gsd.cap_resolution(self.params.resolution, tree.opensfm_reconstruction, ignore_gsd=args.ignore_gsd) / 100.0),
'verbose': verbose
}
# Have geo coordinates?
georef = reconstruction.georef
# Check if the georef object is initialized
# (during a --rerun this might not be)
# TODO: we should move this to a more central
# location (perhaps during the dataset initialization)
if georef and not georef.utm_east_offset:
georeferencing_dir = tree.odm_georeferencing if args.use_3dmesh and not args.skip_3dmodel else tree.odm_25dgeoreferencing
odm_georeferencing_model_txt_geo_file = os.path.join(georeferencing_dir, tree.odm_georeferencing_model_txt_geo)
if io.file_exists(odm_georeferencing_model_txt_geo_file):
georef.extract_offsets(odm_georeferencing_model_txt_geo_file)
else:
log.ODM_WARNING('Cannot read UTM offset from {}. An orthophoto will not be generated.'.format(odm_georeferencing_model_txt_geo_file))
if georef:
if args.use_3dmesh:
kwargs['model_geo'] = os.path.join(tree.odm_texturing, tree.odm_georeferencing_model_obj_geo)
else:
kwargs['model_geo'] = os.path.join(tree.odm_25dtexturing, tree.odm_georeferencing_model_obj_geo)
else:
if args.use_3dmesh:
kwargs['model_geo'] = os.path.join(tree.odm_texturing, tree.odm_textured_model_obj)
else:
kwargs['model_geo'] = os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj)
# run odm_orthophoto
system.run('{bin}/odm_orthophoto -inputFile {model_geo} '
'-logFile {log} -outputFile {ortho} -resolution {res} {verbose} '
'-outputCornerFile {corners}'.format(**kwargs))
# Create georeferenced GeoTiff
geotiffcreated = False
if georef and georef.projection and georef.utm_east_offset and georef.utm_north_offset:
ulx = uly = lrx = lry = 0.0
with open(tree.odm_orthophoto_corners) as f:
for lineNumber, line in enumerate(f):
if lineNumber == 0:
tokens = line.split(' ')
if len(tokens) == 4:
ulx = float(tokens[0]) + \
float(georef.utm_east_offset)
lry = float(tokens[1]) + \
float(georef.utm_north_offset)
lrx = float(tokens[2]) + \
float(georef.utm_east_offset)
uly = float(tokens[3]) + \
float(georef.utm_north_offset)
log.ODM_INFO('Creating GeoTIFF')
kwargs = {
'ulx': ulx,
'uly': uly,
'lrx': lrx,
'lry': lry,
'tiled': '' if self.params.no_tiled else '-co TILED=yes ',
'compress': self.params.compress,
'predictor': '-co PREDICTOR=2 ' if self.params.compress in
['LZW', 'DEFLATE'] else '',
'proj': georef.projection.srs,
'bigtiff': self.params.bigtiff,
'png': tree.odm_orthophoto_file,
'tiff': tree.odm_orthophoto_tif,
'log': tree.odm_orthophoto_tif_log,
'max_memory': get_max_memory(),
'threads': self.params.max_concurrency
}
system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
'{tiled} '
'-co BIGTIFF={bigtiff} '
'-co COMPRESS={compress} '
'{predictor} '
'-co BLOCKXSIZE=512 '
'-co BLOCKYSIZE=512 '
'-co NUM_THREADS={threads} '
'-a_srs \"{proj}\" '
'--config GDAL_CACHEMAX {max_memory}% '
'{png} {tiff} > {log}'.format(**kwargs))
if args.crop > 0:
shapefile_path = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.shp')
Cropper.crop(shapefile_path, tree.odm_orthophoto_tif, {
'TILED': 'NO' if self.params.no_tiled else 'YES',
'COMPRESS': self.params.compress,
'PREDICTOR': '2' if self.params.compress in ['LZW', 'DEFLATE'] else '1',
'BIGTIFF': self.params.bigtiff,
'BLOCKXSIZE': 512,
'BLOCKYSIZE': 512,
'NUM_THREADS': self.params.max_concurrency
})
if self.params.build_overviews:
log.ODM_DEBUG("Building Overviews")
kwargs = {
'orthophoto': tree.odm_orthophoto_tif,
'log': tree.odm_orthophoto_gdaladdo_log
}
# Run gdaladdo
system.run('gdaladdo -ro -r average '
'--config BIGTIFF_OVERVIEW IF_SAFER '
'--config COMPRESS_OVERVIEW JPEG '
'{orthophoto} 2 4 8 16 > {log}'.format(**kwargs))
geotiffcreated = True
if not geotiffcreated:
log.ODM_WARNING('No geo-referenced orthophoto created due '
'to missing geo-referencing or corner coordinates.')
else:
log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file)
if args.time:
system.benchmark(start_time, tree.benchmarking, 'Orthophoto')
log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished')
return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT
def process(args, tree, reconstruction, current_path):
odm_orthophoto = io.join_paths(current_path, 'orthophoto')
odm_orthophoto_path = odm_orthophoto
odm_orthophoto_render = io.join_paths(odm_orthophoto_path, 'odm_orthophoto_render.tif')
odm_orthophoto_tif = io.join_paths(odm_orthophoto_path, 'odm_orthophoto.tif')
odm_orthophoto_corners = io.join_paths(odm_orthophoto_path, 'odm_orthophoto_corners.tif')
odm_orthophoto_log = io.join_paths(odm_orthophoto_path, 'odm_orthophoto_log.tif')
odm_orthophoto_tif_log = io.join_paths(odm_orthophoto_path, 'gdal_translate_log.txt')
odm_25dgeoreferencing = io.join_paths(current_path, 'odm_georeferencing')
odm_georeferencing = io.join_paths(current_path, 'odm_georeferencing')
odm_georeferencing_coords = io.join_paths(
odm_georeferencing, 'coords.txt')
odm_georeferencing_gcp = io.find('gcp_list.txt', current_path)
odm_georeferencing_gcp_utm = io.join_paths(odm_georeferencing, 'gcp_list_utm.txt')
odm_georeferencing_utm_log = io.join_paths(
odm_georeferencing, 'odm_georeferencing_utm_log.txt')
odm_georeferencing_log = 'odm_georeferencing_log.txt'
odm_georeferencing_transform_file = 'odm_georeferencing_transform.txt'
odm_georeferencing_proj = 'proj.txt'
odm_georeferencing_model_txt_geo = 'odm_georeferencing_model_geo.txt'
odm_georeferencing_model_obj_geo = 'odm_textured_model_geo.obj'
odm_georeferencing_xyz_file = io.join_paths(
odm_georeferencing, 'odm_georeferenced_model.csv')
odm_georeferencing_las_json = io.join_paths(
odm_georeferencing, 'las.json')
odm_georeferencing_model_laz = io.join_paths(
odm_georeferencing, 'odm_georeferenced_model.laz')
odm_georeferencing_model_las = io.join_paths(
odm_georeferencing, 'odm_georeferenced_model.las')
odm_georeferencing_dem = io.join_paths(
odm_georeferencing, 'odm_georeferencing_model_dem.tif')
opensfm_reconstruction = io.join_paths(current_path,'reconstruction.json')
odm_texturing = io.join_paths(current_path,'mvs')
odm_textured_model_obj = io.join_paths(odm_texturing, 'odm_textured_model.obj')
images_dir = io.join_paths(current_path, 'images')
tree = tree
opensfm_bundle = os.path.join(current_path, 'opensfm', 'bundle_r000.out')
opensfm_bundle_list = os.path.join(current_path, 'opensfm', 'list_r000.out')
opensfm_transformation = os.path.join(current_path, 'opensfm', 'geocoords_transformation.txt')
filtered_point_cloud = os.path.join(current_path, 'filterpoints', 'point_cloud.ply')
doPointCloudGeo = True
transformPointCloud = True
verbose =''
class nonloc:
runs = []
def add_run(primary=True, band=None):
subdir = ""
if not primary and band is not None:
subdir = band
# Make sure 2.5D mesh is georeferenced before the 3D mesh
# Because it will be used to calculate a transform
# for the point cloud. If we use the 3D model transform,
# DEMs and orthophoto might not align!
b = True
if b:
nonloc.runs += [{
'georeferencing_dir': os.path.join(odm_georeferencing, subdir),
'texturing_dir': os.path.join(odm_texturing, subdir),
}]
if not args.skip_3dmodel and (primary or args.use_3dmesh):
nonloc.runs += [{
'georeferencing_dir': odm_georeferencing,
'texturing_dir': os.path.join(odm_texturing, subdir),
}]
if reconstruction.multi_camera:
for band in reconstruction.multi_camera:
primary = band == reconstruction.multi_camera[0]
add_run(primary, band['name'].lower())
else:
add_run()
#progress_per_run = 100.0 / len(nonloc.runs)
#progress = 0.0
for r in nonloc.runs:
if not io.dir_exists(r['georeferencing_dir']):
system.mkdir_p(r['georeferencing_dir'])
odm_georeferencing_model_obj_geo = os.path.join(r['texturing_dir'], odm_georeferencing_model_obj_geo)
odm_georeferencing_model_obj = os.path.join(r['texturing_dir'], odm_textured_model_obj)
odm_georeferencing_log = os.path.join(r['georeferencing_dir'], odm_georeferencing_log)
odm_georeferencing_transform_file = os.path.join(r['georeferencing_dir'], odm_georeferencing_transform_file)
odm_georeferencing_model_txt_geo_file = os.path.join(r['georeferencing_dir'], odm_georeferencing_model_txt_geo)
pio = True
if pio:
#if not io.file_exists(odm_georeferencing_model_obj_geo) or \
#not io.file_exists(odm_georeferencing_model_laz)
# odm_georeference definitions
kwargs = {
'bin': context.odm_modules_path,
'input_pc_file': filtered_point_cloud,
'bundle': opensfm_bundle,
'imgs': images_dir,
'imgs_list': opensfm_bundle_list,
'model': odm_georeferencing_model_obj,
'log': odm_georeferencing_log,
'input_trans_file': opensfm_transformation,
'transform_file': odm_georeferencing_transform_file,
'coords': odm_georeferencing_coords,
'output_pc_file': odm_georeferencing_model_laz,
'geo_sys': odm_georeferencing_model_txt_geo_file,
'model_geo': odm_georeferencing_model_obj_geo,
'verbose': verbose
}
if transformPointCloud:
kwargs['pc_params'] = '-inputPointCloudFile {input_pc_file} -outputPointCloudFile {output_pc_file}'.format(**kwargs)
if reconstruction.is_georeferenced():
kwargs['pc_params'] += ' -outputPointCloudSrs %s' % pipes.quote(reconstruction.georef.proj4())
else:
log.ODM_WARNING('NO SRS: The output point cloud will not have a SRS.')
else:
kwargs['pc_params'] = ''
if io.file_exists(opensfm_transformation) and io.file_exists(odm_georeferencing_coords):
log.ODM_INFO('Running georeferencing with OpenSfM transformation matrix')
system.run('{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs))
elif io.file_exists(odm_georeferencing_coords):
print('running georeferencing')
log.ODM_INFO('Running georeferencing with generated coords file.')
system.run('{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs))
else:
log.ODM_WARNING('Georeferencing failed. Make sure your '
'photos have geotags in the EXIF or you have '
'provided a GCP file. ')
doPointCloudGeo = False # skip the rest of the georeferencing
if doPointCloudGeo:
reconstruction.georef.extract_offsets(odm_georeferencing_model_txt_geo_file)
point_cloud.post_point_cloud_steps(args, tree)
if args.crop > 0:
log.ODM_INFO("Calculating cropping area and generating bounds shapefile from point cloud")
cropper = Cropper(odm_georeferencing, 'odm_georeferenced_model')
decimation_step = 40 if args.fast_orthophoto or args.use_opensfm_dense else 90
# More aggressive decimation for large datasets
if not args.fast_orthophoto:
decimation_step *= int(len(reconstruction.photos) / 1000) + 1
cropper.create_bounds_gpkg(odm_georeferencing_model_laz, args.crop,
decimation_step=decimation_step)
# Do not execute a second time, since
# We might be doing georeferencing for
# multiple models (3D, 2.5D, ...)
doPointCloudGeo = False
transformPointCloud = False
else:
log.ODM_WARNING('Found a valid georeferenced model in: %s'
% odm_georeferencing_model_laz)
if args.optimize_disk_space and io.file_exists(odm_georeferencing_model_laz) and io.file_exists(filtered_point_cloud):
os.remove(filtered_point_cloud)
def process(self, inputs, outputs):
# Benchmarking
start_time = system.now_raw()
log.ODM_INFO('Running ODM Georeferencing Cell')
# get inputs
args = inputs.args
tree = inputs.tree
reconstruction = inputs.reconstruction
gcpfile = tree.odm_georeferencing_gcp
doPointCloudGeo = True
verbose = '-verbose' if self.params.verbose else ''
# check if we rerun cell or not
rerun_cell = (args.rerun is not None and
args.rerun == 'odm_georeferencing') or \
(args.rerun_all) or \
(args.rerun_from is not None and
'odm_georeferencing' in args.rerun_from)
runs = [{
'georeferencing_dir': tree.odm_georeferencing,
'texturing_dir': tree.odm_texturing,
'model': os.path.join(tree.odm_texturing, tree.odm_textured_model_obj)
}]
if args.fast_orthophoto:
runs = []
if args.use_25dmesh:
runs += [{
'georeferencing_dir': tree.odm_25dgeoreferencing,
'texturing_dir': tree.odm_25dtexturing,
'model': os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj)
}]
for r in runs:
odm_georeferencing_model_obj_geo = os.path.join(r['texturing_dir'], tree.odm_georeferencing_model_obj_geo)
odm_georeferencing_model_ply_geo = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_model_ply_geo)
odm_georeferencing_log = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_log)
odm_georeferencing_transform_file = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_transform_file)
if not io.file_exists(odm_georeferencing_model_obj_geo) or \
not io.file_exists(odm_georeferencing_model_ply_geo) or rerun_cell:
# odm_georeference definitions
kwargs = {
'bin': context.odm_modules_path,
'bundle': tree.opensfm_bundle,
'imgs': tree.dataset_raw,
'imgs_list': tree.opensfm_bundle_list,
'model': r['model'],
'log': odm_georeferencing_log,
'input_trans_file': tree.opensfm_transformation,
'transform_file': odm_georeferencing_transform_file,
'coords': tree.odm_georeferencing_coords,
'pc_geo': odm_georeferencing_model_ply_geo,
'geo_sys': os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo),
'model_geo': odm_georeferencing_model_obj_geo,
'gcp': gcpfile,
'verbose': verbose
}
if not args.use_pmvs:
if args.fast_orthophoto:
kwargs['pc'] = os.path.join(tree.opensfm, 'reconstruction.ply')
else:
kwargs['pc'] = tree.opensfm_model
else:
kwargs['pc'] = tree.pmvs_model
# Check to see if the GCP file exists
#if not self.params.use_exif and (self.params.gcp_file or tree.odm_georeferencing_gcp):
# log.ODM_INFO('Found %s' % gcpfile)
# try:
# system.run('{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} '
# '-inputFile {model} -outputFile {model_geo} '
# '-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} '
# '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys} -gcpFile {gcp} '
# '-outputCoordFile {coords}'.format(**kwargs))
# except Exception:
# log.ODM_EXCEPTION('Georeferencing failed. ')
# return ecto.QUIT
if io.file_exists(tree.opensfm_transformation):
log.ODM_INFO('Running georeferencing with OpenSfM transformation matrix')
system.run('{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} '
'-inputFile {model} -outputFile {model_geo} '
'-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs))
elif io.file_exists(tree.odm_georeferencing_coords):
log.ODM_INFO('Running georeferencing with generated coords file.')
system.run('{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} '
'-inputFile {model} -outputFile {model_geo} '
'-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs))
else:
log.ODM_WARNING('Georeferencing failed. Make sure your '
'photos have geotags in the EXIF or you have '
'provided a GCP file. ')
doPointCloudGeo = False # skip the rest of the georeferencing
odm_georeferencing_model_ply_geo = os.path.join(tree.odm_georeferencing,
tree.odm_georeferencing_model_ply_geo)
if doPointCloudGeo:
# update images metadata
geo_ref = reconstruction.georef
geo_ref.parse_transformation_matrix(tree.opensfm_transformation)
# convert ply model to LAS reference system
geo_ref.convert_to_las(odm_georeferencing_model_ply_geo,
tree.odm_georeferencing_model_las,
tree.odm_georeferencing_las_json)
reconstruction.georef = geo_ref
# XYZ point cloud output
log.ODM_INFO("Creating geo-referenced CSV file (XYZ format)")
with open(tree.odm_georeferencing_xyz_file, "wb") as csvfile:
csvfile_writer = csv.writer(csvfile, delimiter=",")
reachedpoints = False
with open(odm_georeferencing_model_ply_geo) as f:
for lineNumber, line in enumerate(f):
if reachedpoints:
tokens = line.split(" ")
csv_line = [float(tokens[0]),
float(tokens[1]),
tokens[2]]
csvfile_writer.writerow(csv_line)
if line.startswith("end_header"):
reachedpoints = True
csvfile.close()
if args.crop > 0:
log.ODM_INFO("Calculating cropping area and generating bounds shapefile from point cloud")
cropper = Cropper(tree.odm_georeferencing, 'odm_georeferenced_model')
cropper.create_bounds_shapefile(tree.odm_georeferencing_model_las, args.crop)
# Do not execute a second time, since
# We might be doing georeferencing for
# multiple models (3D, 2.5D, ...)
doPointCloudGeo = False
else:
log.ODM_WARNING('Found a valid georeferenced model in: %s'
% odm_georeferencing_model_ply_geo)
outputs.reconstruction = reconstruction
if args.time:
system.benchmark(start_time, tree.benchmarking, 'Georeferencing')
log.ODM_INFO('Running ODM Georeferencing Cell - Finished')
return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
def process(self, inputs, outputs):
# Benchmarking
start_time = system.now_raw()
log.ODM_INFO('Running ODM DEM Cell')
# get inputs
args = self.inputs.args
tree = self.inputs.tree
las_model_found = io.file_exists(tree.odm_georeferencing_model_laz)
# check if we rerun cell or not
rerun_cell = (args.rerun is not None and
args.rerun == 'odm_dem') or \
(args.rerun_all) or \
(args.rerun_from is not None and
'odm_dem' in args.rerun_from)
log.ODM_INFO('Classify: ' + str(args.pc_classify))
log.ODM_INFO('Create DSM: ' + str(args.dsm))
log.ODM_INFO('Create DTM: ' + str(args.dtm))
log.ODM_INFO('DEM input file {0} found: {1}'.format(
tree.odm_georeferencing_model_laz, str(las_model_found)))
slope, cellsize = (0.15, 1)
# define paths and create working directories
odm_dem_root = tree.path('odm_dem')
if not io.dir_exists(odm_dem_root):
system.mkdir_p(odm_dem_root)
if args.pc_classify and las_model_found:
pc_classify_marker = os.path.join(odm_dem_root,
'pc_classify_done.txt')
if not io.file_exists(pc_classify_marker) or rerun_cell:
log.ODM_INFO(
"Classifying {} using Simple Morphological Filter".format(
tree.odm_georeferencing_model_laz))
commands.classify(tree.odm_georeferencing_model_laz,
slope,
cellsize,
verbose=args.verbose)
with open(pc_classify_marker, 'w') as f:
f.write('Classify: smrf\n')
f.write('Slope: {}\n'.format(slope))
f.write('Cellsize: {}\n'.format(cellsize))
# Do we need to process anything here?
if (args.dsm or args.dtm) and las_model_found:
dsm_output_filename = os.path.join(odm_dem_root, 'dsm.tif')
dtm_output_filename = os.path.join(odm_dem_root, 'dtm.tif')
if (args.dtm and not io.file_exists(dtm_output_filename)) or \
(args.dsm and not io.file_exists(dsm_output_filename)) or \
rerun_cell:
products = []
if args.dsm: products.append('dsm')
if args.dtm: products.append('dtm')
resolution = gsd.cap_resolution(args.dem_resolution,
tree.opensfm_reconstruction,
gsd_error_estimate=-3,
ignore_gsd=args.ignore_gsd)
radius_steps = [(resolution / 100.0) / 2.0]
for _ in range(args.dem_gapfill_steps - 1):
radius_steps.append(
radius_steps[-1] *
2) # 2 is arbitrary, maybe there's a better value?
for product in products:
commands.create_dems(
[tree.odm_georeferencing_model_laz],
product,
radius=map(str, radius_steps),
gapfill=True,
outdir=odm_dem_root,
resolution=resolution / 100.0,
decimation=args.dem_decimation,
verbose=args.verbose,
max_workers=get_max_concurrency_for_dem(
args.max_concurrency,
tree.odm_georeferencing_model_laz))
if args.crop > 0:
bounds_shapefile_path = os.path.join(
tree.odm_georeferencing,
'odm_georeferenced_model.bounds.shp')
if os.path.exists(bounds_shapefile_path):
Cropper.crop(
bounds_shapefile_path,
os.path.join(odm_dem_root,
"{}.tif".format(product)),
{
'TILED': 'YES',
'COMPRESS': 'LZW',
'BLOCKXSIZE': 512,
'BLOCKYSIZE': 512,
'NUM_THREADS': self.params.max_concurrency
})
else:
log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root)
else:
log.ODM_WARNING('DEM will not be generated')
if args.time:
system.benchmark(start_time, tree.benchmarking, 'Dem')
log.ODM_INFO('Running ODM DEM Cell - Finished')
return ecto.OK if args.end_with != 'odm_dem' else ecto.QUIT
def process(self, args, outputs):
tree = outputs['tree']
reconstruction = outputs['reconstruction']
if not io.file_exists(
tree.odm_georeferencing_model_laz) or self.rerun():
cmd = (
'pdal translate -i "%s" -o \"%s\"' %
(tree.filtered_point_cloud, tree.odm_georeferencing_model_laz))
stages = ["ferry"]
params = [
'--filters.ferry.dimensions="views => UserData"',
'--writers.las.compression="lazip"',
]
if reconstruction.is_georeferenced():
log.ODM_INFO("Georeferencing point cloud")
stages.append("transformation")
params += [
'--filters.transformation.matrix="1 0 0 %s 0 1 0 %s 0 0 1 0 0 0 0 1"'
% reconstruction.georef.utm_offset(),
'--writers.las.offset_x=%s' %
reconstruction.georef.utm_east_offset,
'--writers.las.offset_y=%s' %
reconstruction.georef.utm_north_offset,
'--writers.las.offset_z=0',
'--writers.las.a_srs="%s"' % reconstruction.georef.proj4()
]
system.run(cmd + ' ' + ' '.join(stages) + ' ' +
' '.join(params))
self.update_progress(50)
if args.crop > 0:
log.ODM_INFO(
"Calculating cropping area and generating bounds shapefile from point cloud"
)
cropper = Cropper(tree.odm_georeferencing,
'odm_georeferenced_model')
if args.fast_orthophoto:
decimation_step = 10
else:
decimation_step = 40
# More aggressive decimation for large datasets
if not args.fast_orthophoto:
decimation_step *= int(
len(reconstruction.photos) / 1000) + 1
decimation_step = min(decimation_step, 95)
try:
cropper.create_bounds_gpkg(
tree.odm_georeferencing_model_laz,
args.crop,
decimation_step=decimation_step)
except:
log.ODM_WARNING(
"Cannot calculate crop bounds! We will skip cropping"
)
args.crop = 0
else:
log.ODM_INFO("Converting point cloud (non-georeferenced)")
system.run(cmd + ' ' + ' '.join(stages) + ' ' +
' '.join(params))
point_cloud.post_point_cloud_steps(args, tree)
else:
log.ODM_WARNING('Found a valid georeferenced model in: %s' %
tree.odm_georeferencing_model_laz)
if args.optimize_disk_space and io.file_exists(
tree.odm_georeferencing_model_laz) and io.file_exists(
tree.filtered_point_cloud):
os.remove(tree.filtered_point_cloud)
def process(self, inputs, outputs):
# Benchmarking
start_time = system.now_raw()
log.ODM_INFO('Running ODM Orthophoto Cell')
# get inputs
args = self.inputs.args
tree = self.inputs.tree
verbose = '-verbose' if self.params.verbose else ''
# define paths and create working directories
system.mkdir_p(tree.odm_orthophoto)
# check if we rerun cell or not
rerun_cell = (args.rerun is not None and
args.rerun == 'odm_orthophoto') or \
(args.rerun_all) or \
(args.rerun_from is not None and
'odm_orthophoto' in args.rerun_from)
if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell:
# odm_orthophoto definitions
kwargs = {
'bin': context.odm_modules_path,
'log': tree.odm_orthophoto_log,
'ortho': tree.odm_orthophoto_file,
'corners': tree.odm_orthophoto_corners,
'res': self.params.resolution,
'verbose': verbose
}
# Have geo coordinates?
if io.file_exists(tree.odm_georeferencing_coords):
if args.use_25dmesh:
kwargs['model_geo'] = os.path.join(
tree.odm_25dtexturing,
tree.odm_georeferencing_model_obj_geo)
else:
kwargs['model_geo'] = os.path.join(
tree.odm_texturing,
tree.odm_georeferencing_model_obj_geo)
else:
if args.use_25dmesh:
kwargs['model_geo'] = os.path.join(
tree.odm_25dtexturing, tree.odm_textured_model_obj)
else:
kwargs['model_geo'] = os.path.join(
tree.odm_texturing, tree.odm_textured_model_obj)
# run odm_orthophoto
system.run(
'{bin}/odm_orthophoto -inputFile {model_geo} '
'-logFile {log} -outputFile {ortho} -resolution {res} {verbose} '
'-outputCornerFile {corners}'.format(**kwargs))
if not io.file_exists(tree.odm_georeferencing_coords):
log.ODM_WARNING('No coordinates file. A georeferenced raster '
'will not be created')
else:
# Create georeferenced GeoTiff
geotiffcreated = False
georef = types.ODM_GeoRef()
# creates the coord refs # TODO I don't want to have to do this twice- after odm_georef
georef.parse_coordinate_system(tree.odm_georeferencing_coords)
if georef.epsg and georef.utm_east_offset and georef.utm_north_offset:
ulx = uly = lrx = lry = 0.0
with open(tree.odm_orthophoto_corners) as f:
for lineNumber, line in enumerate(f):
if lineNumber == 0:
tokens = line.split(' ')
if len(tokens) == 4:
ulx = float(tokens[0]) + \
float(georef.utm_east_offset)
lry = float(tokens[1]) + \
float(georef.utm_north_offset)
lrx = float(tokens[2]) + \
float(georef.utm_east_offset)
uly = float(tokens[3]) + \
float(georef.utm_north_offset)
log.ODM_INFO('Creating GeoTIFF')
kwargs = {
'ulx':
ulx,
'uly':
uly,
'lrx':
lrx,
'lry':
lry,
'tiled':
'' if self.params.no_tiled else '-co TILED=yes ',
'compress':
self.params.compress,
'predictor':
'-co PREDICTOR=2 '
if self.params.compress in ['LZW', 'DEFLATE'] else '',
'epsg':
georef.epsg,
't_srs':
self.params.t_srs or "EPSG:{0}".format(georef.epsg),
'bigtiff':
self.params.bigtiff,
'png':
tree.odm_orthophoto_file,
'tiff':
tree.odm_orthophoto_tif,
'log':
tree.odm_orthophoto_tif_log
}
system.run(
'gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
'{tiled} '
'-co BIGTIFF={bigtiff} '
'-co COMPRESS={compress} '
'{predictor} '
'-co BLOCKXSIZE=512 '
'-co BLOCKYSIZE=512 '
'-co NUM_THREADS=ALL_CPUS '
'-a_srs \"EPSG:{epsg}\" '
'{png} {tiff} > {log}'.format(**kwargs))
if args.crop > 0:
shapefile_path = os.path.join(
tree.odm_georeferencing,
'odm_georeferenced_model.bounds.shp')
Cropper.crop(
shapefile_path, tree.odm_orthophoto_tif, {
'TILED':
'NO' if self.params.no_tiled else 'YES',
'COMPRESS':
self.params.compress,
'PREDICTOR':
'2' if self.params.compress
in ['LZW', 'DEFLATE'] else '1',
'BIGTIFF':
self.params.bigtiff,
'BLOCKXSIZE':
512,
'BLOCKYSIZE':
512,
'NUM_THREADS':
'ALL_CPUS'
})
if self.params.build_overviews:
log.ODM_DEBUG("Building Overviews")
kwargs = {
'orthophoto': tree.odm_orthophoto_tif,
'log': tree.odm_orthophoto_gdaladdo_log
}
# Run gdaladdo
system.run(
'gdaladdo -ro -r average '
'--config BIGTIFF_OVERVIEW IF_SAFER '
'--config COMPRESS_OVERVIEW JPEG '
'{orthophoto} 2 4 8 16 > {log}'.format(**kwargs))
geotiffcreated = True
if not geotiffcreated:
log.ODM_WARNING(
'No geo-referenced orthophoto created due '
'to missing geo-referencing or corner coordinates.')
else:
log.ODM_WARNING('Found a valid orthophoto in: %s' %
tree.odm_orthophoto_file)
if args.time:
system.benchmark(start_time, tree.benchmarking, 'Orthophoto')
log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished')
return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT
def process(self, inputs, outputs):
# Benchmarking
start_time = system.now_raw()
log.ODM_INFO('Running ODM Georeferencing Cell')
# get inputs
args = inputs.args
tree = inputs.tree
reconstruction = inputs.reconstruction
gcpfile = tree.odm_georeferencing_gcp
doPointCloudGeo = True
transformPointCloud = True
verbose = '-verbose' if self.params.verbose else ''
geo_ref = reconstruction.georef
# check if we rerun cell or not
rerun_cell = (args.rerun is not None and
args.rerun == 'odm_georeferencing') or \
(args.rerun_all) or \
(args.rerun_from is not None and
'odm_georeferencing' in args.rerun_from)
runs = [{
'georeferencing_dir': tree.odm_georeferencing,
'texturing_dir': tree.odm_texturing,
'model': os.path.join(tree.odm_texturing, tree.odm_textured_model_obj)
}]
if args.skip_3dmodel:
runs = []
if not args.use_3dmesh:
# Make sure 2.5D mesh is georeferenced before the 3D mesh
# Because it will be used to calculate a transform
# for the point cloud. If we use the 3D model transform,
# DEMs and orthophoto might not align!
runs.insert(0, {
'georeferencing_dir': tree.odm_25dgeoreferencing,
'texturing_dir': tree.odm_25dtexturing,
'model': os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj)
})
for r in runs:
odm_georeferencing_model_obj_geo = os.path.join(r['texturing_dir'], tree.odm_georeferencing_model_obj_geo)
odm_georeferencing_log = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_log)
odm_georeferencing_transform_file = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_transform_file)
odm_georeferencing_model_txt_geo_file = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo)
if not io.file_exists(odm_georeferencing_model_obj_geo) or \
not io.file_exists(tree.odm_georeferencing_model_laz) or rerun_cell:
# odm_georeference definitions
kwargs = {
'bin': context.odm_modules_path,
'input_pc_file': tree.filtered_point_cloud,
'bundle': tree.opensfm_bundle,
'imgs': tree.dataset_raw,
'imgs_list': tree.opensfm_bundle_list,
'model': r['model'],
'log': odm_georeferencing_log,
'input_trans_file': tree.opensfm_transformation,
'transform_file': odm_georeferencing_transform_file,
'coords': tree.odm_georeferencing_coords,
'output_pc_file': tree.odm_georeferencing_model_laz,
'geo_sys': odm_georeferencing_model_txt_geo_file,
'model_geo': odm_georeferencing_model_obj_geo,
'gcp': gcpfile,
'verbose': verbose
}
if transformPointCloud:
kwargs['pc_params'] = '-inputPointCloudFile {input_pc_file} -outputPointCloudFile {output_pc_file}'.format(**kwargs)
if geo_ref and geo_ref.projection and geo_ref.projection.srs:
kwargs['pc_params'] += ' -outputPointCloudSrs %s' % pipes.quote(geo_ref.projection.srs)
else:
log.ODM_WARNING('NO SRS: The output point cloud will not have a SRS.')
else:
kwargs['pc_params'] = ''
# Check to see if the GCP file exists
if not self.params.use_exif and (self.params.gcp_file or tree.odm_georeferencing_gcp):
log.ODM_INFO('Found %s' % gcpfile)
try:
system.run('{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys} -gcpFile {gcp} '
'-outputCoordFile {coords}'.format(**kwargs))
except Exception:
log.ODM_EXCEPTION('Georeferencing failed. ')
return ecto.QUIT
elif io.file_exists(tree.opensfm_transformation) and io.file_exists(tree.odm_georeferencing_coords):
log.ODM_INFO('Running georeferencing with OpenSfM transformation matrix')
system.run('{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs))
elif io.file_exists(tree.odm_georeferencing_coords):
log.ODM_INFO('Running georeferencing with generated coords file.')
system.run('{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} '
'-inputFile {model} -outputFile {model_geo} '
'{pc_params} {verbose} '
'-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs))
else:
log.ODM_WARNING('Georeferencing failed. Make sure your '
'photos have geotags in the EXIF or you have '
'provided a GCP file. ')
doPointCloudGeo = False # skip the rest of the georeferencing
if doPointCloudGeo:
# update images metadata
geo_ref.extract_offsets(odm_georeferencing_model_txt_geo_file)
reconstruction.georef = geo_ref
# XYZ point cloud output
if args.pc_csv:
log.ODM_INFO("Creating geo-referenced CSV file (XYZ format)")
system.run("pdal translate -i \"{}\" "
"-o \"{}\" "
"--writers.text.format=csv "
"--writers.text.order=\"X,Y,Z\" "
"--writers.text.keep_unspecified=false ".format(
tree.odm_georeferencing_model_laz,
tree.odm_georeferencing_xyz_file))
# LAS point cloud output
if args.pc_las:
log.ODM_INFO("Creating geo-referenced LAS file")
system.run("pdal translate -i \"{}\" "
"-o \"{}\" ".format(
tree.odm_georeferencing_model_laz,
tree.odm_georeferencing_model_las))
if args.crop > 0:
log.ODM_INFO("Calculating cropping area and generating bounds shapefile from point cloud")
cropper = Cropper(tree.odm_georeferencing, 'odm_georeferenced_model')
decimation_step = 40 if args.fast_orthophoto or args.use_opensfm_dense else 90
# More aggressive decimation for large datasets
if not args.fast_orthophoto:
decimation_step *= int(len(reconstruction.photos) / 1000) + 1
cropper.create_bounds_shapefile(tree.odm_georeferencing_model_laz, args.crop,
decimation_step=decimation_step,
outlier_radius=20 if args.fast_orthophoto else 2)
# Do not execute a second time, since
# We might be doing georeferencing for
# multiple models (3D, 2.5D, ...)
doPointCloudGeo = False
transformPointCloud = False
else:
log.ODM_WARNING('Found a valid georeferenced model in: %s'
% tree.odm_georeferencing_model_laz)
outputs.reconstruction = reconstruction
if args.time:
system.benchmark(start_time, tree.benchmarking, 'Georeferencing')
log.ODM_INFO('Running ODM Georeferencing Cell - Finished')
return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
