def process(self, args, outputs):
tree = outputs['tree']
if outputs['large']:
if not os.path.exists(tree.submodels_path):
log.ODM_ERROR(
"We reached the merge stage, but %s folder does not exist. Something must have gone wrong at an earlier stage. Check the log and fix possible problem before restarting?" % tree.submodels_path)
exit(1)
# Merge point clouds
if args.merge in ['all', 'pointcloud']:
if not io.file_exists(tree.odm_georeferencing_model_laz) or self.rerun():
all_point_clouds = get_submodel_paths(tree.submodels_path, "odm_georeferencing",
"odm_georeferenced_model.laz")
try:
point_cloud.merge(all_point_clouds, tree.odm_georeferencing_model_laz)
point_cloud.post_point_cloud_steps(args, tree)
except Exception as e:
log.ODM_WARNING("Could not merge point cloud: %s (skipping)" % str(e))
else:
log.ODM_WARNING("Found merged point cloud in %s" % tree.odm_georeferencing_model_laz)
self.update_progress(25)
# Merge crop bounds
merged_bounds_file = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg')
if not io.file_exists(merged_bounds_file) or self.rerun():
all_bounds = get_submodel_paths(tree.submodels_path, 'odm_georeferencing',
'odm_georeferenced_model.bounds.gpkg')
log.ODM_INFO("Merging all crop bounds: %s" % all_bounds)
if len(all_bounds) > 0:
# Calculate a new crop area
# based on the convex hull of all crop areas of all submodels
# (without a buffer, otherwise we are double-cropping)
Cropper.merge_bounds(all_bounds, merged_bounds_file, 0)
else:
log.ODM_WARNING("No bounds found for any submodel.")
# Merge orthophotos
if args.merge in ['all', 'orthophoto']:
if not io.dir_exists(tree.odm_orthophoto):
system.mkdir_p(tree.odm_orthophoto)
if not io.file_exists(tree.odm_orthophoto_tif) or self.rerun():
all_orthos_and_ortho_cuts = get_all_submodel_paths(tree.submodels_path,
os.path.join("odm_orthophoto",
"odm_orthophoto_feathered.tif"),
os.path.join("odm_orthophoto",
"odm_orthophoto_cut.tif"),
)
if len(all_orthos_and_ortho_cuts) > 1:
log.ODM_INFO(
"Found %s submodels with valid orthophotos and cutlines" % len(all_orthos_and_ortho_cuts))
# TODO: histogram matching via rasterio
# currently parts have different color tones
if io.file_exists(tree.odm_orthophoto_tif):
os.remove(tree.odm_orthophoto_tif)
orthophoto_vars = orthophoto.get_orthophoto_vars(args)
orthophoto.merge(all_orthos_and_ortho_cuts, tree.odm_orthophoto_tif, orthophoto_vars)
orthophoto.post_orthophoto_steps(args, merged_bounds_file, tree.odm_orthophoto_tif,
tree.orthophoto_tiles)
elif len(all_orthos_and_ortho_cuts) == 1:
# Simply copy
log.ODM_WARNING("A single orthophoto/cutline pair was found between all submodels.")
shutil.copyfile(all_orthos_and_ortho_cuts[0][0], tree.odm_orthophoto_tif)
else:
log.ODM_WARNING(
"No orthophoto/cutline pairs were found in any of the submodels. No orthophoto will be generated.")
else:
log.ODM_WARNING("Found merged orthophoto in %s" % tree.odm_orthophoto_tif)
self.update_progress(75)
# Merge DEMs
def merge_dems(dem_filename, human_name):
if not io.dir_exists(tree.path('odm_dem')):
system.mkdir_p(tree.path('odm_dem'))
dem_file = tree.path("odm_dem", dem_filename)
if not io.file_exists(dem_file) or self.rerun():
all_dems = get_submodel_paths(tree.submodels_path, "odm_dem", dem_filename)
log.ODM_INFO("Merging %ss" % human_name)
# Merge
dem_vars = utils.get_dem_vars(args)
eu_map_source = None # Default
# Use DSM's euclidean map for DTMs
# (requires the DSM to be computed)
if human_name == "DTM":
eu_map_source = "dsm"
euclidean_merge_dems(all_dems, dem_file, dem_vars, euclidean_map_source=eu_map_source)
if io.file_exists(dem_file):
# Crop
if args.crop > 0:
Cropper.crop(merged_bounds_file, dem_file, dem_vars,
keep_original=not args.optimize_disk_space)
log.ODM_INFO("Created %s" % dem_file)
if args.tiles:
generate_dem_tiles(dem_file, tree.path("%s_tiles" % human_name.lower()),
args.max_concurrency)
else:
log.ODM_WARNING("Cannot merge %s, %s was not created" % (human_name, dem_file))
else:
log.ODM_WARNING("Found merged %s in %s" % (human_name, dem_filename))
if args.merge in ['all', 'dem'] and args.dsm:
merge_dems("dsm.tif", "DSM")
if args.merge in ['all', 'dem'] and args.dtm:
merge_dems("dtm.tif", "DTM")
self.update_progress(95)
# Merge reports
if not io.dir_exists(tree.odm_report):
system.mkdir_p(tree.odm_report)
geojson_shots = tree.path(tree.odm_report, "shots.geojson")
if not io.file_exists(geojson_shots) or self.rerun():
geojson_shots_files = get_submodel_paths(tree.submodels_path, "odm_report", "shots.geojson")
log.ODM_INFO("Merging %s shots.geojson files" % len(geojson_shots_files))
merge_geojson_shots(geojson_shots_files, geojson_shots)
else:
log.ODM_WARNING("Found merged shots.geojson in %s" % tree.odm_report)
# Stop the pipeline short! We're done.
self.next_stage = None
else:
log.ODM_INFO("Normal dataset, nothing to merge.")
self.progress = 0.0
def process(self, args, outputs):
tree = outputs['tree']
reconstruction = outputs['reconstruction']
if not os.path.exists(tree.odm_report): system.mkdir_p(tree.odm_report)
log.ODM_INFO("Exporting shots.geojson")
shots_geojson = os.path.join(tree.odm_report, "shots.geojson")
if not io.file_exists(shots_geojson) or self.rerun():
# Extract geographical camera shots
if reconstruction.is_georeferenced():
shots = get_geojson_shots_from_opensfm(
tree.opensfm_reconstruction,
utm_srs=reconstruction.get_proj_srs(),
utm_offset=reconstruction.georef.utm_offset())
else:
# Pseudo geo
shots = get_geojson_shots_from_opensfm(
tree.opensfm_reconstruction,
pseudo_geotiff=tree.odm_orthophoto_tif)
if shots:
with open(shots_geojson, "w") as fout:
fout.write(json.dumps(shots))
log.ODM_INFO("Wrote %s" % shots_geojson)
else:
log.ODM_WARNING("Cannot extract shots")
else:
log.ODM_WARNING('Found a valid shots file in: %s' % shots_geojson)
if args.skip_report:
# Stop right here
log.ODM_WARNING("Skipping report generation as requested")
return
# Augment OpenSfM stats file with our own stats
odm_stats_json = os.path.join(tree.odm_report, "stats.json")
octx = OSFMContext(tree.opensfm)
osfm_stats_json = octx.path("stats", "stats.json")
odm_stats = None
point_cloud_file = None
views_dimension = None
if not os.path.exists(odm_stats_json) or self.rerun():
if os.path.exists(osfm_stats_json):
with open(osfm_stats_json, 'r') as f:
odm_stats = json.loads(f.read())
# Add point cloud stats
if os.path.exists(tree.odm_georeferencing_model_laz):
point_cloud_file = tree.odm_georeferencing_model_laz
views_dimension = "UserData"
# pc_info_file should have been generated by cropper
pc_info_file = os.path.join(
tree.odm_georeferencing,
"odm_georeferenced_model.info.json")
odm_stats[
'point_cloud_statistics'] = generate_point_cloud_stats(
tree.odm_georeferencing_model_laz, pc_info_file,
self.rerun())
else:
ply_pc = os.path.join(tree.odm_filterpoints,
"point_cloud.ply")
if os.path.exists(ply_pc):
point_cloud_file = ply_pc
views_dimension = "views"
pc_info_file = os.path.join(tree.odm_filterpoints,
"point_cloud.info.json")
odm_stats[
'point_cloud_statistics'] = generate_point_cloud_stats(
ply_pc, pc_info_file, self.rerun())
else:
log.ODM_WARNING("No point cloud found")
odm_stats['point_cloud_statistics'][
'dense'] = not args.fast_orthophoto
# Add runtime stats
total_time = (system.now_raw() -
outputs['start_time']).total_seconds()
odm_stats['odm_processing_statistics'] = {
'total_time':
total_time,
'total_time_human':
hms(total_time),
'average_gsd':
gsd.opensfm_reconstruction_average_gsd(
octx.recon_file(),
use_all_shots=reconstruction.has_gcp()),
}
with open(odm_stats_json, 'w') as f:
f.write(json.dumps(odm_stats))
else:
log.ODM_WARNING(
"Cannot generate report, OpenSfM stats are missing")
else:
log.ODM_WARNING("Reading existing stats %s" % odm_stats_json)
with open(odm_stats_json, 'r') as f:
odm_stats = json.loads(f.read())
# Generate overlap diagram
if odm_stats.get('point_cloud_statistics'
) and point_cloud_file and views_dimension:
bounds = odm_stats['point_cloud_statistics'].get('stats', {}).get(
'bbox', {}).get('native', {}).get('bbox')
if bounds:
image_target_size = 1400 # pixels
osfm_stats_dir = os.path.join(tree.opensfm, "stats")
diagram_tiff = os.path.join(osfm_stats_dir, "overlap.tif")
diagram_png = os.path.join(osfm_stats_dir, "overlap.png")
width = bounds.get('maxx') - bounds.get('minx')
height = bounds.get('maxy') - bounds.get('miny')
max_dim = max(width, height)
resolution = float(max_dim) / float(image_target_size)
radius = resolution * math.sqrt(2)
# Larger radius for sparse point cloud diagram
if not odm_stats['point_cloud_statistics']['dense']:
radius *= 10
system.run("pdal translate -i \"{}\" "
"-o \"{}\" "
"--writer gdal "
"--writers.gdal.resolution={} "
"--writers.gdal.data_type=uint8_t "
"--writers.gdal.dimension={} "
"--writers.gdal.output_type=max "
"--writers.gdal.radius={} ".format(
point_cloud_file, diagram_tiff, resolution,
views_dimension, radius))
report_assets = os.path.abspath(
os.path.join(os.path.dirname(__file__),
"../opendm/report"))
overlap_color_map = os.path.join(report_assets,
"overlap_color_map.txt")
bounds_file_path = os.path.join(
tree.odm_georeferencing,
'odm_georeferenced_model.bounds.gpkg')
if (args.crop > 0
or args.boundary) and os.path.isfile(bounds_file_path):
Cropper.crop(bounds_file_path,
diagram_tiff,
get_orthophoto_vars(args),
keep_original=False)
system.run(
"gdaldem color-relief \"{}\" \"{}\" \"{}\" -of PNG -alpha".
format(diagram_tiff, overlap_color_map, diagram_png))
# Copy assets
for asset in [
"overlap_diagram_legend.png", "dsm_gradient.png"
]:
shutil.copy(os.path.join(report_assets, asset),
os.path.join(osfm_stats_dir, asset))
# Generate previews of ortho/dsm
if os.path.isfile(tree.odm_orthophoto_tif):
osfm_ortho = os.path.join(osfm_stats_dir, "ortho.png")
generate_png(tree.odm_orthophoto_tif, osfm_ortho,
image_target_size)
dems = []
if args.dsm:
dems.append("dsm")
if args.dtm:
dems.append("dtm")
for dem in dems:
dem_file = tree.path("odm_dem", "%s.tif" % dem)
if os.path.isfile(dem_file):
# Resize first (faster)
resized_dem_file = io.related_file_path(
dem_file, postfix=".preview")
system.run(
"gdal_translate -outsize {} 0 \"{}\" \"{}\" --config GDAL_CACHEMAX {}%"
.format(image_target_size, dem_file,
resized_dem_file, get_max_memory()))
log.ODM_INFO("Computing raster stats for %s" %
resized_dem_file)
dem_stats = get_raster_stats(resized_dem_file)
if len(dem_stats) > 0:
odm_stats[dem + '_statistics'] = dem_stats[0]
osfm_dem = os.path.join(osfm_stats_dir, "%s.png" % dem)
colored_dem, hillshade_dem, colored_hillshade_dem = generate_colored_hillshade(
resized_dem_file)
system.run(
"gdal_translate -outsize {} 0 -of png \"{}\" \"{}\" --config GDAL_CACHEMAX {}%"
.format(image_target_size, colored_hillshade_dem,
osfm_dem, get_max_memory()))
for f in [
resized_dem_file, colored_dem, hillshade_dem,
colored_hillshade_dem
]:
if os.path.isfile(f):
os.remove(f)
else:
log.ODM_WARNING(
"Cannot generate overlap diagram, cannot compute point cloud bounds"
)
else:
log.ODM_WARNING(
"Cannot generate overlap diagram, point cloud stats missing")
octx.export_report(os.path.join(tree.odm_report, "report.pdf"),
odm_stats, self.rerun())
def process(self, args, outputs):
tree = outputs['tree']
reconstruction = outputs['reconstruction']
verbose = '-verbose' if args.verbose else ''
# define paths and create working directories
system.mkdir_p(tree.odm_orthophoto)
if not io.file_exists(tree.odm_orthophoto_file) or self.rerun():
# odm_orthophoto definitions
kwargs = {
'bin': context.odm_modules_path,
'log': tree.odm_orthophoto_log,
'ortho': tree.odm_orthophoto_file,
'corners': tree.odm_orthophoto_corners,
'res': 1.0 / (gsd.cap_resolution(args.orthophoto_resolution, tree.opensfm_reconstruction, ignore_gsd=args.ignore_gsd) / 100.0),
'verbose': verbose
}
# Have geo coordinates?
georef = reconstruction.georef
# Check if the georef object is initialized
# (during a --rerun this might not be)
# TODO: this should be moved to a more central location?
if reconstruction.is_georeferenced() and not reconstruction.georef.valid_utm_offsets():
georeferencing_dir = tree.odm_georeferencing if args.use_3dmesh and not args.skip_3dmodel else tree.odm_25dgeoreferencing
odm_georeferencing_model_txt_geo_file = os.path.join(georeferencing_dir, tree.odm_georeferencing_model_txt_geo)
if io.file_exists(odm_georeferencing_model_txt_geo_file):
reconstruction.georef.extract_offsets(odm_georeferencing_model_txt_geo_file)
else:
log.ODM_WARNING('Cannot read UTM offset from {}. An orthophoto will not be generated.'.format(odm_georeferencing_model_txt_geo_file))
if reconstruction.is_georeferenced():
if args.use_3dmesh:
kwargs['model_geo'] = os.path.join(tree.odm_texturing, tree.odm_georeferencing_model_obj_geo)
else:
kwargs['model_geo'] = os.path.join(tree.odm_25dtexturing, tree.odm_georeferencing_model_obj_geo)
else:
if args.use_3dmesh:
kwargs['model_geo'] = os.path.join(tree.odm_texturing, tree.odm_textured_model_obj)
else:
kwargs['model_geo'] = os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj)
# run odm_orthophoto
system.run('{bin}/odm_orthophoto -inputFile {model_geo} '
'-logFile {log} -outputFile {ortho} -resolution {res} {verbose} '
'-outputCornerFile {corners}'.format(**kwargs))
# Create georeferenced GeoTiff
geotiffcreated = False
if reconstruction.is_georeferenced() and reconstruction.georef.valid_utm_offsets():
ulx = uly = lrx = lry = 0.0
with open(tree.odm_orthophoto_corners) as f:
for lineNumber, line in enumerate(f):
if lineNumber == 0:
tokens = line.split(' ')
if len(tokens) == 4:
ulx = float(tokens[0]) + \
float(reconstruction.georef.utm_east_offset)
lry = float(tokens[1]) + \
float(reconstruction.georef.utm_north_offset)
lrx = float(tokens[2]) + \
float(reconstruction.georef.utm_east_offset)
uly = float(tokens[3]) + \
float(reconstruction.georef.utm_north_offset)
log.ODM_INFO('Creating GeoTIFF')
orthophoto_vars = orthophoto.get_orthophoto_vars(args)
kwargs = {
'ulx': ulx,
'uly': uly,
'lrx': lrx,
'lry': lry,
'vars': ' '.join(['-co %s=%s' % (k, orthophoto_vars[k]) for k in orthophoto_vars]),
'proj': reconstruction.georef.proj4(),
'png': tree.odm_orthophoto_file,
'tiff': tree.odm_orthophoto_tif,
'log': tree.odm_orthophoto_tif_log,
'max_memory': get_max_memory(),
}
system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
'{vars} '
'-a_srs \"{proj}\" '
'--config GDAL_CACHEMAX {max_memory}% '
'{png} {tiff} > {log}'.format(**kwargs))
bounds_file_path = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg')
# Cutline computation, before cropping
# We want to use the full orthophoto, not the cropped one.
if args.orthophoto_cutline:
compute_cutline(tree.odm_orthophoto_tif,
bounds_file_path,
os.path.join(tree.odm_orthophoto, "cutline.gpkg"),
args.max_concurrency,
tmpdir=os.path.join(tree.odm_orthophoto, "grass_cutline_tmpdir"),
scale=0.25)
if args.crop > 0:
Cropper.crop(bounds_file_path, tree.odm_orthophoto_tif, orthophoto_vars)
if args.build_overviews:
orthophoto.build_overviews(tree.odm_orthophoto_tif)
geotiffcreated = True
if not geotiffcreated:
log.ODM_WARNING('No geo-referenced orthophoto created due '
'to missing geo-referencing or corner coordinates.')
else:
log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file)
def process(args, current_path, max_concurrency, reconstruction):
#args = vars(args)
orthophoto_cutline = True
odm_orthophoto = io.join_paths(current_path, 'orthophoto')
odm_orthophoto_path = odm_orthophoto
odm_orthophoto_render = io.join_paths(odm_orthophoto_path,
'odm_orthophoto_render.tif')
odm_orthophoto_tif = io.join_paths(odm_orthophoto_path,
'odm_orthophoto.tif')
odm_orthophoto_corners = io.join_paths(odm_orthophoto_path,
'odm_orthophoto_corners.tif')
odm_orthophoto_log = io.join_paths(odm_orthophoto_path,
'odm_orthophoto_log.tif')
odm_orthophoto_tif_log = io.join_paths(odm_orthophoto_path,
'gdal_translate_log.txt')
odm_25dgeoreferencing = io.join_paths(current_path, 'odm_georeferencing')
odm_georeferencing = io.join_paths(current_path, 'odm_georeferencing')
odm_georeferencing_coords = io.join_paths(odm_georeferencing, 'coords.txt')
odm_georeferencing_gcp = io.find('gcp_list.txt', current_path)
odm_georeferencing_gcp_utm = io.join_paths(odm_georeferencing,
'gcp_list_utm.txt')
odm_georeferencing_utm_log = io.join_paths(
odm_georeferencing, 'odm_georeferencing_utm_log.txt')
odm_georeferencing_log = 'odm_georeferencing_log.txt'
odm_georeferencing_transform_file = 'odm_georeferencing_transform.txt'
odm_georeferencing_proj = 'proj.txt'
odm_georeferencing_model_txt_geo = 'odm_georeferencing_model_geo.txt'
odm_georeferencing_model_obj_geo = 'odm_textured_model_geo.obj'
odm_georeferencing_xyz_file = io.join_paths(odm_georeferencing,
'odm_georeferenced_model.csv')
odm_georeferencing_las_json = io.join_paths(odm_georeferencing, 'las.json')
odm_georeferencing_model_laz = io.join_paths(
odm_georeferencing, 'odm_georeferenced_model.laz')
odm_georeferencing_model_las = io.join_paths(
odm_georeferencing, 'odm_georeferenced_model.las')
odm_georeferencing_dem = io.join_paths(odm_georeferencing,
'odm_georeferencing_model_dem.tif')
opensfm_reconstruction = io.join_paths(current_path, 'reconstruction.json')
odm_texturing = io.join_paths(current_path, 'mvs')
odm_textured_model_obj = io.join_paths(odm_texturing,
'odm_textured_model.obj')
images_dir = io.join_paths(current_path, 'images')
reconstruction = reconstruction
verbose = ''
#"-verbose"
# define paths and create working directories
system.mkdir_p(odm_orthophoto)
if not io.file_exists(odm_orthophoto_tif):
gsd_error_estimate = 0.1
ignore_resolution = False
if not reconstruction.is_georeferenced():
# Match DEMs
gsd_error_estimate = -3
ignore_resolution = True
orthophoto_resolution = 5
resolution = 1.0 / (
gsd.cap_resolution(orthophoto_resolution,
opensfm_reconstruction,
gsd_error_estimate=gsd_error_estimate,
ignore_gsd=True,
ignore_resolution=ignore_resolution,
has_gcp=reconstruction.has_gcp()) / 100.0)
# odm_orthophoto definitions
kwargs = {
'bin': context.odm_modules_path,
'log': odm_orthophoto_log,
'ortho': odm_orthophoto_render,
'corners': odm_orthophoto_corners,
'res': resolution,
'bands': '',
'verbose': verbose
}
# Check if the georef object is initialized
# (during a --rerun this might not be)
# TODO: this should be moved to a more central location?
if reconstruction.is_georeferenced(
) and not reconstruction.georef.valid_utm_offsets():
georeferencing_dir = odm_georeferencing #if args.use_3dmesh and not args.skip_3dmodel else odm_25dgeoreferencing
odm_georeferencing_model_txt_geo_file = os.path.join(
georeferencing_dir, odm_georeferencing_model_txt_geo)
if io.file_exists(odm_georeferencing_model_txt_geo_file):
reconstruction.georef.extract_offsets(
odm_georeferencing_model_txt_geo_file)
else:
log.ODM_WARNING('Cannot read UTM offset from {}.'.format(
odm_georeferencing_model_txt_geo_file))
models = []
base_dir = odm_texturing
if reconstruction.is_georeferenced():
model_file = odm_georeferencing_model_obj_geo
else:
model_file = odm_textured_model_obj
if reconstruction.multi_camera:
for band in reconstruction.multi_camera:
primary = band == reconstruction.multi_camera[0]
subdir = ""
if not primary:
subdir = band['name'].lower()
models.append(os.path.join(base_dir, subdir, model_file))
kwargs['bands'] = '-bands %s' % (','.join([
quote(b['name'].lower()) for b in reconstruction.multi_camera
]))
else:
models.append(os.path.join(base_dir, model_file))
kwargs['models'] = ','.join(map(quote, models))
# run odm_orthophoto
system.run(
'{bin}/odm_orthophoto -inputFiles {models} '
'-logFile {log} -outputFile {ortho} -resolution {res} {verbose} '
'-outputCornerFile {corners} {bands}'.format(**kwargs))
# Create georeferenced GeoTiff
geotiffcreated = False
if reconstruction.is_georeferenced(
) and reconstruction.georef.valid_utm_offsets():
ulx = uly = lrx = lry = 0.0
with open(odm_orthophoto_corners) as f:
for lineNumber, line in enumerate(f):
if lineNumber == 0:
tokens = line.split(' ')
if len(tokens) == 4:
ulx = float(tokens[0]) + \
float(reconstruction.georef.utm_east_offset)
lry = float(tokens[1]) + \
float(reconstruction.georef.utm_north_offset)
lrx = float(tokens[2]) + \
float(reconstruction.georef.utm_east_offset)
uly = float(tokens[3]) + \
float(reconstruction.georef.utm_north_offset)
log.ODM_INFO('Creating GeoTIFF')
orthophoto_vars = orthophoto.get_orthophoto_vars(args)
kwargs = {
'ulx':
ulx,
'uly':
uly,
'lrx':
lrx,
'lry':
lry,
'vars':
' '.join([
'-co %s=%s' % (k, orthophoto_vars[k])
for k in orthophoto_vars
]),
'proj':
reconstruction.georef.proj4(),
'input':
odm_orthophoto_render,
'output':
odm_orthophoto_tif,
'log':
odm_orthophoto_tif_log,
'max_memory':
get_max_memory(),
}
system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
'{vars} '
'-a_srs \"{proj}\" '
'--config GDAL_CACHEMAX {max_memory}% '
'--config GDAL_TIFF_INTERNAL_MASK YES '
'{input} {output} > {log}'.format(**kwargs))
bounds_file_path = os.path.join(
odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg')
# Cutline computation, before cropping
# We want to use the full orthophoto, not the cropped one.
pio = True
if pio:
cutline_file = os.path.join(odm_orthophoto, "cutline.gpkg")
compute_cutline(odm_orthophoto_tif,
bounds_file_path,
cutline_file,
max_concurrency,
tmpdir=os.path.join(odm_orthophoto,
"grass_cutline_tmpdir"),
scale=0.25)
orthophoto.compute_mask_raster(odm_orthophoto_tif,
cutline_file,
os.path.join(
odm_orthophoto,
"odm_orthophoto_cut.tif"),
blend_distance=20,
only_max_coords_feature=True)
orthophoto.post_orthophoto_steps(args, bounds_file_path,
odm_orthophoto_tif)
# Generate feathered orthophoto also
if pio:
orthophoto.feather_raster(odm_orthophoto_tif,
os.path.join(
odm_orthophoto,
"odm_orthophoto_feathered.tif"),
blend_distance=20)
geotiffcreated = True
if not geotiffcreated:
if io.file_exists(odm_orthophoto_render):
pseudogeo.add_pseudo_georeferencing(odm_orthophoto_render)
log.ODM_INFO("Renaming %s --> %s" %
(odm_orthophoto_render, odm_orthophoto_tif))
os.rename(odm_orthophoto_render, odm_orthophoto_tif)
else:
log.ODM_WARNING(
"Could not generate an orthophoto (it did not render)")
else:
log.ODM_WARNING('Found a valid orthophoto in: %s' % odm_orthophoto_tif)
#generate png
orthophoto.generate_png(odm_orthophoto_tif)
def process(self, args, outputs):
tree = outputs['tree']
reconstruction = outputs['reconstruction']
if outputs['large']:
if not os.path.exists(tree.submodels_path):
log.ODM_ERROR(
"We reached the merge stage, but %s folder does not exist. Something must have gone wrong at an earlier stage. Check the log and fix possible problem before restarting?"
% tree.submodels_path)
exit(1)
# Merge point clouds
if args.merge in ['all', 'pointcloud']:
if not io.file_exists(
tree.odm_georeferencing_model_laz) or self.rerun():
all_point_clouds = get_submodel_paths(
tree.submodels_path, "odm_georeferencing",
"odm_georeferenced_model.laz")
try:
# pdal.merge_point_clouds(all_point_clouds, tree.odm_georeferencing_model_laz, args.verbose)
entwine.build(all_point_clouds,
tree.entwine_pointcloud,
max_concurrency=args.max_concurrency,
rerun=self.rerun())
except Exception as e:
log.ODM_WARNING(
"Could not merge point cloud: %s (skipping)" %
str(e))
if io.dir_exists(tree.entwine_pointcloud):
try:
system.run('pdal translate "ept://{}" "{}"'.format(
tree.entwine_pointcloud,
tree.odm_georeferencing_model_laz))
except Exception as e:
log.ODM_WARNING(
"Cannot export EPT dataset to LAZ: %s" %
str(e))
else:
log.ODM_WARNING("Found merged point cloud in %s" %
tree.odm_georeferencing_model_laz)
self.update_progress(25)
# Merge crop bounds
merged_bounds_file = os.path.join(
tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg')
if not io.file_exists(merged_bounds_file) or self.rerun():
all_bounds = get_submodel_paths(
tree.submodels_path, 'odm_georeferencing',
'odm_georeferenced_model.bounds.gpkg')
log.ODM_INFO("Merging all crop bounds: %s" % all_bounds)
if len(all_bounds) > 0:
# Calculate a new crop area
# based on the convex hull of all crop areas of all submodels
# (without a buffer, otherwise we are double-cropping)
Cropper.merge_bounds(all_bounds, merged_bounds_file, 0)
else:
log.ODM_WARNING("No bounds found for any submodel.")
# Merge orthophotos
if args.merge in ['all', 'orthophoto']:
if not io.dir_exists(tree.odm_orthophoto):
system.mkdir_p(tree.odm_orthophoto)
if not io.file_exists(tree.odm_orthophoto_tif) or self.rerun():
all_orthos_and_cutlines = get_all_submodel_paths(
tree.submodels_path,
os.path.join("odm_orthophoto", "odm_orthophoto.tif"),
os.path.join("odm_orthophoto", "cutline.gpkg"),
)
if len(all_orthos_and_cutlines) > 1:
log.ODM_INFO(
"Found %s submodels with valid orthophotos and cutlines"
% len(all_orthos_and_cutlines))
# TODO: histogram matching via rasterio
# currently parts have different color tones
merged_geotiff = os.path.join(
tree.odm_orthophoto, "odm_orthophoto.merged.tif")
kwargs = {
'orthophoto_merged':
merged_geotiff,
'input_files':
' '.join(
map(lambda i: quote(i[0]),
all_orthos_and_cutlines)),
'max_memory':
get_max_memory(),
'threads':
args.max_concurrency,
}
# use bounds as cutlines (blending)
if io.file_exists(merged_geotiff):
os.remove(merged_geotiff)
system.run('gdal_merge.py -o {orthophoto_merged} '
#'-createonly '
'-co "BIGTIFF=YES" '
'-co "BLOCKXSIZE=512" '
'-co "BLOCKYSIZE=512" '
'--config GDAL_CACHEMAX {max_memory}% '
'{input_files} '.format(**kwargs))
for ortho_cutline in all_orthos_and_cutlines:
kwargs['input_file'], kwargs[
'cutline'] = ortho_cutline
# Note: cblend has a high performance penalty
system.run(
'gdalwarp -cutline {cutline} '
'-cblend 20 '
'-r bilinear -multi '
'-wo NUM_THREADS={threads} '
'--config GDAL_CACHEMAX {max_memory}% '
'{input_file} {orthophoto_merged}'.format(
**kwargs))
# Apply orthophoto settings (compression, tiling, etc.)
orthophoto_vars = orthophoto.get_orthophoto_vars(args)
if io.file_exists(tree.odm_orthophoto_tif):
os.remove(tree.odm_orthophoto_tif)
kwargs = {
'vars':
' '.join([
'-co %s=%s' % (k, orthophoto_vars[k])
for k in orthophoto_vars
]),
'max_memory':
get_max_memory(),
'merged':
merged_geotiff,
'log':
tree.odm_orthophoto_tif_log,
'orthophoto':
tree.odm_orthophoto_tif,
}
system.run(
'gdal_translate '
'{vars} '
'--config GDAL_CACHEMAX {max_memory}% '
'{merged} {orthophoto} > {log}'.format(**kwargs))
os.remove(merged_geotiff)
# Crop
if args.crop > 0:
Cropper.crop(merged_bounds_file,
tree.odm_orthophoto_tif,
orthophoto_vars)
# Overviews
if args.build_overviews:
orthophoto.build_overviews(tree.odm_orthophoto_tif)
elif len(all_orthos_and_cutlines) == 1:
# Simply copy
log.ODM_WARNING(
"A single orthophoto/cutline pair was found between all submodels."
)
shutil.copyfile(all_orthos_and_cutlines[0][0],
tree.odm_orthophoto_tif)
else:
log.ODM_WARNING(
"No orthophoto/cutline pairs were found in any of the submodels. No orthophoto will be generated."
)
else:
log.ODM_WARNING("Found merged orthophoto in %s" %
tree.odm_orthophoto_tif)
self.update_progress(75)
# Merge DEMs
def merge_dems(dem_filename, human_name):
if not io.dir_exists(tree.path('odm_dem')):
system.mkdir_p(tree.path('odm_dem'))
dem_file = tree.path("odm_dem", dem_filename)
if not io.file_exists(dem_file) or self.rerun():
all_dems = get_submodel_paths(tree.submodels_path,
"odm_dem", dem_filename)
log.ODM_INFO("Merging %ss" % human_name)
# Merge
dem_vars = utils.get_dem_vars(args)
euclidean_merge_dems(all_dems, dem_file, dem_vars)
if io.file_exists(dem_file):
# Crop
if args.crop > 0:
Cropper.crop(merged_bounds_file, dem_file,
dem_vars)
log.ODM_INFO("Created %s" % dem_file)
else:
log.ODM_WARNING("Cannot merge %s, %s was not created" %
(human_name, dem_file))
else:
log.ODM_WARNING("Found merged %s in %s" %
(human_name, dem_filename))
if args.merge in ['all', 'dem'] and args.dsm:
merge_dems("dsm.tif", "DSM")
if args.merge in ['all', 'dem'] and args.dtm:
merge_dems("dtm.tif", "DTM")
# Stop the pipeline short! We're done.
self.next_stage = None
else:
log.ODM_INFO("Normal dataset, nothing to merge.")
self.progress = 0.0
def process(self, args, outputs):
tree = outputs['tree']
reconstruction = outputs['reconstruction']
verbose = '-verbose' if args.verbose else ''
# define paths and create working directories
system.mkdir_p(tree.odm_orthophoto)
if not io.file_exists(tree.odm_orthophoto_tif) or self.rerun():
gsd_error_estimate = 0.1
ignore_resolution = False
if not reconstruction.is_georeferenced():
# Match DEMs
gsd_error_estimate = -3
ignore_resolution = True
resolution = 1.0 / (
gsd.cap_resolution(args.orthophoto_resolution,
tree.opensfm_reconstruction,
gsd_error_estimate=gsd_error_estimate,
ignore_gsd=args.ignore_gsd,
ignore_resolution=ignore_resolution,
has_gcp=reconstruction.has_gcp()) / 100.0)
# odm_orthophoto definitions
kwargs = {
'bin': context.odm_modules_path,
'log': tree.odm_orthophoto_log,
'ortho': tree.odm_orthophoto_render,
'corners': tree.odm_orthophoto_corners,
'res': resolution,
'bands': '',
'verbose': verbose
}
models = []
if args.use_3dmesh:
base_dir = tree.odm_texturing
else:
base_dir = tree.odm_25dtexturing
model_file = tree.odm_textured_model_obj
if reconstruction.multi_camera:
for band in reconstruction.multi_camera:
primary = band['name'] == get_primary_band_name(
reconstruction.multi_camera, args.primary_band)
subdir = ""
if not primary:
subdir = band['name'].lower()
models.append(os.path.join(base_dir, subdir, model_file))
kwargs['bands'] = '-bands %s' % (','.join(
[quote(b['name']) for b in reconstruction.multi_camera]))
else:
models.append(os.path.join(base_dir, model_file))
kwargs['models'] = ','.join(map(quote, models))
# run odm_orthophoto
system.run(
'{bin}/odm_orthophoto -inputFiles {models} '
'-logFile {log} -outputFile {ortho} -resolution {res} {verbose} '
'-outputCornerFile {corners} {bands}'.format(**kwargs))
# Create georeferenced GeoTiff
geotiffcreated = False
if reconstruction.is_georeferenced():
ulx = uly = lrx = lry = 0.0
with open(tree.odm_orthophoto_corners) as f:
for lineNumber, line in enumerate(f):
if lineNumber == 0:
tokens = line.split(' ')
if len(tokens) == 4:
ulx = float(tokens[0]) + \
float(reconstruction.georef.utm_east_offset)
lry = float(tokens[1]) + \
float(reconstruction.georef.utm_north_offset)
lrx = float(tokens[2]) + \
float(reconstruction.georef.utm_east_offset)
uly = float(tokens[3]) + \
float(reconstruction.georef.utm_north_offset)
log.ODM_INFO('Creating GeoTIFF')
orthophoto_vars = orthophoto.get_orthophoto_vars(args)
kwargs = {
'ulx':
ulx,
'uly':
uly,
'lrx':
lrx,
'lry':
lry,
'vars':
' '.join([
'-co %s=%s' % (k, orthophoto_vars[k])
for k in orthophoto_vars
]),
'proj':
reconstruction.georef.proj4(),
'input':
tree.odm_orthophoto_render,
'output':
tree.odm_orthophoto_tif,
'log':
tree.odm_orthophoto_tif_log,
'max_memory':
get_max_memory(),
}
system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} '
'{vars} '
'-a_srs \"{proj}\" '
'--config GDAL_CACHEMAX {max_memory}% '
'--config GDAL_TIFF_INTERNAL_MASK YES '
'{input} {output} > {log}'.format(**kwargs))
bounds_file_path = os.path.join(
tree.odm_georeferencing,
'odm_georeferenced_model.bounds.gpkg')
# Cutline computation, before cropping
# We want to use the full orthophoto, not the cropped one.
if args.orthophoto_cutline:
cutline_file = os.path.join(tree.odm_orthophoto,
"cutline.gpkg")
compute_cutline(tree.odm_orthophoto_tif,
bounds_file_path,
cutline_file,
args.max_concurrency,
tmpdir=os.path.join(
tree.odm_orthophoto,
"grass_cutline_tmpdir"),
scale=0.25)
orthophoto.compute_mask_raster(
tree.odm_orthophoto_tif,
cutline_file,
os.path.join(tree.odm_orthophoto,
"odm_orthophoto_cut.tif"),
blend_distance=20,
only_max_coords_feature=True)
orthophoto.post_orthophoto_steps(args, bounds_file_path,
tree.odm_orthophoto_tif,
tree.orthophoto_tiles)
# Generate feathered orthophoto also
if args.orthophoto_cutline:
orthophoto.feather_raster(
tree.odm_orthophoto_tif,
os.path.join(tree.odm_orthophoto,
"odm_orthophoto_feathered.tif"),
blend_distance=20)
geotiffcreated = True
if not geotiffcreated:
if io.file_exists(tree.odm_orthophoto_render):
pseudogeo.add_pseudo_georeferencing(
tree.odm_orthophoto_render)
log.ODM_INFO(
"Renaming %s --> %s" %
(tree.odm_orthophoto_render, tree.odm_orthophoto_tif))
os.rename(tree.odm_orthophoto_render,
tree.odm_orthophoto_tif)
else:
log.ODM_WARNING(
"Could not generate an orthophoto (it did not render)")
else:
log.ODM_WARNING('Found a valid orthophoto in: %s' %
tree.odm_orthophoto_tif)
if args.optimize_disk_space and io.file_exists(
tree.odm_orthophoto_render):
os.remove(tree.odm_orthophoto_render)
