def process(self, inputs, outputs): # Benchmarking start_time = system.now_raw() log.ODM_INFO('Running ODM DEM Cell') # get inputs args = self.inputs.args tree = self.inputs.tree las_model_found = io.file_exists(tree.odm_georeferencing_model_las) # check if we rerun cell or not rerun_cell = (args.rerun is not None and args.rerun == 'odm_dem') or \ (args.rerun_all) or \ (args.rerun_from is not None and 'odm_dem' in args.rerun_from) log.ODM_INFO('Classify: ' + str(args.pc_classify != "none")) log.ODM_INFO('Create DSM: ' + str(args.dsm)) log.ODM_INFO('Create DTM: ' + str(args.dtm)) log.ODM_INFO('DEM input file {0} found: {1}'.format( tree.odm_georeferencing_model_las, str(las_model_found))) # Setup terrain parameters terrain_params_map = { 'flatnonforest': (1, 3), 'flatforest': (1, 2), 'complexnonforest': (5, 2), 'complexforest': (10, 2) } terrain_params = terrain_params_map[args.dem_terrain_type.lower()] slope, cellsize = terrain_params # define paths and create working directories odm_dem_root = tree.path('odm_dem') if not io.dir_exists(odm_dem_root): system.mkdir_p(odm_dem_root) if args.pc_classify != "none" and las_model_found: pc_classify_marker = os.path.join(odm_dem_root, 'pc_classify_done.txt') if not io.file_exists(pc_classify_marker) or rerun_cell: log.ODM_INFO("Classifying {} using {}".format( tree.odm_georeferencing_model_las, args.pc_classify)) commands.classify(tree.odm_georeferencing_model_las, args.pc_classify == "smrf", slope, cellsize, approximate=args.dem_approximate, initialDistance=args.dem_initial_distance, verbose=args.verbose) with open(pc_classify_marker, 'w') as f: f.write('Classify: {}\n'.format(args.pc_classify)) f.write('Slope: {}\n'.format(slope)) f.write('Cellsize: {}\n'.format(cellsize)) f.write('Approximate: {}\n'.format(args.dem_approximate)) f.write('InitialDistance: {}\n'.format( args.dem_initial_distance)) # Do we need to process anything here? if (args.dsm or args.dtm) and las_model_found: dsm_output_filename = os.path.join(odm_dem_root, 'dsm.tif') dtm_output_filename = os.path.join(odm_dem_root, 'dtm.tif') if (args.dtm and not io.file_exists(dtm_output_filename)) or \ (args.dsm and not io.file_exists(dsm_output_filename)) or \ rerun_cell: products = [] if args.dsm: products.append('dsm') if args.dtm: products.append('dtm') radius_steps = [args.dem_resolution] for _ in range(args.dem_gapfill_steps - 1): radius_steps.append( radius_steps[-1] * 3) # 3 is arbitrary, maybe there's a better value? for product in products: commands.create_dems([tree.odm_georeferencing_model_las], product, radius=map(str, radius_steps), gapfill=True, outdir=odm_dem_root, resolution=args.dem_resolution, maxsd=args.dem_maxsd, maxangle=args.dem_maxangle, decimation=args.dem_decimation, verbose=args.verbose) if args.crop > 0: bounds_shapefile_path = os.path.join( tree.odm_georeferencing, 'odm_georeferenced_model.bounds.shp') if os.path.exists(bounds_shapefile_path): Cropper.crop( bounds_shapefile_path, os.path.join(odm_dem_root, "{}.tif".format(product)), { 'TILED': 'YES', 'COMPRESS': 'LZW', 'BLOCKXSIZE': 512, 'BLOCKYSIZE': 512, 'NUM_THREADS': 'ALL_CPUS' }) else: log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root) else: log.ODM_WARNING('DEM will not be generated') if args.time: system.benchmark(start_time, tree.benchmarking, 'Dem') log.ODM_INFO('Running ODM DEM Cell - Finished') return ecto.OK if args.end_with != 'odm_dem' else ecto.QUIT
def process(self, args, outputs): tree = outputs['tree'] reconstruction = outputs['reconstruction'] if outputs['large']: if not os.path.exists(tree.submodels_path): log.ODM_ERROR( "We reached the merge stage, but %s folder does not exist. Something must have gone wrong at an earlier stage. Check the log and fix possible problem before restarting?" % tree.submodels_path) exit(1) # Merge point clouds if args.merge in ['all', 'pointcloud']: if not io.dir_exists(tree.entwine_pointcloud) or self.rerun(): all_point_clouds = get_submodel_paths( tree.submodels_path, "odm_georeferencing", "odm_georeferenced_model.laz") try: entwine.build(all_point_clouds, tree.entwine_pointcloud, max_concurrency=args.max_concurrency, rerun=self.rerun()) except Exception as e: log.ODM_WARNING( "Could not merge EPT point cloud: %s (skipping)" % str(e)) else: log.ODM_WARNING("Found merged EPT point cloud in %s" % tree.entwine_pointcloud) if not io.file_exists( tree.odm_georeferencing_model_laz) or self.rerun(): if io.dir_exists(tree.entwine_pointcloud): try: system.run('pdal translate "ept://{}" "{}"'.format( tree.entwine_pointcloud, tree.odm_georeferencing_model_laz)) except Exception as e: log.ODM_WARNING( "Cannot export EPT dataset to LAZ: %s" % str(e)) else: log.ODM_WARNING( "No EPT point cloud found (%s), skipping LAZ conversion)" % tree.entwine_pointcloud) else: log.ODM_WARNING("Found merged point cloud in %s" % tree.odm_georeferencing_model_laz) self.update_progress(25) # Merge crop bounds merged_bounds_file = os.path.join( tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg') if not io.file_exists(merged_bounds_file) or self.rerun(): all_bounds = get_submodel_paths( tree.submodels_path, 'odm_georeferencing', 'odm_georeferenced_model.bounds.gpkg') log.ODM_INFO("Merging all crop bounds: %s" % all_bounds) if len(all_bounds) > 0: # Calculate a new crop area # based on the convex hull of all crop areas of all submodels # (without a buffer, otherwise we are double-cropping) Cropper.merge_bounds(all_bounds, merged_bounds_file, 0) else: log.ODM_WARNING("No bounds found for any submodel.") # Merge orthophotos if args.merge in ['all', 'orthophoto']: if not io.dir_exists(tree.odm_orthophoto): system.mkdir_p(tree.odm_orthophoto) if not io.file_exists(tree.odm_orthophoto_tif) or self.rerun(): all_orthos_and_cutlines = get_all_submodel_paths( tree.submodels_path, os.path.join("odm_orthophoto", "odm_orthophoto.tif"), os.path.join("odm_orthophoto", "cutline.gpkg"), ) if len(all_orthos_and_cutlines) > 1: log.ODM_INFO( "Found %s submodels with valid orthophotos and cutlines" % len(all_orthos_and_cutlines)) # TODO: histogram matching via rasterio # currently parts have different color tones merged_geotiff = os.path.join( tree.odm_orthophoto, "odm_orthophoto.merged.tif") kwargs = { 'orthophoto_merged': merged_geotiff, 'input_files': ' '.join( map(lambda i: quote(i[0]), all_orthos_and_cutlines)), 'max_memory': get_max_memory(), 'threads': args.max_concurrency, } # use bounds as cutlines (blending) if io.file_exists(merged_geotiff): os.remove(merged_geotiff) system.run('gdal_merge.py -o {orthophoto_merged} ' #'-createonly ' '-co "BIGTIFF=YES" ' '-co "BLOCKXSIZE=512" ' '-co "BLOCKYSIZE=512" ' '--config GDAL_CACHEMAX {max_memory}% ' '{input_files} '.format(**kwargs)) for ortho_cutline in all_orthos_and_cutlines: kwargs['input_file'], kwargs[ 'cutline'] = ortho_cutline # Note: cblend has a high performance penalty system.run( 'gdalwarp -cutline {cutline} ' '-cblend 20 ' '-r bilinear -multi ' '-wo NUM_THREADS={threads} ' '--config GDAL_CACHEMAX {max_memory}% ' '{input_file} {orthophoto_merged}'.format( **kwargs)) # Apply orthophoto settings (compression, tiling, etc.) orthophoto_vars = orthophoto.get_orthophoto_vars(args) if io.file_exists(tree.odm_orthophoto_tif): os.remove(tree.odm_orthophoto_tif) kwargs = { 'vars': ' '.join([ '-co %s=%s' % (k, orthophoto_vars[k]) for k in orthophoto_vars ]), 'max_memory': get_max_memory(), 'merged': merged_geotiff, 'log': tree.odm_orthophoto_tif_log, 'orthophoto': tree.odm_orthophoto_tif, } system.run( 'gdal_translate ' '{vars} ' '--config GDAL_CACHEMAX {max_memory}% ' '{merged} {orthophoto} > {log}'.format(**kwargs)) os.remove(merged_geotiff) # Crop if args.crop > 0: Cropper.crop(merged_bounds_file, tree.odm_orthophoto_tif, orthophoto_vars) # Overviews if args.build_overviews: orthophoto.build_overviews(tree.odm_orthophoto_tif) elif len(all_orthos_and_cutlines) == 1: # Simply copy log.ODM_WARNING( "A single orthophoto/cutline pair was found between all submodels." ) shutil.copyfile(all_orthos_and_cutlines[0][0], tree.odm_orthophoto_tif) else: log.ODM_WARNING( "No orthophoto/cutline pairs were found in any of the submodels. No orthophoto will be generated." ) else: log.ODM_WARNING("Found merged orthophoto in %s" % tree.odm_orthophoto_tif) self.update_progress(75) # Merge DEMs def merge_dems(dem_filename, human_name): if not io.dir_exists(tree.path('odm_dem')): system.mkdir_p(tree.path('odm_dem')) dem_file = tree.path("odm_dem", dem_filename) if not io.file_exists(dem_file) or self.rerun(): all_dems = get_submodel_paths(tree.submodels_path, "odm_dem", dem_filename) log.ODM_INFO("Merging %ss" % human_name) # Merge dem_vars = utils.get_dem_vars(args) eu_map_source = None # Default # Use DSM's euclidean map for DTMs # (requires the DSM to be computed) if human_name == "DTM": eu_map_source = "dsm" euclidean_merge_dems(all_dems, dem_file, dem_vars, euclidean_map_source=eu_map_source) if io.file_exists(dem_file): # Crop if args.crop > 0: Cropper.crop(merged_bounds_file, dem_file, dem_vars) log.ODM_INFO("Created %s" % dem_file) else: log.ODM_WARNING("Cannot merge %s, %s was not created" % (human_name, dem_file)) else: log.ODM_WARNING("Found merged %s in %s" % (human_name, dem_filename)) if args.merge in ['all', 'dem'] and args.dsm: merge_dems("dsm.tif", "DSM") if args.merge in ['all', 'dem'] and args.dtm: merge_dems("dtm.tif", "DTM") # Stop the pipeline short! We're done. self.next_stage = None else: log.ODM_INFO("Normal dataset, nothing to merge.") self.progress = 0.0
def process(self, args, outputs): tree = outputs['tree'] reconstruction = outputs['reconstruction'] if outputs['large']: if not os.path.exists(tree.submodels_path): log.ODM_ERROR( "We reached the merge stage, but %s folder does not exist. Something must have gone wrong at an earlier stage. Check the log and fix possible problem before restarting?" % tree.submodels_path) exit(1) # Merge point clouds if args.merge in ['all', 'pointcloud']: if not io.file_exists( tree.odm_georeferencing_model_laz) or self.rerun(): all_point_clouds = get_submodel_paths( tree.submodels_path, "odm_georeferencing", "odm_georeferenced_model.laz") try: point_cloud.merge(all_point_clouds, tree.odm_georeferencing_model_laz, rerun=self.rerun()) point_cloud.post_point_cloud_steps(args, tree) except Exception as e: log.ODM_WARNING( "Could not merge point cloud: %s (skipping)" % str(e)) else: log.ODM_WARNING("Found merged point cloud in %s" % tree.odm_georeferencing_model_laz) self.update_progress(25) # Merge crop bounds merged_bounds_file = os.path.join( tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg') if not io.file_exists(merged_bounds_file) or self.rerun(): all_bounds = get_submodel_paths( tree.submodels_path, 'odm_georeferencing', 'odm_georeferenced_model.bounds.gpkg') log.ODM_INFO("Merging all crop bounds: %s" % all_bounds) if len(all_bounds) > 0: # Calculate a new crop area # based on the convex hull of all crop areas of all submodels # (without a buffer, otherwise we are double-cropping) Cropper.merge_bounds(all_bounds, merged_bounds_file, 0) else: log.ODM_WARNING("No bounds found for any submodel.") # Merge orthophotos if args.merge in ['all', 'orthophoto']: if not io.dir_exists(tree.odm_orthophoto): system.mkdir_p(tree.odm_orthophoto) if not io.file_exists(tree.odm_orthophoto_tif) or self.rerun(): all_orthos_and_ortho_cuts = get_all_submodel_paths( tree.submodels_path, os.path.join("odm_orthophoto", "odm_orthophoto_feathered.tif"), os.path.join("odm_orthophoto", "odm_orthophoto_cut.tif"), ) if len(all_orthos_and_ortho_cuts) > 1: log.ODM_INFO( "Found %s submodels with valid orthophotos and cutlines" % len(all_orthos_and_ortho_cuts)) # TODO: histogram matching via rasterio # currently parts have different color tones if io.file_exists(tree.odm_orthophoto_tif): os.remove(tree.odm_orthophoto_tif) orthophoto_vars = orthophoto.get_orthophoto_vars(args) orthophoto.merge(all_orthos_and_ortho_cuts, tree.odm_orthophoto_tif, orthophoto_vars) orthophoto.post_orthophoto_steps( args, merged_bounds_file, tree.odm_orthophoto_tif) elif len(all_orthos_and_ortho_cuts) == 1: # Simply copy log.ODM_WARNING( "A single orthophoto/cutline pair was found between all submodels." ) shutil.copyfile(all_orthos_and_ortho_cuts[0][0], tree.odm_orthophoto_tif) else: log.ODM_WARNING( "No orthophoto/cutline pairs were found in any of the submodels. No orthophoto will be generated." ) else: log.ODM_WARNING("Found merged orthophoto in %s" % tree.odm_orthophoto_tif) self.update_progress(75) # Merge DEMs def merge_dems(dem_filename, human_name): if not io.dir_exists(tree.path('odm_dem')): system.mkdir_p(tree.path('odm_dem')) dem_file = tree.path("odm_dem", dem_filename) if not io.file_exists(dem_file) or self.rerun(): all_dems = get_submodel_paths(tree.submodels_path, "odm_dem", dem_filename) log.ODM_INFO("Merging %ss" % human_name) # Merge dem_vars = utils.get_dem_vars(args) eu_map_source = None # Default # Use DSM's euclidean map for DTMs # (requires the DSM to be computed) if human_name == "DTM": eu_map_source = "dsm" euclidean_merge_dems(all_dems, dem_file, dem_vars, euclidean_map_source=eu_map_source) if io.file_exists(dem_file): # Crop if args.crop > 0: Cropper.crop( merged_bounds_file, dem_file, dem_vars, keep_original=not args.optimize_disk_space) log.ODM_INFO("Created %s" % dem_file) else: log.ODM_WARNING("Cannot merge %s, %s was not created" % (human_name, dem_file)) else: log.ODM_WARNING("Found merged %s in %s" % (human_name, dem_filename)) if args.merge in ['all', 'dem'] and args.dsm: merge_dems("dsm.tif", "DSM") if args.merge in ['all', 'dem'] and args.dtm: merge_dems("dtm.tif", "DTM") # Stop the pipeline short! We're done. self.next_stage = None else: log.ODM_INFO("Normal dataset, nothing to merge.") self.progress = 0.0
def process(self, args, outputs): tree = outputs['tree'] reconstruction = outputs['reconstruction'] dem_input = tree.odm_georeferencing_model_laz pc_model_found = io.file_exists(dem_input) ignore_resolution = False pseudo_georeference = False if not reconstruction.is_georeferenced(): log.ODM_WARNING("Not georeferenced, using ungeoreferenced point cloud...") ignore_resolution = True pseudo_georeference = True resolution = gsd.cap_resolution(args.dem_resolution, tree.opensfm_reconstruction, gsd_error_estimate=-3, ignore_gsd=args.ignore_gsd, ignore_resolution=ignore_resolution, has_gcp=reconstruction.has_gcp()) log.ODM_INFO('Classify: ' + str(args.pc_classify)) log.ODM_INFO('Create DSM: ' + str(args.dsm)) log.ODM_INFO('Create DTM: ' + str(args.dtm)) log.ODM_INFO('DEM input file {0} found: {1}'.format(dem_input, str(pc_model_found))) # define paths and create working directories odm_dem_root = tree.path('odm_dem') if not io.dir_exists(odm_dem_root): system.mkdir_p(odm_dem_root) if args.pc_classify and pc_model_found: pc_classify_marker = os.path.join(odm_dem_root, 'pc_classify_done.txt') if not io.file_exists(pc_classify_marker) or self.rerun(): log.ODM_INFO("Classifying {} using Simple Morphological Filter".format(dem_input)) commands.classify(dem_input, args.smrf_scalar, args.smrf_slope, args.smrf_threshold, args.smrf_window, verbose=args.verbose ) with open(pc_classify_marker, 'w') as f: f.write('Classify: smrf\n') f.write('Scalar: {}\n'.format(args.smrf_scalar)) f.write('Slope: {}\n'.format(args.smrf_slope)) f.write('Threshold: {}\n'.format(args.smrf_threshold)) f.write('Window: {}\n'.format(args.smrf_window)) progress = 20 self.update_progress(progress) if args.pc_rectify: commands.rectify(dem_input, args.debug) # Do we need to process anything here? if (args.dsm or args.dtm) and pc_model_found: dsm_output_filename = os.path.join(odm_dem_root, 'dsm.tif') dtm_output_filename = os.path.join(odm_dem_root, 'dtm.tif') if (args.dtm and not io.file_exists(dtm_output_filename)) or \ (args.dsm and not io.file_exists(dsm_output_filename)) or \ self.rerun(): products = [] if args.dsm or (args.dtm and args.dem_euclidean_map): products.append('dsm') if args.dtm: products.append('dtm') radius_steps = [(resolution / 100.0) / 2.0] for _ in range(args.dem_gapfill_steps - 1): radius_steps.append(radius_steps[-1] * 2) # 2 is arbitrary, maybe there's a better value? for product in products: commands.create_dem( dem_input, product, output_type='idw' if product == 'dtm' else 'max', radiuses=list(map(str, radius_steps)), gapfill=args.dem_gapfill_steps > 0, outdir=odm_dem_root, resolution=resolution / 100.0, decimation=args.dem_decimation, verbose=args.verbose, max_workers=args.max_concurrency, keep_unfilled_copy=args.dem_euclidean_map ) dem_geotiff_path = os.path.join(odm_dem_root, "{}.tif".format(product)) bounds_file_path = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg') if args.crop > 0: # Crop DEM Cropper.crop(bounds_file_path, dem_geotiff_path, utils.get_dem_vars(args), keep_original=not args.optimize_disk_space) if args.dem_euclidean_map: unfilled_dem_path = io.related_file_path(dem_geotiff_path, postfix=".unfilled") if args.crop > 0: # Crop unfilled DEM Cropper.crop(bounds_file_path, unfilled_dem_path, utils.get_dem_vars(args), keep_original=not args.optimize_disk_space) commands.compute_euclidean_map(unfilled_dem_path, io.related_file_path(dem_geotiff_path, postfix=".euclideand"), overwrite=True) if pseudo_georeference: # 0.1 is arbitrary pseudogeo.add_pseudo_georeferencing(dem_geotiff_path, 0.1) if pseudo_georeference: pseudogeo.add_pseudo_georeferencing(dem_geotiff_path) if args.tiles: generate_dem_tiles(dem_geotiff_path, tree.path("%s_tiles" % product), args.max_concurrency) progress += 30 self.update_progress(progress) else: log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root) else: log.ODM_WARNING('DEM will not be generated')
def process(self, args, outputs): tree = outputs['tree'] reconstruction = outputs['reconstruction'] verbose = '-verbose' if args.verbose else '' # define paths and create working directories system.mkdir_p(tree.odm_orthophoto) if not io.file_exists(tree.odm_orthophoto_file) or self.rerun(): # odm_orthophoto definitions kwargs = { 'bin': context.odm_modules_path, 'log': tree.odm_orthophoto_log, 'ortho': tree.odm_orthophoto_file, 'corners': tree.odm_orthophoto_corners, 'res': 1.0 / (gsd.cap_resolution(args.orthophoto_resolution, tree.opensfm_reconstruction, ignore_gsd=args.ignore_gsd) / 100.0), 'verbose': verbose } # Have geo coordinates? georef = reconstruction.georef # Check if the georef object is initialized # (during a --rerun this might not be) # TODO: we should move this to a more central # location (perhaps during the dataset initialization) if georef and not georef.utm_east_offset: georeferencing_dir = tree.odm_georeferencing if args.use_3dmesh and not args.skip_3dmodel else tree.odm_25dgeoreferencing odm_georeferencing_model_txt_geo_file = os.path.join( georeferencing_dir, tree.odm_georeferencing_model_txt_geo) if io.file_exists(odm_georeferencing_model_txt_geo_file): georef.extract_offsets( odm_georeferencing_model_txt_geo_file) else: log.ODM_WARNING( 'Cannot read UTM offset from {}. An orthophoto will not be generated.' .format(odm_georeferencing_model_txt_geo_file)) if georef: if args.use_3dmesh: kwargs['model_geo'] = os.path.join( tree.odm_texturing, tree.odm_georeferencing_model_obj_geo) else: kwargs['model_geo'] = os.path.join( tree.odm_25dtexturing, tree.odm_georeferencing_model_obj_geo) else: if args.use_3dmesh: kwargs['model_geo'] = os.path.join( tree.odm_texturing, tree.odm_textured_model_obj) else: kwargs['model_geo'] = os.path.join( tree.odm_25dtexturing, tree.odm_textured_model_obj) # run odm_orthophoto system.run( '{bin}/odm_orthophoto -inputFile {model_geo} ' '-logFile {log} -outputFile {ortho} -resolution {res} {verbose} ' '-outputCornerFile {corners}'.format(**kwargs)) # Create georeferenced GeoTiff geotiffcreated = False if georef and georef.projection and georef.utm_east_offset and georef.utm_north_offset: ulx = uly = lrx = lry = 0.0 with open(tree.odm_orthophoto_corners) as f: for lineNumber, line in enumerate(f): if lineNumber == 0: tokens = line.split(' ') if len(tokens) == 4: ulx = float(tokens[0]) + \ float(georef.utm_east_offset) lry = float(tokens[1]) + \ float(georef.utm_north_offset) lrx = float(tokens[2]) + \ float(georef.utm_east_offset) uly = float(tokens[3]) + \ float(georef.utm_north_offset) log.ODM_INFO('Creating GeoTIFF') orthophoto_vars = orthophoto.get_orthophoto_vars(args) kwargs = { 'ulx': ulx, 'uly': uly, 'lrx': lrx, 'lry': lry, 'vars': ' '.join([ '-co %s=%s' % (k, orthophoto_vars[k]) for k in orthophoto_vars ]), 'proj': georef.projection.srs, 'png': tree.odm_orthophoto_file, 'tiff': tree.odm_orthophoto_tif, 'log': tree.odm_orthophoto_tif_log, 'max_memory': get_max_memory(), } system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} ' '{vars} ' '-a_srs \"{proj}\" ' '--config GDAL_CACHEMAX {max_memory}% ' '{png} {tiff} > {log}'.format(**kwargs)) bounds_file_path = os.path.join( tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg') # Cutline computation, before cropping # We want to use the full orthophoto, not the cropped one. if args.orthophoto_cutline: compute_cutline(tree.odm_orthophoto_tif, bounds_file_path, os.path.join(tree.odm_orthophoto, "cutline.gpkg"), args.max_concurrency, tmpdir=os.path.join( tree.odm_orthophoto, "grass_cutline_tmpdir")) if args.crop > 0: Cropper.crop(bounds_file_path, tree.odm_orthophoto_tif, orthophoto_vars) if args.build_overviews: orthophoto.build_overviews(tree.odm_orthophoto_tif) geotiffcreated = True if not geotiffcreated: log.ODM_WARNING( 'No geo-referenced orthophoto created due ' 'to missing geo-referencing or corner coordinates.') else: log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file)
def process(self, args, outputs): tree = outputs['tree'] reconstruction = outputs['reconstruction'] if not os.path.exists(tree.odm_report): system.mkdir_p(tree.odm_report) log.ODM_INFO("Exporting shots.geojson") shots_geojson = os.path.join(tree.odm_report, "shots.geojson") if not io.file_exists(shots_geojson) or self.rerun(): # Extract geographical camera shots if reconstruction.is_georeferenced(): shots = get_geojson_shots_from_opensfm( tree.opensfm_reconstruction, tree.opensfm_transformation, reconstruction.get_proj_srs()) else: # Pseudo geo shots = get_geojson_shots_from_opensfm( tree.opensfm_reconstruction, pseudo_geotiff=tree.odm_orthophoto_tif) if shots: with open(shots_geojson, "w") as fout: fout.write(json.dumps(shots)) log.ODM_INFO("Wrote %s" % shots_geojson) else: log.ODM_WARNING("Cannot extract shots") else: log.ODM_WARNING('Found a valid shots file in: %s' % shots_geojson) if args.skip_report: # Stop right here log.ODM_WARNING("Skipping report generation as requested") return # Augment OpenSfM stats file with our own stats odm_stats_json = os.path.join(tree.odm_report, "stats.json") octx = OSFMContext(tree.opensfm) osfm_stats_json = octx.path("stats", "stats.json") odm_stats = None point_cloud_file = None views_dimension = None if not os.path.exists(odm_stats_json) or self.rerun(): if os.path.exists(osfm_stats_json): with open(osfm_stats_json, 'r') as f: odm_stats = json.loads(f.read()) # Add point cloud stats if os.path.exists(tree.odm_georeferencing_model_laz): point_cloud_file = tree.odm_georeferencing_model_laz views_dimension = "UserData" # pc_info_file should have been generated by cropper pc_info_file = os.path.join( tree.odm_georeferencing, "odm_georeferenced_model.info.json") odm_stats[ 'point_cloud_statistics'] = generate_point_cloud_stats( tree.odm_georeferencing_model_laz, pc_info_file) else: ply_pc = os.path.join(tree.odm_filterpoints, "point_cloud.ply") if os.path.exists(ply_pc): point_cloud_file = ply_pc views_dimension = "views" pc_info_file = os.path.join(tree.odm_filterpoints, "point_cloud.info.json") odm_stats[ 'point_cloud_statistics'] = generate_point_cloud_stats( ply_pc, pc_info_file) else: log.ODM_WARNING("No point cloud found") odm_stats['point_cloud_statistics'][ 'dense'] = not args.fast_orthophoto # Add runtime stats total_time = (system.now_raw() - outputs['start_time']).total_seconds() odm_stats['odm_processing_statistics'] = { 'total_time': total_time, 'total_time_human': hms(total_time), 'average_gsd': gsd.opensfm_reconstruction_average_gsd( octx.recon_file(), use_all_shots=reconstruction.has_gcp()), } with open(odm_stats_json, 'w') as f: f.write(json.dumps(odm_stats)) else: log.ODM_WARNING( "Cannot generate report, OpenSfM stats are missing") else: log.ODM_WARNING("Reading existing stats %s" % odm_stats_json) with open(odm_stats_json, 'r') as f: odm_stats = json.loads(f.read()) # Generate overlap diagram if odm_stats.get('point_cloud_statistics' ) and point_cloud_file and views_dimension: bounds = odm_stats['point_cloud_statistics'].get('stats', {}).get( 'bbox', {}).get('native', {}).get('bbox') if bounds: image_target_size = 1400 # pixels osfm_stats_dir = os.path.join(tree.opensfm, "stats") diagram_tiff = os.path.join(osfm_stats_dir, "overlap.tif") diagram_png = os.path.join(osfm_stats_dir, "overlap.png") width = bounds.get('maxx') - bounds.get('minx') height = bounds.get('maxy') - bounds.get('miny') max_dim = max(width, height) resolution = float(max_dim) / float(image_target_size) radius = resolution * math.sqrt(2) # Larger radius for sparse point cloud diagram if not odm_stats['point_cloud_statistics']['dense']: radius *= 10 system.run("pdal translate -i \"{}\" " "-o \"{}\" " "--writer gdal " "--writers.gdal.resolution={} " "--writers.gdal.data_type=uint8_t " "--writers.gdal.dimension={} " "--writers.gdal.output_type=max " "--writers.gdal.radius={} ".format( point_cloud_file, diagram_tiff, resolution, views_dimension, radius)) report_assets = os.path.abspath( os.path.join(os.path.dirname(__file__), "../opendm/report")) overlap_color_map = os.path.join(report_assets, "overlap_color_map.txt") bounds_file_path = os.path.join( tree.odm_georeferencing, 'odm_georeferenced_model.bounds.gpkg') if args.crop > 0 and os.path.isfile(bounds_file_path): Cropper.crop(bounds_file_path, diagram_tiff, get_orthophoto_vars(args), keep_original=False) system.run( "gdaldem color-relief \"{}\" \"{}\" \"{}\" -of PNG -alpha". format(diagram_tiff, overlap_color_map, diagram_png)) # Copy assets for asset in [ "overlap_diagram_legend.png", "dsm_gradient.png" ]: shutil.copy(os.path.join(report_assets, asset), os.path.join(osfm_stats_dir, asset)) # Generate previews of ortho/dsm if os.path.isfile(tree.odm_orthophoto_tif): osfm_ortho = os.path.join(osfm_stats_dir, "ortho.png") generate_png(tree.odm_orthophoto_tif, osfm_ortho, image_target_size) dems = [] if args.dsm: dems.append("dsm") if args.dtm: dems.append("dtm") for dem in dems: dem_file = tree.path("odm_dem", "%s.tif" % dem) if os.path.isfile(dem_file): # Resize first (faster) resized_dem_file = io.related_file_path( dem_file, postfix=".preview") system.run( "gdal_translate -outsize {} 0 \"{}\" \"{}\" --config GDAL_CACHEMAX {}%" .format(image_target_size, dem_file, resized_dem_file, get_max_memory())) log.ODM_INFO("Computing raster stats for %s" % resized_dem_file) dem_stats = get_raster_stats(resized_dem_file) if len(dem_stats) > 0: odm_stats[dem + '_statistics'] = dem_stats[0] osfm_dem = os.path.join(osfm_stats_dir, "%s.png" % dem) colored_dem, hillshade_dem, colored_hillshade_dem = generate_colored_hillshade( resized_dem_file) system.run( "gdal_translate -outsize {} 0 -of png \"{}\" \"{}\" --config GDAL_CACHEMAX {}%" .format(image_target_size, colored_hillshade_dem, osfm_dem, get_max_memory())) for f in [ resized_dem_file, colored_dem, hillshade_dem, colored_hillshade_dem ]: if os.path.isfile(f): os.remove(f) else: log.ODM_WARNING( "Cannot generate overlap diagram, cannot compute point cloud bounds" ) else: log.ODM_WARNING( "Cannot generate overlap diagram, point cloud stats missing") octx.export_report(os.path.join(tree.odm_report, "report.pdf"), odm_stats, self.rerun())
def process(self, inputs, outputs): # Benchmarking start_time = system.now_raw() log.ODM_INFO('Running ODM Orthophoto Cell') # get inputs args = self.inputs.args tree = self.inputs.tree reconstruction = inputs.reconstruction verbose = '-verbose' if self.params.verbose else '' # define paths and create working directories system.mkdir_p(tree.odm_orthophoto) # check if we rerun cell or not rerun_cell = (args.rerun is not None and args.rerun == 'odm_orthophoto') or \ (args.rerun_all) or \ (args.rerun_from is not None and 'odm_orthophoto' in args.rerun_from) if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell: # odm_orthophoto definitions kwargs = { 'bin': context.odm_modules_path, 'log': tree.odm_orthophoto_log, 'ortho': tree.odm_orthophoto_file, 'corners': tree.odm_orthophoto_corners, 'res': 1.0 / (gsd.cap_resolution(self.params.resolution, tree.opensfm_reconstruction, ignore_gsd=args.ignore_gsd) / 100.0), 'verbose': verbose } # Have geo coordinates? georef = reconstruction.georef # Check if the georef object is initialized # (during a --rerun this might not be) # TODO: we should move this to a more central # location (perhaps during the dataset initialization) if georef and not georef.utm_east_offset: odm_georeferencing_model_txt_geo_file = os.path.join( tree.odm_georeferencing, tree.odm_georeferencing_model_txt_geo) if io.file_exists(odm_georeferencing_model_txt_geo_file): georef.extract_offsets( odm_georeferencing_model_txt_geo_file) else: log.ODM_WARNING( 'Cannot read UTM offset from {}. An orthophoto will not be generated.' .format(odm_georeferencing_model_txt_geo_file)) if georef: if args.use_3dmesh: kwargs['model_geo'] = os.path.join( tree.odm_texturing, tree.odm_georeferencing_model_obj_geo) else: kwargs['model_geo'] = os.path.join( tree.odm_25dtexturing, tree.odm_georeferencing_model_obj_geo) else: if args.use_3dmesh: kwargs['model_geo'] = os.path.join( tree.odm_texturing, tree.odm_textured_model_obj) else: kwargs['model_geo'] = os.path.join( tree.odm_25dtexturing, tree.odm_textured_model_obj) # run odm_orthophoto system.run( '{bin}/odm_orthophoto -inputFile {model_geo} ' '-logFile {log} -outputFile {ortho} -resolution {res} {verbose} ' '-outputCornerFile {corners}'.format(**kwargs)) # Create georeferenced GeoTiff geotiffcreated = False if georef and georef.projection and georef.utm_east_offset and georef.utm_north_offset: ulx = uly = lrx = lry = 0.0 with open(tree.odm_orthophoto_corners) as f: for lineNumber, line in enumerate(f): if lineNumber == 0: tokens = line.split(' ') if len(tokens) == 4: ulx = float(tokens[0]) + \ float(georef.utm_east_offset) lry = float(tokens[1]) + \ float(georef.utm_north_offset) lrx = float(tokens[2]) + \ float(georef.utm_east_offset) uly = float(tokens[3]) + \ float(georef.utm_north_offset) log.ODM_INFO('Creating GeoTIFF') kwargs = { 'ulx': ulx, 'uly': uly, 'lrx': lrx, 'lry': lry, 'tiled': '' if self.params.no_tiled else '-co TILED=yes ', 'compress': self.params.compress, 'predictor': '-co PREDICTOR=2 ' if self.params.compress in ['LZW', 'DEFLATE'] else '', 'proj': georef.projection.srs, 'bigtiff': self.params.bigtiff, 'png': tree.odm_orthophoto_file, 'tiff': tree.odm_orthophoto_tif, 'log': tree.odm_orthophoto_tif_log, 'max_memory': get_max_memory(), 'threads': self.params.max_concurrency } system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} ' '{tiled} ' '-co BIGTIFF={bigtiff} ' '-co COMPRESS={compress} ' '{predictor} ' '-co BLOCKXSIZE=512 ' '-co BLOCKYSIZE=512 ' '-co NUM_THREADS={threads} ' '-a_srs \"{proj}\" ' '--config GDAL_CACHEMAX {max_memory}% ' '{png} {tiff} > {log}'.format(**kwargs)) if args.crop > 0: shapefile_path = os.path.join( tree.odm_georeferencing, 'odm_georeferenced_model.bounds.shp') Cropper.crop( shapefile_path, tree.odm_orthophoto_tif, { 'TILED': 'NO' if self.params.no_tiled else 'YES', 'COMPRESS': self.params.compress, 'PREDICTOR': '2' if self.params.compress in ['LZW', 'DEFLATE'] else '1', 'BIGTIFF': self.params.bigtiff, 'BLOCKXSIZE': 512, 'BLOCKYSIZE': 512, 'NUM_THREADS': self.params.max_concurrency }) if self.params.build_overviews: log.ODM_DEBUG("Building Overviews") kwargs = { 'orthophoto': tree.odm_orthophoto_tif, 'log': tree.odm_orthophoto_gdaladdo_log } # Run gdaladdo system.run( 'gdaladdo -ro -r average ' '--config BIGTIFF_OVERVIEW IF_SAFER ' '--config COMPRESS_OVERVIEW JPEG ' '{orthophoto} 2 4 8 16 > {log}'.format(**kwargs)) geotiffcreated = True if not geotiffcreated: log.ODM_WARNING( 'No geo-referenced orthophoto created due ' 'to missing geo-referencing or corner coordinates.') else: log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file) if args.time: system.benchmark(start_time, tree.benchmarking, 'Orthophoto') log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished') return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT
def process(self, inputs, outputs): # Benchmarking start_time = system.now_raw() log.ODM_INFO('Running ODM Georeferencing Cell') # get inputs args = inputs.args tree = inputs.tree reconstruction = inputs.reconstruction gcpfile = tree.odm_georeferencing_gcp doPointCloudGeo = True transformPointCloud = True verbose = '-verbose' if self.params.verbose else '' geo_ref = reconstruction.georef # check if we rerun cell or not rerun_cell = (args.rerun is not None and args.rerun == 'odm_georeferencing') or \ (args.rerun_all) or \ (args.rerun_from is not None and 'odm_georeferencing' in args.rerun_from) runs = [{ 'georeferencing_dir': tree.odm_georeferencing, 'texturing_dir': tree.odm_texturing, 'model': os.path.join(tree.odm_texturing, tree.odm_textured_model_obj) }] if args.skip_3dmodel: runs = [] if not args.use_3dmesh: # Make sure 2.5D mesh is georeferenced before the 3D mesh # Because it will be used to calculate a transform # for the point cloud. If we use the 3D model transform, # DEMs and orthophoto might not align! runs.insert( 0, { 'georeferencing_dir': tree.odm_25dgeoreferencing, 'texturing_dir': tree.odm_25dtexturing, 'model': os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj) }) for r in runs: odm_georeferencing_model_obj_geo = os.path.join( r['texturing_dir'], tree.odm_georeferencing_model_obj_geo) odm_georeferencing_log = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_log) odm_georeferencing_transform_file = os.path.join( r['georeferencing_dir'], tree.odm_georeferencing_transform_file) odm_georeferencing_model_txt_geo_file = os.path.join( r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo) if not io.file_exists(odm_georeferencing_model_obj_geo) or \ not io.file_exists(tree.odm_georeferencing_model_laz) or rerun_cell: # odm_georeference definitions kwargs = { 'bin': context.odm_modules_path, 'bundle': tree.opensfm_bundle, 'imgs': tree.dataset_raw, 'imgs_list': tree.opensfm_bundle_list, 'model': r['model'], 'log': odm_georeferencing_log, 'input_trans_file': tree.opensfm_transformation, 'transform_file': odm_georeferencing_transform_file, 'coords': tree.odm_georeferencing_coords, 'output_pc_file': tree.odm_georeferencing_model_laz, 'geo_sys': odm_georeferencing_model_txt_geo_file, 'model_geo': odm_georeferencing_model_obj_geo, 'gcp': gcpfile, 'verbose': verbose } if args.fast_orthophoto: kwargs['input_pc_file'] = os.path.join( tree.opensfm, 'reconstruction.ply') elif args.use_opensfm_dense: kwargs['input_pc_file'] = tree.opensfm_model else: kwargs['input_pc_file'] = tree.smvs_model if transformPointCloud: kwargs[ 'pc_params'] = '-inputPointCloudFile {input_pc_file} -outputPointCloudFile {output_pc_file}'.format( **kwargs) if geo_ref and geo_ref.projection and geo_ref.projection.srs: kwargs[ 'pc_params'] += ' -outputPointCloudSrs %s' % pipes.quote( geo_ref.projection.srs) else: log.ODM_WARNING( 'NO SRS: The output point cloud will not have a SRS.' ) else: kwargs['pc_params'] = '' # Check to see if the GCP file exists if not self.params.use_exif and (self.params.gcp_file or tree.odm_georeferencing_gcp): log.ODM_INFO('Found %s' % gcpfile) try: system.run( '{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} ' '-inputFile {model} -outputFile {model_geo} ' '{pc_params} {verbose} ' '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys} -gcpFile {gcp} ' '-outputCoordFile {coords}'.format(**kwargs)) except Exception: log.ODM_EXCEPTION('Georeferencing failed. ') return ecto.QUIT elif io.file_exists( tree.opensfm_transformation) and io.file_exists( tree.odm_georeferencing_coords): log.ODM_INFO( 'Running georeferencing with OpenSfM transformation matrix' ) system.run( '{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} ' '-inputFile {model} -outputFile {model_geo} ' '{pc_params} {verbose} ' '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}' .format(**kwargs)) elif io.file_exists(tree.odm_georeferencing_coords): log.ODM_INFO( 'Running georeferencing with generated coords file.') system.run( '{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} ' '-inputFile {model} -outputFile {model_geo} ' '{pc_params} {verbose} ' '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}' .format(**kwargs)) else: log.ODM_WARNING( 'Georeferencing failed. Make sure your ' 'photos have geotags in the EXIF or you have ' 'provided a GCP file. ') doPointCloudGeo = False # skip the rest of the georeferencing if doPointCloudGeo: # update images metadata geo_ref.extract_offsets( odm_georeferencing_model_txt_geo_file) reconstruction.georef = geo_ref # XYZ point cloud output if args.pc_csv: log.ODM_INFO( "Creating geo-referenced CSV file (XYZ format)") system.run( "pdal translate -i \"{}\" " "-o \"{}\" " "--writers.text.format=csv " "--writers.text.order=\"X,Y,Z\" " "--writers.text.keep_unspecified=false ".format( tree.odm_georeferencing_model_laz, tree.odm_georeferencing_xyz_file)) if args.crop > 0: log.ODM_INFO( "Calculating cropping area and generating bounds shapefile from point cloud" ) cropper = Cropper(tree.odm_georeferencing, 'odm_georeferenced_model') decimation_step = 40 if args.fast_orthophoto or args.use_opensfm_dense else 90 # More aggressive decimation for large datasets if not args.fast_orthophoto: decimation_step *= int( len(reconstruction.photos) / 1000) + 1 cropper.create_bounds_shapefile( tree.odm_georeferencing_model_laz, args.crop, decimation_step=decimation_step, outlier_radius=20 if args.fast_orthophoto else 2) # Do not execute a second time, since # We might be doing georeferencing for # multiple models (3D, 2.5D, ...) doPointCloudGeo = False transformPointCloud = False else: log.ODM_WARNING('Found a valid georeferenced model in: %s' % tree.odm_georeferencing_model_laz) outputs.reconstruction = reconstruction if args.time: system.benchmark(start_time, tree.benchmarking, 'Georeferencing') log.ODM_INFO('Running ODM Georeferencing Cell - Finished') return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
def process(self, inputs, outputs): # find a file in the root directory def find(file, dir): for root, dirs, files in os.walk(dir): return '/'.join((root, file)) if file in files else None # Benchmarking start_time = system.now_raw() log.ODM_INFO('Running ODM Georeferencing Cell') # get inputs args = self.inputs.args tree = self.inputs.tree gcpfile = io.join_paths(tree.root_path, self.params.gcp_file) \ if self.params.gcp_file else find('gcp_list.txt', tree.root_path) doPointCloudGeo = True verbose = '-verbose' if self.params.verbose else '' # define paths and create working directories system.mkdir_p(tree.odm_georeferencing) if args.use_25dmesh: system.mkdir_p(tree.odm_25dgeoreferencing) # in case a gcp file it's not provided, let's try to generate it using # images metadata. Internally calls jhead. log.ODM_DEBUG(self.params.gcp_file) if not self.params.gcp_file: # and \ # not io.file_exists(tree.odm_georeferencing_coords): log.ODM_WARNING('No coordinates file. ' 'Generating coordinates file: %s' % tree.odm_georeferencing_coords) # odm_georeference definitions kwargs = { 'bin': context.odm_modules_path, 'imgs': tree.dataset_raw, 'imgs_list': tree.opensfm_bundle_list, 'coords': tree.odm_georeferencing_coords, 'log': tree.odm_georeferencing_utm_log, 'verbose': verbose } # run UTM extraction binary extract_utm = system.run_and_return( '{bin}/odm_extract_utm -imagesPath {imgs}/ ' '-imageListFile {imgs_list} -outputCoordFile {coords} {verbose} ' '-logFile {log}'.format(**kwargs)) if extract_utm != '': log.ODM_WARNING('Could not generate coordinates file. ' 'Ignore if there is a GCP file. Error: %s' % extract_utm) # check if we rerun cell or not rerun_cell = (args.rerun is not None and args.rerun == 'odm_georeferencing') or \ (args.rerun_all) or \ (args.rerun_from is not None and 'odm_georeferencing' in args.rerun_from) runs = [{ 'georeferencing_dir': tree.odm_georeferencing, 'texturing_dir': tree.odm_texturing, 'model': os.path.join(tree.odm_texturing, tree.odm_textured_model_obj) }] if args.fast_orthophoto: runs = [] if args.use_25dmesh: runs += [{ 'georeferencing_dir': tree.odm_25dgeoreferencing, 'texturing_dir': tree.odm_25dtexturing, 'model': os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj) }] for r in runs: odm_georeferencing_model_obj_geo = os.path.join( r['texturing_dir'], tree.odm_georeferencing_model_obj_geo) odm_georeferencing_model_ply_geo = os.path.join( r['georeferencing_dir'], tree.odm_georeferencing_model_ply_geo) odm_georeferencing_log = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_log) odm_georeferencing_transform_file = os.path.join( r['georeferencing_dir'], tree.odm_georeferencing_transform_file) if not io.file_exists(odm_georeferencing_model_obj_geo) or \ not io.file_exists(odm_georeferencing_model_ply_geo) or rerun_cell: # odm_georeference definitions kwargs = { 'bin': context.odm_modules_path, 'bundle': tree.opensfm_bundle, 'imgs': tree.dataset_raw, 'imgs_list': tree.opensfm_bundle_list, 'model': r['model'], 'log': odm_georeferencing_log, 'transform_file': odm_georeferencing_transform_file, 'coords': tree.odm_georeferencing_coords, 'pc_geo': odm_georeferencing_model_ply_geo, 'geo_sys': os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo), 'model_geo': odm_georeferencing_model_obj_geo, 'gcp': gcpfile, 'verbose': verbose } if not args.use_pmvs: if args.fast_orthophoto: kwargs['pc'] = os.path.join(tree.opensfm, 'reconstruction.ply') else: kwargs['pc'] = tree.opensfm_model else: kwargs['pc'] = tree.pmvs_model # Check to see if the GCP file exists if not self.params.use_exif and (self.params.gcp_file or find( 'gcp_list.txt', tree.root_path)): log.ODM_INFO('Found %s' % gcpfile) try: system.run( '{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} ' '-inputFile {model} -outputFile {model_geo} ' '-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} ' '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys} -gcpFile {gcp} ' '-outputCoordFile {coords}'.format(**kwargs)) except Exception: log.ODM_EXCEPTION('Georeferencing failed. ') return ecto.QUIT elif io.file_exists(tree.odm_georeferencing_coords): log.ODM_INFO( 'Running georeferencing with generated coords file.') system.run( '{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} ' '-inputFile {model} -outputFile {model_geo} ' '-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} ' '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}' .format(**kwargs)) else: log.ODM_WARNING( 'Georeferencing failed. Make sure your ' 'photos have geotags in the EXIF or you have ' 'provided a GCP file. ') doPointCloudGeo = False # skip the rest of the georeferencing if doPointCloudGeo: geo_ref = types.ODM_GeoRef() geo_ref.parse_coordinate_system( tree.odm_georeferencing_coords) # convert ply model to LAS reference system geo_ref.convert_to_las(odm_georeferencing_model_ply_geo, tree.odm_georeferencing_model_las, tree.odm_georeferencing_las_json) # XYZ point cloud output log.ODM_INFO( "Creating geo-referenced CSV file (XYZ format)") with open(tree.odm_georeferencing_xyz_file, "wb") as csvfile: csvfile_writer = csv.writer(csvfile, delimiter=",") reachedpoints = False with open(odm_georeferencing_model_ply_geo) as f: for lineNumber, line in enumerate(f): if reachedpoints: tokens = line.split(" ") csv_line = [ float(tokens[0]) + geo_ref.utm_east_offset, float(tokens[1]) + geo_ref.utm_north_offset, tokens[2] ] csvfile_writer.writerow(csv_line) if line.startswith("end_header"): reachedpoints = True csvfile.close() if args.crop > 0: log.ODM_INFO( "Calculating cropping area and generating bounds shapefile from point cloud" ) cropper = Cropper(tree.odm_georeferencing, 'odm_georeferenced_model') cropper.create_bounds_shapefile( tree.odm_georeferencing_model_las, args.crop) # Do not execute a second time, since # We might be doing georeferencing for # multiple models (3D, 2.5D, ...) doPointCloudGeo = False else: log.ODM_WARNING('Found a valid georeferenced model in: %s' % odm_georeferencing_model_ply_geo) if args.time: system.benchmark(start_time, tree.benchmarking, 'Georeferencing') log.ODM_INFO('Running ODM Georeferencing Cell - Finished') return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
def process(self, inputs, outputs): # Benchmarking start_time = system.now_raw() log.ODM_INFO('Running ODM Orthophoto Cell') # get inputs args = self.inputs.args tree = self.inputs.tree reconstruction = inputs.reconstruction verbose = '-verbose' if self.params.verbose else '' # define paths and create working directories system.mkdir_p(tree.odm_orthophoto) # check if we rerun cell or not rerun_cell = (args.rerun is not None and args.rerun == 'odm_orthophoto') or \ (args.rerun_all) or \ (args.rerun_from is not None and 'odm_orthophoto' in args.rerun_from) if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell: # odm_orthophoto definitions kwargs = { 'bin': context.odm_modules_path, 'log': tree.odm_orthophoto_log, 'ortho': tree.odm_orthophoto_file, 'corners': tree.odm_orthophoto_corners, 'res': 1.0 / (gsd.cap_resolution(self.params.resolution, tree.opensfm_reconstruction, ignore_gsd=args.ignore_gsd) / 100.0), 'verbose': verbose } # Have geo coordinates? georef = reconstruction.georef # Check if the georef object is initialized # (during a --rerun this might not be) # TODO: we should move this to a more central # location (perhaps during the dataset initialization) if georef and not georef.utm_east_offset: georeferencing_dir = tree.odm_georeferencing if args.use_3dmesh and not args.skip_3dmodel else tree.odm_25dgeoreferencing odm_georeferencing_model_txt_geo_file = os.path.join(georeferencing_dir, tree.odm_georeferencing_model_txt_geo) if io.file_exists(odm_georeferencing_model_txt_geo_file): georef.extract_offsets(odm_georeferencing_model_txt_geo_file) else: log.ODM_WARNING('Cannot read UTM offset from {}. An orthophoto will not be generated.'.format(odm_georeferencing_model_txt_geo_file)) if georef: if args.use_3dmesh: kwargs['model_geo'] = os.path.join(tree.odm_texturing, tree.odm_georeferencing_model_obj_geo) else: kwargs['model_geo'] = os.path.join(tree.odm_25dtexturing, tree.odm_georeferencing_model_obj_geo) else: if args.use_3dmesh: kwargs['model_geo'] = os.path.join(tree.odm_texturing, tree.odm_textured_model_obj) else: kwargs['model_geo'] = os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj) # run odm_orthophoto system.run('{bin}/odm_orthophoto -inputFile {model_geo} ' '-logFile {log} -outputFile {ortho} -resolution {res} {verbose} ' '-outputCornerFile {corners}'.format(**kwargs)) # Create georeferenced GeoTiff geotiffcreated = False if georef and georef.projection and georef.utm_east_offset and georef.utm_north_offset: ulx = uly = lrx = lry = 0.0 with open(tree.odm_orthophoto_corners) as f: for lineNumber, line in enumerate(f): if lineNumber == 0: tokens = line.split(' ') if len(tokens) == 4: ulx = float(tokens[0]) + \ float(georef.utm_east_offset) lry = float(tokens[1]) + \ float(georef.utm_north_offset) lrx = float(tokens[2]) + \ float(georef.utm_east_offset) uly = float(tokens[3]) + \ float(georef.utm_north_offset) log.ODM_INFO('Creating GeoTIFF') kwargs = { 'ulx': ulx, 'uly': uly, 'lrx': lrx, 'lry': lry, 'tiled': '' if self.params.no_tiled else '-co TILED=yes ', 'compress': self.params.compress, 'predictor': '-co PREDICTOR=2 ' if self.params.compress in ['LZW', 'DEFLATE'] else '', 'proj': georef.projection.srs, 'bigtiff': self.params.bigtiff, 'png': tree.odm_orthophoto_file, 'tiff': tree.odm_orthophoto_tif, 'log': tree.odm_orthophoto_tif_log, 'max_memory': get_max_memory(), 'threads': self.params.max_concurrency } system.run('gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} ' '{tiled} ' '-co BIGTIFF={bigtiff} ' '-co COMPRESS={compress} ' '{predictor} ' '-co BLOCKXSIZE=512 ' '-co BLOCKYSIZE=512 ' '-co NUM_THREADS={threads} ' '-a_srs \"{proj}\" ' '--config GDAL_CACHEMAX {max_memory}% ' '{png} {tiff} > {log}'.format(**kwargs)) if args.crop > 0: shapefile_path = os.path.join(tree.odm_georeferencing, 'odm_georeferenced_model.bounds.shp') Cropper.crop(shapefile_path, tree.odm_orthophoto_tif, { 'TILED': 'NO' if self.params.no_tiled else 'YES', 'COMPRESS': self.params.compress, 'PREDICTOR': '2' if self.params.compress in ['LZW', 'DEFLATE'] else '1', 'BIGTIFF': self.params.bigtiff, 'BLOCKXSIZE': 512, 'BLOCKYSIZE': 512, 'NUM_THREADS': self.params.max_concurrency }) if self.params.build_overviews: log.ODM_DEBUG("Building Overviews") kwargs = { 'orthophoto': tree.odm_orthophoto_tif, 'log': tree.odm_orthophoto_gdaladdo_log } # Run gdaladdo system.run('gdaladdo -ro -r average ' '--config BIGTIFF_OVERVIEW IF_SAFER ' '--config COMPRESS_OVERVIEW JPEG ' '{orthophoto} 2 4 8 16 > {log}'.format(**kwargs)) geotiffcreated = True if not geotiffcreated: log.ODM_WARNING('No geo-referenced orthophoto created due ' 'to missing geo-referencing or corner coordinates.') else: log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file) if args.time: system.benchmark(start_time, tree.benchmarking, 'Orthophoto') log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished') return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT
def process(args, tree, reconstruction, current_path): odm_orthophoto = io.join_paths(current_path, 'orthophoto') odm_orthophoto_path = odm_orthophoto odm_orthophoto_render = io.join_paths(odm_orthophoto_path, 'odm_orthophoto_render.tif') odm_orthophoto_tif = io.join_paths(odm_orthophoto_path, 'odm_orthophoto.tif') odm_orthophoto_corners = io.join_paths(odm_orthophoto_path, 'odm_orthophoto_corners.tif') odm_orthophoto_log = io.join_paths(odm_orthophoto_path, 'odm_orthophoto_log.tif') odm_orthophoto_tif_log = io.join_paths(odm_orthophoto_path, 'gdal_translate_log.txt') odm_25dgeoreferencing = io.join_paths(current_path, 'odm_georeferencing') odm_georeferencing = io.join_paths(current_path, 'odm_georeferencing') odm_georeferencing_coords = io.join_paths( odm_georeferencing, 'coords.txt') odm_georeferencing_gcp = io.find('gcp_list.txt', current_path) odm_georeferencing_gcp_utm = io.join_paths(odm_georeferencing, 'gcp_list_utm.txt') odm_georeferencing_utm_log = io.join_paths( odm_georeferencing, 'odm_georeferencing_utm_log.txt') odm_georeferencing_log = 'odm_georeferencing_log.txt' odm_georeferencing_transform_file = 'odm_georeferencing_transform.txt' odm_georeferencing_proj = 'proj.txt' odm_georeferencing_model_txt_geo = 'odm_georeferencing_model_geo.txt' odm_georeferencing_model_obj_geo = 'odm_textured_model_geo.obj' odm_georeferencing_xyz_file = io.join_paths( odm_georeferencing, 'odm_georeferenced_model.csv') odm_georeferencing_las_json = io.join_paths( odm_georeferencing, 'las.json') odm_georeferencing_model_laz = io.join_paths( odm_georeferencing, 'odm_georeferenced_model.laz') odm_georeferencing_model_las = io.join_paths( odm_georeferencing, 'odm_georeferenced_model.las') odm_georeferencing_dem = io.join_paths( odm_georeferencing, 'odm_georeferencing_model_dem.tif') opensfm_reconstruction = io.join_paths(current_path,'reconstruction.json') odm_texturing = io.join_paths(current_path,'mvs') odm_textured_model_obj = io.join_paths(odm_texturing, 'odm_textured_model.obj') images_dir = io.join_paths(current_path, 'images') tree = tree opensfm_bundle = os.path.join(current_path, 'opensfm', 'bundle_r000.out') opensfm_bundle_list = os.path.join(current_path, 'opensfm', 'list_r000.out') opensfm_transformation = os.path.join(current_path, 'opensfm', 'geocoords_transformation.txt') filtered_point_cloud = os.path.join(current_path, 'filterpoints', 'point_cloud.ply') doPointCloudGeo = True transformPointCloud = True verbose ='' class nonloc: runs = [] def add_run(primary=True, band=None): subdir = "" if not primary and band is not None: subdir = band # Make sure 2.5D mesh is georeferenced before the 3D mesh # Because it will be used to calculate a transform # for the point cloud. If we use the 3D model transform, # DEMs and orthophoto might not align! b = True if b: nonloc.runs += [{ 'georeferencing_dir': os.path.join(odm_georeferencing, subdir), 'texturing_dir': os.path.join(odm_texturing, subdir), }] if not args.skip_3dmodel and (primary or args.use_3dmesh): nonloc.runs += [{ 'georeferencing_dir': odm_georeferencing, 'texturing_dir': os.path.join(odm_texturing, subdir), }] if reconstruction.multi_camera: for band in reconstruction.multi_camera: primary = band == reconstruction.multi_camera[0] add_run(primary, band['name'].lower()) else: add_run() #progress_per_run = 100.0 / len(nonloc.runs) #progress = 0.0 for r in nonloc.runs: if not io.dir_exists(r['georeferencing_dir']): system.mkdir_p(r['georeferencing_dir']) odm_georeferencing_model_obj_geo = os.path.join(r['texturing_dir'], odm_georeferencing_model_obj_geo) odm_georeferencing_model_obj = os.path.join(r['texturing_dir'], odm_textured_model_obj) odm_georeferencing_log = os.path.join(r['georeferencing_dir'], odm_georeferencing_log) odm_georeferencing_transform_file = os.path.join(r['georeferencing_dir'], odm_georeferencing_transform_file) odm_georeferencing_model_txt_geo_file = os.path.join(r['georeferencing_dir'], odm_georeferencing_model_txt_geo) pio = True if pio: #if not io.file_exists(odm_georeferencing_model_obj_geo) or \ #not io.file_exists(odm_georeferencing_model_laz) # odm_georeference definitions kwargs = { 'bin': context.odm_modules_path, 'input_pc_file': filtered_point_cloud, 'bundle': opensfm_bundle, 'imgs': images_dir, 'imgs_list': opensfm_bundle_list, 'model': odm_georeferencing_model_obj, 'log': odm_georeferencing_log, 'input_trans_file': opensfm_transformation, 'transform_file': odm_georeferencing_transform_file, 'coords': odm_georeferencing_coords, 'output_pc_file': odm_georeferencing_model_laz, 'geo_sys': odm_georeferencing_model_txt_geo_file, 'model_geo': odm_georeferencing_model_obj_geo, 'verbose': verbose } if transformPointCloud: kwargs['pc_params'] = '-inputPointCloudFile {input_pc_file} -outputPointCloudFile {output_pc_file}'.format(**kwargs) if reconstruction.is_georeferenced(): kwargs['pc_params'] += ' -outputPointCloudSrs %s' % pipes.quote(reconstruction.georef.proj4()) else: log.ODM_WARNING('NO SRS: The output point cloud will not have a SRS.') else: kwargs['pc_params'] = '' if io.file_exists(opensfm_transformation) and io.file_exists(odm_georeferencing_coords): log.ODM_INFO('Running georeferencing with OpenSfM transformation matrix') system.run('{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} ' '-inputFile {model} -outputFile {model_geo} ' '{pc_params} {verbose} ' '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs)) elif io.file_exists(odm_georeferencing_coords): print('running georeferencing') log.ODM_INFO('Running georeferencing with generated coords file.') system.run('{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} ' '-inputFile {model} -outputFile {model_geo} ' '{pc_params} {verbose} ' '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs)) else: log.ODM_WARNING('Georeferencing failed. Make sure your ' 'photos have geotags in the EXIF or you have ' 'provided a GCP file. ') doPointCloudGeo = False # skip the rest of the georeferencing if doPointCloudGeo: reconstruction.georef.extract_offsets(odm_georeferencing_model_txt_geo_file) point_cloud.post_point_cloud_steps(args, tree) if args.crop > 0: log.ODM_INFO("Calculating cropping area and generating bounds shapefile from point cloud") cropper = Cropper(odm_georeferencing, 'odm_georeferenced_model') decimation_step = 40 if args.fast_orthophoto or args.use_opensfm_dense else 90 # More aggressive decimation for large datasets if not args.fast_orthophoto: decimation_step *= int(len(reconstruction.photos) / 1000) + 1 cropper.create_bounds_gpkg(odm_georeferencing_model_laz, args.crop, decimation_step=decimation_step) # Do not execute a second time, since # We might be doing georeferencing for # multiple models (3D, 2.5D, ...) doPointCloudGeo = False transformPointCloud = False else: log.ODM_WARNING('Found a valid georeferenced model in: %s' % odm_georeferencing_model_laz) if args.optimize_disk_space and io.file_exists(odm_georeferencing_model_laz) and io.file_exists(filtered_point_cloud): os.remove(filtered_point_cloud)
def process(self, inputs, outputs): # Benchmarking start_time = system.now_raw() log.ODM_INFO('Running ODM Georeferencing Cell') # get inputs args = inputs.args tree = inputs.tree reconstruction = inputs.reconstruction gcpfile = tree.odm_georeferencing_gcp doPointCloudGeo = True verbose = '-verbose' if self.params.verbose else '' # check if we rerun cell or not rerun_cell = (args.rerun is not None and args.rerun == 'odm_georeferencing') or \ (args.rerun_all) or \ (args.rerun_from is not None and 'odm_georeferencing' in args.rerun_from) runs = [{ 'georeferencing_dir': tree.odm_georeferencing, 'texturing_dir': tree.odm_texturing, 'model': os.path.join(tree.odm_texturing, tree.odm_textured_model_obj) }] if args.fast_orthophoto: runs = [] if args.use_25dmesh: runs += [{ 'georeferencing_dir': tree.odm_25dgeoreferencing, 'texturing_dir': tree.odm_25dtexturing, 'model': os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj) }] for r in runs: odm_georeferencing_model_obj_geo = os.path.join(r['texturing_dir'], tree.odm_georeferencing_model_obj_geo) odm_georeferencing_model_ply_geo = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_model_ply_geo) odm_georeferencing_log = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_log) odm_georeferencing_transform_file = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_transform_file) if not io.file_exists(odm_georeferencing_model_obj_geo) or \ not io.file_exists(odm_georeferencing_model_ply_geo) or rerun_cell: # odm_georeference definitions kwargs = { 'bin': context.odm_modules_path, 'bundle': tree.opensfm_bundle, 'imgs': tree.dataset_raw, 'imgs_list': tree.opensfm_bundle_list, 'model': r['model'], 'log': odm_georeferencing_log, 'input_trans_file': tree.opensfm_transformation, 'transform_file': odm_georeferencing_transform_file, 'coords': tree.odm_georeferencing_coords, 'pc_geo': odm_georeferencing_model_ply_geo, 'geo_sys': os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo), 'model_geo': odm_georeferencing_model_obj_geo, 'gcp': gcpfile, 'verbose': verbose } if not args.use_pmvs: if args.fast_orthophoto: kwargs['pc'] = os.path.join(tree.opensfm, 'reconstruction.ply') else: kwargs['pc'] = tree.opensfm_model else: kwargs['pc'] = tree.pmvs_model # Check to see if the GCP file exists #if not self.params.use_exif and (self.params.gcp_file or tree.odm_georeferencing_gcp): # log.ODM_INFO('Found %s' % gcpfile) # try: # system.run('{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} ' # '-inputFile {model} -outputFile {model_geo} ' # '-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} ' # '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys} -gcpFile {gcp} ' # '-outputCoordFile {coords}'.format(**kwargs)) # except Exception: # log.ODM_EXCEPTION('Georeferencing failed. ') # return ecto.QUIT if io.file_exists(tree.opensfm_transformation): log.ODM_INFO('Running georeferencing with OpenSfM transformation matrix') system.run('{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} ' '-inputFile {model} -outputFile {model_geo} ' '-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} ' '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs)) elif io.file_exists(tree.odm_georeferencing_coords): log.ODM_INFO('Running georeferencing with generated coords file.') system.run('{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} ' '-inputFile {model} -outputFile {model_geo} ' '-inputPointCloudFile {pc} -outputPointCloudFile {pc_geo} {verbose} ' '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs)) else: log.ODM_WARNING('Georeferencing failed. Make sure your ' 'photos have geotags in the EXIF or you have ' 'provided a GCP file. ') doPointCloudGeo = False # skip the rest of the georeferencing odm_georeferencing_model_ply_geo = os.path.join(tree.odm_georeferencing, tree.odm_georeferencing_model_ply_geo) if doPointCloudGeo: # update images metadata geo_ref = reconstruction.georef geo_ref.parse_transformation_matrix(tree.opensfm_transformation) # convert ply model to LAS reference system geo_ref.convert_to_las(odm_georeferencing_model_ply_geo, tree.odm_georeferencing_model_las, tree.odm_georeferencing_las_json) reconstruction.georef = geo_ref # XYZ point cloud output log.ODM_INFO("Creating geo-referenced CSV file (XYZ format)") with open(tree.odm_georeferencing_xyz_file, "wb") as csvfile: csvfile_writer = csv.writer(csvfile, delimiter=",") reachedpoints = False with open(odm_georeferencing_model_ply_geo) as f: for lineNumber, line in enumerate(f): if reachedpoints: tokens = line.split(" ") csv_line = [float(tokens[0]), float(tokens[1]), tokens[2]] csvfile_writer.writerow(csv_line) if line.startswith("end_header"): reachedpoints = True csvfile.close() if args.crop > 0: log.ODM_INFO("Calculating cropping area and generating bounds shapefile from point cloud") cropper = Cropper(tree.odm_georeferencing, 'odm_georeferenced_model') cropper.create_bounds_shapefile(tree.odm_georeferencing_model_las, args.crop) # Do not execute a second time, since # We might be doing georeferencing for # multiple models (3D, 2.5D, ...) doPointCloudGeo = False else: log.ODM_WARNING('Found a valid georeferenced model in: %s' % odm_georeferencing_model_ply_geo) outputs.reconstruction = reconstruction if args.time: system.benchmark(start_time, tree.benchmarking, 'Georeferencing') log.ODM_INFO('Running ODM Georeferencing Cell - Finished') return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT
def process(self, inputs, outputs): # Benchmarking start_time = system.now_raw() log.ODM_INFO('Running ODM DEM Cell') # get inputs args = self.inputs.args tree = self.inputs.tree las_model_found = io.file_exists(tree.odm_georeferencing_model_laz) # check if we rerun cell or not rerun_cell = (args.rerun is not None and args.rerun == 'odm_dem') or \ (args.rerun_all) or \ (args.rerun_from is not None and 'odm_dem' in args.rerun_from) log.ODM_INFO('Classify: ' + str(args.pc_classify)) log.ODM_INFO('Create DSM: ' + str(args.dsm)) log.ODM_INFO('Create DTM: ' + str(args.dtm)) log.ODM_INFO('DEM input file {0} found: {1}'.format( tree.odm_georeferencing_model_laz, str(las_model_found))) slope, cellsize = (0.15, 1) # define paths and create working directories odm_dem_root = tree.path('odm_dem') if not io.dir_exists(odm_dem_root): system.mkdir_p(odm_dem_root) if args.pc_classify and las_model_found: pc_classify_marker = os.path.join(odm_dem_root, 'pc_classify_done.txt') if not io.file_exists(pc_classify_marker) or rerun_cell: log.ODM_INFO( "Classifying {} using Simple Morphological Filter".format( tree.odm_georeferencing_model_laz)) commands.classify(tree.odm_georeferencing_model_laz, slope, cellsize, verbose=args.verbose) with open(pc_classify_marker, 'w') as f: f.write('Classify: smrf\n') f.write('Slope: {}\n'.format(slope)) f.write('Cellsize: {}\n'.format(cellsize)) # Do we need to process anything here? if (args.dsm or args.dtm) and las_model_found: dsm_output_filename = os.path.join(odm_dem_root, 'dsm.tif') dtm_output_filename = os.path.join(odm_dem_root, 'dtm.tif') if (args.dtm and not io.file_exists(dtm_output_filename)) or \ (args.dsm and not io.file_exists(dsm_output_filename)) or \ rerun_cell: products = [] if args.dsm: products.append('dsm') if args.dtm: products.append('dtm') resolution = gsd.cap_resolution(args.dem_resolution, tree.opensfm_reconstruction, gsd_error_estimate=-3, ignore_gsd=args.ignore_gsd) radius_steps = [(resolution / 100.0) / 2.0] for _ in range(args.dem_gapfill_steps - 1): radius_steps.append( radius_steps[-1] * 2) # 2 is arbitrary, maybe there's a better value? for product in products: commands.create_dems( [tree.odm_georeferencing_model_laz], product, radius=map(str, radius_steps), gapfill=True, outdir=odm_dem_root, resolution=resolution / 100.0, decimation=args.dem_decimation, verbose=args.verbose, max_workers=get_max_concurrency_for_dem( args.max_concurrency, tree.odm_georeferencing_model_laz)) if args.crop > 0: bounds_shapefile_path = os.path.join( tree.odm_georeferencing, 'odm_georeferenced_model.bounds.shp') if os.path.exists(bounds_shapefile_path): Cropper.crop( bounds_shapefile_path, os.path.join(odm_dem_root, "{}.tif".format(product)), { 'TILED': 'YES', 'COMPRESS': 'LZW', 'BLOCKXSIZE': 512, 'BLOCKYSIZE': 512, 'NUM_THREADS': self.params.max_concurrency }) else: log.ODM_WARNING('Found existing outputs in: %s' % odm_dem_root) else: log.ODM_WARNING('DEM will not be generated') if args.time: system.benchmark(start_time, tree.benchmarking, 'Dem') log.ODM_INFO('Running ODM DEM Cell - Finished') return ecto.OK if args.end_with != 'odm_dem' else ecto.QUIT
def process(self, args, outputs): tree = outputs['tree'] reconstruction = outputs['reconstruction'] if not io.file_exists( tree.odm_georeferencing_model_laz) or self.rerun(): cmd = ( 'pdal translate -i "%s" -o \"%s\"' % (tree.filtered_point_cloud, tree.odm_georeferencing_model_laz)) stages = ["ferry"] params = [ '--filters.ferry.dimensions="views => UserData"', '--writers.las.compression="lazip"', ] if reconstruction.is_georeferenced(): log.ODM_INFO("Georeferencing point cloud") stages.append("transformation") params += [ '--filters.transformation.matrix="1 0 0 %s 0 1 0 %s 0 0 1 0 0 0 0 1"' % reconstruction.georef.utm_offset(), '--writers.las.offset_x=%s' % reconstruction.georef.utm_east_offset, '--writers.las.offset_y=%s' % reconstruction.georef.utm_north_offset, '--writers.las.offset_z=0', '--writers.las.a_srs="%s"' % reconstruction.georef.proj4() ] system.run(cmd + ' ' + ' '.join(stages) + ' ' + ' '.join(params)) self.update_progress(50) if args.crop > 0: log.ODM_INFO( "Calculating cropping area and generating bounds shapefile from point cloud" ) cropper = Cropper(tree.odm_georeferencing, 'odm_georeferenced_model') if args.fast_orthophoto: decimation_step = 10 else: decimation_step = 40 # More aggressive decimation for large datasets if not args.fast_orthophoto: decimation_step *= int( len(reconstruction.photos) / 1000) + 1 decimation_step = min(decimation_step, 95) try: cropper.create_bounds_gpkg( tree.odm_georeferencing_model_laz, args.crop, decimation_step=decimation_step) except: log.ODM_WARNING( "Cannot calculate crop bounds! We will skip cropping" ) args.crop = 0 else: log.ODM_INFO("Converting point cloud (non-georeferenced)") system.run(cmd + ' ' + ' '.join(stages) + ' ' + ' '.join(params)) point_cloud.post_point_cloud_steps(args, tree) else: log.ODM_WARNING('Found a valid georeferenced model in: %s' % tree.odm_georeferencing_model_laz) if args.optimize_disk_space and io.file_exists( tree.odm_georeferencing_model_laz) and io.file_exists( tree.filtered_point_cloud): os.remove(tree.filtered_point_cloud)
def process(self, inputs, outputs): # Benchmarking start_time = system.now_raw() log.ODM_INFO('Running ODM Orthophoto Cell') # get inputs args = self.inputs.args tree = self.inputs.tree verbose = '-verbose' if self.params.verbose else '' # define paths and create working directories system.mkdir_p(tree.odm_orthophoto) # check if we rerun cell or not rerun_cell = (args.rerun is not None and args.rerun == 'odm_orthophoto') or \ (args.rerun_all) or \ (args.rerun_from is not None and 'odm_orthophoto' in args.rerun_from) if not io.file_exists(tree.odm_orthophoto_file) or rerun_cell: # odm_orthophoto definitions kwargs = { 'bin': context.odm_modules_path, 'log': tree.odm_orthophoto_log, 'ortho': tree.odm_orthophoto_file, 'corners': tree.odm_orthophoto_corners, 'res': self.params.resolution, 'verbose': verbose } # Have geo coordinates? if io.file_exists(tree.odm_georeferencing_coords): if args.use_25dmesh: kwargs['model_geo'] = os.path.join( tree.odm_25dtexturing, tree.odm_georeferencing_model_obj_geo) else: kwargs['model_geo'] = os.path.join( tree.odm_texturing, tree.odm_georeferencing_model_obj_geo) else: if args.use_25dmesh: kwargs['model_geo'] = os.path.join( tree.odm_25dtexturing, tree.odm_textured_model_obj) else: kwargs['model_geo'] = os.path.join( tree.odm_texturing, tree.odm_textured_model_obj) # run odm_orthophoto system.run( '{bin}/odm_orthophoto -inputFile {model_geo} ' '-logFile {log} -outputFile {ortho} -resolution {res} {verbose} ' '-outputCornerFile {corners}'.format(**kwargs)) if not io.file_exists(tree.odm_georeferencing_coords): log.ODM_WARNING('No coordinates file. A georeferenced raster ' 'will not be created') else: # Create georeferenced GeoTiff geotiffcreated = False georef = types.ODM_GeoRef() # creates the coord refs # TODO I don't want to have to do this twice- after odm_georef georef.parse_coordinate_system(tree.odm_georeferencing_coords) if georef.epsg and georef.utm_east_offset and georef.utm_north_offset: ulx = uly = lrx = lry = 0.0 with open(tree.odm_orthophoto_corners) as f: for lineNumber, line in enumerate(f): if lineNumber == 0: tokens = line.split(' ') if len(tokens) == 4: ulx = float(tokens[0]) + \ float(georef.utm_east_offset) lry = float(tokens[1]) + \ float(georef.utm_north_offset) lrx = float(tokens[2]) + \ float(georef.utm_east_offset) uly = float(tokens[3]) + \ float(georef.utm_north_offset) log.ODM_INFO('Creating GeoTIFF') kwargs = { 'ulx': ulx, 'uly': uly, 'lrx': lrx, 'lry': lry, 'tiled': '' if self.params.no_tiled else '-co TILED=yes ', 'compress': self.params.compress, 'predictor': '-co PREDICTOR=2 ' if self.params.compress in ['LZW', 'DEFLATE'] else '', 'epsg': georef.epsg, 't_srs': self.params.t_srs or "EPSG:{0}".format(georef.epsg), 'bigtiff': self.params.bigtiff, 'png': tree.odm_orthophoto_file, 'tiff': tree.odm_orthophoto_tif, 'log': tree.odm_orthophoto_tif_log } system.run( 'gdal_translate -a_ullr {ulx} {uly} {lrx} {lry} ' '{tiled} ' '-co BIGTIFF={bigtiff} ' '-co COMPRESS={compress} ' '{predictor} ' '-co BLOCKXSIZE=512 ' '-co BLOCKYSIZE=512 ' '-co NUM_THREADS=ALL_CPUS ' '-a_srs \"EPSG:{epsg}\" ' '{png} {tiff} > {log}'.format(**kwargs)) if args.crop > 0: shapefile_path = os.path.join( tree.odm_georeferencing, 'odm_georeferenced_model.bounds.shp') Cropper.crop( shapefile_path, tree.odm_orthophoto_tif, { 'TILED': 'NO' if self.params.no_tiled else 'YES', 'COMPRESS': self.params.compress, 'PREDICTOR': '2' if self.params.compress in ['LZW', 'DEFLATE'] else '1', 'BIGTIFF': self.params.bigtiff, 'BLOCKXSIZE': 512, 'BLOCKYSIZE': 512, 'NUM_THREADS': 'ALL_CPUS' }) if self.params.build_overviews: log.ODM_DEBUG("Building Overviews") kwargs = { 'orthophoto': tree.odm_orthophoto_tif, 'log': tree.odm_orthophoto_gdaladdo_log } # Run gdaladdo system.run( 'gdaladdo -ro -r average ' '--config BIGTIFF_OVERVIEW IF_SAFER ' '--config COMPRESS_OVERVIEW JPEG ' '{orthophoto} 2 4 8 16 > {log}'.format(**kwargs)) geotiffcreated = True if not geotiffcreated: log.ODM_WARNING( 'No geo-referenced orthophoto created due ' 'to missing geo-referencing or corner coordinates.') else: log.ODM_WARNING('Found a valid orthophoto in: %s' % tree.odm_orthophoto_file) if args.time: system.benchmark(start_time, tree.benchmarking, 'Orthophoto') log.ODM_INFO('Running ODM OrthoPhoto Cell - Finished') return ecto.OK if args.end_with != 'odm_orthophoto' else ecto.QUIT
def process(self, inputs, outputs): # Benchmarking start_time = system.now_raw() log.ODM_INFO('Running ODM Georeferencing Cell') # get inputs args = inputs.args tree = inputs.tree reconstruction = inputs.reconstruction gcpfile = tree.odm_georeferencing_gcp doPointCloudGeo = True transformPointCloud = True verbose = '-verbose' if self.params.verbose else '' geo_ref = reconstruction.georef # check if we rerun cell or not rerun_cell = (args.rerun is not None and args.rerun == 'odm_georeferencing') or \ (args.rerun_all) or \ (args.rerun_from is not None and 'odm_georeferencing' in args.rerun_from) runs = [{ 'georeferencing_dir': tree.odm_georeferencing, 'texturing_dir': tree.odm_texturing, 'model': os.path.join(tree.odm_texturing, tree.odm_textured_model_obj) }] if args.skip_3dmodel: runs = [] if not args.use_3dmesh: # Make sure 2.5D mesh is georeferenced before the 3D mesh # Because it will be used to calculate a transform # for the point cloud. If we use the 3D model transform, # DEMs and orthophoto might not align! runs.insert(0, { 'georeferencing_dir': tree.odm_25dgeoreferencing, 'texturing_dir': tree.odm_25dtexturing, 'model': os.path.join(tree.odm_25dtexturing, tree.odm_textured_model_obj) }) for r in runs: odm_georeferencing_model_obj_geo = os.path.join(r['texturing_dir'], tree.odm_georeferencing_model_obj_geo) odm_georeferencing_log = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_log) odm_georeferencing_transform_file = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_transform_file) odm_georeferencing_model_txt_geo_file = os.path.join(r['georeferencing_dir'], tree.odm_georeferencing_model_txt_geo) if not io.file_exists(odm_georeferencing_model_obj_geo) or \ not io.file_exists(tree.odm_georeferencing_model_laz) or rerun_cell: # odm_georeference definitions kwargs = { 'bin': context.odm_modules_path, 'input_pc_file': tree.filtered_point_cloud, 'bundle': tree.opensfm_bundle, 'imgs': tree.dataset_raw, 'imgs_list': tree.opensfm_bundle_list, 'model': r['model'], 'log': odm_georeferencing_log, 'input_trans_file': tree.opensfm_transformation, 'transform_file': odm_georeferencing_transform_file, 'coords': tree.odm_georeferencing_coords, 'output_pc_file': tree.odm_georeferencing_model_laz, 'geo_sys': odm_georeferencing_model_txt_geo_file, 'model_geo': odm_georeferencing_model_obj_geo, 'gcp': gcpfile, 'verbose': verbose } if transformPointCloud: kwargs['pc_params'] = '-inputPointCloudFile {input_pc_file} -outputPointCloudFile {output_pc_file}'.format(**kwargs) if geo_ref and geo_ref.projection and geo_ref.projection.srs: kwargs['pc_params'] += ' -outputPointCloudSrs %s' % pipes.quote(geo_ref.projection.srs) else: log.ODM_WARNING('NO SRS: The output point cloud will not have a SRS.') else: kwargs['pc_params'] = '' # Check to see if the GCP file exists if not self.params.use_exif and (self.params.gcp_file or tree.odm_georeferencing_gcp): log.ODM_INFO('Found %s' % gcpfile) try: system.run('{bin}/odm_georef -bundleFile {bundle} -imagesPath {imgs} -imagesListPath {imgs_list} ' '-inputFile {model} -outputFile {model_geo} ' '{pc_params} {verbose} ' '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys} -gcpFile {gcp} ' '-outputCoordFile {coords}'.format(**kwargs)) except Exception: log.ODM_EXCEPTION('Georeferencing failed. ') return ecto.QUIT elif io.file_exists(tree.opensfm_transformation) and io.file_exists(tree.odm_georeferencing_coords): log.ODM_INFO('Running georeferencing with OpenSfM transformation matrix') system.run('{bin}/odm_georef -bundleFile {bundle} -inputTransformFile {input_trans_file} -inputCoordFile {coords} ' '-inputFile {model} -outputFile {model_geo} ' '{pc_params} {verbose} ' '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs)) elif io.file_exists(tree.odm_georeferencing_coords): log.ODM_INFO('Running georeferencing with generated coords file.') system.run('{bin}/odm_georef -bundleFile {bundle} -inputCoordFile {coords} ' '-inputFile {model} -outputFile {model_geo} ' '{pc_params} {verbose} ' '-logFile {log} -outputTransformFile {transform_file} -georefFileOutputPath {geo_sys}'.format(**kwargs)) else: log.ODM_WARNING('Georeferencing failed. Make sure your ' 'photos have geotags in the EXIF or you have ' 'provided a GCP file. ') doPointCloudGeo = False # skip the rest of the georeferencing if doPointCloudGeo: # update images metadata geo_ref.extract_offsets(odm_georeferencing_model_txt_geo_file) reconstruction.georef = geo_ref # XYZ point cloud output if args.pc_csv: log.ODM_INFO("Creating geo-referenced CSV file (XYZ format)") system.run("pdal translate -i \"{}\" " "-o \"{}\" " "--writers.text.format=csv " "--writers.text.order=\"X,Y,Z\" " "--writers.text.keep_unspecified=false ".format( tree.odm_georeferencing_model_laz, tree.odm_georeferencing_xyz_file)) # LAS point cloud output if args.pc_las: log.ODM_INFO("Creating geo-referenced LAS file") system.run("pdal translate -i \"{}\" " "-o \"{}\" ".format( tree.odm_georeferencing_model_laz, tree.odm_georeferencing_model_las)) if args.crop > 0: log.ODM_INFO("Calculating cropping area and generating bounds shapefile from point cloud") cropper = Cropper(tree.odm_georeferencing, 'odm_georeferenced_model') decimation_step = 40 if args.fast_orthophoto or args.use_opensfm_dense else 90 # More aggressive decimation for large datasets if not args.fast_orthophoto: decimation_step *= int(len(reconstruction.photos) / 1000) + 1 cropper.create_bounds_shapefile(tree.odm_georeferencing_model_laz, args.crop, decimation_step=decimation_step, outlier_radius=20 if args.fast_orthophoto else 2) # Do not execute a second time, since # We might be doing georeferencing for # multiple models (3D, 2.5D, ...) doPointCloudGeo = False transformPointCloud = False else: log.ODM_WARNING('Found a valid georeferenced model in: %s' % tree.odm_georeferencing_model_laz) outputs.reconstruction = reconstruction if args.time: system.benchmark(start_time, tree.benchmarking, 'Georeferencing') log.ODM_INFO('Running ODM Georeferencing Cell - Finished') return ecto.OK if args.end_with != 'odm_georeferencing' else ecto.QUIT