def unit_distributed_plyflatten(config): print('Configuration file: ',config) print('Running end2end with distributed plyflatten dsm ...') test_cfg = s2p.read_config_file(config) test_cfg['skip_existing'] = True s2p.main(test_cfg) outdir = test_cfg['out_dir'] computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif')) print('Running plyflatten dsm reference ...') clouds = '\n'.join(glob.glob(os.path.join(outdir, "tiles", "*", "*", "cloud.ply"))) out_dsm = os.path.join(outdir, "dsm_ref.tif") cmd = ['plyflatten', str(test_cfg['dsm_resolution']), out_dsm] if 'utm_bbx' in test_cfg: bbx = test_cfg['utm_bbx'] global_xoff = bbx[0] global_yoff = bbx[3] global_xsize = int(np.ceil((bbx[1]-bbx[0]) / test_cfg['dsm_resolution'])) global_ysize = int(np.ceil((bbx[3]-bbx[2]) / test_cfg['dsm_resolution'])) cmd += ['-srcwin', '"{} {} {} {}"'.format(global_xoff, global_yoff, global_xsize, global_ysize)] run_cmd = "ls %s | %s" % (clouds.replace('\n', ' '), " ".join(cmd)) s2plib.common.run(run_cmd) expected = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm_ref.tif')) end2end_compare_dsm(computed,expected,0,0)
def unit_distributed_plyflatten(config): print('Configuration file: ',config) print('Running end2end with distributed plyflatten dsm ...') test_cfg = s2p.read_config_file(config) test_cfg['skip_existing'] = True s2p.main(test_cfg) outdir = test_cfg['out_dir'] computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif')) print('Running plyflatten dsm reference ...') clouds = '\n'.join(glob.glob(os.path.join(outdir, "tiles", "*", "*", "cloud.ply"))) out_dsm = os.path.join(outdir, "dsm_ref.tif") cmd = ['plyflatten', str(test_cfg['dsm_resolution']), out_dsm] if 'utm_bbx' in test_cfg: bbx = test_cfg['utm_bbx'] global_xoff = bbx[0] global_yoff = bbx[3] global_xsize = int(np.ceil((bbx[1]-bbx[0]) / test_cfg['dsm_resolution'])) global_ysize = int(np.ceil((bbx[3]-bbx[2]) / test_cfg['dsm_resolution'])) cmd += ['-srcwin', '"{} {} {} {}"'.format(global_xoff, global_yoff, global_xsize, global_ysize)] run_cmd = "ls %s | %s" % (clouds.replace('\n', ' '), " ".join(cmd)) s2plib.common.run(run_cmd) expected = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm_ref.tif')) end2end_compare_dsm(computed,expected,0,0)
def end2end(config,ref_dsm,absmean_tol=0.025,percentile_tol=1.): print('Configuration file: ',config) print('Reference DSM:',ref_dsm,os.linesep) test_cfg = s2p.read_config_file(config) s2p.main(test_cfg) outdir = test_cfg['out_dir'] computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif')) expected = s2plib.common.gdal_read_as_array_with_nans(ref_dsm) end2end_compare_dsm(computed,expected,absmean_tol,percentile_tol)
def end2end(config,ref_dsm,absmean_tol=0.025,percentile_tol=1.): print('Configuration file: ',config) print('Reference DSM:',ref_dsm,os.linesep) test_cfg = s2p.read_config_file(config) s2p.main(test_cfg) outdir = test_cfg['out_dir'] computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif')) expected = s2plib.common.gdal_read_as_array_with_nans(ref_dsm) end2end_compare_dsm(computed,expected,absmean_tol,percentile_tol)
def end2end_mosaic(config,ref_height_map,absmean_tol=0.025,percentile_tol=1.): test_cfg = s2p.read_config_file(config) outdir = test_cfg['out_dir'] s2p.main(test_cfg) tiles_file = os.path.join(outdir,'tiles.txt') global_height_map = os.path.join(outdir,'height_map.tif') s2p_mosaic.main(tiles_file,global_height_map,'pair_1/height_map.tif') computed = s2p.common.gdal_read_as_array_with_nans(global_height_map) expected = s2p.common.gdal_read_as_array_with_nans(ref_height_map) end2end_compare_dsm(computed,expected,absmean_tol,percentile_tol)
def end2end_mosaic(config,ref_height_map,absmean_tol=0.025,percentile_tol=1.): test_cfg = s2p.read_config_file(config) outdir = test_cfg['out_dir'] test_cfg['skip_existing'] = True s2p.main(test_cfg) tiles_file = os.path.join(outdir,'tiles.txt') global_height_map = os.path.join(outdir,'height_map.tif') s2p_mosaic.main(tiles_file,global_height_map,'pair_1/height_map.tif') computed = s2plib.common.gdal_read_as_array_with_nans(global_height_map) expected = s2plib.common.gdal_read_as_array_with_nans(ref_height_map) end2end_compare_dsm(computed,expected,absmean_tol,percentile_tol)
def end2end(config_file, ref_dsm, absmean_tol=0.025, percentile_tol=1.): print('Configuration file: ', config_file) print('Reference DSM:', ref_dsm, os.linesep) # TODO: this is ugly, and will be fixed once we'll have implemented a better # way to control the config parameters if 'out_crs' in s2p.cfg: del s2p.cfg['out_crs'] test_cfg = s2p.read_config_file(config_file) s2p.main(test_cfg) outdir = test_cfg['out_dir'] computed = common.gdal_read_as_array_with_nans( os.path.join(outdir, 'dsm.tif')) expected = common.gdal_read_as_array_with_nans(ref_dsm) compare_dsm(computed, expected, absmean_tol, percentile_tol)
def main(): """ Command line parsing for s2p command line interface. """ parser = argparse.ArgumentParser(description=('S2P: Satellite Stereo ' 'Pipeline')) parser.add_argument('config', metavar='config.json', help=('path to a json file containing the paths to ' 'input and output files and the algorithm ' 'parameters')) args = parser.parse_args() user_cfg = s2p.read_config_file(args.config) s2p.main(user_cfg) # Backup input file for sanity check if not args.config.startswith( os.path.abspath(s2p.cfg['out_dir'] + os.sep)): shutil.copy2(args.config, os.path.join(s2p.cfg['out_dir'], 'config.json.orig'))
def test_distributed_plyflatten(): config_file = data_path('input_triplet/config.json') print('Running end2end with distributed plyflatten dsm ...') test_cfg = s2p.read_config_file(config_file) s2p.main(test_cfg) outdir = test_cfg['out_dir'] computed = common.gdal_read_as_array_with_nans( os.path.join(outdir, 'dsm.tif')) print('Running plyflatten dsm reference ...') clouds_list = glob.glob( os.path.join(outdir, "tiles", "*", "*", "cloud.ply")) out_dsm = os.path.join(outdir, "dsm_ref.tif") res = test_cfg['dsm_resolution'] roi = None if 'utm_bbx' in test_cfg: bbx = test_cfg['utm_bbx'] global_xoff = bbx[0] global_yoff = bbx[3] global_xsize = int( np.ceil((bbx[1] - bbx[0]) / test_cfg['dsm_resolution'])) global_ysize = int( np.ceil((bbx[3] - bbx[2]) / test_cfg['dsm_resolution'])) roi = (global_xoff, global_yoff, global_xsize, global_ysize) raster, profile = s2p.rasterization.plyflatten_from_plyfiles_list( clouds_list, resolution=res, roi=roi) s2p.common.rasterio_write(out_dsm, raster[:, :, 0], profile=profile) expected = common.gdal_read_as_array_with_nans( os.path.join(outdir, 'dsm_ref.tif')) compare_dsm(computed, expected, 0, 0)
def test_distributed_plyflatten(): print('Running end2end with distributed plyflatten dsm ...') test_cfg = s2p.read_config_file(data_path('input_triplet/config.json')) s2p.main(test_cfg) outdir = test_cfg['out_dir'] computed = common.gdal_read_as_array_with_nans( os.path.join(outdir, 'dsm.tif')) print('Running plyflatten dsm reference ...') clouds_list = glob.glob( os.path.join(outdir, "tiles", "*", "*", "cloud.ply")) res = test_cfg['dsm_resolution'] roi = None raster, _ = plyflatten_from_plyfiles_list(clouds_list, resolution=res, roi=roi) expected = raster[:, :, 0] compare_dsm(computed, expected, 0, 0)
def end2end_cluster(config): print('Configuration file: ',config) print('Running end2end in sequential mode to get reference DSM ...') test_cfg = s2p.read_config_file(config) test_cfg['skip_existing'] = True s2p.main(test_cfg) outdir = test_cfg['out_dir'] expected = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif')) print('Running end2end in cluster mode ...') test_cfg_cluster = dict() test_cfg_cluster.update(test_cfg) test_cfg_cluster['out_dir'] = test_cfg_cluster['out_dir'] + "_cluster" test_cfg_cluster['skip_existing'] = True print("Running initialisation step ...") s2p.main(test_cfg_cluster,["initialisation"]) # Retrieve tiles list outdir = test_cfg_cluster['out_dir'] tiles_file = os.path.join(outdir,'tiles.txt') tiles = s2p.read_tiles(tiles_file) print('Found '+str(len(tiles))+' tiles to process') for step in s2p.ALL_STEPS: if s2p.ALL_STEPS[step] is True: print('Running %s on each tile...' % step) for tile in tiles: print('tile : %s' % tile) tile_cfg_cluster = s2p.read_config_file(tile) s2p.main(tile_cfg_cluster, [step]) else: print('Running %s...' % step) print('test_cfg_cluster : %s' % test_cfg_cluster) s2p.main(test_cfg_cluster, [step]) computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif')) end2end_compare_dsm(computed,expected,0,0)
def end2end_cluster(config): print('Configuration file: ',config) print('Running end2end in sequential mode to get reference DSM ...') test_cfg = s2p.read_config_file(config) test_cfg['skip_existing'] = True s2p.main(test_cfg) outdir = test_cfg['out_dir'] expected = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif')) print('Running end2end in cluster mode ...') test_cfg_cluster = dict() test_cfg_cluster.update(test_cfg) test_cfg_cluster['out_dir'] = test_cfg_cluster['out_dir'] + "_cluster" test_cfg_cluster['skip_existing'] = True print("Running initialisation step ...") s2p.main(test_cfg_cluster,["initialisation"]) # Retrieve tiles list outdir = test_cfg_cluster['out_dir'] tiles_file = os.path.join(outdir,'tiles.txt') tiles = s2p.read_tiles(tiles_file) print('Found '+str(len(tiles))+' tiles to process') for step in s2p.ALL_STEPS: if s2p.ALL_STEPS[step] is True: print('Running %s on each tile...' % step) for tile in tiles: print('tile : %s' % tile) tile_cfg_cluster = s2p.read_config_file(tile) s2p.main(tile_cfg_cluster, [step]) else: print('Running %s...' % step) print('test_cfg_cluster : %s' % test_cfg_cluster) s2p.main(test_cfg_cluster, [step]) computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif')) end2end_compare_dsm(computed,expected,0,0)