def unit_distributed_plyflatten(config): print('Configuration file: ',config) print('Running end2end with distributed plyflatten dsm ...') test_cfg = s2p.read_config_file(config) test_cfg['skip_existing'] = True s2p.main(test_cfg) outdir = test_cfg['out_dir'] computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif')) print('Running plyflatten dsm reference ...') clouds = '\n'.join(glob.glob(os.path.join(outdir, "tiles", "*", "*", "cloud.ply"))) out_dsm = os.path.join(outdir, "dsm_ref.tif") cmd = ['plyflatten', str(test_cfg['dsm_resolution']), out_dsm] if 'utm_bbx' in test_cfg: bbx = test_cfg['utm_bbx'] global_xoff = bbx[0] global_yoff = bbx[3] global_xsize = int(np.ceil((bbx[1]-bbx[0]) / test_cfg['dsm_resolution'])) global_ysize = int(np.ceil((bbx[3]-bbx[2]) / test_cfg['dsm_resolution'])) cmd += ['-srcwin', '"{} {} {} {}"'.format(global_xoff, global_yoff, global_xsize, global_ysize)] run_cmd = "ls %s | %s" % (clouds.replace('\n', ' '), " ".join(cmd)) s2plib.common.run(run_cmd) expected = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm_ref.tif')) end2end_compare_dsm(computed,expected,0,0)
def get_coordinates_with_config(tile, m, M): tile_cfg = s2p.read_config_file(os.path.join(tile, "config.json")) x = tile_cfg['roi']['x'] y = tile_cfg['roi']['y'] w = tile_cfg['roi']['w'] h = tile_cfg['roi']['h'] rpcfile = tile_cfg['images'][0]['rpc'] rpc = rpc_model.RPCModel(rpcfile) a = np.array([x, x, x, x, x + w, x + w, x + w, x + w]) b = np.array([y, y, y + h, y + h, y, y, y + h, y + h]) c = np.array([m, M, m, M, m, M, m, M]) lon, lat, __ = rpc.direct_estimate(a, b, c) out = list(common.bounding_box2D(np.vstack([lon, lat]).T)) out[2] += out[0] out[3] += out[1] latlon = [[out[0], out[3], 0], [out[2], out[3], 0], [out[2], out[1], 0], [out[0], out[1], 0], [out[0], out[3], 0]] return latlon
def get_coordinates_with_config(tile, m, M): tile_cfg = s2p.read_config_file(os.path.join(tile, "config.json")) x = tile_cfg['roi']['x'] y = tile_cfg['roi']['y'] w = tile_cfg['roi']['w'] h = tile_cfg['roi']['h'] rpcfile = tile_cfg['images'][0]['rpc'] rpc = rpc_model.RPCModel(rpcfile) a = np.array([x, x, x, x, x+w, x+w, x+w, x+w]) b = np.array([y, y, y+h, y+h, y, y, y+h, y+h]) c = np.array([m, M, m, M, m, M, m, M]) lon, lat, __ = rpc.direct_estimate(a, b, c) out = list(common.bounding_box2D(np.vstack([lon, lat]).T)) out[2] += out[0] out[3] += out[1] latlon = [[out[0], out[3], 0], [out[2], out[3], 0], [out[2], out[1], 0], [out[0], out[1], 0], [out[0], out[3], 0]] return latlon
def test_rpc_path(data, mocks): """ Initialize s2p with `rpc` keys that are paths to text files. The RPCs should be loaded from the text files. """ tmp_config, tmp_path, rpc1, rpc2 = data with open(tmp_config) as f: cfg = json.load(f) rpc1_path = str(tmp_path / "rpc1.txt") rpc1.write_to_file(rpc1_path) cfg["images"][0]["rpc"] = rpc1_path rpc2_path = str(tmp_path / "rpc2.txt") rpc2.write_to_file(rpc2_path) cfg["images"][1]["rpc"] = rpc2_path with open(tmp_config, "w") as f: json.dump(cfg, f) user_cfg = s2p.read_config_file(tmp_config) s2p.initialization.build_cfg(user_cfg) rpcm.rpc_from_geotiff.assert_not_called() rpcm.rpc_from_rpc_file.assert_called() assert rpcm.rpc_from_rpc_file.call_count == 2
def end2end_cluster(config): print('Configuration file: ',config) print('Running end2end in sequential mode to get reference DSM ...') test_cfg = s2p.read_config_file(config) test_cfg['skip_existing'] = True s2p.main(test_cfg) outdir = test_cfg['out_dir'] expected = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif')) print('Running end2end in cluster mode ...') test_cfg_cluster = dict() test_cfg_cluster.update(test_cfg) test_cfg_cluster['out_dir'] = test_cfg_cluster['out_dir'] + "_cluster" test_cfg_cluster['skip_existing'] = True print("Running initialisation step ...") s2p.main(test_cfg_cluster,["initialisation"]) # Retrieve tiles list outdir = test_cfg_cluster['out_dir'] tiles_file = os.path.join(outdir,'tiles.txt') tiles = s2p.read_tiles(tiles_file) print('Found '+str(len(tiles))+' tiles to process') for step in s2p.ALL_STEPS: if s2p.ALL_STEPS[step] is True: print('Running %s on each tile...' % step) for tile in tiles: print('tile : %s' % tile) tile_cfg_cluster = s2p.read_config_file(tile) s2p.main(tile_cfg_cluster, [step]) else: print('Running %s...' % step) print('test_cfg_cluster : %s' % test_cfg_cluster) s2p.main(test_cfg_cluster, [step]) computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif')) end2end_compare_dsm(computed,expected,0,0)
def test_no_rpc(data, mocks): """ Initialize s2p with no `rpc` key. The RPCs should be read from the geotiff tags. """ tmp_config, _, _, _ = data user_cfg = s2p.read_config_file(tmp_config) s2p.initialization.build_cfg(user_cfg) rpcm.rpc_from_geotiff.assert_called() assert rpcm.rpc_from_geotiff.call_count == 2 rpcm.rpc_from_rpc_file.assert_not_called()
def end2end(config,ref_dsm,absmean_tol=0.025,percentile_tol=1.): print('Configuration file: ',config) print('Reference DSM:',ref_dsm,os.linesep) test_cfg = s2p.read_config_file(config) s2p.main(test_cfg) outdir = test_cfg['out_dir'] computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif')) expected = s2plib.common.gdal_read_as_array_with_nans(ref_dsm) end2end_compare_dsm(computed,expected,absmean_tol,percentile_tol)
def end2end_mosaic(config,ref_height_map,absmean_tol=0.025,percentile_tol=1.): test_cfg = s2p.read_config_file(config) outdir = test_cfg['out_dir'] s2p.main(test_cfg) tiles_file = os.path.join(outdir,'tiles.txt') global_height_map = os.path.join(outdir,'height_map.tif') s2p_mosaic.main(tiles_file,global_height_map,'pair_1/height_map.tif') computed = s2p.common.gdal_read_as_array_with_nans(global_height_map) expected = s2p.common.gdal_read_as_array_with_nans(ref_height_map) end2end_compare_dsm(computed,expected,absmean_tol,percentile_tol)
def test_roi_geojson(data): tmp_config, _, _, _ = data user_cfg = s2p.read_config_file(tmp_config) user_cfg["roi_geojson"] = { "coordinates": [[[55.64943405, -21.23207174], [55.65212062, -21.23207174], [55.65212062, -21.23460474], [55.64943405, -21.23460474], [55.64943405, -21.23207174]]], "type": "Polygon" } s2p.initialization.build_cfg(user_cfg) assert user_cfg["roi"] == {'x': 150, 'y': 150, 'w': 700, 'h': 700}
def end2end_mosaic(config,ref_height_map,absmean_tol=0.025,percentile_tol=1.): test_cfg = s2p.read_config_file(config) outdir = test_cfg['out_dir'] test_cfg['skip_existing'] = True s2p.main(test_cfg) tiles_file = os.path.join(outdir,'tiles.txt') global_height_map = os.path.join(outdir,'height_map.tif') s2p_mosaic.main(tiles_file,global_height_map,'pair_1/height_map.tif') computed = s2plib.common.gdal_read_as_array_with_nans(global_height_map) expected = s2plib.common.gdal_read_as_array_with_nans(ref_height_map) end2end_compare_dsm(computed,expected,absmean_tol,percentile_tol)
def end2end(config_file, ref_dsm, absmean_tol=0.025, percentile_tol=1.): print('Configuration file: ', config_file) print('Reference DSM:', ref_dsm, os.linesep) # TODO: this is ugly, and will be fixed once we'll have implemented a better # way to control the config parameters if 'out_crs' in s2p.cfg: del s2p.cfg['out_crs'] test_cfg = s2p.read_config_file(config_file) s2p.main(test_cfg) outdir = test_cfg['out_dir'] computed = common.gdal_read_as_array_with_nans( os.path.join(outdir, 'dsm.tif')) expected = common.gdal_read_as_array_with_nans(ref_dsm) compare_dsm(computed, expected, absmean_tol, percentile_tol)
def test_rpc_dict(data, mocks): """ Initialize s2p with `rpc` keys that are dicts with the RPC contents. The RPCs should be loaded from the dicts. """ tmp_config, _, rpc1, rpc2 = data with open(tmp_config) as f: cfg = json.load(f) cfg["images"][0]["rpc"] = rpc1.__dict__ cfg["images"][1]["rpc"] = rpc2.__dict__ with open(tmp_config, "w") as f: json.dump(cfg, f) user_cfg = s2p.read_config_file(tmp_config) s2p.initialization.build_cfg(user_cfg) rpcm.rpc_from_geotiff.assert_not_called() rpcm.rpc_from_rpc_file.assert_not_called()
def test_disparity_to_ply(tmp_path, out_crs): """ Check that disparity_to_ply() functions correctly when given different out_crs parameters """ # Setup test data tile_dir = str(tmp_path / "tile_dir") shutil.copytree(data_path("input_triangulation"), tile_dir) # Initialize s2p's state config_file = data_path(os.path.join("input_pair", "config.json")) test_cfg = read_config_file(config_file) test_cfg["out_crs"] = out_crs build_cfg(test_cfg) tile = {"coordinates": [500, 150, 350, 350], "dir": tile_dir} disparity_to_ply(tile) _, comments = read_3d_point_cloud_from_ply( os.path.join(tile_dir, "cloud.ply")) expected_crs = out_crs or "epsg:32740" assert comments[-1] == "projection: CRS {}".format(expected_crs)
def main(): """ Command line parsing for s2p command line interface. """ parser = argparse.ArgumentParser(description=('S2P: Satellite Stereo ' 'Pipeline')) parser.add_argument('config', metavar='config.json', help=('path to a json file containing the paths to ' 'input and output files and the algorithm ' 'parameters')) args = parser.parse_args() user_cfg = s2p.read_config_file(args.config) s2p.main(user_cfg) # Backup input file for sanity check if not args.config.startswith( os.path.abspath(s2p.cfg['out_dir'] + os.sep)): shutil.copy2(args.config, os.path.join(s2p.cfg['out_dir'], 'config.json.orig'))
def test_distributed_plyflatten(): config_file = data_path('input_triplet/config.json') print('Running end2end with distributed plyflatten dsm ...') test_cfg = s2p.read_config_file(config_file) s2p.main(test_cfg) outdir = test_cfg['out_dir'] computed = common.gdal_read_as_array_with_nans( os.path.join(outdir, 'dsm.tif')) print('Running plyflatten dsm reference ...') clouds_list = glob.glob( os.path.join(outdir, "tiles", "*", "*", "cloud.ply")) out_dsm = os.path.join(outdir, "dsm_ref.tif") res = test_cfg['dsm_resolution'] roi = None if 'utm_bbx' in test_cfg: bbx = test_cfg['utm_bbx'] global_xoff = bbx[0] global_yoff = bbx[3] global_xsize = int( np.ceil((bbx[1] - bbx[0]) / test_cfg['dsm_resolution'])) global_ysize = int( np.ceil((bbx[3] - bbx[2]) / test_cfg['dsm_resolution'])) roi = (global_xoff, global_yoff, global_xsize, global_ysize) raster, profile = s2p.rasterization.plyflatten_from_plyfiles_list( clouds_list, resolution=res, roi=roi) s2p.common.rasterio_write(out_dsm, raster[:, :, 0], profile=profile) expected = common.gdal_read_as_array_with_nans( os.path.join(outdir, 'dsm_ref.tif')) compare_dsm(computed, expected, 0, 0)
def test_distributed_plyflatten(): print('Running end2end with distributed plyflatten dsm ...') test_cfg = s2p.read_config_file(data_path('input_triplet/config.json')) s2p.main(test_cfg) outdir = test_cfg['out_dir'] computed = common.gdal_read_as_array_with_nans( os.path.join(outdir, 'dsm.tif')) print('Running plyflatten dsm reference ...') clouds_list = glob.glob( os.path.join(outdir, "tiles", "*", "*", "cloud.ply")) res = test_cfg['dsm_resolution'] roi = None raster, _ = plyflatten_from_plyfiles_list(clouds_list, resolution=res, roi=roi) expected = raster[:, :, 0] compare_dsm(computed, expected, 0, 0)
user_cfg: user config dictionary """ common.print_elapsed_time.t0 = datetime.datetime.now() initialization.build_cfg(user_cfg) tw, th = initialization.adjust_tile_size() tiles_txt = os.path.join(cfg['out_dir'],'tiles.txt') tiles = initialization.tiles_full_info(tw, th, tiles_txt) # generate svg tile map write_svg_tilemap(os.path.join(cfg['out_dir'],'tiles.svg'), cfg, tiles) print("\n\n svg tilemap saved in: %s\n"%os.path.join(cfg['out_dir'],'tiles.svg')) # cleanup common.garbage_cleanup() if __name__ == '__main__': parser = argparse.ArgumentParser(description=('generate svg tilemap for S2P')) parser.add_argument('config', metavar='config.json', help=('path to a json file containing the paths to ' 'input and output files and the s2p algorithm ' 'parameters')) args = parser.parse_args() user_cfg = s2p.read_config_file(args.config) main(user_cfg)
Args: user_cfg: user config dictionary """ common.print_elapsed_time.t0 = datetime.datetime.now() initialization.build_cfg(user_cfg) tw, th = initialization.adjust_tile_size() tiles = initialization.tiles_full_info(tw, th) # generate svg tile map write_svg_tilemap(os.path.join(cfg['out_dir'],'tiles.svg'), cfg, tiles) print("\n\n svg tilemap saved in: %s\n"%os.path.join(cfg['out_dir'],'tiles.svg')) # cleanup common.garbage_cleanup() if __name__ == '__main__': parser = argparse.ArgumentParser(description=('generate svg tilemap for S2P')) parser.add_argument('config', metavar='config.json', help=('path to a json file containing the paths to ' 'input and output files and the s2p algorithm ' 'parameters')) args = parser.parse_args() user_cfg = s2p.read_config_file(args.config) main(user_cfg)
def write_tiles_polygon(tile, kml, m=None, M=None, message=None, error_mode=False): dsm = os.path.join(tile, 'dsm.tif') dico = [] green_style = "background-color: #4CAF50; color: white;" red_style = "background-color: #FF4C4C; color: white;" blue_style = "background-color: #9999EB; color:white;" if error_mode is True: latlon = get_coordinates_with_config(tile, m, M) color = simplekml.Color.red head_style = red_style else: latlon = get_coordinates_with_img(dsm) color = simplekml.Color.green head_style = green_style tile_cfg = s2p.read_config_file(os.path.join(tile, "config.json")) x = tile_cfg['roi']['x'] y = tile_cfg['roi']['y'] w = tile_cfg['roi']['w'] h = tile_cfg['roi']['h'] pair_info = [{} for i in range(len(tile_cfg['images']) - 1)] for i in range(len(tile_cfg['images']) - 1): pair_dir = os.path.join(tile, "pair_%d" % (i + 1)) disp = os.path.join(pair_dir, "rectified_disp.tif") if os.path.exists(disp) is False: pair_info[i]['status'] = "failure" if error_mode is False: color = simplekml.Color.blue pair_info[i]['style'] = red_style head_style = blue_style else: pair_info[i]['style'] = None else: pair_info[i]['status'] = "success" pair_info[i]['style'] = None disp_min_max = os.path.join(pair_dir, "disp_min_max.txt") if os.path.exists(disp_min_max) is True: disp_min, disp_max = np.loadtxt(disp_min_max) else: disp_min, disp_max = None, None pair_info[i]['disp_min'] = disp_min pair_info[i]['disp_max'] = disp_max dico += [('roi', { "value": "x : %s - y : %s - w : %s - h : %s" % (x, y, w, h), "style": head_style })] for i in range(len(tile_cfg['images']) - 1): disp_min = pair_info[i]['disp_min'] disp_max = pair_info[i]['disp_max'] status = pair_info[i]['status'] style = pair_info[i]['style'] dico += [('pair_%d' % (i + 1), { "value": ' - '.join([ 'disp_min : %s' % disp_min, 'disp_max : %s' % disp_max, 'status : %s' % status ]), "style": style })] if message != None: dico += [('message', {"value": message, "style": None})] pol = kml.newpolygon(name=tile) pol.outerboundaryis = latlon pol.style.linestyle.color = color pol.style.linestyle.width = 5 pol.style.polystyle.color = simplekml.Color.changealphaint(100, color) dico = collections.OrderedDict(dico) pol.description = get_polygon_description(dico)
def write_tiles_polygon(tile, kml, m=None, M=None, message=None, error_mode=False): dsm = os.path.join(tile, 'dsm.tif') dico = [] green_style = "background-color: #4CAF50; color: white;" red_style = "background-color: #FF4C4C; color: white;" blue_style = "background-color: #9999EB; color:white;" if error_mode is True: latlon = get_coordinates_with_config(tile, m, M) color = simplekml.Color.red head_style = red_style else: latlon = get_coordinates_with_img(dsm) color = simplekml.Color.green head_style = green_style tile_cfg = s2p.read_config_file(os.path.join(tile, "config.json")) x = tile_cfg['roi']['x'] y = tile_cfg['roi']['y'] w = tile_cfg['roi']['w'] h = tile_cfg['roi']['h'] pair_info = [{} for i in range(len(tile_cfg['images'])-1)] for i in range(len(tile_cfg['images'])-1): pair_dir = os.path.join(tile, "pair_%d" % (i+1)) disp = os.path.join(pair_dir, "rectified_disp.tif") if os.path.exists(disp) is False: pair_info[i]['status'] = "failure" if error_mode is False: color = simplekml.Color.blue pair_info[i]['style'] = red_style head_style = blue_style else: pair_info[i]['style'] = None else: pair_info[i]['status'] = "success" pair_info[i]['style'] = None disp_min_max = os.path.join(pair_dir, "disp_min_max.txt") if os.path.exists(disp_min_max) is True: disp_min, disp_max = np.loadtxt(disp_min_max) else: disp_min, disp_max = None, None pair_info[i]['disp_min'] = disp_min pair_info[i]['disp_max'] = disp_max dico += [('roi', {"value" : "x : %s - y : %s - w : %s - h : %s" % (x, y, w, h), "style" : head_style})] for i in range(len(tile_cfg['images'])-1): disp_min = pair_info[i]['disp_min'] disp_max = pair_info[i]['disp_max'] status = pair_info[i]['status'] style = pair_info[i]['style'] dico += [('pair_%d' % (i+1), {"value":' - '.join(['disp_min : %s' % disp_min, 'disp_max : %s' % disp_max, 'status : %s' % status]), "style":style})] if message != None: dico += [('message', {"value": message, "style": None})] pol = kml.newpolygon(name=tile) pol.outerboundaryis = latlon pol.style.linestyle.color = color pol.style.linestyle.width = 5 pol.style.polystyle.color = simplekml.Color.changealphaint(100, color) dico = collections.OrderedDict(dico) pol.description = get_polygon_description(dico)