def unit_distributed_plyflatten(config):
print('Configuration file: ',config)
print('Running end2end with distributed plyflatten dsm ...')
test_cfg = s2p.read_config_file(config)
test_cfg['skip_existing'] = True
s2p.main(test_cfg)
outdir = test_cfg['out_dir']
computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif'))
print('Running plyflatten dsm reference ...')
clouds = '\n'.join(glob.glob(os.path.join(outdir, "tiles", "*", "*", "cloud.ply")))
out_dsm = os.path.join(outdir, "dsm_ref.tif")
cmd = ['plyflatten', str(test_cfg['dsm_resolution']), out_dsm]
if 'utm_bbx' in test_cfg:
bbx = test_cfg['utm_bbx']
global_xoff = bbx[0]
global_yoff = bbx[3]
global_xsize = int(np.ceil((bbx[1]-bbx[0]) / test_cfg['dsm_resolution']))
global_ysize = int(np.ceil((bbx[3]-bbx[2]) / test_cfg['dsm_resolution']))
cmd += ['-srcwin', '"{} {} {} {}"'.format(global_xoff, global_yoff,
global_xsize, global_ysize)]
run_cmd = "ls %s | %s" % (clouds.replace('\n', ' '), " ".join(cmd))
s2plib.common.run(run_cmd)
expected = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm_ref.tif'))
end2end_compare_dsm(computed,expected,0,0)
def unit_distributed_plyflatten(config):
print('Configuration file: ',config)
print('Running end2end with distributed plyflatten dsm ...')
test_cfg = s2p.read_config_file(config)
test_cfg['skip_existing'] = True
s2p.main(test_cfg)
outdir = test_cfg['out_dir']
computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif'))
print('Running plyflatten dsm reference ...')
clouds = '\n'.join(glob.glob(os.path.join(outdir, "tiles", "*", "*", "cloud.ply")))
out_dsm = os.path.join(outdir, "dsm_ref.tif")
cmd = ['plyflatten', str(test_cfg['dsm_resolution']), out_dsm]
if 'utm_bbx' in test_cfg:
bbx = test_cfg['utm_bbx']
global_xoff = bbx[0]
global_yoff = bbx[3]
global_xsize = int(np.ceil((bbx[1]-bbx[0]) / test_cfg['dsm_resolution']))
global_ysize = int(np.ceil((bbx[3]-bbx[2]) / test_cfg['dsm_resolution']))
cmd += ['-srcwin', '"{} {} {} {}"'.format(global_xoff, global_yoff,
global_xsize, global_ysize)]
run_cmd = "ls %s | %s" % (clouds.replace('\n', ' '), " ".join(cmd))
s2plib.common.run(run_cmd)
expected = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm_ref.tif'))
end2end_compare_dsm(computed,expected,0,0)
def get_coordinates_with_config(tile, m, M):
tile_cfg = s2p.read_config_file(os.path.join(tile, "config.json"))
x = tile_cfg['roi']['x']
y = tile_cfg['roi']['y']
w = tile_cfg['roi']['w']
h = tile_cfg['roi']['h']
rpcfile = tile_cfg['images'][0]['rpc']
rpc = rpc_model.RPCModel(rpcfile)
a = np.array([x, x, x, x, x + w, x + w, x + w, x + w])
b = np.array([y, y, y + h, y + h, y, y, y + h, y + h])
c = np.array([m, M, m, M, m, M, m, M])
lon, lat, __ = rpc.direct_estimate(a, b, c)
out = list(common.bounding_box2D(np.vstack([lon, lat]).T))
out[2] += out[0]
out[3] += out[1]
latlon = [[out[0], out[3], 0], [out[2], out[3], 0], [out[2], out[1], 0],
[out[0], out[1], 0], [out[0], out[3], 0]]
return latlon
def get_coordinates_with_config(tile, m, M):
tile_cfg = s2p.read_config_file(os.path.join(tile, "config.json"))
x = tile_cfg['roi']['x']
y = tile_cfg['roi']['y']
w = tile_cfg['roi']['w']
h = tile_cfg['roi']['h']
rpcfile = tile_cfg['images'][0]['rpc']
rpc = rpc_model.RPCModel(rpcfile)
a = np.array([x, x, x, x, x+w, x+w, x+w, x+w])
b = np.array([y, y, y+h, y+h, y, y, y+h, y+h])
c = np.array([m, M, m, M, m, M, m, M])
lon, lat, __ = rpc.direct_estimate(a, b, c)
out = list(common.bounding_box2D(np.vstack([lon, lat]).T))
out[2] += out[0]
out[3] += out[1]
latlon = [[out[0], out[3], 0],
[out[2], out[3], 0],
[out[2], out[1], 0],
[out[0], out[1], 0],
[out[0], out[3], 0]]
return latlon
def test_rpc_path(data, mocks):
"""
Initialize s2p with `rpc` keys that are paths to text files.
The RPCs should be loaded from the text files.
"""
tmp_config, tmp_path, rpc1, rpc2 = data
with open(tmp_config) as f:
cfg = json.load(f)
rpc1_path = str(tmp_path / "rpc1.txt")
rpc1.write_to_file(rpc1_path)
cfg["images"][0]["rpc"] = rpc1_path
rpc2_path = str(tmp_path / "rpc2.txt")
rpc2.write_to_file(rpc2_path)
cfg["images"][1]["rpc"] = rpc2_path
with open(tmp_config, "w") as f:
json.dump(cfg, f)
user_cfg = s2p.read_config_file(tmp_config)
s2p.initialization.build_cfg(user_cfg)
rpcm.rpc_from_geotiff.assert_not_called()
rpcm.rpc_from_rpc_file.assert_called()
assert rpcm.rpc_from_rpc_file.call_count == 2
def end2end_cluster(config):
print('Configuration file: ',config)
print('Running end2end in sequential mode to get reference DSM ...')
test_cfg = s2p.read_config_file(config)
test_cfg['skip_existing'] = True
s2p.main(test_cfg)
outdir = test_cfg['out_dir']
expected = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif'))
print('Running end2end in cluster mode ...')
test_cfg_cluster = dict()
test_cfg_cluster.update(test_cfg)
test_cfg_cluster['out_dir'] = test_cfg_cluster['out_dir'] + "_cluster"
test_cfg_cluster['skip_existing'] = True
print("Running initialisation step ...")
s2p.main(test_cfg_cluster,["initialisation"])
# Retrieve tiles list
outdir = test_cfg_cluster['out_dir']
tiles_file = os.path.join(outdir,'tiles.txt')
tiles = s2p.read_tiles(tiles_file)
print('Found '+str(len(tiles))+' tiles to process')
for step in s2p.ALL_STEPS:
if s2p.ALL_STEPS[step] is True:
print('Running %s on each tile...' % step)
for tile in tiles:
print('tile : %s' % tile)
tile_cfg_cluster = s2p.read_config_file(tile)
s2p.main(tile_cfg_cluster, [step])
else:
print('Running %s...' % step)
print('test_cfg_cluster : %s' % test_cfg_cluster)
s2p.main(test_cfg_cluster, [step])
computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif'))
end2end_compare_dsm(computed,expected,0,0)
def end2end_cluster(config):
print('Configuration file: ',config)
print('Running end2end in sequential mode to get reference DSM ...')
test_cfg = s2p.read_config_file(config)
test_cfg['skip_existing'] = True
s2p.main(test_cfg)
outdir = test_cfg['out_dir']
expected = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif'))
print('Running end2end in cluster mode ...')
test_cfg_cluster = dict()
test_cfg_cluster.update(test_cfg)
test_cfg_cluster['out_dir'] = test_cfg_cluster['out_dir'] + "_cluster"
test_cfg_cluster['skip_existing'] = True
print("Running initialisation step ...")
s2p.main(test_cfg_cluster,["initialisation"])
# Retrieve tiles list
outdir = test_cfg_cluster['out_dir']
tiles_file = os.path.join(outdir,'tiles.txt')
tiles = s2p.read_tiles(tiles_file)
print('Found '+str(len(tiles))+' tiles to process')
for step in s2p.ALL_STEPS:
if s2p.ALL_STEPS[step] is True:
print('Running %s on each tile...' % step)
for tile in tiles:
print('tile : %s' % tile)
tile_cfg_cluster = s2p.read_config_file(tile)
s2p.main(tile_cfg_cluster, [step])
else:
print('Running %s...' % step)
print('test_cfg_cluster : %s' % test_cfg_cluster)
s2p.main(test_cfg_cluster, [step])
computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif'))
end2end_compare_dsm(computed,expected,0,0)
def test_no_rpc(data, mocks):
"""
Initialize s2p with no `rpc` key.
The RPCs should be read from the geotiff tags.
"""
tmp_config, _, _, _ = data
user_cfg = s2p.read_config_file(tmp_config)
s2p.initialization.build_cfg(user_cfg)
rpcm.rpc_from_geotiff.assert_called()
assert rpcm.rpc_from_geotiff.call_count == 2
rpcm.rpc_from_rpc_file.assert_not_called()
def end2end(config,ref_dsm,absmean_tol=0.025,percentile_tol=1.):
print('Configuration file: ',config)
print('Reference DSM:',ref_dsm,os.linesep)
test_cfg = s2p.read_config_file(config)
s2p.main(test_cfg)
outdir = test_cfg['out_dir']
computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif'))
expected = s2plib.common.gdal_read_as_array_with_nans(ref_dsm)
end2end_compare_dsm(computed,expected,absmean_tol,percentile_tol)
def end2end(config,ref_dsm,absmean_tol=0.025,percentile_tol=1.):
print('Configuration file: ',config)
print('Reference DSM:',ref_dsm,os.linesep)
test_cfg = s2p.read_config_file(config)
s2p.main(test_cfg)
outdir = test_cfg['out_dir']
computed = s2plib.common.gdal_read_as_array_with_nans(os.path.join(outdir,'dsm.tif'))
expected = s2plib.common.gdal_read_as_array_with_nans(ref_dsm)
end2end_compare_dsm(computed,expected,absmean_tol,percentile_tol)
def end2end_mosaic(config,ref_height_map,absmean_tol=0.025,percentile_tol=1.):
test_cfg = s2p.read_config_file(config)
outdir = test_cfg['out_dir']
s2p.main(test_cfg)
tiles_file = os.path.join(outdir,'tiles.txt')
global_height_map = os.path.join(outdir,'height_map.tif')
s2p_mosaic.main(tiles_file,global_height_map,'pair_1/height_map.tif')
computed = s2p.common.gdal_read_as_array_with_nans(global_height_map)
expected = s2p.common.gdal_read_as_array_with_nans(ref_height_map)
end2end_compare_dsm(computed,expected,absmean_tol,percentile_tol)
def test_roi_geojson(data):
tmp_config, _, _, _ = data
user_cfg = s2p.read_config_file(tmp_config)
user_cfg["roi_geojson"] = {
"coordinates": [[[55.64943405, -21.23207174],
[55.65212062, -21.23207174],
[55.65212062, -21.23460474],
[55.64943405, -21.23460474],
[55.64943405, -21.23207174]]],
"type":
"Polygon"
}
s2p.initialization.build_cfg(user_cfg)
assert user_cfg["roi"] == {'x': 150, 'y': 150, 'w': 700, 'h': 700}
def end2end_mosaic(config,ref_height_map,absmean_tol=0.025,percentile_tol=1.):
test_cfg = s2p.read_config_file(config)
outdir = test_cfg['out_dir']
test_cfg['skip_existing'] = True
s2p.main(test_cfg)
tiles_file = os.path.join(outdir,'tiles.txt')
global_height_map = os.path.join(outdir,'height_map.tif')
s2p_mosaic.main(tiles_file,global_height_map,'pair_1/height_map.tif')
computed = s2plib.common.gdal_read_as_array_with_nans(global_height_map)
expected = s2plib.common.gdal_read_as_array_with_nans(ref_height_map)
end2end_compare_dsm(computed,expected,absmean_tol,percentile_tol)
def end2end(config_file, ref_dsm, absmean_tol=0.025, percentile_tol=1.):
print('Configuration file: ', config_file)
print('Reference DSM:', ref_dsm, os.linesep)
# TODO: this is ugly, and will be fixed once we'll have implemented a better
# way to control the config parameters
if 'out_crs' in s2p.cfg: del s2p.cfg['out_crs']
test_cfg = s2p.read_config_file(config_file)
s2p.main(test_cfg)
outdir = test_cfg['out_dir']
computed = common.gdal_read_as_array_with_nans(
os.path.join(outdir, 'dsm.tif'))
expected = common.gdal_read_as_array_with_nans(ref_dsm)
compare_dsm(computed, expected, absmean_tol, percentile_tol)
def test_rpc_dict(data, mocks):
"""
Initialize s2p with `rpc` keys that are dicts with the RPC contents.
The RPCs should be loaded from the dicts.
"""
tmp_config, _, rpc1, rpc2 = data
with open(tmp_config) as f:
cfg = json.load(f)
cfg["images"][0]["rpc"] = rpc1.__dict__
cfg["images"][1]["rpc"] = rpc2.__dict__
with open(tmp_config, "w") as f:
json.dump(cfg, f)
user_cfg = s2p.read_config_file(tmp_config)
s2p.initialization.build_cfg(user_cfg)
rpcm.rpc_from_geotiff.assert_not_called()
rpcm.rpc_from_rpc_file.assert_not_called()
def test_disparity_to_ply(tmp_path, out_crs):
"""
Check that disparity_to_ply() functions correctly when given
different out_crs parameters
"""
# Setup test data
tile_dir = str(tmp_path / "tile_dir")
shutil.copytree(data_path("input_triangulation"), tile_dir)
# Initialize s2p's state
config_file = data_path(os.path.join("input_pair", "config.json"))
test_cfg = read_config_file(config_file)
test_cfg["out_crs"] = out_crs
build_cfg(test_cfg)
tile = {"coordinates": [500, 150, 350, 350], "dir": tile_dir}
disparity_to_ply(tile)
_, comments = read_3d_point_cloud_from_ply(
os.path.join(tile_dir, "cloud.ply"))
expected_crs = out_crs or "epsg:32740"
assert comments[-1] == "projection: CRS {}".format(expected_crs)
def main():
"""
Command line parsing for s2p command line interface.
"""
parser = argparse.ArgumentParser(description=('S2P: Satellite Stereo '
'Pipeline'))
parser.add_argument('config',
metavar='config.json',
help=('path to a json file containing the paths to '
'input and output files and the algorithm '
'parameters'))
args = parser.parse_args()
user_cfg = s2p.read_config_file(args.config)
s2p.main(user_cfg)
# Backup input file for sanity check
if not args.config.startswith(
os.path.abspath(s2p.cfg['out_dir'] + os.sep)):
shutil.copy2(args.config,
os.path.join(s2p.cfg['out_dir'], 'config.json.orig'))
def test_distributed_plyflatten():
config_file = data_path('input_triplet/config.json')
print('Running end2end with distributed plyflatten dsm ...')
test_cfg = s2p.read_config_file(config_file)
s2p.main(test_cfg)
outdir = test_cfg['out_dir']
computed = common.gdal_read_as_array_with_nans(
os.path.join(outdir, 'dsm.tif'))
print('Running plyflatten dsm reference ...')
clouds_list = glob.glob(
os.path.join(outdir, "tiles", "*", "*", "cloud.ply"))
out_dsm = os.path.join(outdir, "dsm_ref.tif")
res = test_cfg['dsm_resolution']
roi = None
if 'utm_bbx' in test_cfg:
bbx = test_cfg['utm_bbx']
global_xoff = bbx[0]
global_yoff = bbx[3]
global_xsize = int(
np.ceil((bbx[1] - bbx[0]) / test_cfg['dsm_resolution']))
global_ysize = int(
np.ceil((bbx[3] - bbx[2]) / test_cfg['dsm_resolution']))
roi = (global_xoff, global_yoff, global_xsize, global_ysize)
raster, profile = s2p.rasterization.plyflatten_from_plyfiles_list(
clouds_list, resolution=res, roi=roi)
s2p.common.rasterio_write(out_dsm, raster[:, :, 0], profile=profile)
expected = common.gdal_read_as_array_with_nans(
os.path.join(outdir, 'dsm_ref.tif'))
compare_dsm(computed, expected, 0, 0)
def test_distributed_plyflatten():
print('Running end2end with distributed plyflatten dsm ...')
test_cfg = s2p.read_config_file(data_path('input_triplet/config.json'))
s2p.main(test_cfg)
outdir = test_cfg['out_dir']
computed = common.gdal_read_as_array_with_nans(
os.path.join(outdir, 'dsm.tif'))
print('Running plyflatten dsm reference ...')
clouds_list = glob.glob(
os.path.join(outdir, "tiles", "*", "*", "cloud.ply"))
res = test_cfg['dsm_resolution']
roi = None
raster, _ = plyflatten_from_plyfiles_list(clouds_list,
resolution=res,
roi=roi)
expected = raster[:, :, 0]
compare_dsm(computed, expected, 0, 0)
user_cfg: user config dictionary
"""
common.print_elapsed_time.t0 = datetime.datetime.now()
initialization.build_cfg(user_cfg)
tw, th = initialization.adjust_tile_size()
tiles_txt = os.path.join(cfg['out_dir'],'tiles.txt')
tiles = initialization.tiles_full_info(tw, th, tiles_txt)
# generate svg tile map
write_svg_tilemap(os.path.join(cfg['out_dir'],'tiles.svg'), cfg, tiles)
print("\n\n svg tilemap saved in: %s\n"%os.path.join(cfg['out_dir'],'tiles.svg'))
# cleanup
common.garbage_cleanup()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=('generate svg tilemap for S2P'))
parser.add_argument('config', metavar='config.json',
help=('path to a json file containing the paths to '
'input and output files and the s2p algorithm '
'parameters'))
args = parser.parse_args()
user_cfg = s2p.read_config_file(args.config)
main(user_cfg)
Args:
user_cfg: user config dictionary
"""
common.print_elapsed_time.t0 = datetime.datetime.now()
initialization.build_cfg(user_cfg)
tw, th = initialization.adjust_tile_size()
tiles = initialization.tiles_full_info(tw, th)
# generate svg tile map
write_svg_tilemap(os.path.join(cfg['out_dir'],'tiles.svg'), cfg, tiles)
print("\n\n svg tilemap saved in: %s\n"%os.path.join(cfg['out_dir'],'tiles.svg'))
# cleanup
common.garbage_cleanup()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=('generate svg tilemap for S2P'))
parser.add_argument('config', metavar='config.json',
help=('path to a json file containing the paths to '
'input and output files and the s2p algorithm '
'parameters'))
args = parser.parse_args()
user_cfg = s2p.read_config_file(args.config)
main(user_cfg)
def write_tiles_polygon(tile,
kml,
m=None,
M=None,
message=None,
error_mode=False):
dsm = os.path.join(tile, 'dsm.tif')
dico = []
green_style = "background-color: #4CAF50; color: white;"
red_style = "background-color: #FF4C4C; color: white;"
blue_style = "background-color: #9999EB; color:white;"
if error_mode is True:
latlon = get_coordinates_with_config(tile, m, M)
color = simplekml.Color.red
head_style = red_style
else:
latlon = get_coordinates_with_img(dsm)
color = simplekml.Color.green
head_style = green_style
tile_cfg = s2p.read_config_file(os.path.join(tile, "config.json"))
x = tile_cfg['roi']['x']
y = tile_cfg['roi']['y']
w = tile_cfg['roi']['w']
h = tile_cfg['roi']['h']
pair_info = [{} for i in range(len(tile_cfg['images']) - 1)]
for i in range(len(tile_cfg['images']) - 1):
pair_dir = os.path.join(tile, "pair_%d" % (i + 1))
disp = os.path.join(pair_dir, "rectified_disp.tif")
if os.path.exists(disp) is False:
pair_info[i]['status'] = "failure"
if error_mode is False:
color = simplekml.Color.blue
pair_info[i]['style'] = red_style
head_style = blue_style
else:
pair_info[i]['style'] = None
else:
pair_info[i]['status'] = "success"
pair_info[i]['style'] = None
disp_min_max = os.path.join(pair_dir, "disp_min_max.txt")
if os.path.exists(disp_min_max) is True:
disp_min, disp_max = np.loadtxt(disp_min_max)
else:
disp_min, disp_max = None, None
pair_info[i]['disp_min'] = disp_min
pair_info[i]['disp_max'] = disp_max
dico += [('roi', {
"value": "x : %s - y : %s - w : %s - h : %s" % (x, y, w, h),
"style": head_style
})]
for i in range(len(tile_cfg['images']) - 1):
disp_min = pair_info[i]['disp_min']
disp_max = pair_info[i]['disp_max']
status = pair_info[i]['status']
style = pair_info[i]['style']
dico += [('pair_%d' % (i + 1), {
"value":
' - '.join([
'disp_min : %s' % disp_min,
'disp_max : %s' % disp_max,
'status : %s' % status
]),
"style":
style
})]
if message != None:
dico += [('message', {"value": message, "style": None})]
pol = kml.newpolygon(name=tile)
pol.outerboundaryis = latlon
pol.style.linestyle.color = color
pol.style.linestyle.width = 5
pol.style.polystyle.color = simplekml.Color.changealphaint(100, color)
dico = collections.OrderedDict(dico)
pol.description = get_polygon_description(dico)
def write_tiles_polygon(tile, kml, m=None, M=None, message=None, error_mode=False):
dsm = os.path.join(tile, 'dsm.tif')
dico = []
green_style = "background-color: #4CAF50; color: white;"
red_style = "background-color: #FF4C4C; color: white;"
blue_style = "background-color: #9999EB; color:white;"
if error_mode is True:
latlon = get_coordinates_with_config(tile, m, M)
color = simplekml.Color.red
head_style = red_style
else:
latlon = get_coordinates_with_img(dsm)
color = simplekml.Color.green
head_style = green_style
tile_cfg = s2p.read_config_file(os.path.join(tile, "config.json"))
x = tile_cfg['roi']['x']
y = tile_cfg['roi']['y']
w = tile_cfg['roi']['w']
h = tile_cfg['roi']['h']
pair_info = [{} for i in range(len(tile_cfg['images'])-1)]
for i in range(len(tile_cfg['images'])-1):
pair_dir = os.path.join(tile,
"pair_%d" % (i+1))
disp = os.path.join(pair_dir,
"rectified_disp.tif")
if os.path.exists(disp) is False:
pair_info[i]['status'] = "failure"
if error_mode is False:
color = simplekml.Color.blue
pair_info[i]['style'] = red_style
head_style = blue_style
else:
pair_info[i]['style'] = None
else:
pair_info[i]['status'] = "success"
pair_info[i]['style'] = None
disp_min_max = os.path.join(pair_dir,
"disp_min_max.txt")
if os.path.exists(disp_min_max) is True:
disp_min, disp_max = np.loadtxt(disp_min_max)
else:
disp_min, disp_max = None, None
pair_info[i]['disp_min'] = disp_min
pair_info[i]['disp_max'] = disp_max
dico += [('roi', {"value" : "x : %s - y : %s - w : %s - h : %s" % (x, y, w, h),
"style" : head_style})]
for i in range(len(tile_cfg['images'])-1):
disp_min = pair_info[i]['disp_min']
disp_max = pair_info[i]['disp_max']
status = pair_info[i]['status']
style = pair_info[i]['style']
dico += [('pair_%d' % (i+1), {"value":' - '.join(['disp_min : %s' % disp_min,
'disp_max : %s' % disp_max,
'status : %s' % status]),
"style":style})]
if message != None:
dico += [('message', {"value": message,
"style": None})]
pol = kml.newpolygon(name=tile)
pol.outerboundaryis = latlon
pol.style.linestyle.color = color
pol.style.linestyle.width = 5
pol.style.polystyle.color = simplekml.Color.changealphaint(100, color)
dico = collections.OrderedDict(dico)
pol.description = get_polygon_description(dico)
