def test_rasterio_handles():
"""
Test to create a file handle(s) depending on whether the output data has a color layer or not.
"""
bounds = (675248.0, 4897075.0, 675460.5, 4897173.0)
resolution = 0.5
geotransform = (bounds[0], resolution, 0.0, bounds[3], 0.0, -resolution)
transform = Affine.from_gdal(*geotransform)
rio_params = dict(
height=196, width=425, driver='GTiff', dtype=np.float32,
transform=transform, crs='EPSG:{}'.format(32631), tiled=True
)
dsm_no_data = -32768
color_no_data = 0
nb_bands = 1
# Create file handles
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
dsm_file = os.path.join(directory, 'dsm.tif')
clr_file = os.path.join(directory, 'clr.tif')
file_handles = readwrite.rasterio_handles(['hgt', 'clr'], [dsm_file, clr_file], [rio_params, rio_params], [dsm_no_data, color_no_data], [1, nb_bands])
with file_handles as rio_handles:
assert isinstance(rio_handles, dict) == True
assert 'hgt' in rio_handles.keys() and 'clr' in rio_handles.keys()
for key in rio_handles.keys():
assert isinstance(rio_handles[key], rio.io.DatasetWriter)
def test_write_ply():
"""
Test write ply file
"""
points = xr.open_dataset(
absolute_data_path("input/intermediate_results/points_ref.nc"))
utils.write_ply(os.path.join(temporary_dir(), 'test.ply'), points)
def test_epipolar_distributions():
cars_path = os.environ.get('CARSPATH')
# uncomment the following lines to regenerate the input files
# input_json = read_input_parameters(
# absolute_data_path("input/phr_ventoux/preproc_input.json"))
# prepare.run(
# input_json,
# absolute_data_path("input/notebooks_input/content_dir/"),
# epi_step=30,
# region_size=250,
# disparity_margin=0.25,
# epipolar_error_upper_bound=43.,
# elevation_delta_lower_bound=-20.,
# elevation_delta_upper_bound=20.,
# mode="local", # Run on a local cluster
# nb_workers=4,
# walltime="00:10:00",
# check_inputs=True)
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
subprocess.run(['jupyter nbconvert --to script {}/notebooks/epipolar_distributions.ipynb --output-dir {}'
.format(cars_path, directory)], shell=True)
for line in fileinput.input('{}/epipolar_distributions.py'.format(directory), inplace=True):
if "cars_home = \"TODO\"" in line:
line = line.replace("TODO", cars_path)
elif "content_dir = \"TODO\"" in line:
line = line.replace("TODO",
absolute_data_path("input/notebooks_input/content_dir/"))
print(line) # keep this print
out = subprocess.run(['ipython {}/epipolar_distributions.py'.format(directory)], shell=True)
out.check_returncode()
def test_lowres_dem_fit():
"""
Low resolution notebook test:
notebook conversion (.ipynb->.py), replace default TODO values,
run the notebook with ipython and check return code
"""
cars_path = os.environ.get('CARSPATH')
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
subprocess.run([
'jupyter nbconvert '
'--to script {}/notebooks/lowres_dem_fit.ipynb '
'--output-dir {}'.format(cars_path, directory)
],
shell=True,
check=True)
for line in fileinput.input('{}/lowres_dem_fit.py'.format(directory),
inplace=True):
if "cars_home = \"TODO\"" in line:
line = line.replace("TODO", cars_path)
elif "content_dir = \"TODO\"" in line:
line = line.replace(
"TODO",
absolute_data_path("input/notebooks_input/lowres_dem/"))
print(line) # keep this print
out = subprocess.run(
['ipython {}/lowres_dem_fit.py'.format(directory)],
shell=True,
check=True)
out.check_returncode()
def test_step_by_step_compute_dsm():
"""
Step by step compute dsm notebook test:
notebook conversion (.ipynb->.py), replace default TODO values,
run the notebook with ipython and check return code
"""
cars_path = os.environ.get('CARSPATH')
# uncomment the following lines to regenerate the input files
# input_json = read_input_parameters(
# absolute_data_path("input/phr_ventoux/preproc_input.json"))
# prepare.run(
# input_json,
# absolute_data_path("input/notebooks_input/content_dir/"),
# epi_step=30,
# region_size=250,
# disparity_margin=0.25,
# epipolar_error_upper_bound=43.,
# elevation_delta_lower_bound=-20.,
# elevation_delta_upper_bound=20.,
# mode="local", # Run on a local cluster
# nb_workers=4,
# walltime="00:10:00",
# check_inputs=True)
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
subprocess.run([
'jupyter nbconvert '
'--to script '
'{}/notebooks/step_by_step_compute_dsm.ipynb '
'--output-dir {}'.format(cars_path, directory)
],
shell=True,
check=True)
for line in fileinput.input(
'{}/step_by_step_compute_dsm.py'.format(directory),
inplace=True):
if "cars_home = \"TODO\"" in line:
line = line.replace("TODO", cars_path)
elif "content_dir = \"TODO\"" in line:
line = line.replace(
"TODO",
absolute_data_path("input/notebooks_input/content_dir/"))
elif "roi_file = \"TODO\"" in line:
line = line.replace(
"TODO",
absolute_data_path("input/notebooks_input/content_dir/"
"envelopes_intersection.gpkg"))
elif "output_dir = \"TODO\"" in line:
line = line.replace("TODO", directory)
print(line) # keep this print
out = subprocess.run(
['ipython {}/step_by_step_compute_dsm.py'.format(directory)],
shell=True,
check=True)
out.check_returncode()
def test_cluster():
"""
End to end cluster management test
"""
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
clus, client = cluster.start_cluster(2, "00:01:00", directory)
link = cluster.get_dashboard_link(clus)
cluster.stop_cluster(clus, client)
def test_image_envelope():
"""
Test image_envelope function
"""
img = absolute_data_path("input/phr_ventoux/left_image.tif")
dem = absolute_data_path("input/phr_ventoux/srtm")
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
shp = os.path.join(directory, "envelope.gpkg")
preprocessing.image_envelope(img, shp, dem)
assert os.path.isfile(shp)
def test_configure_pandora_with_file():
"""
Test configure pandora correlator
"""
json_content = """{
"image":{
"nodata1":"np.nan",
"nodata2":"np.nan"
},
"stereo" : {
"stereo_method": "census",
"window_size": 5,
"subpix": 1
},
"aggregation" : {
"aggregation_method": "none"
},
"optimization" : {
"optimization_method": "sgm",
"P1": 8,
"P2": 24,
"p2_method": "constant",
"penalty_method": "sgm_penalty",
"overcounting": false,
"min_cost_paths": false
},
"refinement": {
"refinement_method": "vfit"
},
"filter" : {
"filter_method": "median",
"filter_size": 3
},
"validation" : {
"validation_method": "cross_checking",
"interpolated_disparity": "none"
},
"invalid_disparity": "np.nan"
}
"""
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
json_path = os.path.join(directory, "corr.json")
with open(json_path, "w") as json_file:
json_file.write(json_content)
corr_config = corr_cfg.configure_correlator(json_path)
assert corr_config["optimization"]["P2"] == 24
def test_prepare_ventoux_bias():
"""
Dask prepare with bias geoms
"""
input_json = read_input_parameters(absolute_data_path(
"input/phr_ventoux/preproc_input_bias.json"))
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
out_preproc = os.path.join(directory, "out_preproc_bias")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
epipolar_error_maximum_bias=50.,
elevation_delta_lower_bound=-120.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00")
# Check preproc properties
preproc_json = os.path.join(out_preproc, "content.json")
assert os.path.isfile(preproc_json)
with open(preproc_json, 'r') as preproc_json_file:
preproc_data = json.load(preproc_json_file)
preproc_output = preproc_data["preprocessing"]["output"]
assert preproc_output["epipolar_size_x"] == 612
assert preproc_output["epipolar_size_y"] == 612
assert preproc_output["minimum_disparity"] > -86
assert preproc_output["minimum_disparity"] < -84
assert preproc_output["maximum_disparity"] > -46
assert preproc_output["maximum_disparity"] < -44
for img in [
"matches",
"right_epipolar_grid",
"left_epipolar_grid"]:
assert os.path.isfile(
os.path.join(
out_preproc,
preproc_data["preprocessing"]["output"][img]))
def test_write_vector():
polys = [
Polygon([(1.0, 1.0), (1.0, 2.0), (2.0, 2.0), (2.0, 1.0)]),
Polygon([(2.0, 2.0), (2.0, 3.0), (3.0, 3.0), (3.0, 2.0)])
]
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
path_to_file = os.path.join(directory, 'test.gpkg')
utils.write_vector(polys, path_to_file, 4326)
assert os.path.exists(path_to_file)
nb_feat = 0
for feat in fiona.open(path_to_file):
poly = shape(feat['geometry'])
nb_feat += 1
assert poly in polys
assert nb_feat == 2
def test_write_geotiff_dsm():
"""
Test to result tiles to GTiff file(s).
"""
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
# Set variables and parameters for create dataset
resolution = 0.5
epsg = 32630
dsm_no_data = -32768
hgt_no_data = dsm_no_data
color_no_data = 0
msk_no_data = 255
xstart, ystart, xsize, ysize = [0, 10, 10, 10]
raster = np.ndarray(shape=(10, 10, 2), dtype=np.float32)
mean = np.ndarray(shape=(10, 10, 2), dtype=np.float32)
stdev = np.ndarray(shape=(10, 10, 2), dtype=np.float32)
n_pts = np.ndarray(shape=(10, 10), dtype=np.uint16)
n_in_cell = np.ndarray(shape=(10, 10), dtype=np.uint16)
msk = np.ndarray(shape=(10, 10), dtype=np.uint16)
delayed_raster_datasets = list()
delayed_raster_datasets.append(
dask.delayed(rasterization.create_raster_dataset)(
raster, xstart, ystart, xsize, ysize, resolution, hgt_no_data,
color_no_data, epsg, mean, stdev, n_pts, n_in_cell, msk))
# Start cluster with a local cluster
nb_workers = 4
cluster, client = start_local_cluster(nb_workers)
future_dsm = client.compute(delayed_raster_datasets)
# Write netcdf with raster_datasets
bounds = [xstart, ystart, xsize, ysize]
nb_bands = 1
dsm_file = os.path.join(directory, "dsm.tif")
clr_file = os.path.join(directory, "clr.tif")
msk_file = os.path.join(directory, "msk.tif")
readwrite.write_geotiff_dsm(future_dsm,
directory,
xsize,
ysize,
bounds,
resolution,
epsg,
nb_bands,
dsm_no_data,
color_no_data,
write_msk=True,
msk_no_data=msk_no_data)
# stop cluster
stop_local_cluster(cluster, client)
width_ref = xsize
height_ref = ysize
crs = osr.SpatialReference()
crs.ImportFromEPSG(epsg)
crs_ref = crs.ExportToWkt()
geotransform = (bounds[0], resolution, 0.0, bounds[3], 0.0,
-resolution)
transform_ref = Affine.from_gdal(*geotransform)
# Compare DSM
with rio.open(dsm_file) as rio_actual:
np.testing.assert_equal(rio_actual.width, width_ref)
np.testing.assert_equal(rio_actual.height, height_ref)
assert rio_actual.transform == transform_ref
assert rio_actual.crs == crs_ref
assert rio_actual.nodata == hgt_no_data
for i in range(rio_actual.width):
for j in range(rio_actual.height):
if np.isnan(raster[i][j][0]):
assert rio_actual.read()[0][i][j] == dsm_no_data
else:
assert rio_actual.read()[0][i][j] == raster[i][j][0]
# Compare CLR
with rio.open(clr_file) as rio_actual:
np.testing.assert_equal(rio_actual.width, width_ref)
np.testing.assert_equal(rio_actual.height, height_ref)
assert rio_actual.transform == transform_ref
assert rio_actual.crs == crs_ref
assert rio_actual.nodata == color_no_data
for i in range(rio_actual.width):
for j in range(rio_actual.height):
if np.isnan(raster[i][j][1]):
assert rio_actual.read()[0][i][j] == color_no_data
else:
assert rio_actual.read()[0][i][j] == raster[i][j][1]
# Compare MSK
with rio.open(msk_file) as rio_actual:
np.testing.assert_equal(rio_actual.width, width_ref)
np.testing.assert_equal(rio_actual.height, height_ref)
assert rio_actual.transform == transform_ref
assert rio_actual.crs == crs_ref
assert rio_actual.nodata == msk_no_data
for i in range(rio_actual.width):
for j in range(rio_actual.height):
if np.isnan(raster[i][j][1]):
assert rio_actual.read()[0][i][j] == msk_no_data
else:
assert rio_actual.read()[0][i][j] == msk[i][j]
def test_compute_dsm_with_roi_ventoux():
"""
Dask compute dsm processing with input roi (cars_stereo)
"""
# Force max RAM to 1000 to get stable tiling in tests
os.environ['OTB_MAX_RAM_HINT'] = '1000'
input_json = read_input_parameters(
absolute_data_path("input/phr_ventoux/preproc_input.json"))
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
out_preproc = os.path.join(directory, "out_preproc")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
elevation_delta_lower_bound=-20.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
check_inputs=True)
preproc_json = os.path.join(out_preproc, "content.json")
out_stereo = os.path.join(directory, "out_preproc")
final_epsg = 32631
resolution = 0.5
roi = [5.194, 44.2059, 5.195, 44.2064]
roi_epsg = 4326
corr_config = corr_cfg.configure_correlator()
compute_dsm.run(
[read_preprocessing_content_file(preproc_json)],
out_stereo,
resolution=resolution,
epsg=final_epsg,
sigma=0.3,
dsm_radius=3,
dsm_no_data=-999,
color_no_data=0,
corr_config=corr_config,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
roi=(roi, roi_epsg))
# Uncomment the 2 following instructions to update reference data
#copy2(os.path.join(out_stereo, 'dsm.tif'),
# absolute_data_path("ref_output/dsm_end2end_ventoux_with_roi.tif"))
#copy2(os.path.join(out_stereo, 'clr.tif'),
# absolute_data_path("ref_output/clr_end2end_ventoux_with_roi.tif"))
assert_same_images(
os.path.join(out_stereo, "dsm.tif"),
absolute_data_path("ref_output/dsm_end2end_ventoux_with_roi.tif"),
atol=0.0001,
rtol=1e-6)
assert_same_images(
os.path.join(out_stereo, "clr.tif"),
absolute_data_path("ref_output/clr_end2end_ventoux_with_roi.tif"),
rtol=1.e-7,
atol=1.e-7)
# check final bounding box
# create reference
[roi_xmin, roi_ymin, roi_xmax, roi_ymax] = roi
roi_poly = Polygon([(roi_xmin, roi_ymin), (roi_xmax, roi_ymin),
(roi_xmax, roi_ymax), (roi_xmin, roi_ymax),
(roi_xmin, roi_ymin)])
project = pyproj.Transformer.from_proj(
pyproj.Proj(init='epsg:{}'.format(roi_epsg)),
pyproj.Proj(init='epsg:{}'.format(final_epsg)))
ref_roi_poly = transform(project.transform, roi_poly)
[ref_xmin, ref_ymin, ref_xmax, ref_ymax] = ref_roi_poly.bounds
# retrieve bounding box of computed dsm
data = rasterio.open(os.path.join(out_stereo, "dsm.tif"))
xmin = min(data.bounds.left, data.bounds.right)
ymin = min(data.bounds.bottom, data.bounds.top)
xmax = max(data.bounds.left, data.bounds.right)
ymax = max(data.bounds.bottom, data.bounds.top)
assert math.floor(ref_xmin / resolution) * resolution == xmin
assert math.ceil(ref_xmax / resolution) * resolution == xmax
assert math.floor(ref_ymin / resolution) * resolution == ymin
assert math.ceil(ref_ymax / resolution) * resolution == ymax
def test_compute_dsm_with_snap_to_img1():
"""
Dask compute dsm processing with input roi (cars_stereo)
"""
# Force max RAM to 1000 to get stable tiling in tests
os.environ['OTB_MAX_RAM_HINT'] = '1000'
input_json = read_input_parameters(
absolute_data_path("input/phr_ventoux/preproc_input.json"))
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
out_preproc = os.path.join(directory, "out_preproc")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
elevation_delta_lower_bound=-20.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00")
# Check preproc properties
preproc_json = os.path.join(out_preproc, "content.json")
assert os.path.isfile(preproc_json)
out_stereo = os.path.join(directory, "out_preproc")
final_epsg = 32631
resolution = 0.5
corr_config = corr_cfg.configure_correlator()
compute_dsm.run(
[read_preprocessing_content_file(preproc_json)],
out_stereo,
resolution=resolution,
epsg=final_epsg,
sigma=0.3,
dsm_radius=3,
dsm_no_data=-999,
color_no_data=0,
corr_config=corr_config,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
snap_to_img1=True)
# Uncomment the 2 following instructions to update reference data
#copy2(os.path.join(out_stereo, 'dsm.tif'),
# absolute_data_path("ref_output/dsm_end2end_ventoux_with_snap_to_img1.tif"))
#copy2(os.path.join(out_stereo, 'clr.tif'),
# absolute_data_path("ref_output/clr_end2end_ventoux_with_snap_to_img1.tif"))
assert_same_images(
os.path.join(out_stereo, "dsm.tif"),
absolute_data_path(
"ref_output/dsm_end2end_ventoux_with_snap_to_img1.tif"),
atol=0.0001,
rtol=1e-6)
assert_same_images(
os.path.join(out_stereo, "clr.tif"),
absolute_data_path(
"ref_output/clr_end2end_ventoux_with_snap_to_img1.tif"),
rtol=1.e-7,
atol=1.e-7)
def test_dsm_compute_arg(compute_dsm_default_args): #pylint: disable=redefined-outer-name
"""
Cars compute_dsm arguments test with default and degraded cases
"""
parser = cars_cli_parser()
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
compute_dsm_default_args.outdir = directory
# test with default args
main_cli(compute_dsm_default_args, parser, check_inputs=True)
# test with mp mode (multiprocessing)
args_mode_mp = copy(compute_dsm_default_args)
args_mode_mp.mode = "mp"
main_cli(args_mode_mp, parser, check_inputs=True)
# test [xmin, ymin, xmax, ymax] roi argument
args_roi_bbox = copy(compute_dsm_default_args)
args_roi_bbox.roi_bbox = ['1.0', '2.0', '3.0', '4.0']
main_cli(args_roi_bbox, parser, check_inputs=True)
# test image roi argument
args_roi_file = copy(compute_dsm_default_args)
args_roi_file.roi_file = absolute_data_path(
'input/cars_cli_input/roi_image.tif')
main_cli(args_roi_file, parser, check_inputs=True)
# test vector roi argument
args_roi_file.roi_file = absolute_data_path(
'input/cars_cli_input/roi_vector.gpkg')
main_cli(args_roi_file, parser, check_inputs=True)
# degraded cases input jsons
args_bad_jsons = copy(compute_dsm_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_jsons.injsons = [
absolute_data_path('input/cars_cli_input/test.txt')
]
main_cli(args_bad_jsons, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
with pytest.raises(SystemExit) as exit_error:
args_bad_jsons.injsons = []
main_cli(args_bad_jsons, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases sigma
args_bad_sigma = copy(compute_dsm_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_sigma.sigma = -10
main_cli(args_bad_sigma, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases dsm radius
args_bad_dsm_radius = copy(compute_dsm_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_dsm_radius.dsm_radius = -10
main_cli(args_bad_dsm_radius, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases resolution
args_bad_resolution = copy(compute_dsm_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_resolution.resolution = 0
main_cli(args_bad_resolution, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
args_bad_resolution = copy(compute_dsm_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_resolution.resolution = -1
main_cli(args_bad_resolution, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases epsg
args_bad_epsg = copy(compute_dsm_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_epsg.epsg = -1
main_cli(args_bad_epsg, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases input ROI file
args_bad_roi_file = copy(compute_dsm_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_roi_file.roi_file = \
absolute_data_path('input/cars_cli_input/test.txt')
main_cli(args_bad_roi_file, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
with pytest.raises(SystemExit) as exit_error:
args_bad_roi_file.roi_file = absolute_data_path(
'input/phr_ventoux/preproc_output/content.json')
main_cli(args_bad_roi_file, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
with pytest.raises(SystemExit) as exit_error:
args_bad_roi_file.roi_file = absolute_data_path(
'input/phr_ventoux/left_image.tif')
main_cli(args_bad_roi_file, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases correlator config file
args_bad_correlator_conf = copy(compute_dsm_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_correlator_conf.corr_config = absolute_data_path(
'input/cars_cli_input/test.txt')
main_cli(args_bad_correlator_conf, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases elevation offsets
args_bad_elevation_offsets = copy(compute_dsm_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_elevation_offsets.min_elevation_offset = 10
args_bad_elevation_offsets.max_elevation_offset = -10
main_cli(args_bad_elevation_offsets, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases log level
args_bad_loglevel = copy(compute_dsm_default_args)
with pytest.raises(ValueError):
args_bad_loglevel.loglevel = 'TEST'
main_cli(args_bad_loglevel, parser, check_inputs=True)
# degraded cases number of workers
args_bad_nb_workers = copy(compute_dsm_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_nb_workers.nb_workers = -1
main_cli(args_bad_nb_workers, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases wall time
args_bad_wall_time = copy(compute_dsm_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_wall_time.walltime = '000:00:00'
main_cli(args_bad_wall_time, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
def test_prepare_args(prepare_default_args): #pylint: disable=redefined-outer-name
"""
Cars prepare arguments test with default and degraded cases
"""
parser = cars_cli_parser()
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
prepare_default_args.outdir = directory
# test default args
main_cli(prepare_default_args, parser, check_inputs=True)
prepare_default_args.loglevel = 'INFO'
main_cli(prepare_default_args, parser, check_inputs=True)
# degraded cases injson
args_bad_json = copy(prepare_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_json.injson = absolute_data_path(
'input/cars_cli_input/test.json')
main_cli(args_bad_json, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases disparity_margin
args_bad_disp_margin = copy(prepare_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_disp_margin.disparity_margin = -1.0
main_cli(args_bad_disp_margin, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
with pytest.raises(SystemExit) as exit_error:
args_bad_disp_margin.disparity_margin = 1.5
main_cli(args_bad_disp_margin, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases elevation bounds
args_bad_elev_delta_bounds = copy(prepare_default_args)
args_bad_elev_delta_bounds.elevation_delta_lower_bound = 50
args_bad_elev_delta_bounds.elevation_delta_upper_bound = -50
with pytest.raises(SystemExit) as exit_error:
main_cli(args_bad_elev_delta_bounds, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases epi_step
args_bad_epi_step = copy(prepare_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_epi_step.epi_step = 0
main_cli(args_bad_epi_step, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases epipolar_error_upper_bound
args_bad_epi_error_bound = copy(prepare_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_epi_error_bound.epipolar_error_upper_bound = -10
main_cli(args_bad_epi_error_bound, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases log level
args_bad_loglevel = copy(prepare_default_args)
with pytest.raises(ValueError):
args_bad_loglevel.loglevel = 'TEST'
main_cli(args_bad_loglevel, parser, check_inputs=True)
# degraded cases number of workers
args_bad_nb_workers = copy(prepare_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_nb_workers.nb_workers = -1
main_cli(args_bad_nb_workers, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
# degraded cases wall time
args_bad_wall_time = copy(prepare_default_args)
with pytest.raises(SystemExit) as exit_error:
args_bad_wall_time.walltime = '000:00:00'
main_cli(args_bad_wall_time, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
with pytest.raises(SystemExit) as exit_error:
args_bad_wall_time.walltime = 'bb:bb:bb'
main_cli(args_bad_wall_time, parser, check_inputs=True)
assert exit_error.type == SystemExit
assert exit_error.value.code == 1
def test_end2end_ventoux_egm96_geoid():
"""
End to end processing
"""
# Force max RAM to 1000 to get stable tiling in tests
os.environ['OTB_MAX_RAM_HINT'] = '1000'
input_json = read_input_parameters(
absolute_data_path("input/phr_ventoux/preproc_input.json"))
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
out_preproc = os.path.join(directory, "out_preproc")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
elevation_delta_lower_bound=-20.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
check_inputs=True)
# Check preproc properties
preproc_json = os.path.join(out_preproc, "content.json")
assert os.path.isfile(preproc_json)
with open(preproc_json, 'r') as f:
preproc_data = json.load(f)
assert preproc_data["preprocessing"]["output"][
"epipolar_size_x"] == 612
assert preproc_data["preprocessing"]["output"][
"epipolar_size_y"] == 612
assert - \
20 < preproc_data["preprocessing"]["output"]["minimum_disparity"] < -18
assert 14 < preproc_data["preprocessing"]["output"][
"maximum_disparity"] < 15
for img in [
"matches", "right_epipolar_grid", "left_epipolar_grid"
]:
assert os.path.isfile(
os.path.join(out_preproc,
preproc_data["preprocessing"]["output"][img]))
out_stereo = os.path.join(directory, "out_preproc")
corr_config = corr_cfg.configure_correlator()
compute_dsm.run(
[read_preprocessing_content_file(preproc_json)],
out_stereo,
resolution=0.5,
epsg=32631,
sigma=0.3,
dsm_radius=3,
dsm_no_data=-999,
color_no_data=0,
corr_config=corr_config,
mode="local_dask", # Run on a local cluster,
nb_workers=4,
walltime="00:10:00",
use_geoid_alt=True,
use_sec_disp=True)
# Uncomment the 2 following instructions to update reference data
#copy2(os.path.join(out_stereo, 'dsm.tif'),
# absolute_data_path("ref_output/dsm_end2end_ventoux_egm96.tif"))
#copy2(os.path.join(out_stereo, 'clr.tif'),
# absolute_data_path("ref_output/clr_end2end_ventoux.tif"))
assert_same_images(
os.path.join(out_stereo, "dsm.tif"),
absolute_data_path("ref_output/dsm_end2end_ventoux_egm96.tif"),
atol=0.0001,
rtol=1e-6)
assert_same_images(
os.path.join(out_stereo, "clr.tif"),
absolute_data_path("ref_output/clr_end2end_ventoux.tif"),
rtol=1.e-7,
atol=1.e-7)
# Test that we have the same results without setting the color1
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
input_json = read_input_parameters(
absolute_data_path(
"input/phr_ventoux/preproc_input_without_color.json"))
out_preproc = os.path.join(directory, "out_preproc")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
elevation_delta_lower_bound=-20.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00")
preproc_json = os.path.join(out_preproc, "content.json")
out_stereo = os.path.join(directory, "out_preproc")
corr_config = corr_cfg.configure_correlator()
compute_dsm.run(
[read_preprocessing_content_file(preproc_json)],
out_stereo,
resolution=0.5,
epsg=32631,
sigma=0.3,
dsm_radius=3,
dsm_no_data=-999,
color_no_data=0,
corr_config=corr_config,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
use_geoid_alt=True,
use_sec_disp=True)
assert_same_images(
os.path.join(out_stereo, "dsm.tif"),
absolute_data_path("ref_output/dsm_end2end_ventoux_egm96.tif"),
atol=0.0001,
rtol=1e-6)
assert_same_images(
os.path.join(out_stereo, "clr.tif"),
absolute_data_path("ref_output/clr_end2end_ventoux.tif"),
rtol=1.e-7,
atol=1.e-7)
def test_end2end_paca_with_mask():
"""
End to end processing
"""
# Force max RAM to 1000 to get stable tiling in tests
os.environ['OTB_MAX_RAM_HINT'] = '1000'
input_json = read_input_parameters(
absolute_data_path("input/phr_paca/preproc_input.json"))
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
out_preproc = os.path.join(directory, "out_preproc")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
elevation_delta_lower_bound=-20.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
check_inputs=True)
# Check preproc properties
preproc_json = os.path.join(out_preproc, "content.json")
out_stereo = os.path.join(directory, "out_stereo")
corr_config = corr_cfg.configure_correlator()
compute_dsm.run(
[read_preprocessing_content_file(preproc_json)],
out_stereo,
resolution=0.5,
epsg=32631,
sigma=0.3,
dsm_radius=3,
dsm_no_data=-999,
color_no_data=0,
msk_no_data=65534,
corr_config=corr_config,
mode="local_dask", # Run on a local cluster,
output_stats=True,
nb_workers=4,
walltime="00:10:00",
use_sec_disp=True)
# Uncomment the 2 following instructions to update reference data
# copy2(os.path.join(out_stereo, 'dsm.tif'),
# absolute_data_path("ref_output/dsm_end2end_paca.tif"))
# copy2(os.path.join(out_stereo, 'clr.tif'),
# absolute_data_path("ref_output/clr_end2end_paca.tif"))
# copy2(os.path.join(out_stereo, 'msk.tif'),
# absolute_data_path("ref_output/msk_end2end_paca.tif"))
assert_same_images(os.path.join(out_stereo, "dsm.tif"),
absolute_data_path(
"ref_output/dsm_end2end_paca.tif"),
atol=0.0001, rtol=1e-6)
assert_same_images(os.path.join(out_stereo, "clr.tif"),
absolute_data_path(
"ref_output/clr_end2end_paca.tif"),
rtol=1.e-7, atol=1.e-7)
assert_same_images(os.path.join(out_stereo, "msk.tif"),
absolute_data_path(
"ref_output/msk_end2end_paca.tif"),
rtol=1.e-7, atol=1.e-7)
# Test we have the same results with multiprocessing
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
out_preproc = os.path.join(directory, "out_preproc")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
elevation_delta_lower_bound=-20.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
check_inputs=True)
# Check preproc properties
preproc_json = os.path.join(out_preproc, "content.json")
out_stereo = os.path.join(directory, "out_stereo")
corr_config = corr_cfg.configure_correlator()
compute_dsm.run(
[read_preprocessing_content_file(preproc_json)],
out_stereo,
resolution=0.5,
epsg=32631,
sigma=0.3,
dsm_radius=3,
dsm_no_data=-999,
color_no_data=0,
msk_no_data=65534,
corr_config=corr_config,
mode="mp",
output_stats=True,
nb_workers=4,
walltime="00:10:00",
use_sec_disp=True)
# Uncomment the 2 following instructions to update reference data
# copy2(os.path.join(out_stereo, 'dsm.tif'),
# absolute_data_path("ref_output/dsm_end2end_paca.tif"))
# copy2(os.path.join(out_stereo, 'clr.tif'),
# absolute_data_path("ref_output/clr_end2end_paca.tif"))
# copy2(os.path.join(out_stereo, 'msk.tif'),
# absolute_data_path("ref_output/msk_end2end_paca.tif"))
assert_same_images(os.path.join(out_stereo, "dsm.tif"),
absolute_data_path(
"ref_output/dsm_end2end_paca.tif"),
atol=0.0001, rtol=1e-6)
assert_same_images(os.path.join(out_stereo, "clr.tif"),
absolute_data_path(
"ref_output/clr_end2end_paca.tif"),
rtol=1.e-7, atol=1.e-7)
assert_same_images(os.path.join(out_stereo, "msk.tif"),
absolute_data_path(
"ref_output/msk_end2end_paca.tif"),
rtol=1.e-7, atol=1.e-7)
