def test_prepare_ventoux_bias():
"""
Dask prepare with bias geoms
"""
input_json = read_input_parameters(absolute_data_path(
"input/phr_ventoux/preproc_input_bias.json"))
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
out_preproc = os.path.join(directory, "out_preproc_bias")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
epipolar_error_maximum_bias=50.,
elevation_delta_lower_bound=-120.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00")
# Check preproc properties
preproc_json = os.path.join(out_preproc, "content.json")
assert os.path.isfile(preproc_json)
with open(preproc_json, 'r') as preproc_json_file:
preproc_data = json.load(preproc_json_file)
preproc_output = preproc_data["preprocessing"]["output"]
assert preproc_output["epipolar_size_x"] == 612
assert preproc_output["epipolar_size_y"] == 612
assert preproc_output["minimum_disparity"] > -86
assert preproc_output["minimum_disparity"] < -84
assert preproc_output["maximum_disparity"] > -46
assert preproc_output["maximum_disparity"] < -44
for img in [
"matches",
"right_epipolar_grid",
"left_epipolar_grid"]:
assert os.path.isfile(
os.path.join(
out_preproc,
preproc_data["preprocessing"]["output"][img]))
def main_cli(args, parser, check_inputs=False):
"""
Main for command line management
:param check_inputs: activate only arguments checking
"""
# TODO : refactor in order to avoid a slow argparse
# Don't move the local function imports for now
import re
import sys
import logging
from cars import prepare
from cars import compute_dsm
from cars import parameters as params
from cars import configuration_correlator as corr_cfg
# logging
numeric_level = getattr(logging, args.loglevel.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % args.loglevel)
logging.basicConfig(
level=numeric_level,
datefmt='%y-%m-%d %H:%M:%S',
format='%(asctime)s :: %(levelname)s :: %(message)s')
# main(s) for each command
if args.command == "prepare":
# Check remaining arguments
stop_now = False
if not os.path.exists(args.injson):
logging.critical('{} does not exist'.format(args.injson))
stop_now = True
if args.epi_step < 1:
logging.critical('{} is an invalid value for --epi_step parameter \
(should be > 1)'.format(args.epi_step))
stop_now = True
if args.disparity_margin < 0 or args.disparity_margin > 1:
logging.critical(
'{} is an invalid value for --disparity_margin parameter \
(should be in range [0,1])'.format(
args.disparity_margin))
stop_now = True
if args.epipolar_error_upper_bound <= 0:
logging.critical(
'{} is an invalid value for --epipolar_error_upper_bound \
parameter (should be > 0)'.format(
args.epipolar_error_upper_bound))
stop_now = True
if args.epipolar_error_maximum_bias < 0:
logging.critical(
'{} is an invalid value for --epipolar_error_maximum_bias \
parameter (should be >= 0)'.format(
args.epipolar_error_maximum_bias))
stop_now = True
if args.nb_workers < 1:
logging.critical(
'{} is an invalid value for --nb_workers parameter \
(should be > 0)'.format(
args.nb_workers))
stop_now = True
if not re.match(r'[0-9]{2}:[0-9]{2}:[0-9]{2}', args.walltime):
logging.critical('{} is an invalid value for --walltime parameter \
(should match HH:MM:SS)'.format(args.walltime))
stop_now = True
if args.elevation_delta_upper_bound <= args.elevation_delta_lower_bound:
logging.critical(
'--elevation_delta_lower_bound = {} is greater than \
--elevation_delta_upper_bound = {}'.format(
args.elevation_delta_lower_bound,
args.elevation_delta_upper_bound))
stop_now = True
# If there are invalid parameters, stop now
if stop_now:
logging.critical(
"Invalid parameters detected, please fix cars_cli \
prepare command-line.")
sys.exit(1)
# Read input json file
in_json = params.read_input_parameters(args.injson)
if not check_inputs:
prepare.run(
in_json,
args.outdir,
epi_step=args.epi_step,
disparity_margin=args.disparity_margin,
epipolar_error_upper_bound=args.epipolar_error_upper_bound,
epipolar_error_maximum_bias=args.epipolar_error_maximum_bias,
elevation_delta_lower_bound=args.elevation_delta_lower_bound,
elevation_delta_upper_bound=args.elevation_delta_upper_bound,
mode=args.mode,
nb_workers=args.nb_workers,
walltime=args.walltime,
check_inputs=args.check_inputs)
elif args.command == "compute_dsm":
# Check remaining arguments
stop_now = False
if len(args.injsons) == 0:
logging.critical('At least one input json file is required')
stop_now = True
for json_file in args.injsons:
if not os.path.exists(json_file):
logging.critical('{} does not exist'.format(json_file))
stop_now = True
if args.sigma is not None and args.sigma < 0:
logging.critical('{} is an invalid value for --sigma parameter \
(should be >= 0)'.format(args.sigma))
stop_now = True
if args.dsm_radius < 0:
logging.critical(
'{} is an invalid value for --dsm_radius parameter \
(should be >= 0)'.format(
args.dsm_radius))
stop_now = True
if args.resolution <= 0:
logging.critical(
'{} is an invalid value for --resolution parameter \
(should be > 0)'.format(
args.resolution))
stop_now = True
if args.epsg is not None and args.epsg < 1:
logging.critical('{} is an invalid value for --epsg parameter \
(should be > 0)'.format(args.epsg))
stop_now = True
if args.corr_config is not None:
if not os.path.exists(args.corr_config):
logging.critical('{} does not exist'.format(args.corr_config))
stop_now = True
if args.nb_workers < 1:
logging.critical(
'{} is an invalid value for --nb_workers parameter \
(should be > 0)'.format(
args.nb_workers))
stop_now = True
if not re.match(r'[0-9]{2}:[0-9]{2}:[0-9]{2}', args.walltime):
logging.critical('{} is an invalid value for --walltime parameter \
(should match HH:MM:SS)'.format(args.walltime))
stop_now = True
if args.max_elevation_offset is not None \
and args.min_elevation_offset is not None \
and args.max_elevation_offset <= args.min_elevation_offset:
logging.critical('--min_elevation_offset = {} is greater than \
--max_elevation_offset = {}'.format(args.min_elevation_offset,
args.max_elevation_offset))
stop_now = True
# By default roi = None if no roi mutually exclusive options
roi = None
if args.roi_bbox is not None:
# if roi_bbox defined, roi = 4 floats bounding box + EPSG code=None
roi = (args.roi_bbox, None)
if args.roi_file is not None:
# If roi_file is defined, generate bouding box roi
roi, stop_now = parse_roi_file(args.roi_file, stop_now)
# If there are invalid parameters, stop now
if stop_now:
logging.critical(
"Invalid parameters detected, please fix cars_cli \
compute_dsm command-line.")
sys.exit(1)
# Read input json files
in_jsons = [params.read_preprocessing_content_file(
f) for f in args.injsons]
# Configure correlator
corr_config = corr_cfg.configure_correlator(args.corr_config)
if not check_inputs:
compute_dsm.run(
in_jsons,
args.outdir,
resolution = args.resolution,
min_elevation_offset = args.min_elevation_offset,
max_elevation_offset = args.max_elevation_offset,
epsg = args.epsg,
sigma = args.sigma,
dsm_radius = args.dsm_radius,
dsm_no_data = args.dsm_no_data,
color_no_data = args.color_no_data,
msk_no_data = args.msk_no_data,
corr_config = corr_config,
output_stats = args.output_stats,
mode = args.mode,
nb_workers = args.nb_workers,
walltime = args.walltime,
roi = roi,
use_geoid_alt = args.use_geoid_as_alt_ref,
snap_to_img1 = args.snap_to_left_image,
align = args.align_with_lowres_dem,
cloud_small_components_filter = \
not args.disable_cloud_small_components_filter,
cloud_statistical_outliers_filter = \
not args.disable_cloud_statistical_outliers_filter,
use_sec_disp = args.use_sec_disp
)
else:
parser.print_help()
sys.exit(1)
def test_end2end_ventoux_egm96_geoid():
"""
End to end processing
"""
# Force max RAM to 1000 to get stable tiling in tests
os.environ['OTB_MAX_RAM_HINT'] = '1000'
input_json = read_input_parameters(
absolute_data_path("input/phr_ventoux/preproc_input.json"))
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
out_preproc = os.path.join(directory, "out_preproc")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
elevation_delta_lower_bound=-20.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
check_inputs=True)
# Check preproc properties
preproc_json = os.path.join(out_preproc, "content.json")
assert os.path.isfile(preproc_json)
with open(preproc_json, 'r') as f:
preproc_data = json.load(f)
assert preproc_data["preprocessing"]["output"][
"epipolar_size_x"] == 612
assert preproc_data["preprocessing"]["output"][
"epipolar_size_y"] == 612
assert - \
20 < preproc_data["preprocessing"]["output"]["minimum_disparity"] < -18
assert 14 < preproc_data["preprocessing"]["output"][
"maximum_disparity"] < 15
for img in [
"matches", "right_epipolar_grid", "left_epipolar_grid"
]:
assert os.path.isfile(
os.path.join(out_preproc,
preproc_data["preprocessing"]["output"][img]))
out_stereo = os.path.join(directory, "out_preproc")
corr_config = corr_cfg.configure_correlator()
compute_dsm.run(
[read_preprocessing_content_file(preproc_json)],
out_stereo,
resolution=0.5,
epsg=32631,
sigma=0.3,
dsm_radius=3,
dsm_no_data=-999,
color_no_data=0,
corr_config=corr_config,
mode="local_dask", # Run on a local cluster,
nb_workers=4,
walltime="00:10:00",
use_geoid_alt=True,
use_sec_disp=True)
# Uncomment the 2 following instructions to update reference data
#copy2(os.path.join(out_stereo, 'dsm.tif'),
# absolute_data_path("ref_output/dsm_end2end_ventoux_egm96.tif"))
#copy2(os.path.join(out_stereo, 'clr.tif'),
# absolute_data_path("ref_output/clr_end2end_ventoux.tif"))
assert_same_images(
os.path.join(out_stereo, "dsm.tif"),
absolute_data_path("ref_output/dsm_end2end_ventoux_egm96.tif"),
atol=0.0001,
rtol=1e-6)
assert_same_images(
os.path.join(out_stereo, "clr.tif"),
absolute_data_path("ref_output/clr_end2end_ventoux.tif"),
rtol=1.e-7,
atol=1.e-7)
# Test that we have the same results without setting the color1
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
input_json = read_input_parameters(
absolute_data_path(
"input/phr_ventoux/preproc_input_without_color.json"))
out_preproc = os.path.join(directory, "out_preproc")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
elevation_delta_lower_bound=-20.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00")
preproc_json = os.path.join(out_preproc, "content.json")
out_stereo = os.path.join(directory, "out_preproc")
corr_config = corr_cfg.configure_correlator()
compute_dsm.run(
[read_preprocessing_content_file(preproc_json)],
out_stereo,
resolution=0.5,
epsg=32631,
sigma=0.3,
dsm_radius=3,
dsm_no_data=-999,
color_no_data=0,
corr_config=corr_config,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
use_geoid_alt=True,
use_sec_disp=True)
assert_same_images(
os.path.join(out_stereo, "dsm.tif"),
absolute_data_path("ref_output/dsm_end2end_ventoux_egm96.tif"),
atol=0.0001,
rtol=1e-6)
assert_same_images(
os.path.join(out_stereo, "clr.tif"),
absolute_data_path("ref_output/clr_end2end_ventoux.tif"),
rtol=1.e-7,
atol=1.e-7)
def test_compute_dsm_with_snap_to_img1():
"""
Dask compute dsm processing with input roi (cars_stereo)
"""
# Force max RAM to 1000 to get stable tiling in tests
os.environ['OTB_MAX_RAM_HINT'] = '1000'
input_json = read_input_parameters(
absolute_data_path("input/phr_ventoux/preproc_input.json"))
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
out_preproc = os.path.join(directory, "out_preproc")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
elevation_delta_lower_bound=-20.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00")
# Check preproc properties
preproc_json = os.path.join(out_preproc, "content.json")
assert os.path.isfile(preproc_json)
out_stereo = os.path.join(directory, "out_preproc")
final_epsg = 32631
resolution = 0.5
corr_config = corr_cfg.configure_correlator()
compute_dsm.run(
[read_preprocessing_content_file(preproc_json)],
out_stereo,
resolution=resolution,
epsg=final_epsg,
sigma=0.3,
dsm_radius=3,
dsm_no_data=-999,
color_no_data=0,
corr_config=corr_config,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
snap_to_img1=True)
# Uncomment the 2 following instructions to update reference data
#copy2(os.path.join(out_stereo, 'dsm.tif'),
# absolute_data_path("ref_output/dsm_end2end_ventoux_with_snap_to_img1.tif"))
#copy2(os.path.join(out_stereo, 'clr.tif'),
# absolute_data_path("ref_output/clr_end2end_ventoux_with_snap_to_img1.tif"))
assert_same_images(
os.path.join(out_stereo, "dsm.tif"),
absolute_data_path(
"ref_output/dsm_end2end_ventoux_with_snap_to_img1.tif"),
atol=0.0001,
rtol=1e-6)
assert_same_images(
os.path.join(out_stereo, "clr.tif"),
absolute_data_path(
"ref_output/clr_end2end_ventoux_with_snap_to_img1.tif"),
rtol=1.e-7,
atol=1.e-7)
def test_compute_dsm_with_roi_ventoux():
"""
Dask compute dsm processing with input roi (cars_stereo)
"""
# Force max RAM to 1000 to get stable tiling in tests
os.environ['OTB_MAX_RAM_HINT'] = '1000'
input_json = read_input_parameters(
absolute_data_path("input/phr_ventoux/preproc_input.json"))
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
out_preproc = os.path.join(directory, "out_preproc")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
elevation_delta_lower_bound=-20.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
check_inputs=True)
preproc_json = os.path.join(out_preproc, "content.json")
out_stereo = os.path.join(directory, "out_preproc")
final_epsg = 32631
resolution = 0.5
roi = [5.194, 44.2059, 5.195, 44.2064]
roi_epsg = 4326
corr_config = corr_cfg.configure_correlator()
compute_dsm.run(
[read_preprocessing_content_file(preproc_json)],
out_stereo,
resolution=resolution,
epsg=final_epsg,
sigma=0.3,
dsm_radius=3,
dsm_no_data=-999,
color_no_data=0,
corr_config=corr_config,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
roi=(roi, roi_epsg))
# Uncomment the 2 following instructions to update reference data
#copy2(os.path.join(out_stereo, 'dsm.tif'),
# absolute_data_path("ref_output/dsm_end2end_ventoux_with_roi.tif"))
#copy2(os.path.join(out_stereo, 'clr.tif'),
# absolute_data_path("ref_output/clr_end2end_ventoux_with_roi.tif"))
assert_same_images(
os.path.join(out_stereo, "dsm.tif"),
absolute_data_path("ref_output/dsm_end2end_ventoux_with_roi.tif"),
atol=0.0001,
rtol=1e-6)
assert_same_images(
os.path.join(out_stereo, "clr.tif"),
absolute_data_path("ref_output/clr_end2end_ventoux_with_roi.tif"),
rtol=1.e-7,
atol=1.e-7)
# check final bounding box
# create reference
[roi_xmin, roi_ymin, roi_xmax, roi_ymax] = roi
roi_poly = Polygon([(roi_xmin, roi_ymin), (roi_xmax, roi_ymin),
(roi_xmax, roi_ymax), (roi_xmin, roi_ymax),
(roi_xmin, roi_ymin)])
project = pyproj.Transformer.from_proj(
pyproj.Proj(init='epsg:{}'.format(roi_epsg)),
pyproj.Proj(init='epsg:{}'.format(final_epsg)))
ref_roi_poly = transform(project.transform, roi_poly)
[ref_xmin, ref_ymin, ref_xmax, ref_ymax] = ref_roi_poly.bounds
# retrieve bounding box of computed dsm
data = rasterio.open(os.path.join(out_stereo, "dsm.tif"))
xmin = min(data.bounds.left, data.bounds.right)
ymin = min(data.bounds.bottom, data.bounds.top)
xmax = max(data.bounds.left, data.bounds.right)
ymax = max(data.bounds.bottom, data.bounds.top)
assert math.floor(ref_xmin / resolution) * resolution == xmin
assert math.ceil(ref_xmax / resolution) * resolution == xmax
assert math.floor(ref_ymin / resolution) * resolution == ymin
assert math.ceil(ref_ymax / resolution) * resolution == ymax
def test_end2end_paca_with_mask():
"""
End to end processing
"""
# Force max RAM to 1000 to get stable tiling in tests
os.environ['OTB_MAX_RAM_HINT'] = '1000'
input_json = read_input_parameters(
absolute_data_path("input/phr_paca/preproc_input.json"))
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
out_preproc = os.path.join(directory, "out_preproc")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
elevation_delta_lower_bound=-20.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
check_inputs=True)
# Check preproc properties
preproc_json = os.path.join(out_preproc, "content.json")
out_stereo = os.path.join(directory, "out_stereo")
corr_config = corr_cfg.configure_correlator()
compute_dsm.run(
[read_preprocessing_content_file(preproc_json)],
out_stereo,
resolution=0.5,
epsg=32631,
sigma=0.3,
dsm_radius=3,
dsm_no_data=-999,
color_no_data=0,
msk_no_data=65534,
corr_config=corr_config,
mode="local_dask", # Run on a local cluster,
output_stats=True,
nb_workers=4,
walltime="00:10:00",
use_sec_disp=True)
# Uncomment the 2 following instructions to update reference data
# copy2(os.path.join(out_stereo, 'dsm.tif'),
# absolute_data_path("ref_output/dsm_end2end_paca.tif"))
# copy2(os.path.join(out_stereo, 'clr.tif'),
# absolute_data_path("ref_output/clr_end2end_paca.tif"))
# copy2(os.path.join(out_stereo, 'msk.tif'),
# absolute_data_path("ref_output/msk_end2end_paca.tif"))
assert_same_images(os.path.join(out_stereo, "dsm.tif"),
absolute_data_path(
"ref_output/dsm_end2end_paca.tif"),
atol=0.0001, rtol=1e-6)
assert_same_images(os.path.join(out_stereo, "clr.tif"),
absolute_data_path(
"ref_output/clr_end2end_paca.tif"),
rtol=1.e-7, atol=1.e-7)
assert_same_images(os.path.join(out_stereo, "msk.tif"),
absolute_data_path(
"ref_output/msk_end2end_paca.tif"),
rtol=1.e-7, atol=1.e-7)
# Test we have the same results with multiprocessing
with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
out_preproc = os.path.join(directory, "out_preproc")
prepare.run(
input_json,
out_preproc,
epi_step=30,
region_size=250,
disparity_margin=0.25,
epipolar_error_upper_bound=43.,
elevation_delta_lower_bound=-20.,
elevation_delta_upper_bound=20.,
mode="local_dask", # Run on a local cluster
nb_workers=4,
walltime="00:10:00",
check_inputs=True)
# Check preproc properties
preproc_json = os.path.join(out_preproc, "content.json")
out_stereo = os.path.join(directory, "out_stereo")
corr_config = corr_cfg.configure_correlator()
compute_dsm.run(
[read_preprocessing_content_file(preproc_json)],
out_stereo,
resolution=0.5,
epsg=32631,
sigma=0.3,
dsm_radius=3,
dsm_no_data=-999,
color_no_data=0,
msk_no_data=65534,
corr_config=corr_config,
mode="mp",
output_stats=True,
nb_workers=4,
walltime="00:10:00",
use_sec_disp=True)
# Uncomment the 2 following instructions to update reference data
# copy2(os.path.join(out_stereo, 'dsm.tif'),
# absolute_data_path("ref_output/dsm_end2end_paca.tif"))
# copy2(os.path.join(out_stereo, 'clr.tif'),
# absolute_data_path("ref_output/clr_end2end_paca.tif"))
# copy2(os.path.join(out_stereo, 'msk.tif'),
# absolute_data_path("ref_output/msk_end2end_paca.tif"))
assert_same_images(os.path.join(out_stereo, "dsm.tif"),
absolute_data_path(
"ref_output/dsm_end2end_paca.tif"),
atol=0.0001, rtol=1e-6)
assert_same_images(os.path.join(out_stereo, "clr.tif"),
absolute_data_path(
"ref_output/clr_end2end_paca.tif"),
rtol=1.e-7, atol=1.e-7)
assert_same_images(os.path.join(out_stereo, "msk.tif"),
absolute_data_path(
"ref_output/msk_end2end_paca.tif"),
rtol=1.e-7, atol=1.e-7)