def test_rasterio_handles(): """ Test to create a file handle(s) depending on whether the output data has a color layer or not. """ bounds = (675248.0, 4897075.0, 675460.5, 4897173.0) resolution = 0.5 geotransform = (bounds[0], resolution, 0.0, bounds[3], 0.0, -resolution) transform = Affine.from_gdal(*geotransform) rio_params = dict( height=196, width=425, driver='GTiff', dtype=np.float32, transform=transform, crs='EPSG:{}'.format(32631), tiled=True ) dsm_no_data = -32768 color_no_data = 0 nb_bands = 1 # Create file handles with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: dsm_file = os.path.join(directory, 'dsm.tif') clr_file = os.path.join(directory, 'clr.tif') file_handles = readwrite.rasterio_handles(['hgt', 'clr'], [dsm_file, clr_file], [rio_params, rio_params], [dsm_no_data, color_no_data], [1, nb_bands]) with file_handles as rio_handles: assert isinstance(rio_handles, dict) == True assert 'hgt' in rio_handles.keys() and 'clr' in rio_handles.keys() for key in rio_handles.keys(): assert isinstance(rio_handles[key], rio.io.DatasetWriter)
def test_write_ply(): """ Test write ply file """ points = xr.open_dataset( absolute_data_path("input/intermediate_results/points_ref.nc")) utils.write_ply(os.path.join(temporary_dir(), 'test.ply'), points)
def test_epipolar_distributions(): cars_path = os.environ.get('CARSPATH') # uncomment the following lines to regenerate the input files # input_json = read_input_parameters( # absolute_data_path("input/phr_ventoux/preproc_input.json")) # prepare.run( # input_json, # absolute_data_path("input/notebooks_input/content_dir/"), # epi_step=30, # region_size=250, # disparity_margin=0.25, # epipolar_error_upper_bound=43., # elevation_delta_lower_bound=-20., # elevation_delta_upper_bound=20., # mode="local", # Run on a local cluster # nb_workers=4, # walltime="00:10:00", # check_inputs=True) with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: subprocess.run(['jupyter nbconvert --to script {}/notebooks/epipolar_distributions.ipynb --output-dir {}' .format(cars_path, directory)], shell=True) for line in fileinput.input('{}/epipolar_distributions.py'.format(directory), inplace=True): if "cars_home = \"TODO\"" in line: line = line.replace("TODO", cars_path) elif "content_dir = \"TODO\"" in line: line = line.replace("TODO", absolute_data_path("input/notebooks_input/content_dir/")) print(line) # keep this print out = subprocess.run(['ipython {}/epipolar_distributions.py'.format(directory)], shell=True) out.check_returncode()
def test_lowres_dem_fit(): """ Low resolution notebook test: notebook conversion (.ipynb->.py), replace default TODO values, run the notebook with ipython and check return code """ cars_path = os.environ.get('CARSPATH') with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: subprocess.run([ 'jupyter nbconvert ' '--to script {}/notebooks/lowres_dem_fit.ipynb ' '--output-dir {}'.format(cars_path, directory) ], shell=True, check=True) for line in fileinput.input('{}/lowres_dem_fit.py'.format(directory), inplace=True): if "cars_home = \"TODO\"" in line: line = line.replace("TODO", cars_path) elif "content_dir = \"TODO\"" in line: line = line.replace( "TODO", absolute_data_path("input/notebooks_input/lowres_dem/")) print(line) # keep this print out = subprocess.run( ['ipython {}/lowres_dem_fit.py'.format(directory)], shell=True, check=True) out.check_returncode()
def test_step_by_step_compute_dsm(): """ Step by step compute dsm notebook test: notebook conversion (.ipynb->.py), replace default TODO values, run the notebook with ipython and check return code """ cars_path = os.environ.get('CARSPATH') # uncomment the following lines to regenerate the input files # input_json = read_input_parameters( # absolute_data_path("input/phr_ventoux/preproc_input.json")) # prepare.run( # input_json, # absolute_data_path("input/notebooks_input/content_dir/"), # epi_step=30, # region_size=250, # disparity_margin=0.25, # epipolar_error_upper_bound=43., # elevation_delta_lower_bound=-20., # elevation_delta_upper_bound=20., # mode="local", # Run on a local cluster # nb_workers=4, # walltime="00:10:00", # check_inputs=True) with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: subprocess.run([ 'jupyter nbconvert ' '--to script ' '{}/notebooks/step_by_step_compute_dsm.ipynb ' '--output-dir {}'.format(cars_path, directory) ], shell=True, check=True) for line in fileinput.input( '{}/step_by_step_compute_dsm.py'.format(directory), inplace=True): if "cars_home = \"TODO\"" in line: line = line.replace("TODO", cars_path) elif "content_dir = \"TODO\"" in line: line = line.replace( "TODO", absolute_data_path("input/notebooks_input/content_dir/")) elif "roi_file = \"TODO\"" in line: line = line.replace( "TODO", absolute_data_path("input/notebooks_input/content_dir/" "envelopes_intersection.gpkg")) elif "output_dir = \"TODO\"" in line: line = line.replace("TODO", directory) print(line) # keep this print out = subprocess.run( ['ipython {}/step_by_step_compute_dsm.py'.format(directory)], shell=True, check=True) out.check_returncode()
def test_cluster(): """ End to end cluster management test """ with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: clus, client = cluster.start_cluster(2, "00:01:00", directory) link = cluster.get_dashboard_link(clus) cluster.stop_cluster(clus, client)
def test_image_envelope(): """ Test image_envelope function """ img = absolute_data_path("input/phr_ventoux/left_image.tif") dem = absolute_data_path("input/phr_ventoux/srtm") with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: shp = os.path.join(directory, "envelope.gpkg") preprocessing.image_envelope(img, shp, dem) assert os.path.isfile(shp)
def test_configure_pandora_with_file(): """ Test configure pandora correlator """ json_content = """{ "image":{ "nodata1":"np.nan", "nodata2":"np.nan" }, "stereo" : { "stereo_method": "census", "window_size": 5, "subpix": 1 }, "aggregation" : { "aggregation_method": "none" }, "optimization" : { "optimization_method": "sgm", "P1": 8, "P2": 24, "p2_method": "constant", "penalty_method": "sgm_penalty", "overcounting": false, "min_cost_paths": false }, "refinement": { "refinement_method": "vfit" }, "filter" : { "filter_method": "median", "filter_size": 3 }, "validation" : { "validation_method": "cross_checking", "interpolated_disparity": "none" }, "invalid_disparity": "np.nan" } """ with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: json_path = os.path.join(directory, "corr.json") with open(json_path, "w") as json_file: json_file.write(json_content) corr_config = corr_cfg.configure_correlator(json_path) assert corr_config["optimization"]["P2"] == 24
def test_prepare_ventoux_bias(): """ Dask prepare with bias geoms """ input_json = read_input_parameters(absolute_data_path( "input/phr_ventoux/preproc_input_bias.json")) with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: out_preproc = os.path.join(directory, "out_preproc_bias") prepare.run( input_json, out_preproc, epi_step=30, region_size=250, disparity_margin=0.25, epipolar_error_upper_bound=43., epipolar_error_maximum_bias=50., elevation_delta_lower_bound=-120., elevation_delta_upper_bound=20., mode="local_dask", # Run on a local cluster nb_workers=4, walltime="00:10:00") # Check preproc properties preproc_json = os.path.join(out_preproc, "content.json") assert os.path.isfile(preproc_json) with open(preproc_json, 'r') as preproc_json_file: preproc_data = json.load(preproc_json_file) preproc_output = preproc_data["preprocessing"]["output"] assert preproc_output["epipolar_size_x"] == 612 assert preproc_output["epipolar_size_y"] == 612 assert preproc_output["minimum_disparity"] > -86 assert preproc_output["minimum_disparity"] < -84 assert preproc_output["maximum_disparity"] > -46 assert preproc_output["maximum_disparity"] < -44 for img in [ "matches", "right_epipolar_grid", "left_epipolar_grid"]: assert os.path.isfile( os.path.join( out_preproc, preproc_data["preprocessing"]["output"][img]))
def test_write_vector(): polys = [ Polygon([(1.0, 1.0), (1.0, 2.0), (2.0, 2.0), (2.0, 1.0)]), Polygon([(2.0, 2.0), (2.0, 3.0), (3.0, 3.0), (3.0, 2.0)]) ] with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: path_to_file = os.path.join(directory, 'test.gpkg') utils.write_vector(polys, path_to_file, 4326) assert os.path.exists(path_to_file) nb_feat = 0 for feat in fiona.open(path_to_file): poly = shape(feat['geometry']) nb_feat += 1 assert poly in polys assert nb_feat == 2
def test_write_geotiff_dsm(): """ Test to result tiles to GTiff file(s). """ with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: # Set variables and parameters for create dataset resolution = 0.5 epsg = 32630 dsm_no_data = -32768 hgt_no_data = dsm_no_data color_no_data = 0 msk_no_data = 255 xstart, ystart, xsize, ysize = [0, 10, 10, 10] raster = np.ndarray(shape=(10, 10, 2), dtype=np.float32) mean = np.ndarray(shape=(10, 10, 2), dtype=np.float32) stdev = np.ndarray(shape=(10, 10, 2), dtype=np.float32) n_pts = np.ndarray(shape=(10, 10), dtype=np.uint16) n_in_cell = np.ndarray(shape=(10, 10), dtype=np.uint16) msk = np.ndarray(shape=(10, 10), dtype=np.uint16) delayed_raster_datasets = list() delayed_raster_datasets.append( dask.delayed(rasterization.create_raster_dataset)( raster, xstart, ystart, xsize, ysize, resolution, hgt_no_data, color_no_data, epsg, mean, stdev, n_pts, n_in_cell, msk)) # Start cluster with a local cluster nb_workers = 4 cluster, client = start_local_cluster(nb_workers) future_dsm = client.compute(delayed_raster_datasets) # Write netcdf with raster_datasets bounds = [xstart, ystart, xsize, ysize] nb_bands = 1 dsm_file = os.path.join(directory, "dsm.tif") clr_file = os.path.join(directory, "clr.tif") msk_file = os.path.join(directory, "msk.tif") readwrite.write_geotiff_dsm(future_dsm, directory, xsize, ysize, bounds, resolution, epsg, nb_bands, dsm_no_data, color_no_data, write_msk=True, msk_no_data=msk_no_data) # stop cluster stop_local_cluster(cluster, client) width_ref = xsize height_ref = ysize crs = osr.SpatialReference() crs.ImportFromEPSG(epsg) crs_ref = crs.ExportToWkt() geotransform = (bounds[0], resolution, 0.0, bounds[3], 0.0, -resolution) transform_ref = Affine.from_gdal(*geotransform) # Compare DSM with rio.open(dsm_file) as rio_actual: np.testing.assert_equal(rio_actual.width, width_ref) np.testing.assert_equal(rio_actual.height, height_ref) assert rio_actual.transform == transform_ref assert rio_actual.crs == crs_ref assert rio_actual.nodata == hgt_no_data for i in range(rio_actual.width): for j in range(rio_actual.height): if np.isnan(raster[i][j][0]): assert rio_actual.read()[0][i][j] == dsm_no_data else: assert rio_actual.read()[0][i][j] == raster[i][j][0] # Compare CLR with rio.open(clr_file) as rio_actual: np.testing.assert_equal(rio_actual.width, width_ref) np.testing.assert_equal(rio_actual.height, height_ref) assert rio_actual.transform == transform_ref assert rio_actual.crs == crs_ref assert rio_actual.nodata == color_no_data for i in range(rio_actual.width): for j in range(rio_actual.height): if np.isnan(raster[i][j][1]): assert rio_actual.read()[0][i][j] == color_no_data else: assert rio_actual.read()[0][i][j] == raster[i][j][1] # Compare MSK with rio.open(msk_file) as rio_actual: np.testing.assert_equal(rio_actual.width, width_ref) np.testing.assert_equal(rio_actual.height, height_ref) assert rio_actual.transform == transform_ref assert rio_actual.crs == crs_ref assert rio_actual.nodata == msk_no_data for i in range(rio_actual.width): for j in range(rio_actual.height): if np.isnan(raster[i][j][1]): assert rio_actual.read()[0][i][j] == msk_no_data else: assert rio_actual.read()[0][i][j] == msk[i][j]
def test_compute_dsm_with_roi_ventoux(): """ Dask compute dsm processing with input roi (cars_stereo) """ # Force max RAM to 1000 to get stable tiling in tests os.environ['OTB_MAX_RAM_HINT'] = '1000' input_json = read_input_parameters( absolute_data_path("input/phr_ventoux/preproc_input.json")) with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: out_preproc = os.path.join(directory, "out_preproc") prepare.run( input_json, out_preproc, epi_step=30, region_size=250, disparity_margin=0.25, epipolar_error_upper_bound=43., elevation_delta_lower_bound=-20., elevation_delta_upper_bound=20., mode="local_dask", # Run on a local cluster nb_workers=4, walltime="00:10:00", check_inputs=True) preproc_json = os.path.join(out_preproc, "content.json") out_stereo = os.path.join(directory, "out_preproc") final_epsg = 32631 resolution = 0.5 roi = [5.194, 44.2059, 5.195, 44.2064] roi_epsg = 4326 corr_config = corr_cfg.configure_correlator() compute_dsm.run( [read_preprocessing_content_file(preproc_json)], out_stereo, resolution=resolution, epsg=final_epsg, sigma=0.3, dsm_radius=3, dsm_no_data=-999, color_no_data=0, corr_config=corr_config, mode="local_dask", # Run on a local cluster nb_workers=4, walltime="00:10:00", roi=(roi, roi_epsg)) # Uncomment the 2 following instructions to update reference data #copy2(os.path.join(out_stereo, 'dsm.tif'), # absolute_data_path("ref_output/dsm_end2end_ventoux_with_roi.tif")) #copy2(os.path.join(out_stereo, 'clr.tif'), # absolute_data_path("ref_output/clr_end2end_ventoux_with_roi.tif")) assert_same_images( os.path.join(out_stereo, "dsm.tif"), absolute_data_path("ref_output/dsm_end2end_ventoux_with_roi.tif"), atol=0.0001, rtol=1e-6) assert_same_images( os.path.join(out_stereo, "clr.tif"), absolute_data_path("ref_output/clr_end2end_ventoux_with_roi.tif"), rtol=1.e-7, atol=1.e-7) # check final bounding box # create reference [roi_xmin, roi_ymin, roi_xmax, roi_ymax] = roi roi_poly = Polygon([(roi_xmin, roi_ymin), (roi_xmax, roi_ymin), (roi_xmax, roi_ymax), (roi_xmin, roi_ymax), (roi_xmin, roi_ymin)]) project = pyproj.Transformer.from_proj( pyproj.Proj(init='epsg:{}'.format(roi_epsg)), pyproj.Proj(init='epsg:{}'.format(final_epsg))) ref_roi_poly = transform(project.transform, roi_poly) [ref_xmin, ref_ymin, ref_xmax, ref_ymax] = ref_roi_poly.bounds # retrieve bounding box of computed dsm data = rasterio.open(os.path.join(out_stereo, "dsm.tif")) xmin = min(data.bounds.left, data.bounds.right) ymin = min(data.bounds.bottom, data.bounds.top) xmax = max(data.bounds.left, data.bounds.right) ymax = max(data.bounds.bottom, data.bounds.top) assert math.floor(ref_xmin / resolution) * resolution == xmin assert math.ceil(ref_xmax / resolution) * resolution == xmax assert math.floor(ref_ymin / resolution) * resolution == ymin assert math.ceil(ref_ymax / resolution) * resolution == ymax
def test_compute_dsm_with_snap_to_img1(): """ Dask compute dsm processing with input roi (cars_stereo) """ # Force max RAM to 1000 to get stable tiling in tests os.environ['OTB_MAX_RAM_HINT'] = '1000' input_json = read_input_parameters( absolute_data_path("input/phr_ventoux/preproc_input.json")) with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: out_preproc = os.path.join(directory, "out_preproc") prepare.run( input_json, out_preproc, epi_step=30, region_size=250, disparity_margin=0.25, epipolar_error_upper_bound=43., elevation_delta_lower_bound=-20., elevation_delta_upper_bound=20., mode="local_dask", # Run on a local cluster nb_workers=4, walltime="00:10:00") # Check preproc properties preproc_json = os.path.join(out_preproc, "content.json") assert os.path.isfile(preproc_json) out_stereo = os.path.join(directory, "out_preproc") final_epsg = 32631 resolution = 0.5 corr_config = corr_cfg.configure_correlator() compute_dsm.run( [read_preprocessing_content_file(preproc_json)], out_stereo, resolution=resolution, epsg=final_epsg, sigma=0.3, dsm_radius=3, dsm_no_data=-999, color_no_data=0, corr_config=corr_config, mode="local_dask", # Run on a local cluster nb_workers=4, walltime="00:10:00", snap_to_img1=True) # Uncomment the 2 following instructions to update reference data #copy2(os.path.join(out_stereo, 'dsm.tif'), # absolute_data_path("ref_output/dsm_end2end_ventoux_with_snap_to_img1.tif")) #copy2(os.path.join(out_stereo, 'clr.tif'), # absolute_data_path("ref_output/clr_end2end_ventoux_with_snap_to_img1.tif")) assert_same_images( os.path.join(out_stereo, "dsm.tif"), absolute_data_path( "ref_output/dsm_end2end_ventoux_with_snap_to_img1.tif"), atol=0.0001, rtol=1e-6) assert_same_images( os.path.join(out_stereo, "clr.tif"), absolute_data_path( "ref_output/clr_end2end_ventoux_with_snap_to_img1.tif"), rtol=1.e-7, atol=1.e-7)
def test_dsm_compute_arg(compute_dsm_default_args): #pylint: disable=redefined-outer-name """ Cars compute_dsm arguments test with default and degraded cases """ parser = cars_cli_parser() with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: compute_dsm_default_args.outdir = directory # test with default args main_cli(compute_dsm_default_args, parser, check_inputs=True) # test with mp mode (multiprocessing) args_mode_mp = copy(compute_dsm_default_args) args_mode_mp.mode = "mp" main_cli(args_mode_mp, parser, check_inputs=True) # test [xmin, ymin, xmax, ymax] roi argument args_roi_bbox = copy(compute_dsm_default_args) args_roi_bbox.roi_bbox = ['1.0', '2.0', '3.0', '4.0'] main_cli(args_roi_bbox, parser, check_inputs=True) # test image roi argument args_roi_file = copy(compute_dsm_default_args) args_roi_file.roi_file = absolute_data_path( 'input/cars_cli_input/roi_image.tif') main_cli(args_roi_file, parser, check_inputs=True) # test vector roi argument args_roi_file.roi_file = absolute_data_path( 'input/cars_cli_input/roi_vector.gpkg') main_cli(args_roi_file, parser, check_inputs=True) # degraded cases input jsons args_bad_jsons = copy(compute_dsm_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_jsons.injsons = [ absolute_data_path('input/cars_cli_input/test.txt') ] main_cli(args_bad_jsons, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 with pytest.raises(SystemExit) as exit_error: args_bad_jsons.injsons = [] main_cli(args_bad_jsons, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases sigma args_bad_sigma = copy(compute_dsm_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_sigma.sigma = -10 main_cli(args_bad_sigma, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases dsm radius args_bad_dsm_radius = copy(compute_dsm_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_dsm_radius.dsm_radius = -10 main_cli(args_bad_dsm_radius, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases resolution args_bad_resolution = copy(compute_dsm_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_resolution.resolution = 0 main_cli(args_bad_resolution, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 args_bad_resolution = copy(compute_dsm_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_resolution.resolution = -1 main_cli(args_bad_resolution, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases epsg args_bad_epsg = copy(compute_dsm_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_epsg.epsg = -1 main_cli(args_bad_epsg, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases input ROI file args_bad_roi_file = copy(compute_dsm_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_roi_file.roi_file = \ absolute_data_path('input/cars_cli_input/test.txt') main_cli(args_bad_roi_file, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 with pytest.raises(SystemExit) as exit_error: args_bad_roi_file.roi_file = absolute_data_path( 'input/phr_ventoux/preproc_output/content.json') main_cli(args_bad_roi_file, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 with pytest.raises(SystemExit) as exit_error: args_bad_roi_file.roi_file = absolute_data_path( 'input/phr_ventoux/left_image.tif') main_cli(args_bad_roi_file, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases correlator config file args_bad_correlator_conf = copy(compute_dsm_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_correlator_conf.corr_config = absolute_data_path( 'input/cars_cli_input/test.txt') main_cli(args_bad_correlator_conf, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases elevation offsets args_bad_elevation_offsets = copy(compute_dsm_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_elevation_offsets.min_elevation_offset = 10 args_bad_elevation_offsets.max_elevation_offset = -10 main_cli(args_bad_elevation_offsets, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases log level args_bad_loglevel = copy(compute_dsm_default_args) with pytest.raises(ValueError): args_bad_loglevel.loglevel = 'TEST' main_cli(args_bad_loglevel, parser, check_inputs=True) # degraded cases number of workers args_bad_nb_workers = copy(compute_dsm_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_nb_workers.nb_workers = -1 main_cli(args_bad_nb_workers, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases wall time args_bad_wall_time = copy(compute_dsm_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_wall_time.walltime = '000:00:00' main_cli(args_bad_wall_time, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1
def test_prepare_args(prepare_default_args): #pylint: disable=redefined-outer-name """ Cars prepare arguments test with default and degraded cases """ parser = cars_cli_parser() with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: prepare_default_args.outdir = directory # test default args main_cli(prepare_default_args, parser, check_inputs=True) prepare_default_args.loglevel = 'INFO' main_cli(prepare_default_args, parser, check_inputs=True) # degraded cases injson args_bad_json = copy(prepare_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_json.injson = absolute_data_path( 'input/cars_cli_input/test.json') main_cli(args_bad_json, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases disparity_margin args_bad_disp_margin = copy(prepare_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_disp_margin.disparity_margin = -1.0 main_cli(args_bad_disp_margin, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 with pytest.raises(SystemExit) as exit_error: args_bad_disp_margin.disparity_margin = 1.5 main_cli(args_bad_disp_margin, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases elevation bounds args_bad_elev_delta_bounds = copy(prepare_default_args) args_bad_elev_delta_bounds.elevation_delta_lower_bound = 50 args_bad_elev_delta_bounds.elevation_delta_upper_bound = -50 with pytest.raises(SystemExit) as exit_error: main_cli(args_bad_elev_delta_bounds, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases epi_step args_bad_epi_step = copy(prepare_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_epi_step.epi_step = 0 main_cli(args_bad_epi_step, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases epipolar_error_upper_bound args_bad_epi_error_bound = copy(prepare_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_epi_error_bound.epipolar_error_upper_bound = -10 main_cli(args_bad_epi_error_bound, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases log level args_bad_loglevel = copy(prepare_default_args) with pytest.raises(ValueError): args_bad_loglevel.loglevel = 'TEST' main_cli(args_bad_loglevel, parser, check_inputs=True) # degraded cases number of workers args_bad_nb_workers = copy(prepare_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_nb_workers.nb_workers = -1 main_cli(args_bad_nb_workers, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 # degraded cases wall time args_bad_wall_time = copy(prepare_default_args) with pytest.raises(SystemExit) as exit_error: args_bad_wall_time.walltime = '000:00:00' main_cli(args_bad_wall_time, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1 with pytest.raises(SystemExit) as exit_error: args_bad_wall_time.walltime = 'bb:bb:bb' main_cli(args_bad_wall_time, parser, check_inputs=True) assert exit_error.type == SystemExit assert exit_error.value.code == 1
def test_end2end_ventoux_egm96_geoid(): """ End to end processing """ # Force max RAM to 1000 to get stable tiling in tests os.environ['OTB_MAX_RAM_HINT'] = '1000' input_json = read_input_parameters( absolute_data_path("input/phr_ventoux/preproc_input.json")) with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: out_preproc = os.path.join(directory, "out_preproc") prepare.run( input_json, out_preproc, epi_step=30, region_size=250, disparity_margin=0.25, epipolar_error_upper_bound=43., elevation_delta_lower_bound=-20., elevation_delta_upper_bound=20., mode="local_dask", # Run on a local cluster nb_workers=4, walltime="00:10:00", check_inputs=True) # Check preproc properties preproc_json = os.path.join(out_preproc, "content.json") assert os.path.isfile(preproc_json) with open(preproc_json, 'r') as f: preproc_data = json.load(f) assert preproc_data["preprocessing"]["output"][ "epipolar_size_x"] == 612 assert preproc_data["preprocessing"]["output"][ "epipolar_size_y"] == 612 assert - \ 20 < preproc_data["preprocessing"]["output"]["minimum_disparity"] < -18 assert 14 < preproc_data["preprocessing"]["output"][ "maximum_disparity"] < 15 for img in [ "matches", "right_epipolar_grid", "left_epipolar_grid" ]: assert os.path.isfile( os.path.join(out_preproc, preproc_data["preprocessing"]["output"][img])) out_stereo = os.path.join(directory, "out_preproc") corr_config = corr_cfg.configure_correlator() compute_dsm.run( [read_preprocessing_content_file(preproc_json)], out_stereo, resolution=0.5, epsg=32631, sigma=0.3, dsm_radius=3, dsm_no_data=-999, color_no_data=0, corr_config=corr_config, mode="local_dask", # Run on a local cluster, nb_workers=4, walltime="00:10:00", use_geoid_alt=True, use_sec_disp=True) # Uncomment the 2 following instructions to update reference data #copy2(os.path.join(out_stereo, 'dsm.tif'), # absolute_data_path("ref_output/dsm_end2end_ventoux_egm96.tif")) #copy2(os.path.join(out_stereo, 'clr.tif'), # absolute_data_path("ref_output/clr_end2end_ventoux.tif")) assert_same_images( os.path.join(out_stereo, "dsm.tif"), absolute_data_path("ref_output/dsm_end2end_ventoux_egm96.tif"), atol=0.0001, rtol=1e-6) assert_same_images( os.path.join(out_stereo, "clr.tif"), absolute_data_path("ref_output/clr_end2end_ventoux.tif"), rtol=1.e-7, atol=1.e-7) # Test that we have the same results without setting the color1 with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: input_json = read_input_parameters( absolute_data_path( "input/phr_ventoux/preproc_input_without_color.json")) out_preproc = os.path.join(directory, "out_preproc") prepare.run( input_json, out_preproc, epi_step=30, region_size=250, disparity_margin=0.25, epipolar_error_upper_bound=43., elevation_delta_lower_bound=-20., elevation_delta_upper_bound=20., mode="local_dask", # Run on a local cluster nb_workers=4, walltime="00:10:00") preproc_json = os.path.join(out_preproc, "content.json") out_stereo = os.path.join(directory, "out_preproc") corr_config = corr_cfg.configure_correlator() compute_dsm.run( [read_preprocessing_content_file(preproc_json)], out_stereo, resolution=0.5, epsg=32631, sigma=0.3, dsm_radius=3, dsm_no_data=-999, color_no_data=0, corr_config=corr_config, mode="local_dask", # Run on a local cluster nb_workers=4, walltime="00:10:00", use_geoid_alt=True, use_sec_disp=True) assert_same_images( os.path.join(out_stereo, "dsm.tif"), absolute_data_path("ref_output/dsm_end2end_ventoux_egm96.tif"), atol=0.0001, rtol=1e-6) assert_same_images( os.path.join(out_stereo, "clr.tif"), absolute_data_path("ref_output/clr_end2end_ventoux.tif"), rtol=1.e-7, atol=1.e-7)
def test_end2end_paca_with_mask(): """ End to end processing """ # Force max RAM to 1000 to get stable tiling in tests os.environ['OTB_MAX_RAM_HINT'] = '1000' input_json = read_input_parameters( absolute_data_path("input/phr_paca/preproc_input.json")) with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: out_preproc = os.path.join(directory, "out_preproc") prepare.run( input_json, out_preproc, epi_step=30, region_size=250, disparity_margin=0.25, epipolar_error_upper_bound=43., elevation_delta_lower_bound=-20., elevation_delta_upper_bound=20., mode="local_dask", # Run on a local cluster nb_workers=4, walltime="00:10:00", check_inputs=True) # Check preproc properties preproc_json = os.path.join(out_preproc, "content.json") out_stereo = os.path.join(directory, "out_stereo") corr_config = corr_cfg.configure_correlator() compute_dsm.run( [read_preprocessing_content_file(preproc_json)], out_stereo, resolution=0.5, epsg=32631, sigma=0.3, dsm_radius=3, dsm_no_data=-999, color_no_data=0, msk_no_data=65534, corr_config=corr_config, mode="local_dask", # Run on a local cluster, output_stats=True, nb_workers=4, walltime="00:10:00", use_sec_disp=True) # Uncomment the 2 following instructions to update reference data # copy2(os.path.join(out_stereo, 'dsm.tif'), # absolute_data_path("ref_output/dsm_end2end_paca.tif")) # copy2(os.path.join(out_stereo, 'clr.tif'), # absolute_data_path("ref_output/clr_end2end_paca.tif")) # copy2(os.path.join(out_stereo, 'msk.tif'), # absolute_data_path("ref_output/msk_end2end_paca.tif")) assert_same_images(os.path.join(out_stereo, "dsm.tif"), absolute_data_path( "ref_output/dsm_end2end_paca.tif"), atol=0.0001, rtol=1e-6) assert_same_images(os.path.join(out_stereo, "clr.tif"), absolute_data_path( "ref_output/clr_end2end_paca.tif"), rtol=1.e-7, atol=1.e-7) assert_same_images(os.path.join(out_stereo, "msk.tif"), absolute_data_path( "ref_output/msk_end2end_paca.tif"), rtol=1.e-7, atol=1.e-7) # Test we have the same results with multiprocessing with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory: out_preproc = os.path.join(directory, "out_preproc") prepare.run( input_json, out_preproc, epi_step=30, region_size=250, disparity_margin=0.25, epipolar_error_upper_bound=43., elevation_delta_lower_bound=-20., elevation_delta_upper_bound=20., mode="local_dask", # Run on a local cluster nb_workers=4, walltime="00:10:00", check_inputs=True) # Check preproc properties preproc_json = os.path.join(out_preproc, "content.json") out_stereo = os.path.join(directory, "out_stereo") corr_config = corr_cfg.configure_correlator() compute_dsm.run( [read_preprocessing_content_file(preproc_json)], out_stereo, resolution=0.5, epsg=32631, sigma=0.3, dsm_radius=3, dsm_no_data=-999, color_no_data=0, msk_no_data=65534, corr_config=corr_config, mode="mp", output_stats=True, nb_workers=4, walltime="00:10:00", use_sec_disp=True) # Uncomment the 2 following instructions to update reference data # copy2(os.path.join(out_stereo, 'dsm.tif'), # absolute_data_path("ref_output/dsm_end2end_paca.tif")) # copy2(os.path.join(out_stereo, 'clr.tif'), # absolute_data_path("ref_output/clr_end2end_paca.tif")) # copy2(os.path.join(out_stereo, 'msk.tif'), # absolute_data_path("ref_output/msk_end2end_paca.tif")) assert_same_images(os.path.join(out_stereo, "dsm.tif"), absolute_data_path( "ref_output/dsm_end2end_paca.tif"), atol=0.0001, rtol=1e-6) assert_same_images(os.path.join(out_stereo, "clr.tif"), absolute_data_path( "ref_output/clr_end2end_paca.tif"), rtol=1.e-7, atol=1.e-7) assert_same_images(os.path.join(out_stereo, "msk.tif"), absolute_data_path( "ref_output/msk_end2end_paca.tif"), rtol=1.e-7, atol=1.e-7)