Пример #1
0
def test_prepare_ventoux_bias():
    """
    Dask prepare with bias geoms
    """
    input_json = read_input_parameters(absolute_data_path(
        "input/phr_ventoux/preproc_input_bias.json"))

    with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
        out_preproc = os.path.join(directory, "out_preproc_bias")
        prepare.run(
            input_json,
            out_preproc,
            epi_step=30,
            region_size=250,
            disparity_margin=0.25,
            epipolar_error_upper_bound=43.,
            epipolar_error_maximum_bias=50.,
            elevation_delta_lower_bound=-120.,
            elevation_delta_upper_bound=20.,
            mode="local_dask",  # Run on a local cluster
            nb_workers=4,
            walltime="00:10:00")

        # Check preproc properties
        preproc_json = os.path.join(out_preproc, "content.json")
        assert os.path.isfile(preproc_json)

        with open(preproc_json, 'r') as preproc_json_file:
            preproc_data = json.load(preproc_json_file)
            preproc_output = preproc_data["preprocessing"]["output"]
            assert preproc_output["epipolar_size_x"] == 612
            assert preproc_output["epipolar_size_y"] == 612
            assert preproc_output["minimum_disparity"] > -86
            assert preproc_output["minimum_disparity"] < -84
            assert preproc_output["maximum_disparity"] > -46
            assert preproc_output["maximum_disparity"] < -44
            for img in [
                "matches",
                "right_epipolar_grid",
                "left_epipolar_grid"]:
                assert os.path.isfile(
                    os.path.join(
                        out_preproc,
                        preproc_data["preprocessing"]["output"][img]))
Пример #2
0
def main_cli(args, parser, check_inputs=False):
    """
    Main for command line management

    :param check_inputs: activate only arguments checking
    """
    # TODO : refactor in order to avoid a slow argparse
    # Don't move the local function imports for now
    import re
    import sys
    import logging

    from cars import prepare
    from cars import compute_dsm
    from cars import parameters as params
    from cars import configuration_correlator as corr_cfg

    # logging
    numeric_level = getattr(logging, args.loglevel.upper(), None)

    if not isinstance(numeric_level, int):
        raise ValueError('Invalid log level: %s' % args.loglevel)

    logging.basicConfig(
        level=numeric_level,
        datefmt='%y-%m-%d %H:%M:%S',
        format='%(asctime)s :: %(levelname)s :: %(message)s')

    # main(s) for each command
    if args.command == "prepare":
        # Check remaining arguments
        stop_now = False
        if not os.path.exists(args.injson):
            logging.critical('{} does not exist'.format(args.injson))
            stop_now = True
        if args.epi_step < 1:
            logging.critical('{} is an invalid value for --epi_step parameter \
            (should be > 1)'.format(args.epi_step))
            stop_now = True
        if args.disparity_margin < 0 or args.disparity_margin > 1:
            logging.critical(
                '{} is an invalid value for --disparity_margin parameter \
            (should be in range [0,1])'.format(
                    args.disparity_margin))
            stop_now = True
        if args.epipolar_error_upper_bound <= 0:
            logging.critical(
                '{} is an invalid value for --epipolar_error_upper_bound \
            parameter (should be > 0)'.format(
                    args.epipolar_error_upper_bound))
            stop_now = True
        if args.epipolar_error_maximum_bias < 0:
            logging.critical(
                '{} is an invalid value for --epipolar_error_maximum_bias \
            parameter (should be >= 0)'.format(
                    args.epipolar_error_maximum_bias))
            stop_now = True
        if args.nb_workers < 1:
            logging.critical(
                '{} is an invalid value for --nb_workers parameter \
            (should be > 0)'.format(
                    args.nb_workers))
            stop_now = True
        if not re.match(r'[0-9]{2}:[0-9]{2}:[0-9]{2}', args.walltime):
            logging.critical('{} is an invalid value for --walltime parameter \
            (should match HH:MM:SS)'.format(args.walltime))
            stop_now = True
        if args.elevation_delta_upper_bound <= args.elevation_delta_lower_bound:
            logging.critical(
                '--elevation_delta_lower_bound = {} is greater than \
            --elevation_delta_upper_bound = {}'.format(
                    args.elevation_delta_lower_bound,
                    args.elevation_delta_upper_bound))
            stop_now = True

        # If there are invalid parameters, stop now
        if stop_now:
            logging.critical(
                "Invalid parameters detected, please fix cars_cli \
            prepare command-line.")
            sys.exit(1)

        # Read input json file
        in_json = params.read_input_parameters(args.injson)

        if not check_inputs:
            prepare.run(
                in_json,
                args.outdir,
                epi_step=args.epi_step,
                disparity_margin=args.disparity_margin,
                epipolar_error_upper_bound=args.epipolar_error_upper_bound,
                epipolar_error_maximum_bias=args.epipolar_error_maximum_bias,
                elevation_delta_lower_bound=args.elevation_delta_lower_bound,
                elevation_delta_upper_bound=args.elevation_delta_upper_bound,
                mode=args.mode,
                nb_workers=args.nb_workers,
                walltime=args.walltime,
                check_inputs=args.check_inputs)

    elif args.command == "compute_dsm":
        # Check remaining arguments
        stop_now = False
        if len(args.injsons) == 0:
            logging.critical('At least one input json file is required')
            stop_now = True
        for json_file in args.injsons:
            if not os.path.exists(json_file):
                logging.critical('{} does not exist'.format(json_file))
                stop_now = True
        if args.sigma is not None and args.sigma < 0:
            logging.critical('{} is an invalid value for --sigma parameter \
            (should be >= 0)'.format(args.sigma))
            stop_now = True
        if args.dsm_radius < 0:
            logging.critical(
                '{} is an invalid value for --dsm_radius parameter \
            (should be >= 0)'.format(
                    args.dsm_radius))
            stop_now = True
        if args.resolution <= 0:
            logging.critical(
                '{} is an invalid value for --resolution parameter \
            (should be > 0)'.format(
                    args.resolution))
            stop_now = True
        if args.epsg is not None and args.epsg < 1:
            logging.critical('{} is an invalid value for --epsg parameter \
            (should be > 0)'.format(args.epsg))
            stop_now = True
        if args.corr_config is not None:
            if not os.path.exists(args.corr_config):
                logging.critical('{} does not exist'.format(args.corr_config))
                stop_now = True
        if args.nb_workers < 1:
            logging.critical(
                '{} is an invalid value for --nb_workers parameter \
            (should be > 0)'.format(
                    args.nb_workers))
            stop_now = True
        if not re.match(r'[0-9]{2}:[0-9]{2}:[0-9]{2}', args.walltime):
            logging.critical('{} is an invalid value for --walltime parameter \
            (should match HH:MM:SS)'.format(args.walltime))
            stop_now = True
        if args.max_elevation_offset is not None \
                and args.min_elevation_offset is not None \
                and args.max_elevation_offset <= args.min_elevation_offset:
            logging.critical('--min_elevation_offset = {} is greater than \
            --max_elevation_offset = {}'.format(args.min_elevation_offset,
                                                args.max_elevation_offset))
            stop_now = True

        # By default roi = None if no roi mutually exclusive options
        roi = None
        if args.roi_bbox is not None:
            # if roi_bbox defined, roi = 4 floats bounding box + EPSG code=None
            roi = (args.roi_bbox, None)
        if args.roi_file is not None:
            # If roi_file is defined, generate bouding box roi
            roi, stop_now = parse_roi_file(args.roi_file, stop_now)

        # If there are invalid parameters, stop now
        if stop_now:
            logging.critical(
                "Invalid parameters detected, please fix cars_cli \
            compute_dsm command-line.")
            sys.exit(1)

        # Read input json files
        in_jsons = [params.read_preprocessing_content_file(
            f) for f in args.injsons]
        # Configure correlator
        corr_config = corr_cfg.configure_correlator(args.corr_config)

        if not check_inputs:
            compute_dsm.run(
                in_jsons,
                args.outdir,
                resolution = args.resolution,
                min_elevation_offset = args.min_elevation_offset,
                max_elevation_offset = args.max_elevation_offset,
                epsg = args.epsg,
                sigma = args.sigma,
                dsm_radius = args.dsm_radius,
                dsm_no_data = args.dsm_no_data,
                color_no_data = args.color_no_data,
                msk_no_data = args.msk_no_data,
                corr_config = corr_config,
                output_stats = args.output_stats,
                mode = args.mode,
                nb_workers = args.nb_workers,
                walltime = args.walltime,
                roi = roi,
                use_geoid_alt = args.use_geoid_as_alt_ref,
                snap_to_img1 = args.snap_to_left_image,
                align = args.align_with_lowres_dem,
                cloud_small_components_filter = \
                    not args.disable_cloud_small_components_filter,
                cloud_statistical_outliers_filter = \
                    not args.disable_cloud_statistical_outliers_filter,
                use_sec_disp = args.use_sec_disp
            )

    else:
        parser.print_help()
        sys.exit(1)
Пример #3
0
def test_end2end_ventoux_egm96_geoid():
    """
    End to end processing
    """
    # Force max RAM to 1000 to get stable tiling in tests
    os.environ['OTB_MAX_RAM_HINT'] = '1000'

    input_json = read_input_parameters(
        absolute_data_path("input/phr_ventoux/preproc_input.json"))

    with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
        out_preproc = os.path.join(directory, "out_preproc")
        prepare.run(
            input_json,
            out_preproc,
            epi_step=30,
            region_size=250,
            disparity_margin=0.25,
            epipolar_error_upper_bound=43.,
            elevation_delta_lower_bound=-20.,
            elevation_delta_upper_bound=20.,
            mode="local_dask",  # Run on a local cluster
            nb_workers=4,
            walltime="00:10:00",
            check_inputs=True)

        # Check preproc properties
        preproc_json = os.path.join(out_preproc, "content.json")
        assert os.path.isfile(preproc_json)

        with open(preproc_json, 'r') as f:
            preproc_data = json.load(f)
            assert preproc_data["preprocessing"]["output"][
                "epipolar_size_x"] == 612
            assert preproc_data["preprocessing"]["output"][
                "epipolar_size_y"] == 612
            assert - \
                20 < preproc_data["preprocessing"]["output"]["minimum_disparity"] < -18
            assert 14 < preproc_data["preprocessing"]["output"][
                "maximum_disparity"] < 15
            for img in [
                    "matches", "right_epipolar_grid", "left_epipolar_grid"
            ]:
                assert os.path.isfile(
                    os.path.join(out_preproc,
                                 preproc_data["preprocessing"]["output"][img]))

        out_stereo = os.path.join(directory, "out_preproc")
        corr_config = corr_cfg.configure_correlator()

        compute_dsm.run(
            [read_preprocessing_content_file(preproc_json)],
            out_stereo,
            resolution=0.5,
            epsg=32631,
            sigma=0.3,
            dsm_radius=3,
            dsm_no_data=-999,
            color_no_data=0,
            corr_config=corr_config,
            mode="local_dask",  # Run on a local cluster,
            nb_workers=4,
            walltime="00:10:00",
            use_geoid_alt=True,
            use_sec_disp=True)

        # Uncomment the 2 following instructions to update reference data
        #copy2(os.path.join(out_stereo, 'dsm.tif'),
        #      absolute_data_path("ref_output/dsm_end2end_ventoux_egm96.tif"))
        #copy2(os.path.join(out_stereo, 'clr.tif'),
        #      absolute_data_path("ref_output/clr_end2end_ventoux.tif"))

        assert_same_images(
            os.path.join(out_stereo, "dsm.tif"),
            absolute_data_path("ref_output/dsm_end2end_ventoux_egm96.tif"),
            atol=0.0001,
            rtol=1e-6)
        assert_same_images(
            os.path.join(out_stereo, "clr.tif"),
            absolute_data_path("ref_output/clr_end2end_ventoux.tif"),
            rtol=1.e-7,
            atol=1.e-7)

    # Test that we have the same results without setting the color1
    with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
        input_json = read_input_parameters(
            absolute_data_path(
                "input/phr_ventoux/preproc_input_without_color.json"))

        out_preproc = os.path.join(directory, "out_preproc")
        prepare.run(
            input_json,
            out_preproc,
            epi_step=30,
            region_size=250,
            disparity_margin=0.25,
            epipolar_error_upper_bound=43.,
            elevation_delta_lower_bound=-20.,
            elevation_delta_upper_bound=20.,
            mode="local_dask",  # Run on a local cluster
            nb_workers=4,
            walltime="00:10:00")

        preproc_json = os.path.join(out_preproc, "content.json")
        out_stereo = os.path.join(directory, "out_preproc")
        corr_config = corr_cfg.configure_correlator()

        compute_dsm.run(
            [read_preprocessing_content_file(preproc_json)],
            out_stereo,
            resolution=0.5,
            epsg=32631,
            sigma=0.3,
            dsm_radius=3,
            dsm_no_data=-999,
            color_no_data=0,
            corr_config=corr_config,
            mode="local_dask",  # Run on a local cluster
            nb_workers=4,
            walltime="00:10:00",
            use_geoid_alt=True,
            use_sec_disp=True)

        assert_same_images(
            os.path.join(out_stereo, "dsm.tif"),
            absolute_data_path("ref_output/dsm_end2end_ventoux_egm96.tif"),
            atol=0.0001,
            rtol=1e-6)
        assert_same_images(
            os.path.join(out_stereo, "clr.tif"),
            absolute_data_path("ref_output/clr_end2end_ventoux.tif"),
            rtol=1.e-7,
            atol=1.e-7)
Пример #4
0
def test_compute_dsm_with_snap_to_img1():
    """
    Dask compute dsm processing with input roi (cars_stereo)
    """
    # Force max RAM to 1000 to get stable tiling in tests
    os.environ['OTB_MAX_RAM_HINT'] = '1000'

    input_json = read_input_parameters(
        absolute_data_path("input/phr_ventoux/preproc_input.json"))

    with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
        out_preproc = os.path.join(directory, "out_preproc")
        prepare.run(
            input_json,
            out_preproc,
            epi_step=30,
            region_size=250,
            disparity_margin=0.25,
            epipolar_error_upper_bound=43.,
            elevation_delta_lower_bound=-20.,
            elevation_delta_upper_bound=20.,
            mode="local_dask",  # Run on a local cluster
            nb_workers=4,
            walltime="00:10:00")

        # Check preproc properties
        preproc_json = os.path.join(out_preproc, "content.json")
        assert os.path.isfile(preproc_json)

        out_stereo = os.path.join(directory, "out_preproc")
        final_epsg = 32631
        resolution = 0.5
        corr_config = corr_cfg.configure_correlator()

        compute_dsm.run(
            [read_preprocessing_content_file(preproc_json)],
            out_stereo,
            resolution=resolution,
            epsg=final_epsg,
            sigma=0.3,
            dsm_radius=3,
            dsm_no_data=-999,
            color_no_data=0,
            corr_config=corr_config,
            mode="local_dask",  # Run on a local cluster
            nb_workers=4,
            walltime="00:10:00",
            snap_to_img1=True)

        # Uncomment the 2 following instructions to update reference data
        #copy2(os.path.join(out_stereo, 'dsm.tif'),
        #      absolute_data_path("ref_output/dsm_end2end_ventoux_with_snap_to_img1.tif"))
        #copy2(os.path.join(out_stereo, 'clr.tif'),
        #      absolute_data_path("ref_output/clr_end2end_ventoux_with_snap_to_img1.tif"))

        assert_same_images(
            os.path.join(out_stereo, "dsm.tif"),
            absolute_data_path(
                "ref_output/dsm_end2end_ventoux_with_snap_to_img1.tif"),
            atol=0.0001,
            rtol=1e-6)
        assert_same_images(
            os.path.join(out_stereo, "clr.tif"),
            absolute_data_path(
                "ref_output/clr_end2end_ventoux_with_snap_to_img1.tif"),
            rtol=1.e-7,
            atol=1.e-7)
Пример #5
0
def test_compute_dsm_with_roi_ventoux():
    """
    Dask compute dsm processing with input roi (cars_stereo)
    """
    # Force max RAM to 1000 to get stable tiling in tests
    os.environ['OTB_MAX_RAM_HINT'] = '1000'

    input_json = read_input_parameters(
        absolute_data_path("input/phr_ventoux/preproc_input.json"))

    with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
        out_preproc = os.path.join(directory, "out_preproc")
        prepare.run(
            input_json,
            out_preproc,
            epi_step=30,
            region_size=250,
            disparity_margin=0.25,
            epipolar_error_upper_bound=43.,
            elevation_delta_lower_bound=-20.,
            elevation_delta_upper_bound=20.,
            mode="local_dask",  # Run on a local cluster
            nb_workers=4,
            walltime="00:10:00",
            check_inputs=True)

        preproc_json = os.path.join(out_preproc, "content.json")
        out_stereo = os.path.join(directory, "out_preproc")
        final_epsg = 32631
        resolution = 0.5

        roi = [5.194, 44.2059, 5.195, 44.2064]
        roi_epsg = 4326

        corr_config = corr_cfg.configure_correlator()

        compute_dsm.run(
            [read_preprocessing_content_file(preproc_json)],
            out_stereo,
            resolution=resolution,
            epsg=final_epsg,
            sigma=0.3,
            dsm_radius=3,
            dsm_no_data=-999,
            color_no_data=0,
            corr_config=corr_config,
            mode="local_dask",  # Run on a local cluster
            nb_workers=4,
            walltime="00:10:00",
            roi=(roi, roi_epsg))

        # Uncomment the 2 following instructions to update reference data
        #copy2(os.path.join(out_stereo, 'dsm.tif'),
        #      absolute_data_path("ref_output/dsm_end2end_ventoux_with_roi.tif"))
        #copy2(os.path.join(out_stereo, 'clr.tif'),
        #      absolute_data_path("ref_output/clr_end2end_ventoux_with_roi.tif"))

        assert_same_images(
            os.path.join(out_stereo, "dsm.tif"),
            absolute_data_path("ref_output/dsm_end2end_ventoux_with_roi.tif"),
            atol=0.0001,
            rtol=1e-6)
        assert_same_images(
            os.path.join(out_stereo, "clr.tif"),
            absolute_data_path("ref_output/clr_end2end_ventoux_with_roi.tif"),
            rtol=1.e-7,
            atol=1.e-7)

        # check final bounding box
        # create reference
        [roi_xmin, roi_ymin, roi_xmax, roi_ymax] = roi
        roi_poly = Polygon([(roi_xmin, roi_ymin), (roi_xmax, roi_ymin),
                            (roi_xmax, roi_ymax), (roi_xmin, roi_ymax),
                            (roi_xmin, roi_ymin)])

        project = pyproj.Transformer.from_proj(
            pyproj.Proj(init='epsg:{}'.format(roi_epsg)),
            pyproj.Proj(init='epsg:{}'.format(final_epsg)))
        ref_roi_poly = transform(project.transform, roi_poly)

        [ref_xmin, ref_ymin, ref_xmax, ref_ymax] = ref_roi_poly.bounds

        # retrieve bounding box of computed dsm
        data = rasterio.open(os.path.join(out_stereo, "dsm.tif"))
        xmin = min(data.bounds.left, data.bounds.right)
        ymin = min(data.bounds.bottom, data.bounds.top)
        xmax = max(data.bounds.left, data.bounds.right)
        ymax = max(data.bounds.bottom, data.bounds.top)

        assert math.floor(ref_xmin / resolution) * resolution == xmin
        assert math.ceil(ref_xmax / resolution) * resolution == xmax
        assert math.floor(ref_ymin / resolution) * resolution == ymin
        assert math.ceil(ref_ymax / resolution) * resolution == ymax
Пример #6
0
def test_end2end_paca_with_mask():
    """
    End to end processing
    """
    # Force max RAM to 1000 to get stable tiling in tests
    os.environ['OTB_MAX_RAM_HINT'] = '1000'

    input_json = read_input_parameters(
        absolute_data_path("input/phr_paca/preproc_input.json"))

    with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
        out_preproc = os.path.join(directory, "out_preproc")
        prepare.run(
            input_json,
            out_preproc,
            epi_step=30,
            region_size=250,
            disparity_margin=0.25,
            epipolar_error_upper_bound=43.,
            elevation_delta_lower_bound=-20.,
            elevation_delta_upper_bound=20.,
            mode="local_dask",  # Run on a local cluster
            nb_workers=4,
            walltime="00:10:00",
            check_inputs=True)

        # Check preproc properties
        preproc_json = os.path.join(out_preproc, "content.json")

        out_stereo = os.path.join(directory, "out_stereo")

        corr_config = corr_cfg.configure_correlator()

        compute_dsm.run(
            [read_preprocessing_content_file(preproc_json)],
            out_stereo,
            resolution=0.5,
            epsg=32631,
            sigma=0.3,
            dsm_radius=3,
            dsm_no_data=-999,
            color_no_data=0,
            msk_no_data=65534,
            corr_config=corr_config,
            mode="local_dask",  # Run on a local cluster,
            output_stats=True,
            nb_workers=4,
            walltime="00:10:00",
            use_sec_disp=True)

        # Uncomment the 2 following instructions to update reference data
        # copy2(os.path.join(out_stereo, 'dsm.tif'),
        #      absolute_data_path("ref_output/dsm_end2end_paca.tif"))
        # copy2(os.path.join(out_stereo, 'clr.tif'),
        #       absolute_data_path("ref_output/clr_end2end_paca.tif"))
        # copy2(os.path.join(out_stereo, 'msk.tif'),
        #      absolute_data_path("ref_output/msk_end2end_paca.tif"))

        assert_same_images(os.path.join(out_stereo, "dsm.tif"),
                           absolute_data_path(
                               "ref_output/dsm_end2end_paca.tif"),
                           atol=0.0001, rtol=1e-6)
        assert_same_images(os.path.join(out_stereo, "clr.tif"),
                           absolute_data_path(
                               "ref_output/clr_end2end_paca.tif"),
                           rtol=1.e-7, atol=1.e-7)
        assert_same_images(os.path.join(out_stereo, "msk.tif"),
                           absolute_data_path(
                               "ref_output/msk_end2end_paca.tif"),
                           rtol=1.e-7, atol=1.e-7)

        # Test we have the same results with multiprocessing
        with tempfile.TemporaryDirectory(dir=temporary_dir()) as directory:
            out_preproc = os.path.join(directory, "out_preproc")
            prepare.run(
                input_json,
                out_preproc,
                epi_step=30,
                region_size=250,
                disparity_margin=0.25,
                epipolar_error_upper_bound=43.,
                elevation_delta_lower_bound=-20.,
                elevation_delta_upper_bound=20.,
                mode="local_dask",  # Run on a local cluster
                nb_workers=4,
                walltime="00:10:00",
                check_inputs=True)

            # Check preproc properties
            preproc_json = os.path.join(out_preproc, "content.json")

            out_stereo = os.path.join(directory, "out_stereo")

            corr_config = corr_cfg.configure_correlator()

            compute_dsm.run(
                [read_preprocessing_content_file(preproc_json)],
                out_stereo,
                resolution=0.5,
                epsg=32631,
                sigma=0.3,
                dsm_radius=3,
                dsm_no_data=-999,
                color_no_data=0,
                msk_no_data=65534,
                corr_config=corr_config,
                mode="mp",
                output_stats=True,
                nb_workers=4,
                walltime="00:10:00",
                use_sec_disp=True)

            # Uncomment the 2 following instructions to update reference data
            # copy2(os.path.join(out_stereo, 'dsm.tif'),
            #      absolute_data_path("ref_output/dsm_end2end_paca.tif"))
            # copy2(os.path.join(out_stereo, 'clr.tif'),
            #       absolute_data_path("ref_output/clr_end2end_paca.tif"))
            # copy2(os.path.join(out_stereo, 'msk.tif'),
            #      absolute_data_path("ref_output/msk_end2end_paca.tif"))

            assert_same_images(os.path.join(out_stereo, "dsm.tif"),
                               absolute_data_path(
                                   "ref_output/dsm_end2end_paca.tif"),
                               atol=0.0001, rtol=1e-6)
            assert_same_images(os.path.join(out_stereo, "clr.tif"),
                               absolute_data_path(
                                   "ref_output/clr_end2end_paca.tif"),
                               rtol=1.e-7, atol=1.e-7)
            assert_same_images(os.path.join(out_stereo, "msk.tif"),
                               absolute_data_path(
                                   "ref_output/msk_end2end_paca.tif"),
                               rtol=1.e-7, atol=1.e-7)