コード例 #1
0
def test_load_collection_common_name(get_jvm):
    catalog = get_layer_catalog()

    jvm_mock = get_jvm.return_value
    raster_layer = MagicMock()
    jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
    raster_layer.layerMetadata.return_value = '{' \
                                              '"crs":"EPSG:4326",\n' \
                                              '"cellType":"uint8",\n' \
                                              '"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
                                              '"layoutDefinition":{\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
                                              '"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
                                              '}' \
                                              '}'

    load_params = LoadParameters(bands=['B03'],
                                 temporal_extent=('2019-01-01', '2019-01-01'),
                                 spatial_extent={'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326}
                                 )
    collection = catalog.load_collection('SENTINEL2_L2A', load_params=load_params, env=EvalEnv())

    assert collection.metadata.get('id') == 'SENTINEL2_L2A'

    load_params.backend_provider = 'sentinelhub'
    collection = catalog.load_collection('SENTINEL2_L2A', load_params=load_params, env=EvalEnv())

    assert collection.metadata.get('id') == 'SENTINEL2_L2A_SENTINELHUB'

    load_params.backend_provider = 'terrascope'
    collection = catalog.load_collection('SENTINEL2_L2A', load_params=load_params, env=EvalEnv())

    assert collection.metadata.get('id') == 'TERRASCOPE_S2_TOC_V2'
コード例 #2
0
def test_load_collection_sar_backscatter_compatible(get_jvm):
    catalog = get_layer_catalog()

    jvm_mock = get_jvm.return_value
    raster_layer = MagicMock()
    jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
    raster_layer.layerMetadata.return_value = '{' \
                                              '"crs":"EPSG:4326",\n' \
                                              '"cellType":"uint8",\n' \
                                              '"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
                                              '"layoutDefinition":{\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
                                              '"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
                                              '}' \
                                              '}'

    load_params = LoadParameters(temporal_extent=("2021-02-08T10:36:00Z", "2021-02-08T10:36:00Z"),
                                 spatial_extent={'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326},
                                 sar_backscatter=SarBackscatterArgs())
    catalog.load_collection('SENTINEL1_GAMMA0_SENTINELHUB', load_params=load_params,
                            env=EvalEnv({'pyramid_levels': 'highest'}))

    factory_mock = jvm_mock.org.openeo.geotrellissentinelhub.PyramidFactory.rateLimited
    sample_type_mock = jvm_mock.org.openeo.geotrellissentinelhub.SampleType.withName.return_value
    cellsize_mock = jvm_mock.geotrellis.raster.CellSize(10, 10)

    factory_mock.assert_called_once_with("https://services.sentinel-hub.com", "sentinel-1-grd", "S1GRD", "???", "!!!",
                                         {"backCoeff": "GAMMA0_TERRAIN", "orthorectify": True}, sample_type_mock, cellsize_mock)

    jvm_mock.org.openeo.geotrellissentinelhub.SampleType.withName.assert_called_once_with("FLOAT32")
    factory_mock.return_value.datacube_seq.assert_called_once()
コード例 #3
0
def test_load_collection_old_and_new_band_names(get_jvm):
    catalog = get_layer_catalog()

    jvm_mock = get_jvm.return_value
    raster_layer = MagicMock()
    jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
    raster_layer.layerMetadata.return_value = '{' \
                                              '"crs":"EPSG:4326",\n' \
                                              '"cellType":"uint8",\n' \
                                              '"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
                                              '"layoutDefinition":{\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
                                              '"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
                                              '}' \
                                              '}'

    temporal_extent = ('2019-01-01', '2019-01-01')
    spatial_extent = {'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326}

    for bands in [['TOC-B03_10M'], ['B03']]:
        load_params = LoadParameters(temporal_extent=temporal_extent, bands=bands, spatial_extent=spatial_extent)
        collection = catalog.load_collection('TERRASCOPE_S2_TOC_V2', load_params=load_params, env=EvalEnv())

        assert len(collection.metadata.bands) == 1
        assert collection.metadata.bands[0].name == bands[0]
        assert collection.metadata.bands[0].aliases == ['TOC-B03_10M']
        assert collection.metadata.temporal_dimension.extent == ('2019-01-01T00:00:00+00:00', '2019-01-01T00:00:00+00:00')
コード例 #4
0
def test_layercatalog_json():
    catalog = get_layer_catalog()
    for layer in catalog.get_all_metadata():
        assert re.match(r'^[A-Za-z0-9_\-\.~\/]+$', layer['id'])
        assert 'stac_version' in layer
        assert 'description' in layer
        assert 'license' in layer
        assert 'extent' in layer
コード例 #5
0
def test_load_collection_sar_backscatter_incompatible():
    catalog = get_layer_catalog()
    load_params = LoadParameters(sar_backscatter=SarBackscatterArgs())
    with pytest.raises(OpenEOApiException) as exc_info:
        catalog.load_collection('TERRASCOPE_S2_TOC_V2', load_params=load_params, env=EvalEnv())

    assert exc_info.value.status_code == 400
    assert (exc_info.value.args[0] ==
            """Process "sar_backscatter" is not applicable for collection TERRASCOPE_S2_TOC_V2.""")
コード例 #6
0
    def __init__(self):
        # TODO: do this with a config instead of hardcoding rules?
        self._service_registry = (InMemoryServiceRegistry()
                                  if ConfigParams().is_ci_context else
                                  ZooKeeperServiceRegistry())

        super().__init__(
            secondary_services=GpsSecondaryServices(
                service_registry=self._service_registry),
            catalog=get_layer_catalog(service_registry=self._service_registry),
            batch_jobs=GpsBatchJobs(),
        )
コード例 #7
0
def test_get_layer_catalog_opensearch_enrich_creodias(requests_mock):
    with mock.patch(
            "openeogeotrellis.layercatalog.ConfigParams") as ConfigParams:
        ConfigParams.return_value.layer_catalog_metadata_files = [
            "tests/data/layercatalog01.json",
            "tests/data/layercatalog04_creodias.json"
        ]
        collections_response = read_json(
            "tests/data/collections_creodias01.json")
        requests_mock.get(
            "https://finder.creodias.test/resto/collections.json",
            json=collections_response)

        all_metadata = get_layer_catalog(
            opensearch_enrich=True).get_all_metadata()

    assert all_metadata == [{
        "id":
        "WUQ",
        "title":
        "Sentinel-1 Collection",
        "description":
        "Sentinel-1 Collection",
        "keywords": ["esa", "sentinel", "sentinel1", "s1", "radar"],
        "_vito": {
            "data_source": {
                "opensearch_collection_id": "Sentinel1",
                "opensearch_endpoint": "https://finder.creodias.test"
            }
        },
        "cube:dimensions": {
            "t": {
                "type": "temporal"
            },
            "x": {
                "axis": "x",
                "type": "spatial"
            },
            "y": {
                "axis": "y",
                "type": "spatial"
            }
        },
    }, {
        "id": "FOO",
        "license": "mit"
    }, {
        "id": "BAR",
        "description": "bar",
        "links": ["example.com/bar"]
    }, {
        "id": "BZZ"
    }]
コード例 #8
0
        def batch_jobs() -> GpsBatchJobs:
            java_opts = [
                "-client",
                "-Dsoftware.amazon.awssdk.http.service.impl=software.amazon.awssdk.http.urlconnection.UrlConnectionSdkHttpService"
            ]

            java_gateway = JavaGateway.launch_gateway(jarpath=args.py4j_jarpath,
                                                      classpath=args.py4j_classpath,
                                                      javaopts=java_opts,
                                                      die_on_exit=True)

            return GpsBatchJobs(get_layer_catalog(opensearch_enrich=True), java_gateway.jvm, args.principal,
                                args.keytab)
コード例 #9
0
def skip_sentinelhub_layer():
    catalog = get_layer_catalog()
    viewingParameters = {}
    viewingParameters["from"] = "2018-01-01"
    viewingParameters["to"] = "2018-01-02"

    viewingParameters["left"] = 4
    viewingParameters["right"] = 4.0001
    viewingParameters["top"] = 50.00001
    viewingParameters["bottom"] = 50.0
    viewingParameters["srs"] = "EPSG:4326"
    datacube = catalog.load_collection("SENTINEL1_GAMMA0_SENTINELHUB",
                                       viewingParameters)
コード例 #10
0
def test_get_layer_catalog_with_updates():
    with mock.patch(
            "openeogeotrellis.layercatalog.ConfigParams") as ConfigParams:
        ConfigParams.return_value.layer_catalog_metadata_files = [
            "tests/data/layercatalog01.json",
            "tests/data/layercatalog02.json",
        ]
        catalog = get_layer_catalog()
        assert sorted(l["id"] for l in catalog.get_all_metadata()) == [
            "BAR", "BZZ", "FOO", "QUU"
        ]
        foo = catalog.get_collection_metadata("FOO")
        assert foo["license"] == "apache"
        assert foo["links"] == ["example.com/foo"]
        bar = catalog.get_collection_metadata("BAR")
        assert bar["description"] == "The BAR layer"
        assert bar["links"] == ["example.com/bar"]
コード例 #11
0
def test_load_collection_data_cube_params(get_jvm):
    catalog = get_layer_catalog()

    jvm_mock = get_jvm.return_value
    raster_layer = MagicMock()
    jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
    raster_layer.layerMetadata.return_value = '{' \
                                              '"crs":"EPSG:4326",\n' \
                                              '"cellType":"uint8",\n' \
                                              '"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
                                              '"layoutDefinition":{\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
                                              '"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
                                              '}' \
                                              '}'

    load_params = LoadParameters(
        temporal_extent=('2019-01-01', '2019-01-01'),
        bands=['temperature-mean'],
        spatial_extent={'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326}

    )
    load_params['featureflags'] = {
        "tilesize": 1,
        "experimental": True
    }
    env = EvalEnv({'require_bounds': True, 'pyramid_levels': 'highest'})
    collection = catalog.load_collection('AGERA5', load_params=load_params, env=env)

    print(collection.metadata)
    assert len(collection.metadata.bands) == 1
    assert collection.metadata.bands[0].name == 'temperature-mean'

    factory_mock = jvm_mock.org.openeo.geotrellis.file.AgEra5PyramidFactory
    projected_polys = jvm_mock.org.openeo.geotrellis.ProjectedPolygons.fromExtent.return_value
    datacubeParams = jvm_mock.org.openeo.geotrelliscommon.DataCubeParameters.return_value

    jvm_mock.geotrellis.vector.Extent.assert_called_once_with(4.0, 51.9999, 4.001, 52.0)

    factory_mock.assert_called_once_with('/data/MEP/ECMWF/AgERA5/*/*/AgERA5_dewpoint-temperature_*.tif', ['temperature-mean'], '.+_(\\d{4})(\\d{2})(\\d{2})\\.tif')
    factory_mock.return_value.datacube_seq.assert_called_once_with(projected_polys, '2019-01-01T00:00:00+00:00', '2019-01-01T00:00:00+00:00', {}, '',datacubeParams)
    getattr(datacubeParams,'tileSize_$eq').assert_called_once_with(1)
    getattr(datacubeParams, 'layoutScheme_$eq').assert_called_once_with('FloatingLayoutScheme')
コード例 #12
0
def test_load_file_oscars_resample(get_jvm):
    catalog = get_layer_catalog()
    jvm_mock = get_jvm.return_value
    raster_layer = MagicMock()
    raster_layer.layerMetadata.return_value = '{' \
                                              '"crs":"EPSG:4326",\n' \
                                              '"cellType":"uint8",\n' \
                                              '"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
                                              '"layoutDefinition":{\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
                                              '"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
                                              '}' \
                                              '}'

    jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
    load_params = LoadParameters(temporal_extent=("2010-01-01T10:36:00Z", "2012-01-01T10:36:00Z"),
                                 spatial_extent={'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326},
                                 target_resolution=[15,15],
                                 target_crs=3857,
                                 featureflags={"experimental":True}
                                 )
    env = EvalEnv()
    env = env.push({"pyramid_levels": "single"})

    factory_mock = jvm_mock.org.openeo.geotrellis.file.Sentinel2PyramidFactory
    extent_mock = jvm_mock.geotrellis.vector.Extent.return_value
    cellsize_call_mock = jvm_mock.geotrellis.raster.CellSize
    cellsize_mock = jvm_mock.geotrellis.raster.CellSize(15, 15)

    datacubeParams = jvm_mock.org.openeo.geotrelliscommon.DataCubeParameters.return_value


    collection = catalog.load_collection('COPERNICUS_30', load_params=load_params, env=env)
    assert(collection.metadata.spatial_dimensions[0].step == 0.002777777777777778)
    assert(collection.metadata.spatial_dimensions[1].step == 0.002777777777777778)

    jvm_mock.geotrellis.vector.Extent.assert_called_once_with(4.0, 51.9999, 4.001, 52.0)
    cellsize_call_mock.assert_called_with(15,15)

    factory_mock.assert_called_once_with('https://services.terrascope.be/catalogue', 'urn:eop:VITO:COP_DEM_GLO_30M_COG', ['DEM'], '/data/MTDA/DEM/COP_DEM_30M_COG', cellsize_mock, True)
    factory_mock.return_value.datacube_seq.assert_called_once_with(ANY, '2010-01-01T10:36:00+00:00', '2012-01-01T10:36:00+00:00', {}, '', datacubeParams)
コード例 #13
0
def test_issue77_band_metadata():
    catalog = get_layer_catalog()
    for layer in catalog.get_all_metadata():
        # print(layer['id'])
        # TODO: stop doing this non-standard band metadata ("bands" item in metadata root)
        old_bands = [
            b if isinstance(b, str) else b["band_id"]
            for b in layer.get("bands", [])
        ]
        eo_bands = [
            b["name"] for b in layer.get("properties", {}).get('eo:bands', [])
        ]
        cube_dimension_bands = []
        for cube_dim in layer.get("properties", {}).get("cube:dimensions",
                                                        {}).values():
            if cube_dim["type"] == "bands":
                cube_dimension_bands = cube_dim["values"]
        if len(old_bands) > 1:
            assert old_bands == eo_bands
            assert old_bands == cube_dimension_bands
        assert eo_bands == cube_dimension_bands
コード例 #14
0
def test_load_file_oscars(get_jvm):
    catalog = get_layer_catalog()
    jvm_mock = get_jvm.return_value
    raster_layer = MagicMock()
    raster_layer.layerMetadata.return_value = '{' \
                                              '"crs":"EPSG:4326",\n' \
                                              '"cellType":"uint8",\n' \
                                              '"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
                                              '"layoutDefinition":{\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
                                              '"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
                                              '}' \
                                              '}'

    jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
    load_params = LoadParameters(temporal_extent=("2010-01-01T10:36:00Z", "2012-01-01T10:36:00Z"),
                                 spatial_extent={'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326})
    env = EvalEnv()
    env = env.push({"pyramid_levels": "single"})
    collection = catalog.load_collection('COPERNICUS_30', load_params=load_params, env=env)
    assert(collection.metadata.spatial_dimensions[0].step == 0.002777777777777778)
    assert(collection.metadata.spatial_dimensions[1].step == 0.002777777777777778)
コード例 #15
0
def test_load_collection_bands_with_required_extent(get_jvm):
    catalog = get_layer_catalog()

    jvm_mock = get_jvm.return_value
    raster_layer = MagicMock()
    jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
    raster_layer.layerMetadata.return_value = '{' \
                                              '"crs":"EPSG:4326",\n' \
                                              '"cellType":"uint8",\n' \
                                              '"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
                                              '"layoutDefinition":{\n' \
                                              '"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
                                              '"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
                                              '}' \
                                              '}'

    load_params = LoadParameters(
        temporal_extent=('2019-01-01', '2019-01-01'),
        bands=['TOC-B03_10M'],
        spatial_extent={'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326}
    )
    env = EvalEnv({'require_bounds': True})
    collection = catalog.load_collection('TERRASCOPE_S2_TOC_V2', load_params=load_params, env=env)

    print(collection.metadata)
    assert len(collection.metadata.bands) == 1
    assert collection.metadata.bands[0].name == 'TOC-B03_10M'

    factory_mock = jvm_mock.org.openeo.geotrellis.file.Sentinel2PyramidFactory
    extent_mock = jvm_mock.geotrellis.vector.Extent.return_value
    cellsize_mock = jvm_mock.geotrellis.raster.CellSize.return_value

    jvm_mock.geotrellis.vector.Extent.assert_called_once_with(4.0, 51.9999, 4.001, 52.0)

    factory_mock.assert_called_once_with("https://services.terrascope.be/catalogue", 'urn:eop:VITO:TERRASCOPE_S2_TOC_V2', ['TOC-B03_10M'], '/data/MTDA/TERRASCOPE_Sentinel2/TOC_V2', cellsize_mock,False)
    factory_mock.return_value.pyramid_seq.assert_called_once_with(extent_mock, "EPSG:4326", '2019-01-01T00:00:00+00:00', '2019-01-01T00:00:00+00:00', {}, '')
コード例 #16
0
def _get_layers() -> List[Tuple[str, dict]]:
    catalog = get_layer_catalog()
    layers = catalog.get_all_metadata()
    return [(layer["id"], layer) for layer in layers]
コード例 #17
0
def test_get_layer_catalog_opensearch_enrich_oscars(requests_mock):
    test_root = Path(__file__).parent / "data"
    with mock.patch(
            "openeogeotrellis.layercatalog.ConfigParams") as ConfigParams:
        ConfigParams.return_value.layer_catalog_metadata_files = [
            test_root / "layercatalog01.json",
            test_root / "layercatalog02.json",
            test_root / "layercatalog03_oscars.json"
        ]

        collections_response = read_json(test_root /
                                         "collections_oscars01.json")
        requests_mock.get(
            "https://services.terrascope.test/catalogue/collections",
            json=collections_response)

        all_metadata = get_layer_catalog(
            opensearch_enrich=True).get_all_metadata()

    assert all_metadata == [{
        "id":
        "XIP",
        "_vito": {
            "data_source": {
                "opensearch_endpoint":
                "https://services.terrascope.test/catalogue",
                "opensearch_collection_id": "urn:eop:VITO:CGS_S1_GRD_SIGMA0_L1"
            }
        },
        "title":
        "Sentinel 1 GRD Sigma0 product, VH, VV and angle.",
        "description":
        "The Sigma0 product describes how much of the radar signal that was sent out by Sentinel-1 "
        "is reflected back to the sensor...",
        "extent": {
            "spatial": {
                "bbox": [[-1.05893, 47.66031, 11.6781, 53.67487]]
            },
            "temporal": {
                "interval": [["2014-10-23", None]]
            }
        },
        'keywords': [
            'VITO', 'C-SAR', 'Orthoimagery', 'SENTINEL-1A', 'SENTINEL-1',
            'SENTINEL-1B', 'RADAR BACKSCATTER', 'RADAR'
        ],
        "links": [{
            "rel": "alternate",
            "href":
            "https://docs.terrascope.be/#/DataProducts/Sentinel-1/ProductsOverview",
            "title": "Online User Documentation"
        }, {
            "rel":
            "alternate",
            "href":
            "https://www.vito-eodata.be/collections/srv/eng/main.home?uuid=urn:eop:VITO:CGS_S1_GRD_SIGMA0_L1"
        }, {
            "rel": "alternate",
            "href":
            "https://services.terrascope.be/catalogue/description.geojson?collection=urn:eop:VITO:CGS_S1_GRD_SIGMA0_L1",
            "title": "OpenSearch entry point"
        }],
        "cube:dimensions": {
            "x": {
                "type": "spatial",
                "axis": "x"
            },
            "y": {
                "type": "spatial",
                "axis": "y"
            },
            "t": {
                "type": "temporal"
            },
            "bands": {
                "type": "bands",
                "values": ["VH"]
            }
        },
        "summaries": {
            "eo:bands": [{
                "description":
                "Calibrated radar backscattering coefficient (unitless), describing the returned radar signal strength in the cross-polarized channel (V transmit, H receive). Values are stored as floats.",
                "type": "VH",
                "title": "VH",
                "resolution": 10,
                "bitPerValue": 32,
                "name": "VH"
            }],
            "platform": [],
            "instruments": ["MSI"]
        },
        "assets": {}
    }, {
        "id": "FOO",
        "license": "apache",
        "links": ["example.com/foo"]
    }, {
        "id": "BAR",
        "description": "The BAR layer",
        "links": ["example.com/bar"]
    }, {
        "id": "BZZ"
    }, {
        "id": "QUU"
    }]
コード例 #18
0
def test_load_collection_bands_missing_required_extent():
    catalog = get_layer_catalog()
    load_params = LoadParameters(bands=['TOC-B03_10M'])
    env = EvalEnv({'require_bounds': True})
    with pytest.raises(OpenEOApiException):
        catalog.load_collection('TERRASCOPE_S2_TOC_V2', load_params=load_params, env=env)