def test_load_collection_common_name(get_jvm):
catalog = get_layer_catalog()
jvm_mock = get_jvm.return_value
raster_layer = MagicMock()
jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
raster_layer.layerMetadata.return_value = '{' \
'"crs":"EPSG:4326",\n' \
'"cellType":"uint8",\n' \
'"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
'"layoutDefinition":{\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
'"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
'}' \
'}'
load_params = LoadParameters(bands=['B03'],
temporal_extent=('2019-01-01', '2019-01-01'),
spatial_extent={'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326}
)
collection = catalog.load_collection('SENTINEL2_L2A', load_params=load_params, env=EvalEnv())
assert collection.metadata.get('id') == 'SENTINEL2_L2A'
load_params.backend_provider = 'sentinelhub'
collection = catalog.load_collection('SENTINEL2_L2A', load_params=load_params, env=EvalEnv())
assert collection.metadata.get('id') == 'SENTINEL2_L2A_SENTINELHUB'
load_params.backend_provider = 'terrascope'
collection = catalog.load_collection('SENTINEL2_L2A', load_params=load_params, env=EvalEnv())
assert collection.metadata.get('id') == 'TERRASCOPE_S2_TOC_V2'
def test_load_collection_sar_backscatter_compatible(get_jvm):
catalog = get_layer_catalog()
jvm_mock = get_jvm.return_value
raster_layer = MagicMock()
jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
raster_layer.layerMetadata.return_value = '{' \
'"crs":"EPSG:4326",\n' \
'"cellType":"uint8",\n' \
'"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
'"layoutDefinition":{\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
'"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
'}' \
'}'
load_params = LoadParameters(temporal_extent=("2021-02-08T10:36:00Z", "2021-02-08T10:36:00Z"),
spatial_extent={'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326},
sar_backscatter=SarBackscatterArgs())
catalog.load_collection('SENTINEL1_GAMMA0_SENTINELHUB', load_params=load_params,
env=EvalEnv({'pyramid_levels': 'highest'}))
factory_mock = jvm_mock.org.openeo.geotrellissentinelhub.PyramidFactory.rateLimited
sample_type_mock = jvm_mock.org.openeo.geotrellissentinelhub.SampleType.withName.return_value
cellsize_mock = jvm_mock.geotrellis.raster.CellSize(10, 10)
factory_mock.assert_called_once_with("https://services.sentinel-hub.com", "sentinel-1-grd", "S1GRD", "???", "!!!",
{"backCoeff": "GAMMA0_TERRAIN", "orthorectify": True}, sample_type_mock, cellsize_mock)
jvm_mock.org.openeo.geotrellissentinelhub.SampleType.withName.assert_called_once_with("FLOAT32")
factory_mock.return_value.datacube_seq.assert_called_once()
def test_load_collection_old_and_new_band_names(get_jvm):
catalog = get_layer_catalog()
jvm_mock = get_jvm.return_value
raster_layer = MagicMock()
jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
raster_layer.layerMetadata.return_value = '{' \
'"crs":"EPSG:4326",\n' \
'"cellType":"uint8",\n' \
'"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
'"layoutDefinition":{\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
'"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
'}' \
'}'
temporal_extent = ('2019-01-01', '2019-01-01')
spatial_extent = {'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326}
for bands in [['TOC-B03_10M'], ['B03']]:
load_params = LoadParameters(temporal_extent=temporal_extent, bands=bands, spatial_extent=spatial_extent)
collection = catalog.load_collection('TERRASCOPE_S2_TOC_V2', load_params=load_params, env=EvalEnv())
assert len(collection.metadata.bands) == 1
assert collection.metadata.bands[0].name == bands[0]
assert collection.metadata.bands[0].aliases == ['TOC-B03_10M']
assert collection.metadata.temporal_dimension.extent == ('2019-01-01T00:00:00+00:00', '2019-01-01T00:00:00+00:00')
def test_layercatalog_json():
catalog = get_layer_catalog()
for layer in catalog.get_all_metadata():
assert re.match(r'^[A-Za-z0-9_\-\.~\/]+$', layer['id'])
assert 'stac_version' in layer
assert 'description' in layer
assert 'license' in layer
assert 'extent' in layer
def test_load_collection_sar_backscatter_incompatible():
catalog = get_layer_catalog()
load_params = LoadParameters(sar_backscatter=SarBackscatterArgs())
with pytest.raises(OpenEOApiException) as exc_info:
catalog.load_collection('TERRASCOPE_S2_TOC_V2', load_params=load_params, env=EvalEnv())
assert exc_info.value.status_code == 400
assert (exc_info.value.args[0] ==
"""Process "sar_backscatter" is not applicable for collection TERRASCOPE_S2_TOC_V2.""")
def __init__(self):
# TODO: do this with a config instead of hardcoding rules?
self._service_registry = (InMemoryServiceRegistry()
if ConfigParams().is_ci_context else
ZooKeeperServiceRegistry())
super().__init__(
secondary_services=GpsSecondaryServices(
service_registry=self._service_registry),
catalog=get_layer_catalog(service_registry=self._service_registry),
batch_jobs=GpsBatchJobs(),
)
def test_get_layer_catalog_opensearch_enrich_creodias(requests_mock):
with mock.patch(
"openeogeotrellis.layercatalog.ConfigParams") as ConfigParams:
ConfigParams.return_value.layer_catalog_metadata_files = [
"tests/data/layercatalog01.json",
"tests/data/layercatalog04_creodias.json"
]
collections_response = read_json(
"tests/data/collections_creodias01.json")
requests_mock.get(
"https://finder.creodias.test/resto/collections.json",
json=collections_response)
all_metadata = get_layer_catalog(
opensearch_enrich=True).get_all_metadata()
assert all_metadata == [{
"id":
"WUQ",
"title":
"Sentinel-1 Collection",
"description":
"Sentinel-1 Collection",
"keywords": ["esa", "sentinel", "sentinel1", "s1", "radar"],
"_vito": {
"data_source": {
"opensearch_collection_id": "Sentinel1",
"opensearch_endpoint": "https://finder.creodias.test"
}
},
"cube:dimensions": {
"t": {
"type": "temporal"
},
"x": {
"axis": "x",
"type": "spatial"
},
"y": {
"axis": "y",
"type": "spatial"
}
},
}, {
"id": "FOO",
"license": "mit"
}, {
"id": "BAR",
"description": "bar",
"links": ["example.com/bar"]
}, {
"id": "BZZ"
}]
def batch_jobs() -> GpsBatchJobs:
java_opts = [
"-client",
"-Dsoftware.amazon.awssdk.http.service.impl=software.amazon.awssdk.http.urlconnection.UrlConnectionSdkHttpService"
]
java_gateway = JavaGateway.launch_gateway(jarpath=args.py4j_jarpath,
classpath=args.py4j_classpath,
javaopts=java_opts,
die_on_exit=True)
return GpsBatchJobs(get_layer_catalog(opensearch_enrich=True), java_gateway.jvm, args.principal,
args.keytab)
def skip_sentinelhub_layer():
catalog = get_layer_catalog()
viewingParameters = {}
viewingParameters["from"] = "2018-01-01"
viewingParameters["to"] = "2018-01-02"
viewingParameters["left"] = 4
viewingParameters["right"] = 4.0001
viewingParameters["top"] = 50.00001
viewingParameters["bottom"] = 50.0
viewingParameters["srs"] = "EPSG:4326"
datacube = catalog.load_collection("SENTINEL1_GAMMA0_SENTINELHUB",
viewingParameters)
def test_get_layer_catalog_with_updates():
with mock.patch(
"openeogeotrellis.layercatalog.ConfigParams") as ConfigParams:
ConfigParams.return_value.layer_catalog_metadata_files = [
"tests/data/layercatalog01.json",
"tests/data/layercatalog02.json",
]
catalog = get_layer_catalog()
assert sorted(l["id"] for l in catalog.get_all_metadata()) == [
"BAR", "BZZ", "FOO", "QUU"
]
foo = catalog.get_collection_metadata("FOO")
assert foo["license"] == "apache"
assert foo["links"] == ["example.com/foo"]
bar = catalog.get_collection_metadata("BAR")
assert bar["description"] == "The BAR layer"
assert bar["links"] == ["example.com/bar"]
def test_load_collection_data_cube_params(get_jvm):
catalog = get_layer_catalog()
jvm_mock = get_jvm.return_value
raster_layer = MagicMock()
jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
raster_layer.layerMetadata.return_value = '{' \
'"crs":"EPSG:4326",\n' \
'"cellType":"uint8",\n' \
'"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
'"layoutDefinition":{\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
'"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
'}' \
'}'
load_params = LoadParameters(
temporal_extent=('2019-01-01', '2019-01-01'),
bands=['temperature-mean'],
spatial_extent={'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326}
)
load_params['featureflags'] = {
"tilesize": 1,
"experimental": True
}
env = EvalEnv({'require_bounds': True, 'pyramid_levels': 'highest'})
collection = catalog.load_collection('AGERA5', load_params=load_params, env=env)
print(collection.metadata)
assert len(collection.metadata.bands) == 1
assert collection.metadata.bands[0].name == 'temperature-mean'
factory_mock = jvm_mock.org.openeo.geotrellis.file.AgEra5PyramidFactory
projected_polys = jvm_mock.org.openeo.geotrellis.ProjectedPolygons.fromExtent.return_value
datacubeParams = jvm_mock.org.openeo.geotrelliscommon.DataCubeParameters.return_value
jvm_mock.geotrellis.vector.Extent.assert_called_once_with(4.0, 51.9999, 4.001, 52.0)
factory_mock.assert_called_once_with('/data/MEP/ECMWF/AgERA5/*/*/AgERA5_dewpoint-temperature_*.tif', ['temperature-mean'], '.+_(\\d{4})(\\d{2})(\\d{2})\\.tif')
factory_mock.return_value.datacube_seq.assert_called_once_with(projected_polys, '2019-01-01T00:00:00+00:00', '2019-01-01T00:00:00+00:00', {}, '',datacubeParams)
getattr(datacubeParams,'tileSize_$eq').assert_called_once_with(1)
getattr(datacubeParams, 'layoutScheme_$eq').assert_called_once_with('FloatingLayoutScheme')
def test_load_file_oscars_resample(get_jvm):
catalog = get_layer_catalog()
jvm_mock = get_jvm.return_value
raster_layer = MagicMock()
raster_layer.layerMetadata.return_value = '{' \
'"crs":"EPSG:4326",\n' \
'"cellType":"uint8",\n' \
'"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
'"layoutDefinition":{\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
'"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
'}' \
'}'
jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
load_params = LoadParameters(temporal_extent=("2010-01-01T10:36:00Z", "2012-01-01T10:36:00Z"),
spatial_extent={'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326},
target_resolution=[15,15],
target_crs=3857,
featureflags={"experimental":True}
)
env = EvalEnv()
env = env.push({"pyramid_levels": "single"})
factory_mock = jvm_mock.org.openeo.geotrellis.file.Sentinel2PyramidFactory
extent_mock = jvm_mock.geotrellis.vector.Extent.return_value
cellsize_call_mock = jvm_mock.geotrellis.raster.CellSize
cellsize_mock = jvm_mock.geotrellis.raster.CellSize(15, 15)
datacubeParams = jvm_mock.org.openeo.geotrelliscommon.DataCubeParameters.return_value
collection = catalog.load_collection('COPERNICUS_30', load_params=load_params, env=env)
assert(collection.metadata.spatial_dimensions[0].step == 0.002777777777777778)
assert(collection.metadata.spatial_dimensions[1].step == 0.002777777777777778)
jvm_mock.geotrellis.vector.Extent.assert_called_once_with(4.0, 51.9999, 4.001, 52.0)
cellsize_call_mock.assert_called_with(15,15)
factory_mock.assert_called_once_with('https://services.terrascope.be/catalogue', 'urn:eop:VITO:COP_DEM_GLO_30M_COG', ['DEM'], '/data/MTDA/DEM/COP_DEM_30M_COG', cellsize_mock, True)
factory_mock.return_value.datacube_seq.assert_called_once_with(ANY, '2010-01-01T10:36:00+00:00', '2012-01-01T10:36:00+00:00', {}, '', datacubeParams)
def test_issue77_band_metadata():
catalog = get_layer_catalog()
for layer in catalog.get_all_metadata():
# print(layer['id'])
# TODO: stop doing this non-standard band metadata ("bands" item in metadata root)
old_bands = [
b if isinstance(b, str) else b["band_id"]
for b in layer.get("bands", [])
]
eo_bands = [
b["name"] for b in layer.get("properties", {}).get('eo:bands', [])
]
cube_dimension_bands = []
for cube_dim in layer.get("properties", {}).get("cube:dimensions",
{}).values():
if cube_dim["type"] == "bands":
cube_dimension_bands = cube_dim["values"]
if len(old_bands) > 1:
assert old_bands == eo_bands
assert old_bands == cube_dimension_bands
assert eo_bands == cube_dimension_bands
def test_load_file_oscars(get_jvm):
catalog = get_layer_catalog()
jvm_mock = get_jvm.return_value
raster_layer = MagicMock()
raster_layer.layerMetadata.return_value = '{' \
'"crs":"EPSG:4326",\n' \
'"cellType":"uint8",\n' \
'"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
'"layoutDefinition":{\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
'"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
'}' \
'}'
jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
load_params = LoadParameters(temporal_extent=("2010-01-01T10:36:00Z", "2012-01-01T10:36:00Z"),
spatial_extent={'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326})
env = EvalEnv()
env = env.push({"pyramid_levels": "single"})
collection = catalog.load_collection('COPERNICUS_30', load_params=load_params, env=env)
assert(collection.metadata.spatial_dimensions[0].step == 0.002777777777777778)
assert(collection.metadata.spatial_dimensions[1].step == 0.002777777777777778)
def test_load_collection_bands_with_required_extent(get_jvm):
catalog = get_layer_catalog()
jvm_mock = get_jvm.return_value
raster_layer = MagicMock()
jvm_mock.geopyspark.geotrellis.TemporalTiledRasterLayer.return_value = raster_layer
raster_layer.layerMetadata.return_value = '{' \
'"crs":"EPSG:4326",\n' \
'"cellType":"uint8",\n' \
'"bounds":{"minKey":{"col":0,"row":0},"maxKey":{"col":1,"row":1}},\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},\n' \
'"layoutDefinition":{\n' \
'"extent":{"xmin":0,"ymin":0,"xmax":1,"ymax":1},' \
'"tileLayout":{"layoutCols":1, "layoutRows":1, "tileCols":256, "tileRows":256}' \
'}' \
'}'
load_params = LoadParameters(
temporal_extent=('2019-01-01', '2019-01-01'),
bands=['TOC-B03_10M'],
spatial_extent={'west': 4, 'east': 4.001, 'north': 52, 'south': 51.9999, 'crs': 4326}
)
env = EvalEnv({'require_bounds': True})
collection = catalog.load_collection('TERRASCOPE_S2_TOC_V2', load_params=load_params, env=env)
print(collection.metadata)
assert len(collection.metadata.bands) == 1
assert collection.metadata.bands[0].name == 'TOC-B03_10M'
factory_mock = jvm_mock.org.openeo.geotrellis.file.Sentinel2PyramidFactory
extent_mock = jvm_mock.geotrellis.vector.Extent.return_value
cellsize_mock = jvm_mock.geotrellis.raster.CellSize.return_value
jvm_mock.geotrellis.vector.Extent.assert_called_once_with(4.0, 51.9999, 4.001, 52.0)
factory_mock.assert_called_once_with("https://services.terrascope.be/catalogue", 'urn:eop:VITO:TERRASCOPE_S2_TOC_V2', ['TOC-B03_10M'], '/data/MTDA/TERRASCOPE_Sentinel2/TOC_V2', cellsize_mock,False)
factory_mock.return_value.pyramid_seq.assert_called_once_with(extent_mock, "EPSG:4326", '2019-01-01T00:00:00+00:00', '2019-01-01T00:00:00+00:00', {}, '')
def _get_layers() -> List[Tuple[str, dict]]:
catalog = get_layer_catalog()
layers = catalog.get_all_metadata()
return [(layer["id"], layer) for layer in layers]
def test_get_layer_catalog_opensearch_enrich_oscars(requests_mock):
test_root = Path(__file__).parent / "data"
with mock.patch(
"openeogeotrellis.layercatalog.ConfigParams") as ConfigParams:
ConfigParams.return_value.layer_catalog_metadata_files = [
test_root / "layercatalog01.json",
test_root / "layercatalog02.json",
test_root / "layercatalog03_oscars.json"
]
collections_response = read_json(test_root /
"collections_oscars01.json")
requests_mock.get(
"https://services.terrascope.test/catalogue/collections",
json=collections_response)
all_metadata = get_layer_catalog(
opensearch_enrich=True).get_all_metadata()
assert all_metadata == [{
"id":
"XIP",
"_vito": {
"data_source": {
"opensearch_endpoint":
"https://services.terrascope.test/catalogue",
"opensearch_collection_id": "urn:eop:VITO:CGS_S1_GRD_SIGMA0_L1"
}
},
"title":
"Sentinel 1 GRD Sigma0 product, VH, VV and angle.",
"description":
"The Sigma0 product describes how much of the radar signal that was sent out by Sentinel-1 "
"is reflected back to the sensor...",
"extent": {
"spatial": {
"bbox": [[-1.05893, 47.66031, 11.6781, 53.67487]]
},
"temporal": {
"interval": [["2014-10-23", None]]
}
},
'keywords': [
'VITO', 'C-SAR', 'Orthoimagery', 'SENTINEL-1A', 'SENTINEL-1',
'SENTINEL-1B', 'RADAR BACKSCATTER', 'RADAR'
],
"links": [{
"rel": "alternate",
"href":
"https://docs.terrascope.be/#/DataProducts/Sentinel-1/ProductsOverview",
"title": "Online User Documentation"
}, {
"rel":
"alternate",
"href":
"https://www.vito-eodata.be/collections/srv/eng/main.home?uuid=urn:eop:VITO:CGS_S1_GRD_SIGMA0_L1"
}, {
"rel": "alternate",
"href":
"https://services.terrascope.be/catalogue/description.geojson?collection=urn:eop:VITO:CGS_S1_GRD_SIGMA0_L1",
"title": "OpenSearch entry point"
}],
"cube:dimensions": {
"x": {
"type": "spatial",
"axis": "x"
},
"y": {
"type": "spatial",
"axis": "y"
},
"t": {
"type": "temporal"
},
"bands": {
"type": "bands",
"values": ["VH"]
}
},
"summaries": {
"eo:bands": [{
"description":
"Calibrated radar backscattering coefficient (unitless), describing the returned radar signal strength in the cross-polarized channel (V transmit, H receive). Values are stored as floats.",
"type": "VH",
"title": "VH",
"resolution": 10,
"bitPerValue": 32,
"name": "VH"
}],
"platform": [],
"instruments": ["MSI"]
},
"assets": {}
}, {
"id": "FOO",
"license": "apache",
"links": ["example.com/foo"]
}, {
"id": "BAR",
"description": "The BAR layer",
"links": ["example.com/bar"]
}, {
"id": "BZZ"
}, {
"id": "QUU"
}]
def test_load_collection_bands_missing_required_extent():
catalog = get_layer_catalog()
load_params = LoadParameters(bands=['TOC-B03_10M'])
env = EvalEnv({'require_bounds': True})
with pytest.raises(OpenEOApiException):
catalog.load_collection('TERRASCOPE_S2_TOC_V2', load_params=load_params, env=env)