def test_get_dimensions_eo_bands_only():
logs = []
dims = CollectionMetadata._parse_dimensions(
{
"summaries": {
"eo:bands": [
{
"name": "y",
"common_name": "yellow",
"center_wavelength": 5
},
{
"name": "c",
"center_wavelength": 8
},
{
"name": "m",
"common_name": "magenta"
},
]
}
},
complain=logs.append)
assert_same_dimensions(dims, [
BandDimension(name="bands",
bands=[
Band("y", "yellow", 5),
Band("c", None, 8),
Band("m", "magenta", None),
])
])
assert logs == [
'No cube:dimensions metadata',
"Assuming name 'bands' for anonymous band dimension."
]
def test_get_dimensions_cube_dimensions_spatial_xyt_bands():
dims = CollectionMetadata._parse_dimensions({
"cube:dimensions": {
"x": {
"type": "spatial",
"extent": [-10, 10]
},
"y": {
"type": "spatial",
"extent": [-56, 83],
"reference_system": 123
},
"t": {
"type": "temporal",
"extent": ["2020-02-20", None]
},
"spectral": {
"type": "bands",
"values": ["red", "green", "blue"]
},
}
})
assert_same_dimensions(dims, [
SpatialDimension(name="x", extent=[-10, 10]),
SpatialDimension(name="y", extent=[-56, 83], crs=123),
TemporalDimension(name="t", extent=["2020-02-20", None]),
BandDimension(name="spectral",
bands=[
Band("red", None, None),
Band("green", None, None),
Band("blue", None, None),
])
])
def test_get_dimensions_cube_dimensions_eo_bands():
dims = CollectionMetadata._parse_dimensions({
"cube:dimensions": {
"x": {"type": "spatial", "extent": [-10, 10]},
"y": {"type": "spatial", "extent": [-56, 83], "reference_system": 123},
"t": {"type": "temporal", "extent": ["2020-02-20", None]},
"spectral": {"type": "bands", "values": ["r", "g", "b"]},
},
"summaries": {
"eo:bands": [
{"name": "r", "common_name": "red", "center_wavelength": 5},
{"name": "g", "center_wavelength": 8},
{"name": "b", "common_name": "blue"},
]
}
})
assert_same_dimensions(dims, [
SpatialDimension(name="x", extent=[-10, 10]),
SpatialDimension(name="y", extent=[-56, 83], crs=123),
TemporalDimension(name="t", extent=["2020-02-20", None]),
BandDimension(name="spectral", bands=[
Band("r", "red", 5),
Band("g", None, 8),
Band("b", "blue", None),
])
])
def test_band_dimension_band_index():
bdim = BandDimension(name="spectral",
bands=[
Band("B02", "blue", 0.490),
Band("B03", "green", 0.560),
Band("B04", "red", 0.665),
Band("B08", "nir", 0.842),
])
assert bdim.band_index(0) == 0
assert bdim.band_index(2) == 2
with pytest.raises(ValueError, match="Invalid band name/index"):
bdim.band_index(-1)
with pytest.raises(ValueError, match="Invalid band name/index"):
bdim.band_index(4)
assert bdim.band_index("B02") == 0
assert bdim.band_index("B03") == 1
assert bdim.band_index("B08") == 3
assert bdim.band_index("blue") == 0
assert bdim.band_index("green") == 1
assert bdim.band_index("red") == 2
assert bdim.band_index("nir") == 3
with pytest.raises(ValueError, match="Invalid band name/index"):
bdim.band_index("B05")
with pytest.raises(ValueError, match="Invalid band name/index"):
bdim.band_index("yellow")
def test_band_dimension_rename_labels_with_source_mismatch():
b02 = Band("B02", "blue", 0.490)
b03 = Band("B03", "green", 0.560)
bdim = BandDimension(name="bs", bands=[b02, b03])
metadata = CollectionMetadata({}, dimensions=[bdim])
with pytest.raises(ValueError,
match="should have same number of labels, but got"):
_ = metadata.rename_labels("bs", target=['2', "3"], source=['B03'])
def test_band_dimension_filter_bands():
b02 = Band("B02", "blue", 0.490)
b03 = Band("B03", "green", 0.560)
b04 = Band("B04", "red", 0.665)
bdim = BandDimension(name="bs", bands=[b02, b03, b04])
assert bdim.filter_bands(["B03", "B04"]) == BandDimension(name="bs", bands=[b03, b04])
assert bdim.filter_bands(["B04", "blue"]) == BandDimension(name="bs", bands=[b04, b02])
assert bdim.filter_bands(["green", 2]) == BandDimension(name="bs", bands=[b03, b04])
def test_band_dimension():
bdim = BandDimension(name="spectral", bands=[
Band("B02", "blue", 0.490),
Band("B03", "green", 0.560),
Band("B04", "red", 0.665),
])
assert bdim.band_names == ["B02", "B03", "B04"]
assert bdim.common_names == ["blue", "green", "red"]
def test_metadata_bands_dimension(spec):
metadata = CollectionMetadata(spec)
assert metadata.band_dimension.name == "b"
assert metadata.bands == [
Band("foo", "F00", 0.543),
Band("bar", None, None)
]
assert metadata.band_names == ["foo", "bar"]
assert metadata.band_common_names == ["F00", None]
def test_band_dimension_rename_labels():
b02 = Band("B02", "blue", 0.490)
b03 = Band("B03", "green", 0.560)
b04 = Band("B04", "red", 0.665)
bdim = BandDimension(name="bs", bands=[b02, b03, b04])
metadata = CollectionMetadata({},dimensions=[bdim])
newdim = metadata.rename_labels("bs",target=['1','2','3']).band_dimension
assert metadata.band_dimension.band_names == ['B02','B03','B04']
assert newdim.band_names == ['1','2','3']
def test_band_dimension_set_labels():
bdim = BandDimension(name="bs", bands=[Band('some_name',None,None)])
metadata = CollectionMetadata({},dimensions=[bdim])
newdim = metadata.rename_labels("bs",target=['1','2','3']).band_dimension
assert metadata.band_dimension.band_names == ['some_name']
assert newdim.band_names == ['1','2','3']
def test_band_dimension_band_name():
bdim = BandDimension(name="spectral", bands=[
Band("B02", "blue", 0.490),
Band("B03", "green", 0.560),
])
assert bdim.band_name("B02") == "B02"
assert bdim.band_name("B03") == "B03"
with pytest.raises(ValueError, match="Invalid band name/index"):
bdim.band_name("B04")
assert bdim.band_name("blue") == "blue"
assert bdim.band_name("green") == "green"
with pytest.raises(ValueError, match="Invalid band name/index"):
bdim.band_name("red")
assert bdim.band_name(0) == "B02"
assert bdim.band_name(1) == "B03"
with pytest.raises(ValueError, match="Invalid band name/index"):
bdim.band_name(2)
def download_no_args(self, tmp_path, format, format_options={}):
input = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands')
imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','',''))
res = imagecollection.save_result(str(tmp_path / "test_download_result.") + format, format=format, format_options=format_options)
print(res)
return res
def test_get_dimensions_multiple_band_dimensions_with_eo_bands():
logs = []
dims = CollectionMetadata._parse_dimensions(
{
"cube:dimensions": {
"x": {
"type": "spatial",
"extent": [-10, 10]
},
"spectral": {
"type": "bands",
"values": ["alpha", "beta"]
},
"bands": {
"type": "bands",
"values": ["r", "g", "b"]
},
},
"summaries": {
"eo:bands": [
{
"name": "zu",
"common_name": "foo"
},
]
},
},
complain=logs.append)
assert_same_dimensions(dims, [
SpatialDimension(name="x", extent=[-10, 10]),
BandDimension(name="spectral",
bands=[
Band("alpha", None, None),
Band("beta", None, None),
]),
BandDimension(name="bands",
bands=[
Band("r", None, None),
Band("g", None, None),
Band("b", None, None),
]),
])
assert logs == ["Multiple dimensions of type 'bands'"]
def test_get_dimensions_cube_dimensions_eo_bands_mismatch():
logs = []
dims = CollectionMetadata._parse_dimensions(
{
"cube:dimensions": {
"x": {
"type": "spatial",
"extent": [-10, 10]
},
"spectral": {
"type": "bands",
"values": ["r", "g", "b"]
},
},
"summaries": {
"eo:bands": [
{
"name": "y",
"common_name": "yellow",
"center_wavelength": 5
},
{
"name": "c",
"center_wavelength": 8
},
{
"name": "m",
"common_name": "magenta"
},
]
}
},
complain=logs.append)
assert_same_dimensions(dims, [
SpatialDimension(name="x", extent=[-10, 10]),
BandDimension(name="spectral",
bands=[
Band("r", None, None),
Band("g", None, None),
Band("b", None, None),
])
])
assert logs == ["Band name mismatch: ['r', 'g', 'b'] != ['y', 'c', 'm']"]
def download_no_args_single_byte_band(self, tmp_path, format, format_options={}):
input = self.create_spacetime_layer().convert_data_type(gps.CellType.UINT8)
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands')
imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','',''))
imagecollection.filter_bands(['band_one'])
res = imagecollection.save_result(str(tmp_path / "test_download_result_single_band.") + format, format=format, format_options=format_options)
print(res)
return res
def download_masked(self, tmp_path, format):
input = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands')
imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','',''))
polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]])
imagecollection = imagecollection.mask_polygon(mask=polygon)
filename = str(tmp_path / "test_download_masked_result.") + format
res = imagecollection.save_result(filename, format=format)
print(res)
def test_write_assets(self, tmp_path):
input = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
imagecollection.metadata = imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands')
imagecollection.metadata = imagecollection.metadata.append_band(Band('band_two', '', ''))
format = 'GTiff'
res = imagecollection.write_assets(str(tmp_path / "test_download_result.") + format, format=format,format_options={
"multidate":True,
"batch_mode":True
})
assert 1 == len(res)
name, asset = res.popitem()
assert Path(asset['href']).parent == tmp_path
assert asset['nodata'] == -1
assert asset['roles'] == ['data']
assert 2 == len(asset['bands'])
assert 'image/tiff; application=geotiff' == asset['type']
assert asset['datetime'] == "2017-09-25T11:37:00Z"
def test_write_assets_samples_netcdf(self, tmp_path):
input = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
imagecollection.metadata = imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands')
imagecollection.metadata = imagecollection.metadata.append_band(Band('band_two', '', ''))
format = 'netCDF'
res = imagecollection.write_assets(str(tmp_path / "test_download_result.") + format, format=format,format_options={
"batch_mode":True,
"geometries":geojson_to_geometry(self.features),
"sample_by_feature": True,
"feature_id_property": 'id'
})
assert len(res) == 3
name,asset = res.popitem()
file = asset['href']
assert asset['nodata'] == -1
assert asset['roles'] == ['data']
assert 2 == len(asset['bands'])
assert 'application/x-netcdf' == asset['type']
def download_masked_reproject(self, tmp_path, format):
input = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands')
imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','',''))
polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]])
import pyproj
from shapely.ops import transform
from functools import partial
project = partial(
pyproj.transform,
pyproj.Proj(init="EPSG:4326"), # source coordinate system
pyproj.Proj(init="EPSG:3857")) # destination coordinate system
reprojected = transform(project, polygon)
imagecollection = imagecollection.mask_polygon(mask=reprojected, srs="EPSG:3857")
filename = str(tmp_path / "test_download_masked_result.3857.") + format
res = imagecollection.save_result(filename, format=format)
print(res)
def sentinel_hub_pyramid():
# TODO: move the metadata manipulation out of this function and get rid of the nonlocal?
nonlocal metadata
dependencies = env.get('dependencies', {})
sar_backscatter_arguments: Optional[SarBackscatterArgs] = (
(load_params.sar_backscatter or SarBackscatterArgs())
if sar_backscatter_compatible else None)
if dependencies:
source_location, card4l = dependencies[(
collection_id, to_hashable(metadata_properties()))]
# date_regex supports:
# - original: _20210223.tif
# - CARD4L: s1_rtc_0446B9_S07E035_2021_02_03_MULTIBAND.tif
# - tiles assembled from cache: 31UDS_7_2-20190921.tif
date_regex = r".+(\d{4})_?(\d{2})_?(\d{2}).*\.tif"
interpret_as_cell_type = "float32ud0"
lat_lon = card4l
if source_location.startswith("file:"):
assembled_uri = source_location
glob_pattern = f"{assembled_uri}/*.tif"
logger.info(f"Sentinel Hub pyramid from {glob_pattern}")
pyramid_factory = jvm.org.openeo.geotrellis.geotiff.PyramidFactory.from_disk(
glob_pattern, date_regex, interpret_as_cell_type,
lat_lon)
else:
s3_uri = source_location
key_regex = r".+\.tif"
recursive = True
logger.info(f"Sentinel Hub pyramid from {s3_uri}")
pyramid_factory = jvm.org.openeo.geotrellis.geotiff.PyramidFactory.from_s3(
s3_uri, key_regex, date_regex, recursive,
interpret_as_cell_type, lat_lon)
if sar_backscatter_arguments and sar_backscatter_arguments.mask:
metadata = metadata.append_band(
Band(name='mask', common_name=None,
wavelength_um=None))
if sar_backscatter_arguments and sar_backscatter_arguments.local_incidence_angle:
metadata = metadata.append_band(
Band(name='local_incidence_angle',
common_name=None,
wavelength_um=None))
return (pyramid_factory.datacube_seq(
projected_polygons_native_crs, from_date, to_date,
metadata_properties(), collection_id, datacubeParams)
if single_level else pyramid_factory.pyramid_seq(
extent, srs, from_date, to_date))
else:
if collection_id == 'PLANETSCOPE':
# note: "byoc-" prefix is optional for the collection ID but dataset ID requires it
shub_collection_id = feature_flags['byoc_collection_id']
dataset_id = shub_collection_id
else:
shub_collection_id = layer_source_info['collection_id']
dataset_id = layer_source_info['dataset_id']
endpoint = layer_source_info['endpoint']
client_id = layer_source_info['client_id']
client_secret = layer_source_info['client_secret']
sample_type = jvm.org.openeo.geotrellissentinelhub.SampleType.withName(
layer_source_info.get('sample_type', 'UINT16'))
shub_band_names = metadata.band_names
if sar_backscatter_arguments and sar_backscatter_arguments.mask:
metadata = metadata.append_band(
Band(name='mask', common_name=None,
wavelength_um=None))
shub_band_names.append('dataMask')
if sar_backscatter_arguments and sar_backscatter_arguments.local_incidence_angle:
metadata = metadata.append_band(
Band(name='local_incidence_angle',
common_name=None,
wavelength_um=None))
shub_band_names.append('localIncidenceAngle')
band_gsds = [
band.gsd['value'] for band in metadata.bands
if band.gsd is not None
]
cell_size = (jvm.geotrellis.raster.CellSize(
min([float(gsd[0]) for gsd in band_gsds]),
min([float(gsd[1])
for gsd in band_gsds])) if len(band_gsds) > 0 else
jvm.geotrellis.raster.CellSize(
cell_width, cell_height))
pyramid_factory = jvm.org.openeo.geotrellissentinelhub.PyramidFactory.rateLimited(
endpoint, shub_collection_id, dataset_id, client_id,
client_secret,
sentinel_hub.processing_options(sar_backscatter_arguments)
if sar_backscatter_arguments else {}, sample_type,
cell_size)
unflattened_metadata_properties = metadata_properties(
flatten_eqs=False)
return (pyramid_factory.datacube_seq(
projected_polygons_native_crs.polygons(),
projected_polygons_native_crs.crs(), from_date, to_date,
shub_band_names, unflattened_metadata_properties,
datacubeParams)
if single_level else pyramid_factory.pyramid_seq(
extent, srs, from_date, to_date, shub_band_names,
unflattened_metadata_properties))
def get_results(self, job_id: str, user_id: str) -> Dict[str, dict]:
if self._get_job_info(job_id=job_id,
user_id=user_id).status != "finished":
raise JobNotFinishedException
return {
"output.tiff": {
"asset":
True,
"output_dir":
str(self._output_root() / job_id),
"type":
"image/tiff; application=geotiff",
"roles": ["data"],
"bands":
[Band(name="NDVI", common_name="NDVI", wavelength_um=1.23)],
"nodata":
123,
"instruments":
"MSI"
},
"randomforest.model": {
"asset": True,
"href": str(Path(job_id) / "randomforest.model"),
},
DriverMlModel.METADATA_FILE_NAME: {
"ml_model_metadata":
True,
"asset":
False,
"stac_version":
"1.0.0",
"stac_extensions": [
"https://stac-extensions.github.io/ml-model/v1.0.0/schema.json"
],
"type":
"Feature",
"id":
str(uuid.uuid4()),
"collection":
"collection-id",
"bbox": [-179.999, -89.999, 179.999, 89.999],
"geometry": {
"type":
"Polygon",
"coordinates": [[[-179.999, -89.999], [179.999, -89.999],
[179.999, 89.999], [-179.999, 89.999],
[-179.999, -89.999]]]
},
'properties': {
"datetime": None,
"start_datetime": "1970-01-01T00:00:00Z",
"end_datetime": "9999-12-31T23:59:59Z",
"ml-model:type": "ml-model",
"ml-model:learning_approach": "supervised",
"ml-model:prediction_type": "classification",
"ml-model:architecture": "random-forest",
"ml-model:training-processor-type": "cpu",
"ml-model:training-os": "linux",
},
'links': [],
'assets': {
'model': {
"href": str(Path(job_id) / "randomforest.model"),
"type": "application/octet-stream",
"title":
"org.apache.spark.mllib.tree.model.RandomForestModel",
"roles": ["ml-model:checkpoint"]
}
}
}
}