def test_get_dimensions_eo_bands_only(): logs = [] dims = CollectionMetadata._parse_dimensions( { "summaries": { "eo:bands": [ { "name": "y", "common_name": "yellow", "center_wavelength": 5 }, { "name": "c", "center_wavelength": 8 }, { "name": "m", "common_name": "magenta" }, ] } }, complain=logs.append) assert_same_dimensions(dims, [ BandDimension(name="bands", bands=[ Band("y", "yellow", 5), Band("c", None, 8), Band("m", "magenta", None), ]) ]) assert logs == [ 'No cube:dimensions metadata', "Assuming name 'bands' for anonymous band dimension." ]
def test_get_dimensions_cube_dimensions_spatial_xyt_bands(): dims = CollectionMetadata._parse_dimensions({ "cube:dimensions": { "x": { "type": "spatial", "extent": [-10, 10] }, "y": { "type": "spatial", "extent": [-56, 83], "reference_system": 123 }, "t": { "type": "temporal", "extent": ["2020-02-20", None] }, "spectral": { "type": "bands", "values": ["red", "green", "blue"] }, } }) assert_same_dimensions(dims, [ SpatialDimension(name="x", extent=[-10, 10]), SpatialDimension(name="y", extent=[-56, 83], crs=123), TemporalDimension(name="t", extent=["2020-02-20", None]), BandDimension(name="spectral", bands=[ Band("red", None, None), Band("green", None, None), Band("blue", None, None), ]) ])
def test_get_dimensions_cube_dimensions_eo_bands(): dims = CollectionMetadata._parse_dimensions({ "cube:dimensions": { "x": {"type": "spatial", "extent": [-10, 10]}, "y": {"type": "spatial", "extent": [-56, 83], "reference_system": 123}, "t": {"type": "temporal", "extent": ["2020-02-20", None]}, "spectral": {"type": "bands", "values": ["r", "g", "b"]}, }, "summaries": { "eo:bands": [ {"name": "r", "common_name": "red", "center_wavelength": 5}, {"name": "g", "center_wavelength": 8}, {"name": "b", "common_name": "blue"}, ] } }) assert_same_dimensions(dims, [ SpatialDimension(name="x", extent=[-10, 10]), SpatialDimension(name="y", extent=[-56, 83], crs=123), TemporalDimension(name="t", extent=["2020-02-20", None]), BandDimension(name="spectral", bands=[ Band("r", "red", 5), Band("g", None, 8), Band("b", "blue", None), ]) ])
def test_band_dimension_band_index(): bdim = BandDimension(name="spectral", bands=[ Band("B02", "blue", 0.490), Band("B03", "green", 0.560), Band("B04", "red", 0.665), Band("B08", "nir", 0.842), ]) assert bdim.band_index(0) == 0 assert bdim.band_index(2) == 2 with pytest.raises(ValueError, match="Invalid band name/index"): bdim.band_index(-1) with pytest.raises(ValueError, match="Invalid band name/index"): bdim.band_index(4) assert bdim.band_index("B02") == 0 assert bdim.band_index("B03") == 1 assert bdim.band_index("B08") == 3 assert bdim.band_index("blue") == 0 assert bdim.band_index("green") == 1 assert bdim.band_index("red") == 2 assert bdim.band_index("nir") == 3 with pytest.raises(ValueError, match="Invalid band name/index"): bdim.band_index("B05") with pytest.raises(ValueError, match="Invalid band name/index"): bdim.band_index("yellow")
def test_band_dimension_rename_labels_with_source_mismatch(): b02 = Band("B02", "blue", 0.490) b03 = Band("B03", "green", 0.560) bdim = BandDimension(name="bs", bands=[b02, b03]) metadata = CollectionMetadata({}, dimensions=[bdim]) with pytest.raises(ValueError, match="should have same number of labels, but got"): _ = metadata.rename_labels("bs", target=['2', "3"], source=['B03'])
def test_band_dimension_filter_bands(): b02 = Band("B02", "blue", 0.490) b03 = Band("B03", "green", 0.560) b04 = Band("B04", "red", 0.665) bdim = BandDimension(name="bs", bands=[b02, b03, b04]) assert bdim.filter_bands(["B03", "B04"]) == BandDimension(name="bs", bands=[b03, b04]) assert bdim.filter_bands(["B04", "blue"]) == BandDimension(name="bs", bands=[b04, b02]) assert bdim.filter_bands(["green", 2]) == BandDimension(name="bs", bands=[b03, b04])
def test_band_dimension(): bdim = BandDimension(name="spectral", bands=[ Band("B02", "blue", 0.490), Band("B03", "green", 0.560), Band("B04", "red", 0.665), ]) assert bdim.band_names == ["B02", "B03", "B04"] assert bdim.common_names == ["blue", "green", "red"]
def test_metadata_bands_dimension(spec): metadata = CollectionMetadata(spec) assert metadata.band_dimension.name == "b" assert metadata.bands == [ Band("foo", "F00", 0.543), Band("bar", None, None) ] assert metadata.band_names == ["foo", "bar"] assert metadata.band_common_names == ["F00", None]
def test_band_dimension_rename_labels(): b02 = Band("B02", "blue", 0.490) b03 = Band("B03", "green", 0.560) b04 = Band("B04", "red", 0.665) bdim = BandDimension(name="bs", bands=[b02, b03, b04]) metadata = CollectionMetadata({},dimensions=[bdim]) newdim = metadata.rename_labels("bs",target=['1','2','3']).band_dimension assert metadata.band_dimension.band_names == ['B02','B03','B04'] assert newdim.band_names == ['1','2','3']
def test_band_dimension_set_labels(): bdim = BandDimension(name="bs", bands=[Band('some_name',None,None)]) metadata = CollectionMetadata({},dimensions=[bdim]) newdim = metadata.rename_labels("bs",target=['1','2','3']).band_dimension assert metadata.band_dimension.band_names == ['some_name'] assert newdim.band_names == ['1','2','3']
def test_band_dimension_band_name(): bdim = BandDimension(name="spectral", bands=[ Band("B02", "blue", 0.490), Band("B03", "green", 0.560), ]) assert bdim.band_name("B02") == "B02" assert bdim.band_name("B03") == "B03" with pytest.raises(ValueError, match="Invalid band name/index"): bdim.band_name("B04") assert bdim.band_name("blue") == "blue" assert bdim.band_name("green") == "green" with pytest.raises(ValueError, match="Invalid band name/index"): bdim.band_name("red") assert bdim.band_name(0) == "B02" assert bdim.band_name(1) == "B03" with pytest.raises(ValueError, match="Invalid band name/index"): bdim.band_name(2)
def download_no_args(self, tmp_path, format, format_options={}): input = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands') imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','','')) res = imagecollection.save_result(str(tmp_path / "test_download_result.") + format, format=format, format_options=format_options) print(res) return res
def test_get_dimensions_multiple_band_dimensions_with_eo_bands(): logs = [] dims = CollectionMetadata._parse_dimensions( { "cube:dimensions": { "x": { "type": "spatial", "extent": [-10, 10] }, "spectral": { "type": "bands", "values": ["alpha", "beta"] }, "bands": { "type": "bands", "values": ["r", "g", "b"] }, }, "summaries": { "eo:bands": [ { "name": "zu", "common_name": "foo" }, ] }, }, complain=logs.append) assert_same_dimensions(dims, [ SpatialDimension(name="x", extent=[-10, 10]), BandDimension(name="spectral", bands=[ Band("alpha", None, None), Band("beta", None, None), ]), BandDimension(name="bands", bands=[ Band("r", None, None), Band("g", None, None), Band("b", None, None), ]), ]) assert logs == ["Multiple dimensions of type 'bands'"]
def test_get_dimensions_cube_dimensions_eo_bands_mismatch(): logs = [] dims = CollectionMetadata._parse_dimensions( { "cube:dimensions": { "x": { "type": "spatial", "extent": [-10, 10] }, "spectral": { "type": "bands", "values": ["r", "g", "b"] }, }, "summaries": { "eo:bands": [ { "name": "y", "common_name": "yellow", "center_wavelength": 5 }, { "name": "c", "center_wavelength": 8 }, { "name": "m", "common_name": "magenta" }, ] } }, complain=logs.append) assert_same_dimensions(dims, [ SpatialDimension(name="x", extent=[-10, 10]), BandDimension(name="spectral", bands=[ Band("r", None, None), Band("g", None, None), Band("b", None, None), ]) ]) assert logs == ["Band name mismatch: ['r', 'g', 'b'] != ['y', 'c', 'm']"]
def download_no_args_single_byte_band(self, tmp_path, format, format_options={}): input = self.create_spacetime_layer().convert_data_type(gps.CellType.UINT8) imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands') imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','','')) imagecollection.filter_bands(['band_one']) res = imagecollection.save_result(str(tmp_path / "test_download_result_single_band.") + format, format=format, format_options=format_options) print(res) return res
def download_masked(self, tmp_path, format): input = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands') imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','','')) polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]]) imagecollection = imagecollection.mask_polygon(mask=polygon) filename = str(tmp_path / "test_download_masked_result.") + format res = imagecollection.save_result(filename, format=format) print(res)
def test_write_assets(self, tmp_path): input = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) imagecollection.metadata = imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands') imagecollection.metadata = imagecollection.metadata.append_band(Band('band_two', '', '')) format = 'GTiff' res = imagecollection.write_assets(str(tmp_path / "test_download_result.") + format, format=format,format_options={ "multidate":True, "batch_mode":True }) assert 1 == len(res) name, asset = res.popitem() assert Path(asset['href']).parent == tmp_path assert asset['nodata'] == -1 assert asset['roles'] == ['data'] assert 2 == len(asset['bands']) assert 'image/tiff; application=geotiff' == asset['type'] assert asset['datetime'] == "2017-09-25T11:37:00Z"
def test_write_assets_samples_netcdf(self, tmp_path): input = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) imagecollection.metadata = imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands') imagecollection.metadata = imagecollection.metadata.append_band(Band('band_two', '', '')) format = 'netCDF' res = imagecollection.write_assets(str(tmp_path / "test_download_result.") + format, format=format,format_options={ "batch_mode":True, "geometries":geojson_to_geometry(self.features), "sample_by_feature": True, "feature_id_property": 'id' }) assert len(res) == 3 name,asset = res.popitem() file = asset['href'] assert asset['nodata'] == -1 assert asset['roles'] == ['data'] assert 2 == len(asset['bands']) assert 'application/x-netcdf' == asset['type']
def download_masked_reproject(self, tmp_path, format): input = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands') imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','','')) polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]]) import pyproj from shapely.ops import transform from functools import partial project = partial( pyproj.transform, pyproj.Proj(init="EPSG:4326"), # source coordinate system pyproj.Proj(init="EPSG:3857")) # destination coordinate system reprojected = transform(project, polygon) imagecollection = imagecollection.mask_polygon(mask=reprojected, srs="EPSG:3857") filename = str(tmp_path / "test_download_masked_result.3857.") + format res = imagecollection.save_result(filename, format=format) print(res)
def sentinel_hub_pyramid(): # TODO: move the metadata manipulation out of this function and get rid of the nonlocal? nonlocal metadata dependencies = env.get('dependencies', {}) sar_backscatter_arguments: Optional[SarBackscatterArgs] = ( (load_params.sar_backscatter or SarBackscatterArgs()) if sar_backscatter_compatible else None) if dependencies: source_location, card4l = dependencies[( collection_id, to_hashable(metadata_properties()))] # date_regex supports: # - original: _20210223.tif # - CARD4L: s1_rtc_0446B9_S07E035_2021_02_03_MULTIBAND.tif # - tiles assembled from cache: 31UDS_7_2-20190921.tif date_regex = r".+(\d{4})_?(\d{2})_?(\d{2}).*\.tif" interpret_as_cell_type = "float32ud0" lat_lon = card4l if source_location.startswith("file:"): assembled_uri = source_location glob_pattern = f"{assembled_uri}/*.tif" logger.info(f"Sentinel Hub pyramid from {glob_pattern}") pyramid_factory = jvm.org.openeo.geotrellis.geotiff.PyramidFactory.from_disk( glob_pattern, date_regex, interpret_as_cell_type, lat_lon) else: s3_uri = source_location key_regex = r".+\.tif" recursive = True logger.info(f"Sentinel Hub pyramid from {s3_uri}") pyramid_factory = jvm.org.openeo.geotrellis.geotiff.PyramidFactory.from_s3( s3_uri, key_regex, date_regex, recursive, interpret_as_cell_type, lat_lon) if sar_backscatter_arguments and sar_backscatter_arguments.mask: metadata = metadata.append_band( Band(name='mask', common_name=None, wavelength_um=None)) if sar_backscatter_arguments and sar_backscatter_arguments.local_incidence_angle: metadata = metadata.append_band( Band(name='local_incidence_angle', common_name=None, wavelength_um=None)) return (pyramid_factory.datacube_seq( projected_polygons_native_crs, from_date, to_date, metadata_properties(), collection_id, datacubeParams) if single_level else pyramid_factory.pyramid_seq( extent, srs, from_date, to_date)) else: if collection_id == 'PLANETSCOPE': # note: "byoc-" prefix is optional for the collection ID but dataset ID requires it shub_collection_id = feature_flags['byoc_collection_id'] dataset_id = shub_collection_id else: shub_collection_id = layer_source_info['collection_id'] dataset_id = layer_source_info['dataset_id'] endpoint = layer_source_info['endpoint'] client_id = layer_source_info['client_id'] client_secret = layer_source_info['client_secret'] sample_type = jvm.org.openeo.geotrellissentinelhub.SampleType.withName( layer_source_info.get('sample_type', 'UINT16')) shub_band_names = metadata.band_names if sar_backscatter_arguments and sar_backscatter_arguments.mask: metadata = metadata.append_band( Band(name='mask', common_name=None, wavelength_um=None)) shub_band_names.append('dataMask') if sar_backscatter_arguments and sar_backscatter_arguments.local_incidence_angle: metadata = metadata.append_band( Band(name='local_incidence_angle', common_name=None, wavelength_um=None)) shub_band_names.append('localIncidenceAngle') band_gsds = [ band.gsd['value'] for band in metadata.bands if band.gsd is not None ] cell_size = (jvm.geotrellis.raster.CellSize( min([float(gsd[0]) for gsd in band_gsds]), min([float(gsd[1]) for gsd in band_gsds])) if len(band_gsds) > 0 else jvm.geotrellis.raster.CellSize( cell_width, cell_height)) pyramid_factory = jvm.org.openeo.geotrellissentinelhub.PyramidFactory.rateLimited( endpoint, shub_collection_id, dataset_id, client_id, client_secret, sentinel_hub.processing_options(sar_backscatter_arguments) if sar_backscatter_arguments else {}, sample_type, cell_size) unflattened_metadata_properties = metadata_properties( flatten_eqs=False) return (pyramid_factory.datacube_seq( projected_polygons_native_crs.polygons(), projected_polygons_native_crs.crs(), from_date, to_date, shub_band_names, unflattened_metadata_properties, datacubeParams) if single_level else pyramid_factory.pyramid_seq( extent, srs, from_date, to_date, shub_band_names, unflattened_metadata_properties))
def get_results(self, job_id: str, user_id: str) -> Dict[str, dict]: if self._get_job_info(job_id=job_id, user_id=user_id).status != "finished": raise JobNotFinishedException return { "output.tiff": { "asset": True, "output_dir": str(self._output_root() / job_id), "type": "image/tiff; application=geotiff", "roles": ["data"], "bands": [Band(name="NDVI", common_name="NDVI", wavelength_um=1.23)], "nodata": 123, "instruments": "MSI" }, "randomforest.model": { "asset": True, "href": str(Path(job_id) / "randomforest.model"), }, DriverMlModel.METADATA_FILE_NAME: { "ml_model_metadata": True, "asset": False, "stac_version": "1.0.0", "stac_extensions": [ "https://stac-extensions.github.io/ml-model/v1.0.0/schema.json" ], "type": "Feature", "id": str(uuid.uuid4()), "collection": "collection-id", "bbox": [-179.999, -89.999, 179.999, 89.999], "geometry": { "type": "Polygon", "coordinates": [[[-179.999, -89.999], [179.999, -89.999], [179.999, 89.999], [-179.999, 89.999], [-179.999, -89.999]]] }, 'properties': { "datetime": None, "start_datetime": "1970-01-01T00:00:00Z", "end_datetime": "9999-12-31T23:59:59Z", "ml-model:type": "ml-model", "ml-model:learning_approach": "supervised", "ml-model:prediction_type": "classification", "ml-model:architecture": "random-forest", "ml-model:training-processor-type": "cpu", "ml-model:training-os": "linux", }, 'links': [], 'assets': { 'model': { "href": str(Path(job_id) / "randomforest.model"), "type": "application/octet-stream", "title": "org.apache.spark.mllib.tree.model.RandomForestModel", "roles": ["ml-model:checkpoint"] } } } }