def test_merge_cubes_exception_if_levels_do_not_match(): red_ramp, nir_ramp = np.mgrid[0:4, 0:4] layer1 = _create_spacetime_layer(cells=np.array([[red_ramp]])) layer2 = _create_spacetime_layer(cells=np.array([[nir_ramp]])) metadata = _build_metadata(bands=["the_band"]) cube1 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer1}), metadata=metadata) cube2 = GeopysparkDataCube(pyramid=gps.Pyramid({14: layer2}), metadata=metadata) with pytest.raises(OpenEOApiException) as excinfo: res = cube1.merge_cubes(cube2, 'sum')
def test_merge_cubes_into_single_band(): red_ramp, nir_ramp = np.mgrid[0:4, 0:4] layer1 = _create_spacetime_layer(cells=np.array([[red_ramp]])) layer2 = _create_spacetime_layer(cells=np.array([[nir_ramp]])) metadata = _build_metadata(bands=["the_band"]) cube1 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer1}), metadata=metadata) cube2 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer2}), metadata=metadata) res = cube1.merge_cubes(cube2, 'sum') stitched = res.pyramid.levels[0].to_spatial_layer().stitch() assert stitched.cells.shape[0] == 1 np.testing.assert_array_equal(red_ramp + nir_ramp, stitched.cells[0, 0:4, 0:4])
def test_reduce_bands_comparison_ops(self): input = self.create_spacetime_layer_singleband() input = gps.Pyramid({0: input}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) visitor = GeotrellisTileProcessGraphVisitor() graph = { "gt": { "arguments": { "x": { "from_argument": "data" }, "y": 6.0 }, "process_id": "gt", "result": True } } visitor.accept_process_graph(graph) stitched = imagecollection.reduce_bands( visitor).pyramid.levels[0].to_spatial_layer().stitch() print(stitched) self.assertEqual(1, stitched.cells[0][0][0])
def test_linear_scale_range(self): red_ramp, nir_ramp = np.mgrid[0:4, 0:4] layer = self._create_spacetime_layer( cells=np.array([[red_ramp], [nir_ramp]])) pyramid = gps.Pyramid({0: layer}) metadata = CollectionMetadata({ "properties": { "eo:bands": [ { "name": "B04", "common_name": "red" }, { "name": "B08", "common_name": "nir" }, ] } }) imagecollection = GeotrellisTimeSeriesImageCollection( pyramid, InMemoryServiceRegistry(), metadata=metadata) stitched = imagecollection.ndvi().linear_scale_range( -1, 1, 0, 100).pyramid.levels[0].to_spatial_layer().stitch() cells = stitched.cells[0, 0:4, 0:4] expected = 50.0 * (1.0 + np.array( [[np.nan, 1 / 1, 2 / 2, 3 / 3], [-1 / 1, 0 / 2, 1 / 3, 2 / 4], [-2 / 2, -1 / 3, 0 / 4, 1 / 5], [-3 / 3, -2 / 4, -1 / 5, 0 / 6]])) expected[0][0] = 255.0 np.testing.assert_array_almost_equal(cells, expected.astype(np.uint8))
def test_zonal_statistics_median_datacube(self): layer = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer}), InMemoryServiceRegistry()) polygon = Polygon(shell=[ (0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0), (0.0, 0.0) ]) result = imagecollection.zonal_statistics(polygon, "median") assert result.data == {'2017-09-25T11:37:00Z': [[1.0, 2.0]]} covjson = result.to_covjson() assert covjson["ranges"] == { "band0": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 1), "values": [1.0] }, "band1": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 1), "values": [2.0] } }
def test_apply_complex_graph(): graph = { "sin": { "arguments": { "x": { "from_argument": "data" } }, "process_id": "sin", "result": False }, "multiply": { "arguments": { "x": { "from_node": "sin" }, "y": 5.0 }, "process_id": "multiply", "result": True } } input = create_spacetime_layer() cube = GeopysparkDataCube(gps.Pyramid({0: input}), InMemoryServiceRegistry()) res = cube.apply(graph) data = res.pyramid.levels[0].to_spatial_layer().stitch().cells np.testing.assert_array_almost_equal(data[0, 2:6, 2:6], 5.0 * np.sin(first[0])) np.testing.assert_array_almost_equal(data[1, 2:6, 2:6], 5.0 * np.sin(second[0]))
def test_apply_cos(): input = create_spacetime_layer() cube = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) res = cube.apply("cos") data = res.pyramid.levels[0].to_spatial_layer().stitch().cells np.testing.assert_array_almost_equal(data[0, 2:6, 2:6], np.cos(first[0])) np.testing.assert_array_almost_equal(data[1, 2:6, 2:6], np.cos(second[0]))
def test_reduce_bands_logical_ops(): input = create_spacetime_layer_singleband() input = gps.Pyramid({0: input}) imagecollection = GeopysparkDataCube(pyramid=input) visitor = GeotrellisTileProcessGraphVisitor() graph = { "eq": { "arguments": { "x": { "from_argument": "data" }, "y": 10 }, "process_id": "eq", }, "not": { "arguments": { "expression": { "from_node": "eq" } }, "process_id": "not", "result": True } } visitor.accept_process_graph(graph) stitched = imagecollection.reduce_bands( visitor).pyramid.levels[0].to_spatial_layer().stitch() print(stitched) assert 0 == stitched.cells[0][0][0]
def imagecollection_with_two_bands_and_three_dates_webmerc(request): from geopyspark.geotrellis import (SpaceTimeKey, Tile, _convert_to_unix_time) from geopyspark.geotrellis.constants import LayerType from geopyspark.geotrellis.layer import TiledRasterLayer import geopyspark as gps from openeogeotrellis.geopysparkdatacube import GeopysparkDataCube,GeopysparkCubeMetadata date1, date3, rdd = numpy_rdd_two_bands_and_three_dates() metadata = {'cellType': 'int32ud-1', 'extent': extent_webmerc, 'crs': '+proj=merc +a=6378137 +b=6378137 +lat_ts=0 +lon_0=0 +x_0=0 +y_0=0 +k=1 +units=m +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(date1)}, 'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(date3)} }, 'layoutDefinition': { 'extent': extent_webmerc, 'tileLayout': layout } } geopyspark_layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata) datacube = GeopysparkDataCube(pyramid=gps.Pyramid({0: geopyspark_layer}), metadata=GeopysparkCubeMetadata(openeo_metadata)) if request.instance: request.instance.imagecollection_with_two_bands_and_three_dates = datacube return datacube
def create_red_nir_layer(): red_ramp, nir_ramp = np.mgrid[0:4, 0:4] layer = _create_spacetime_layer(cells=np.array([[red_ramp], [nir_ramp]])) pyramid = gps.Pyramid({0: layer}) metadata = GeopysparkCubeMetadata({ "cube:dimensions": { "x": { "type": "spatial", "axis": "x" }, "y": { "type": "spatial", "axis": "y" }, "bands": { "type": "bands", "values": ["B04", "B08"] } }, "summaries": { "eo:bands": [ { "name": "B04", "common_name": "red" }, { "name": "B08", "common_name": "nir" }, ] } }) imagecollection = GeopysparkDataCube(pyramid=pyramid, metadata=metadata) return imagecollection
def test_apply_if(): input = create_spacetime_layer_singleband() input = gps.Pyramid({0: input}) imagecollection = GeopysparkDataCube(pyramid=input) graph = { "6": { "arguments": { "reject": { "from_parameter": "x" }, "value": { "from_node": "10" }, "accept": 2.0 }, "process_id": "if", "result": True }, "10": { "process_id": "gt", "arguments": { "x": { "from_parameter": "x" }, "y": 7.0 } } } stitched = imagecollection.apply( graph).pyramid.levels[0].to_spatial_layer().stitch() print(stitched) assert 2.0 == stitched.cells[0][0][0]
def test_zonal_statistics(self): layer = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer})) polygon = Polygon(shell=[ (0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0), (0.0, 0.0) ]) result = imagecollection.zonal_statistics(polygon, "mean") assert result.data == {'2017-09-25T11:37:00Z': [[1.0, 2.0]]} covjson = result.to_covjson() assert covjson["ranges"] == { "band0": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 1), "values": [1.0] }, "band1": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 1), "values": [2.0] }, }
def test_download_as_catalog(self): input = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) imagecollection.download("catalogresult.tiff", format="GTIFF", parameters={"catalog": True})
def download_no_args(self, tmp_path, format, format_options={}): input = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands') imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','','')) res = imagecollection.save_result(str(tmp_path / "test_download_result.") + format, format=format, format_options=format_options) print(res) return res
def test_polygon_series(self): input = self.create_spacetime_layer() polygon = Polygon([(0, 0), (0, 2), (2, 2), (2, 0), (0, 0)]) imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry(), self.openeo_metadata) means = imagecollection.polygonal_mean_timeseries(polygon) assert means == {'2017-09-25T11:37:00': [[1.0, 2.0]]}
def test_download_geotiff_no_args(self): input = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) geotiffs = imagecollection.download( str(self.temp_folder / "test_download_result.geotiff")) print(geotiffs)
def test_download_masked_geotiff(self): input = self.create_spacetime_layer() polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]]) imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) imagecollection = imagecollection.mask(polygon) geotiffs = imagecollection.download( str(self.temp_folder / "test_download_masked_result.geotiff")) print(geotiffs)
def download_no_args_single_byte_band(self, tmp_path, format, format_options={}): input = self.create_spacetime_layer().convert_data_type(gps.CellType.UINT8) imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands') imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','','')) imagecollection.filter_bands(['band_one']) res = imagecollection.save_result(str(tmp_path / "test_download_result_single_band.") + format, format=format, format_options=format_options) print(res) return res
def test_another_polygon_series(self): input = self._create_spacetime_layer(no_data=-1.0) imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) polygon = Polygon(shell=[(2.0, 6.0), (6.0, 6.0), (6.0, 2.0), (2.0, 2.0), (2.0, 6.0)]) means = imagecollection.zonal_statistics(regions=polygon, func="mean") assert means.data == { '2017-09-25T11:37:00Z': [[(0 + 0 + 0 + 0 + 1 + 1 + 1 + 1 + 2 + 2 + 2 + 2) / 12]] }
def download_masked(self, tmp_path, format): input = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands') imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','','')) polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]]) imagecollection = imagecollection.mask_polygon(mask=polygon) filename = str(tmp_path / "test_download_masked_result.") + format res = imagecollection.save_result(filename, format=format) print(res)
def test_another_polygon_series(self): input = self._create_spacetime_layer(no_data=-1.0) imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) polygon = Polygon(shell=[(2.0, 6.0), (6.0, 6.0), (6.0, 2.0), (2.0, 2.0), (2.0, 6.0)]) means = imagecollection.polygonal_mean_timeseries(polygon) assert means == { '2017-09-25T11:37:00': [[(0 + 0 + 0 + 0 + 1 + 1 + 1 + 1 + 2 + 2 + 2 + 2) / 12]] }
def test_point_series(): input = create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) transformed_collection = imagecollection.apply("cos") for p in points[0:3]: result = transformed_collection.timeseries(p.x, p.y) print(result) value = result.popitem() assert math.cos(10) == value[1][0] assert math.cos(5) == value[1][1]
def load_disk_data(self, format: str, glob_pattern: str, options: dict, viewing_parameters: dict) -> object: if format != 'GTiff': raise NotImplementedError( "The format is not supported by the backend: " + format) date_regex = options['date_regex'] if glob_pattern.startswith("hdfs:"): kerberos() from_date = normalize_date(viewing_parameters.get("from", None)) to_date = normalize_date(viewing_parameters.get("to", None)) left = viewing_parameters.get("left", None) right = viewing_parameters.get("right", None) top = viewing_parameters.get("top", None) bottom = viewing_parameters.get("bottom", None) srs = viewing_parameters.get("srs", None) band_indices = viewing_parameters.get("bands") sc = gps.get_spark_context() gateway = JavaGateway( eager_load=True, gateway_parameters=sc._gateway.gateway_parameters) jvm = gateway.jvm extent = jvm.geotrellis.vector.Extent(float(left), float(bottom), float(right), float(top)) \ if left is not None and right is not None and top is not None and bottom is not None else None pyramid = jvm.org.openeo.geotrellis.geotiff.PyramidFactory.from_disk(glob_pattern, date_regex) \ .pyramid_seq(extent, srs, from_date, to_date) temporal_tiled_raster_layer = jvm.geopyspark.geotrellis.TemporalTiledRasterLayer option = jvm.scala.Option levels = { pyramid.apply(index)._1(): TiledRasterLayer( LayerType.SPACETIME, temporal_tiled_raster_layer( option.apply(pyramid.apply(index)._1()), pyramid.apply(index)._2())) for index in range(0, pyramid.size()) } image_collection = GeotrellisTimeSeriesImageCollection( pyramid=gps.Pyramid(levels), service_registry=self._service_registry, metadata={}) return image_collection.band_filter( band_indices) if band_indices else image_collection
def test_reduce_bands(): input = create_spacetime_layer() input = gps.Pyramid({0: input}) collection_metadata = GeopysparkCubeMetadata({ "cube:dimensions": { "my_bands": { "type": "bands", "values": ["B04", "B08"] }, } }) imagecollection = GeopysparkDataCube(pyramid=input, metadata=collection_metadata) visitor = GeotrellisTileProcessGraphVisitor() graph = { "sum": { "arguments": { "data": { "from_argument": "dimension_data" }, "ignore_nodata": True }, "process_id": "sum" }, "subtract": { "arguments": { "data": { "from_argument": "dimension_data" } }, "process_id": "subtract" }, "divide": { "arguments": { "data": [{ "from_node": "sum" }, { "from_node": "subtract" }] }, "process_id": "divide", "result": True } } visitor.accept_process_graph(graph) stitched = imagecollection.reduce_dimension( dimension='my_bands', reducer=visitor, env=EvalEnv()).pyramid.levels[0].to_spatial_layer().stitch() print(stitched) assert 3.0 == stitched.cells[0][0][0]
def test_merge_cubes(self): red_ramp, nir_ramp = np.mgrid[0:4, 0:4] layer1 = self._create_spacetime_layer(cells=np.array([[red_ramp]])) layer2 = self._create_spacetime_layer(cells=np.array([[nir_ramp]])) metadata = CollectionMetadata( {"properties": { "eo:bands": [{ "name": "the_band" }] }}) cube1 = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer1}), InMemoryServiceRegistry(), metadata=metadata) cube2 = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer2}), InMemoryServiceRegistry(), metadata=metadata) sum = cube1.merge(cube2, 'sum') stitched = sum.pyramid.levels[0].to_spatial_layer().stitch() np.testing.assert_array_equal(red_ramp + nir_ramp, stitched.cells[0, 0:4, 0:4])
def test_point_series(self): input = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) transformed_collection = imagecollection.apply("cos") for p in self.points[0:3]: result = transformed_collection.timeseries(p.x, p.y) print(result) value = result.popitem() self.assertEqual(math.cos(10), value[1][0]) self.assertEqual(math.cos(5), value[1][1])
def _test_merge_cubes_subtract_spatial(left_spatial=False, right_spatial=False): # TODO: this would be cleaner with @pytest.mark.parameterize but that's not supported on TestCase methods red_ramp, nir_ramp = np.mgrid[0:4, 0:4] layer1 = _create_spacetime_layer(cells=np.array([[red_ramp]])) if left_spatial: layer1 = layer1.to_spatial_layer() layer2 = _create_spacetime_layer(cells=np.array([[nir_ramp]])) if right_spatial: layer2 = layer2.to_spatial_layer() metadata = _build_metadata() cube1 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer1}), metadata=metadata) cube2 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer2}), metadata=metadata) res = cube1.merge_cubes(cube2, 'subtract') layer = res.pyramid.levels[0] if layer.layer_type != LayerType.SPATIAL: layer = layer.to_spatial_layer() actual = layer.stitch().cells[0, 0:4, 0:4] expected = red_ramp - nir_ramp np.testing.assert_array_equal(expected, actual)
def test_zonal_statistics_datacube(self): layer = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer}), InMemoryServiceRegistry()) polygon = Polygon(shell=[ (0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0), (0.0, 0.0) ]) polygon2 = Polygon(shell=[ (2.0, 2.0), (3.0, 2.0), (3.0, 3.0), (2.0, 3.0), (2.0, 2.0) ]) regions = GeometryCollection([polygon, MultiPolygon([polygon2])]) for use_file in [True,False]: with self.subTest(): if use_file: with NamedTemporaryFile(delete=False,suffix='.json',mode='r+') as fp: json.dump(mapping(regions),fp) regions_serialized = fp.name else: regions_serialized = regions result = imagecollection.zonal_statistics(regions_serialized, "mean") assert result.data == { '2017-09-25T11:37:00Z': [[1.0, 2.0], [1.0, 2.0]] } result._regions = regions covjson = result.to_covjson() assert covjson["ranges"] == { "band0": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 2), "values": [1.0, 1.0] }, "band1": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 2), "values": [2.0, 2.0] }, }
def _build_cube(): openeo_metadata = { "cube:dimensions": { "x": {"type": "spatial", "axis": "x"}, "y": {"type": "spatial", "axis": "y"}, "bands": {"type": "bands", "values": ["red", "nir"]}, "t": {"type": "temporal"} }, "bands": [ { "band_id": "red", "name": "red", "offset": 0, "res_m": 10, "scale": 0.0001, "type": "int16", "unit": "1", "wavelength_nm": 664.5 }, { "band_id": "nir", "name": "nir", "offset": 0, "res_m": 10, "scale": 0.0001, "type": "int16", "unit": "1", "wavelength_nm": 835.1 } ], "description": "Sentinel 2 Level-2: Bottom-of-atmosphere reflectances in cartographic geometry", "extent": { "bottom": 39, "crs": "EPSG:4326", "left": -34, "right": 35, "top": 71 }, "product_id": "CGS_SENTINEL2_RADIOMETRY_V101", "time": { "from": "2016-01-01", "to": "2019-10-01" } } # TODO: avoid instantiating TestTimeSeries? e.g. use pytest fixtures or simple builder functions. layer = TestTimeSeries().create_spacetime_layer() cube = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer}),metadata=openeo_metadata) return cube
def test_point_series(self): def custom_function(cells: np.ndarray, nd): return cells[0] + cells[1] input = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) transformed_collection = imagecollection.apply_pixel([0, 1], custom_function) for p in self.points[0:3]: result = transformed_collection.timeseries(p.x, p.y) print(result) value = result.popitem() self.assertEqual(3.0, value[1][0])