def test_apply_if(): input = create_spacetime_layer_singleband() input = gps.Pyramid({0: input}) imagecollection = GeopysparkDataCube(pyramid=input) graph = { "6": { "arguments": { "reject": { "from_parameter": "x" }, "value": { "from_node": "10" }, "accept": 2.0 }, "process_id": "if", "result": True }, "10": { "process_id": "gt", "arguments": { "x": { "from_parameter": "x" }, "y": 7.0 } } } stitched = imagecollection.apply( graph).pyramid.levels[0].to_spatial_layer().stitch() print(stitched) assert 2.0 == stitched.cells[0][0][0]
def test_zonal_statistics(self): layer = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer})) polygon = Polygon(shell=[ (0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0), (0.0, 0.0) ]) result = imagecollection.zonal_statistics(polygon, "mean") assert result.data == {'2017-09-25T11:37:00Z': [[1.0, 2.0]]} covjson = result.to_covjson() assert covjson["ranges"] == { "band0": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 1), "values": [1.0] }, "band1": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 1), "values": [2.0] }, }
def test_apply_cos(): input = create_spacetime_layer() cube = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) res = cube.apply("cos") data = res.pyramid.levels[0].to_spatial_layer().stitch().cells np.testing.assert_array_almost_equal(data[0, 2:6, 2:6], np.cos(first[0])) np.testing.assert_array_almost_equal(data[1, 2:6, 2:6], np.cos(second[0]))
def test_reduce_bands_logical_ops(): input = create_spacetime_layer_singleband() input = gps.Pyramid({0: input}) imagecollection = GeopysparkDataCube(pyramid=input) visitor = GeotrellisTileProcessGraphVisitor() graph = { "eq": { "arguments": { "x": { "from_argument": "data" }, "y": 10 }, "process_id": "eq", }, "not": { "arguments": { "expression": { "from_node": "eq" } }, "process_id": "not", "result": True } } visitor.accept_process_graph(graph) stitched = imagecollection.reduce_bands( visitor).pyramid.levels[0].to_spatial_layer().stitch() print(stitched) assert 0 == stitched.cells[0][0][0]
def test_min_time(self): input = Pyramid({0: self.tiled_raster_rdd}) cube = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) env = EvalEnv() min_time = cube.reduce_dimension(reducer=reducer('min'), dimension='t', env=env) max_time = cube.reduce_dimension(reducer=reducer('max'), dimension='t', env=env) stitched = min_time.pyramid.levels[0].stitch() print(stitched) self.assertEquals(2.0, stitched.cells[0][0][0]) for p in self.points[1:3]: result = min_time.timeseries(p.x, p.y, srs="EPSG:3857") print(result) print(cube.timeseries(p.x, p.y, srs="EPSG:3857")) max_result = max_time.timeseries(p.x, p.y, srs="EPSG:3857") self.assertEqual(1.0, result['NoDate']) self.assertEqual(2.0, max_result['NoDate'])
def test_reproject_spatial(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) ref_path = str(self.temp_folder / "reproj_ref.tiff") imagecollection.reduce('max', dimension="t").save_result(ref_path, format="GTIFF") resampled = imagecollection.resample_spatial(resolution=0, projection="EPSG:3395", method="max") metadata = resampled.pyramid.levels[0].layer_metadata print(metadata) self.assertTrue("proj=merc" in metadata.crs) path = str(self.temp_folder / "reprojected.tiff") res = resampled.reduce('max', dimension="t") res.save_result(path, format="GTIFF") with rasterio.open(ref_path) as ref_ds: with rasterio.open(path) as ds: print(ds.profile) #this reprojection does not change the shape, so we can compare assert ds.read().shape == ref_ds.read().shape assert (ds.crs.to_epsg() == 3395)
def test_apply_complex_graph(): graph = { "sin": { "arguments": { "x": { "from_argument": "data" } }, "process_id": "sin", "result": False }, "multiply": { "arguments": { "x": { "from_node": "sin" }, "y": 5.0 }, "process_id": "multiply", "result": True } } input = create_spacetime_layer() cube = GeopysparkDataCube(gps.Pyramid({0: input}), InMemoryServiceRegistry()) res = cube.apply(graph) data = res.pyramid.levels[0].to_spatial_layer().stitch().cells np.testing.assert_array_almost_equal(data[0, 2:6, 2:6], 5.0 * np.sin(first[0])) np.testing.assert_array_almost_equal(data[1, 2:6, 2:6], 5.0 * np.sin(second[0]))
def test_rename_dimension(self): imagecollection = GeopysparkDataCube(pyramid=Pyramid( {0: self.tiled_raster_rdd}), metadata=self.collection_metadata) dim_renamed = imagecollection.rename_dimension('t', 'myNewTimeDim') dim_renamed.metadata.assert_valid_dimension('myNewTimeDim')
def download_no_args(self, tmp_path, format, format_options={}): input = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands') imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','','')) res = imagecollection.save_result(str(tmp_path / "test_download_result.") + format, format=format, format_options=format_options) print(res) return res
def test_reduce_nontemporal(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) with self.assertRaises(FeatureUnsupportedException) as context: imagecollection.reduce( "max", dimension="gender").pyramid.levels[0].stitch() print(context.exception)
def test_aggregate_max_time(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) layer = imagecollection.reduce('max', dimension='t').pyramid.levels[0] stitched = layer.stitch() assert CellType.FLOAT32.value == layer.layer_metadata.cell_type print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0])
def _test_aggregate_temporal(self, interval_list): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) stitched = (imagecollection.aggregate_temporal( interval_list, ["2017-01-03"], "min", dimension="t").pyramid.levels[0].to_spatial_layer().stitch()) print(stitched) expected_max = np.min([self.tile2.cells, self.tile.cells], axis=0) assert_array_almost_equal(stitched.cells[0, 0:5, 0:5], expected_max)
def test_merge_cubes_exception_if_levels_do_not_match(): red_ramp, nir_ramp = np.mgrid[0:4, 0:4] layer1 = _create_spacetime_layer(cells=np.array([[red_ramp]])) layer2 = _create_spacetime_layer(cells=np.array([[nir_ramp]])) metadata = _build_metadata(bands=["the_band"]) cube1 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer1}), metadata=metadata) cube2 = GeopysparkDataCube(pyramid=gps.Pyramid({14: layer2}), metadata=metadata) with pytest.raises(OpenEOApiException) as excinfo: res = cube1.merge_cubes(cube2, 'sum')
def test_apply_kernel_int(self): kernel = np.array([[0, 1, 0], [1, 1, 1], [0, 1, 0]]) input = Pyramid({0: self.tiled_raster_rdd}) img = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) stitched = img.apply_kernel(kernel).reduce( 'max', dimension="t").pyramid.levels[0].stitch() assert stitched.cells[0][0][0] == 6.0 assert stitched.cells[0][0][1] == 8.0 assert stitched.cells[0][1][1] == 10.0
def test_another_polygon_series(self): input = self._create_spacetime_layer(no_data=-1.0) imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) polygon = Polygon(shell=[(2.0, 6.0), (6.0, 6.0), (6.0, 2.0), (2.0, 2.0), (2.0, 6.0)]) means = imagecollection.zonal_statistics(regions=polygon, func="mean") assert means.data == { '2017-09-25T11:37:00Z': [[(0 + 0 + 0 + 0 + 1 + 1 + 1 + 1 + 2 + 2 + 2 + 2) / 12]] }
def test_apply_dimension_spatiotemporal(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube( pyramid=input, metadata=GeopysparkCubeMetadata({ "cube:dimensions": { # TODO: also specify other dimensions? "bands": { "type": "bands", "values": ["2"] } }, "summaries": { "eo:bands": [{ "name": "2", "common_name": "blue", "wavelength_nm": 496.6, "res_m": 10, "scale": 0.0001, "offset": 0, "type": "int16", "unit": "1" }] } })) udf_code = """ def rct_savitzky_golay(udf_data:UdfData): from scipy.signal import savgol_filter print(udf_data.get_datacube_list()) return udf_data """ result = imagecollection.apply_tiles_spatiotemporal(udf_code) local_tiles = result.pyramid.levels[0].to_numpy_rdd().collect() print(local_tiles) self.assertEquals(len(TestMultipleDates.layer), len(local_tiles)) ref_dict = { e[0]: e[1] for e in imagecollection.pyramid.levels[0].convert_data_type( CellType.FLOAT64).to_numpy_rdd().collect() } result_dict = {e[0]: e[1] for e in local_tiles} for k, v in ref_dict.items(): tile = result_dict[k] assert_array_almost_equal(np.squeeze(v.cells), np.squeeze(tile.cells), decimal=2)
def test_point_series(): input = create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) transformed_collection = imagecollection.apply("cos") for p in points[0:3]: result = transformed_collection.timeseries(p.x, p.y) print(result) value = result.popitem() assert math.cos(10) == value[1][0] assert math.cos(5) == value[1][1]
def test_reduce_bands(): input = create_spacetime_layer() input = gps.Pyramid({0: input}) collection_metadata = GeopysparkCubeMetadata({ "cube:dimensions": { "my_bands": { "type": "bands", "values": ["B04", "B08"] }, } }) imagecollection = GeopysparkDataCube(pyramid=input, metadata=collection_metadata) visitor = GeotrellisTileProcessGraphVisitor() graph = { "sum": { "arguments": { "data": { "from_argument": "dimension_data" }, "ignore_nodata": True }, "process_id": "sum" }, "subtract": { "arguments": { "data": { "from_argument": "dimension_data" } }, "process_id": "subtract" }, "divide": { "arguments": { "data": [{ "from_node": "sum" }, { "from_node": "subtract" }] }, "process_id": "divide", "result": True } } visitor.accept_process_graph(graph) stitched = imagecollection.reduce_dimension( dimension='my_bands', reducer=visitor, env=EvalEnv()).pyramid.levels[0].to_spatial_layer().stitch() print(stitched) assert 3.0 == stitched.cells[0][0][0]
def test_aggregate_temporal_median(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) stitched = (imagecollection.aggregate_temporal( ["2015-01-01", "2018-01-01"], ["2017-01-03"], self._median_reducer(), dimension="t").pyramid.levels[0].to_spatial_layer().stitch()) print(stitched) expected_median = np.median( [self.tile.cells, self.tile2.cells, self.tile.cells], axis=0) #TODO nodata handling?? assert_array_almost_equal(stitched.cells[0, 1:2, 1:2], expected_median[1:2, 1:2])
def test_merge_cubes_into_single_band(): red_ramp, nir_ramp = np.mgrid[0:4, 0:4] layer1 = _create_spacetime_layer(cells=np.array([[red_ramp]])) layer2 = _create_spacetime_layer(cells=np.array([[nir_ramp]])) metadata = _build_metadata(bands=["the_band"]) cube1 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer1}), metadata=metadata) cube2 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer2}), metadata=metadata) res = cube1.merge_cubes(cube2, 'sum') stitched = res.pyramid.levels[0].to_spatial_layer().stitch() assert stitched.cells.shape[0] == 1 np.testing.assert_array_equal(red_ramp + nir_ramp, stitched.cells[0, 0:4, 0:4])
def test_zonal_statistics_datacube(self): layer = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer})) polygon = Polygon(shell=[ (0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0), (0.0, 0.0) ]) polygon2 = Polygon(shell=[ (2.0, 2.0), (3.0, 2.0), (3.0, 3.0), (2.0, 3.0), (2.0, 2.0) ]) regions = GeometryCollection([polygon, MultiPolygon([polygon2])]) for use_file in [True,False]: with self.subTest(): if use_file: with NamedTemporaryFile(delete=False,suffix='.json',mode='r+') as fp: json.dump(mapping(regions),fp) regions_serialized = fp.name else: regions_serialized = regions result = imagecollection.zonal_statistics(regions_serialized, "mean") assert result.data == { '2017-09-25T11:37:00Z': [[1.0, 2.0], [1.0, 2.0]] } result._regions = regions covjson = result.to_covjson() assert covjson["ranges"] == { "band0": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 2), "values": [1.0, 1.0] }, "band1": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 2), "values": [2.0, 2.0] }, }
def imagecollection_with_two_bands_and_three_dates_webmerc(request): from geopyspark.geotrellis import (SpaceTimeKey, Tile, _convert_to_unix_time) from geopyspark.geotrellis.constants import LayerType from geopyspark.geotrellis.layer import TiledRasterLayer import geopyspark as gps from openeogeotrellis.geopysparkdatacube import GeopysparkDataCube,GeopysparkCubeMetadata date1, date3, rdd = numpy_rdd_two_bands_and_three_dates() metadata = {'cellType': 'int32ud-1', 'extent': extent_webmerc, 'crs': '+proj=merc +a=6378137 +b=6378137 +lat_ts=0 +lon_0=0 +x_0=0 +y_0=0 +k=1 +units=m +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(date1)}, 'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(date3)} }, 'layoutDefinition': { 'extent': extent_webmerc, 'tileLayout': layout } } geopyspark_layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata) datacube = GeopysparkDataCube(pyramid=gps.Pyramid({0: geopyspark_layer}), metadata=GeopysparkCubeMetadata(openeo_metadata)) if request.instance: request.instance.imagecollection_with_two_bands_and_three_dates = datacube return datacube
def create_red_nir_layer(): red_ramp, nir_ramp = np.mgrid[0:4, 0:4] layer = _create_spacetime_layer(cells=np.array([[red_ramp], [nir_ramp]])) pyramid = gps.Pyramid({0: layer}) metadata = GeopysparkCubeMetadata({ "cube:dimensions": { "x": { "type": "spatial", "axis": "x" }, "y": { "type": "spatial", "axis": "y" }, "bands": { "type": "bands", "values": ["B04", "B08"] } }, "summaries": { "eo:bands": [ { "name": "B04", "common_name": "red" }, { "name": "B08", "common_name": "nir" }, ] } }) imagecollection = GeopysparkDataCube(pyramid=pyramid, metadata=metadata) return imagecollection
def test_resample_spatial(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) resampled = imagecollection.resample_spatial(resolution=0.05) path = str(self.temp_folder / "resampled.tiff") res = resampled.reduce('max', dimension="t") res.save_result(path, format="GTIFF") import rasterio with rasterio.open(path) as ds: print(ds.profile) self.assertAlmostEqual(0.05, ds.res[0], 3)
def test_mask_raster_replacement_int(self): def createMask(tile): tile.cells[0][0][0] = 0.0 return tile input = Pyramid({0: self.tiled_raster_rdd}) mask_layer = self.tiled_raster_rdd.map_tiles(createMask) mask = Pyramid({0: mask_layer}) cube = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) mask_cube = GeopysparkDataCube(pyramid=mask) stitched = cube.mask(mask=mask_cube, replacement=10).reduce( 'max', dimension="t").pyramid.levels[0].stitch() print(stitched) assert stitched.cells[0][0][0] == 2.0 assert stitched.cells[0][0][1] == 10.0
def test_multiband_with_tile_coordinates(self): metadata = GeopysparkCubeMetadata({ "cube:dimensions": { # TODO: also specify other dimensions? "bands": { "type": "bands", "values": ["2", "3", "4"] } }, "summaries": { "eo:bands": [{ 'name': '2', 'common_name': 'blue', 'wavelength_nm': 496.6, 'res_m': 10, 'scale': 0.0001, 'offset': 0, 'type': 'int16', 'unit': '1' }, { 'name': '3', 'common_name': 'green', 'wavelength_nm': 560, 'res_m': 10, 'scale': 0.0001, 'offset': 0, 'type': 'int16', 'unit': '1' }, { 'name': '4', 'common_name': 'red', 'wavelength_nm': 664.5, 'res_m': 10, 'scale': 0.0001, 'offset': 0, 'type': 'int16', 'unit': '1' }] } }) datacube = GeopysparkDataCube._numpy_to_xarraydatacube( TestMultiBandUDF.tile.cells, SpatialExtent(bottom=100, top=110, left=200, right=220, height=1, width=2), band_coordinates=metadata.band_dimension.band_names) the_array = datacube.get_array() assert the_array is not None np.testing.assert_almost_equal( the_array.x.values, np.array([185.0, 195.0, 205.0, 215.0, 225.0, 235.0]), 5) np.testing.assert_almost_equal(the_array.y.values, np.array([120.0, 110.0, 100.0, 90.0]), 5) print(the_array)
def test_write_assets(self, tmp_path): input = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) imagecollection.metadata = imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands') imagecollection.metadata = imagecollection.metadata.append_band(Band('band_two', '', '')) format = 'GTiff' res = imagecollection.write_assets(str(tmp_path / "test_download_result.") + format, format=format,format_options={ "multidate":True, "batch_mode":True }) assert 1 == len(res) name, asset = res.popitem() assert Path(asset['href']).parent == tmp_path assert asset['nodata'] == -1 assert asset['roles'] == ['data'] assert 2 == len(asset['bands']) assert 'image/tiff; application=geotiff' == asset['type'] assert asset['datetime'] == "2017-09-25T11:37:00Z"
def test_reprojection(): """ It is important that reprojection in Python and Geotrellis give the same result, for alignment of bounding boxes! @return: """ reprojected = GeopysparkDataCube._reproject_extent("EPSG:4326","EPSG:32631",5.071, 51.21,5.1028,51.23) print(reprojected) assert reprojected.xmin == 644594.8230399278 assert reprojected.ymin == 5675216.271413178 assert reprojected.xmax == 646878.5028127492 assert reprojected.ymax == 5677503.191395153
def test_write_assets_samples_netcdf(self, tmp_path): input = self.create_spacetime_layer() imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input})) imagecollection.metadata = imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands') imagecollection.metadata = imagecollection.metadata.append_band(Band('band_two', '', '')) format = 'netCDF' res = imagecollection.write_assets(str(tmp_path / "test_download_result.") + format, format=format,format_options={ "batch_mode":True, "geometries":geojson_to_geometry(self.features), "sample_by_feature": True, "feature_id_property": 'id' }) assert len(res) == 3 name,asset = res.popitem() file = asset['href'] assert asset['nodata'] == -1 assert asset['roles'] == ['data'] assert 2 == len(asset['bands']) assert 'application/x-netcdf' == asset['type']
def test_viewing(self): geotrellis_layer = self.create_spacetime_layer() imagecollection = GeopysparkDataCube( pyramid=gps.Pyramid({0: geotrellis_layer})) metadata = imagecollection.tiled_viewing_service( user_id='u9876', service_id='s1234', service_type="TMS", api_version='0.4.0', process_graph={}).service_metadata print(metadata) assert metadata.type == "TMS" assert isinstance(metadata.attributes["bounds"], dict) tileresponse = requests.get(metadata.url.format(x=0, y=0, z=0), timeout=2) assert tileresponse.status_code == 200 assert tileresponse.content.startswith(PNG_SIGNATURE) tileresponse = requests.get(metadata.url.format(x=1, y=1, z=0), timeout=2) assert tileresponse.status_code == 200 assert tileresponse.content.startswith(PNG_SIGNATURE)