def test_apply_spatiotemporal(self): import openeo_udf.functions input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry(), { "bands": [{ "band_id": "2", "name": "blue", "wavelength_nm": 496.6, "res_m": 10, "scale": 0.0001, "offset": 0, "type": "int16", "unit": "1" }] }) import os, openeo_udf dir = os.path.dirname(openeo_udf.functions.__file__) file_name = os.path.join(dir, "datacube_reduce_time_sum.py") with open(file_name, "r") as f: udf_code = f.read() result = imagecollection.apply_tiles_spatiotemporal(udf_code) stitched = result.pyramid.levels[0].to_spatial_layer().stitch() print(stitched) self.assertEqual(2, stitched.cells[0][0][0]) self.assertEqual(6, stitched.cells[0][0][5]) self.assertEqual(4, stitched.cells[0][5][6])
def test_zonal_statistics_median_datacube(self): layer = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer}), InMemoryServiceRegistry()) polygon = Polygon(shell=[ (0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0), (0.0, 0.0) ]) result = imagecollection.zonal_statistics(polygon, "median") assert result.data == {'2017-09-25T11:37:00Z': [[1.0, 2.0]]} covjson = result.to_covjson() assert covjson["ranges"] == { "band0": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 1), "values": [1.0] }, "band1": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 1), "values": [2.0] } }
def test_linear_scale_range(self): red_ramp, nir_ramp = np.mgrid[0:4, 0:4] layer = self._create_spacetime_layer( cells=np.array([[red_ramp], [nir_ramp]])) pyramid = gps.Pyramid({0: layer}) metadata = CollectionMetadata({ "properties": { "eo:bands": [ { "name": "B04", "common_name": "red" }, { "name": "B08", "common_name": "nir" }, ] } }) imagecollection = GeotrellisTimeSeriesImageCollection( pyramid, InMemoryServiceRegistry(), metadata=metadata) stitched = imagecollection.ndvi().linear_scale_range( -1, 1, 0, 100).pyramid.levels[0].to_spatial_layer().stitch() cells = stitched.cells[0, 0:4, 0:4] expected = 50.0 * (1.0 + np.array( [[np.nan, 1 / 1, 2 / 2, 3 / 3], [-1 / 1, 0 / 2, 1 / 3, 2 / 4], [-2 / 2, -1 / 3, 0 / 4, 1 / 5], [-3 / 3, -2 / 4, -1 / 5, 0 / 6]])) expected[0][0] = 255.0 np.testing.assert_array_almost_equal(cells, expected.astype(np.uint8))
def test_reduce_bands_comparison_ops(self): input = self.create_spacetime_layer_singleband() input = gps.Pyramid({0: input}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) visitor = GeotrellisTileProcessGraphVisitor() graph = { "gt": { "arguments": { "x": { "from_argument": "data" }, "y": 6.0 }, "process_id": "gt", "result": True } } visitor.accept_process_graph(graph) stitched = imagecollection.reduce_bands( visitor).pyramid.levels[0].to_spatial_layer().stitch() print(stitched) self.assertEqual(1, stitched.cells[0][0][0])
def test_download_as_catalog(self): input = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) imagecollection.download("catalogresult.tiff", format="GTIFF", parameters={"catalog": True})
def test_reduce_nontemporal(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) with self.assertRaises(AttributeError) as context: imagecollection.reduce("max", "spectral").pyramid.levels[0].stitch() print(context.exception)
def test_download_geotiff_no_args(self): input = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) geotiffs = imagecollection.download( str(self.temp_folder / "test_download_result.geotiff")) print(geotiffs)
def test_polygon_series(self): input = self.create_spacetime_layer() polygon = Polygon([(0, 0), (0, 2), (2, 2), (2, 0), (0, 0)]) imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry(), self.openeo_metadata) means = imagecollection.polygonal_mean_timeseries(polygon) assert means == {'2017-09-25T11:37:00': [[1.0, 2.0]]}
def test_aggregate_temporal(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.aggregate_temporal( ["2017-01-01", "2018-01-01"], ["2017-01-03"], "max").pyramid.levels[0].to_spatial_layer().stitch() print(stitched)
def test_mask_raster(self): input = Pyramid({0: self.tiled_raster_rdd}) def createMask(tile): tile.cells[0][0][0] = 0.0 return tile mask_layer = self.tiled_raster_rdd.map_tiles(createMask) mask = Pyramid({0: mask_layer}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.mask( rastermask=GeotrellisTimeSeriesImageCollection( mask, InMemoryServiceRegistry()), replacement=10.0).reduce('max', 'temporal').pyramid.levels[0].stitch() print(stitched) self.assertEquals(2.0, stitched.cells[0][0][0]) self.assertEquals(10.0, stitched.cells[0][0][1])
def test_download_masked_geotiff(self): input = self.create_spacetime_layer() polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]]) imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) imagecollection = imagecollection.mask(polygon) geotiffs = imagecollection.download( str(self.temp_folder / "test_download_masked_result.geotiff")) print(geotiffs)
def test_aggregate_max_time(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.reduce( 'max', 'temporal').pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0])
def test_apply_kernel(self): kernel = np.array([[0.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 0.0]]) input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.apply_kernel(kernel, 2.0).reduce( 'max', 'temporal').pyramid.levels[0].stitch() self.assertEquals(12.0, stitched.cells[0][0][0]) self.assertEquals(16.0, stitched.cells[0][0][1]) self.assertEquals(20.0, stitched.cells[0][1][1])
def test_another_polygon_series(self): input = self._create_spacetime_layer(no_data=-1.0) imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) polygon = Polygon(shell=[(2.0, 6.0), (6.0, 6.0), (6.0, 2.0), (2.0, 2.0), (2.0, 6.0)]) means = imagecollection.polygonal_mean_timeseries(polygon) assert means == { '2017-09-25T11:37:00': [[(0 + 0 + 0 + 0 + 1 + 1 + 1 + 1 + 2 + 2 + 2 + 2) / 12]] }
def load_disk_data(self, format: str, glob_pattern: str, options: dict, viewing_parameters: dict) -> object: if format != 'GTiff': raise NotImplementedError( "The format is not supported by the backend: " + format) date_regex = options['date_regex'] if glob_pattern.startswith("hdfs:"): kerberos() from_date = normalize_date(viewing_parameters.get("from", None)) to_date = normalize_date(viewing_parameters.get("to", None)) left = viewing_parameters.get("left", None) right = viewing_parameters.get("right", None) top = viewing_parameters.get("top", None) bottom = viewing_parameters.get("bottom", None) srs = viewing_parameters.get("srs", None) band_indices = viewing_parameters.get("bands") sc = gps.get_spark_context() gateway = JavaGateway( eager_load=True, gateway_parameters=sc._gateway.gateway_parameters) jvm = gateway.jvm extent = jvm.geotrellis.vector.Extent(float(left), float(bottom), float(right), float(top)) \ if left is not None and right is not None and top is not None and bottom is not None else None pyramid = jvm.org.openeo.geotrellis.geotiff.PyramidFactory.from_disk(glob_pattern, date_regex) \ .pyramid_seq(extent, srs, from_date, to_date) temporal_tiled_raster_layer = jvm.geopyspark.geotrellis.TemporalTiledRasterLayer option = jvm.scala.Option levels = { pyramid.apply(index)._1(): TiledRasterLayer( LayerType.SPACETIME, temporal_tiled_raster_layer( option.apply(pyramid.apply(index)._1()), pyramid.apply(index)._2())) for index in range(0, pyramid.size()) } image_collection = GeotrellisTimeSeriesImageCollection( pyramid=gps.Pyramid(levels), service_registry=self._service_registry, metadata={}) return image_collection.band_filter( band_indices) if band_indices else image_collection
def test_zonal_statistics_datacube(self): layer = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer}), InMemoryServiceRegistry()) polygon = Polygon(shell=[ (0.0, 0.0), (1.0, 0.0), (1.0, 1.0), (0.0, 1.0), (0.0, 0.0) ]) polygon2 = Polygon(shell=[ (2.0, 2.0), (3.0, 2.0), (3.0, 3.0), (2.0, 3.0), (2.0, 2.0) ]) regions = GeometryCollection([polygon, MultiPolygon([polygon2])]) for use_file in [True,False]: with self.subTest(): if use_file: with NamedTemporaryFile(delete=False,suffix='.json',mode='r+') as fp: json.dump(mapping(regions),fp) regions_serialized = fp.name else: regions_serialized = regions result = imagecollection.zonal_statistics(regions_serialized, "mean") assert result.data == { '2017-09-25T11:37:00Z': [[1.0, 2.0], [1.0, 2.0]] } result._regions = regions covjson = result.to_covjson() assert covjson["ranges"] == { "band0": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 2), "values": [1.0, 1.0] }, "band1": { "type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"], "shape": (1, 2), "values": [2.0, 2.0] }, }
def test_merge_cubes(self): red_ramp, nir_ramp = np.mgrid[0:4, 0:4] layer1 = self._create_spacetime_layer(cells=np.array([[red_ramp]])) layer2 = self._create_spacetime_layer(cells=np.array([[nir_ramp]])) metadata = CollectionMetadata( {"properties": { "eo:bands": [{ "name": "the_band" }] }}) cube1 = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer1}), InMemoryServiceRegistry(), metadata=metadata) cube2 = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer2}), InMemoryServiceRegistry(), metadata=metadata) sum = cube1.merge(cube2, 'sum') stitched = sum.pyramid.levels[0].to_spatial_layer().stitch() np.testing.assert_array_equal(red_ramp + nir_ramp, stitched.cells[0, 0:4, 0:4])
def test_point_series(self): input = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) transformed_collection = imagecollection.apply("cos") for p in self.points[0:3]: result = transformed_collection.timeseries(p.x, p.y) print(result) value = result.popitem() self.assertEqual(math.cos(10), value[1][0]) self.assertEqual(math.cos(5), value[1][1])
def test_point_series(self): def custom_function(cells: np.ndarray, nd): return cells[0] + cells[1] input = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) transformed_collection = imagecollection.apply_pixel([0, 1], custom_function) for p in self.points[0:3]: result = transformed_collection.timeseries(p.x, p.y) print(result) value = result.popitem() self.assertEqual(3.0, value[1][0])
def test_viewing(self): geotrellis_layer = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection( pyramid=gps.Pyramid({0: geotrellis_layer}), service_registry=InMemoryServiceRegistry()) metadata = imagecollection.tiled_viewing_service(service_type="TMS", process_graph={}) print(metadata) assert metadata.type == "TMS" assert isinstance(metadata.attributes["bounds"], dict) tileresponse = requests.get(metadata.url.format(x=0, y=0, z=0), timeout=2) assert tileresponse.status_code == 200 assert tileresponse.content.startswith(PNG_SIGNATURE) tileresponse = requests.get(metadata.url.format(x=1, y=1, z=0), timeout=2) assert tileresponse.status_code == 200 assert tileresponse.content.startswith(PNG_SIGNATURE)
def test_resample_spatial(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) resampled = imagecollection.resample_spatial(resolution=0.05) path = str(self.temp_folder / "resampled.tiff") resampled.reduce('max', 'temporal').download(path, format="GTIFF", parameters={'tiled': True}) import rasterio with rasterio.open(path) as ds: print(ds.profile) self.assertAlmostEqual(0.05, ds.res[0], 3)
def test_point_series_apply_tile(self): import os, openeo_udf dir = os.path.dirname(openeo_udf.functions.__file__) file_name = os.path.join(dir, "datacube_ndvi.py") with open(file_name, "r") as f: udf_code = f.read() input = self.create_spacetime_layer() imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry(), self.openeo_metadata) transformed_collection = imagecollection.apply_tiles(udf_code) for p in self.points[0:3]: result = transformed_collection.timeseries(p.x, p.y) print(result) value = result.popitem() print(value)
def test_apply_dimension_spatiotemporal(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry(), { "bands": [{ "band_id": "2", "name": "blue", "wavelength_nm": 496.6, "res_m": 10, "scale": 0.0001, "offset": 0, "type": "int16", "unit": "1" }] }) udf_code = """ def rct_savitzky_golay(udf_data:UdfData): from scipy.signal import savgol_filter print(udf_data.get_datacube_list()) return udf_data """ result = imagecollection.apply_tiles_spatiotemporal(udf_code) local_tiles = result.pyramid.levels[0].to_numpy_rdd().collect() print(local_tiles) self.assertEquals(len(TestMultipleDates.layer), len(local_tiles)) ref_dict = { e[0]: e[1] for e in imagecollection.pyramid.levels[0].convert_data_type( CellType.FLOAT64).to_numpy_rdd().collect() } result_dict = {e[0]: e[1] for e in local_tiles} for k, v in ref_dict.items(): tile = result_dict[k] assert_array_almost_equal(np.squeeze(v.cells), np.squeeze(tile.cells), decimal=2)
def test_reduce_bands(self): input = self.create_spacetime_layer() input = gps.Pyramid({0: input}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) visitor = GeotrellisTileProcessGraphVisitor() graph = { "sum": { "arguments": { "data": { "from_argument": "dimension_data" } }, "process_id": "sum" }, "subtract": { "arguments": { "data": { "from_argument": "dimension_data" } }, "process_id": "subtract" }, "divide": { "arguments": { "data": [{ "from_node": "sum" }, { "from_node": "subtract" }] }, "process_id": "divide", "result": True } } visitor.accept_process_graph(graph) stitched = imagecollection.reduce_bands( visitor).pyramid.levels[0].to_spatial_layer().stitch() print(stitched) self.assertEqual(3.0, stitched.cells[0][0][0])
def test_min_time(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) min_time = imagecollection.reduce('min', 'temporal') max_time = imagecollection.reduce('max', 'temporal') stitched = min_time.pyramid.levels[0].stitch() print(stitched) self.assertEquals(2.0, stitched.cells[0][0][0]) for p in self.points[1:3]: result = min_time.timeseries(p.x, p.y, srs="EPSG:3857") print(result) print(imagecollection.timeseries(p.x, p.y, srs="EPSG:3857")) max_result = max_time.timeseries(p.x, p.y, srs="EPSG:3857") self.assertEqual(1.0, result['NoDate']) self.assertEqual(2.0, max_result['NoDate'])
def test_reduce(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.reduce( "max", "temporal").pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) self.assertEqual(2.0, stitched.cells[0][0][1]) stitched = imagecollection.reduce( "min", "temporal").pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) self.assertEqual(1.0, stitched.cells[0][0][1]) stitched = imagecollection.reduce( "sum", "temporal").pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) self.assertEqual(4.0, stitched.cells[0][0][1]) stitched = imagecollection.reduce( "mean", "temporal").pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) self.assertAlmostEqual(1.3333333, stitched.cells[0][0][1]) stitched = imagecollection.reduce( "variance", "temporal").pyramid.levels[0].stitch() print(stitched) self.assertEqual(0.0, stitched.cells[0][0][0]) self.assertAlmostEqual(0.2222222, stitched.cells[0][0][1]) stitched = imagecollection.reduce( "sd", "temporal").pyramid.levels[0].stitch() print(stitched) self.assertEqual(0.0, stitched.cells[0][0][0]) self.assertAlmostEqual(0.4714045, stitched.cells[0][0][1])
def test_reduce_some_nodata(self): no_data = -1.0 input = Pyramid({ 0: self._single_pixel_layer( { datetime.datetime.strptime("2016-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'): no_data, datetime.datetime.strptime("2017-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'): 5.0 }, no_data) }) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.reduce( "min", "temporal").pyramid.levels[0].stitch() #print(stitched) self.assertEqual(5.0, stitched.cells[0][0][0]) stitched = imagecollection.reduce( "max", "temporal").pyramid.levels[0].stitch() self.assertEqual(5.0, stitched.cells[0][0][0]) stitched = imagecollection.reduce( "sum", "temporal").pyramid.levels[0].stitch() self.assertEqual(5.0, stitched.cells[0][0][0]) stitched = imagecollection.reduce( "mean", "temporal").pyramid.levels[0].stitch() self.assertAlmostEqual(5.0, stitched.cells[0][0][0], delta=0.001) stitched = imagecollection.reduce( "variance", "temporal").pyramid.levels[0].stitch() self.assertAlmostEqual(0.0, stitched.cells[0][0][0], delta=0.001) stitched = imagecollection.reduce( "sd", "temporal").pyramid.levels[0].stitch() self.assertAlmostEqual(0.0, stitched.cells[0][0][0], delta=0.001)
def test_reproject_spatial(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) resampled = imagecollection.resample_spatial(resolution=0, projection="EPSG:3857", method="max") metadata = resampled.pyramid.levels[0].layer_metadata print(metadata) self.assertTrue("proj=merc" in metadata.crs) path = str(self.temp_folder / "reprojected.tiff") resampled.reduce('max', 'temporal').download(path, format="GTIFF", parameters={'tiled': True}) import rasterio with rasterio.open(path) as ds: print(ds.profile)
def test_convert_multiband_tile_hypercube(self): from openeo_udf.api.datacube \ import DataCube metadata = CollectionMetadata({ 'bands': [{ 'band_id': '2', 'name': 'blue', 'wavelength_nm': 496.6, 'res_m': 10, 'scale': 0.0001, 'offset': 0, 'type': 'int16', 'unit': '1' }, { 'band_id': '3', 'name': 'green', 'wavelength_nm': 560, 'res_m': 10, 'scale': 0.0001, 'offset': 0, 'type': 'int16', 'unit': '1' }, { 'band_id': '4', 'name': 'red', 'wavelength_nm': 664.5, 'res_m': 10, 'scale': 0.0001, 'offset': 0, 'type': 'int16', 'unit': '1' }] }) imagecollection = GeotrellisTimeSeriesImageCollection( "test", InMemoryServiceRegistry(), metadata=metadata) datacube = GeotrellisTimeSeriesImageCollection._tile_to_datacube( TestMultiBandUDF.tile.cells, None, bands_metadata=metadata.bands) the_array = datacube.get_array() assert the_array is not None print(the_array)
def test_download_masked_geotiff_reproject(self): input = self.create_spacetime_layer() polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]]) import pyproj from shapely.ops import transform from functools import partial project = partial( pyproj.transform, pyproj.Proj(init="EPSG:4326"), # source coordinate system pyproj.Proj(init="EPSG:3857")) # destination coordinate system reprojected = transform(project, polygon) imagecollection = GeotrellisTimeSeriesImageCollection( gps.Pyramid({0: input}), InMemoryServiceRegistry()) imagecollection = imagecollection.mask(reprojected, "EPSG:3857") geotiffs = imagecollection.download( str(self.temp_folder / "test_download_masked_result.3857")) print(geotiffs)