def numpy_rdd_two_bands_and_three_dates(): from geopyspark.geotrellis import (SpaceTimeKey, Tile, _convert_to_unix_time) from pyspark import SparkContext two_band_one_two = np.array([matrix_of_one, matrix_of_two], dtype='int') first_tile = Tile.from_numpy_array(two_band_one_two, -1) second_tile = Tile.from_numpy_array(np.array([matrix_of_two, matrix_of_one], dtype='int'), -1) nodata_tile = Tile.from_numpy_array(np.array([matrix_of_nodata, matrix_of_nodata], dtype='int'), -1) date1 = datetime.datetime.strptime("2017-09-25T11:37:00Z", '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC) date2 = datetime.datetime.strptime("2017-09-30T00:37:00Z", '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC) date3 = datetime.datetime.strptime("2017-10-25T11:37:00Z", '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC) layer = [(SpaceTimeKey(0, 0, date1), first_tile), (SpaceTimeKey(1, 0, date1), first_tile), (SpaceTimeKey(0, 1, date1), first_tile), (SpaceTimeKey(1, 1, date1), first_tile), (SpaceTimeKey(0, 0, date2), nodata_tile), (SpaceTimeKey(1, 0, date2), nodata_tile), (SpaceTimeKey(0, 1, date2), nodata_tile), (SpaceTimeKey(1, 1, date2), nodata_tile), (SpaceTimeKey(0, 0, date3), second_tile), (SpaceTimeKey(1, 0, date3), second_tile), (SpaceTimeKey(0, 1, date3), second_tile), (SpaceTimeKey(1, 1, date3), second_tile) ] rdd = SparkContext.getOrCreate().parallelize(layer) return date1, date3, rdd
class ByteTileSchemaTest(BaseTestClass): tiles = [ Tile.from_numpy_array(np.int8([0, 0, 1, 1]).reshape(2, 2), -128), Tile.from_numpy_array(np.int8([1, 2, 3, 4]).reshape(2, 2), -128), Tile.from_numpy_array(np.int8([5, 6, 7, 8]).reshape(2, 2), -128) ] sc = BaseTestClass.pysc._jsc.sc() tw = BaseTestClass.pysc._jvm.geopyspark.geotrellis.tests.schemas.ByteArrayTileWrapper java_rdd = tw.testOut(sc) ser = ProtoBufSerializer(tile_decoder, tile_encoder) rdd = RDD(java_rdd, BaseTestClass.pysc, AutoBatchedSerializer(ser)) collected = rdd.collect() def test_encoded_tiles(self): expected_encoded = [to_pb_tile(x) for x in self.collected] for actual, expected in zip(self.tiles, expected_encoded): cells = actual.cells rows, cols = cells.shape self.assertEqual(expected.cols, cols) self.assertEqual(expected.rows, rows) self.assertEqual(expected.cellType.nd, actual.no_data_value) self.assertEqual(expected.cellType.dataType, mapped_data_types[actual.cell_type]) def test_decoded_tiles(self): for actual, expected in zip(self.collected, self.tiles): self.assertTrue((actual.cells == expected.cells).all()) self.assertTrue(actual.cells.dtype == expected.cells.dtype) self.assertEqual(actual.cells.shape, actual.cells.shape)
def create_spacetime_layer(self): cells = np.array([self.first, self.second], dtype='int') tile = Tile.from_numpy_array(cells, -1) layer = [(SpaceTimeKey(0, 0, self.now), tile), (SpaceTimeKey(1, 0, self.now), tile), (SpaceTimeKey(0, 1, self.now), tile), (SpaceTimeKey(1, 1, self.now), tile)] rdd = SparkContext.getOrCreate().parallelize(layer) metadata = {'cellType': 'int32ud-1', 'extent': self.extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(self.now)}, 'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(self.now)} }, 'layoutDefinition': { 'extent': self.extent, 'tileLayout': self.layout } } return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
def get_tile(self, bands: List[str], col: int, row: int, date: datetime) -> Tile: array = np.array([ self.get_band_tile(band=band, col=col, row=row, date=date) for band in bands ]) return Tile.from_numpy_array(array)
def create_spatial_layer(self): cells = np.array([self.first, self.second], dtype='int') tile = Tile.from_numpy_array(cells, -1) layer = [(SpatialKey(0, 0), tile), (SpatialKey(1, 0), tile), (SpatialKey(0, 1), tile), (SpatialKey(1, 1), tile)] rdd = BaseTestClass.pysc.parallelize(layer) metadata = { 'cellType': 'int32ud-1', 'extent': self.extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0 }, 'maxKey': { 'col': 1, 'row': 1 } }, 'layoutDefinition': { 'extent': self.extent, 'tileLayout': self.layout } } return TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, metadata)
def _create_spacetime_layer(cells: np.ndarray = None) -> TiledRasterLayer: # TODO all these "create_spacetime_layer" functions are duplicated across all tests # and better should be moved to some kind of general factory or test fixture assert len(cells.shape) == 4 tile = Tile.from_numpy_array(cells, -1) layer = [(SpaceTimeKey(0, 0, now), tile), (SpaceTimeKey(1, 0, now), tile), (SpaceTimeKey(0, 1, now), tile), (SpaceTimeKey(1, 1, now), tile)] rdd = SparkContext.getOrCreate().parallelize(layer) metadata = { 'cellType': 'int32ud-1', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(now) }, 'maxKey': { 'col': 1, 'row': 1, 'instant': _convert_to_unix_time(now) } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': layout } } return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
def test_apply_to_tile(self): def custom_function(cells: np.ndarray, nd): return cells[0] + cells[1] cells = np.array([self.first, self.second], dtype='int') tile = Tile.from_numpy_array(cells, -1) custom_function(tile.cells, 0)
def test_no_data_of_zero(self): no_data_layer = [(t[0], Tile.from_numpy_array(t[1].cells, 1)) for t in self.layers] rdd = BaseTestClass.pysc.parallelize(no_data_layer) nd_layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd) nd_metadata = nd_layer.collect_metadata() self.assertTrue('ud1' in nd_metadata.cell_type) self.assertEqual(nd_metadata.no_data_value, 1)
def test_apply_openeo_udf_to_tile(self): import os, openeo_udf dir = os.path.dirname(openeo_udf.functions.__file__) file_name = os.path.join(dir, "datacube_ndvi.py") with open(file_name, "r") as f: udf_code = f.read() cells = np.array([self.first, self.second], dtype='int') tile = Tile.from_numpy_array(cells, -1)
def imagecollection_with_two_bands_and_one_date(request): import geopyspark as gps from geopyspark.geotrellis import (SpaceTimeKey, Tile, _convert_to_unix_time) from geopyspark.geotrellis.constants import LayerType from geopyspark.geotrellis.layer import TiledRasterLayer from pyspark import SparkContext from openeogeotrellis.geopysparkdatacube import GeopysparkDataCube print(request) two_band_one_two = np.array([matrix_of_one, matrix_of_two], dtype='int') tile = Tile.from_numpy_array(two_band_one_two, -1) date1 = datetime.datetime.strptime("2017-09-25T11:37:00Z", '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC) layer = [(SpaceTimeKey(0, 0, date1), tile), (SpaceTimeKey(1, 0, date1), tile), (SpaceTimeKey(0, 1, date1), tile), (SpaceTimeKey(1, 1, date1), tile)] rdd = SparkContext.getOrCreate().parallelize(layer) metadata = {'cellType': 'int32ud-1', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(date1)}, 'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(date1)} }, 'layoutDefinition': { 'extent': extent, 'tileLayout': layout } } geopyspark_layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata) datacube = GeopysparkDataCube(pyramid=gps.Pyramid({0: geopyspark_layer}), metadata=openeo_metadata) if request.instance: request.instance.imagecollection_with_two_bands_and_one_date = datacube return datacube
def create_spacetime_unsigned_byte_layer(self): """ Returns a single-band uint8ud255 layer consisting of four tiles that each look like this: ND 220 220 220 220 220 220 220 220 220 220 220 220 220 220 220 The extent is (0.0, 0.0) to (4.0, 4.0). """ no_data = 255 single_band = np.zeros((1, 4, 4)) single_band.fill(220) single_band[0, 0, 0] = no_data cells = np.array([single_band], dtype='uint8') tile = Tile.from_numpy_array(cells, no_data) layer = [(SpaceTimeKey(0, 0, self.now), tile), (SpaceTimeKey(1, 0, self.now), tile), (SpaceTimeKey(0, 1, self.now), tile), (SpaceTimeKey(1, 1, self.now), tile)] rdd = SparkContext.getOrCreate().parallelize(layer) metadata = { 'cellType': 'uint8ud255', 'extent': self.extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(self.now)}, 'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(self.now)} }, 'layoutDefinition': { 'extent': self.extent, 'tileLayout': self.layout } } return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
class SlopeTest(BaseTestClass): cells = np.array([[ [0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 1.0, 0.0], [0.0, 0.0, 1.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0]]]) tile = Tile.from_numpy_array(cells, -1.0) layer = [(SpatialKey(0, 0), tile), (SpatialKey(1, 0), tile), (SpatialKey(0, 1), tile), (SpatialKey(1, 1), tile)] rdd = BaseTestClass.pysc.parallelize(layer) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0} layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5} metadata = {'cellType': 'float32ud-1.0', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0}, 'maxKey': {'col': 1, 'row': 1}}, 'layoutDefinition': { 'extent': extent, 'tileLayout': {'tileCols': 5, 'tileRows': 5, 'layoutCols': 2, 'layoutRows': 2}}} raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, metadata) @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_slope(self): calc = zfactor_lat_lng_calculator(Unit.METERS) result = self.raster_rdd.slope(calc).to_numpy_rdd().values().first().cells self.assertEqual(result[0, 0, 0], 0.0)
class TestMultipleDates(TestCase): band1 = np.array([[-1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0]]) band2 = np.array([[2.0, 2.0, 2.0, 2.0, 2.0], [2.0, 2.0, 2.0, 2.0, 2.0], [2.0, 2.0, -1.0, 2.0, 2.0], [2.0, 2.0, 2.0, 2.0, 2.0], [2.0, 2.0, 2.0, 2.0, 2.0]]) tile = Tile.from_numpy_array(band1, no_data_value=-1.0) tile2 = Tile.from_numpy_array(band2, no_data_value=-1.0) time_1 = datetime.datetime.strptime("2016-08-24T09:00:00Z", '%Y-%m-%dT%H:%M:%SZ') time_2 = datetime.datetime.strptime("2017-08-24T09:00:00Z", '%Y-%m-%dT%H:%M:%SZ') time_3 = datetime.datetime.strptime("2017-10-17T09:00:00Z", '%Y-%m-%dT%H:%M:%SZ') layer = [(SpaceTimeKey(0, 0, time_1), tile), (SpaceTimeKey(1, 0, time_1), tile2), (SpaceTimeKey(0, 1, time_1), tile), (SpaceTimeKey(1, 1, time_1), tile), (SpaceTimeKey(0, 0, time_2), tile2), (SpaceTimeKey(1, 0, time_2), tile2), (SpaceTimeKey(0, 1, time_2), tile2), (SpaceTimeKey(1, 1, time_2), tile2), (SpaceTimeKey(0, 0, time_3), tile), (SpaceTimeKey(1, 0, time_3), tile2), (SpaceTimeKey(0, 1, time_3), tile), (SpaceTimeKey(1, 1, time_3), tile)] rdd = SparkContext.getOrCreate().parallelize(layer) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0} layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5} metadata = { 'cellType': 'float32ud-1.0', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(time_1) }, 'maxKey': { 'col': 1, 'row': 1, 'instant': _convert_to_unix_time(time_3) } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': { 'tileCols': 5, 'tileRows': 5, 'layoutCols': 2, 'layoutRows': 2 } } } tiled_raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata) layer2 = [(TemporalProjectedExtent(Extent(0, 0, 1, 1), epsg=3857, instant=time_1), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_1), tile), (TemporalProjectedExtent(Extent(0, 1, 1, 2), epsg=3857, instant=time_1), tile), (TemporalProjectedExtent(Extent(1, 1, 2, 2), epsg=3857, instant=time_1), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_2), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_2), tile), (TemporalProjectedExtent(Extent(0, 1, 1, 2), epsg=3857, instant=time_2), tile), (TemporalProjectedExtent(Extent(1, 1, 2, 2), epsg=3857, instant=time_2), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_3), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_3), tile), (TemporalProjectedExtent(Extent(0, 1, 1, 2), epsg=3857, instant=time_3), tile), (TemporalProjectedExtent(Extent(1, 1, 2, 2), epsg=3857, instant=time_3), tile)] rdd2 = SparkContext.getOrCreate().parallelize(layer2) raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd2) points = [ Point(1.0, -3.0), Point(0.5, 0.5), Point(20.0, 3.0), Point(1.0, -2.0), Point(-10.0, 15.0) ] def setUp(self): # TODO: make this reusable (or a pytest fixture) self.temp_folder = Path.cwd() / 'tmp' if not self.temp_folder.exists(): self.temp_folder.mkdir() assert self.temp_folder.is_dir() def test_reproject_spatial(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) resampled = imagecollection.resample_spatial(resolution=0, projection="EPSG:3857", method="max") metadata = resampled.pyramid.levels[0].layer_metadata print(metadata) self.assertTrue("proj=merc" in metadata.crs) path = str(self.temp_folder / "reprojected.tiff") resampled.reduce('max', 'temporal').download(path, format="GTIFF", parameters={'tiled': True}) import rasterio with rasterio.open(path) as ds: print(ds.profile) def test_reduce(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.reduce( "max", "temporal").pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) self.assertEqual(2.0, stitched.cells[0][0][1]) stitched = imagecollection.reduce( "min", "temporal").pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) self.assertEqual(1.0, stitched.cells[0][0][1]) stitched = imagecollection.reduce( "sum", "temporal").pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) self.assertEqual(4.0, stitched.cells[0][0][1]) stitched = imagecollection.reduce( "mean", "temporal").pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) self.assertAlmostEqual(1.3333333, stitched.cells[0][0][1]) stitched = imagecollection.reduce( "variance", "temporal").pyramid.levels[0].stitch() print(stitched) self.assertEqual(0.0, stitched.cells[0][0][0]) self.assertAlmostEqual(0.2222222, stitched.cells[0][0][1]) stitched = imagecollection.reduce( "sd", "temporal").pyramid.levels[0].stitch() print(stitched) self.assertEqual(0.0, stitched.cells[0][0][0]) self.assertAlmostEqual(0.4714045, stitched.cells[0][0][1]) def test_reduce_all_data(self): input = Pyramid({ 0: self._single_pixel_layer({ datetime.datetime.strptime("2016-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'): 1.0, datetime.datetime.strptime("2017-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'): 5.0 }) }) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.reduce( "min", "temporal").pyramid.levels[0].stitch() self.assertEqual(1.0, stitched.cells[0][0][0]) stitched = imagecollection.reduce( "max", "temporal").pyramid.levels[0].stitch() self.assertEqual(5.0, stitched.cells[0][0][0]) stitched = imagecollection.reduce( "sum", "temporal").pyramid.levels[0].stitch() self.assertEqual(6.0, stitched.cells[0][0][0]) stitched = imagecollection.reduce( "mean", "temporal").pyramid.levels[0].stitch() self.assertAlmostEqual(3.0, stitched.cells[0][0][0], delta=0.001) stitched = imagecollection.reduce( "variance", "temporal").pyramid.levels[0].stitch() self.assertAlmostEqual(4.0, stitched.cells[0][0][0], delta=0.001) stitched = imagecollection.reduce( "sd", "temporal").pyramid.levels[0].stitch() self.assertAlmostEqual(2.0, stitched.cells[0][0][0], delta=0.001) def test_reduce_some_nodata(self): no_data = -1.0 input = Pyramid({ 0: self._single_pixel_layer( { datetime.datetime.strptime("2016-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'): no_data, datetime.datetime.strptime("2017-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'): 5.0 }, no_data) }) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.reduce( "min", "temporal").pyramid.levels[0].stitch() #print(stitched) self.assertEqual(5.0, stitched.cells[0][0][0]) stitched = imagecollection.reduce( "max", "temporal").pyramid.levels[0].stitch() self.assertEqual(5.0, stitched.cells[0][0][0]) stitched = imagecollection.reduce( "sum", "temporal").pyramid.levels[0].stitch() self.assertEqual(5.0, stitched.cells[0][0][0]) stitched = imagecollection.reduce( "mean", "temporal").pyramid.levels[0].stitch() self.assertAlmostEqual(5.0, stitched.cells[0][0][0], delta=0.001) stitched = imagecollection.reduce( "variance", "temporal").pyramid.levels[0].stitch() self.assertAlmostEqual(0.0, stitched.cells[0][0][0], delta=0.001) stitched = imagecollection.reduce( "sd", "temporal").pyramid.levels[0].stitch() self.assertAlmostEqual(0.0, stitched.cells[0][0][0], delta=0.001) def test_reduce_tiles(self): print("======") tile1 = self._single_pixel_tile(1) tile2 = self._single_pixel_tile(5) cube = np.array([tile1.cells, tile2.cells]) # "MIN", "MAX", "SUM", "MEAN", "VARIANCE" std = np.std(cube, axis=0) var = np.var(cube, axis=0) print(var) @staticmethod def _single_pixel_tile(value, no_data=-1.0): cells = np.array([[value]]) return Tile.from_numpy_array(cells, no_data) def _single_pixel_layer(self, grid_value_by_datetime, no_data=-1.0): from collections import OrderedDict sorted_by_datetime = OrderedDict(sorted( grid_value_by_datetime.items())) def elem(timestamp, value): tile = self._single_pixel_tile(value, no_data) return [(SpaceTimeKey(0, 0, timestamp), tile)] layer = [ elem(timestamp, value) for timestamp, value in sorted_by_datetime.items() ] rdd = SparkContext.getOrCreate().parallelize(layer) datetimes = list(sorted_by_datetime.keys()) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 1.0, 'ymax': 1.0} layout = { 'layoutCols': 1, 'layoutRows': 1, 'tileCols': 1, 'tileRows': 1 } metadata = { 'cellType': 'float32ud%f' % no_data, 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(datetimes[0]) }, 'maxKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(datetimes[-1]) } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': layout } } return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata) def test_reduce_nontemporal(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) with self.assertRaises(AttributeError) as context: imagecollection.reduce("max", "spectral").pyramid.levels[0].stitch() print(context.exception) def test_aggregate_temporal(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.aggregate_temporal( ["2017-01-01", "2018-01-01"], ["2017-01-03"], "max").pyramid.levels[0].to_spatial_layer().stitch() print(stitched) def test_max_aggregator(self): tiles = [self.tile, self.tile2] composite = max_composite(tiles) self.assertEqual(2.0, composite.cells[0][0]) def test_aggregate_max_time(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.reduce( 'max', 'temporal').pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) def test_min_time(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) min_time = imagecollection.reduce('min', 'temporal') max_time = imagecollection.reduce('max', 'temporal') stitched = min_time.pyramid.levels[0].stitch() print(stitched) self.assertEquals(2.0, stitched.cells[0][0][0]) for p in self.points[1:3]: result = min_time.timeseries(p.x, p.y, srs="EPSG:3857") print(result) print(imagecollection.timeseries(p.x, p.y, srs="EPSG:3857")) max_result = max_time.timeseries(p.x, p.y, srs="EPSG:3857") self.assertEqual(1.0, result['NoDate']) self.assertEqual(2.0, max_result['NoDate']) def test_apply_spatiotemporal(self): import openeo_udf.functions input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry(), { "bands": [{ "band_id": "2", "name": "blue", "wavelength_nm": 496.6, "res_m": 10, "scale": 0.0001, "offset": 0, "type": "int16", "unit": "1" }] }) import os, openeo_udf dir = os.path.dirname(openeo_udf.functions.__file__) file_name = os.path.join(dir, "datacube_reduce_time_sum.py") with open(file_name, "r") as f: udf_code = f.read() result = imagecollection.apply_tiles_spatiotemporal(udf_code) stitched = result.pyramid.levels[0].to_spatial_layer().stitch() print(stitched) self.assertEqual(2, stitched.cells[0][0][0]) self.assertEqual(6, stitched.cells[0][0][5]) self.assertEqual(4, stitched.cells[0][5][6]) def test_apply_dimension_spatiotemporal(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry(), { "bands": [{ "band_id": "2", "name": "blue", "wavelength_nm": 496.6, "res_m": 10, "scale": 0.0001, "offset": 0, "type": "int16", "unit": "1" }] }) udf_code = """ def rct_savitzky_golay(udf_data:UdfData): from scipy.signal import savgol_filter print(udf_data.get_datacube_list()) return udf_data """ result = imagecollection.apply_tiles_spatiotemporal(udf_code) local_tiles = result.pyramid.levels[0].to_numpy_rdd().collect() print(local_tiles) self.assertEquals(len(TestMultipleDates.layer), len(local_tiles)) ref_dict = { e[0]: e[1] for e in imagecollection.pyramid.levels[0].convert_data_type( CellType.FLOAT64).to_numpy_rdd().collect() } result_dict = {e[0]: e[1] for e in local_tiles} for k, v in ref_dict.items(): tile = result_dict[k] assert_array_almost_equal(np.squeeze(v.cells), np.squeeze(tile.cells), decimal=2) def test_mask_raster(self): input = Pyramid({0: self.tiled_raster_rdd}) def createMask(tile): tile.cells[0][0][0] = 0.0 return tile mask_layer = self.tiled_raster_rdd.map_tiles(createMask) mask = Pyramid({0: mask_layer}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.mask( rastermask=GeotrellisTimeSeriesImageCollection( mask, InMemoryServiceRegistry()), replacement=10.0).reduce('max', 'temporal').pyramid.levels[0].stitch() print(stitched) self.assertEquals(2.0, stitched.cells[0][0][0]) self.assertEquals(10.0, stitched.cells[0][0][1]) def test_apply_kernel(self): kernel = np.array([[0.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 0.0]]) input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) stitched = imagecollection.apply_kernel(kernel, 2.0).reduce( 'max', 'temporal').pyramid.levels[0].stitch() self.assertEquals(12.0, stitched.cells[0][0][0]) self.assertEquals(16.0, stitched.cells[0][0][1]) self.assertEquals(20.0, stitched.cells[0][1][1]) def test_resample_spatial(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeotrellisTimeSeriesImageCollection( input, InMemoryServiceRegistry()) resampled = imagecollection.resample_spatial(resolution=0.05) path = str(self.temp_folder / "resampled.tiff") resampled.reduce('max', 'temporal').download(path, format="GTIFF", parameters={'tiled': True}) import rasterio with rasterio.open(path) as ds: print(ds.profile) self.assertAlmostEqual(0.05, ds.res[0], 3)
class TestMultipleDates(TestCase): band1 = np.array([[-1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0]]) band2 = np.array([[2.0, 2.0, 2.0, 2.0, 2.0], [2.0, 2.0, 2.0, 2.0, 2.0], [2.0, 2.0, -1.0, 2.0, 2.0], [2.0, 2.0, 2.0, 2.0, 2.0], [2.0, 2.0, 2.0, 2.0, 2.0]]) tile = Tile.from_numpy_array(band1, no_data_value=-1.0) tile2 = Tile.from_numpy_array(band2, no_data_value=-1.0) time_1 = datetime.datetime.strptime("2016-08-24T09:00:00Z", '%Y-%m-%dT%H:%M:%SZ') time_2 = datetime.datetime.strptime("2017-08-24T09:00:00Z", '%Y-%m-%dT%H:%M:%SZ') time_3 = datetime.datetime.strptime("2017-10-17T09:00:00Z", '%Y-%m-%dT%H:%M:%SZ') layer = [(SpaceTimeKey(0, 0, time_1), tile), (SpaceTimeKey(1, 0, time_1), tile2), (SpaceTimeKey(0, 1, time_1), tile), (SpaceTimeKey(1, 1, time_1), tile), (SpaceTimeKey(0, 0, time_2), tile2), (SpaceTimeKey(1, 0, time_2), tile2), (SpaceTimeKey(0, 1, time_2), tile2), (SpaceTimeKey(1, 1, time_2), tile2), (SpaceTimeKey(0, 0, time_3), tile), (SpaceTimeKey(1, 0, time_3), tile2), (SpaceTimeKey(0, 1, time_3), tile), (SpaceTimeKey(1, 1, time_3), tile)] rdd = SparkContext.getOrCreate().parallelize(layer) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0} layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5} metadata = { 'cellType': 'float32ud-1.0', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(time_1) }, 'maxKey': { 'col': 1, 'row': 1, 'instant': _convert_to_unix_time(time_3) } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': { 'tileCols': 5, 'tileRows': 5, 'layoutCols': 2, 'layoutRows': 2 } } } collection_metadata = GeopysparkCubeMetadata( {"cube:dimensions": { "t": { "type": "temporal" }, }}) tiled_raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata) layer2 = [(TemporalProjectedExtent(Extent(0, 0, 1, 1), epsg=3857, instant=time_1), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_1), tile), (TemporalProjectedExtent(Extent(0, 1, 1, 2), epsg=3857, instant=time_1), tile), (TemporalProjectedExtent(Extent(1, 1, 2, 2), epsg=3857, instant=time_1), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_2), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_2), tile), (TemporalProjectedExtent(Extent(0, 1, 1, 2), epsg=3857, instant=time_2), tile), (TemporalProjectedExtent(Extent(1, 1, 2, 2), epsg=3857, instant=time_2), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_3), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_3), tile), (TemporalProjectedExtent(Extent(0, 1, 1, 2), epsg=3857, instant=time_3), tile), (TemporalProjectedExtent(Extent(1, 1, 2, 2), epsg=3857, instant=time_3), tile)] rdd2 = SparkContext.getOrCreate().parallelize(layer2) raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd2) points = [ Point(1.0, -3.0), Point(0.5, 0.5), Point(20.0, 3.0), Point(1.0, -2.0), Point(-10.0, 15.0) ] def setUp(self): # TODO: make this reusable (or a pytest fixture) self.temp_folder = Path.cwd() / 'tmp' if not self.temp_folder.exists(): self.temp_folder.mkdir() assert self.temp_folder.is_dir() def test_reproject_spatial(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) ref_path = str(self.temp_folder / "reproj_ref.tiff") imagecollection.reduce('max', dimension="t").save_result(ref_path, format="GTIFF") resampled = imagecollection.resample_spatial(resolution=0, projection="EPSG:3395", method="max") metadata = resampled.pyramid.levels[0].layer_metadata print(metadata) self.assertTrue("proj=merc" in metadata.crs) path = str(self.temp_folder / "reprojected.tiff") res = resampled.reduce('max', dimension="t") res.save_result(path, format="GTIFF") with rasterio.open(ref_path) as ref_ds: with rasterio.open(path) as ds: print(ds.profile) #this reprojection does not change the shape, so we can compare assert ds.read().shape == ref_ds.read().shape assert (ds.crs.to_epsg() == 3395) def test_reduce(self): input = Pyramid({0: self.tiled_raster_rdd}) cube = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) env = EvalEnv() stitched = cube.reduce_dimension(dimension="t", reducer=reducer("max"), env=env).pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) self.assertEqual(2.0, stitched.cells[0][0][1]) stitched = cube.reduce_dimension(dimension="t", reducer=reducer("min"), env=env).pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) self.assertEqual(1.0, stitched.cells[0][0][1]) stitched = cube.reduce_dimension(dimension="t", reducer=reducer("sum"), env=env).pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) self.assertEqual(4.0, stitched.cells[0][0][1]) stitched = cube.reduce_dimension(dimension="t", reducer=reducer("mean"), env=env).pyramid.levels[0].stitch() print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) self.assertAlmostEqual(1.3333333, stitched.cells[0][0][1]) stitched = cube.reduce_dimension(reducer=reducer("variance"), dimension="t", env=env).pyramid.levels[0].stitch() print(stitched) self.assertEqual(0.0, stitched.cells[0][0][0]) self.assertAlmostEqual(0.2222222, stitched.cells[0][0][1]) stitched = cube.reduce_dimension(reducer=reducer("sd"), dimension="t", env=env).pyramid.levels[0].stitch() print(stitched) self.assertEqual(0.0, stitched.cells[0][0][0]) self.assertAlmostEqual(0.4714045, stitched.cells[0][0][1]) def test_reduce_all_data(self): input = Pyramid({ 0: self._single_pixel_layer({ datetime.datetime.strptime("2016-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'): 1.0, datetime.datetime.strptime("2017-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'): 5.0 }) }) cube = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) env = EvalEnv() stitched = cube.reduce_dimension(reducer=reducer("min"), dimension="t", env=env).pyramid.levels[0].stitch() self.assertEqual(1.0, stitched.cells[0][0][0]) stitched = cube.reduce_dimension(reducer=reducer("max"), dimension="t", env=env).pyramid.levels[0].stitch() self.assertEqual(5.0, stitched.cells[0][0][0]) stitched = cube.reduce_dimension(reducer=reducer("sum"), dimension="t", env=env).pyramid.levels[0].stitch() self.assertEqual(6.0, stitched.cells[0][0][0]) stitched = cube.reduce_dimension(reducer=reducer("mean"), dimension="t", env=env).pyramid.levels[0].stitch() self.assertAlmostEqual(3.0, stitched.cells[0][0][0], delta=0.001) stitched = cube.reduce_dimension(reducer=reducer("variance"), dimension="t", env=env).pyramid.levels[0].stitch() self.assertAlmostEqual(4.0, stitched.cells[0][0][0], delta=0.001) stitched = cube.reduce_dimension(reducer=reducer("sd"), dimension="t", env=env).pyramid.levels[0].stitch() self.assertAlmostEqual(2.0, stitched.cells[0][0][0], delta=0.001) def test_reduce_some_nodata(self): no_data = -1.0 input = Pyramid({ 0: self._single_pixel_layer( { datetime.datetime.strptime("2016-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'): no_data, datetime.datetime.strptime("2017-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'): 5.0 }, no_data) }) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) stitched = imagecollection.reduce( "min", dimension="t").pyramid.levels[0].stitch() #print(stitched) self.assertEqual(5.0, stitched.cells[0][0][0]) stitched = imagecollection.reduce( "max", dimension="t").pyramid.levels[0].stitch() self.assertEqual(5.0, stitched.cells[0][0][0]) stitched = imagecollection.reduce( "sum", dimension="t").pyramid.levels[0].stitch() self.assertEqual(5.0, stitched.cells[0][0][0]) stitched = imagecollection.reduce( "mean", dimension="t").pyramid.levels[0].stitch() self.assertAlmostEqual(5.0, stitched.cells[0][0][0], delta=0.001) stitched = imagecollection.reduce( "variance", dimension="t").pyramid.levels[0].stitch() self.assertAlmostEqual(0.0, stitched.cells[0][0][0], delta=0.001) stitched = imagecollection.reduce( "sd", dimension="t").pyramid.levels[0].stitch() self.assertAlmostEqual(0.0, stitched.cells[0][0][0], delta=0.001) def test_reduce_tiles(self): print("======") tile1 = self._single_pixel_tile(1) tile2 = self._single_pixel_tile(5) cube = np.array([tile1.cells, tile2.cells]) # "MIN", "MAX", "SUM", "MEAN", "VARIANCE" std = np.std(cube, axis=0) var = np.var(cube, axis=0) print(var) @staticmethod def _single_pixel_tile(value, no_data=-1.0): cells = np.array([[value]]) return Tile.from_numpy_array(cells, no_data) def _single_pixel_layer(self, grid_value_by_datetime, no_data=-1.0): from collections import OrderedDict sorted_by_datetime = OrderedDict(sorted( grid_value_by_datetime.items())) def elem(timestamp, value): tile = self._single_pixel_tile(value, no_data) return [(SpaceTimeKey(0, 0, timestamp), tile)] layer = [ elem(timestamp, value) for timestamp, value in sorted_by_datetime.items() ] rdd = SparkContext.getOrCreate().parallelize(layer) datetimes = list(sorted_by_datetime.keys()) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 1.0, 'ymax': 1.0} layout = { 'layoutCols': 1, 'layoutRows': 1, 'tileCols': 1, 'tileRows': 1 } metadata = { 'cellType': 'float32ud%f' % no_data, 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(datetimes[0]) }, 'maxKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(datetimes[-1]) } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': layout } } return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata) def test_reduce_nontemporal(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) with self.assertRaises(FeatureUnsupportedException) as context: imagecollection.reduce( "max", dimension="gender").pyramid.levels[0].stitch() print(context.exception) def test_aggregate_temporal(self): """ Tests deprecated process spec! To be phased out. @return: """ interval_list = ["2017-01-01", "2018-01-01"] self._test_aggregate_temporal(interval_list) def _median_reducer(self): from openeo.processes import median builder = median({"from_argument": "data"}) return builder.flat_graph() def test_aggregate_temporal_median(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) stitched = (imagecollection.aggregate_temporal( ["2015-01-01", "2018-01-01"], ["2017-01-03"], self._median_reducer(), dimension="t").pyramid.levels[0].to_spatial_layer().stitch()) print(stitched) expected_median = np.median( [self.tile.cells, self.tile2.cells, self.tile.cells], axis=0) #TODO nodata handling?? assert_array_almost_equal(stitched.cells[0, 1:2, 1:2], expected_median[1:2, 1:2]) def _test_aggregate_temporal(self, interval_list): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) stitched = (imagecollection.aggregate_temporal( interval_list, ["2017-01-03"], "min", dimension="t").pyramid.levels[0].to_spatial_layer().stitch()) print(stitched) expected_max = np.min([self.tile2.cells, self.tile.cells], axis=0) assert_array_almost_equal(stitched.cells[0, 0:5, 0:5], expected_max) def test_aggregate_temporal_100(self): self._test_aggregate_temporal([["2017-01-01", "2018-01-01"]]) def test_max_aggregator(self): tiles = [self.tile, self.tile2] composite = max_composite(tiles) self.assertEqual(2.0, composite.cells[0][0]) def test_aggregate_max_time(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) layer = imagecollection.reduce('max', dimension='t').pyramid.levels[0] stitched = layer.stitch() assert CellType.FLOAT32.value == layer.layer_metadata.cell_type print(stitched) self.assertEqual(2.0, stitched.cells[0][0][0]) def test_min_time(self): input = Pyramid({0: self.tiled_raster_rdd}) cube = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) env = EvalEnv() min_time = cube.reduce_dimension(reducer=reducer('min'), dimension='t', env=env) max_time = cube.reduce_dimension(reducer=reducer('max'), dimension='t', env=env) stitched = min_time.pyramid.levels[0].stitch() print(stitched) self.assertEquals(2.0, stitched.cells[0][0][0]) for p in self.points[1:3]: result = min_time.timeseries(p.x, p.y, srs="EPSG:3857") print(result) print(cube.timeseries(p.x, p.y, srs="EPSG:3857")) max_result = max_time.timeseries(p.x, p.y, srs="EPSG:3857") self.assertEqual(1.0, result['NoDate']) self.assertEqual(2.0, max_result['NoDate']) def test_apply_dimension_spatiotemporal(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube( pyramid=input, metadata=GeopysparkCubeMetadata({ "cube:dimensions": { # TODO: also specify other dimensions? "bands": { "type": "bands", "values": ["2"] } }, "summaries": { "eo:bands": [{ "name": "2", "common_name": "blue", "wavelength_nm": 496.6, "res_m": 10, "scale": 0.0001, "offset": 0, "type": "int16", "unit": "1" }] } })) udf_code = """ def rct_savitzky_golay(udf_data:UdfData): from scipy.signal import savgol_filter print(udf_data.get_datacube_list()) return udf_data """ result = imagecollection.apply_tiles_spatiotemporal(udf_code) local_tiles = result.pyramid.levels[0].to_numpy_rdd().collect() print(local_tiles) self.assertEquals(len(TestMultipleDates.layer), len(local_tiles)) ref_dict = { e[0]: e[1] for e in imagecollection.pyramid.levels[0].convert_data_type( CellType.FLOAT64).to_numpy_rdd().collect() } result_dict = {e[0]: e[1] for e in local_tiles} for k, v in ref_dict.items(): tile = result_dict[k] assert_array_almost_equal(np.squeeze(v.cells), np.squeeze(tile.cells), decimal=2) def test_mask_raster_replacement_default_none(self): def createMask(tile): tile.cells[0][0][0] = 0.0 return tile input = Pyramid({0: self.tiled_raster_rdd}) mask_layer = self.tiled_raster_rdd.map_tiles(createMask) mask = Pyramid({0: mask_layer}) cube = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) mask_cube = GeopysparkDataCube(pyramid=mask) stitched = cube.mask(mask=mask_cube).reduce( 'max', dimension="t").pyramid.levels[0].stitch() print(stitched) assert stitched.cells[0][0][0] == 2.0 assert np.isnan(stitched.cells[0][0][1]) def test_mask_raster_replacement_float(self): def createMask(tile): tile.cells[0][0][0] = 0.0 return tile input = Pyramid({0: self.tiled_raster_rdd}) mask_layer = self.tiled_raster_rdd.map_tiles(createMask) mask = Pyramid({0: mask_layer}) cube = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) mask_cube = GeopysparkDataCube(pyramid=mask) stitched = cube.mask(mask=mask_cube, replacement=10.0).reduce( 'max', dimension="t").pyramid.levels[0].stitch() print(stitched) assert stitched.cells[0][0][0] == 2.0 assert stitched.cells[0][0][1] == 10.0 def test_mask_raster_replacement_int(self): def createMask(tile): tile.cells[0][0][0] = 0.0 return tile input = Pyramid({0: self.tiled_raster_rdd}) mask_layer = self.tiled_raster_rdd.map_tiles(createMask) mask = Pyramid({0: mask_layer}) cube = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) mask_cube = GeopysparkDataCube(pyramid=mask) stitched = cube.mask(mask=mask_cube, replacement=10).reduce( 'max', dimension="t").pyramid.levels[0].stitch() print(stitched) assert stitched.cells[0][0][0] == 2.0 assert stitched.cells[0][0][1] == 10.0 def test_apply_kernel_float(self): kernel = np.array([[0.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 0.0]]) input = Pyramid({0: self.tiled_raster_rdd}) img = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) stitched = img.apply_kernel(kernel, 2.0).reduce( 'max', dimension="t").pyramid.levels[0].stitch() assert stitched.cells[0][0][0] == 12.0 assert stitched.cells[0][0][1] == 16.0 assert stitched.cells[0][1][1] == 20.0 def test_apply_kernel_int(self): kernel = np.array([[0, 1, 0], [1, 1, 1], [0, 1, 0]]) input = Pyramid({0: self.tiled_raster_rdd}) img = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) stitched = img.apply_kernel(kernel).reduce( 'max', dimension="t").pyramid.levels[0].stitch() assert stitched.cells[0][0][0] == 6.0 assert stitched.cells[0][0][1] == 8.0 assert stitched.cells[0][1][1] == 10.0 def test_resample_spatial(self): input = Pyramid({0: self.tiled_raster_rdd}) imagecollection = GeopysparkDataCube(pyramid=input, metadata=self.collection_metadata) resampled = imagecollection.resample_spatial(resolution=0.05) path = str(self.temp_folder / "resampled.tiff") res = resampled.reduce('max', dimension="t") res.save_result(path, format="GTIFF") import rasterio with rasterio.open(path) as ds: print(ds.profile) self.assertAlmostEqual(0.05, ds.res[0], 3) def test_rename_dimension(self): imagecollection = GeopysparkDataCube(pyramid=Pyramid( {0: self.tiled_raster_rdd}), metadata=self.collection_metadata) dim_renamed = imagecollection.rename_dimension('t', 'myNewTimeDim') dim_renamed.metadata.assert_valid_dimension('myNewTimeDim')
class FocalTest(BaseTestClass): cells = np.array([[[1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 0.0]]]) tile = Tile.from_numpy_array(cells, -1.0) layer = [(SpatialKey(0, 0), tile), (SpatialKey(1, 0), tile), (SpatialKey(0, 1), tile), (SpatialKey(1, 1), tile)] rdd = BaseTestClass.pysc.parallelize(layer) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0} layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5} metadata = { 'cellType': 'float32ud-1.0', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0 }, 'maxKey': { 'col': 1, 'row': 1 } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': { 'tileCols': 5, 'tileRows': 5, 'layoutCols': 2, 'layoutRows': 2 } } } raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, metadata) @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_focal_sum_square(self): result = self.raster_rdd.focal( operation=Operation.SUM, neighborhood=Neighborhood.SQUARE, param_1=1.0, partition_strategy=HashPartitionStrategy()) self.assertTrue(result.to_numpy_rdd().first()[1].cells[0][1][0] >= 6) def test_focal_sum_wedge(self): neighborhood = Wedge(radius=1.0, start_angle=0.0, end_angle=180.0) self.assertEqual(str(neighborhood), repr(neighborhood)) result = self.raster_rdd.focal( operation=Operation.SUM, neighborhood=neighborhood, partition_strategy=HashPartitionStrategy(2)) self.assertTrue(result.to_numpy_rdd().first()[1].cells[0][1][0] >= 3) def test_focal_sum_circle(self): neighborhood = Circle(radius=1.0) self.assertEqual(str(neighborhood), repr(neighborhood)) result = self.raster_rdd.focal( operation=Operation.SUM, neighborhood=neighborhood, partition_strategy=SpatialPartitionStrategy(2)) self.assertTrue(result.to_numpy_rdd().first()[1].cells[0][1][0] >= 4) def test_focal_sum_nesw(self): neighborhood = Nesw(extent=1.0) self.assertEqual(str(neighborhood), repr(neighborhood)) result = self.raster_rdd.focal( operation=Operation.SUM, neighborhood=neighborhood, partition_strategy=SpatialPartitionStrategy()) self.assertTrue(result.to_numpy_rdd().first()[1].cells[0][1][0] >= 4) def test_focal_sum_annulus(self): neighborhood = Annulus(inner_radius=0.5, outer_radius=1.5) self.assertEqual(str(neighborhood), repr(neighborhood)) result = self.raster_rdd.focal(operation=Operation.SUM, neighborhood=neighborhood) self.assertTrue(result.to_numpy_rdd().first()[1].cells[0][1][0] >= 5.0) def test_square(self): neighborhood = Square(extent=1.0) self.assertEqual(str(neighborhood), repr(neighborhood)) result = self.raster_rdd.focal(operation=Operation.SUM, neighborhood=neighborhood) self.assertTrue(result.to_numpy_rdd().first()[1].cells[0][1][0] >= 6.0) def test_focal_sum_int(self): result = self.raster_rdd.focal(operation=Operation.SUM, neighborhood=Neighborhood.SQUARE, param_1=1) self.assertTrue(result.to_numpy_rdd().first()[1].cells[0][1][0] >= 6) def test_focal_sum_square_with_square(self): square = Square(extent=1.0) result = self.raster_rdd.focal(operation=Operation.SUM, neighborhood=square) self.assertTrue(result.to_numpy_rdd().first()[1].cells[0][1][0] >= 6) def test_focal_min(self): result = self.raster_rdd.focal(operation=Operation.MIN, neighborhood=Neighborhood.ANNULUS, param_1=2.0, param_2=1.0) self.assertEqual(result.to_numpy_rdd().first()[1].cells[0][0][0], -1) def test_focal_min_annulus(self): annulus = Annulus(inner_radius=2.0, outer_radius=1.0) result = self.raster_rdd.focal(operation=Operation.MIN, neighborhood=annulus) self.assertEqual(result.to_numpy_rdd().first()[1].cells[0][0][0], -1) def test_focal_min_int(self): result = self.raster_rdd.focal(operation=Operation.MIN, neighborhood=Neighborhood.ANNULUS, param_1=2, param_2=1) self.assertEqual(result.to_numpy_rdd().first()[1].cells[0][0][0], -1) def test_tobler(self): result = self.raster_rdd.tobler()
first = np.array([[[1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]]) second = np.array([[[0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0]]]) third = np.array([[[10.0, 10.0, 10.0, 10.0, 10.0], [10.0, 10.0, 10.0, 10.0, 10.0], [10.0, 10.0, 10.0, 10.0, 10.0], [10.0, 10.0, 10.0, 10.0, 10.0], [10.0, 10.0, 10.0, 10.0, 10.0]]]) tile_1 = Tile.from_numpy_array(first, -1.0) tile_2 = Tile.from_numpy_array(second, -1.0) tile_3 = Tile.from_numpy_array(third, -1.0) class CombineSpatialBandsTest(BaseTestClass): layer_1 = create_spatial_layer(tile_1) layer_2 = create_spatial_layer(tile_2) layer_3 = create_spatial_layer(tile_3) r1 = BaseTestClass.pysc.parallelize(layer_1) r2 = BaseTestClass.pysc.parallelize(layer_2) r3 = BaseTestClass.pysc.parallelize(layer_3) rdd_1 = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, r1) rdd_2 = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, r2)
class ToSpatialLayerTest(BaseTestClass): band_1 = np.array([[1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0]]) band_2 = np.array([[2.0, 2.0, 2.0, 2.0, 2.0], [2.0, 2.0, 2.0, 2.0, 2.0], [2.0, 2.0, 2.0, 2.0, 2.0], [2.0, 2.0, 2.0, 2.0, 2.0], [2.0, 2.0, 2.0, 2.0, 2.0]]) tile_1 = Tile.from_numpy_array(np.array([band_1])) tile_2 = Tile.from_numpy_array(np.array([band_2])) time_1 = datetime.datetime.strptime("2016-08-24T09:00:00Z", '%Y-%m-%dT%H:%M:%SZ') time_2 = datetime.datetime.strptime("2017-08-24T09:00:00Z", '%Y-%m-%dT%H:%M:%SZ') layer = [(SpaceTimeKey(0, 0, time_1), tile_1), (SpaceTimeKey(1, 0, time_1), tile_1), (SpaceTimeKey(0, 1, time_1), tile_1), (SpaceTimeKey(1, 1, time_1), tile_1), (SpaceTimeKey(0, 0, time_2), tile_2), (SpaceTimeKey(1, 0, time_2), tile_2), (SpaceTimeKey(0, 1, time_2), tile_2), (SpaceTimeKey(1, 1, time_2), tile_2)] rdd = BaseTestClass.pysc.parallelize(layer) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0} layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5} metadata = { 'cellType': 'float32ud-1.0', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(time_1) }, 'maxKey': { 'col': 1, 'row': 1, 'instant': _convert_to_unix_time(time_2) } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': { 'tileCols': 5, 'tileRows': 5, 'layoutCols': 2, 'layoutRows': 2 } } } tiled_raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata) layer2 = [(TemporalProjectedExtent(Extent(0, 0, 1, 1), epsg=3857, instant=time_1), tile_1), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_1), tile_1), (TemporalProjectedExtent(Extent(0, 1, 1, 2), epsg=3857, instant=time_1), tile_1), (TemporalProjectedExtent(Extent(1, 1, 2, 2), epsg=3857, instant=time_1), tile_1), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_2), tile_2), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_2), tile_2), (TemporalProjectedExtent(Extent(0, 1, 1, 2), epsg=3857, instant=time_2), tile_2), (TemporalProjectedExtent(Extent(1, 1, 2, 2), epsg=3857, instant=time_2), tile_2)] rdd2 = BaseTestClass.pysc.parallelize(layer2) raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd2) @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() # This test should be moved to a more appropriate file once more spatial-temporal # tests are made. def test_spatial_metadata(self): metadata = self.raster_rdd.collect_metadata() min_key = metadata.bounds.minKey max_key = metadata.bounds.maxKey self.assertEqual(min_key.instant, self.time_1) self.assertEqual(max_key.instant, self.time_2) def test_to_spatial_raster_layer(self): actual = self.raster_rdd.to_spatial_layer().to_numpy_rdd().keys( ).collect() expected = [ ProjectedExtent(Extent(0, 0, 1, 1), 3857), ProjectedExtent(Extent(1, 0, 2, 1), 3857), ProjectedExtent(Extent(0, 1, 1, 2), 3857), ProjectedExtent(Extent(1, 1, 2, 2), 3857) ] for x in actual: self.assertTrue(x in expected) def test_to_spatial_target_time_raster_layer(self): converted = self.raster_rdd.to_spatial_layer(target_time=self.time_1) keys = converted.to_numpy_rdd().keys().collect() values = converted.to_numpy_rdd().values().collect() expected = [ ProjectedExtent(Extent(0, 0, 1, 1), 3857), ProjectedExtent(Extent(1, 0, 2, 1), 3857), ProjectedExtent(Extent(0, 1, 1, 2), 3857), ProjectedExtent(Extent(1, 1, 2, 2), 3857) ] for x in keys: self.assertTrue(x in expected) for x in values: self.assertEqual(x.cells.shape, self.tile_1.cells.shape) self.assertTrue((x.cells == 1.0).all()) def test_to_spatial_tiled_layer(self): actual = self.tiled_raster_rdd.to_spatial_layer().to_numpy_rdd().keys( ).collect() expected = [ SpatialKey(0, 0), SpatialKey(1, 0), SpatialKey(0, 1), SpatialKey(1, 1) ] for x in actual: self.assertTrue(x in expected) def test_to_spatial_target_time_tiled_layer(self): converted = self.tiled_raster_rdd.to_spatial_layer( target_time=self.time_2) keys = converted.to_numpy_rdd().keys().collect() values = converted.to_numpy_rdd().values().collect() expected = [ SpatialKey(0, 0), SpatialKey(1, 0), SpatialKey(0, 1), SpatialKey(1, 1) ] for x in keys: self.assertTrue(x in expected) for x in values: self.assertEqual(x.cells.shape, self.tile_2.cells.shape) self.assertTrue((x.cells == 2.0).all())
class CostDistanceTest(BaseTestClass): cells = np.array([[ [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 0.0]]]) tile = Tile.from_numpy_array(cells, -1.0) layer = [(SpatialKey(0, 0), tile), (SpatialKey(1, 0), tile), (SpatialKey(0, 1), tile), (SpatialKey(1, 1), tile)] rdd = BaseTestClass.pysc.parallelize(layer) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0} layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5} metadata = {'cellType': 'float32ud-1.0', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0}, 'maxKey': {'col': 1, 'row': 1}}, 'layoutDefinition': { 'extent': extent, 'tileLayout': {'tileCols': 5, 'tileRows': 5, 'layoutCols': 2, 'layoutRows': 2}}} raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, metadata) @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_costdistance_finite(self): def zero_one(kv): k = kv[0] return (k.col == 0 and k.row == 1) result = cost_distance(self.raster_rdd, geometries=[Point(13, 13)], max_distance=144000.0) tile = result.to_numpy_rdd().filter(zero_one).first()[1] point_distance = tile.cells[0][1][3] self.assertEqual(point_distance, 0.0) def test_costdistance_finite_int(self): def zero_one(kv): k = kv[0] return (k.col == 0 and k.row == 1) result = cost_distance(self.raster_rdd, geometries=[Point(13, 13)], max_distance=144000) tile = result.to_numpy_rdd().filter(zero_one).first()[1] point_distance = tile.cells[0][1][3] self.assertEqual(point_distance, 0.0) def test_costdistance_infinite(self): def zero_one(kv): k = kv[0] return (k.col == 0 and k.row == 1) result = cost_distance(self.raster_rdd, geometries=[Point(13, 13)], max_distance=float('inf')) tile = result.to_numpy_rdd().filter(zero_one).first()[1] point_distance = tile.cells[0][0][0] self.assertTrue(point_distance > 1250000)
def tile(value): cells = np.zeros((4, 4), dtype=float) cells.fill(value) return Tile.from_numpy_array(cells, no_data)
def _single_pixel_tile(value, no_data=-1.0): cells = np.array([[value]]) return Tile.from_numpy_array(cells, no_data)
class AggregateCellsTest(BaseTestClass): first = np.array([[ [1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0], [1.0, 2.0, 3.0, 4.0, 5.0]]]) second = np.array([[ [1.0, 1.0, 1.0, 1.0, 1.0], [2.0, 2.0, 2.0, 2.0, 2.0], [3.0, 3.0, 3.0, 3.0, 3.0], [4.0, 4.0, 4.0, 4.0, 4.0], [5.0, 5.0, 5.0, 5.0, 5.0]]]) cells_1 = np.array([first, second]) cells_2 = np.array([second, first]) tile_1 = Tile.from_numpy_array(cells_1, -1.0) tile_2 = Tile.from_numpy_array(cells_2, -1.0) layer = [(SpatialKey(0, 0), tile_1), (SpatialKey(1, 0), tile_1), (SpatialKey(1, 0), tile_2), (SpatialKey(0, 1), tile_1), (SpatialKey(1, 1), tile_1)] rdd = BaseTestClass.pysc.parallelize(layer) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0} layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5} metadata = {'cellType': 'float32ud-1.0', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0}, 'maxKey': {'col': 1, 'row': 1}}, 'layoutDefinition': { 'extent': extent, 'tileLayout': {'tileCols': 5, 'tileRows': 5, 'layoutCols': 2, 'layoutRows': 2}}} raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, metadata) @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_aggregate_sum(self): result = self.raster_rdd.aggregate_by_cell(operation=Operation.SUM) expected = np.array([self.first + self.second, self.first + self.second]) self.assertTrue((result.lookup(1, 0)[0].cells == expected).all()) self.assertTrue((result.lookup(0, 0)[0].cells[0] == self.first).all()) self.assertTrue((result.lookup(0, 0)[0].cells[1] == self.second).all()) def test_aggregate_min(self): result = self.raster_rdd.aggregate_by_cell(operation=Operation.MIN) band = np.array([[ [1, 1, 1, 1, 1], [1, 2, 2, 2, 2], [1, 2, 3, 3, 3], [1, 2, 3, 4, 4], [1, 2, 3, 4, 5]]]) expected = np.array([band, band]) self.assertTrue((result.lookup(1, 0)[0].cells == expected).all()) self.assertTrue((result.lookup(0, 0)[0].cells[0] == self.first).all()) self.assertTrue((result.lookup(0, 0)[0].cells[1] == self.second).all()) def test_aggregate_max(self): result = self.raster_rdd.aggregate_by_cell(operation=Operation.MAX) band = np.array([[ [1, 2, 3, 4, 5], [2, 2, 3, 4, 5], [3, 3, 3, 4, 5], [4, 4, 4, 4, 5], [5, 5, 5, 5, 5]]]) expected = np.array([band, band]) self.assertTrue((result.lookup(1, 0)[0].cells == expected).all()) self.assertTrue((result.lookup(0, 0)[0].cells[0] == self.first).all()) self.assertTrue((result.lookup(0, 0)[0].cells[1] == self.second).all()) def test_aggregate_mean(self): result = self.raster_rdd.aggregate_by_cell(Operation.MEAN) band = np.array([[ [1, 1.5, 2, 2.5, 3], [1.5, 2, 2.5, 3, 3.5], [2, 2.5, 3, 3.5, 4], [2.5, 3, 3.5, 4, 4.5], [3, 3.5, 4, 4.5, 5]]]) expected = np.array([band, band]) self.assertTrue((result.lookup(1, 0)[0].cells == expected).all()) self.assertTrue((result.lookup(0, 0)[0].cells[0] == self.first).all()) self.assertTrue((result.lookup(0, 0)[0].cells[1] == self.second).all()) def test_aggregate_variance(self): result = self.raster_rdd.aggregate_by_cell(Operation.VARIANCE) band = np.array([[ [1, 1.5, 2, 2.5, 3], [1.5, 2, 2.5, 3, 3.5], [2, 2.5, 3, 3.5, 4], [2.5, 3, 3.5, 4, 4.5], [3, 3.5, 4, 4.5, 5]]]) expected = np.array([ ((self.first - band) ** 2) + ((self.second - band) ** 2), ((self.first - band) ** 2) + ((self.second - band) ** 2) ]) expected_2 = np.full((5, 5), -1.0) self.assertTrue((result.lookup(1, 0)[0].cells == expected).all()) self.assertTrue((result.lookup(0, 0)[0].cells == expected_2).all()) def test_aggregate_std(self): result = self.raster_rdd.aggregate_by_cell(Operation.STANDARD_DEVIATION) band = np.array([[ [1, 1.5, 2, 2.5, 3], [1.5, 2, 2.5, 3, 3.5], [2, 2.5, 3, 3.5, 4], [2.5, 3, 3.5, 4, 4.5], [3, 3.5, 4, 4.5, 5]]]) expected = np.array([ (((self.first - band) ** 2) + ((self.second - band) ** 2)) ** (1/2), (((self.first - band) ** 2) + ((self.second - band) ** 2)) ** (1/2) ]) expected_2 = np.full((5, 5), -1.0) self.assertTrue((result.lookup(1, 0)[0].cells == expected).all()) self.assertTrue((result.lookup(0, 0)[0].cells == expected_2).all())
class MergeTest(BaseTestClass): arr_1 = np.zeros((1, 4, 4)) arr_2 = np.ones((1, 4, 4)) tile_1 = Tile.from_numpy_array(arr_1) tile_2 = Tile.from_numpy_array(arr_2) crs = 4326 time = datetime.datetime.strptime("2016-08-24T09:00:00Z", '%Y-%m-%dT%H:%M:%SZ') extents = [ Extent(0.0, 0.0, 4.0, 4.0), Extent(0.0, 4.0, 4.0, 8.0), ] extent = Extent(0.0, 0.0, 8.0, 8.0) layout = TileLayout(2, 2, 5, 5) ct = 'float32ud-1.0' md_proj = '+proj=longlat +datum=WGS84 +no_defs ' ld = LayoutDefinition(extent, layout) @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_projected_extent(self): pes = [ ProjectedExtent(extent=self.extents[0], epsg=self.crs), ProjectedExtent(extent=self.extents[1], epsg=self.crs), ] pe_layer = [(pes[0], self.tile_1), (pes[0], self.tile_2), (pes[1], self.tile_1), (pes[1], self.tile_2)] rdd = self.pysc.parallelize(pe_layer) layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd) actual = layer.merge() self.assertEqual(actual.srdd.rdd().count(), 2) for k, v in actual.to_numpy_rdd().collect(): self.assertTrue((v.cells == self.arr_2).all()) def test_temporal_projected_extent(self): pes = [ TemporalProjectedExtent(extent=self.extents[0], epsg=self.crs, instant=self.time), TemporalProjectedExtent(extent=self.extents[1], epsg=self.crs, instant=self.time), ] pe_layer = [(pes[0], self.tile_1), (pes[1], self.tile_1), (pes[0], self.tile_2), (pes[1], self.tile_2)] rdd = self.pysc.parallelize(pe_layer) layer = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd) actual = layer.merge() self.assertEqual(actual.srdd.rdd().count(), 2) for k, v in actual.to_numpy_rdd().collect(): self.assertTrue((v.cells == self.arr_2).all()) def test_spatial_keys(self): keys = [SpatialKey(0, 0), SpatialKey(0, 1)] key_layer = [(keys[0], self.tile_1), (keys[1], self.tile_1), (keys[0], self.tile_2), (keys[1], self.tile_2)] bounds = Bounds(keys[0], keys[1]) md = Metadata(bounds=bounds, crs=self.md_proj, cell_type=self.ct, extent=self.extent, layout_definition=self.ld) rdd = self.pysc.parallelize(key_layer) layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, md) actual = layer.merge() self.assertEqual(actual.srdd.rdd().count(), 2) for k, v in actual.to_numpy_rdd().collect(): self.assertTrue((v.cells == self.arr_2).all()) def test_space_time_keys(self): temp_keys = [ SpaceTimeKey(0, 0, instant=self.time), SpaceTimeKey(0, 1, instant=self.time) ] temp_key_layer = [(temp_keys[0], self.tile_2), (temp_keys[1], self.tile_2), (temp_keys[0], self.tile_2), (temp_keys[1], self.tile_2)] temp_bounds = Bounds(temp_keys[0], temp_keys[1]) temp_md = Metadata(bounds=temp_bounds, crs=self.md_proj, cell_type=self.ct, extent=self.extent, layout_definition=self.ld) rdd = self.pysc.parallelize(temp_key_layer) layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, temp_md) actual = layer.merge() self.assertEqual(actual.srdd.rdd().count(), 2) for k, v in actual.to_numpy_rdd().collect(): self.assertTrue((v.cells == self.arr_2).all())
class FilterByTimesTest(BaseTestClass): band = np.array([[1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0]]) tile = Tile.from_numpy_array(band) time_1 = datetime.datetime.strptime("2016-08-24T09:00:00Z", '%Y-%m-%dT%H:%M:%SZ') time_2 = datetime.datetime.strptime("2017-08-24T09:00:00Z", '%Y-%m-%dT%H:%M:%SZ') time_3 = datetime.datetime.strptime("2017-10-17T09:00:00Z", '%Y-%m-%dT%H:%M:%SZ') layer = [(SpaceTimeKey(0, 0, time_1), tile), (SpaceTimeKey(1, 0, time_1), tile), (SpaceTimeKey(0, 1, time_1), tile), (SpaceTimeKey(1, 1, time_1), tile), (SpaceTimeKey(0, 0, time_2), tile), (SpaceTimeKey(1, 0, time_2), tile), (SpaceTimeKey(0, 1, time_2), tile), (SpaceTimeKey(1, 1, time_2), tile), (SpaceTimeKey(0, 0, time_3), tile), (SpaceTimeKey(1, 0, time_3), tile), (SpaceTimeKey(0, 1, time_3), tile), (SpaceTimeKey(1, 1, time_3), tile)] rdd = BaseTestClass.pysc.parallelize(layer) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0} layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5} metadata = { 'cellType': 'float32ud-1.0', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(time_1) }, 'maxKey': { 'col': 1, 'row': 1, 'instant': _convert_to_unix_time(time_3) } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': { 'tileCols': 5, 'tileRows': 5, 'layoutCols': 2, 'layoutRows': 2 } } } tiled_raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata) layer2 = [(TemporalProjectedExtent(Extent(0, 0, 1, 1), epsg=3857, instant=time_1), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_1), tile), (TemporalProjectedExtent(Extent(0, 1, 1, 2), epsg=3857, instant=time_1), tile), (TemporalProjectedExtent(Extent(1, 1, 2, 2), epsg=3857, instant=time_1), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_2), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_2), tile), (TemporalProjectedExtent(Extent(0, 1, 1, 2), epsg=3857, instant=time_2), tile), (TemporalProjectedExtent(Extent(1, 1, 2, 2), epsg=3857, instant=time_2), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_3), tile), (TemporalProjectedExtent(Extent(1, 0, 2, 1), epsg=3857, instant=time_3), tile), (TemporalProjectedExtent(Extent(0, 1, 1, 2), epsg=3857, instant=time_3), tile), (TemporalProjectedExtent(Extent(1, 1, 2, 2), epsg=3857, instant=time_3), tile)] rdd2 = BaseTestClass.pysc.parallelize(layer2) raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd2) @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_filter_temporal_projected_extent_single_time(self): result = self.raster_rdd.filter_by_times([self.time_1]) expected = self.layer2[:4] actual = result.to_numpy_rdd().collect() self.assertEqual(len(expected), len(actual)) for x, y in zip(expected, actual): self.assertEqual(x[0], y[0]) self.assertTrue((x[1].cells == y[1].cells).all()) def test_filter_temporal_projected_extent_multi_intervals(self): result = self.raster_rdd.filter_by_times([self.time_2, self.time_3]) expected = self.layer2[4:] actual = result.to_numpy_rdd().collect() self.assertEqual(len(expected), len(actual)) for x, y in zip(expected, actual): self.assertEqual(x[0], y[0]) self.assertTrue((x[1].cells == y[1].cells).all()) def test_filter_spacetime_key_single_time(self): result = self.tiled_raster_rdd.filter_by_times([self.time_3]) expected = self.layer[8:] actual = result.to_numpy_rdd().collect() self.assertEqual(len(expected), len(actual)) for x, y in zip(expected, actual): self.assertEqual(x[0], y[0]) self.assertTrue((x[1].cells == y[1].cells).all()) def test_filter_spacetime_key_multi_intervals(self): result = self.tiled_raster_rdd.filter_by_times( [self.time_1, self.time_2]) expected = self.layer[:8] actual = result.to_numpy_rdd().collect() self.assertEqual(len(expected), len(actual)) for x, y in zip(expected, actual): self.assertEqual(x[0], y[0]) self.assertTrue((x[1].cells == y[1].cells).all())