class CellValueCountsTest(BaseTestClass): pysc = BaseTestClass.pysc cells = np.array([[ [1.0, 1.0, 1.0, 1.0, 1.0], [2.0, 2.0, 2.0, 2.0, 2.0], [3.0, 3.0, 3.0, 3.0, 3.0], [4.0, 4.0, 4.0, 4.0, 4.0], [5.0, 5.0, 5.0, 5.0, 5.0]]]) layer = [(SpatialKey(0, 0), Tile(cells, 'FLOAT', -1.0)), (SpatialKey(1, 0), Tile(cells, 'FLOAT', -1.0,)), (SpatialKey(0, 1), Tile(cells, 'FLOAT', -1.0,)), (SpatialKey(1, 1), Tile(cells, 'FLOAT', -1.0,))] rdd = pysc.parallelize(layer) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 4.0, 'ymax': 4.0} layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5} metadata = {'cellType': 'float32ud-1.0', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0}, 'maxKey': {'col': 1, 'row': 1}}, 'layoutDefinition': { 'extent': extent, 'tileLayout': layout}} raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, metadata) @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_counts_with_no_area(self): actual = self.raster_rdd.get_cell_value_counts() expected = { 1: 20, 2: 20, 3: 20, 4: 20, 5: 20 } self.assertDictEqual(actual, expected) def test_counts_with_polygon(self): area_of_interest = box(0.0, 0.0, 2.0, 2.0) actual = self.raster_rdd.get_cell_value_counts(area_of_interest) expected = { 1: 5, 2: 5, 3: 5, 4: 5, 5: 5 } self.assertDictEqual(actual, expected)
def rasterize(geoms, crs, zoom, fill_value, cell_type=CellType.FLOAT64, options=None, num_partitions=None): """Rasterizes a Shapely geometries. Args: geoms ([shapely.geometry]): List of shapely geometries to rasterize. crs (str or int): The CRS of the input geometry. zoom (int): The zoom level of the output raster. fill_value (int or float): Value to burn into pixels intersectiong geometry cell_type (str or :class:`~geopyspark.geotrellis.constants.CellType`): Which data type the cells should be when created. Defaults to ``CellType.FLOAT64``. options (:class:`~geopyspark.geotrellis.RasterizerOptions`): Pixel intersection options. Returns: :class:`~geopyspark.geotrellis.rdd.TiledRasterLayer` """ if isinstance(crs, int): crs = str(crs) pysc = get_spark_context() wkb_geoms = [shapely.wkb.dumps(g) for g in geoms] srdd = pysc._gateway.jvm.geopyspark.geotrellis.SpatialTiledRasterLayer.rasterizeGeometry( pysc._jsc.sc(), wkb_geoms, crs, zoom, float(fill_value), CellType(cell_type).value, options, num_partitions) return TiledRasterLayer(LayerType.SPATIAL, srdd)
def euclidean_distance(geometry, source_crs, zoom, cell_type=CellType.FLOAT64): """Calculates the Euclidean distance of a Shapely geometry. Args: geometry (shapely.geometry): The input geometry to compute the Euclidean distance for. source_crs (str or int): The CRS of the input geometry. zoom (int): The zoom level of the output raster. cell_type (str or :class:`~geopyspark.geotrellis.constants.CellType`, optional): The data type of the cells for the new layer. If not specified, then ``CellType.FLOAT64`` is used. Note: This function may run very slowly for polygonal inputs if they cover many cells of the output raster. Returns: :class:`~geopyspark.geotrellis.rdd.TiledRasterLayer` """ if isinstance(source_crs, int): source_crs = str(source_crs) pysc = get_spark_context() srdd = pysc._gateway.jvm.geopyspark.geotrellis.SpatialTiledRasterLayer.euclideanDistance( pysc._jsc.sc(), shapely.wkb.dumps(geometry), source_crs, CellType(cell_type).value, zoom) return TiledRasterLayer(LayerType.SPATIAL, srdd)
def _create_spacetime_layer(cells: np.ndarray = None) -> TiledRasterLayer: # TODO all these "create_spacetime_layer" functions are duplicated across all tests # and better should be moved to some kind of general factory or test fixture assert len(cells.shape) == 4 tile = Tile.from_numpy_array(cells, -1) layer = [(SpaceTimeKey(0, 0, now), tile), (SpaceTimeKey(1, 0, now), tile), (SpaceTimeKey(0, 1, now), tile), (SpaceTimeKey(1, 1, now), tile)] rdd = SparkContext.getOrCreate().parallelize(layer) metadata = { 'cellType': 'int32ud-1', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(now) }, 'maxKey': { 'col': 1, 'row': 1, 'instant': _convert_to_unix_time(now) } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': layout } } return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
def create_spacetime_layer(self): cells = np.array([self.first, self.second], dtype='int') tile = Tile.from_numpy_array(cells, -1) layer = [(SpaceTimeKey(0, 0, self.now), tile), (SpaceTimeKey(1, 0, self.now), tile), (SpaceTimeKey(0, 1, self.now), tile), (SpaceTimeKey(1, 1, self.now), tile)] rdd = SparkContext.getOrCreate().parallelize(layer) metadata = {'cellType': 'int32ud-1', 'extent': self.extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(self.now)}, 'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(self.now)} }, 'layoutDefinition': { 'extent': self.extent, 'tileLayout': self.layout } } return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
def create_spatial_layer(self): cells = np.array([self.first, self.second], dtype='int') tile = Tile.from_numpy_array(cells, -1) layer = [(SpatialKey(0, 0), tile), (SpatialKey(1, 0), tile), (SpatialKey(0, 1), tile), (SpatialKey(1, 1), tile)] rdd = BaseTestClass.pysc.parallelize(layer) metadata = { 'cellType': 'int32ud-1', 'extent': self.extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0 }, 'maxKey': { 'col': 1, 'row': 1 } }, 'layoutDefinition': { 'extent': self.extent, 'tileLayout': self.layout } } return TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, metadata)
def imagecollection_with_two_bands_and_three_dates_webmerc(request): from geopyspark.geotrellis import (SpaceTimeKey, Tile, _convert_to_unix_time) from geopyspark.geotrellis.constants import LayerType from geopyspark.geotrellis.layer import TiledRasterLayer import geopyspark as gps from openeogeotrellis.geopysparkdatacube import GeopysparkDataCube,GeopysparkCubeMetadata date1, date3, rdd = numpy_rdd_two_bands_and_three_dates() metadata = {'cellType': 'int32ud-1', 'extent': extent_webmerc, 'crs': '+proj=merc +a=6378137 +b=6378137 +lat_ts=0 +lon_0=0 +x_0=0 +y_0=0 +k=1 +units=m +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(date1)}, 'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(date3)} }, 'layoutDefinition': { 'extent': extent_webmerc, 'tileLayout': layout } } geopyspark_layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata) datacube = GeopysparkDataCube(pyramid=gps.Pyramid({0: geopyspark_layer}), metadata=GeopysparkCubeMetadata(openeo_metadata)) if request.instance: request.instance.imagecollection_with_two_bands_and_three_dates = datacube return datacube
def test_space_time_keys(self): temp_keys = [ SpaceTimeKey(0, 0, instant=self.time), SpaceTimeKey(0, 1, instant=self.time) ] temp_key_layer = [(temp_keys[0], self.tile_2), (temp_keys[1], self.tile_2), (temp_keys[0], self.tile_2), (temp_keys[1], self.tile_2)] temp_bounds = Bounds(temp_keys[0], temp_keys[1]) temp_md = Metadata(bounds=temp_bounds, crs=self.md_proj, cell_type=self.ct, extent=self.extent, layout_definition=self.ld) rdd = self.pysc.parallelize(temp_key_layer) layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, temp_md) actual = layer.merge() self.assertEqual(actual.srdd.rdd().count(), 2) for k, v in actual.to_numpy_rdd().collect(): self.assertTrue((v.cells == self.arr_2).all())
def hillshade(tiled_raster_layer, band=0, azimuth=315.0, altitude=45.0, z_factor=1.0): """Computes Hillshade (shaded relief) from a raster. The resulting raster will be a shaded relief map (a hill shading) based on the sun altitude, azimuth, and the z factor. The z factor is a conversion factor from map units to elevation units. Returns a raster of ShortConstantNoDataCellType. For descriptions of parameters, please see Esri Desktop's `description <http://goo.gl/DtVDQ>`_ of Hillshade. Args: band (int): The band of the raster to base the hillshade calculation on. Default is 0. azimuth (float): The azimuth angle of the source of light. Default value is 315.0. altitude (float): The angle of the altitude of the light above the horizon. Default is 45.0. z_factor (float): How many x and y units in a single z unit. Default value is 1.0. Returns: :class:`~geopyspark.geotrellis.layer.TiledRasterLayer` """ srdd = tiled_raster_layer.srdd.hillshade(tiled_raster_layer.pysc._jsc.sc(), azimuth, altitude, z_factor, band) return TiledRasterLayer(tiled_raster_layer.layer_type, srdd)
def test_bin_counts(self): metadata2 = {'cellType': 'int32ud-500', 'extent': self.extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0}, 'maxKey': {'col': 0, 'row': 0}}, 'layoutDefinition': { 'extent': self.extent, 'tileLayout': {'tileCols': 4, 'tileRows': 4, 'layoutCols': 1, 'layoutRows': 1}}} arr2 = np.int8([[[1, 1, 1, 1], [3, 1, 1, 1], [4, 3, 1, 1], [5, 4, 3, 1]]]) tile2 = Tile(arr2, 'INT', -500) rdd2 = BaseTestClass.pysc.parallelize([(self.spatial_key, tile2)]) tiled2 = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd2, metadata2) hist2 = tiled2.get_class_histogram() bin_counts = hist2.bin_counts() self.assertEqual(bin_counts, [(1, 10), (3, 3), (4, 2), (5, 1)])
def test_spatial_keys(self): keys = [ SpatialKey(0, 0), SpatialKey(0, 1), SpatialKey(1, 0), SpatialKey(1, 1) ] key_layer = [(keys[0], self.tile), (keys[1], self.tile), (keys[2], self.tile), (keys[3], self.tile)] bounds = Bounds(keys[0], keys[3]) md = Metadata(bounds=bounds, crs=self.md_proj, cell_type=self.ct, extent=self.extent, layout_definition=self.ld) rdd = self.pysc.parallelize(key_layer) layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, md) actual = layer.collect_keys() for x in actual: self.assertTrue(x in keys)
def test_space_time_keys(self): temp_keys = [ SpaceTimeKey(0, 0, instant=self.time), SpaceTimeKey(0, 1, instant=self.time), SpaceTimeKey(1, 0, instant=self.time), SpaceTimeKey(1, 1, instant=self.time) ] temp_key_layer = [(temp_keys[0], self.tile), (temp_keys[1], self.tile), (temp_keys[2], self.tile), (temp_keys[3], self.tile)] temp_bounds = Bounds(temp_keys[0], temp_keys[3]) temp_md = Metadata(bounds=temp_bounds, crs=self.md_proj, cell_type=self.ct, extent=self.extent, layout_definition=self.ld) rdd = self.pysc.parallelize(temp_key_layer) layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, temp_md) actual = layer.collect_keys() for x in actual: self.assertTrue(x in temp_keys)
class StitchTest(BaseTestClass): cells = np.array([[[1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 0.0]]]) layer = [(SpatialKey(0, 0), Tile(cells, 'FLOAT', -1.0)), (SpatialKey(1, 0), Tile( cells, 'FLOAT', -1.0, )), (SpatialKey(0, 1), Tile( cells, 'FLOAT', -1.0, )), (SpatialKey(1, 1), Tile( cells, 'FLOAT', -1.0, ))] rdd = BaseTestClass.pysc.parallelize(layer) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0} layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5} metadata = { 'cellType': 'float32ud-1.0', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0 }, 'maxKey': { 'col': 1, 'row': 1 } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': { 'tileCols': 5, 'tileRows': 5, 'layoutCols': 2, 'layoutRows': 2 } } } raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, metadata) @pytest.fixture(scope='class', autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_stitch(self): result = self.raster_rdd.stitch() self.assertTrue(result.cells.shape == (1, 10, 10))
def combine_bands(layers): """Combines the bands of values that share the same key in two or more ``TiledRasterLayer``\s. This method will concat the bands of two or more values with the same key. For example, ``layer a`` has values that have 2 bands and ``layer b`` has values with 1 band. When ``combine_bands`` is used on both of these layers, then the resulting layer will have values with 3 bands, 2 from ``layer a`` and 1 from ``layer b``. Note: All layers must have the same ``layer_type``. If the layers are ``TiledRasterLayer``\s, then all of the layers must also have the same :class:`~geopyspark.geotrellis.TileLayout` and ``CRS``. Args: layers ([:class:`~geopyspark.RasterLayer`] or [:class:`~geopyspark.TiledRasterLayer`] or (:class:`~geopyspark.RasterLayer`) or (:class:`~geopyspark.TiledRasterLayer`)): A colection of two or more ``RasterLayer``\s or ``TiledRasterLayer``\s. **The order of the layers determines the order in which the bands are concatenated**. With the bands being ordered based on the position of their respective layer. For example, the first layer in ``layers`` is ``layer a`` which contains 2 bands and the second layer is ``layer b`` whose values have 1 band. The resulting layer will have values with 3 bands: the first 2 are from ``layer a`` and the third from ``layer b``. If the positions of ``layer a`` and ``layer b`` are reversed, then the resulting values' first band will be from ``layer b`` and the last 2 will be from ``layer a``. Returns: :class:`~geopyspark.RasterLayer` or :class:`~geopyspark.TiledRasterLayer` """ if len(layers) == 1: raise ValueError( "combine_bands can only be performed on 2 or more layers") base_layer = layers[0] base_layer_type = base_layer.layer_type check_layers(base_layer, base_layer_type, layers) pysc = get_spark_context() if isinstance(base_layer, RasterLayer): if base_layer_type == LayerType.SPATIAL: result = pysc._gateway.jvm.geopyspark.geotrellis.ProjectedRasterLayer.combineBands( pysc._jsc.sc(), [x.srdd for x in layers]) else: result = pysc._gateway.jvm.geopyspark.geotrellis.TemporalRasterLayer.combineBands( pysc._jsc.sc(), [x.srdd for x in layers]) return RasterLayer(base_layer_type, result) else: if base_layer_type == LayerType.SPATIAL: result = pysc._gateway.jvm.geopyspark.geotrellis.SpatialTiledRasterLayer.combineBands( pysc._jsc.sc(), [x.srdd for x in layers]) else: result = pysc._gateway.jvm.geopyspark.geotrellis.TemporalTiledRasterLayer.combineBands( pysc._jsc.sc(), [x.srdd for x in layers]) return TiledRasterLayer(base_layer_type, result)
class HillshadeTest(BaseTestClass): cells = np.array([[[1.0, 1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 3.0, 3.0, 2.0, 1.0, -1.0], [1.0, 1.0, 3.0, 2.0, 2.0, 2.0], [1.0, 2.0, 2.0, 2.0, 2.0, 2.0], [1.0, 1.0, 1.0, 2.0, 2.0, 2.0], [1.0, 1.0, 1.0, 1.0, 1.0, 2.0]]]) layer = [(SpatialKey(0, 0), Tile(cells, 'FLOAT', -1.0))] rdd = BaseTestClass.pysc.parallelize(layer) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0} layout = {'layoutCols': 1, 'layoutRows': 1, 'tileCols': 6, 'tileRows': 6} metadata = { 'cellType': 'float32ud-1.0', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0 }, 'maxKey': { 'col': 0, 'row': 0 } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': { 'tileCols': 6, 'tileRows': 6, 'layoutCols': 1, 'layoutRows': 1 } } } raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, metadata) @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_hillshade(self): calc = zfactor_lat_lng_calculator(Unit.METERS) result = hillshade(self.raster_rdd, calc, band=0, azimuth=99.0, altitude=33.0) data = result.to_numpy_rdd().first()[1].cells[0][0][0] self.assertEqual(data, 63)
def test_add_int(self): arr = np.zeros((1, 4, 4)) tile = Tile(arr, 'FLOAT', -500) rdd = BaseTestClass.pysc.parallelize([(self.spatial_key, tile)]) tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, self.metadata) result = tiled + 1 actual = result.to_numpy_rdd().first()[1].cells self.assertTrue((actual == 1).all())
def test_rpow_double(self): arr = np.full((1, 4, 4), 3.0, dtype='int64') tile = Tile(arr, 'FLOAT', -500) rdd = BaseTestClass.pysc.parallelize([(self.spatial_key, tile)]) tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, self.metadata) result = 0.0 ** tiled actual = result.to_numpy_rdd().first()[1].cells self.assertTrue((actual == 0.0).all())
def test_mode(self): arr2 = np.array([[[1.0, 1.0, 1.0, 1.0], [2.0, 2.0, 2.0, 2.0], [1.0, 3.0, 3.0, 3.0], [4.0, 4.0, 4.0, 4.0]]], dtype=float) tile2 = Tile(arr2, 'FLOAT', -500) rdd2 = BaseTestClass.pysc.parallelize([(self.spatial_key, tile2)]) tiled2 = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd2, self.metadata) hist2 = tiled2.get_histogram() self.assertEqual(hist2.mode(), 1.0)
class MaskTest(BaseTestClass): pysc = BaseTestClass.pysc cells = np.zeros((1, 2, 2)) cells.fill(1) layer = [(SpatialKey(0, 0), Tile(cells, 'FLOAT', -1.0)), (SpatialKey(1, 0), Tile(cells, 'FLOAT', -1.0,)), (SpatialKey(0, 1), Tile(cells, 'FLOAT', -1.0,)), (SpatialKey(1, 1), Tile(cells, 'FLOAT', -1.0,))] rdd = pysc.parallelize(layer) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 4.0, 'ymax': 4.0} layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 2, 'tileRows': 2} metadata = {'cellType': 'float32ud-1.0', 'extent': extent, 'crs': 4326, 'bounds': { 'minKey': {'col': 0, 'row': 0}, 'maxKey': {'col': 1, 'row': 1}}, 'layoutDefinition': { 'extent': extent, 'tileLayout': layout}} geoms = [box(0.0, 0.0, 2.0, 2.0), box(3.0, 3.0, 4.0, 4.0)] raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, Metadata.from_dict(metadata)) @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_geotrellis_mask(self): result = self.raster_rdd.mask(geometries=self.geoms).to_numpy_rdd() n = result.map(lambda kv: np.sum(kv[1].cells)).reduce(lambda a, b: a + b) self.assertEqual(n, 2.0) def test_rdd_mask_no_partition_strategy(self): rdd = BaseTestClass.pysc.parallelize(self.geoms) result = self.raster_rdd.mask(rdd, options=RasterizerOptions(True, 'PixelIsArea')).to_numpy_rdd() n = result.map(lambda kv: np.sum(kv[1].cells)).reduce(lambda a, b: a + b) self.assertEqual(n, 2.0) def test_rdd_mask_with_partition_strategy(self): rdd = BaseTestClass.pysc.parallelize(self.geoms) result = self.raster_rdd.mask(rdd, partition_strategy=SpatialPartitionStrategy()).to_numpy_rdd() n = result.map(lambda kv: np.sum(kv[1].cells)).reduce(lambda a, b: a + b) self.assertEqual(n, 2.0)
def read(self, layout, read_method, target_crs=None, multiplex=False): expected_tiled = self.rdd.tile_to_layout(layout, target_crs=target_crs) expected_collected = expected_tiled.to_numpy_rdd().collect() if multiplex: sources = [ SourceInfo(self.path, {0: 0}), SourceInfo(self.path, {0: 1}) ] actual_tiled = TiledRasterLayer.read(sources, layout_type=layout, target_crs=target_crs) else: actual_tiled = TiledRasterLayer.read([self.path], layout_type=layout, target_crs=target_crs) actual_collected = actual_tiled.to_numpy_rdd().collect() self.assertEqual(len(expected_collected), len(actual_collected)) expected_collected.sort(key=lambda tup: (tup[0].col, tup[0].row)) actual_collected.sort(key=lambda tup: (tup[0].col, tup[0].row)) if multiplex: bands = (0, 1) else: bands = [0] for expected, actual in zip(expected_collected, actual_collected): for x in bands: self.assertEqual(expected[0], actual[0]) self.assertTrue(expected[1].cells.shape[1:] == actual[1].cells[ x, :, :].shape) diff = abs(expected[1].cells - actual[1].cells[x, :, :]) off_values_count = (diff > self.difference).sum() self.assertTrue( off_values_count / expected[1].cells.size <= 0.025)
def rdd_of_rasters(cls, uri, extent, days, num_partitions=None): if not isinstance(uri, str): raise Exception sc = get_spark_context() int_days = list(map(lambda day: int(day), days)) float_extent = list(map(lambda coord: float(coord), extent)) jvm = sc._gateway.jvm rdd = jvm.geopyspark.netcdf.datasets.Gddp.rasters( uri, float_extent, int_days, num_partitions, sc._jsc.sc()) return TiledRasterLayer(LayerType.SPACETIME, rdd)
def _single_pixel_layer(self, grid_value_by_datetime, no_data=-1.0): from collections import OrderedDict sorted_by_datetime = OrderedDict(sorted( grid_value_by_datetime.items())) def elem(timestamp, value): tile = self._single_pixel_tile(value, no_data) return [(SpaceTimeKey(0, 0, timestamp), tile)] layer = [ elem(timestamp, value) for timestamp, value in sorted_by_datetime.items() ] rdd = SparkContext.getOrCreate().parallelize(layer) datetimes = list(sorted_by_datetime.keys()) extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 1.0, 'ymax': 1.0} layout = { 'layoutCols': 1, 'layoutRows': 1, 'tileCols': 1, 'tileRows': 1 } metadata = { 'cellType': 'float32ud%f' % no_data, 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(datetimes[0]) }, 'maxKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(datetimes[-1]) } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': layout } } return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
def _create_spacetime_layer(self, no_data): def tile(value): cells = np.zeros((4, 4), dtype=float) cells.fill(value) return Tile.from_numpy_array(cells, no_data) tiles = [(SpaceTimeKey(0, 0, self.now), tile(0)), (SpaceTimeKey(1, 0, self.now), tile(1)), (SpaceTimeKey(0, 1, self.now), tile(2)), (SpaceTimeKey(1, 1, self.now), tile(no_data))] for tile in tiles: print(tile) layout = { 'layoutCols': 2, 'layoutRows': 2, 'tileCols': 4, 'tileRows': 4 } extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 8.0, 'ymax': 8.0} rdd = SparkContext.getOrCreate().parallelize(tiles) print(rdd.count()) metadata = { 'cellType': 'float64ud-1', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0, 'instant': _convert_to_unix_time(self.now) }, 'maxKey': { 'col': 1, 'row': 1, 'instant': _convert_to_unix_time(self.now) } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': layout } } return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
def test_combined_operations(self): arr = np.array([[[10, 10, 10, 10], [20, 20, 20, 20], [10, 10, 10, 10], [20, 20, 20, 20]]], dtype=int) tile = Tile(arr, 'INT', -500) rdd = BaseTestClass.pysc.parallelize([(self.spatial_key, tile)]) tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, self.metadata) result = (tiled + tiled) / 2 actual = result.to_numpy_rdd().first()[1].cells self.assertTrue((actual == arr).all())
def test_divide_tiled_rdd(self): arr = np.array([[[5.0, 5.0, 5.0, 5.0], [5.0, 5.0, 5.0, 5.0], [5.0, 5.0, 5.0, 5.0], [5.0, 5.0, 5.0, 5.0]]], dtype=float) divider = np.array([[[1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0]]], dtype=float) tile = Tile(arr, 'FLOAT', float('nan')) tile2 = Tile(divider, 'FLOAT', float('nan')) rdd = BaseTestClass.pysc.parallelize([(self.spatial_key, tile)]) rdd2 = BaseTestClass.pysc.parallelize([(self.spatial_key, tile2)]) tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, self.metadata) tiled2 = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd2, self.metadata) result = tiled / tiled2 actual = result.to_numpy_rdd().first()[1].cells self.assertTrue((actual == 5.0).all())
def test_from_histogram(self): extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0} layout = { 'layoutCols': 1, 'layoutRows': 1, 'tileCols': 2, 'tileRows': 4 } metadata = { 'cellType': 'float32ud-1.0', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': { 'col': 0, 'row': 0 }, 'maxKey': { 'col': 0, 'row': 0 } }, 'layoutDefinition': { 'extent': extent, 'tileLayout': { 'tileCols': 2, 'tileRows': 4, 'layoutCols': 1, 'layoutRows': 1 } } } spatial_key = SpatialKey(0, 0) arr = np.array([[[1.0, 5.0, 2.0, 3.0], [4.0, 6.0, 7.0, 0.0]]], dtype=float) tile = Tile(arr, 'FLOAT', -500) rdd = BaseTestClass.pysc.parallelize([(spatial_key, tile)]) tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, metadata) hist = tiled.get_histogram() color_list = self.color_list result = ColorMap.from_histogram(hist, color_list) self.assertTrue(isinstance(result, ColorMap))
def test_multiply_double(self): arr = np.array([[[1.0, 1.0, 1.0, 1.0], [2.0, 2.0, 2.0, 2.0], [3.0, 3.0, 3.0, 3.0], [4.0, 4.0, 4.0, 4.0]]], dtype=float) tile = Tile(arr, 'FLOAT', float('nan')) rdd = BaseTestClass.pysc.parallelize([(self.spatial_key, tile)]) tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, self.metadata) result = 5.0 * tiled actual = result.to_numpy_rdd().first()[1].cells expected = np.array([[[5.0, 5.0, 5.0, 5.0], [10.0, 10.0, 10.0, 10.0], [15.0, 15.0, 15.0, 15.0], [20.0, 20.0, 20.0, 20.0]]], dtype=float) self.assertTrue((actual == expected).all())
def imagecollection_with_two_bands_and_one_date(request): import geopyspark as gps from geopyspark.geotrellis import (SpaceTimeKey, Tile, _convert_to_unix_time) from geopyspark.geotrellis.constants import LayerType from geopyspark.geotrellis.layer import TiledRasterLayer from pyspark import SparkContext from openeogeotrellis.geopysparkdatacube import GeopysparkDataCube print(request) two_band_one_two = np.array([matrix_of_one, matrix_of_two], dtype='int') tile = Tile.from_numpy_array(two_band_one_two, -1) date1 = datetime.datetime.strptime("2017-09-25T11:37:00Z", '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC) layer = [(SpaceTimeKey(0, 0, date1), tile), (SpaceTimeKey(1, 0, date1), tile), (SpaceTimeKey(0, 1, date1), tile), (SpaceTimeKey(1, 1, date1), tile)] rdd = SparkContext.getOrCreate().parallelize(layer) metadata = {'cellType': 'int32ud-1', 'extent': extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(date1)}, 'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(date1)} }, 'layoutDefinition': { 'extent': extent, 'tileLayout': layout } } geopyspark_layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata) datacube = GeopysparkDataCube(pyramid=gps.Pyramid({0: geopyspark_layer}), metadata=openeo_metadata) if request.instance: request.instance.imagecollection_with_two_bands_and_one_date = datacube return datacube
def hillshade(tiled_raster_layer, zfactor_calculator, band=0, azimuth=315.0, altitude=45.0): """Computes Hillshade (shaded relief) from a raster. The resulting raster will be a shaded relief map (a hill shading) based on the sun altitude, azimuth, and the ``zfactor``. The ``zfactor`` is a conversion factor from map units to elevation units. The ``hillshade``` operation will be carried out in a ``SQUARE`` neighborhood with with an ``extent`` of 1. The ``zfactor`` will be derived from the ``zfactor_calculator`` for each ``Tile`` in the Layer. The resulting Layer will have a ``cell_type`` of ``INT16`` regardless of the input Layer's ``cell_type``; as well as have a single band, that represents the calculated ``hillshade``. Returns a raster of ShortConstantNoDataCellType. For descriptions of parameters, please see Esri Desktop's `description <http://goo.gl/DtVDQ>`_ of Hillshade. Args: tiled_raster_layer (:class:`~geopyspark.geotrellis.layer.TiledRasterLayer`): The base layer that contains the rasters used to compute the hillshade. zfactor_calculator (py4j.JavaObject): A ``JavaObject`` that represents the Scala ``ZFactorCalculator`` class. This can be created using either the :meth:`~geopyspark.geotrellis.zfactor_lat_lng_calculator` or the :meth:`~geopyspark.geotrellis.zfactor_calculator` methods. band (int, optional): The band of the raster to base the hillshade calculation on. Default is 0. azimuth (float, optional): The azimuth angle of the source of light. Default value is 315.0. altitude (float, optional): The angle of the altitude of the light above the horizon. Default is 45.0. Returns: :class:`~geopyspark.geotrellis.layer.TiledRasterLayer` """ srdd = tiled_raster_layer.srdd.hillshade(azimuth, altitude, zfactor_calculator, band) return TiledRasterLayer(tiled_raster_layer.layer_type, srdd)
def create_spacetime_unsigned_byte_layer(self): """ Returns a single-band uint8ud255 layer consisting of four tiles that each look like this: ND 220 220 220 220 220 220 220 220 220 220 220 220 220 220 220 The extent is (0.0, 0.0) to (4.0, 4.0). """ no_data = 255 single_band = np.zeros((1, 4, 4)) single_band.fill(220) single_band[0, 0, 0] = no_data cells = np.array([single_band], dtype='uint8') tile = Tile.from_numpy_array(cells, no_data) layer = [(SpaceTimeKey(0, 0, self.now), tile), (SpaceTimeKey(1, 0, self.now), tile), (SpaceTimeKey(0, 1, self.now), tile), (SpaceTimeKey(1, 1, self.now), tile)] rdd = SparkContext.getOrCreate().parallelize(layer) metadata = { 'cellType': 'uint8ud255', 'extent': self.extent, 'crs': '+proj=longlat +datum=WGS84 +no_defs ', 'bounds': { 'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(self.now)}, 'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(self.now)} }, 'layoutDefinition': { 'extent': self.extent, 'tileLayout': self.layout } } return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)