class CellValueCountsTest(BaseTestClass):
pysc = BaseTestClass.pysc
cells = np.array([[
[1.0, 1.0, 1.0, 1.0, 1.0],
[2.0, 2.0, 2.0, 2.0, 2.0],
[3.0, 3.0, 3.0, 3.0, 3.0],
[4.0, 4.0, 4.0, 4.0, 4.0],
[5.0, 5.0, 5.0, 5.0, 5.0]]])
layer = [(SpatialKey(0, 0), Tile(cells, 'FLOAT', -1.0)),
(SpatialKey(1, 0), Tile(cells, 'FLOAT', -1.0,)),
(SpatialKey(0, 1), Tile(cells, 'FLOAT', -1.0,)),
(SpatialKey(1, 1), Tile(cells, 'FLOAT', -1.0,))]
rdd = pysc.parallelize(layer)
extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 4.0, 'ymax': 4.0}
layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5}
metadata = {'cellType': 'float32ud-1.0',
'extent': extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {'col': 0, 'row': 0},
'maxKey': {'col': 1, 'row': 1}},
'layoutDefinition': {
'extent': extent,
'tileLayout': layout}}
raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, metadata)
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_counts_with_no_area(self):
actual = self.raster_rdd.get_cell_value_counts()
expected = {
1: 20,
2: 20,
3: 20,
4: 20,
5: 20
}
self.assertDictEqual(actual, expected)
def test_counts_with_polygon(self):
area_of_interest = box(0.0, 0.0, 2.0, 2.0)
actual = self.raster_rdd.get_cell_value_counts(area_of_interest)
expected = {
1: 5,
2: 5,
3: 5,
4: 5,
5: 5
}
self.assertDictEqual(actual, expected)
def rasterize(geoms,
crs,
zoom,
fill_value,
cell_type=CellType.FLOAT64,
options=None,
num_partitions=None):
"""Rasterizes a Shapely geometries.
Args:
geoms ([shapely.geometry]): List of shapely geometries to rasterize.
crs (str or int): The CRS of the input geometry.
zoom (int): The zoom level of the output raster.
fill_value (int or float): Value to burn into pixels intersectiong geometry
cell_type (str or :class:`~geopyspark.geotrellis.constants.CellType`): Which data type the
cells should be when created. Defaults to ``CellType.FLOAT64``.
options (:class:`~geopyspark.geotrellis.RasterizerOptions`): Pixel intersection options.
Returns:
:class:`~geopyspark.geotrellis.rdd.TiledRasterLayer`
"""
if isinstance(crs, int):
crs = str(crs)
pysc = get_spark_context()
wkb_geoms = [shapely.wkb.dumps(g) for g in geoms]
srdd = pysc._gateway.jvm.geopyspark.geotrellis.SpatialTiledRasterLayer.rasterizeGeometry(
pysc._jsc.sc(), wkb_geoms, crs, zoom, float(fill_value),
CellType(cell_type).value, options, num_partitions)
return TiledRasterLayer(LayerType.SPATIAL, srdd)
def euclidean_distance(geometry, source_crs, zoom, cell_type=CellType.FLOAT64):
"""Calculates the Euclidean distance of a Shapely geometry.
Args:
geometry (shapely.geometry): The input geometry to compute the Euclidean distance
for.
source_crs (str or int): The CRS of the input geometry.
zoom (int): The zoom level of the output raster.
cell_type (str or :class:`~geopyspark.geotrellis.constants.CellType`, optional): The data
type of the cells for the new layer. If not specified, then ``CellType.FLOAT64`` is used.
Note:
This function may run very slowly for polygonal inputs if they cover many cells of
the output raster.
Returns:
:class:`~geopyspark.geotrellis.rdd.TiledRasterLayer`
"""
if isinstance(source_crs, int):
source_crs = str(source_crs)
pysc = get_spark_context()
srdd = pysc._gateway.jvm.geopyspark.geotrellis.SpatialTiledRasterLayer.euclideanDistance(
pysc._jsc.sc(), shapely.wkb.dumps(geometry), source_crs,
CellType(cell_type).value, zoom)
return TiledRasterLayer(LayerType.SPATIAL, srdd)
def _create_spacetime_layer(cells: np.ndarray = None) -> TiledRasterLayer:
# TODO all these "create_spacetime_layer" functions are duplicated across all tests
# and better should be moved to some kind of general factory or test fixture
assert len(cells.shape) == 4
tile = Tile.from_numpy_array(cells, -1)
layer = [(SpaceTimeKey(0, 0, now), tile), (SpaceTimeKey(1, 0, now), tile),
(SpaceTimeKey(0, 1, now), tile), (SpaceTimeKey(1, 1, now), tile)]
rdd = SparkContext.getOrCreate().parallelize(layer)
metadata = {
'cellType': 'int32ud-1',
'extent': extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {
'col': 0,
'row': 0,
'instant': _convert_to_unix_time(now)
},
'maxKey': {
'col': 1,
'row': 1,
'instant': _convert_to_unix_time(now)
}
},
'layoutDefinition': {
'extent': extent,
'tileLayout': layout
}
}
return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
def create_spacetime_layer(self):
cells = np.array([self.first, self.second], dtype='int')
tile = Tile.from_numpy_array(cells, -1)
layer = [(SpaceTimeKey(0, 0, self.now), tile),
(SpaceTimeKey(1, 0, self.now), tile),
(SpaceTimeKey(0, 1, self.now), tile),
(SpaceTimeKey(1, 1, self.now), tile)]
rdd = SparkContext.getOrCreate().parallelize(layer)
metadata = {'cellType': 'int32ud-1',
'extent': self.extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(self.now)},
'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(self.now)}
},
'layoutDefinition': {
'extent': self.extent,
'tileLayout': self.layout
}
}
return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
def create_spatial_layer(self):
cells = np.array([self.first, self.second], dtype='int')
tile = Tile.from_numpy_array(cells, -1)
layer = [(SpatialKey(0, 0), tile), (SpatialKey(1, 0), tile),
(SpatialKey(0, 1), tile), (SpatialKey(1, 1), tile)]
rdd = BaseTestClass.pysc.parallelize(layer)
metadata = {
'cellType': 'int32ud-1',
'extent': self.extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {
'col': 0,
'row': 0
},
'maxKey': {
'col': 1,
'row': 1
}
},
'layoutDefinition': {
'extent': self.extent,
'tileLayout': self.layout
}
}
return TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd,
metadata)
def imagecollection_with_two_bands_and_three_dates_webmerc(request):
from geopyspark.geotrellis import (SpaceTimeKey, Tile, _convert_to_unix_time)
from geopyspark.geotrellis.constants import LayerType
from geopyspark.geotrellis.layer import TiledRasterLayer
import geopyspark as gps
from openeogeotrellis.geopysparkdatacube import GeopysparkDataCube,GeopysparkCubeMetadata
date1, date3, rdd = numpy_rdd_two_bands_and_three_dates()
metadata = {'cellType': 'int32ud-1',
'extent': extent_webmerc,
'crs': '+proj=merc +a=6378137 +b=6378137 +lat_ts=0 +lon_0=0 +x_0=0 +y_0=0 +k=1 +units=m +no_defs ',
'bounds': {
'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(date1)},
'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(date3)}
},
'layoutDefinition': {
'extent': extent_webmerc,
'tileLayout': layout
}
}
geopyspark_layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
datacube = GeopysparkDataCube(pyramid=gps.Pyramid({0: geopyspark_layer}), metadata=GeopysparkCubeMetadata(openeo_metadata))
if request.instance:
request.instance.imagecollection_with_two_bands_and_three_dates = datacube
return datacube
def test_space_time_keys(self):
temp_keys = [
SpaceTimeKey(0, 0, instant=self.time),
SpaceTimeKey(0, 1, instant=self.time)
]
temp_key_layer = [(temp_keys[0], self.tile_2),
(temp_keys[1], self.tile_2),
(temp_keys[0], self.tile_2),
(temp_keys[1], self.tile_2)]
temp_bounds = Bounds(temp_keys[0], temp_keys[1])
temp_md = Metadata(bounds=temp_bounds,
crs=self.md_proj,
cell_type=self.ct,
extent=self.extent,
layout_definition=self.ld)
rdd = self.pysc.parallelize(temp_key_layer)
layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd,
temp_md)
actual = layer.merge()
self.assertEqual(actual.srdd.rdd().count(), 2)
for k, v in actual.to_numpy_rdd().collect():
self.assertTrue((v.cells == self.arr_2).all())
def hillshade(tiled_raster_layer,
band=0,
azimuth=315.0,
altitude=45.0,
z_factor=1.0):
"""Computes Hillshade (shaded relief) from a raster.
The resulting raster will be a shaded relief map (a hill shading) based
on the sun altitude, azimuth, and the z factor. The z factor is a
conversion factor from map units to elevation units.
Returns a raster of ShortConstantNoDataCellType.
For descriptions of parameters, please see Esri Desktop's
`description <http://goo.gl/DtVDQ>`_ of Hillshade.
Args:
band (int): The band of the raster to base the hillshade calculation on. Default is 0.
azimuth (float): The azimuth angle of the source of light. Default value is 315.0.
altitude (float): The angle of the altitude of the light above the horizon. Default is
45.0.
z_factor (float): How many x and y units in a single z unit. Default value is 1.0.
Returns:
:class:`~geopyspark.geotrellis.layer.TiledRasterLayer`
"""
srdd = tiled_raster_layer.srdd.hillshade(tiled_raster_layer.pysc._jsc.sc(),
azimuth, altitude, z_factor, band)
return TiledRasterLayer(tiled_raster_layer.layer_type, srdd)
def test_bin_counts(self):
metadata2 = {'cellType': 'int32ud-500',
'extent': self.extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {'col': 0, 'row': 0},
'maxKey': {'col': 0, 'row': 0}},
'layoutDefinition': {
'extent': self.extent,
'tileLayout': {'tileCols': 4, 'tileRows': 4, 'layoutCols': 1, 'layoutRows': 1}}}
arr2 = np.int8([[[1, 1, 1, 1],
[3, 1, 1, 1],
[4, 3, 1, 1],
[5, 4, 3, 1]]])
tile2 = Tile(arr2, 'INT', -500)
rdd2 = BaseTestClass.pysc.parallelize([(self.spatial_key, tile2)])
tiled2 = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd2,
metadata2)
hist2 = tiled2.get_class_histogram()
bin_counts = hist2.bin_counts()
self.assertEqual(bin_counts, [(1, 10), (3, 3), (4, 2), (5, 1)])
def test_spatial_keys(self):
keys = [
SpatialKey(0, 0),
SpatialKey(0, 1),
SpatialKey(1, 0),
SpatialKey(1, 1)
]
key_layer = [(keys[0], self.tile), (keys[1], self.tile),
(keys[2], self.tile), (keys[3], self.tile)]
bounds = Bounds(keys[0], keys[3])
md = Metadata(bounds=bounds,
crs=self.md_proj,
cell_type=self.ct,
extent=self.extent,
layout_definition=self.ld)
rdd = self.pysc.parallelize(key_layer)
layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, md)
actual = layer.collect_keys()
for x in actual:
self.assertTrue(x in keys)
def test_space_time_keys(self):
temp_keys = [
SpaceTimeKey(0, 0, instant=self.time),
SpaceTimeKey(0, 1, instant=self.time),
SpaceTimeKey(1, 0, instant=self.time),
SpaceTimeKey(1, 1, instant=self.time)
]
temp_key_layer = [(temp_keys[0], self.tile), (temp_keys[1], self.tile),
(temp_keys[2], self.tile), (temp_keys[3], self.tile)]
temp_bounds = Bounds(temp_keys[0], temp_keys[3])
temp_md = Metadata(bounds=temp_bounds,
crs=self.md_proj,
cell_type=self.ct,
extent=self.extent,
layout_definition=self.ld)
rdd = self.pysc.parallelize(temp_key_layer)
layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd,
temp_md)
actual = layer.collect_keys()
for x in actual:
self.assertTrue(x in temp_keys)
class StitchTest(BaseTestClass):
cells = np.array([[[1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 0.0]]])
layer = [(SpatialKey(0, 0), Tile(cells, 'FLOAT', -1.0)),
(SpatialKey(1, 0), Tile(
cells,
'FLOAT',
-1.0,
)), (SpatialKey(0, 1), Tile(
cells,
'FLOAT',
-1.0,
)), (SpatialKey(1, 1), Tile(
cells,
'FLOAT',
-1.0,
))]
rdd = BaseTestClass.pysc.parallelize(layer)
extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0}
layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5}
metadata = {
'cellType': 'float32ud-1.0',
'extent': extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {
'col': 0,
'row': 0
},
'maxKey': {
'col': 1,
'row': 1
}
},
'layoutDefinition': {
'extent': extent,
'tileLayout': {
'tileCols': 5,
'tileRows': 5,
'layoutCols': 2,
'layoutRows': 2
}
}
}
raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd,
metadata)
@pytest.fixture(scope='class', autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_stitch(self):
result = self.raster_rdd.stitch()
self.assertTrue(result.cells.shape == (1, 10, 10))
def combine_bands(layers):
"""Combines the bands of values that share the same key in two or more ``TiledRasterLayer``\s.
This method will concat the bands of two or more values with the same key. For example,
``layer a`` has values that have 2 bands and ``layer b`` has values with 1 band. When
``combine_bands`` is used on both of these layers, then the resulting layer will have
values with 3 bands, 2 from ``layer a`` and 1 from ``layer b``.
Note:
All layers must have the same ``layer_type``. If the layers are ``TiledRasterLayer``\s,
then all of the layers must also have the same :class:`~geopyspark.geotrellis.TileLayout`
and ``CRS``.
Args:
layers ([:class:`~geopyspark.RasterLayer`] or [:class:`~geopyspark.TiledRasterLayer`] or (:class:`~geopyspark.RasterLayer`) or (:class:`~geopyspark.TiledRasterLayer`)): A
colection of two or more ``RasterLayer``\s or ``TiledRasterLayer``\s. **The order of the
layers determines the order in which the bands are concatenated**. With the bands being
ordered based on the position of their respective layer.
For example, the first layer in ``layers`` is ``layer a`` which contains 2 bands and
the second layer is ``layer b`` whose values have 1 band. The resulting layer will
have values with 3 bands: the first 2 are from ``layer a`` and the third from ``layer b``.
If the positions of ``layer a`` and ``layer b`` are reversed, then the resulting values'
first band will be from ``layer b`` and the last 2 will be from ``layer a``.
Returns:
:class:`~geopyspark.RasterLayer` or :class:`~geopyspark.TiledRasterLayer`
"""
if len(layers) == 1:
raise ValueError(
"combine_bands can only be performed on 2 or more layers")
base_layer = layers[0]
base_layer_type = base_layer.layer_type
check_layers(base_layer, base_layer_type, layers)
pysc = get_spark_context()
if isinstance(base_layer, RasterLayer):
if base_layer_type == LayerType.SPATIAL:
result = pysc._gateway.jvm.geopyspark.geotrellis.ProjectedRasterLayer.combineBands(
pysc._jsc.sc(), [x.srdd for x in layers])
else:
result = pysc._gateway.jvm.geopyspark.geotrellis.TemporalRasterLayer.combineBands(
pysc._jsc.sc(), [x.srdd for x in layers])
return RasterLayer(base_layer_type, result)
else:
if base_layer_type == LayerType.SPATIAL:
result = pysc._gateway.jvm.geopyspark.geotrellis.SpatialTiledRasterLayer.combineBands(
pysc._jsc.sc(), [x.srdd for x in layers])
else:
result = pysc._gateway.jvm.geopyspark.geotrellis.TemporalTiledRasterLayer.combineBands(
pysc._jsc.sc(), [x.srdd for x in layers])
return TiledRasterLayer(base_layer_type, result)
class HillshadeTest(BaseTestClass):
cells = np.array([[[1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 3.0, 3.0, 2.0, 1.0, -1.0],
[1.0, 1.0, 3.0, 2.0, 2.0, 2.0],
[1.0, 2.0, 2.0, 2.0, 2.0, 2.0],
[1.0, 1.0, 1.0, 2.0, 2.0, 2.0],
[1.0, 1.0, 1.0, 1.0, 1.0, 2.0]]])
layer = [(SpatialKey(0, 0), Tile(cells, 'FLOAT', -1.0))]
rdd = BaseTestClass.pysc.parallelize(layer)
extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0}
layout = {'layoutCols': 1, 'layoutRows': 1, 'tileCols': 6, 'tileRows': 6}
metadata = {
'cellType': 'float32ud-1.0',
'extent': extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {
'col': 0,
'row': 0
},
'maxKey': {
'col': 0,
'row': 0
}
},
'layoutDefinition': {
'extent': extent,
'tileLayout': {
'tileCols': 6,
'tileRows': 6,
'layoutCols': 1,
'layoutRows': 1
}
}
}
raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd,
metadata)
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_hillshade(self):
calc = zfactor_lat_lng_calculator(Unit.METERS)
result = hillshade(self.raster_rdd,
calc,
band=0,
azimuth=99.0,
altitude=33.0)
data = result.to_numpy_rdd().first()[1].cells[0][0][0]
self.assertEqual(data, 63)
def test_add_int(self):
arr = np.zeros((1, 4, 4))
tile = Tile(arr, 'FLOAT', -500)
rdd = BaseTestClass.pysc.parallelize([(self.spatial_key, tile)])
tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, self.metadata)
result = tiled + 1
actual = result.to_numpy_rdd().first()[1].cells
self.assertTrue((actual == 1).all())
def test_rpow_double(self):
arr = np.full((1, 4, 4), 3.0, dtype='int64')
tile = Tile(arr, 'FLOAT', -500)
rdd = BaseTestClass.pysc.parallelize([(self.spatial_key, tile)])
tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, self.metadata)
result = 0.0 ** tiled
actual = result.to_numpy_rdd().first()[1].cells
self.assertTrue((actual == 0.0).all())
def test_mode(self):
arr2 = np.array([[[1.0, 1.0, 1.0, 1.0],
[2.0, 2.0, 2.0, 2.0],
[1.0, 3.0, 3.0, 3.0],
[4.0, 4.0, 4.0, 4.0]]], dtype=float)
tile2 = Tile(arr2, 'FLOAT', -500)
rdd2 = BaseTestClass.pysc.parallelize([(self.spatial_key, tile2)])
tiled2 = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd2, self.metadata)
hist2 = tiled2.get_histogram()
self.assertEqual(hist2.mode(), 1.0)
class MaskTest(BaseTestClass):
pysc = BaseTestClass.pysc
cells = np.zeros((1, 2, 2))
cells.fill(1)
layer = [(SpatialKey(0, 0), Tile(cells, 'FLOAT', -1.0)),
(SpatialKey(1, 0), Tile(cells, 'FLOAT', -1.0,)),
(SpatialKey(0, 1), Tile(cells, 'FLOAT', -1.0,)),
(SpatialKey(1, 1), Tile(cells, 'FLOAT', -1.0,))]
rdd = pysc.parallelize(layer)
extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 4.0, 'ymax': 4.0}
layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 2, 'tileRows': 2}
metadata = {'cellType': 'float32ud-1.0',
'extent': extent,
'crs': 4326,
'bounds': {
'minKey': {'col': 0, 'row': 0},
'maxKey': {'col': 1, 'row': 1}},
'layoutDefinition': {
'extent': extent,
'tileLayout': layout}}
geoms = [box(0.0, 0.0, 2.0, 2.0), box(3.0, 3.0, 4.0, 4.0)]
raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, Metadata.from_dict(metadata))
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_geotrellis_mask(self):
result = self.raster_rdd.mask(geometries=self.geoms).to_numpy_rdd()
n = result.map(lambda kv: np.sum(kv[1].cells)).reduce(lambda a, b: a + b)
self.assertEqual(n, 2.0)
def test_rdd_mask_no_partition_strategy(self):
rdd = BaseTestClass.pysc.parallelize(self.geoms)
result = self.raster_rdd.mask(rdd, options=RasterizerOptions(True, 'PixelIsArea')).to_numpy_rdd()
n = result.map(lambda kv: np.sum(kv[1].cells)).reduce(lambda a, b: a + b)
self.assertEqual(n, 2.0)
def test_rdd_mask_with_partition_strategy(self):
rdd = BaseTestClass.pysc.parallelize(self.geoms)
result = self.raster_rdd.mask(rdd, partition_strategy=SpatialPartitionStrategy()).to_numpy_rdd()
n = result.map(lambda kv: np.sum(kv[1].cells)).reduce(lambda a, b: a + b)
self.assertEqual(n, 2.0)
def read(self, layout, read_method, target_crs=None, multiplex=False):
expected_tiled = self.rdd.tile_to_layout(layout, target_crs=target_crs)
expected_collected = expected_tiled.to_numpy_rdd().collect()
if multiplex:
sources = [
SourceInfo(self.path, {0: 0}),
SourceInfo(self.path, {0: 1})
]
actual_tiled = TiledRasterLayer.read(sources,
layout_type=layout,
target_crs=target_crs)
else:
actual_tiled = TiledRasterLayer.read([self.path],
layout_type=layout,
target_crs=target_crs)
actual_collected = actual_tiled.to_numpy_rdd().collect()
self.assertEqual(len(expected_collected), len(actual_collected))
expected_collected.sort(key=lambda tup: (tup[0].col, tup[0].row))
actual_collected.sort(key=lambda tup: (tup[0].col, tup[0].row))
if multiplex:
bands = (0, 1)
else:
bands = [0]
for expected, actual in zip(expected_collected, actual_collected):
for x in bands:
self.assertEqual(expected[0], actual[0])
self.assertTrue(expected[1].cells.shape[1:] == actual[1].cells[
x, :, :].shape)
diff = abs(expected[1].cells - actual[1].cells[x, :, :])
off_values_count = (diff > self.difference).sum()
self.assertTrue(
off_values_count / expected[1].cells.size <= 0.025)
def rdd_of_rasters(cls, uri, extent, days, num_partitions=None):
if not isinstance(uri, str):
raise Exception
sc = get_spark_context()
int_days = list(map(lambda day: int(day), days))
float_extent = list(map(lambda coord: float(coord), extent))
jvm = sc._gateway.jvm
rdd = jvm.geopyspark.netcdf.datasets.Gddp.rasters(
uri, float_extent, int_days, num_partitions, sc._jsc.sc())
return TiledRasterLayer(LayerType.SPACETIME, rdd)
def _single_pixel_layer(self, grid_value_by_datetime, no_data=-1.0):
from collections import OrderedDict
sorted_by_datetime = OrderedDict(sorted(
grid_value_by_datetime.items()))
def elem(timestamp, value):
tile = self._single_pixel_tile(value, no_data)
return [(SpaceTimeKey(0, 0, timestamp), tile)]
layer = [
elem(timestamp, value)
for timestamp, value in sorted_by_datetime.items()
]
rdd = SparkContext.getOrCreate().parallelize(layer)
datetimes = list(sorted_by_datetime.keys())
extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 1.0, 'ymax': 1.0}
layout = {
'layoutCols': 1,
'layoutRows': 1,
'tileCols': 1,
'tileRows': 1
}
metadata = {
'cellType': 'float32ud%f' % no_data,
'extent': extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {
'col': 0,
'row': 0,
'instant': _convert_to_unix_time(datetimes[0])
},
'maxKey': {
'col': 0,
'row': 0,
'instant': _convert_to_unix_time(datetimes[-1])
}
},
'layoutDefinition': {
'extent': extent,
'tileLayout': layout
}
}
return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd,
metadata)
def _create_spacetime_layer(self, no_data):
def tile(value):
cells = np.zeros((4, 4), dtype=float)
cells.fill(value)
return Tile.from_numpy_array(cells, no_data)
tiles = [(SpaceTimeKey(0, 0, self.now), tile(0)),
(SpaceTimeKey(1, 0, self.now), tile(1)),
(SpaceTimeKey(0, 1, self.now), tile(2)),
(SpaceTimeKey(1, 1, self.now), tile(no_data))]
for tile in tiles:
print(tile)
layout = {
'layoutCols': 2,
'layoutRows': 2,
'tileCols': 4,
'tileRows': 4
}
extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 8.0, 'ymax': 8.0}
rdd = SparkContext.getOrCreate().parallelize(tiles)
print(rdd.count())
metadata = {
'cellType': 'float64ud-1',
'extent': extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {
'col': 0,
'row': 0,
'instant': _convert_to_unix_time(self.now)
},
'maxKey': {
'col': 1,
'row': 1,
'instant': _convert_to_unix_time(self.now)
}
},
'layoutDefinition': {
'extent': extent,
'tileLayout': layout
}
}
return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd,
metadata)
def test_combined_operations(self):
arr = np.array([[[10, 10, 10, 10],
[20, 20, 20, 20],
[10, 10, 10, 10],
[20, 20, 20, 20]]], dtype=int)
tile = Tile(arr, 'INT', -500)
rdd = BaseTestClass.pysc.parallelize([(self.spatial_key, tile)])
tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, self.metadata)
result = (tiled + tiled) / 2
actual = result.to_numpy_rdd().first()[1].cells
self.assertTrue((actual == arr).all())
def test_divide_tiled_rdd(self):
arr = np.array([[[5.0, 5.0, 5.0, 5.0],
[5.0, 5.0, 5.0, 5.0],
[5.0, 5.0, 5.0, 5.0],
[5.0, 5.0, 5.0, 5.0]]], dtype=float)
divider = np.array([[[1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0]]], dtype=float)
tile = Tile(arr, 'FLOAT', float('nan'))
tile2 = Tile(divider, 'FLOAT', float('nan'))
rdd = BaseTestClass.pysc.parallelize([(self.spatial_key, tile)])
rdd2 = BaseTestClass.pysc.parallelize([(self.spatial_key, tile2)])
tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, self.metadata)
tiled2 = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd2, self.metadata)
result = tiled / tiled2
actual = result.to_numpy_rdd().first()[1].cells
self.assertTrue((actual == 5.0).all())
def test_from_histogram(self):
extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0}
layout = {
'layoutCols': 1,
'layoutRows': 1,
'tileCols': 2,
'tileRows': 4
}
metadata = {
'cellType': 'float32ud-1.0',
'extent': extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {
'col': 0,
'row': 0
},
'maxKey': {
'col': 0,
'row': 0
}
},
'layoutDefinition': {
'extent': extent,
'tileLayout': {
'tileCols': 2,
'tileRows': 4,
'layoutCols': 1,
'layoutRows': 1
}
}
}
spatial_key = SpatialKey(0, 0)
arr = np.array([[[1.0, 5.0, 2.0, 3.0], [4.0, 6.0, 7.0, 0.0]]],
dtype=float)
tile = Tile(arr, 'FLOAT', -500)
rdd = BaseTestClass.pysc.parallelize([(spatial_key, tile)])
tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd,
metadata)
hist = tiled.get_histogram()
color_list = self.color_list
result = ColorMap.from_histogram(hist, color_list)
self.assertTrue(isinstance(result, ColorMap))
def test_multiply_double(self):
arr = np.array([[[1.0, 1.0, 1.0, 1.0],
[2.0, 2.0, 2.0, 2.0],
[3.0, 3.0, 3.0, 3.0],
[4.0, 4.0, 4.0, 4.0]]], dtype=float)
tile = Tile(arr, 'FLOAT', float('nan'))
rdd = BaseTestClass.pysc.parallelize([(self.spatial_key, tile)])
tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, self.metadata)
result = 5.0 * tiled
actual = result.to_numpy_rdd().first()[1].cells
expected = np.array([[[5.0, 5.0, 5.0, 5.0],
[10.0, 10.0, 10.0, 10.0],
[15.0, 15.0, 15.0, 15.0],
[20.0, 20.0, 20.0, 20.0]]], dtype=float)
self.assertTrue((actual == expected).all())
def imagecollection_with_two_bands_and_one_date(request):
import geopyspark as gps
from geopyspark.geotrellis import (SpaceTimeKey, Tile, _convert_to_unix_time)
from geopyspark.geotrellis.constants import LayerType
from geopyspark.geotrellis.layer import TiledRasterLayer
from pyspark import SparkContext
from openeogeotrellis.geopysparkdatacube import GeopysparkDataCube
print(request)
two_band_one_two = np.array([matrix_of_one, matrix_of_two], dtype='int')
tile = Tile.from_numpy_array(two_band_one_two, -1)
date1 = datetime.datetime.strptime("2017-09-25T11:37:00Z", '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC)
layer = [(SpaceTimeKey(0, 0, date1), tile),
(SpaceTimeKey(1, 0, date1), tile),
(SpaceTimeKey(0, 1, date1), tile),
(SpaceTimeKey(1, 1, date1), tile)]
rdd = SparkContext.getOrCreate().parallelize(layer)
metadata = {'cellType': 'int32ud-1',
'extent': extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(date1)},
'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(date1)}
},
'layoutDefinition': {
'extent': extent,
'tileLayout': layout
}
}
geopyspark_layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
datacube = GeopysparkDataCube(pyramid=gps.Pyramid({0: geopyspark_layer}), metadata=openeo_metadata)
if request.instance:
request.instance.imagecollection_with_two_bands_and_one_date = datacube
return datacube
def hillshade(tiled_raster_layer,
zfactor_calculator,
band=0,
azimuth=315.0,
altitude=45.0):
"""Computes Hillshade (shaded relief) from a raster.
The resulting raster will be a shaded relief map (a hill shading) based
on the sun altitude, azimuth, and the ``zfactor``. The ``zfactor`` is a
conversion factor from map units to elevation units.
The ``hillshade``` operation will be carried out in a ``SQUARE`` neighborhood with with an
``extent`` of 1. The ``zfactor`` will be derived from the ``zfactor_calculator``
for each ``Tile`` in the Layer. The resulting Layer will have a ``cell_type``
of ``INT16`` regardless of the input Layer's ``cell_type``; as well as
have a single band, that represents the calculated ``hillshade``.
Returns a raster of ShortConstantNoDataCellType.
For descriptions of parameters, please see Esri Desktop's
`description <http://goo.gl/DtVDQ>`_ of Hillshade.
Args:
tiled_raster_layer (:class:`~geopyspark.geotrellis.layer.TiledRasterLayer`): The base layer
that contains the rasters used to compute the hillshade.
zfactor_calculator (py4j.JavaObject): A ``JavaObject`` that represents the
Scala ``ZFactorCalculator`` class. This can be created using either the
:meth:`~geopyspark.geotrellis.zfactor_lat_lng_calculator` or the
:meth:`~geopyspark.geotrellis.zfactor_calculator` methods.
band (int, optional): The band of the raster to base the hillshade calculation on. Default is 0.
azimuth (float, optional): The azimuth angle of the source of light. Default value is 315.0.
altitude (float, optional): The angle of the altitude of the light above the horizon. Default is
45.0.
Returns:
:class:`~geopyspark.geotrellis.layer.TiledRasterLayer`
"""
srdd = tiled_raster_layer.srdd.hillshade(azimuth, altitude,
zfactor_calculator, band)
return TiledRasterLayer(tiled_raster_layer.layer_type, srdd)
def create_spacetime_unsigned_byte_layer(self):
"""
Returns a single-band uint8ud255 layer consisting of four tiles that each look like this:
ND 220 220 220
220 220 220 220
220 220 220 220
220 220 220 220
The extent is (0.0, 0.0) to (4.0, 4.0).
"""
no_data = 255
single_band = np.zeros((1, 4, 4))
single_band.fill(220)
single_band[0, 0, 0] = no_data
cells = np.array([single_band], dtype='uint8')
tile = Tile.from_numpy_array(cells, no_data)
layer = [(SpaceTimeKey(0, 0, self.now), tile),
(SpaceTimeKey(1, 0, self.now), tile),
(SpaceTimeKey(0, 1, self.now), tile),
(SpaceTimeKey(1, 1, self.now), tile)]
rdd = SparkContext.getOrCreate().parallelize(layer)
metadata = {
'cellType': 'uint8ud255',
'extent': self.extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(self.now)},
'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(self.now)}
},
'layoutDefinition': {
'extent': self.extent,
'tileLayout': self.layout
}
}
return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)