def numpy_rdd_two_bands_and_three_dates():
from geopyspark.geotrellis import (SpaceTimeKey, Tile, _convert_to_unix_time)
from pyspark import SparkContext
two_band_one_two = np.array([matrix_of_one, matrix_of_two], dtype='int')
first_tile = Tile.from_numpy_array(two_band_one_two, -1)
second_tile = Tile.from_numpy_array(np.array([matrix_of_two, matrix_of_one], dtype='int'), -1)
nodata_tile = Tile.from_numpy_array(np.array([matrix_of_nodata, matrix_of_nodata], dtype='int'), -1)
date1 = datetime.datetime.strptime("2017-09-25T11:37:00Z", '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC)
date2 = datetime.datetime.strptime("2017-09-30T00:37:00Z", '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC)
date3 = datetime.datetime.strptime("2017-10-25T11:37:00Z", '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.UTC)
layer = [(SpaceTimeKey(0, 0, date1), first_tile),
(SpaceTimeKey(1, 0, date1), first_tile),
(SpaceTimeKey(0, 1, date1), first_tile),
(SpaceTimeKey(1, 1, date1), first_tile),
(SpaceTimeKey(0, 0, date2), nodata_tile),
(SpaceTimeKey(1, 0, date2), nodata_tile),
(SpaceTimeKey(0, 1, date2), nodata_tile),
(SpaceTimeKey(1, 1, date2), nodata_tile),
(SpaceTimeKey(0, 0, date3), second_tile),
(SpaceTimeKey(1, 0, date3), second_tile),
(SpaceTimeKey(0, 1, date3), second_tile),
(SpaceTimeKey(1, 1, date3), second_tile)
]
rdd = SparkContext.getOrCreate().parallelize(layer)
return date1, date3, rdd
class CellValueCountsTest(BaseTestClass):
pysc = BaseTestClass.pysc
cells = np.array([[
[1.0, 1.0, 1.0, 1.0, 1.0],
[2.0, 2.0, 2.0, 2.0, 2.0],
[3.0, 3.0, 3.0, 3.0, 3.0],
[4.0, 4.0, 4.0, 4.0, 4.0],
[5.0, 5.0, 5.0, 5.0, 5.0]]])
layer = [(SpatialKey(0, 0), Tile(cells, 'FLOAT', -1.0)),
(SpatialKey(1, 0), Tile(cells, 'FLOAT', -1.0,)),
(SpatialKey(0, 1), Tile(cells, 'FLOAT', -1.0,)),
(SpatialKey(1, 1), Tile(cells, 'FLOAT', -1.0,))]
rdd = pysc.parallelize(layer)
extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 4.0, 'ymax': 4.0}
layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5}
metadata = {'cellType': 'float32ud-1.0',
'extent': extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {'col': 0, 'row': 0},
'maxKey': {'col': 1, 'row': 1}},
'layoutDefinition': {
'extent': extent,
'tileLayout': layout}}
raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, metadata)
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_counts_with_no_area(self):
actual = self.raster_rdd.get_cell_value_counts()
expected = {
1: 20,
2: 20,
3: 20,
4: 20,
5: 20
}
self.assertDictEqual(actual, expected)
def test_counts_with_polygon(self):
area_of_interest = box(0.0, 0.0, 2.0, 2.0)
actual = self.raster_rdd.get_cell_value_counts(area_of_interest)
expected = {
1: 5,
2: 5,
3: 5,
4: 5,
5: 5
}
self.assertDictEqual(actual, expected)
class ByteTileSchemaTest(BaseTestClass):
tiles = [
Tile.from_numpy_array(np.int8([0, 0, 1, 1]).reshape(2, 2), -128),
Tile.from_numpy_array(np.int8([1, 2, 3, 4]).reshape(2, 2), -128),
Tile.from_numpy_array(np.int8([5, 6, 7, 8]).reshape(2, 2), -128)
]
sc = BaseTestClass.pysc._jsc.sc()
tw = BaseTestClass.pysc._jvm.geopyspark.geotrellis.tests.schemas.ByteArrayTileWrapper
java_rdd = tw.testOut(sc)
ser = ProtoBufSerializer(tile_decoder, tile_encoder)
rdd = RDD(java_rdd, BaseTestClass.pysc, AutoBatchedSerializer(ser))
collected = rdd.collect()
def test_encoded_tiles(self):
expected_encoded = [to_pb_tile(x) for x in self.collected]
for actual, expected in zip(self.tiles, expected_encoded):
cells = actual.cells
rows, cols = cells.shape
self.assertEqual(expected.cols, cols)
self.assertEqual(expected.rows, rows)
self.assertEqual(expected.cellType.nd, actual.no_data_value)
self.assertEqual(expected.cellType.dataType,
mapped_data_types[actual.cell_type])
def test_decoded_tiles(self):
for actual, expected in zip(self.collected, self.tiles):
self.assertTrue((actual.cells == expected.cells).all())
self.assertTrue(actual.cells.dtype == expected.cells.dtype)
self.assertEqual(actual.cells.shape, actual.cells.shape)
class StitchTest(BaseTestClass):
cells = np.array([[[1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 0.0]]])
layer = [(SpatialKey(0, 0), Tile(cells, 'FLOAT', -1.0)),
(SpatialKey(1, 0), Tile(
cells,
'FLOAT',
-1.0,
)), (SpatialKey(0, 1), Tile(
cells,
'FLOAT',
-1.0,
)), (SpatialKey(1, 1), Tile(
cells,
'FLOAT',
-1.0,
))]
rdd = BaseTestClass.pysc.parallelize(layer)
extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0}
layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 5, 'tileRows': 5}
metadata = {
'cellType': 'float32ud-1.0',
'extent': extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {
'col': 0,
'row': 0
},
'maxKey': {
'col': 1,
'row': 1
}
},
'layoutDefinition': {
'extent': extent,
'tileLayout': {
'tileCols': 5,
'tileRows': 5,
'layoutCols': 2,
'layoutRows': 2
}
}
}
raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd,
metadata)
@pytest.fixture(scope='class', autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_stitch(self):
result = self.raster_rdd.stitch()
self.assertTrue(result.cells.shape == (1, 10, 10))
class MultibandSchemaTest(BaseTestClass):
arr = np.int8([0, 0, 1, 1]).reshape(2, 2)
no_data = -128
arr_dict = Tile(arr, 'BYTE', no_data)
band_dicts = [arr_dict, arr_dict, arr_dict]
bands = [arr, arr, arr]
multiband_tile = np.array(bands)
multiband_dict = Tile(multiband_tile, 'BYTE', no_data)
sc = BaseTestClass.pysc._jsc.sc()
mw = BaseTestClass.pysc._jvm.geopyspark.geotrellis.tests.schemas.ArrayMultibandTileWrapper
java_rdd = mw.testOut(sc)
ser = ProtoBufSerializer(multibandtile_decoder, multibandtile_encoder)
rdd = RDD(java_rdd, BaseTestClass.pysc, AutoBatchedSerializer(ser))
collected = rdd.collect()
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_encoded_multibands(self):
actual_encoded = [multibandtile_encoder(x) for x in self.collected]
proto_tile = tileMessages_pb2.ProtoTile()
cell_type = tileMessages_pb2.ProtoCellType()
cell_type.nd = self.no_data
cell_type.hasNoData = True
cell_type.dataType = 1
proto_tile.cols = 2
proto_tile.rows = 2
proto_tile.sint32Cells.extend(self.arr.flatten().tolist())
proto_tile.cellType.CopyFrom(cell_type)
proto_multiband = tileMessages_pb2.ProtoMultibandTile()
proto_multiband.tiles.extend([proto_tile, proto_tile, proto_tile])
bs = proto_multiband.SerializeToString()
expected_encoded = [bs, bs, bs]
for actual, expected in zip(actual_encoded, expected_encoded):
self.assertEqual(actual, expected)
def test_decoded_multibands(self):
expected_multibands = [
self.multiband_dict, self.multiband_dict, self.multiband_dict
]
for actual, expected in zip(self.collected, expected_multibands):
self.assertTrue((actual.cells == expected.cells).all())
class MaskTest(BaseTestClass):
pysc = BaseTestClass.pysc
cells = np.zeros((1, 2, 2))
cells.fill(1)
layer = [(SpatialKey(0, 0), Tile(cells, 'FLOAT', -1.0)),
(SpatialKey(1, 0), Tile(cells, 'FLOAT', -1.0,)),
(SpatialKey(0, 1), Tile(cells, 'FLOAT', -1.0,)),
(SpatialKey(1, 1), Tile(cells, 'FLOAT', -1.0,))]
rdd = pysc.parallelize(layer)
extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 4.0, 'ymax': 4.0}
layout = {'layoutCols': 2, 'layoutRows': 2, 'tileCols': 2, 'tileRows': 2}
metadata = {'cellType': 'float32ud-1.0',
'extent': extent,
'crs': 4326,
'bounds': {
'minKey': {'col': 0, 'row': 0},
'maxKey': {'col': 1, 'row': 1}},
'layoutDefinition': {
'extent': extent,
'tileLayout': layout}}
geoms = [box(0.0, 0.0, 2.0, 2.0), box(3.0, 3.0, 4.0, 4.0)]
raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, Metadata.from_dict(metadata))
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_geotrellis_mask(self):
result = self.raster_rdd.mask(geometries=self.geoms).to_numpy_rdd()
n = result.map(lambda kv: np.sum(kv[1].cells)).reduce(lambda a, b: a + b)
self.assertEqual(n, 2.0)
def test_rdd_mask_no_partition_strategy(self):
rdd = BaseTestClass.pysc.parallelize(self.geoms)
result = self.raster_rdd.mask(rdd, options=RasterizerOptions(True, 'PixelIsArea')).to_numpy_rdd()
n = result.map(lambda kv: np.sum(kv[1].cells)).reduce(lambda a, b: a + b)
self.assertEqual(n, 2.0)
def test_rdd_mask_with_partition_strategy(self):
rdd = BaseTestClass.pysc.parallelize(self.geoms)
result = self.raster_rdd.mask(rdd, partition_strategy=SpatialPartitionStrategy()).to_numpy_rdd()
n = result.map(lambda kv: np.sum(kv[1].cells)).reduce(lambda a, b: a + b)
self.assertEqual(n, 2.0)
def create_spatial_layer(self):
cells = np.array([self.first, self.second], dtype='int')
tile = Tile.from_numpy_array(cells, -1)
layer = [(SpatialKey(0, 0), tile), (SpatialKey(1, 0), tile),
(SpatialKey(0, 1), tile), (SpatialKey(1, 1), tile)]
rdd = BaseTestClass.pysc.parallelize(layer)
metadata = {
'cellType': 'int32ud-1',
'extent': self.extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {
'col': 0,
'row': 0
},
'maxKey': {
'col': 1,
'row': 1
}
},
'layoutDefinition': {
'extent': self.extent,
'tileLayout': self.layout
}
}
return TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd,
metadata)
def test_bin_counts(self):
metadata2 = {'cellType': 'int32ud-500',
'extent': self.extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {'col': 0, 'row': 0},
'maxKey': {'col': 0, 'row': 0}},
'layoutDefinition': {
'extent': self.extent,
'tileLayout': {'tileCols': 4, 'tileRows': 4, 'layoutCols': 1, 'layoutRows': 1}}}
arr2 = np.int8([[[1, 1, 1, 1],
[3, 1, 1, 1],
[4, 3, 1, 1],
[5, 4, 3, 1]]])
tile2 = Tile(arr2, 'INT', -500)
rdd2 = BaseTestClass.pysc.parallelize([(self.spatial_key, tile2)])
tiled2 = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd2,
metadata2)
hist2 = tiled2.get_class_histogram()
bin_counts = hist2.bin_counts()
self.assertEqual(bin_counts, [(1, 10), (3, 3), (4, 2), (5, 1)])
def create_spacetime_layer(self):
cells = np.array([self.first, self.second], dtype='int')
tile = Tile.from_numpy_array(cells, -1)
layer = [(SpaceTimeKey(0, 0, self.now), tile),
(SpaceTimeKey(1, 0, self.now), tile),
(SpaceTimeKey(0, 1, self.now), tile),
(SpaceTimeKey(1, 1, self.now), tile)]
rdd = SparkContext.getOrCreate().parallelize(layer)
metadata = {'cellType': 'int32ud-1',
'extent': self.extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(self.now)},
'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(self.now)}
},
'layoutDefinition': {
'extent': self.extent,
'tileLayout': self.layout
}
}
return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
def test_map_tiles_lambda_tiled(self):
mapped_layer = self.tiled_raster_rdd.map_tiles(lambda tile: Tile(tile.cells[0], tile.cell_type, tile.no_data_value))
actual = mapped_layer.to_numpy_rdd().first()[1]
expected = np.array([self.band_1])
self.assertEqual(mapped_layer.zoom_level, self.tiled_raster_rdd.zoom_level)
self.assertTrue((expected == actual.cells).all())
def _create_spacetime_layer(cells: np.ndarray = None) -> TiledRasterLayer:
# TODO all these "create_spacetime_layer" functions are duplicated across all tests
# and better should be moved to some kind of general factory or test fixture
assert len(cells.shape) == 4
tile = Tile.from_numpy_array(cells, -1)
layer = [(SpaceTimeKey(0, 0, now), tile), (SpaceTimeKey(1, 0, now), tile),
(SpaceTimeKey(0, 1, now), tile), (SpaceTimeKey(1, 1, now), tile)]
rdd = SparkContext.getOrCreate().parallelize(layer)
metadata = {
'cellType': 'int32ud-1',
'extent': extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {
'col': 0,
'row': 0,
'instant': _convert_to_unix_time(now)
},
'maxKey': {
'col': 1,
'row': 1,
'instant': _convert_to_unix_time(now)
}
},
'layoutDefinition': {
'extent': extent,
'tileLayout': layout
}
}
return TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
def get_tile(self, bands: List[str], col: int, row: int,
date: datetime) -> Tile:
array = np.array([
self.get_band_tile(band=band, col=col, row=row, date=date)
for band in bands
])
return Tile.from_numpy_array(array)
def test_local_pyramid(self):
arr = np.zeros((1, 250, 250))
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
tile = Tile(arr, 'FLOAT', None)
projected_extent = ProjectedExtent(extent, epsg_code)
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
laid_out = raster_rdd.tile_to_layout(LocalLayout(250))
# Single tile is at level 0
result = laid_out.pyramid()
assert result.max_zoom == 0
laid_out = raster_rdd.tile_to_layout(LocalLayout(25))
result = laid_out.pyramid()
assert result.max_zoom == 4
assert result.levels[4].layer_metadata.tile_layout.layoutCols == 10
assert result.levels[3].layer_metadata.tile_layout.layoutCols == 5
assert result.levels[2].layer_metadata.tile_layout.layoutCols == 3
assert result.levels[1].layer_metadata.tile_layout.layoutCols == 2
assert result.levels[0].layer_metadata.tile_layout.layoutCols == 1
def test_apply_to_tile(self):
def custom_function(cells: np.ndarray, nd):
return cells[0] + cells[1]
cells = np.array([self.first, self.second], dtype='int')
tile = Tile.from_numpy_array(cells, -1)
custom_function(tile.cells, 0)
class TupleSchemaTest(BaseTestClass):
extent = {
'epsg': 2004,
'extent': {
'xmax': 1.0,
'xmin': 0.0,
'ymax': 1.0,
'ymin': 0.0
},
'proj4': None
}
arr = np.int8([0, 0, 1, 1]).reshape(2, 2)
bands = [arr, arr, arr]
multiband_tile = np.array(bands)
multiband_dict = Tile(multiband_tile, 'BYTE', -128)
sc = BaseTestClass.pysc._jsc.sc()
ew = BaseTestClass.pysc._jvm.geopyspark.geotrellis.tests.schemas.TupleWrapper
java_rdd = ew.testOut(sc)
decoder = create_partial_tuple_decoder(key_type="ProjectedExtent")
encoder = create_partial_tuple_encoder(key_type="ProjectedExtent")
ser = ProtoBufSerializer(decoder, encoder)
rdd = RDD(java_rdd, BaseTestClass.pysc, AutoBatchedSerializer(ser))
collected = rdd.collect()
@pytest.mark.skipif(
'TRAVIS' in os.environ,
reason="Encoding using methods in Main causes issues on Travis")
def test_encoded_tuples(self):
proto_tuple = tupleMessages_pb2.ProtoTuple()
self.extent['extent'] = Extent(**self.extent['extent'])
proto_extent = to_pb_projected_extent(ProjectedExtent(**self.extent))
proto_multiband = to_pb_multibandtile(self.multiband_dict)
proto_tuple.projectedExtent.CopyFrom(proto_extent)
proto_tuple.tiles.CopyFrom(proto_multiband)
bs = proto_tuple.SerializeToString()
expected_encoded = [self.ser.dumps(x) for x in self.collected]
for expected in expected_encoded:
self.assertEqual(bs, expected)
def test_decoded_tuples(self):
expected_tuples = [(self.extent, self.multiband_dict),
(self.extent, self.multiband_dict),
(self.extent, self.multiband_dict)]
for actual, expected in zip(self.collected, expected_tuples):
(actual_extent, actual_tile) = actual
(expected_extent, expected_tile) = expected
self.assertTrue((actual_tile.cells == expected_tile.cells).all())
self.assertDictEqual(actual_extent._asdict(), expected_extent)
def test_apply_openeo_udf_to_tile(self):
import os, openeo_udf
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_ndvi.py")
with open(file_name, "r") as f:
udf_code = f.read()
cells = np.array([self.first, self.second], dtype='int')
tile = Tile.from_numpy_array(cells, -1)
def test_all_zeros(self):
arr = np.zeros((1, 16, 16)).astype('int')
tile = Tile(arr, 'INT', -500)
rdd = BaseTestClass.pysc.parallelize([(self.projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
min_max = raster_rdd.get_min_max()
self.assertEqual((0.0, 0.0), min_max)
def test_no_data_of_zero(self):
no_data_layer = [(t[0], Tile.from_numpy_array(t[1].cells, 1)) for t in self.layers]
rdd = BaseTestClass.pysc.parallelize(no_data_layer)
nd_layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
nd_metadata = nd_layer.collect_metadata()
self.assertTrue('ud1' in nd_metadata.cell_type)
self.assertEqual(nd_metadata.no_data_value, 1)
class HillshadeTest(BaseTestClass):
cells = np.array([[[1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 3.0, 3.0, 2.0, 1.0, -1.0],
[1.0, 1.0, 3.0, 2.0, 2.0, 2.0],
[1.0, 2.0, 2.0, 2.0, 2.0, 2.0],
[1.0, 1.0, 1.0, 2.0, 2.0, 2.0],
[1.0, 1.0, 1.0, 1.0, 1.0, 2.0]]])
layer = [(SpatialKey(0, 0), Tile(cells, 'FLOAT', -1.0))]
rdd = BaseTestClass.pysc.parallelize(layer)
extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 33.0, 'ymax': 33.0}
layout = {'layoutCols': 1, 'layoutRows': 1, 'tileCols': 6, 'tileRows': 6}
metadata = {
'cellType': 'float32ud-1.0',
'extent': extent,
'crs': '+proj=longlat +datum=WGS84 +no_defs ',
'bounds': {
'minKey': {
'col': 0,
'row': 0
},
'maxKey': {
'col': 0,
'row': 0
}
},
'layoutDefinition': {
'extent': extent,
'tileLayout': {
'tileCols': 6,
'tileRows': 6,
'layoutCols': 1,
'layoutRows': 1
}
}
}
raster_rdd = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd,
metadata)
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_hillshade(self):
calc = zfactor_lat_lng_calculator(Unit.METERS)
result = hillshade(self.raster_rdd,
calc,
band=0,
azimuth=99.0,
altitude=33.0)
data = result.to_numpy_rdd().first()[1].cells[0][0][0]
self.assertEqual(data, 63)
def tile_decoder(proto_bytes):
"""Deserializes the ``ProtoTile`` bytes into Python.
Args:
proto_bytes (bytes): The ProtoBuf encoded bytes of the ProtoBuf class.
Returns:
:class:`~geopyspark.geotrellis.Tile`
"""
tile = ProtoTile.FromString(proto_bytes)
cell_type = _mapped_data_types[tile.cellType.dataType]
if tile.cellType.hasNoData:
nd = tile.cellType.nd
return Tile(np.array([from_pb_tile(tile, nd)]), cell_type, nd)
else:
return Tile(np.array([from_pb_tile(tile)]), cell_type, None)
def from_pb_multibandtile(multibandtile):
"""Creates a ``Tile`` from ``ProtoMultibandTile``.
Args:
multibandtile (ProtoTile): The ``ProtoMultibandTile`` instance to be converted.
Returns:
:class:`~geopyspark.geotrellis.Tile`
"""
cell_type = _mapped_data_types[multibandtile.tiles[0].cellType.dataType]
if multibandtile.tiles[0].cellType.hasNoData:
nd = multibandtile.tiles[0].cellType.nd
bands = np.array([from_pb_tile(tile, nd) for tile in multibandtile.tiles])
return Tile(bands, cell_type, nd)
else:
bands = np.array([from_pb_tile(tile) for tile in multibandtile.tiles])
return Tile(bands, cell_type, None)
def to_geotiff(x):
with tempfile.NamedTemporaryFile() as temp:
temp.write(x)
temp_path = pathlib.Path(temp.name)
with rasterio.open(str(temp_path)) as src:
self.assertFalse(src.is_tiled)
data = src.read()
return Tile(data, self.mapped_types[str(data.dtype)],
src.nodata)
def test_rpow_double(self):
arr = np.full((1, 4, 4), 3.0, dtype='int64')
tile = Tile(arr, 'FLOAT', -500)
rdd = BaseTestClass.pysc.parallelize([(self.spatial_key, tile)])
tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, self.metadata)
result = 0.0 ** tiled
actual = result.to_numpy_rdd().first()[1].cells
self.assertTrue((actual == 0.0).all())
def test_floating(self):
arr = np.array([[[0.0, 0.0, 0.0, 0.0], [1.0, 1.0, 1.0, 1.0],
[1.5, 1.5, 1.5, 1.5], [2.0, 2.0, 2.0, 2.0]]],
dtype=float)
tile = Tile(arr, 'FLOAT', float('nan'))
rdd = BaseTestClass.pysc.parallelize([(self.projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
min_max = raster_rdd.get_min_max()
self.assertEqual((0.0, 2.0), min_max)
def test_add_int(self):
arr = np.zeros((1, 4, 4))
tile = Tile(arr, 'FLOAT', -500)
rdd = BaseTestClass.pysc.parallelize([(self.spatial_key, tile)])
tiled = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, self.metadata)
result = tiled + 1
actual = result.to_numpy_rdd().first()[1].cells
self.assertTrue((actual == 1).all())
def test_multibands(self):
arr = np.array(
[[[1, 1, 1, 1]], [[2, 2, 2, 2]], [[3, 3, 3, 3]], [[4, 4, 4, 4]]],
dtype=int)
tile = Tile(arr, 'INT', -500)
rdd = BaseTestClass.pysc.parallelize([(self.projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
min_max = raster_rdd.get_min_max()
self.assertEqual((1.0, 4.0), min_max)
def test_mode(self):
arr2 = np.array([[[1.0, 1.0, 1.0, 1.0],
[2.0, 2.0, 2.0, 2.0],
[1.0, 3.0, 3.0, 3.0],
[4.0, 4.0, 4.0, 4.0]]], dtype=float)
tile2 = Tile(arr2, 'FLOAT', -500)
rdd2 = BaseTestClass.pysc.parallelize([(self.spatial_key, tile2)])
tiled2 = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd2, self.metadata)
hist2 = tiled2.get_histogram()
self.assertEqual(hist2.mode(), 1.0)
def test_collection_python_rdd(self):
data = rasterio.open(self.path)
tile_dict = Tile(data.read(), 'FLOAT', data.nodata)
rasterio_rdd = self.pysc.parallelize([(self.projected_extent, tile_dict)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rasterio_rdd)
result = raster_rdd.collect_metadata(layout=self.layout_def)
self.assertEqual(result.extent, self.extent)
self.assertEqual(result.layout_definition.extent, self.extent)
self.assertEqual(result.layout_definition.tileLayout, self.layout)
def test_list_bad(self):
arr = np.zeros((1, 16, 16))
tile = Tile(arr, 'FLOAT', -500)
rdd = BaseTestClass.pysc.parallelize([(self.projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
value_map = {'apple, orange, banana': 1}
with pytest.raises(TypeError):
result = raster_rdd.reclassify(value_map,
int).to_numpy_rdd().first()[1].cells
class UnionTemporalTest(BaseTestClass):
time_1 = datetime.datetime.strptime("1993-09-19T07:01:00Z",
'%Y-%m-%dT%H:%M:%SZ')
time_2 = datetime.datetime.strptime("2017-09-19T07:01:00Z",
'%Y-%m-%dT%H:%M:%SZ')
temp_projected_extent_1 = TemporalProjectedExtent(extent, time_1,
epsg_code)
temp_projected_extent_2 = TemporalProjectedExtent(extent, time_2,
epsg_code)
arr = np.zeros((1, 16, 16))
tile = Tile(arr, 'FLOAT', -500.0)
rdd_1 = BaseTestClass.pysc.parallelize([(temp_projected_extent_1, tile)])
rdd_2 = BaseTestClass.pysc.parallelize([(temp_projected_extent_2, tile)])
layer_1 = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd_1)
layer_2 = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd_2)
tiled_layer_1 = layer_1.tile_to_layout(GlobalLayout())
tiled_layer_2 = layer_2.tile_to_layout(GlobalLayout())
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_union_of_raster_layers(self):
result = union(self.layer_1, self.layer_2)
self.assertTrue(result.srdd.rdd().count(), 2)
def test_union_of_tiled_raster_layers(self):
result = union(self.tiled_layer_1, self.tiled_layer_2)
bounds_1 = self.tiled_layer_1.layer_metadata.bounds
bounds_2 = self.tiled_layer_2.layer_metadata.bounds
min_col = min(bounds_1.minKey.col, bounds_2.minKey.col)
min_row = min(bounds_1.minKey.row, bounds_2.minKey.row)
min_instant = min(bounds_1.minKey.instant, bounds_2.minKey.instant)
max_col = max(bounds_1.maxKey.col, bounds_2.maxKey.col)
max_row = max(bounds_1.maxKey.row, bounds_2.maxKey.row)
max_instant = max(bounds_1.maxKey.instant, bounds_2.maxKey.instant)
min_key = SpaceTimeKey(min_col, min_row, min_instant)
max_key = SpaceTimeKey(max_col, max_row, max_instant)
self.assertTrue(result.srdd.rdd().count(), 2)
self.assertEqual(result.layer_metadata.bounds,
Bounds(min_key, max_key))
