class BaseTestClass(unittest.TestCase):
if 'TRAVIS' in os.environ:
master_str = "local[2]"
else:
master_str = "local[*]"
conf = geopyspark_conf(master=master_str, appName="test")
conf.set('spark.kryoserializer.buffer.max', value='1G')
conf.set('spark.ui.enabled', True)
if 'TRAVIS' in os.environ:
conf.set(key='spark.driver.memory', value='2G')
conf.set(key='spark.executor.memory', value='2G')
pysc = SparkContext(conf=conf)
dir_path = geotiff_test_path("all-ones.tif")
rdd = get(LayerType.SPATIAL, dir_path)
value = rdd.to_numpy_rdd().collect()[0]
projected_extent = value[0]
extent = projected_extent.extent
expected_tile = value[1].cells
(_, rows, cols) = expected_tile.shape
layout = TileLayout(1, 1, cols, rows)
class Multiband(S3GeoTiffIOTest, BaseTestClass):
mock_wrapper = BaseTestClass.geopysc._jvm.geopyspark.geotrellis.testkit.MockS3ClientWrapper
client = mock_wrapper.mockClient()
key = "one-month-tiles-multiband/result.tif"
bucket = "test"
uri = "s3://test/one-month-tiles-multiband/result.tif"
file_path = geotiff_test_path(key)
options = {"s3Client": "mock"}
in_file = open(file_path, "rb")
data = in_file.read()
in_file.close()
@pytest.fixture(scope='class', autouse=True)
def tearDown(self):
yield
BaseTestClass.geopysc.pysc._gateway.close()
def read_multiband_geotrellis(self, opt=options):
self.client.putObject(self.bucket, self.key, self.data)
result = get(BaseTestClass.geopysc,
SPATIAL,
self.uri,
opt)
return [tile[1] for tile in result.to_numpy_rdd().collect()]
def test_whole_tiles(self):
geotrellis_tiles = self.read_multiband_geotrellis()
rasterio_tiles = self.read_geotiff_rasterio([self.file_path], False)
for x, y in zip(geotrellis_tiles, rasterio_tiles):
self.assertTrue((x['data'] == y['data']).all())
def windowed_result_checker(self, windowed_tiles):
self.assertEqual(len(windowed_tiles), 4)
def test_windowed_tiles(self):
geotrellis_tiles = self.read_multiband_geotrellis({"s3Client": "mock", "maxTileSize": 256})
rasterio_tiles = self.read_geotiff_rasterio([self.file_path], True)
self.windowed_result_checker(geotrellis_tiles)
for x, y in zip(geotrellis_tiles, rasterio_tiles):
self.assertTrue((x['data'] == y['data']).all())
class BaseTestClass(unittest.TestCase):
if 'TRAVIS' in os.environ:
master_str = "local[2]"
else:
master_str = "local[*]"
geopysc = GeoPyContext(master=master_str, appName="test")
dir_path = geotiff_test_path("all-ones.tif")
rdd = get(geopysc, SPATIAL, dir_path)
value = rdd.to_numpy_rdd().collect()[0]
projected_extent = value[0]
extent = projected_extent.extent
expected_tile = value[1]['data']
(_, rows, cols) = expected_tile.shape
layout = TileLayout(1, 1, cols, rows)
class TiledRasterLayerTest(BaseTestClass):
dir_path = geotiff_test_path("all-ones.tif")
result = get(LayerType.SPATIAL, dir_path)
tiled_layer = result.tile_to_layout()
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_tile_to_layout_layout_definition(self):
layout_definition = self.tiled_layer.layer_metadata.layout_definition
new_extent = Extent(layout_definition.extent.xmin,
layout_definition.extent.ymin,
layout_definition.extent.xmax + 15.0,
layout_definition.extent.ymax + 15.0)
new_layout_definition = LayoutDefinition(extent=new_extent, tileLayout=layout_definition.tileLayout)
actual = self.tiled_layer.tile_to_layout(new_layout_definition).layer_metadata.layout_definition.extent
self.assertEqual(actual, new_extent)
def test_tile_to_layout_tiled_layer(self):
actual = self.tiled_layer.tile_to_layout(self.tiled_layer).layer_metadata
expected = self.tiled_layer.layer_metadata
self.assertDictEqual(actual.to_dict(), expected.to_dict())
def test_tile_to_layout_global_layout(self):
actual = self.tiled_layer.tile_to_layout(layout=GlobalLayout(zoom=5))
self.assertEqual(actual.zoom_level, 5)
def test_tile_to_layout_with_reproject(self):
proj4 = crs_to_proj4(3857)
actual = self.result.tile_to_layout(layout=GlobalLayout(), target_crs=proj4).layer_metadata.crs
self.assertEqual(proj4, actual)
def test_tile_to_layout_bad_crs(self):
with pytest.raises(ValueError):
self.result.tile_to_layout(layout=self.tiled_layer, target_crs=3857)
class Singleband(GeoTiffIOTest, BaseTestClass):
dir_path = geotiff_test_path("one-month-tiles/")
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.geopysc.pysc._gateway.close()
def read_singleband_geotrellis(self, options=None):
if options is None:
result = get(BaseTestClass.geopysc, SPATIAL, self.dir_path)
else:
result = get(BaseTestClass.geopysc,
SPATIAL,
self.dir_path,
maxTileSize=256)
return [tile[1] for tile in result.to_numpy_rdd().collect()]
def test_whole_tiles(self):
geotrellis_tiles = self.read_singleband_geotrellis()
file_paths = self.get_filepaths(self.dir_path)
rasterio_tiles = self.read_geotiff_rasterio(file_paths, False)
for x, y in zip(geotrellis_tiles, rasterio_tiles):
self.assertTrue((x['data'] == y['data']).all())
self.assertEqual(x['no_data_value'], y['no_data_value'])
def windowed_result_checker(self, windowed_tiles):
self.assertEqual(len(windowed_tiles), 4)
def test_windowed_tiles(self):
geotrellis_tiles = self.read_singleband_geotrellis(True)
file_paths = self.get_filepaths(self.dir_path)
rasterio_tiles = self.read_geotiff_rasterio(file_paths, True)
self.windowed_result_checker(geotrellis_tiles)
for x, y in zip(geotrellis_tiles, rasterio_tiles):
self.assertTrue((x['data'] == y['data']).all())
self.assertEqual(x['no_data_value'], y['no_data_value'])
class Multiband(S3GeoTiffIOTest, BaseTestClass):
mock_wrapper = BaseTestClass.pysc._gateway.jvm.geopyspark.geotrellis.testkit.MockS3ClientWrapper
client = mock_wrapper.mockClient()
key = "one-month-tiles-multiband/result.tif"
bucket = "test"
uri = "s3://test/one-month-tiles-multiband/result.tif"
file_path = geotiff_test_path(key)
options = {"s3Client": "mock"}
in_file = open(file_path, "rb")
cells = in_file.read()
in_file.close()
@pytest.fixture(scope='class', autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def read_multiband_geotrellis(self, opt=options):
self.client.putObject(self.bucket, self.key, self.cells)
result = get(LayerType.SPATIAL,
self.uri,
s3_client=opt['s3Client'],
max_tile_size=opt.get('maxTileSize'))
return result
def test_segment_tiles(self):
# GeoTrellis will read GeoTiff Segments given no window size
# Retile them to match Rasterio read and check the cell values
geotrellis_tiles = self.read_multiband_geotrellis()\
.tile_to_layout(LocalLayout(512))\
.to_numpy_rdd().collect()
# TODO: assert there is only one geotrellis tile
geotrellis_tile = dict(geotrellis_tiles)[SpatialKey(0, 0)]
rasterio_tiles = self.read_geotiff_rasterio([self.file_path], False)
self.assertEquals(len(rasterio_tiles), 1)
rasterio_tile = rasterio_tiles[0]
self.assertTilesEqual(geotrellis_tile.cells, rasterio_tile['cells'])
def windowed_result_checker(self, windowed_tiles):
self.assertEqual(len(windowed_tiles), 4)
def test_windowed_tiles(self):
geotrellis_tiles = self.read_multiband_geotrellis({
"s3Client": "mock",
"maxTileSize": 256
})
geotrellis_tiles = [
tile[1] for tile in geotrellis_tiles.to_numpy_rdd().collect()
]
rasterio_tiles = self.read_geotiff_rasterio([self.file_path], True)
self.windowed_result_checker(geotrellis_tiles)
for x, y in zip(geotrellis_tiles, rasterio_tiles):
print('\n')
print('This is read in from geotrellis', x.cells.shape)
print('This is read in from rasterio', y['cells'].shape)
self.assertTrue(np.array_equal(x.cells, y['cells']))
class Multiband(BaseTestClass):
dir_path = geotiff_test_path("all-ones.tif")
result = get(LayerType.SPATIAL, dir_path, max_tile_size=256)
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_to_numpy_rdd(self, option=None):
pyrdd = self.result.to_numpy_rdd()
(key, tile) = pyrdd.first()
self.assertEqual(tile.cells.shape, (1, 256, 256))
def test_collect_metadata(self, options=None):
md = self.result.collect_metadata()
self.assertTrue('+proj=longlat' in md.crs)
self.assertTrue('+datum=WGS84' in md.crs)
def test_reproject(self, options=None):
tiles = self.result.reproject("EPSG:3857")
md = tiles.collect_metadata()
self.assertTrue('+proj=merc' in md.crs)
def test_to_ud_ubyte(self):
arr = np.array([[0.4324323432124, 0.0, 0.0], [1.0, 1.0, 1.0]],
dtype=float)
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
projected_extent = ProjectedExtent(extent, epsg_code)
tile = Tile(arr, 'FLOAT', float('nan'))
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
converted = raster_rdd.convert_data_type(CellType.UINT8,
no_data_value=-1)
tile = converted.to_numpy_rdd().first()
no_data = tile[1].no_data_value
self.assertEqual(no_data, -1)
def test_no_data_deserialization(self):
arr = np.int16([[[-32768, -32768, -32768, -32768],
[-32768, -32768, -32768, -32768],
[-32768, -32768, -32768, -32768],
[-32768, -32768, -32768, -32768]]])
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
projected_extent = ProjectedExtent(extent, epsg_code)
tile = Tile(arr, 'SHORT', -32768)
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
actual_tile = raster_layer.to_numpy_rdd().first()[1]
self.assertEqual(actual_tile.cell_type, tile.cell_type)
self.assertEqual(actual_tile.no_data_value, tile.no_data_value)
self.assertTrue((actual_tile.cells == tile.cells).all())
class CatalogTest(BaseTestClass):
rdd = get(BaseTestClass.geopysc, SPATIAL,
geotiff_test_path("srtm_52_11.tif"))
metadata = rdd.collect_metadata()
laid_out = rdd.tile_to_layout(metadata)
reprojected = laid_out.reproject(target_crs="EPSG:3857", scheme=ZOOM)
result = reprojected.pyramid(start_zoom=11, end_zoom=1)
dir_path = geotiff_test_path("catalog/")
uri = "file://{}".format(dir_path)
layer_name = "catalog-test"
@pytest.fixture(scope='class', autouse=True)
def tearDown(self):
yield
BaseTestClass.geopysc.pysc._gateway.close()
def test_read(self):
for x in range(11, 0, -1):
actual_layer = read(BaseTestClass.geopysc, SPATIAL, self.uri,
self.layer_name, x)
expected_layer = self.result[11 - x]
self.assertDictEqual(actual_layer.layer_metadata.to_dict(),
expected_layer.layer_metadata.to_dict())
def test_read_value(self):
tiled = read_value(BaseTestClass.geopysc, SPATIAL, self.uri,
self.layer_name, 11, 1450, 966)
self.assertEqual(tiled['data'].shape, (1, 256, 256))
def test_bad_read_value(self):
tiled = read_value(BaseTestClass.geopysc, SPATIAL, self.uri,
self.layer_name, 11, 1450, 2000)
self.assertEqual(tiled, None)
def test_query(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(BaseTestClass.geopysc, SPATIAL, self.uri,
self.layer_name, 11, intersection)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query_partitions(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(BaseTestClass.geopysc,
SPATIAL,
self.uri,
self.layer_name,
11,
intersection,
numPartitions=2)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query_crs(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(BaseTestClass.geopysc,
SPATIAL,
self.uri,
self.layer_name,
11,
intersection,
proj_query=3857)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_read_metadata(self):
layer = read(BaseTestClass.geopysc, SPATIAL, self.uri, self.layer_name,
5)
actual_metadata = layer.layer_metadata
expected_metadata = read_layer_metadata(BaseTestClass.geopysc, SPATIAL,
self.uri, self.layer_name, 5)
self.assertEqual(actual_metadata.to_dict(),
expected_metadata.to_dict())
def test_layer_ids(self):
ids = get_layer_ids(BaseTestClass.geopysc, self.uri)
self.assertTrue(len(ids) == 11)
class CatalogTest(BaseTestClass):
rdd = get(LayerType.SPATIAL, geotiff_test_path("srtm_52_11.tif"))
metadata = rdd.collect_metadata()
reprojected = rdd.tile_to_layout(layout=GlobalLayout(zoom=11),
target_crs="EPSG:3857")
result = reprojected.pyramid()
dir_path = geotiff_test_path("catalog/")
uri = "file://{}".format(dir_path)
layer_name = "catalog-test"
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_read(self):
for x in range(11, 0, -1):
actual_layer = query(self.uri, self.layer_name, x)
expected_layer = self.result.levels[x]
self.assertDictEqual(actual_layer.layer_metadata.to_dict(),
expected_layer.layer_metadata.to_dict())
def test_read_value(self):
tiled = read_value(self.uri, self.layer_name, 11, 1450, 966)
self.assertEqual(tiled.cells.shape, (1, 256, 256))
def test_bad_read_value(self):
tiled = read_value(self.uri, self.layer_name, 11, 1450, 2000)
self.assertEqual(tiled, None)
@pytest.mark.skipif('TRAVIS' in os.environ,
reason="test_query_1 causes issues on Travis")
def test_query1(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(self.uri, self.layer_name, 11, intersection)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query2(self):
intersection = Extent(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(self.uri,
self.layer_name,
11,
intersection,
query_proj=3857)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query3(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520).wkb
queried = query(self.uri, self.layer_name, 11, intersection)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query4(self):
intersection = 42
with pytest.raises(TypeError):
queried = query(self.uri,
self.layer_name,
11,
query_geom=intersection,
num_partitions=2)
result = queried.to_numpy_rdd().first()[0]
def test_query_partitions(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(self.uri,
self.layer_name,
11,
intersection,
num_partitions=2)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query_crs(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(self.uri,
self.layer_name,
11,
intersection,
query_proj=3857)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_read_metadata_exception(self):
uri = "abcxyz://123"
with pytest.raises(ValueError):
layer = read_layer_metadata(uri, self.layer_name, 5)
def test_read_metadata1(self):
layer = query(self.uri, self.layer_name, 5)
actual_metadata = layer.layer_metadata
expected_metadata = read_layer_metadata(self.uri, self.layer_name, 5)
def test_read_metadata2(self):
layer = query(self.uri, self.layer_name, 5)
actual_metadata = layer.layer_metadata
expected_metadata = read_layer_metadata(self.uri, self.layer_name, 5)
self.assertEqual(actual_metadata.to_dict(),
expected_metadata.to_dict())
def test_layer_ids(self):
ids = AttributeStore(self.uri).layers()
self.assertTrue(len(ids) == 11)
def test_attributestore(self):
store = AttributeStore(self.uri)
layer_name = "boop-epsg-bop"
value = {"first": 113, "second": "44two"}
store.layer(layer_name, 34).write("val", value)
self.assertEqual(value, store.layer(layer_name, 34).read("val"))
self.assertEqual(value, store.layer(layer_name, 34)["val"])
store.layer(layer_name, 34).delete("val")
with pytest.raises(KeyError):
store.layer(layer_name, 34)["val"]
class Multiband(GeoTiffIOTest, BaseTestClass):
dir_path = geotiff_test_path("one-month-tiles-multiband/")
result = get(BaseTestClass.geopysc, SPATIAL, dir_path)
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.geopysc.pysc._gateway.close()
def test_to_numpy_rdd(self, option=None):
pyrdd = self.result.to_numpy_rdd()
(key, tile) = pyrdd.first()
self.assertEqual(tile['data'].shape, (2, 512, 512))
def test_collect_metadata(self, options=None):
md = self.result.collect_metadata()
self.assertTrue('+proj=longlat' in md.crs)
self.assertTrue('+datum=WGS84' in md.crs)
def test_collect_metadata_crs_override(self, options=None):
md = self.result.collect_metadata(crs='EPSG:3857')
self.assertTrue('+proj=merc' in md.crs)
def test_cut_tiles(self, options=None):
md = self.result.collect_metadata(tile_size=100)
tiles = self.result.cut_tiles(md)
records_before = self.result.srdd.rdd().count()
records_after = tiles.srdd.rdd().count()
self.assertTrue(records_after > records_before)
def test_reproject(self, options=None):
tiles = self.result.reproject("EPSG:3857")
md = tiles.collect_metadata()
self.assertTrue('+proj=merc' in md.crs)
def test_to_tiled_raster(self):
md = self.result.collect_metadata()
tiled = self.result.tile_to_layout(md)
converted = self.result.to_tiled_layer()
self.assertDictEqual(tiled.layer_metadata.to_dict(),
converted.layer_metadata.to_dict())
def test_to_int(self):
arr = np.array([[0.4324323432124, 0.0, 0.0],
[1.0, 1.0, 1.0]], dtype=float)
epsg_code = 3857
extent = {'xmin': 0.0, 'ymin': 0.0, 'xmax': 10.0, 'ymax': 10.0}
projected_extent = {'extent': extent, 'epsg': epsg_code}
tile = {'data': arr, 'no_data_value': float('nan')}
rdd = BaseTestClass.geopysc.pysc.parallelize([(self.projected_extent, tile)])
raster_rdd = RasterRDD.from_numpy_rdd(BaseTestClass.geopysc, SPATIAL, rdd)
converted = raster_rdd.convert_data_type(INT32)
arr = converted.to_numpy_rdd().first()[1]['data']
self.assertEqual(arr.dtype, np.int64)
def test_to_boolraw(self):
converted = self.result.convert_data_type(BOOLRAW)
arr = converted.to_numpy_rdd().first()[1]['data']
self.assertEqual(arr.dtype, np.uint8)
class ToGeoTiffTest(BaseTestClass):
dir_path = geotiff_test_path("all-ones.tif")
rdd = get(LayerType.SPATIAL, dir_path)
metadata = rdd.collect_metadata()
mapped_types = {
'int8': 'BYTE',
'uint8': 'UBYTE',
'int16': 'SHORT',
'uint16': 'USHORT',
'int32': 'INT',
'float': 'FLOAT',
'float32': 'FLOAT',
'double': 'DOUBLE'
}
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_to_geotiff_rdd_rasterlayer(self):
geotiff_rdd = self.rdd.to_geotiff_rdd(storage_method="Tiled",
compression="DeflateCompression",
color_space=0,
head_tags={'INTERLEAVE': 'BAND'})
geotiff_bytes = geotiff_rdd.first()[1]
with tempfile.NamedTemporaryFile() as temp:
temp.write(geotiff_bytes)
temp_path = pathlib.Path(temp.name)
with rasterio.open(str(temp_path)) as src:
self.assertTrue(src.is_tiled)
profile = src.profile
self.assertEqual(profile['blockxsize'], 256)
self.assertEqual(profile['blockysize'], 256)
self.assertEqual(profile['interleave'], 'band')
self.assertEqual(src.compression,
rasterio.enums.Compression.deflate)
def test_to_geotiff_rdd_tiledrasterlayer(self):
tiled_rdd = self.rdd.tile_to_layout()
tiled_collected = tiled_rdd.to_numpy_rdd().first()[1]
geotiff_rdd = tiled_rdd.to_geotiff_rdd()
geotiff_collected = geotiff_rdd.first()[1]
def to_geotiff(x):
with tempfile.NamedTemporaryFile() as temp:
temp.write(x)
temp_path = pathlib.Path(temp.name)
with rasterio.open(str(temp_path)) as src:
self.assertFalse(src.is_tiled)
data = src.read()
return Tile(data, self.mapped_types[str(data.dtype)],
src.nodata)
rasterio_geotiff = to_geotiff(geotiff_collected)
self.assertTrue(
(tiled_collected.cells == rasterio_geotiff.cells).all())
self.assertEqual(tiled_collected.cell_type, rasterio_geotiff.cell_type)
self.assertEqual(tiled_collected.no_data_value,
rasterio_geotiff.no_data_value)
