def read_singleband_geotrellis(self, options=None):
if options is None:
result = get(LayerType.SPATIAL, self.dir_path)
else:
result = get(LayerType.SPATIAL, self.dir_path, max_tile_size=256)
return [tile[1] for tile in result.to_numpy_rdd().collect()]
def test_s3_uri_type_and_credential_mismatch():
local_path = file_path("all-ones.tif")
with pytest.raises(RuntimeError):
get(LayerType.SPATIAL,
local_path,
max_tile_size=256,
s3_credentials=Credentials('123', 'abc'))
class BaseTestClass(unittest.TestCase):
if 'TRAVIS' in os.environ:
master_str = "local[2]"
else:
master_str = "local[*]"
conf = geopyspark_conf(master=master_str, appName="test")
conf.set('spark.kryoserializer.buffer.max', value='1G')
conf.set('spark.ui.enabled', True)
if 'TRAVIS' in os.environ:
conf.set(key='spark.driver.memory', value='1G')
conf.set(key='spark.executor.memory', value='1G')
pysc = SparkContext(conf=conf)
dir_path = file_path("all-ones.tif")
rdd = get(LayerType.SPATIAL, dir_path, max_tile_size=1024)
value = rdd.to_numpy_rdd().collect()[0]
projected_extent = value[0]
extent = projected_extent.extent
expected_tile = value[1].cells
(_, rows, cols) = expected_tile.shape
layout = TileLayout(1, 1, cols, rows)
def read_multiband_geotrellis(self, opt=options):
self.client.putObject(self.bucket, self.key, self.cells)
result = get(LayerType.SPATIAL,
self.uri,
s3_client=opt['s3Client'],
max_tile_size=opt.get('maxTileSize'))
return result
class TiledRasterLayerTest(BaseTestClass):
dir_path = file_path("all-ones.tif")
result = get(LayerType.SPATIAL, dir_path)
tiled_layer = result.tile_to_layout()
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_tile_to_layout_layout_definition(self):
layout_definition = self.tiled_layer.layer_metadata.layout_definition
new_extent = Extent(layout_definition.extent.xmin,
layout_definition.extent.ymin,
layout_definition.extent.xmax + 15.0,
layout_definition.extent.ymax + 15.0)
new_layout_definition = LayoutDefinition(
extent=new_extent, tileLayout=layout_definition.tileLayout)
actual = self.tiled_layer.tile_to_layout(
new_layout_definition).layer_metadata.layout_definition.extent
self.assertEqual(actual, new_extent)
def test_tile_to_layout_tiled_layer(self):
actual = self.tiled_layer.tile_to_layout(
self.tiled_layer).layer_metadata
expected = self.tiled_layer.layer_metadata
self.assertDictEqual(actual.to_dict(), expected.to_dict())
def test_tile_to_layout_global_layout(self):
actual = self.tiled_layer.tile_to_layout(layout=GlobalLayout(zoom=5))
self.assertEqual(actual.zoom_level, 5)
def test_tile_to_layout_with_reproject(self):
proj4 = crs_to_proj4(3857)
actual = self.result.tile_to_layout(
layout=GlobalLayout(), target_crs=proj4).layer_metadata.crs
self.assertEqual(proj4, actual)
def test_tile_to_layout_bad_crs(self):
with pytest.raises(ValueError):
self.result.tile_to_layout(layout=self.tiled_layer,
target_crs=3857)
class PartitionPreservationTest(BaseTestClass):
rdd = get(LayerType.SPATIAL,
file_path("srtm_52_11.tif"),
max_tile_size=6001)
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_partition_preservation(self):
partition_states = []
strategy = SpatialPartitionStrategy(16)
tiled = self.rdd.tile_to_layout()
tiled2 = self.rdd.tile_to_layout(partition_strategy=strategy)
partition_states.append(tiled2.get_partition_strategy())
added_layer = (tiled + tiled2) * 0.75
partition_states.append(added_layer.get_partition_strategy())
local_max_layer = added_layer.local_max(tiled)
partition_states.append(local_max_layer.get_partition_strategy())
focal_layer = local_max_layer.focal(Operation.MAX, Square(1))
partition_states.append(focal_layer.get_partition_strategy())
reprojected_layer = focal_layer.tile_to_layout(
layout=LocalLayout(), target_crs=3857, partition_strategy=strategy)
partition_states.append(reprojected_layer.get_partition_strategy())
pyramided = reprojected_layer.pyramid()
partition_states.append(
pyramided.levels[pyramided.max_zoom].get_partition_strategy())
self.assertTrue(all(x == partition_states[0]
for x in partition_states))
class Multiband(BaseTestClass):
dir_path = geotiff_test_path("all-ones.tif")
result = get(LayerType.SPATIAL, dir_path, max_tile_size=256)
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_to_numpy_rdd(self, option=None):
pyrdd = self.result.to_numpy_rdd()
(key, tile) = pyrdd.first()
self.assertEqual(tile.cells.shape, (1, 256, 256))
def test_collect_metadata(self, options=None):
md = self.result.collect_metadata()
self.assertTrue('+proj=longlat' in md.crs)
self.assertTrue('+datum=WGS84' in md.crs)
def test_reproject(self, options=None):
tiles = self.result.reproject("EPSG:3857")
md = tiles.collect_metadata()
self.assertTrue('+proj=merc' in md.crs)
def test_to_ud_ubyte(self):
arr = np.array([[0.4324323432124, 0.0, 0.0], [1.0, 1.0, 1.0]],
dtype=float)
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
projected_extent = ProjectedExtent(extent, epsg_code)
tile = Tile(arr, 'FLOAT', float('nan'))
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
converted = raster_rdd.convert_data_type(CellType.UINT8,
no_data_value=-1)
tile = converted.to_numpy_rdd().first()
no_data = tile[1].no_data_value
self.assertEqual(no_data, -1)
def test_no_data_deserialization(self):
arr = np.int16([[[-32768, -32768, -32768, -32768],
[-32768, -32768, -32768, -32768],
[-32768, -32768, -32768, -32768],
[-32768, -32768, -32768, -32768]]])
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
projected_extent = ProjectedExtent(extent, epsg_code)
tile = Tile(arr, 'SHORT', -32768)
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
actual_tile = raster_layer.to_numpy_rdd().first()[1]
self.assertEqual(actual_tile.cell_type, tile.cell_type)
self.assertEqual(actual_tile.no_data_value, tile.no_data_value)
self.assertTrue((actual_tile.cells == tile.cells).all())
class CatalogTest(BaseTestClass):
rdd = get(LayerType.SPATIAL, geotiff_test_path("srtm_52_11.tif"))
metadata = rdd.collect_metadata()
reprojected = rdd.tile_to_layout(layout=GlobalLayout(zoom=11),
target_crs="EPSG:3857")
result = reprojected.pyramid()
dir_path = geotiff_test_path("catalog/")
uri = "file://{}".format(dir_path)
layer_name = "catalog-test"
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_read(self):
for x in range(11, 0, -1):
actual_layer = query(self.uri, self.layer_name, x)
expected_layer = self.result.levels[x]
self.assertDictEqual(actual_layer.layer_metadata.to_dict(),
expected_layer.layer_metadata.to_dict())
def test_read_value(self):
tiled = read_value(self.uri, self.layer_name, 11, 1450, 966)
self.assertEqual(tiled.cells.shape, (1, 256, 256))
def test_bad_read_value(self):
tiled = read_value(self.uri, self.layer_name, 11, 1450, 2000)
self.assertEqual(tiled, None)
@pytest.mark.skipif('TRAVIS' in os.environ,
reason="test_query_1 causes issues on Travis")
def test_query1(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(self.uri, self.layer_name, 11, intersection)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query2(self):
intersection = Extent(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(self.uri,
self.layer_name,
11,
intersection,
query_proj=3857)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query3(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520).wkb
queried = query(self.uri, self.layer_name, 11, intersection)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query4(self):
intersection = 42
with pytest.raises(TypeError):
queried = query(self.uri,
self.layer_name,
11,
query_geom=intersection,
num_partitions=2)
result = queried.to_numpy_rdd().first()[0]
def test_query_partitions(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(self.uri,
self.layer_name,
11,
intersection,
num_partitions=2)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query_crs(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(self.uri,
self.layer_name,
11,
intersection,
query_proj=3857)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_read_metadata_exception(self):
uri = "abcxyz://123"
with pytest.raises(ValueError):
layer = read_layer_metadata(uri, self.layer_name, 5)
def test_read_metadata1(self):
layer = query(self.uri, self.layer_name, 5)
actual_metadata = layer.layer_metadata
expected_metadata = read_layer_metadata(self.uri, self.layer_name, 5)
def test_read_metadata2(self):
layer = query(self.uri, self.layer_name, 5)
actual_metadata = layer.layer_metadata
expected_metadata = read_layer_metadata(self.uri, self.layer_name, 5)
self.assertEqual(actual_metadata.to_dict(),
expected_metadata.to_dict())
def test_layer_ids(self):
ids = AttributeStore(self.uri).layers()
self.assertTrue(len(ids) == 11)
def test_attributestore(self):
store = AttributeStore(self.uri)
layer_name = "boop-epsg-bop"
value = {"first": 113, "second": "44two"}
store.layer(layer_name, 34).write("val", value)
self.assertEqual(value, store.layer(layer_name, 34).read("val"))
self.assertEqual(value, store.layer(layer_name, 34)["val"])
store.layer(layer_name, 34).delete("val")
with pytest.raises(KeyError):
store.layer(layer_name, 34)["val"]
class ToGeoTiffTest(BaseTestClass):
dir_path = file_path("all-ones.tif")
rdd = get(LayerType.SPATIAL, dir_path, max_tile_size=1024)
metadata = rdd.collect_metadata()
mapped_types = {
'int8': 'BYTE',
'uint8': 'UBYTE',
'int16': 'SHORT',
'uint16': 'USHORT',
'int32': 'INT',
'float': 'FLOAT',
'float32': 'FLOAT',
'double': 'DOUBLE'
}
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_to_geotiff_rdd_rasterlayer(self):
geotiff_rdd = self.rdd.to_geotiff_rdd(storage_method="Tiled",
compression="DeflateCompression",
color_space=0,
head_tags={'INTERLEAVE': 'BAND'})
geotiff_bytes = geotiff_rdd.first()[1]
with tempfile.NamedTemporaryFile() as temp:
temp.write(geotiff_bytes)
temp_path = pathlib.Path(temp.name)
with rasterio.open(str(temp_path)) as src:
self.assertTrue(src.is_tiled)
profile = src.profile
self.assertEqual(profile['blockxsize'], 256)
self.assertEqual(profile['blockysize'], 256)
self.assertEqual(profile['interleave'], 'band')
self.assertEqual(src.compression,
rasterio.enums.Compression.deflate)
def test_to_geotiff_rdd_tiledrasterlayer(self):
tiled_rdd = self.rdd.tile_to_layout()
tiled_collected = tiled_rdd.to_numpy_rdd().first()[1]
geotiff_rdd = tiled_rdd.to_geotiff_rdd()
geotiff_collected = geotiff_rdd.first()[1]
def to_geotiff(x):
with tempfile.NamedTemporaryFile() as temp:
temp.write(x)
temp_path = pathlib.Path(temp.name)
with rasterio.open(str(temp_path)) as src:
self.assertFalse(src.is_tiled)
data = src.read()
return Tile(data, self.mapped_types[str(data.dtype)],
src.nodata)
rasterio_geotiff = to_geotiff(geotiff_collected)
self.assertTrue(
(tiled_collected.cells == rasterio_geotiff.cells).all())
self.assertEqual(tiled_collected.cell_type, rasterio_geotiff.cell_type)
self.assertEqual(tiled_collected.no_data_value,
rasterio_geotiff.no_data_value)
