class UnionTemporalTest(BaseTestClass):
time_1 = datetime.datetime.strptime("1993-09-19T07:01:00Z",
'%Y-%m-%dT%H:%M:%SZ')
time_2 = datetime.datetime.strptime("2017-09-19T07:01:00Z",
'%Y-%m-%dT%H:%M:%SZ')
temp_projected_extent_1 = TemporalProjectedExtent(extent, time_1,
epsg_code)
temp_projected_extent_2 = TemporalProjectedExtent(extent, time_2,
epsg_code)
arr = np.zeros((1, 16, 16))
tile = Tile(arr, 'FLOAT', -500.0)
rdd_1 = BaseTestClass.pysc.parallelize([(temp_projected_extent_1, tile)])
rdd_2 = BaseTestClass.pysc.parallelize([(temp_projected_extent_2, tile)])
layer_1 = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd_1)
layer_2 = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd_2)
tiled_layer_1 = layer_1.tile_to_layout(GlobalLayout())
tiled_layer_2 = layer_2.tile_to_layout(GlobalLayout())
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_union_of_raster_layers(self):
result = union(self.layer_1, self.layer_2)
self.assertTrue(result.srdd.rdd().count(), 2)
def test_union_of_tiled_raster_layers(self):
result = union(self.tiled_layer_1, self.tiled_layer_2)
bounds_1 = self.tiled_layer_1.layer_metadata.bounds
bounds_2 = self.tiled_layer_2.layer_metadata.bounds
min_col = min(bounds_1.minKey.col, bounds_2.minKey.col)
min_row = min(bounds_1.minKey.row, bounds_2.minKey.row)
min_instant = min(bounds_1.minKey.instant, bounds_2.minKey.instant)
max_col = max(bounds_1.maxKey.col, bounds_2.maxKey.col)
max_row = max(bounds_1.maxKey.row, bounds_2.maxKey.row)
max_instant = max(bounds_1.maxKey.instant, bounds_2.maxKey.instant)
min_key = SpaceTimeKey(min_col, min_row, min_instant)
max_key = SpaceTimeKey(max_col, max_row, max_instant)
self.assertTrue(result.srdd.rdd().count(), 2)
self.assertEqual(result.layer_metadata.bounds,
Bounds(min_key, max_key))
class TileToLayoutTest(BaseTestClass):
layers = [
make_raster(0, 0, v=1),
make_raster(3, 2, v=2),
make_raster(6, 0, v=3)
]
numpy_rdd = BaseTestClass.pysc.parallelize(layers)
layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, numpy_rdd)
metadata = layer.collect_metadata(GlobalLayout(5))
def test_to_to_layout_with_partitioner(self):
strategy = SpatialPartitionStrategy(4)
tiled = self.layer.tile_to_layout(LocalLayout(5),
partition_strategy=strategy)
self.assertEqual(tiled.get_partition_strategy(), strategy)
def test_tile_to_local_layout(self):
tiled = self.layer.tile_to_layout(LocalLayout(5))
assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6)
assert tiled.layer_metadata.tile_layout == TileLayout(2, 2, 5, 5)
def test_tile_to_global_layout(self):
tiled = self.layer.tile_to_layout(GlobalLayout(5))
assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6)
assert tiled.layer_metadata.tile_layout == TileLayout(128, 128, 5, 5)
assert tiled.zoom_level == 7
def test_tile_to_metadata_layout(self):
tiled = self.layer.tile_to_layout(layout=self.metadata)
self.assertEqual(tiled.layer_metadata.extent, Extent(0, 0, 10, 6))
self.assertDictEqual(tiled.layer_metadata.to_dict(),
self.metadata.to_dict())
def test_tile_to_tiled_layer_layout(self):
extent = Extent(0., 0., 10., 6.)
tile_layout = TileLayout(2, 2, 5, 5)
layout_definition = LayoutDefinition(extent, tile_layout)
base = self.layer.tile_to_layout(layout_definition)
tiled = self.layer.tile_to_layout(layout=base)
self.assertDictEqual(tiled.layer_metadata.to_dict(),
base.layer_metadata.to_dict())
def test_tile_to_layout_definition(self):
tiled = self.layer.tile_to_layout(
layout=self.metadata.layout_definition)
self.assertDictEqual(tiled.layer_metadata.to_dict(),
self.metadata.to_dict())
@pytest.fixture(scope='class', autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
class UnionSpatialTest(BaseTestClass):
projected_extent_1 = ProjectedExtent(extent, epsg_code)
projected_extent_2 = ProjectedExtent(extent_2, epsg_code)
arr = np.zeros((1, 16, 16))
tile = Tile(arr, 'FLOAT', -500.0)
rdd_1 = BaseTestClass.pysc.parallelize([(projected_extent_1, tile)])
rdd_2 = BaseTestClass.pysc.parallelize([(projected_extent_2, tile)])
layer_1 = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd_1)
layer_2 = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd_2)
tiled_layer_1 = layer_1.tile_to_layout(GlobalLayout())
tiled_layer_2 = layer_2.tile_to_layout(GlobalLayout())
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_union_of_raster_layers(self):
result = union(self.layer_1, self.layer_2)
self.assertTrue(result.srdd.rdd().count(), 2)
def test_union_of_tiled_raster_layers(self):
result = union(self.tiled_layer_1, self.tiled_layer_2)
bounds_1 = self.tiled_layer_1.layer_metadata.bounds
bounds_2 = self.tiled_layer_2.layer_metadata.bounds
min_col = min(bounds_1.minKey.col, bounds_2.minKey.col)
min_row = min(bounds_1.minKey.row, bounds_2.minKey.row)
max_col = max(bounds_1.maxKey.col, bounds_2.maxKey.col)
max_row = max(bounds_1.maxKey.row, bounds_2.maxKey.row)
min_key = SpatialKey(min_col, min_row)
max_key = SpatialKey(max_col, max_row)
self.assertTrue(result.srdd.rdd().count(), 2)
self.assertEqual(result.layer_metadata.bounds,
Bounds(min_key, max_key))
def test_correct_base(self):
arr = np.zeros((1, 16, 16))
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
tile = Tile(arr, 'FLOAT', False)
projected_extent = ProjectedExtent(extent, epsg_code)
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
tile_layout = TileLayout(32, 32, 16, 16)
new_extent = Extent(-20037508.342789244, -20037508.342789244,
20037508.342789244, 20037508.342789244)
layout_def = LayoutDefinition(new_extent, tile_layout)
laid_out = raster_rdd.tile_to_layout(GlobalLayout(tile_size=16))
result = laid_out.pyramid()
self.pyramid_building_check(result)
def test_pyramid_class(self):
arr = np.zeros((1, 16, 16))
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
tile = Tile(arr, 'FLOAT', False)
projected_extent = ProjectedExtent(extent, epsg_code)
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
tile_layout = TileLayout(1, 1, 16, 16)
reprojected = raster_rdd.tile_to_layout(layout=GlobalLayout(tile_size=16), target_crs=3857)
result = reprojected.pyramid()
hist = result.get_histogram()
self.assertEqual(result.max_zoom, reprojected.zoom_level)
self.assertTrue(set(result.levels.keys()).issuperset(range(1, 13)))
self.assertEqual(hist.mean(), 0.0)
self.assertEqual(hist.min_max(), (0.0, 0.0))
def test_write_pyramid_layers(self):
max_zoom = 5
tif = file_path('srtm_52_11.tif')
raster_layer = geotiff.get(layer_type=LayerType.SPATIAL, uri=tif)
tiled_raster_layer = raster_layer.tile_to_layout(
GlobalLayout(zoom=max_zoom), target_crs=3857)
pyramided_layer = tiled_raster_layer.pyramid()
layer_name = 'pyramid-test-layer'
path = file_path('pyramid-test-catalog')
uri = 'file:///' + path
if os.path.isdir(path):
import shutil
shutil.rmtree(path)
pyramided_layer.write(uri, layer_name)
self.assertTrue(catalog.read_layer_metadata(uri, layer_name, 0))
self.assertTrue(catalog.read_layer_metadata(uri, layer_name, max_zoom))
def test_pyraminding_with_partitioner(self):
arr = np.zeros((1, 16, 16))
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
tile = Tile(arr, 'FLOAT', False)
projected_extent = ProjectedExtent(extent, epsg_code)
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
tile_layout = TileLayout(32, 32, 16, 16)
new_extent = Extent(-20037508.342789244, -20037508.342789244, 20037508.342789244,
20037508.342789244)
layout_def = LayoutDefinition(new_extent, tile_layout)
laid_out = raster_rdd.tile_to_layout(GlobalLayout(tile_size=16))
strategy = SpatialPartitionStrategy(4)
pyramided = laid_out.pyramid(partition_strategy=strategy)
self.assertEqual(pyramided.levels[0].get_partition_strategy(), strategy)
def test_read_no_reproject_global_gdal(self):
self.read(GlobalLayout(tile_size=128, zoom=4), ReadMethod.GDAL)
class RasterLayerTest(BaseTestClass):
layers = [
make_raster(0, 0, v=1),
make_raster(3, 2, v=2),
make_raster(6, 0, v=3)
]
# TODO: Have Travis be able to run the GDAL tests
numpy_rdd = BaseTestClass.pysc.parallelize(layers)
layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, numpy_rdd)
metadata = layer.collect_metadata(GlobalLayout(5))
def read_no_reproject(self, read_method):
actual_raster_layer = RasterLayer.read([self.path], read_method=read_method)
collected = actual_raster_layer.to_numpy_rdd().first()
(projected_extent, tile) = collected
self.assertEqual(projected_extent.proj4, self.projected_extent.proj4)
self.assertTrue((self.expected_tile == tile.cells).all())
def read_with_reproject(self, read_method):
expected_raster_layer = self.rdd.reproject(target_crs=3857)
expected_collected = expected_raster_layer.to_numpy_rdd().first()
(expected_projected_extent, expected_tile) = expected_collected
actual_raster_layer = RasterLayer.read([self.path], target_crs=3857, read_method=read_method)
actual_collected = actual_raster_layer.to_numpy_rdd().first()
(actual_projected_extent, tile) = actual_collected
self.assertEqual(actual_projected_extent.epsg, expected_projected_extent.epsg)
self.assertTrue((expected_tile.cells == tile.cells).all())
# No reprojection
def test_read_no_reproject_geotrellis(self):
self.read_no_reproject(ReadMethod.GEOTRELLIS)
def test_read_no_reproject_gdal(self):
self.read_no_reproject(ReadMethod.GDAL)
# With reprojection
def test_read_with_reproject_geotrellis(self):
self.read_with_reproject(ReadMethod.GEOTRELLIS)
def test_read_with_reproject_gdal(self):
self.read_with_reproject(ReadMethod.GDAL)
def test_no_data_of_zero(self):
no_data_layer = [(t[0], Tile.from_numpy_array(t[1].cells, 1)) for t in self.layers]
rdd = BaseTestClass.pysc.parallelize(no_data_layer)
nd_layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
nd_metadata = nd_layer.collect_metadata()
self.assertTrue('ud1' in nd_metadata.cell_type)
self.assertEqual(nd_metadata.no_data_value, 1)
@pytest.fixture(scope='class', autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
class CatalogTest(BaseTestClass):
rdd = get(LayerType.SPATIAL, geotiff_test_path("srtm_52_11.tif"))
metadata = rdd.collect_metadata()
reprojected = rdd.tile_to_layout(layout=GlobalLayout(zoom=11),
target_crs="EPSG:3857")
result = reprojected.pyramid()
dir_path = geotiff_test_path("catalog/")
uri = "file://{}".format(dir_path)
layer_name = "catalog-test"
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_read(self):
for x in range(11, 0, -1):
actual_layer = query(self.uri, self.layer_name, x)
expected_layer = self.result.levels[x]
self.assertDictEqual(actual_layer.layer_metadata.to_dict(),
expected_layer.layer_metadata.to_dict())
def test_read_value(self):
tiled = read_value(self.uri, self.layer_name, 11, 1450, 966)
self.assertEqual(tiled.cells.shape, (1, 256, 256))
def test_bad_read_value(self):
tiled = read_value(self.uri, self.layer_name, 11, 1450, 2000)
self.assertEqual(tiled, None)
@pytest.mark.skipif('TRAVIS' in os.environ,
reason="test_query_1 causes issues on Travis")
def test_query1(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(self.uri, self.layer_name, 11, intersection)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query2(self):
intersection = Extent(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(self.uri,
self.layer_name,
11,
intersection,
query_proj=3857)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query3(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520).wkb
queried = query(self.uri, self.layer_name, 11, intersection)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query4(self):
intersection = 42
with pytest.raises(TypeError):
queried = query(self.uri,
self.layer_name,
11,
query_geom=intersection,
num_partitions=2)
result = queried.to_numpy_rdd().first()[0]
def test_query_partitions(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(self.uri,
self.layer_name,
11,
intersection,
num_partitions=2)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_query_crs(self):
intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
543988.943201520)
queried = query(self.uri,
self.layer_name,
11,
intersection,
query_proj=3857)
self.assertEqual(queried.to_numpy_rdd().first()[0],
SpatialKey(1450, 996))
def test_read_metadata_exception(self):
uri = "abcxyz://123"
with pytest.raises(ValueError):
layer = read_layer_metadata(uri, self.layer_name, 5)
def test_read_metadata1(self):
layer = query(self.uri, self.layer_name, 5)
actual_metadata = layer.layer_metadata
expected_metadata = read_layer_metadata(self.uri, self.layer_name, 5)
def test_read_metadata2(self):
layer = query(self.uri, self.layer_name, 5)
actual_metadata = layer.layer_metadata
expected_metadata = read_layer_metadata(self.uri, self.layer_name, 5)
self.assertEqual(actual_metadata.to_dict(),
expected_metadata.to_dict())
def test_layer_ids(self):
ids = AttributeStore(self.uri).layers()
self.assertTrue(len(ids) == 11)
def test_attributestore(self):
store = AttributeStore(self.uri)
layer_name = "boop-epsg-bop"
value = {"first": 113, "second": "44two"}
store.layer(layer_name, 34).write("val", value)
self.assertEqual(value, store.layer(layer_name, 34).read("val"))
self.assertEqual(value, store.layer(layer_name, 34)["val"])
store.layer(layer_name, 34).delete("val")
with pytest.raises(KeyError):
store.layer(layer_name, 34)["val"]
def test_tile_to_global_layout(self):
tiled = self.layer.tile_to_layout(GlobalLayout(5))
assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6)
assert tiled.layer_metadata.tile_layout == TileLayout(128, 128, 5, 5)
assert tiled.zoom_level == 7
def test_tile_to_layout_global_layout(self):
actual = self.tiled_layer.tile_to_layout(layout=GlobalLayout(zoom=5))
self.assertEqual(actual.zoom_level, 5)
def test_read_with_reproject_global_gdal(self):
self.read(GlobalLayout(tile_size=128, zoom=4), ReadMethod.GDAL, target_crs=3857)
def test_tile_to_layout_with_reproject(self):
proj4 = crs_to_proj4(3857)
actual = self.result.tile_to_layout(layout=GlobalLayout(), target_crs=proj4).layer_metadata.crs
self.assertEqual(proj4, actual)
def test_read_with_reproject_global_geotrellis(self):
self.read(GlobalLayout(tile_size=128, zoom=4), ReadMethod.GEOTRELLIS, target_crs=3857)
def test_ordered_read_no_reproject_global_geotrellis(self):
self.read(GlobalLayout(tile_size=16, zoom=4),
ReadMethod.GEOTRELLIS,
multiplex=True)
def test_read_no_reproject_global_geotrellis(self):
self.read(GlobalLayout(tile_size=16, zoom=4), ReadMethod.GEOTRELLIS)
def test_different_crs_zoom(self):
result = BaseTestClass.rdd.tile_to_layout(
layout=GlobalLayout(tile_size=self.cols), target_crs="EPSG:4324")
new_metadata = result.layer_metadata
self.assertEqual(self.expected_crs, new_metadata.crs)
class RasterLayerTest(BaseTestClass):
layers = [
make_raster(0, 0, v=1),
make_raster(3, 2, v=2),
make_raster(6, 0, v=3)
]
numpy_rdd = BaseTestClass.pysc.parallelize(layers)
layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, numpy_rdd)
metadata = layer.collect_metadata(GlobalLayout(5))
def test_tile_to_local_layout(self):
tiled = self.layer.tile_to_layout(LocalLayout(5))
assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6)
assert tiled.layer_metadata.tile_layout == TileLayout(2, 2, 5, 5)
def test_tile_to_global_layout(self):
tiled = self.layer.tile_to_layout(GlobalLayout(5))
assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6)
assert tiled.layer_metadata.tile_layout == TileLayout(128, 128, 5, 5)
assert tiled.zoom_level == 7
def test_tile_to_metadata_layout(self):
tiled = self.layer.tile_to_layout(layout=self.metadata)
self.assertEqual(tiled.layer_metadata.extent, Extent(0, 0, 10, 6))
self.assertDictEqual(tiled.layer_metadata.to_dict(),
self.metadata.to_dict())
def test_tile_to_tiled_layer_layout(self):
extent = Extent(0., 0., 10., 6.)
tile_layout = TileLayout(2, 2, 5, 5)
layout_definition = LayoutDefinition(extent, tile_layout)
base = self.layer.tile_to_layout(layout_definition)
tiled = self.layer.tile_to_layout(layout=base)
self.assertDictEqual(tiled.layer_metadata.to_dict(),
base.layer_metadata.to_dict())
def test_tile_to_layout_definition(self):
tiled = self.layer.tile_to_layout(
layout=self.metadata.layout_definition)
self.assertDictEqual(tiled.layer_metadata.to_dict(),
self.metadata.to_dict())
def test_no_data_of_zero(self):
no_data_layer = [(t[0], Tile.from_numpy_array(t[1].cells, 1))
for t in self.layers]
rdd = BaseTestClass.pysc.parallelize(no_data_layer)
nd_layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
nd_metadata = nd_layer.collect_metadata()
self.assertTrue('ud1' in nd_metadata.cell_type)
self.assertEqual(nd_metadata.no_data_value, 1)
@pytest.fixture(scope='class', autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
