def test_integer_crs(self):
result = BaseTestClass.rdd.tile_to_layout(layout=self.layout_def,
target_crs=4326)
new_metadata = result.layer_metadata
layout_definition = LayoutDefinition(BaseTestClass.extent,
BaseTestClass.layout)
self.assertEqual(layout_definition, new_metadata.layout_definition)
def test_tile_to_tiled_layer_layout(self):
extent = Extent(0., 0., 10., 6.)
tile_layout = TileLayout(2, 2, 5, 5)
layout_definition = LayoutDefinition(extent, tile_layout)
base = self.layer.tile_to_layout(layout_definition)
tiled = self.layer.tile_to_layout(layout=base)
self.assertDictEqual(tiled.layer_metadata.to_dict(),
base.layer_metadata.to_dict())
def test_same_crs_layout(self):
result = self.laid_out_rdd.reproject("EPSG:4326",
extent=self.extent,
layout=self.layout)
new_metadata = result.layer_metadata
layout_definition = LayoutDefinition(BaseTestClass.extent,
BaseTestClass.layout)
self.assertEqual(layout_definition, new_metadata.layout_definition)
def test_tile_to_layout_layout_definition(self):
layout_definition = self.tiled_layer.layer_metadata.layout_definition
new_extent = Extent(layout_definition.extent.xmin,
layout_definition.extent.ymin,
layout_definition.extent.xmax + 15.0,
layout_definition.extent.ymax + 15.0)
new_layout_definition = LayoutDefinition(extent=new_extent, tileLayout=layout_definition.tileLayout)
actual = self.tiled_layer.tile_to_layout(new_layout_definition).layer_metadata.layout_definition.extent
self.assertEqual(actual, new_extent)
def test_correct_base(self):
arr = np.zeros((1, 16, 16))
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
tile = Tile(arr, 'FLOAT', False)
projected_extent = ProjectedExtent(extent, epsg_code)
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
tile_layout = TileLayout(32, 32, 16, 16)
new_extent = Extent(-20037508.342789244, -20037508.342789244,
20037508.342789244, 20037508.342789244)
layout_def = LayoutDefinition(new_extent, tile_layout)
laid_out = raster_rdd.tile_to_layout(GlobalLayout(tile_size=16))
result = laid_out.pyramid()
self.pyramid_building_check(result)
class TileLayerMetadataTest(BaseTestClass):
extent = BaseTestClass.extent
layout = BaseTestClass.layout
layout_def = LayoutDefinition(extent, layout)
rdd = BaseTestClass.rdd
projected_extent = BaseTestClass.projected_extent
cols = BaseTestClass.cols
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_collection_avro_rdd(self):
result = self.rdd.collect_metadata(layout=self.layout_def)
self.assertEqual(result.extent, self.extent)
self.assertEqual(result.layout_definition.extent, self.extent)
self.assertEqual(result.layout_definition.tileLayout, self.layout)
@pytest.mark.skipif('TRAVIS' in os.environ,
reason="Test causes memory errors on Travis")
def test_collection_python_rdd(self):
data = rasterio.open(self.dir_path)
tile_dict = Tile(data.read(), 'FLOAT', data.nodata)
rasterio_rdd = self.pysc.parallelize([(self.projected_extent,
tile_dict)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL,
rasterio_rdd)
result = raster_rdd.collect_metadata(layout=self.layout_def)
self.assertEqual(result.extent, self.extent)
self.assertEqual(result.layout_definition.extent, self.extent)
self.assertEqual(result.layout_definition.tileLayout, self.layout)
def test_collection_floating(self):
result = self.rdd.collect_metadata(LocalLayout(self.cols))
self.assertEqual(result.extent, self.extent)
self.assertEqual(result.layout_definition.extent, self.extent)
self.assertEqual(result.layout_definition.tileLayout, self.layout)
def test_pyraminding_with_partitioner(self):
arr = np.zeros((1, 16, 16))
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
tile = Tile(arr, 'FLOAT', False)
projected_extent = ProjectedExtent(extent, epsg_code)
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
tile_layout = TileLayout(32, 32, 16, 16)
new_extent = Extent(-20037508.342789244, -20037508.342789244, 20037508.342789244,
20037508.342789244)
layout_def = LayoutDefinition(new_extent, tile_layout)
laid_out = raster_rdd.tile_to_layout(GlobalLayout(tile_size=16))
strategy = SpatialPartitionStrategy(4)
pyramided = laid_out.pyramid(partition_strategy=strategy)
self.assertEqual(pyramided.levels[0].get_partition_strategy(), strategy)
class MergeTest(BaseTestClass):
arr_1 = np.zeros((1, 4, 4))
arr_2 = np.ones((1, 4, 4))
tile_1 = Tile.from_numpy_array(arr_1)
tile_2 = Tile.from_numpy_array(arr_2)
crs = 4326
time = datetime.datetime.strptime("2016-08-24T09:00:00Z",
'%Y-%m-%dT%H:%M:%SZ')
extents = [
Extent(0.0, 0.0, 4.0, 4.0),
Extent(0.0, 4.0, 4.0, 8.0),
]
extent = Extent(0.0, 0.0, 8.0, 8.0)
layout = TileLayout(2, 2, 5, 5)
ct = 'float32ud-1.0'
md_proj = '+proj=longlat +datum=WGS84 +no_defs '
ld = LayoutDefinition(extent, layout)
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_projected_extent(self):
pes = [
ProjectedExtent(extent=self.extents[0], epsg=self.crs),
ProjectedExtent(extent=self.extents[1], epsg=self.crs),
]
pe_layer = [(pes[0], self.tile_1), (pes[0], self.tile_2),
(pes[1], self.tile_1), (pes[1], self.tile_2)]
rdd = self.pysc.parallelize(pe_layer)
layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
actual = layer.merge()
self.assertEqual(actual.srdd.rdd().count(), 2)
for k, v in actual.to_numpy_rdd().collect():
self.assertTrue((v.cells == self.arr_2).all())
def test_temporal_projected_extent(self):
pes = [
TemporalProjectedExtent(extent=self.extents[0],
epsg=self.crs,
instant=self.time),
TemporalProjectedExtent(extent=self.extents[1],
epsg=self.crs,
instant=self.time),
]
pe_layer = [(pes[0], self.tile_1), (pes[1], self.tile_1),
(pes[0], self.tile_2), (pes[1], self.tile_2)]
rdd = self.pysc.parallelize(pe_layer)
layer = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd)
actual = layer.merge()
self.assertEqual(actual.srdd.rdd().count(), 2)
for k, v in actual.to_numpy_rdd().collect():
self.assertTrue((v.cells == self.arr_2).all())
def test_spatial_keys(self):
keys = [SpatialKey(0, 0), SpatialKey(0, 1)]
key_layer = [(keys[0], self.tile_1), (keys[1], self.tile_1),
(keys[0], self.tile_2), (keys[1], self.tile_2)]
bounds = Bounds(keys[0], keys[1])
md = Metadata(bounds=bounds,
crs=self.md_proj,
cell_type=self.ct,
extent=self.extent,
layout_definition=self.ld)
rdd = self.pysc.parallelize(key_layer)
layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, md)
actual = layer.merge()
self.assertEqual(actual.srdd.rdd().count(), 2)
for k, v in actual.to_numpy_rdd().collect():
self.assertTrue((v.cells == self.arr_2).all())
def test_space_time_keys(self):
temp_keys = [
SpaceTimeKey(0, 0, instant=self.time),
SpaceTimeKey(0, 1, instant=self.time)
]
temp_key_layer = [(temp_keys[0], self.tile_2),
(temp_keys[1], self.tile_2),
(temp_keys[0], self.tile_2),
(temp_keys[1], self.tile_2)]
temp_bounds = Bounds(temp_keys[0], temp_keys[1])
temp_md = Metadata(bounds=temp_bounds,
crs=self.md_proj,
cell_type=self.ct,
extent=self.extent,
layout_definition=self.ld)
rdd = self.pysc.parallelize(temp_key_layer)
layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd,
temp_md)
actual = layer.merge()
self.assertEqual(actual.srdd.rdd().count(), 2)
for k, v in actual.to_numpy_rdd().collect():
self.assertTrue((v.cells == self.arr_2).all())
def load_test_collection(
collection_id: str,
collection_metadata: GeopysparkCubeMetadata,
extent,
srs: str,
from_date: str,
to_date: str,
bands=None,
correlation_id: str = "NA",
) -> Dict[int, geopyspark.TiledRasterLayer]:
"""
Load synthetic data as test collection
:param collection_id:
:param collection_metadata:
:param extent:
:param srs:
:param from_date:
:param to_date:
:param bands:
:param correlation_id:
:return:
"""
# TODO: support more test collections
assert collection_id == "TestCollection-LonLat4x4"
grid_size: float = 1.0
tile_size = 4
# TODO: support other srs'es?
assert srs == "EPSG:4326"
# Get bounds of tiling layout
extent = geopyspark.Extent(extent.xmin(), extent.ymin(), extent.xmax(),
extent.ymax())
col_min = int(math.floor(extent.xmin / grid_size))
row_min = int(math.floor(extent.ymin / grid_size))
col_max = int(math.ceil(extent.xmax / grid_size) - 1)
row_max = int(math.ceil(extent.ymax / grid_size) - 1)
# Simulate sparse range of observation dates
from_date = rfc3339.parse_datetime(rfc3339.datetime(from_date))
to_date = rfc3339.parse_datetime(rfc3339.datetime(to_date))
dates = dates_between(from_date, to_date)
# Build RDD of tiles with requested bands.
tile_builder = TestCollectionLonLat(tile_size=tile_size,
grid_size=grid_size)
bands = bands or [b.name for b in collection_metadata.bands]
rdd_data = [(SpaceTimeKey(col, row, date),
tile_builder.get_tile(bands=bands,
col=col,
row=row,
date=date))
for col in range(col_min, col_max + 1)
for row in range(row_min, row_max + 1) for date in dates]
rdd = SparkContext.getOrCreate().parallelize(rdd_data)
metadata = Metadata(
bounds=Bounds(SpaceTimeKey(col_min, row_min, min(dates)),
SpaceTimeKey(col_max, row_max, max(dates))),
crs="+proj=longlat +datum=WGS84 +no_defs ",
cell_type=CellType.FLOAT64,
extent=extent,
layout_definition=LayoutDefinition(
extent=geopyspark.Extent(col_min * grid_size, row_min * grid_size,
(col_max + 1) * grid_size,
(row_max + 1) * grid_size),
tileLayout=TileLayout(layoutCols=col_max - col_min + 1,
layoutRows=row_max - row_min + 1,
tileCols=tile_size,
tileRows=tile_size)))
layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
return {0: layer}
class ReprojectTest(BaseTestClass):
layout_def = LayoutDefinition(BaseTestClass.extent, BaseTestClass.layout)
metadata = BaseTestClass.rdd.collect_metadata(layout=layout_def)
crs = metadata.crs
expected_crs = "+proj=longlat +ellps=WGS72 +towgs84=0,0,1.9,0,0,0.814,-0.38 +no_defs "
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_repartition(self):
laid_out_rdd = BaseTestClass.rdd.tile_to_layout(self.metadata)
result = laid_out_rdd.repartition(2)
self.assertEqual(result.getNumPartitions(), 2)
def test_same_crs_layout(self):
result = BaseTestClass.rdd.tile_to_layout(layout=self.layout_def,
target_crs="EPSG:4326")
new_metadata = result.layer_metadata
layout_definition = LayoutDefinition(BaseTestClass.extent,
BaseTestClass.layout)
self.assertEqual(layout_definition, new_metadata.layout_definition)
def test_integer_crs(self):
result = BaseTestClass.rdd.tile_to_layout(layout=self.layout_def,
target_crs=4326)
new_metadata = result.layer_metadata
layout_definition = LayoutDefinition(BaseTestClass.extent,
BaseTestClass.layout)
self.assertEqual(layout_definition, new_metadata.layout_definition)
def test_same_crs_zoom(self):
result = BaseTestClass.rdd.tile_to_layout(layout=LocalLayout(
self.cols),
target_crs="EPSG:4326")
new_metadata = result.layer_metadata
self.assertTrue("+datum=WGS84" in new_metadata.crs)
def test_different_crs_layout(self):
result = BaseTestClass.rdd.tile_to_layout(layout=self.layout_def,
target_crs="EPSG:4324")
new_metadata = result.layer_metadata
self.assertEqual(self.expected_crs, new_metadata.crs)
def test_different_crs_zoom(self):
result = BaseTestClass.rdd.tile_to_layout(
layout=GlobalLayout(tile_size=self.cols), target_crs="EPSG:4324")
new_metadata = result.layer_metadata
self.assertEqual(self.expected_crs, new_metadata.crs)
def test_different_crs_float(self):
result = BaseTestClass.rdd.tile_to_layout(layout=LocalLayout(
self.cols),
target_crs="EPSG:4324")
new_metadata = result.layer_metadata
self.assertEqual(self.expected_crs, new_metadata.crs)