class BaseTestClass(unittest.TestCase):
if 'TRAVIS' in os.environ:
master_str = "local[2]"
else:
master_str = "local[*]"
conf = geopyspark_conf(master=master_str, appName="test")
conf.set('spark.kryoserializer.buffer.max', value='1G')
conf.set('spark.ui.enabled', True)
if 'TRAVIS' in os.environ:
conf.set(key='spark.driver.memory', value='1G')
conf.set(key='spark.executor.memory', value='1G')
pysc = SparkContext(conf=conf)
dir_path = file_path("all-ones.tif")
rdd = get(LayerType.SPATIAL, dir_path, max_tile_size=1024)
value = rdd.to_numpy_rdd().collect()[0]
projected_extent = value[0]
extent = projected_extent.extent
expected_tile = value[1].cells
(_, rows, cols) = expected_tile.shape
layout = TileLayout(1, 1, cols, rows)
def test_tile_to_tiled_layer_layout(self):
extent = Extent(0., 0., 10., 6.)
tile_layout = TileLayout(2, 2, 5, 5)
layout_definition = LayoutDefinition(extent, tile_layout)
base = self.layer.tile_to_layout(layout_definition)
tiled = self.layer.tile_to_layout(layout=base)
self.assertDictEqual(tiled.layer_metadata.to_dict(),
base.layer_metadata.to_dict())
def test_correct_base(self):
arr = np.zeros((1, 16, 16))
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
tile = Tile(arr, 'FLOAT', False)
projected_extent = ProjectedExtent(extent, epsg_code)
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
tile_layout = TileLayout(32, 32, 16, 16)
new_extent = Extent(-20037508.342789244, -20037508.342789244,
20037508.342789244, 20037508.342789244)
layout_def = LayoutDefinition(new_extent, tile_layout)
laid_out = raster_rdd.tile_to_layout(GlobalLayout(tile_size=16))
result = laid_out.pyramid()
self.pyramid_building_check(result)
class BaseTestClass(unittest.TestCase):
if 'TRAVIS' in os.environ:
master_str = "local[2]"
else:
master_str = "local[*]"
geopysc = GeoPyContext(master=master_str, appName="test")
dir_path = geotiff_test_path("all-ones.tif")
rdd = get(geopysc, SPATIAL, dir_path)
value = rdd.to_numpy_rdd().collect()[0]
projected_extent = value[0]
extent = projected_extent.extent
expected_tile = value[1]['data']
(_, rows, cols) = expected_tile.shape
layout = TileLayout(1, 1, cols, rows)
def test_pyramid_class(self):
arr = np.zeros((1, 16, 16))
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
tile = Tile(arr, 'FLOAT', False)
projected_extent = ProjectedExtent(extent, epsg_code)
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
tile_layout = TileLayout(1, 1, 16, 16)
reprojected = raster_rdd.tile_to_layout(layout=GlobalLayout(tile_size=16), target_crs=3857)
result = reprojected.pyramid()
hist = result.get_histogram()
self.assertEqual(result.max_zoom, reprojected.zoom_level)
self.assertTrue(set(result.levels.keys()).issuperset(range(1, 13)))
self.assertEqual(hist.mean(), 0.0)
self.assertEqual(hist.min_max(), (0.0, 0.0))
def test_pyraminding_with_partitioner(self):
arr = np.zeros((1, 16, 16))
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
tile = Tile(arr, 'FLOAT', False)
projected_extent = ProjectedExtent(extent, epsg_code)
rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
tile_layout = TileLayout(32, 32, 16, 16)
new_extent = Extent(-20037508.342789244, -20037508.342789244, 20037508.342789244,
20037508.342789244)
layout_def = LayoutDefinition(new_extent, tile_layout)
laid_out = raster_rdd.tile_to_layout(GlobalLayout(tile_size=16))
strategy = SpatialPartitionStrategy(4)
pyramided = laid_out.pyramid(partition_strategy=strategy)
self.assertEqual(pyramided.levels[0].get_partition_strategy(), strategy)
def test_correct_base(self):
arr = np.zeros((1, 16, 16))
epsg_code = 3857
extent = Extent(0.0, 0.0, 10.0, 10.0)
tile = {'data': arr, 'no_data_value': False}
projected_extent = {'extent': extent, 'epsg': epsg_code}
rdd = BaseTestClass.geopysc.pysc.parallelize([(projected_extent, tile)
])
raster_rdd = RasterRDD.from_numpy_rdd(BaseTestClass.geopysc, SPATIAL,
rdd)
tile_layout = TileLayout(32, 32, 16, 16)
new_extent = Extent(-20037508.342789244, -20037508.342789244,
20037508.342789244, 20037508.342789244)
metadata = raster_rdd.collect_metadata(extent=new_extent,
layout=tile_layout)
laid_out = raster_rdd.tile_to_layout(metadata)
result = laid_out.pyramid(start_zoom=5, end_zoom=1)
self.pyramid_building_check(result)
class MergeTest(BaseTestClass):
arr_1 = np.zeros((1, 4, 4))
arr_2 = np.ones((1, 4, 4))
tile_1 = Tile.from_numpy_array(arr_1)
tile_2 = Tile.from_numpy_array(arr_2)
crs = 4326
time = datetime.datetime.strptime("2016-08-24T09:00:00Z",
'%Y-%m-%dT%H:%M:%SZ')
extents = [
Extent(0.0, 0.0, 4.0, 4.0),
Extent(0.0, 4.0, 4.0, 8.0),
]
extent = Extent(0.0, 0.0, 8.0, 8.0)
layout = TileLayout(2, 2, 5, 5)
ct = 'float32ud-1.0'
md_proj = '+proj=longlat +datum=WGS84 +no_defs '
ld = LayoutDefinition(extent, layout)
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.pysc._gateway.close()
def test_projected_extent(self):
pes = [
ProjectedExtent(extent=self.extents[0], epsg=self.crs),
ProjectedExtent(extent=self.extents[1], epsg=self.crs),
]
pe_layer = [(pes[0], self.tile_1), (pes[0], self.tile_2),
(pes[1], self.tile_1), (pes[1], self.tile_2)]
rdd = self.pysc.parallelize(pe_layer)
layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
actual = layer.merge()
self.assertEqual(actual.srdd.rdd().count(), 2)
for k, v in actual.to_numpy_rdd().collect():
self.assertTrue((v.cells == self.arr_2).all())
def test_temporal_projected_extent(self):
pes = [
TemporalProjectedExtent(extent=self.extents[0],
epsg=self.crs,
instant=self.time),
TemporalProjectedExtent(extent=self.extents[1],
epsg=self.crs,
instant=self.time),
]
pe_layer = [(pes[0], self.tile_1), (pes[1], self.tile_1),
(pes[0], self.tile_2), (pes[1], self.tile_2)]
rdd = self.pysc.parallelize(pe_layer)
layer = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd)
actual = layer.merge()
self.assertEqual(actual.srdd.rdd().count(), 2)
for k, v in actual.to_numpy_rdd().collect():
self.assertTrue((v.cells == self.arr_2).all())
def test_spatial_keys(self):
keys = [SpatialKey(0, 0), SpatialKey(0, 1)]
key_layer = [(keys[0], self.tile_1), (keys[1], self.tile_1),
(keys[0], self.tile_2), (keys[1], self.tile_2)]
bounds = Bounds(keys[0], keys[1])
md = Metadata(bounds=bounds,
crs=self.md_proj,
cell_type=self.ct,
extent=self.extent,
layout_definition=self.ld)
rdd = self.pysc.parallelize(key_layer)
layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd, md)
actual = layer.merge()
self.assertEqual(actual.srdd.rdd().count(), 2)
for k, v in actual.to_numpy_rdd().collect():
self.assertTrue((v.cells == self.arr_2).all())
def test_space_time_keys(self):
temp_keys = [
SpaceTimeKey(0, 0, instant=self.time),
SpaceTimeKey(0, 1, instant=self.time)
]
temp_key_layer = [(temp_keys[0], self.tile_2),
(temp_keys[1], self.tile_2),
(temp_keys[0], self.tile_2),
(temp_keys[1], self.tile_2)]
temp_bounds = Bounds(temp_keys[0], temp_keys[1])
temp_md = Metadata(bounds=temp_bounds,
crs=self.md_proj,
cell_type=self.ct,
extent=self.extent,
layout_definition=self.ld)
rdd = self.pysc.parallelize(temp_key_layer)
layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd,
temp_md)
actual = layer.merge()
self.assertEqual(actual.srdd.rdd().count(), 2)
for k, v in actual.to_numpy_rdd().collect():
self.assertTrue((v.cells == self.arr_2).all())
def test_tile_to_global_layout(self):
tiled = self.layer.tile_to_layout(GlobalLayout(5))
assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6)
assert tiled.layer_metadata.tile_layout == TileLayout(128, 128, 5, 5)
assert tiled.zoom_level == 7
def test_tile_to_local_layout(self):
tiled = self.layer.tile_to_layout(LocalLayout(5))
assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6)
assert tiled.layer_metadata.tile_layout == TileLayout(2, 2, 5, 5)
def load_test_collection(
collection_id: str,
collection_metadata: GeopysparkCubeMetadata,
extent,
srs: str,
from_date: str,
to_date: str,
bands=None,
correlation_id: str = "NA",
) -> Dict[int, geopyspark.TiledRasterLayer]:
"""
Load synthetic data as test collection
:param collection_id:
:param collection_metadata:
:param extent:
:param srs:
:param from_date:
:param to_date:
:param bands:
:param correlation_id:
:return:
"""
# TODO: support more test collections
assert collection_id == "TestCollection-LonLat4x4"
grid_size: float = 1.0
tile_size = 4
# TODO: support other srs'es?
assert srs == "EPSG:4326"
# Get bounds of tiling layout
extent = geopyspark.Extent(extent.xmin(), extent.ymin(), extent.xmax(),
extent.ymax())
col_min = int(math.floor(extent.xmin / grid_size))
row_min = int(math.floor(extent.ymin / grid_size))
col_max = int(math.ceil(extent.xmax / grid_size) - 1)
row_max = int(math.ceil(extent.ymax / grid_size) - 1)
# Simulate sparse range of observation dates
from_date = rfc3339.parse_datetime(rfc3339.datetime(from_date))
to_date = rfc3339.parse_datetime(rfc3339.datetime(to_date))
dates = dates_between(from_date, to_date)
# Build RDD of tiles with requested bands.
tile_builder = TestCollectionLonLat(tile_size=tile_size,
grid_size=grid_size)
bands = bands or [b.name for b in collection_metadata.bands]
rdd_data = [(SpaceTimeKey(col, row, date),
tile_builder.get_tile(bands=bands,
col=col,
row=row,
date=date))
for col in range(col_min, col_max + 1)
for row in range(row_min, row_max + 1) for date in dates]
rdd = SparkContext.getOrCreate().parallelize(rdd_data)
metadata = Metadata(
bounds=Bounds(SpaceTimeKey(col_min, row_min, min(dates)),
SpaceTimeKey(col_max, row_max, max(dates))),
crs="+proj=longlat +datum=WGS84 +no_defs ",
cell_type=CellType.FLOAT64,
extent=extent,
layout_definition=LayoutDefinition(
extent=geopyspark.Extent(col_min * grid_size, row_min * grid_size,
(col_max + 1) * grid_size,
(row_max + 1) * grid_size),
tileLayout=TileLayout(layoutCols=col_max - col_min + 1,
layoutRows=row_max - row_min + 1,
tileCols=tile_size,
tileRows=tile_size)))
layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
return {0: layer}
