class UnionTemporalTest(BaseTestClass): time_1 = datetime.datetime.strptime("1993-09-19T07:01:00Z", '%Y-%m-%dT%H:%M:%SZ') time_2 = datetime.datetime.strptime("2017-09-19T07:01:00Z", '%Y-%m-%dT%H:%M:%SZ') temp_projected_extent_1 = TemporalProjectedExtent(extent, time_1, epsg_code) temp_projected_extent_2 = TemporalProjectedExtent(extent, time_2, epsg_code) arr = np.zeros((1, 16, 16)) tile = Tile(arr, 'FLOAT', -500.0) rdd_1 = BaseTestClass.pysc.parallelize([(temp_projected_extent_1, tile)]) rdd_2 = BaseTestClass.pysc.parallelize([(temp_projected_extent_2, tile)]) layer_1 = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd_1) layer_2 = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd_2) tiled_layer_1 = layer_1.tile_to_layout(GlobalLayout()) tiled_layer_2 = layer_2.tile_to_layout(GlobalLayout()) @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_union_of_raster_layers(self): result = union(self.layer_1, self.layer_2) self.assertTrue(result.srdd.rdd().count(), 2) def test_union_of_tiled_raster_layers(self): result = union(self.tiled_layer_1, self.tiled_layer_2) bounds_1 = self.tiled_layer_1.layer_metadata.bounds bounds_2 = self.tiled_layer_2.layer_metadata.bounds min_col = min(bounds_1.minKey.col, bounds_2.minKey.col) min_row = min(bounds_1.minKey.row, bounds_2.minKey.row) min_instant = min(bounds_1.minKey.instant, bounds_2.minKey.instant) max_col = max(bounds_1.maxKey.col, bounds_2.maxKey.col) max_row = max(bounds_1.maxKey.row, bounds_2.maxKey.row) max_instant = max(bounds_1.maxKey.instant, bounds_2.maxKey.instant) min_key = SpaceTimeKey(min_col, min_row, min_instant) max_key = SpaceTimeKey(max_col, max_row, max_instant) self.assertTrue(result.srdd.rdd().count(), 2) self.assertEqual(result.layer_metadata.bounds, Bounds(min_key, max_key))
class TileToLayoutTest(BaseTestClass): layers = [ make_raster(0, 0, v=1), make_raster(3, 2, v=2), make_raster(6, 0, v=3) ] numpy_rdd = BaseTestClass.pysc.parallelize(layers) layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, numpy_rdd) metadata = layer.collect_metadata(GlobalLayout(5)) def test_to_to_layout_with_partitioner(self): strategy = SpatialPartitionStrategy(4) tiled = self.layer.tile_to_layout(LocalLayout(5), partition_strategy=strategy) self.assertEqual(tiled.get_partition_strategy(), strategy) def test_tile_to_local_layout(self): tiled = self.layer.tile_to_layout(LocalLayout(5)) assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6) assert tiled.layer_metadata.tile_layout == TileLayout(2, 2, 5, 5) def test_tile_to_global_layout(self): tiled = self.layer.tile_to_layout(GlobalLayout(5)) assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6) assert tiled.layer_metadata.tile_layout == TileLayout(128, 128, 5, 5) assert tiled.zoom_level == 7 def test_tile_to_metadata_layout(self): tiled = self.layer.tile_to_layout(layout=self.metadata) self.assertEqual(tiled.layer_metadata.extent, Extent(0, 0, 10, 6)) self.assertDictEqual(tiled.layer_metadata.to_dict(), self.metadata.to_dict()) def test_tile_to_tiled_layer_layout(self): extent = Extent(0., 0., 10., 6.) tile_layout = TileLayout(2, 2, 5, 5) layout_definition = LayoutDefinition(extent, tile_layout) base = self.layer.tile_to_layout(layout_definition) tiled = self.layer.tile_to_layout(layout=base) self.assertDictEqual(tiled.layer_metadata.to_dict(), base.layer_metadata.to_dict()) def test_tile_to_layout_definition(self): tiled = self.layer.tile_to_layout( layout=self.metadata.layout_definition) self.assertDictEqual(tiled.layer_metadata.to_dict(), self.metadata.to_dict()) @pytest.fixture(scope='class', autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close()
class UnionSpatialTest(BaseTestClass): projected_extent_1 = ProjectedExtent(extent, epsg_code) projected_extent_2 = ProjectedExtent(extent_2, epsg_code) arr = np.zeros((1, 16, 16)) tile = Tile(arr, 'FLOAT', -500.0) rdd_1 = BaseTestClass.pysc.parallelize([(projected_extent_1, tile)]) rdd_2 = BaseTestClass.pysc.parallelize([(projected_extent_2, tile)]) layer_1 = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd_1) layer_2 = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd_2) tiled_layer_1 = layer_1.tile_to_layout(GlobalLayout()) tiled_layer_2 = layer_2.tile_to_layout(GlobalLayout()) @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_union_of_raster_layers(self): result = union(self.layer_1, self.layer_2) self.assertTrue(result.srdd.rdd().count(), 2) def test_union_of_tiled_raster_layers(self): result = union(self.tiled_layer_1, self.tiled_layer_2) bounds_1 = self.tiled_layer_1.layer_metadata.bounds bounds_2 = self.tiled_layer_2.layer_metadata.bounds min_col = min(bounds_1.minKey.col, bounds_2.minKey.col) min_row = min(bounds_1.minKey.row, bounds_2.minKey.row) max_col = max(bounds_1.maxKey.col, bounds_2.maxKey.col) max_row = max(bounds_1.maxKey.row, bounds_2.maxKey.row) min_key = SpatialKey(min_col, min_row) max_key = SpatialKey(max_col, max_row) self.assertTrue(result.srdd.rdd().count(), 2) self.assertEqual(result.layer_metadata.bounds, Bounds(min_key, max_key))
def test_correct_base(self): arr = np.zeros((1, 16, 16)) epsg_code = 3857 extent = Extent(0.0, 0.0, 10.0, 10.0) tile = Tile(arr, 'FLOAT', False) projected_extent = ProjectedExtent(extent, epsg_code) rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)]) raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd) tile_layout = TileLayout(32, 32, 16, 16) new_extent = Extent(-20037508.342789244, -20037508.342789244, 20037508.342789244, 20037508.342789244) layout_def = LayoutDefinition(new_extent, tile_layout) laid_out = raster_rdd.tile_to_layout(GlobalLayout(tile_size=16)) result = laid_out.pyramid() self.pyramid_building_check(result)
def test_pyramid_class(self): arr = np.zeros((1, 16, 16)) epsg_code = 3857 extent = Extent(0.0, 0.0, 10.0, 10.0) tile = Tile(arr, 'FLOAT', False) projected_extent = ProjectedExtent(extent, epsg_code) rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)]) raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd) tile_layout = TileLayout(1, 1, 16, 16) reprojected = raster_rdd.tile_to_layout(layout=GlobalLayout(tile_size=16), target_crs=3857) result = reprojected.pyramid() hist = result.get_histogram() self.assertEqual(result.max_zoom, reprojected.zoom_level) self.assertTrue(set(result.levels.keys()).issuperset(range(1, 13))) self.assertEqual(hist.mean(), 0.0) self.assertEqual(hist.min_max(), (0.0, 0.0))
def test_write_pyramid_layers(self): max_zoom = 5 tif = file_path('srtm_52_11.tif') raster_layer = geotiff.get(layer_type=LayerType.SPATIAL, uri=tif) tiled_raster_layer = raster_layer.tile_to_layout( GlobalLayout(zoom=max_zoom), target_crs=3857) pyramided_layer = tiled_raster_layer.pyramid() layer_name = 'pyramid-test-layer' path = file_path('pyramid-test-catalog') uri = 'file:///' + path if os.path.isdir(path): import shutil shutil.rmtree(path) pyramided_layer.write(uri, layer_name) self.assertTrue(catalog.read_layer_metadata(uri, layer_name, 0)) self.assertTrue(catalog.read_layer_metadata(uri, layer_name, max_zoom))
def test_pyraminding_with_partitioner(self): arr = np.zeros((1, 16, 16)) epsg_code = 3857 extent = Extent(0.0, 0.0, 10.0, 10.0) tile = Tile(arr, 'FLOAT', False) projected_extent = ProjectedExtent(extent, epsg_code) rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)]) raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd) tile_layout = TileLayout(32, 32, 16, 16) new_extent = Extent(-20037508.342789244, -20037508.342789244, 20037508.342789244, 20037508.342789244) layout_def = LayoutDefinition(new_extent, tile_layout) laid_out = raster_rdd.tile_to_layout(GlobalLayout(tile_size=16)) strategy = SpatialPartitionStrategy(4) pyramided = laid_out.pyramid(partition_strategy=strategy) self.assertEqual(pyramided.levels[0].get_partition_strategy(), strategy)
def test_read_no_reproject_global_gdal(self): self.read(GlobalLayout(tile_size=128, zoom=4), ReadMethod.GDAL)
class RasterLayerTest(BaseTestClass): layers = [ make_raster(0, 0, v=1), make_raster(3, 2, v=2), make_raster(6, 0, v=3) ] # TODO: Have Travis be able to run the GDAL tests numpy_rdd = BaseTestClass.pysc.parallelize(layers) layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, numpy_rdd) metadata = layer.collect_metadata(GlobalLayout(5)) def read_no_reproject(self, read_method): actual_raster_layer = RasterLayer.read([self.path], read_method=read_method) collected = actual_raster_layer.to_numpy_rdd().first() (projected_extent, tile) = collected self.assertEqual(projected_extent.proj4, self.projected_extent.proj4) self.assertTrue((self.expected_tile == tile.cells).all()) def read_with_reproject(self, read_method): expected_raster_layer = self.rdd.reproject(target_crs=3857) expected_collected = expected_raster_layer.to_numpy_rdd().first() (expected_projected_extent, expected_tile) = expected_collected actual_raster_layer = RasterLayer.read([self.path], target_crs=3857, read_method=read_method) actual_collected = actual_raster_layer.to_numpy_rdd().first() (actual_projected_extent, tile) = actual_collected self.assertEqual(actual_projected_extent.epsg, expected_projected_extent.epsg) self.assertTrue((expected_tile.cells == tile.cells).all()) # No reprojection def test_read_no_reproject_geotrellis(self): self.read_no_reproject(ReadMethod.GEOTRELLIS) def test_read_no_reproject_gdal(self): self.read_no_reproject(ReadMethod.GDAL) # With reprojection def test_read_with_reproject_geotrellis(self): self.read_with_reproject(ReadMethod.GEOTRELLIS) def test_read_with_reproject_gdal(self): self.read_with_reproject(ReadMethod.GDAL) def test_no_data_of_zero(self): no_data_layer = [(t[0], Tile.from_numpy_array(t[1].cells, 1)) for t in self.layers] rdd = BaseTestClass.pysc.parallelize(no_data_layer) nd_layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd) nd_metadata = nd_layer.collect_metadata() self.assertTrue('ud1' in nd_metadata.cell_type) self.assertEqual(nd_metadata.no_data_value, 1) @pytest.fixture(scope='class', autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close()
class CatalogTest(BaseTestClass): rdd = get(LayerType.SPATIAL, geotiff_test_path("srtm_52_11.tif")) metadata = rdd.collect_metadata() reprojected = rdd.tile_to_layout(layout=GlobalLayout(zoom=11), target_crs="EPSG:3857") result = reprojected.pyramid() dir_path = geotiff_test_path("catalog/") uri = "file://{}".format(dir_path) layer_name = "catalog-test" @pytest.fixture(autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close() def test_read(self): for x in range(11, 0, -1): actual_layer = query(self.uri, self.layer_name, x) expected_layer = self.result.levels[x] self.assertDictEqual(actual_layer.layer_metadata.to_dict(), expected_layer.layer_metadata.to_dict()) def test_read_value(self): tiled = read_value(self.uri, self.layer_name, 11, 1450, 966) self.assertEqual(tiled.cells.shape, (1, 256, 256)) def test_bad_read_value(self): tiled = read_value(self.uri, self.layer_name, 11, 1450, 2000) self.assertEqual(tiled, None) @pytest.mark.skipif('TRAVIS' in os.environ, reason="test_query_1 causes issues on Travis") def test_query1(self): intersection = box(8348915.46680623, 543988.943201519, 8348915.4669, 543988.943201520) queried = query(self.uri, self.layer_name, 11, intersection) self.assertEqual(queried.to_numpy_rdd().first()[0], SpatialKey(1450, 996)) def test_query2(self): intersection = Extent(8348915.46680623, 543988.943201519, 8348915.4669, 543988.943201520) queried = query(self.uri, self.layer_name, 11, intersection, query_proj=3857) self.assertEqual(queried.to_numpy_rdd().first()[0], SpatialKey(1450, 996)) def test_query3(self): intersection = box(8348915.46680623, 543988.943201519, 8348915.4669, 543988.943201520).wkb queried = query(self.uri, self.layer_name, 11, intersection) self.assertEqual(queried.to_numpy_rdd().first()[0], SpatialKey(1450, 996)) def test_query4(self): intersection = 42 with pytest.raises(TypeError): queried = query(self.uri, self.layer_name, 11, query_geom=intersection, num_partitions=2) result = queried.to_numpy_rdd().first()[0] def test_query_partitions(self): intersection = box(8348915.46680623, 543988.943201519, 8348915.4669, 543988.943201520) queried = query(self.uri, self.layer_name, 11, intersection, num_partitions=2) self.assertEqual(queried.to_numpy_rdd().first()[0], SpatialKey(1450, 996)) def test_query_crs(self): intersection = box(8348915.46680623, 543988.943201519, 8348915.4669, 543988.943201520) queried = query(self.uri, self.layer_name, 11, intersection, query_proj=3857) self.assertEqual(queried.to_numpy_rdd().first()[0], SpatialKey(1450, 996)) def test_read_metadata_exception(self): uri = "abcxyz://123" with pytest.raises(ValueError): layer = read_layer_metadata(uri, self.layer_name, 5) def test_read_metadata1(self): layer = query(self.uri, self.layer_name, 5) actual_metadata = layer.layer_metadata expected_metadata = read_layer_metadata(self.uri, self.layer_name, 5) def test_read_metadata2(self): layer = query(self.uri, self.layer_name, 5) actual_metadata = layer.layer_metadata expected_metadata = read_layer_metadata(self.uri, self.layer_name, 5) self.assertEqual(actual_metadata.to_dict(), expected_metadata.to_dict()) def test_layer_ids(self): ids = AttributeStore(self.uri).layers() self.assertTrue(len(ids) == 11) def test_attributestore(self): store = AttributeStore(self.uri) layer_name = "boop-epsg-bop" value = {"first": 113, "second": "44two"} store.layer(layer_name, 34).write("val", value) self.assertEqual(value, store.layer(layer_name, 34).read("val")) self.assertEqual(value, store.layer(layer_name, 34)["val"]) store.layer(layer_name, 34).delete("val") with pytest.raises(KeyError): store.layer(layer_name, 34)["val"]
def test_tile_to_global_layout(self): tiled = self.layer.tile_to_layout(GlobalLayout(5)) assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6) assert tiled.layer_metadata.tile_layout == TileLayout(128, 128, 5, 5) assert tiled.zoom_level == 7
def test_tile_to_layout_global_layout(self): actual = self.tiled_layer.tile_to_layout(layout=GlobalLayout(zoom=5)) self.assertEqual(actual.zoom_level, 5)
def test_read_with_reproject_global_gdal(self): self.read(GlobalLayout(tile_size=128, zoom=4), ReadMethod.GDAL, target_crs=3857)
def test_tile_to_layout_with_reproject(self): proj4 = crs_to_proj4(3857) actual = self.result.tile_to_layout(layout=GlobalLayout(), target_crs=proj4).layer_metadata.crs self.assertEqual(proj4, actual)
def test_read_with_reproject_global_geotrellis(self): self.read(GlobalLayout(tile_size=128, zoom=4), ReadMethod.GEOTRELLIS, target_crs=3857)
def test_ordered_read_no_reproject_global_geotrellis(self): self.read(GlobalLayout(tile_size=16, zoom=4), ReadMethod.GEOTRELLIS, multiplex=True)
def test_read_no_reproject_global_geotrellis(self): self.read(GlobalLayout(tile_size=16, zoom=4), ReadMethod.GEOTRELLIS)
def test_different_crs_zoom(self): result = BaseTestClass.rdd.tile_to_layout( layout=GlobalLayout(tile_size=self.cols), target_crs="EPSG:4324") new_metadata = result.layer_metadata self.assertEqual(self.expected_crs, new_metadata.crs)
class RasterLayerTest(BaseTestClass): layers = [ make_raster(0, 0, v=1), make_raster(3, 2, v=2), make_raster(6, 0, v=3) ] numpy_rdd = BaseTestClass.pysc.parallelize(layers) layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, numpy_rdd) metadata = layer.collect_metadata(GlobalLayout(5)) def test_tile_to_local_layout(self): tiled = self.layer.tile_to_layout(LocalLayout(5)) assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6) assert tiled.layer_metadata.tile_layout == TileLayout(2, 2, 5, 5) def test_tile_to_global_layout(self): tiled = self.layer.tile_to_layout(GlobalLayout(5)) assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6) assert tiled.layer_metadata.tile_layout == TileLayout(128, 128, 5, 5) assert tiled.zoom_level == 7 def test_tile_to_metadata_layout(self): tiled = self.layer.tile_to_layout(layout=self.metadata) self.assertEqual(tiled.layer_metadata.extent, Extent(0, 0, 10, 6)) self.assertDictEqual(tiled.layer_metadata.to_dict(), self.metadata.to_dict()) def test_tile_to_tiled_layer_layout(self): extent = Extent(0., 0., 10., 6.) tile_layout = TileLayout(2, 2, 5, 5) layout_definition = LayoutDefinition(extent, tile_layout) base = self.layer.tile_to_layout(layout_definition) tiled = self.layer.tile_to_layout(layout=base) self.assertDictEqual(tiled.layer_metadata.to_dict(), base.layer_metadata.to_dict()) def test_tile_to_layout_definition(self): tiled = self.layer.tile_to_layout( layout=self.metadata.layout_definition) self.assertDictEqual(tiled.layer_metadata.to_dict(), self.metadata.to_dict()) def test_no_data_of_zero(self): no_data_layer = [(t[0], Tile.from_numpy_array(t[1].cells, 1)) for t in self.layers] rdd = BaseTestClass.pysc.parallelize(no_data_layer) nd_layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd) nd_metadata = nd_layer.collect_metadata() self.assertTrue('ud1' in nd_metadata.cell_type) self.assertEqual(nd_metadata.no_data_value, 1) @pytest.fixture(scope='class', autouse=True) def tearDown(self): yield BaseTestClass.pysc._gateway.close()