Esempio n. 1
0
class UnionTemporalTest(BaseTestClass):
    time_1 = datetime.datetime.strptime("1993-09-19T07:01:00Z",
                                        '%Y-%m-%dT%H:%M:%SZ')
    time_2 = datetime.datetime.strptime("2017-09-19T07:01:00Z",
                                        '%Y-%m-%dT%H:%M:%SZ')

    temp_projected_extent_1 = TemporalProjectedExtent(extent, time_1,
                                                      epsg_code)
    temp_projected_extent_2 = TemporalProjectedExtent(extent, time_2,
                                                      epsg_code)

    arr = np.zeros((1, 16, 16))
    tile = Tile(arr, 'FLOAT', -500.0)

    rdd_1 = BaseTestClass.pysc.parallelize([(temp_projected_extent_1, tile)])
    rdd_2 = BaseTestClass.pysc.parallelize([(temp_projected_extent_2, tile)])

    layer_1 = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd_1)
    layer_2 = RasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd_2)

    tiled_layer_1 = layer_1.tile_to_layout(GlobalLayout())
    tiled_layer_2 = layer_2.tile_to_layout(GlobalLayout())

    @pytest.fixture(autouse=True)
    def tearDown(self):
        yield
        BaseTestClass.pysc._gateway.close()

    def test_union_of_raster_layers(self):
        result = union(self.layer_1, self.layer_2)

        self.assertTrue(result.srdd.rdd().count(), 2)

    def test_union_of_tiled_raster_layers(self):
        result = union(self.tiled_layer_1, self.tiled_layer_2)

        bounds_1 = self.tiled_layer_1.layer_metadata.bounds
        bounds_2 = self.tiled_layer_2.layer_metadata.bounds

        min_col = min(bounds_1.minKey.col, bounds_2.minKey.col)
        min_row = min(bounds_1.minKey.row, bounds_2.minKey.row)
        min_instant = min(bounds_1.minKey.instant, bounds_2.minKey.instant)

        max_col = max(bounds_1.maxKey.col, bounds_2.maxKey.col)
        max_row = max(bounds_1.maxKey.row, bounds_2.maxKey.row)
        max_instant = max(bounds_1.maxKey.instant, bounds_2.maxKey.instant)

        min_key = SpaceTimeKey(min_col, min_row, min_instant)
        max_key = SpaceTimeKey(max_col, max_row, max_instant)

        self.assertTrue(result.srdd.rdd().count(), 2)
        self.assertEqual(result.layer_metadata.bounds,
                         Bounds(min_key, max_key))
Esempio n. 2
0
class TileToLayoutTest(BaseTestClass):
    layers = [
        make_raster(0, 0, v=1),
        make_raster(3, 2, v=2),
        make_raster(6, 0, v=3)
    ]

    numpy_rdd = BaseTestClass.pysc.parallelize(layers)
    layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, numpy_rdd)
    metadata = layer.collect_metadata(GlobalLayout(5))

    def test_to_to_layout_with_partitioner(self):
        strategy = SpatialPartitionStrategy(4)
        tiled = self.layer.tile_to_layout(LocalLayout(5),
                                          partition_strategy=strategy)

        self.assertEqual(tiled.get_partition_strategy(), strategy)

    def test_tile_to_local_layout(self):
        tiled = self.layer.tile_to_layout(LocalLayout(5))
        assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6)
        assert tiled.layer_metadata.tile_layout == TileLayout(2, 2, 5, 5)

    def test_tile_to_global_layout(self):
        tiled = self.layer.tile_to_layout(GlobalLayout(5))
        assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6)
        assert tiled.layer_metadata.tile_layout == TileLayout(128, 128, 5, 5)
        assert tiled.zoom_level == 7

    def test_tile_to_metadata_layout(self):
        tiled = self.layer.tile_to_layout(layout=self.metadata)

        self.assertEqual(tiled.layer_metadata.extent, Extent(0, 0, 10, 6))
        self.assertDictEqual(tiled.layer_metadata.to_dict(),
                             self.metadata.to_dict())

    def test_tile_to_tiled_layer_layout(self):
        extent = Extent(0., 0., 10., 6.)
        tile_layout = TileLayout(2, 2, 5, 5)
        layout_definition = LayoutDefinition(extent, tile_layout)

        base = self.layer.tile_to_layout(layout_definition)
        tiled = self.layer.tile_to_layout(layout=base)

        self.assertDictEqual(tiled.layer_metadata.to_dict(),
                             base.layer_metadata.to_dict())

    def test_tile_to_layout_definition(self):
        tiled = self.layer.tile_to_layout(
            layout=self.metadata.layout_definition)

        self.assertDictEqual(tiled.layer_metadata.to_dict(),
                             self.metadata.to_dict())

    @pytest.fixture(scope='class', autouse=True)
    def tearDown(self):
        yield
        BaseTestClass.pysc._gateway.close()
Esempio n. 3
0
class UnionSpatialTest(BaseTestClass):
    projected_extent_1 = ProjectedExtent(extent, epsg_code)
    projected_extent_2 = ProjectedExtent(extent_2, epsg_code)

    arr = np.zeros((1, 16, 16))
    tile = Tile(arr, 'FLOAT', -500.0)

    rdd_1 = BaseTestClass.pysc.parallelize([(projected_extent_1, tile)])
    rdd_2 = BaseTestClass.pysc.parallelize([(projected_extent_2, tile)])

    layer_1 = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd_1)
    layer_2 = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd_2)

    tiled_layer_1 = layer_1.tile_to_layout(GlobalLayout())
    tiled_layer_2 = layer_2.tile_to_layout(GlobalLayout())

    @pytest.fixture(autouse=True)
    def tearDown(self):
        yield
        BaseTestClass.pysc._gateway.close()

    def test_union_of_raster_layers(self):
        result = union(self.layer_1, self.layer_2)

        self.assertTrue(result.srdd.rdd().count(), 2)

    def test_union_of_tiled_raster_layers(self):
        result = union(self.tiled_layer_1, self.tiled_layer_2)

        bounds_1 = self.tiled_layer_1.layer_metadata.bounds
        bounds_2 = self.tiled_layer_2.layer_metadata.bounds

        min_col = min(bounds_1.minKey.col, bounds_2.minKey.col)
        min_row = min(bounds_1.minKey.row, bounds_2.minKey.row)
        max_col = max(bounds_1.maxKey.col, bounds_2.maxKey.col)
        max_row = max(bounds_1.maxKey.row, bounds_2.maxKey.row)

        min_key = SpatialKey(min_col, min_row)
        max_key = SpatialKey(max_col, max_row)

        self.assertTrue(result.srdd.rdd().count(), 2)
        self.assertEqual(result.layer_metadata.bounds,
                         Bounds(min_key, max_key))
Esempio n. 4
0
    def test_correct_base(self):
        arr = np.zeros((1, 16, 16))
        epsg_code = 3857
        extent = Extent(0.0, 0.0, 10.0, 10.0)

        tile = Tile(arr, 'FLOAT', False)
        projected_extent = ProjectedExtent(extent, epsg_code)

        rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
        raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
        tile_layout = TileLayout(32, 32, 16, 16)
        new_extent = Extent(-20037508.342789244, -20037508.342789244,
                            20037508.342789244, 20037508.342789244)
        layout_def = LayoutDefinition(new_extent, tile_layout)

        laid_out = raster_rdd.tile_to_layout(GlobalLayout(tile_size=16))
        result = laid_out.pyramid()
        self.pyramid_building_check(result)
Esempio n. 5
0
    def test_pyramid_class(self):
        arr = np.zeros((1, 16, 16))
        epsg_code = 3857
        extent = Extent(0.0, 0.0, 10.0, 10.0)

        tile = Tile(arr, 'FLOAT', False)
        projected_extent = ProjectedExtent(extent, epsg_code)

        rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
        raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
        tile_layout = TileLayout(1, 1, 16, 16)
        reprojected = raster_rdd.tile_to_layout(layout=GlobalLayout(tile_size=16), target_crs=3857)

        result = reprojected.pyramid()
        hist = result.get_histogram()

        self.assertEqual(result.max_zoom, reprojected.zoom_level)
        self.assertTrue(set(result.levels.keys()).issuperset(range(1, 13)))
        self.assertEqual(hist.mean(), 0.0)
        self.assertEqual(hist.min_max(), (0.0, 0.0))
Esempio n. 6
0
    def test_write_pyramid_layers(self):
        max_zoom = 5
        tif = file_path('srtm_52_11.tif')
        raster_layer = geotiff.get(layer_type=LayerType.SPATIAL, uri=tif)
        tiled_raster_layer = raster_layer.tile_to_layout(
            GlobalLayout(zoom=max_zoom), target_crs=3857)
        pyramided_layer = tiled_raster_layer.pyramid()

        layer_name = 'pyramid-test-layer'
        path = file_path('pyramid-test-catalog')
        uri = 'file:///' + path

        if os.path.isdir(path):
            import shutil
            shutil.rmtree(path)

        pyramided_layer.write(uri, layer_name)

        self.assertTrue(catalog.read_layer_metadata(uri, layer_name, 0))
        self.assertTrue(catalog.read_layer_metadata(uri, layer_name, max_zoom))
Esempio n. 7
0
    def test_pyraminding_with_partitioner(self):
        arr = np.zeros((1, 16, 16))
        epsg_code = 3857
        extent = Extent(0.0, 0.0, 10.0, 10.0)

        tile = Tile(arr, 'FLOAT', False)
        projected_extent = ProjectedExtent(extent, epsg_code)

        rdd = BaseTestClass.pysc.parallelize([(projected_extent, tile)])
        raster_rdd = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
        tile_layout = TileLayout(32, 32, 16, 16)
        new_extent = Extent(-20037508.342789244, -20037508.342789244, 20037508.342789244,
                            20037508.342789244)

        layout_def = LayoutDefinition(new_extent, tile_layout)
        laid_out = raster_rdd.tile_to_layout(GlobalLayout(tile_size=16))

        strategy = SpatialPartitionStrategy(4)

        pyramided = laid_out.pyramid(partition_strategy=strategy)

        self.assertEqual(pyramided.levels[0].get_partition_strategy(), strategy)
 def test_read_no_reproject_global_gdal(self):
     self.read(GlobalLayout(tile_size=128, zoom=4), ReadMethod.GDAL)
Esempio n. 9
0
class RasterLayerTest(BaseTestClass):
    layers = [
        make_raster(0, 0, v=1),
        make_raster(3, 2, v=2),
        make_raster(6, 0, v=3)
    ]

    # TODO: Have Travis be able to run the GDAL tests

    numpy_rdd = BaseTestClass.pysc.parallelize(layers)
    layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, numpy_rdd)
    metadata = layer.collect_metadata(GlobalLayout(5))

    def read_no_reproject(self, read_method):
        actual_raster_layer = RasterLayer.read([self.path], read_method=read_method)

        collected = actual_raster_layer.to_numpy_rdd().first()

        (projected_extent, tile) = collected

        self.assertEqual(projected_extent.proj4, self.projected_extent.proj4)

        self.assertTrue((self.expected_tile == tile.cells).all())

    def read_with_reproject(self, read_method):
        expected_raster_layer = self.rdd.reproject(target_crs=3857)

        expected_collected = expected_raster_layer.to_numpy_rdd().first()
        (expected_projected_extent, expected_tile) = expected_collected

        actual_raster_layer = RasterLayer.read([self.path], target_crs=3857, read_method=read_method)

        actual_collected = actual_raster_layer.to_numpy_rdd().first()
        (actual_projected_extent, tile) = actual_collected

        self.assertEqual(actual_projected_extent.epsg, expected_projected_extent.epsg)

        self.assertTrue((expected_tile.cells == tile.cells).all())


    # No reprojection
    def test_read_no_reproject_geotrellis(self):
        self.read_no_reproject(ReadMethod.GEOTRELLIS)

    def test_read_no_reproject_gdal(self):
        self.read_no_reproject(ReadMethod.GDAL)

    # With reprojection
    def test_read_with_reproject_geotrellis(self):
        self.read_with_reproject(ReadMethod.GEOTRELLIS)

    def test_read_with_reproject_gdal(self):
        self.read_with_reproject(ReadMethod.GDAL)

    def test_no_data_of_zero(self):
        no_data_layer = [(t[0], Tile.from_numpy_array(t[1].cells, 1)) for t in self.layers]

        rdd = BaseTestClass.pysc.parallelize(no_data_layer)
        nd_layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
        nd_metadata = nd_layer.collect_metadata()

        self.assertTrue('ud1' in nd_metadata.cell_type)
        self.assertEqual(nd_metadata.no_data_value, 1)

    @pytest.fixture(scope='class', autouse=True)
    def tearDown(self):
        yield
        BaseTestClass.pysc._gateway.close()
Esempio n. 10
0
class CatalogTest(BaseTestClass):
    rdd = get(LayerType.SPATIAL, geotiff_test_path("srtm_52_11.tif"))

    metadata = rdd.collect_metadata()
    reprojected = rdd.tile_to_layout(layout=GlobalLayout(zoom=11),
                                     target_crs="EPSG:3857")
    result = reprojected.pyramid()

    dir_path = geotiff_test_path("catalog/")
    uri = "file://{}".format(dir_path)
    layer_name = "catalog-test"

    @pytest.fixture(autouse=True)
    def tearDown(self):
        yield
        BaseTestClass.pysc._gateway.close()

    def test_read(self):
        for x in range(11, 0, -1):
            actual_layer = query(self.uri, self.layer_name, x)
            expected_layer = self.result.levels[x]

            self.assertDictEqual(actual_layer.layer_metadata.to_dict(),
                                 expected_layer.layer_metadata.to_dict())

    def test_read_value(self):
        tiled = read_value(self.uri, self.layer_name, 11, 1450, 966)

        self.assertEqual(tiled.cells.shape, (1, 256, 256))

    def test_bad_read_value(self):
        tiled = read_value(self.uri, self.layer_name, 11, 1450, 2000)

        self.assertEqual(tiled, None)

    @pytest.mark.skipif('TRAVIS' in os.environ,
                        reason="test_query_1 causes issues on Travis")
    def test_query1(self):
        intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
                           543988.943201520)
        queried = query(self.uri, self.layer_name, 11, intersection)

        self.assertEqual(queried.to_numpy_rdd().first()[0],
                         SpatialKey(1450, 996))

    def test_query2(self):
        intersection = Extent(8348915.46680623, 543988.943201519, 8348915.4669,
                              543988.943201520)
        queried = query(self.uri,
                        self.layer_name,
                        11,
                        intersection,
                        query_proj=3857)

        self.assertEqual(queried.to_numpy_rdd().first()[0],
                         SpatialKey(1450, 996))

    def test_query3(self):
        intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
                           543988.943201520).wkb
        queried = query(self.uri, self.layer_name, 11, intersection)

        self.assertEqual(queried.to_numpy_rdd().first()[0],
                         SpatialKey(1450, 996))

    def test_query4(self):
        intersection = 42
        with pytest.raises(TypeError):
            queried = query(self.uri,
                            self.layer_name,
                            11,
                            query_geom=intersection,
                            num_partitions=2)
            result = queried.to_numpy_rdd().first()[0]

    def test_query_partitions(self):
        intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
                           543988.943201520)
        queried = query(self.uri,
                        self.layer_name,
                        11,
                        intersection,
                        num_partitions=2)

        self.assertEqual(queried.to_numpy_rdd().first()[0],
                         SpatialKey(1450, 996))

    def test_query_crs(self):
        intersection = box(8348915.46680623, 543988.943201519, 8348915.4669,
                           543988.943201520)
        queried = query(self.uri,
                        self.layer_name,
                        11,
                        intersection,
                        query_proj=3857)

        self.assertEqual(queried.to_numpy_rdd().first()[0],
                         SpatialKey(1450, 996))

    def test_read_metadata_exception(self):
        uri = "abcxyz://123"
        with pytest.raises(ValueError):
            layer = read_layer_metadata(uri, self.layer_name, 5)

    def test_read_metadata1(self):
        layer = query(self.uri, self.layer_name, 5)
        actual_metadata = layer.layer_metadata

        expected_metadata = read_layer_metadata(self.uri, self.layer_name, 5)

    def test_read_metadata2(self):
        layer = query(self.uri, self.layer_name, 5)
        actual_metadata = layer.layer_metadata

        expected_metadata = read_layer_metadata(self.uri, self.layer_name, 5)

        self.assertEqual(actual_metadata.to_dict(),
                         expected_metadata.to_dict())

    def test_layer_ids(self):
        ids = AttributeStore(self.uri).layers()
        self.assertTrue(len(ids) == 11)

    def test_attributestore(self):
        store = AttributeStore(self.uri)
        layer_name = "boop-epsg-bop"
        value = {"first": 113, "second": "44two"}
        store.layer(layer_name, 34).write("val", value)
        self.assertEqual(value, store.layer(layer_name, 34).read("val"))

        self.assertEqual(value, store.layer(layer_name, 34)["val"])
        store.layer(layer_name, 34).delete("val")
        with pytest.raises(KeyError):
            store.layer(layer_name, 34)["val"]
Esempio n. 11
0
 def test_tile_to_global_layout(self):
     tiled = self.layer.tile_to_layout(GlobalLayout(5))
     assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6)
     assert tiled.layer_metadata.tile_layout == TileLayout(128, 128, 5, 5)
     assert tiled.zoom_level == 7
Esempio n. 12
0
    def test_tile_to_layout_global_layout(self):
        actual = self.tiled_layer.tile_to_layout(layout=GlobalLayout(zoom=5))

        self.assertEqual(actual.zoom_level, 5)
Esempio n. 13
0
 def test_read_with_reproject_global_gdal(self):
     self.read(GlobalLayout(tile_size=128, zoom=4), ReadMethod.GDAL, target_crs=3857)
Esempio n. 14
0
    def test_tile_to_layout_with_reproject(self):
        proj4 = crs_to_proj4(3857)
        actual = self.result.tile_to_layout(layout=GlobalLayout(), target_crs=proj4).layer_metadata.crs

        self.assertEqual(proj4, actual)
Esempio n. 15
0
 def test_read_with_reproject_global_geotrellis(self):
     self.read(GlobalLayout(tile_size=128, zoom=4), ReadMethod.GEOTRELLIS, target_crs=3857)
Esempio n. 16
0
 def test_ordered_read_no_reproject_global_geotrellis(self):
     self.read(GlobalLayout(tile_size=16, zoom=4),
               ReadMethod.GEOTRELLIS,
               multiplex=True)
Esempio n. 17
0
 def test_read_no_reproject_global_geotrellis(self):
     self.read(GlobalLayout(tile_size=16, zoom=4), ReadMethod.GEOTRELLIS)
Esempio n. 18
0
    def test_different_crs_zoom(self):
        result = BaseTestClass.rdd.tile_to_layout(
            layout=GlobalLayout(tile_size=self.cols), target_crs="EPSG:4324")
        new_metadata = result.layer_metadata

        self.assertEqual(self.expected_crs, new_metadata.crs)
Esempio n. 19
0
class RasterLayerTest(BaseTestClass):
    layers = [
        make_raster(0, 0, v=1),
        make_raster(3, 2, v=2),
        make_raster(6, 0, v=3)
    ]

    numpy_rdd = BaseTestClass.pysc.parallelize(layers)
    layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, numpy_rdd)
    metadata = layer.collect_metadata(GlobalLayout(5))

    def test_tile_to_local_layout(self):
        tiled = self.layer.tile_to_layout(LocalLayout(5))
        assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6)
        assert tiled.layer_metadata.tile_layout == TileLayout(2, 2, 5, 5)

    def test_tile_to_global_layout(self):
        tiled = self.layer.tile_to_layout(GlobalLayout(5))
        assert tiled.layer_metadata.extent == Extent(0, 0, 10, 6)
        assert tiled.layer_metadata.tile_layout == TileLayout(128, 128, 5, 5)
        assert tiled.zoom_level == 7

    def test_tile_to_metadata_layout(self):
        tiled = self.layer.tile_to_layout(layout=self.metadata)

        self.assertEqual(tiled.layer_metadata.extent, Extent(0, 0, 10, 6))
        self.assertDictEqual(tiled.layer_metadata.to_dict(),
                             self.metadata.to_dict())

    def test_tile_to_tiled_layer_layout(self):
        extent = Extent(0., 0., 10., 6.)
        tile_layout = TileLayout(2, 2, 5, 5)
        layout_definition = LayoutDefinition(extent, tile_layout)

        base = self.layer.tile_to_layout(layout_definition)
        tiled = self.layer.tile_to_layout(layout=base)

        self.assertDictEqual(tiled.layer_metadata.to_dict(),
                             base.layer_metadata.to_dict())

    def test_tile_to_layout_definition(self):
        tiled = self.layer.tile_to_layout(
            layout=self.metadata.layout_definition)

        self.assertDictEqual(tiled.layer_metadata.to_dict(),
                             self.metadata.to_dict())

    def test_no_data_of_zero(self):
        no_data_layer = [(t[0], Tile.from_numpy_array(t[1].cells, 1))
                         for t in self.layers]

        rdd = BaseTestClass.pysc.parallelize(no_data_layer)
        nd_layer = RasterLayer.from_numpy_rdd(LayerType.SPATIAL, rdd)
        nd_metadata = nd_layer.collect_metadata()

        self.assertTrue('ud1' in nd_metadata.cell_type)
        self.assertEqual(nd_metadata.no_data_value, 1)

    @pytest.fixture(scope='class', autouse=True)
    def tearDown(self):
        yield
        BaseTestClass.pysc._gateway.close()