Exemplo n.º 1
0
 def test_stop_service(self):
     reg = InMemoryServiceRegistry()
     server = mock.Mock()
     reg.register(SecondaryService(service_metadata=dummy_service_metadata, host='oeo.net', port=5678, server=server))
     # Stop service
     reg.stop_service('s1234')
     server.stop.assert_called_with()
     # service should be gone now
     with pytest.raises(ServiceNotFoundException):
         reg.get_metadata('s1234')
     with pytest.raises(ServiceNotFoundException):
         reg.stop_service('s1234')
Exemplo n.º 2
0
def test_apply_complex_graph():
    graph = {
        "sin": {
            "arguments": {
                "x": {
                    "from_argument": "data"
                }
            },
            "process_id": "sin",
            "result": False
        },
        "multiply": {
            "arguments": {
                "x": {
                    "from_node": "sin"
                },
                "y": 5.0
            },
            "process_id": "multiply",
            "result": True
        }
    }

    input = create_spacetime_layer()
    cube = GeopysparkDataCube(gps.Pyramid({0: input}),
                              InMemoryServiceRegistry())
    res = cube.apply(graph)
    data = res.pyramid.levels[0].to_spatial_layer().stitch().cells
    np.testing.assert_array_almost_equal(data[0, 2:6, 2:6],
                                         5.0 * np.sin(first[0]))
    np.testing.assert_array_almost_equal(data[1, 2:6, 2:6],
                                         5.0 * np.sin(second[0]))
Exemplo n.º 3
0
    def test_apply_spatiotemporal(self):
        import openeo_udf.functions

        input = Pyramid({0: self.tiled_raster_rdd})

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry(), {
                "bands": [{
                    "band_id": "2",
                    "name": "blue",
                    "wavelength_nm": 496.6,
                    "res_m": 10,
                    "scale": 0.0001,
                    "offset": 0,
                    "type": "int16",
                    "unit": "1"
                }]
            })
        import os, openeo_udf
        dir = os.path.dirname(openeo_udf.functions.__file__)
        file_name = os.path.join(dir, "datacube_reduce_time_sum.py")
        with open(file_name, "r") as f:
            udf_code = f.read()

        result = imagecollection.apply_tiles_spatiotemporal(udf_code)
        stitched = result.pyramid.levels[0].to_spatial_layer().stitch()
        print(stitched)
        self.assertEqual(2, stitched.cells[0][0][0])
        self.assertEqual(6, stitched.cells[0][0][5])
        self.assertEqual(4, stitched.cells[0][5][6])
Exemplo n.º 4
0
    def test_linear_scale_range(self):
        red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
        layer = self._create_spacetime_layer(
            cells=np.array([[red_ramp], [nir_ramp]]))
        pyramid = gps.Pyramid({0: layer})
        metadata = CollectionMetadata({
            "properties": {
                "eo:bands": [
                    {
                        "name": "B04",
                        "common_name": "red"
                    },
                    {
                        "name": "B08",
                        "common_name": "nir"
                    },
                ]
            }
        })
        imagecollection = GeotrellisTimeSeriesImageCollection(
            pyramid, InMemoryServiceRegistry(), metadata=metadata)

        stitched = imagecollection.ndvi().linear_scale_range(
            -1, 1, 0, 100).pyramid.levels[0].to_spatial_layer().stitch()
        cells = stitched.cells[0, 0:4, 0:4]
        expected = 50.0 * (1.0 + np.array(
            [[np.nan, 1 / 1, 2 / 2, 3 / 3], [-1 / 1, 0 / 2, 1 / 3, 2 / 4],
             [-2 / 2, -1 / 3, 0 / 4, 1 / 5], [-3 / 3, -2 / 4, -1 / 5, 0 / 6]]))
        expected[0][0] = 255.0
        np.testing.assert_array_almost_equal(cells, expected.astype(np.uint8))
Exemplo n.º 5
0
    def test_reduce_bands_comparison_ops(self):
        input = self.create_spacetime_layer_singleband()
        input = gps.Pyramid({0: input})

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry())

        visitor = GeotrellisTileProcessGraphVisitor()
        graph = {
            "gt": {
                "arguments": {
                    "x": {
                        "from_argument": "data"
                    },
                    "y": 6.0
                },
                "process_id": "gt",
                "result": True
            }
        }
        visitor.accept_process_graph(graph)
        stitched = imagecollection.reduce_bands(
            visitor).pyramid.levels[0].to_spatial_layer().stitch()
        print(stitched)
        self.assertEqual(1, stitched.cells[0][0][0])
    def test_download_as_catalog(self):
        input = self.create_spacetime_layer()

        imagecollection = GeotrellisTimeSeriesImageCollection(
            gps.Pyramid({0: input}), InMemoryServiceRegistry())
        imagecollection.download("catalogresult.tiff",
                                 format="GTIFF",
                                 parameters={"catalog": True})
Exemplo n.º 7
0
    def test_reduce_nontemporal(self):
        input = Pyramid({0: self.tiled_raster_rdd})

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry())
        with self.assertRaises(AttributeError) as context:
            imagecollection.reduce("max",
                                   "spectral").pyramid.levels[0].stitch()
        print(context.exception)
Exemplo n.º 8
0
    def test_aggregate_temporal(self):
        input = Pyramid({0: self.tiled_raster_rdd})

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry())
        stitched = imagecollection.aggregate_temporal(
            ["2017-01-01", "2018-01-01"], ["2017-01-03"],
            "max").pyramid.levels[0].to_spatial_layer().stitch()
        print(stitched)
    def test_download_geotiff_no_args(self):

        input = self.create_spacetime_layer()

        imagecollection = GeotrellisTimeSeriesImageCollection(
            gps.Pyramid({0: input}), InMemoryServiceRegistry())
        geotiffs = imagecollection.download(
            str(self.temp_folder / "test_download_result.geotiff"))
        print(geotiffs)
    def test_polygon_series(self):
        input = self.create_spacetime_layer()
        polygon = Polygon([(0, 0), (0, 2), (2, 2), (2, 0), (0, 0)])
        imagecollection = GeotrellisTimeSeriesImageCollection(
            gps.Pyramid({0: input}), InMemoryServiceRegistry(),
            self.openeo_metadata)

        means = imagecollection.polygonal_mean_timeseries(polygon)
        assert means == {'2017-09-25T11:37:00': [[1.0, 2.0]]}
    def test_download_masked_geotiff(self):

        input = self.create_spacetime_layer()
        polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]])

        imagecollection = GeotrellisTimeSeriesImageCollection(
            gps.Pyramid({0: input}), InMemoryServiceRegistry())
        imagecollection = imagecollection.mask(polygon)
        geotiffs = imagecollection.download(
            str(self.temp_folder / "test_download_masked_result.geotiff"))
        print(geotiffs)
Exemplo n.º 12
0
    def test_aggregate_max_time(self):

        input = Pyramid({0: self.tiled_raster_rdd})

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry())

        stitched = imagecollection.reduce(
            'max', 'temporal').pyramid.levels[0].stitch()
        print(stitched)
        self.assertEqual(2.0, stitched.cells[0][0][0])
Exemplo n.º 13
0
    def test_mask_raster(self):
        input = Pyramid({0: self.tiled_raster_rdd})

        def createMask(tile):
            tile.cells[0][0][0] = 0.0
            return tile

        mask_layer = self.tiled_raster_rdd.map_tiles(createMask)
        mask = Pyramid({0: mask_layer})

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry())
        stitched = imagecollection.mask(
            rastermask=GeotrellisTimeSeriesImageCollection(
                mask, InMemoryServiceRegistry()),
            replacement=10.0).reduce('max',
                                     'temporal').pyramid.levels[0].stitch()
        print(stitched)
        self.assertEquals(2.0, stitched.cells[0][0][0])
        self.assertEquals(10.0, stitched.cells[0][0][1])
Exemplo n.º 14
0
    def __init__(self):
        # TODO: do this with a config instead of hardcoding rules?
        self._service_registry = (InMemoryServiceRegistry()
                                  if ConfigParams().is_ci_context else
                                  ZooKeeperServiceRegistry())

        super().__init__(
            secondary_services=GpsSecondaryServices(
                service_registry=self._service_registry),
            catalog=get_layer_catalog(service_registry=self._service_registry),
            batch_jobs=GpsBatchJobs(),
        )
Exemplo n.º 15
0
    def test_apply_kernel(self):
        kernel = np.array([[0.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 0.0]])

        input = Pyramid({0: self.tiled_raster_rdd})
        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry())
        stitched = imagecollection.apply_kernel(kernel, 2.0).reduce(
            'max', 'temporal').pyramid.levels[0].stitch()

        self.assertEquals(12.0, stitched.cells[0][0][0])
        self.assertEquals(16.0, stitched.cells[0][0][1])
        self.assertEquals(20.0, stitched.cells[0][1][1])
 def test_another_polygon_series(self):
     input = self._create_spacetime_layer(no_data=-1.0)
     imagecollection = GeotrellisTimeSeriesImageCollection(
         gps.Pyramid({0: input}), InMemoryServiceRegistry())
     polygon = Polygon(shell=[(2.0, 6.0), (6.0,
                                           6.0), (6.0,
                                                  2.0), (2.0,
                                                         2.0), (2.0, 6.0)])
     means = imagecollection.polygonal_mean_timeseries(polygon)
     assert means == {
         '2017-09-25T11:37:00':
         [[(0 + 0 + 0 + 0 + 1 + 1 + 1 + 1 + 2 + 2 + 2 + 2) / 12]]
     }
Exemplo n.º 17
0
    def test_merge_cubes(self):
        red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
        layer1 = self._create_spacetime_layer(cells=np.array([[red_ramp]]))
        layer2 = self._create_spacetime_layer(cells=np.array([[nir_ramp]]))

        metadata = CollectionMetadata(
            {"properties": {
                "eo:bands": [{
                    "name": "the_band"
                }]
            }})

        cube1 = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer1}),
                                                    InMemoryServiceRegistry(),
                                                    metadata=metadata)
        cube2 = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer2}),
                                                    InMemoryServiceRegistry(),
                                                    metadata=metadata)
        sum = cube1.merge(cube2, 'sum')
        stitched = sum.pyramid.levels[0].to_spatial_layer().stitch()

        np.testing.assert_array_equal(red_ramp + nir_ramp,
                                      stitched.cells[0, 0:4, 0:4])
Exemplo n.º 18
0
    def test_point_series(self):

        input = self.create_spacetime_layer()

        imagecollection = GeotrellisTimeSeriesImageCollection(
            gps.Pyramid({0: input}), InMemoryServiceRegistry())
        transformed_collection = imagecollection.apply("cos")
        for p in self.points[0:3]:
            result = transformed_collection.timeseries(p.x, p.y)
            print(result)
            value = result.popitem()

            self.assertEqual(math.cos(10), value[1][0])
            self.assertEqual(math.cos(5), value[1][1])
    def test_point_series(self):
        def custom_function(cells: np.ndarray, nd):
            return cells[0] + cells[1]

        input = self.create_spacetime_layer()

        imagecollection = GeotrellisTimeSeriesImageCollection(
            gps.Pyramid({0: input}), InMemoryServiceRegistry())
        transformed_collection = imagecollection.apply_pixel([0, 1],
                                                             custom_function)

        for p in self.points[0:3]:
            result = transformed_collection.timeseries(p.x, p.y)
            print(result)
            value = result.popitem()
            self.assertEqual(3.0, value[1][0])
Exemplo n.º 20
0
    def test_resample_spatial(self):
        input = Pyramid({0: self.tiled_raster_rdd})

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry())

        resampled = imagecollection.resample_spatial(resolution=0.05)

        path = str(self.temp_folder / "resampled.tiff")
        resampled.reduce('max',
                         'temporal').download(path,
                                              format="GTIFF",
                                              parameters={'tiled': True})

        import rasterio
        with rasterio.open(path) as ds:
            print(ds.profile)
            self.assertAlmostEqual(0.05, ds.res[0], 3)
Exemplo n.º 21
0
 def test_viewing(self):
     geotrellis_layer = self.create_spacetime_layer()
     imagecollection = GeotrellisTimeSeriesImageCollection(
         pyramid=gps.Pyramid({0: geotrellis_layer}),
         service_registry=InMemoryServiceRegistry())
     metadata = imagecollection.tiled_viewing_service(service_type="TMS",
                                                      process_graph={})
     print(metadata)
     assert metadata.type == "TMS"
     assert isinstance(metadata.attributes["bounds"], dict)
     tileresponse = requests.get(metadata.url.format(x=0, y=0, z=0),
                                 timeout=2)
     assert tileresponse.status_code == 200
     assert tileresponse.content.startswith(PNG_SIGNATURE)
     tileresponse = requests.get(metadata.url.format(x=1, y=1, z=0),
                                 timeout=2)
     assert tileresponse.status_code == 200
     assert tileresponse.content.startswith(PNG_SIGNATURE)
    def test_point_series_apply_tile(self):
        import os, openeo_udf
        dir = os.path.dirname(openeo_udf.functions.__file__)
        file_name = os.path.join(dir, "datacube_ndvi.py")
        with open(file_name, "r") as f:
            udf_code = f.read()

        input = self.create_spacetime_layer()

        imagecollection = GeotrellisTimeSeriesImageCollection(
            gps.Pyramid({0: input}), InMemoryServiceRegistry(),
            self.openeo_metadata)
        transformed_collection = imagecollection.apply_tiles(udf_code)

        for p in self.points[0:3]:
            result = transformed_collection.timeseries(p.x, p.y)
            print(result)
            value = result.popitem()
            print(value)
Exemplo n.º 23
0
    def test_reduce_bands(self):
        input = self.create_spacetime_layer()
        input = gps.Pyramid({0: input})

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry())

        visitor = GeotrellisTileProcessGraphVisitor()
        graph = {
            "sum": {
                "arguments": {
                    "data": {
                        "from_argument": "dimension_data"
                    }
                },
                "process_id": "sum"
            },
            "subtract": {
                "arguments": {
                    "data": {
                        "from_argument": "dimension_data"
                    }
                },
                "process_id": "subtract"
            },
            "divide": {
                "arguments": {
                    "data": [{
                        "from_node": "sum"
                    }, {
                        "from_node": "subtract"
                    }]
                },
                "process_id": "divide",
                "result": True
            }
        }
        visitor.accept_process_graph(graph)
        stitched = imagecollection.reduce_bands(
            visitor).pyramid.levels[0].to_spatial_layer().stitch()
        print(stitched)
        self.assertEqual(3.0, stitched.cells[0][0][0])
Exemplo n.º 24
0
    def test_apply_dimension_spatiotemporal(self):

        input = Pyramid({0: self.tiled_raster_rdd})

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry(), {
                "bands": [{
                    "band_id": "2",
                    "name": "blue",
                    "wavelength_nm": 496.6,
                    "res_m": 10,
                    "scale": 0.0001,
                    "offset": 0,
                    "type": "int16",
                    "unit": "1"
                }]
            })

        udf_code = """
def rct_savitzky_golay(udf_data:UdfData):
    from scipy.signal import savgol_filter

    print(udf_data.get_datacube_list())
    return udf_data
        
        """

        result = imagecollection.apply_tiles_spatiotemporal(udf_code)
        local_tiles = result.pyramid.levels[0].to_numpy_rdd().collect()
        print(local_tiles)
        self.assertEquals(len(TestMultipleDates.layer), len(local_tiles))
        ref_dict = {
            e[0]: e[1]
            for e in imagecollection.pyramid.levels[0].convert_data_type(
                CellType.FLOAT64).to_numpy_rdd().collect()
        }
        result_dict = {e[0]: e[1] for e in local_tiles}
        for k, v in ref_dict.items():
            tile = result_dict[k]
            assert_array_almost_equal(np.squeeze(v.cells),
                                      np.squeeze(tile.cells),
                                      decimal=2)
Exemplo n.º 25
0
    def test_min_time(self):
        input = Pyramid({0: self.tiled_raster_rdd})

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry())
        min_time = imagecollection.reduce('min', 'temporal')
        max_time = imagecollection.reduce('max', 'temporal')

        stitched = min_time.pyramid.levels[0].stitch()
        print(stitched)

        self.assertEquals(2.0, stitched.cells[0][0][0])

        for p in self.points[1:3]:
            result = min_time.timeseries(p.x, p.y, srs="EPSG:3857")
            print(result)
            print(imagecollection.timeseries(p.x, p.y, srs="EPSG:3857"))
            max_result = max_time.timeseries(p.x, p.y, srs="EPSG:3857")
            self.assertEqual(1.0, result['NoDate'])
            self.assertEqual(2.0, max_result['NoDate'])
Exemplo n.º 26
0
    def test_reduce_some_nodata(self):
        no_data = -1.0

        input = Pyramid({
            0:
            self._single_pixel_layer(
                {
                    datetime.datetime.strptime("2016-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'):
                    no_data,
                    datetime.datetime.strptime("2017-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'):
                    5.0
                }, no_data)
        })

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry())

        stitched = imagecollection.reduce(
            "min", "temporal").pyramid.levels[0].stitch()
        #print(stitched)
        self.assertEqual(5.0, stitched.cells[0][0][0])

        stitched = imagecollection.reduce(
            "max", "temporal").pyramid.levels[0].stitch()
        self.assertEqual(5.0, stitched.cells[0][0][0])

        stitched = imagecollection.reduce(
            "sum", "temporal").pyramid.levels[0].stitch()
        self.assertEqual(5.0, stitched.cells[0][0][0])

        stitched = imagecollection.reduce(
            "mean", "temporal").pyramid.levels[0].stitch()
        self.assertAlmostEqual(5.0, stitched.cells[0][0][0], delta=0.001)

        stitched = imagecollection.reduce(
            "variance", "temporal").pyramid.levels[0].stitch()
        self.assertAlmostEqual(0.0, stitched.cells[0][0][0], delta=0.001)

        stitched = imagecollection.reduce(
            "sd", "temporal").pyramid.levels[0].stitch()
        self.assertAlmostEqual(0.0, stitched.cells[0][0][0], delta=0.001)
Exemplo n.º 27
0
    def test_reduce(self):
        input = Pyramid({0: self.tiled_raster_rdd})

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry())

        stitched = imagecollection.reduce(
            "max", "temporal").pyramid.levels[0].stitch()
        print(stitched)
        self.assertEqual(2.0, stitched.cells[0][0][0])
        self.assertEqual(2.0, stitched.cells[0][0][1])

        stitched = imagecollection.reduce(
            "min", "temporal").pyramid.levels[0].stitch()
        print(stitched)
        self.assertEqual(2.0, stitched.cells[0][0][0])
        self.assertEqual(1.0, stitched.cells[0][0][1])

        stitched = imagecollection.reduce(
            "sum", "temporal").pyramid.levels[0].stitch()
        print(stitched)
        self.assertEqual(2.0, stitched.cells[0][0][0])
        self.assertEqual(4.0, stitched.cells[0][0][1])

        stitched = imagecollection.reduce(
            "mean", "temporal").pyramid.levels[0].stitch()
        print(stitched)
        self.assertEqual(2.0, stitched.cells[0][0][0])
        self.assertAlmostEqual(1.3333333, stitched.cells[0][0][1])

        stitched = imagecollection.reduce(
            "variance", "temporal").pyramid.levels[0].stitch()
        print(stitched)
        self.assertEqual(0.0, stitched.cells[0][0][0])
        self.assertAlmostEqual(0.2222222, stitched.cells[0][0][1])

        stitched = imagecollection.reduce(
            "sd", "temporal").pyramid.levels[0].stitch()
        print(stitched)
        self.assertEqual(0.0, stitched.cells[0][0][0])
        self.assertAlmostEqual(0.4714045, stitched.cells[0][0][1])
Exemplo n.º 28
0
    def test_reproject_spatial(self):
        input = Pyramid({0: self.tiled_raster_rdd})

        imagecollection = GeotrellisTimeSeriesImageCollection(
            input, InMemoryServiceRegistry())

        resampled = imagecollection.resample_spatial(resolution=0,
                                                     projection="EPSG:3857",
                                                     method="max")
        metadata = resampled.pyramid.levels[0].layer_metadata
        print(metadata)
        self.assertTrue("proj=merc" in metadata.crs)
        path = str(self.temp_folder / "reprojected.tiff")
        resampled.reduce('max',
                         'temporal').download(path,
                                              format="GTIFF",
                                              parameters={'tiled': True})

        import rasterio
        with rasterio.open(path) as ds:
            print(ds.profile)
Exemplo n.º 29
0
 def test_convert_multiband_tile_hypercube(self):
     from openeo_udf.api.datacube \
         import DataCube
     metadata = CollectionMetadata({
         'bands': [{
             'band_id': '2',
             'name': 'blue',
             'wavelength_nm': 496.6,
             'res_m': 10,
             'scale': 0.0001,
             'offset': 0,
             'type': 'int16',
             'unit': '1'
         }, {
             'band_id': '3',
             'name': 'green',
             'wavelength_nm': 560,
             'res_m': 10,
             'scale': 0.0001,
             'offset': 0,
             'type': 'int16',
             'unit': '1'
         }, {
             'band_id': '4',
             'name': 'red',
             'wavelength_nm': 664.5,
             'res_m': 10,
             'scale': 0.0001,
             'offset': 0,
             'type': 'int16',
             'unit': '1'
         }]
     })
     imagecollection = GeotrellisTimeSeriesImageCollection(
         "test", InMemoryServiceRegistry(), metadata=metadata)
     datacube = GeotrellisTimeSeriesImageCollection._tile_to_datacube(
         TestMultiBandUDF.tile.cells, None, bands_metadata=metadata.bands)
     the_array = datacube.get_array()
     assert the_array is not None
     print(the_array)
    def test_download_masked_geotiff_reproject(self):

        input = self.create_spacetime_layer()
        polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]])

        import pyproj
        from shapely.ops import transform
        from functools import partial

        project = partial(
            pyproj.transform,
            pyproj.Proj(init="EPSG:4326"),  # source coordinate system
            pyproj.Proj(init="EPSG:3857"))  # destination coordinate system

        reprojected = transform(project, polygon)

        imagecollection = GeotrellisTimeSeriesImageCollection(
            gps.Pyramid({0: input}), InMemoryServiceRegistry())
        imagecollection = imagecollection.mask(reprojected, "EPSG:3857")
        geotiffs = imagecollection.download(
            str(self.temp_folder / "test_download_masked_result.3857"))
        print(geotiffs)