def test_stop_service(self):
reg = InMemoryServiceRegistry()
server = mock.Mock()
reg.register(SecondaryService(service_metadata=dummy_service_metadata, host='oeo.net', port=5678, server=server))
# Stop service
reg.stop_service('s1234')
server.stop.assert_called_with()
# service should be gone now
with pytest.raises(ServiceNotFoundException):
reg.get_metadata('s1234')
with pytest.raises(ServiceNotFoundException):
reg.stop_service('s1234')
def test_apply_complex_graph():
graph = {
"sin": {
"arguments": {
"x": {
"from_argument": "data"
}
},
"process_id": "sin",
"result": False
},
"multiply": {
"arguments": {
"x": {
"from_node": "sin"
},
"y": 5.0
},
"process_id": "multiply",
"result": True
}
}
input = create_spacetime_layer()
cube = GeopysparkDataCube(gps.Pyramid({0: input}),
InMemoryServiceRegistry())
res = cube.apply(graph)
data = res.pyramid.levels[0].to_spatial_layer().stitch().cells
np.testing.assert_array_almost_equal(data[0, 2:6, 2:6],
5.0 * np.sin(first[0]))
np.testing.assert_array_almost_equal(data[1, 2:6, 2:6],
5.0 * np.sin(second[0]))
def test_apply_spatiotemporal(self):
import openeo_udf.functions
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry(), {
"bands": [{
"band_id": "2",
"name": "blue",
"wavelength_nm": 496.6,
"res_m": 10,
"scale": 0.0001,
"offset": 0,
"type": "int16",
"unit": "1"
}]
})
import os, openeo_udf
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_reduce_time_sum.py")
with open(file_name, "r") as f:
udf_code = f.read()
result = imagecollection.apply_tiles_spatiotemporal(udf_code)
stitched = result.pyramid.levels[0].to_spatial_layer().stitch()
print(stitched)
self.assertEqual(2, stitched.cells[0][0][0])
self.assertEqual(6, stitched.cells[0][0][5])
self.assertEqual(4, stitched.cells[0][5][6])
def test_linear_scale_range(self):
red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
layer = self._create_spacetime_layer(
cells=np.array([[red_ramp], [nir_ramp]]))
pyramid = gps.Pyramid({0: layer})
metadata = CollectionMetadata({
"properties": {
"eo:bands": [
{
"name": "B04",
"common_name": "red"
},
{
"name": "B08",
"common_name": "nir"
},
]
}
})
imagecollection = GeotrellisTimeSeriesImageCollection(
pyramid, InMemoryServiceRegistry(), metadata=metadata)
stitched = imagecollection.ndvi().linear_scale_range(
-1, 1, 0, 100).pyramid.levels[0].to_spatial_layer().stitch()
cells = stitched.cells[0, 0:4, 0:4]
expected = 50.0 * (1.0 + np.array(
[[np.nan, 1 / 1, 2 / 2, 3 / 3], [-1 / 1, 0 / 2, 1 / 3, 2 / 4],
[-2 / 2, -1 / 3, 0 / 4, 1 / 5], [-3 / 3, -2 / 4, -1 / 5, 0 / 6]]))
expected[0][0] = 255.0
np.testing.assert_array_almost_equal(cells, expected.astype(np.uint8))
def test_reduce_bands_comparison_ops(self):
input = self.create_spacetime_layer_singleband()
input = gps.Pyramid({0: input})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
visitor = GeotrellisTileProcessGraphVisitor()
graph = {
"gt": {
"arguments": {
"x": {
"from_argument": "data"
},
"y": 6.0
},
"process_id": "gt",
"result": True
}
}
visitor.accept_process_graph(graph)
stitched = imagecollection.reduce_bands(
visitor).pyramid.levels[0].to_spatial_layer().stitch()
print(stitched)
self.assertEqual(1, stitched.cells[0][0][0])
def test_download_as_catalog(self):
input = self.create_spacetime_layer()
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
imagecollection.download("catalogresult.tiff",
format="GTIFF",
parameters={"catalog": True})
def test_reduce_nontemporal(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
with self.assertRaises(AttributeError) as context:
imagecollection.reduce("max",
"spectral").pyramid.levels[0].stitch()
print(context.exception)
def test_aggregate_temporal(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
stitched = imagecollection.aggregate_temporal(
["2017-01-01", "2018-01-01"], ["2017-01-03"],
"max").pyramid.levels[0].to_spatial_layer().stitch()
print(stitched)
def test_download_geotiff_no_args(self):
input = self.create_spacetime_layer()
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
geotiffs = imagecollection.download(
str(self.temp_folder / "test_download_result.geotiff"))
print(geotiffs)
def test_polygon_series(self):
input = self.create_spacetime_layer()
polygon = Polygon([(0, 0), (0, 2), (2, 2), (2, 0), (0, 0)])
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry(),
self.openeo_metadata)
means = imagecollection.polygonal_mean_timeseries(polygon)
assert means == {'2017-09-25T11:37:00': [[1.0, 2.0]]}
def test_download_masked_geotiff(self):
input = self.create_spacetime_layer()
polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]])
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
imagecollection = imagecollection.mask(polygon)
geotiffs = imagecollection.download(
str(self.temp_folder / "test_download_masked_result.geotiff"))
print(geotiffs)
def test_aggregate_max_time(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
stitched = imagecollection.reduce(
'max', 'temporal').pyramid.levels[0].stitch()
print(stitched)
self.assertEqual(2.0, stitched.cells[0][0][0])
def test_mask_raster(self):
input = Pyramid({0: self.tiled_raster_rdd})
def createMask(tile):
tile.cells[0][0][0] = 0.0
return tile
mask_layer = self.tiled_raster_rdd.map_tiles(createMask)
mask = Pyramid({0: mask_layer})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
stitched = imagecollection.mask(
rastermask=GeotrellisTimeSeriesImageCollection(
mask, InMemoryServiceRegistry()),
replacement=10.0).reduce('max',
'temporal').pyramid.levels[0].stitch()
print(stitched)
self.assertEquals(2.0, stitched.cells[0][0][0])
self.assertEquals(10.0, stitched.cells[0][0][1])
def __init__(self):
# TODO: do this with a config instead of hardcoding rules?
self._service_registry = (InMemoryServiceRegistry()
if ConfigParams().is_ci_context else
ZooKeeperServiceRegistry())
super().__init__(
secondary_services=GpsSecondaryServices(
service_registry=self._service_registry),
catalog=get_layer_catalog(service_registry=self._service_registry),
batch_jobs=GpsBatchJobs(),
)
def test_apply_kernel(self):
kernel = np.array([[0.0, 1.0, 0.0], [1.0, 1.0, 1.0], [0.0, 1.0, 0.0]])
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
stitched = imagecollection.apply_kernel(kernel, 2.0).reduce(
'max', 'temporal').pyramid.levels[0].stitch()
self.assertEquals(12.0, stitched.cells[0][0][0])
self.assertEquals(16.0, stitched.cells[0][0][1])
self.assertEquals(20.0, stitched.cells[0][1][1])
def test_another_polygon_series(self):
input = self._create_spacetime_layer(no_data=-1.0)
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
polygon = Polygon(shell=[(2.0, 6.0), (6.0,
6.0), (6.0,
2.0), (2.0,
2.0), (2.0, 6.0)])
means = imagecollection.polygonal_mean_timeseries(polygon)
assert means == {
'2017-09-25T11:37:00':
[[(0 + 0 + 0 + 0 + 1 + 1 + 1 + 1 + 2 + 2 + 2 + 2) / 12]]
}
def test_merge_cubes(self):
red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
layer1 = self._create_spacetime_layer(cells=np.array([[red_ramp]]))
layer2 = self._create_spacetime_layer(cells=np.array([[nir_ramp]]))
metadata = CollectionMetadata(
{"properties": {
"eo:bands": [{
"name": "the_band"
}]
}})
cube1 = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer1}),
InMemoryServiceRegistry(),
metadata=metadata)
cube2 = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer2}),
InMemoryServiceRegistry(),
metadata=metadata)
sum = cube1.merge(cube2, 'sum')
stitched = sum.pyramid.levels[0].to_spatial_layer().stitch()
np.testing.assert_array_equal(red_ramp + nir_ramp,
stitched.cells[0, 0:4, 0:4])
def test_point_series(self):
input = self.create_spacetime_layer()
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
transformed_collection = imagecollection.apply("cos")
for p in self.points[0:3]:
result = transformed_collection.timeseries(p.x, p.y)
print(result)
value = result.popitem()
self.assertEqual(math.cos(10), value[1][0])
self.assertEqual(math.cos(5), value[1][1])
def test_point_series(self):
def custom_function(cells: np.ndarray, nd):
return cells[0] + cells[1]
input = self.create_spacetime_layer()
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
transformed_collection = imagecollection.apply_pixel([0, 1],
custom_function)
for p in self.points[0:3]:
result = transformed_collection.timeseries(p.x, p.y)
print(result)
value = result.popitem()
self.assertEqual(3.0, value[1][0])
def test_resample_spatial(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
resampled = imagecollection.resample_spatial(resolution=0.05)
path = str(self.temp_folder / "resampled.tiff")
resampled.reduce('max',
'temporal').download(path,
format="GTIFF",
parameters={'tiled': True})
import rasterio
with rasterio.open(path) as ds:
print(ds.profile)
self.assertAlmostEqual(0.05, ds.res[0], 3)
def test_viewing(self):
geotrellis_layer = self.create_spacetime_layer()
imagecollection = GeotrellisTimeSeriesImageCollection(
pyramid=gps.Pyramid({0: geotrellis_layer}),
service_registry=InMemoryServiceRegistry())
metadata = imagecollection.tiled_viewing_service(service_type="TMS",
process_graph={})
print(metadata)
assert metadata.type == "TMS"
assert isinstance(metadata.attributes["bounds"], dict)
tileresponse = requests.get(metadata.url.format(x=0, y=0, z=0),
timeout=2)
assert tileresponse.status_code == 200
assert tileresponse.content.startswith(PNG_SIGNATURE)
tileresponse = requests.get(metadata.url.format(x=1, y=1, z=0),
timeout=2)
assert tileresponse.status_code == 200
assert tileresponse.content.startswith(PNG_SIGNATURE)
def test_point_series_apply_tile(self):
import os, openeo_udf
dir = os.path.dirname(openeo_udf.functions.__file__)
file_name = os.path.join(dir, "datacube_ndvi.py")
with open(file_name, "r") as f:
udf_code = f.read()
input = self.create_spacetime_layer()
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry(),
self.openeo_metadata)
transformed_collection = imagecollection.apply_tiles(udf_code)
for p in self.points[0:3]:
result = transformed_collection.timeseries(p.x, p.y)
print(result)
value = result.popitem()
print(value)
def test_reduce_bands(self):
input = self.create_spacetime_layer()
input = gps.Pyramid({0: input})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
visitor = GeotrellisTileProcessGraphVisitor()
graph = {
"sum": {
"arguments": {
"data": {
"from_argument": "dimension_data"
}
},
"process_id": "sum"
},
"subtract": {
"arguments": {
"data": {
"from_argument": "dimension_data"
}
},
"process_id": "subtract"
},
"divide": {
"arguments": {
"data": [{
"from_node": "sum"
}, {
"from_node": "subtract"
}]
},
"process_id": "divide",
"result": True
}
}
visitor.accept_process_graph(graph)
stitched = imagecollection.reduce_bands(
visitor).pyramid.levels[0].to_spatial_layer().stitch()
print(stitched)
self.assertEqual(3.0, stitched.cells[0][0][0])
def test_apply_dimension_spatiotemporal(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry(), {
"bands": [{
"band_id": "2",
"name": "blue",
"wavelength_nm": 496.6,
"res_m": 10,
"scale": 0.0001,
"offset": 0,
"type": "int16",
"unit": "1"
}]
})
udf_code = """
def rct_savitzky_golay(udf_data:UdfData):
from scipy.signal import savgol_filter
print(udf_data.get_datacube_list())
return udf_data
"""
result = imagecollection.apply_tiles_spatiotemporal(udf_code)
local_tiles = result.pyramid.levels[0].to_numpy_rdd().collect()
print(local_tiles)
self.assertEquals(len(TestMultipleDates.layer), len(local_tiles))
ref_dict = {
e[0]: e[1]
for e in imagecollection.pyramid.levels[0].convert_data_type(
CellType.FLOAT64).to_numpy_rdd().collect()
}
result_dict = {e[0]: e[1] for e in local_tiles}
for k, v in ref_dict.items():
tile = result_dict[k]
assert_array_almost_equal(np.squeeze(v.cells),
np.squeeze(tile.cells),
decimal=2)
def test_min_time(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
min_time = imagecollection.reduce('min', 'temporal')
max_time = imagecollection.reduce('max', 'temporal')
stitched = min_time.pyramid.levels[0].stitch()
print(stitched)
self.assertEquals(2.0, stitched.cells[0][0][0])
for p in self.points[1:3]:
result = min_time.timeseries(p.x, p.y, srs="EPSG:3857")
print(result)
print(imagecollection.timeseries(p.x, p.y, srs="EPSG:3857"))
max_result = max_time.timeseries(p.x, p.y, srs="EPSG:3857")
self.assertEqual(1.0, result['NoDate'])
self.assertEqual(2.0, max_result['NoDate'])
def test_reduce_some_nodata(self):
no_data = -1.0
input = Pyramid({
0:
self._single_pixel_layer(
{
datetime.datetime.strptime("2016-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'):
no_data,
datetime.datetime.strptime("2017-04-24T04:00:00Z", '%Y-%m-%dT%H:%M:%SZ'):
5.0
}, no_data)
})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
stitched = imagecollection.reduce(
"min", "temporal").pyramid.levels[0].stitch()
#print(stitched)
self.assertEqual(5.0, stitched.cells[0][0][0])
stitched = imagecollection.reduce(
"max", "temporal").pyramid.levels[0].stitch()
self.assertEqual(5.0, stitched.cells[0][0][0])
stitched = imagecollection.reduce(
"sum", "temporal").pyramid.levels[0].stitch()
self.assertEqual(5.0, stitched.cells[0][0][0])
stitched = imagecollection.reduce(
"mean", "temporal").pyramid.levels[0].stitch()
self.assertAlmostEqual(5.0, stitched.cells[0][0][0], delta=0.001)
stitched = imagecollection.reduce(
"variance", "temporal").pyramid.levels[0].stitch()
self.assertAlmostEqual(0.0, stitched.cells[0][0][0], delta=0.001)
stitched = imagecollection.reduce(
"sd", "temporal").pyramid.levels[0].stitch()
self.assertAlmostEqual(0.0, stitched.cells[0][0][0], delta=0.001)
def test_reduce(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
stitched = imagecollection.reduce(
"max", "temporal").pyramid.levels[0].stitch()
print(stitched)
self.assertEqual(2.0, stitched.cells[0][0][0])
self.assertEqual(2.0, stitched.cells[0][0][1])
stitched = imagecollection.reduce(
"min", "temporal").pyramid.levels[0].stitch()
print(stitched)
self.assertEqual(2.0, stitched.cells[0][0][0])
self.assertEqual(1.0, stitched.cells[0][0][1])
stitched = imagecollection.reduce(
"sum", "temporal").pyramid.levels[0].stitch()
print(stitched)
self.assertEqual(2.0, stitched.cells[0][0][0])
self.assertEqual(4.0, stitched.cells[0][0][1])
stitched = imagecollection.reduce(
"mean", "temporal").pyramid.levels[0].stitch()
print(stitched)
self.assertEqual(2.0, stitched.cells[0][0][0])
self.assertAlmostEqual(1.3333333, stitched.cells[0][0][1])
stitched = imagecollection.reduce(
"variance", "temporal").pyramid.levels[0].stitch()
print(stitched)
self.assertEqual(0.0, stitched.cells[0][0][0])
self.assertAlmostEqual(0.2222222, stitched.cells[0][0][1])
stitched = imagecollection.reduce(
"sd", "temporal").pyramid.levels[0].stitch()
print(stitched)
self.assertEqual(0.0, stitched.cells[0][0][0])
self.assertAlmostEqual(0.4714045, stitched.cells[0][0][1])
def test_reproject_spatial(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
resampled = imagecollection.resample_spatial(resolution=0,
projection="EPSG:3857",
method="max")
metadata = resampled.pyramid.levels[0].layer_metadata
print(metadata)
self.assertTrue("proj=merc" in metadata.crs)
path = str(self.temp_folder / "reprojected.tiff")
resampled.reduce('max',
'temporal').download(path,
format="GTIFF",
parameters={'tiled': True})
import rasterio
with rasterio.open(path) as ds:
print(ds.profile)
def test_convert_multiband_tile_hypercube(self):
from openeo_udf.api.datacube \
import DataCube
metadata = CollectionMetadata({
'bands': [{
'band_id': '2',
'name': 'blue',
'wavelength_nm': 496.6,
'res_m': 10,
'scale': 0.0001,
'offset': 0,
'type': 'int16',
'unit': '1'
}, {
'band_id': '3',
'name': 'green',
'wavelength_nm': 560,
'res_m': 10,
'scale': 0.0001,
'offset': 0,
'type': 'int16',
'unit': '1'
}, {
'band_id': '4',
'name': 'red',
'wavelength_nm': 664.5,
'res_m': 10,
'scale': 0.0001,
'offset': 0,
'type': 'int16',
'unit': '1'
}]
})
imagecollection = GeotrellisTimeSeriesImageCollection(
"test", InMemoryServiceRegistry(), metadata=metadata)
datacube = GeotrellisTimeSeriesImageCollection._tile_to_datacube(
TestMultiBandUDF.tile.cells, None, bands_metadata=metadata.bands)
the_array = datacube.get_array()
assert the_array is not None
print(the_array)
def test_download_masked_geotiff_reproject(self):
input = self.create_spacetime_layer()
polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]])
import pyproj
from shapely.ops import transform
from functools import partial
project = partial(
pyproj.transform,
pyproj.Proj(init="EPSG:4326"), # source coordinate system
pyproj.Proj(init="EPSG:3857")) # destination coordinate system
reprojected = transform(project, polygon)
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
imagecollection = imagecollection.mask(reprojected, "EPSG:3857")
geotiffs = imagecollection.download(
str(self.temp_folder / "test_download_masked_result.3857"))
print(geotiffs)