def test_merge_cubes_exception_if_levels_do_not_match():
red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
layer1 = _create_spacetime_layer(cells=np.array([[red_ramp]]))
layer2 = _create_spacetime_layer(cells=np.array([[nir_ramp]]))
metadata = _build_metadata(bands=["the_band"])
cube1 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer1}),
metadata=metadata)
cube2 = GeopysparkDataCube(pyramid=gps.Pyramid({14: layer2}),
metadata=metadata)
with pytest.raises(OpenEOApiException) as excinfo:
res = cube1.merge_cubes(cube2, 'sum')
def test_merge_cubes_into_single_band():
red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
layer1 = _create_spacetime_layer(cells=np.array([[red_ramp]]))
layer2 = _create_spacetime_layer(cells=np.array([[nir_ramp]]))
metadata = _build_metadata(bands=["the_band"])
cube1 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer1}),
metadata=metadata)
cube2 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer2}),
metadata=metadata)
res = cube1.merge_cubes(cube2, 'sum')
stitched = res.pyramid.levels[0].to_spatial_layer().stitch()
assert stitched.cells.shape[0] == 1
np.testing.assert_array_equal(red_ramp + nir_ramp, stitched.cells[0, 0:4,
0:4])
def test_reduce_bands_comparison_ops(self):
input = self.create_spacetime_layer_singleband()
input = gps.Pyramid({0: input})
imagecollection = GeotrellisTimeSeriesImageCollection(
input, InMemoryServiceRegistry())
visitor = GeotrellisTileProcessGraphVisitor()
graph = {
"gt": {
"arguments": {
"x": {
"from_argument": "data"
},
"y": 6.0
},
"process_id": "gt",
"result": True
}
}
visitor.accept_process_graph(graph)
stitched = imagecollection.reduce_bands(
visitor).pyramid.levels[0].to_spatial_layer().stitch()
print(stitched)
self.assertEqual(1, stitched.cells[0][0][0])
def test_linear_scale_range(self):
red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
layer = self._create_spacetime_layer(
cells=np.array([[red_ramp], [nir_ramp]]))
pyramid = gps.Pyramid({0: layer})
metadata = CollectionMetadata({
"properties": {
"eo:bands": [
{
"name": "B04",
"common_name": "red"
},
{
"name": "B08",
"common_name": "nir"
},
]
}
})
imagecollection = GeotrellisTimeSeriesImageCollection(
pyramid, InMemoryServiceRegistry(), metadata=metadata)
stitched = imagecollection.ndvi().linear_scale_range(
-1, 1, 0, 100).pyramid.levels[0].to_spatial_layer().stitch()
cells = stitched.cells[0, 0:4, 0:4]
expected = 50.0 * (1.0 + np.array(
[[np.nan, 1 / 1, 2 / 2, 3 / 3], [-1 / 1, 0 / 2, 1 / 3, 2 / 4],
[-2 / 2, -1 / 3, 0 / 4, 1 / 5], [-3 / 3, -2 / 4, -1 / 5, 0 / 6]]))
expected[0][0] = 255.0
np.testing.assert_array_almost_equal(cells, expected.astype(np.uint8))
def test_zonal_statistics_median_datacube(self):
layer = self.create_spacetime_layer()
imagecollection = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer}), InMemoryServiceRegistry())
polygon = Polygon(shell=[
(0.0, 0.0),
(1.0, 0.0),
(1.0, 1.0),
(0.0, 1.0),
(0.0, 0.0)
])
result = imagecollection.zonal_statistics(polygon, "median")
assert result.data == {'2017-09-25T11:37:00Z': [[1.0, 2.0]]}
covjson = result.to_covjson()
assert covjson["ranges"] == {
"band0": {
"type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"],
"shape": (1, 1),
"values": [1.0]
},
"band1": {
"type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"],
"shape": (1, 1),
"values": [2.0]
}
}
def test_apply_complex_graph():
graph = {
"sin": {
"arguments": {
"x": {
"from_argument": "data"
}
},
"process_id": "sin",
"result": False
},
"multiply": {
"arguments": {
"x": {
"from_node": "sin"
},
"y": 5.0
},
"process_id": "multiply",
"result": True
}
}
input = create_spacetime_layer()
cube = GeopysparkDataCube(gps.Pyramid({0: input}),
InMemoryServiceRegistry())
res = cube.apply(graph)
data = res.pyramid.levels[0].to_spatial_layer().stitch().cells
np.testing.assert_array_almost_equal(data[0, 2:6, 2:6],
5.0 * np.sin(first[0]))
np.testing.assert_array_almost_equal(data[1, 2:6, 2:6],
5.0 * np.sin(second[0]))
def test_apply_cos():
input = create_spacetime_layer()
cube = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
res = cube.apply("cos")
data = res.pyramid.levels[0].to_spatial_layer().stitch().cells
np.testing.assert_array_almost_equal(data[0, 2:6, 2:6], np.cos(first[0]))
np.testing.assert_array_almost_equal(data[1, 2:6, 2:6], np.cos(second[0]))
def test_reduce_bands_logical_ops():
input = create_spacetime_layer_singleband()
input = gps.Pyramid({0: input})
imagecollection = GeopysparkDataCube(pyramid=input)
visitor = GeotrellisTileProcessGraphVisitor()
graph = {
"eq": {
"arguments": {
"x": {
"from_argument": "data"
},
"y": 10
},
"process_id": "eq",
},
"not": {
"arguments": {
"expression": {
"from_node": "eq"
}
},
"process_id": "not",
"result": True
}
}
visitor.accept_process_graph(graph)
stitched = imagecollection.reduce_bands(
visitor).pyramid.levels[0].to_spatial_layer().stitch()
print(stitched)
assert 0 == stitched.cells[0][0][0]
def imagecollection_with_two_bands_and_three_dates_webmerc(request):
from geopyspark.geotrellis import (SpaceTimeKey, Tile, _convert_to_unix_time)
from geopyspark.geotrellis.constants import LayerType
from geopyspark.geotrellis.layer import TiledRasterLayer
import geopyspark as gps
from openeogeotrellis.geopysparkdatacube import GeopysparkDataCube,GeopysparkCubeMetadata
date1, date3, rdd = numpy_rdd_two_bands_and_three_dates()
metadata = {'cellType': 'int32ud-1',
'extent': extent_webmerc,
'crs': '+proj=merc +a=6378137 +b=6378137 +lat_ts=0 +lon_0=0 +x_0=0 +y_0=0 +k=1 +units=m +no_defs ',
'bounds': {
'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(date1)},
'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(date3)}
},
'layoutDefinition': {
'extent': extent_webmerc,
'tileLayout': layout
}
}
geopyspark_layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
datacube = GeopysparkDataCube(pyramid=gps.Pyramid({0: geopyspark_layer}), metadata=GeopysparkCubeMetadata(openeo_metadata))
if request.instance:
request.instance.imagecollection_with_two_bands_and_three_dates = datacube
return datacube
def create_red_nir_layer():
red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
layer = _create_spacetime_layer(cells=np.array([[red_ramp], [nir_ramp]]))
pyramid = gps.Pyramid({0: layer})
metadata = GeopysparkCubeMetadata({
"cube:dimensions": {
"x": {
"type": "spatial",
"axis": "x"
},
"y": {
"type": "spatial",
"axis": "y"
},
"bands": {
"type": "bands",
"values": ["B04", "B08"]
}
},
"summaries": {
"eo:bands": [
{
"name": "B04",
"common_name": "red"
},
{
"name": "B08",
"common_name": "nir"
},
]
}
})
imagecollection = GeopysparkDataCube(pyramid=pyramid, metadata=metadata)
return imagecollection
def test_apply_if():
input = create_spacetime_layer_singleband()
input = gps.Pyramid({0: input})
imagecollection = GeopysparkDataCube(pyramid=input)
graph = {
"6": {
"arguments": {
"reject": {
"from_parameter": "x"
},
"value": {
"from_node": "10"
},
"accept": 2.0
},
"process_id": "if",
"result": True
},
"10": {
"process_id": "gt",
"arguments": {
"x": {
"from_parameter": "x"
},
"y": 7.0
}
}
}
stitched = imagecollection.apply(
graph).pyramid.levels[0].to_spatial_layer().stitch()
print(stitched)
assert 2.0 == stitched.cells[0][0][0]
def test_zonal_statistics(self):
layer = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer}))
polygon = Polygon(shell=[
(0.0, 0.0),
(1.0, 0.0),
(1.0, 1.0),
(0.0, 1.0),
(0.0, 0.0)
])
result = imagecollection.zonal_statistics(polygon, "mean")
assert result.data == {'2017-09-25T11:37:00Z': [[1.0, 2.0]]}
covjson = result.to_covjson()
assert covjson["ranges"] == {
"band0": {
"type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"],
"shape": (1, 1),
"values": [1.0]
},
"band1": {
"type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"],
"shape": (1, 1),
"values": [2.0]
},
}
def test_download_as_catalog(self):
input = self.create_spacetime_layer()
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
imagecollection.download("catalogresult.tiff",
format="GTIFF",
parameters={"catalog": True})
def download_no_args(self, tmp_path, format, format_options={}):
input = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands')
imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','',''))
res = imagecollection.save_result(str(tmp_path / "test_download_result.") + format, format=format, format_options=format_options)
print(res)
return res
def test_polygon_series(self):
input = self.create_spacetime_layer()
polygon = Polygon([(0, 0), (0, 2), (2, 2), (2, 0), (0, 0)])
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry(),
self.openeo_metadata)
means = imagecollection.polygonal_mean_timeseries(polygon)
assert means == {'2017-09-25T11:37:00': [[1.0, 2.0]]}
def test_download_geotiff_no_args(self):
input = self.create_spacetime_layer()
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
geotiffs = imagecollection.download(
str(self.temp_folder / "test_download_result.geotiff"))
print(geotiffs)
def test_download_masked_geotiff(self):
input = self.create_spacetime_layer()
polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]])
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
imagecollection = imagecollection.mask(polygon)
geotiffs = imagecollection.download(
str(self.temp_folder / "test_download_masked_result.geotiff"))
print(geotiffs)
def download_no_args_single_byte_band(self, tmp_path, format, format_options={}):
input = self.create_spacetime_layer().convert_data_type(gps.CellType.UINT8)
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands')
imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','',''))
imagecollection.filter_bands(['band_one'])
res = imagecollection.save_result(str(tmp_path / "test_download_result_single_band.") + format, format=format, format_options=format_options)
print(res)
return res
def test_another_polygon_series(self):
input = self._create_spacetime_layer(no_data=-1.0)
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
polygon = Polygon(shell=[(2.0, 6.0), (6.0,
6.0), (6.0,
2.0), (2.0,
2.0), (2.0, 6.0)])
means = imagecollection.zonal_statistics(regions=polygon, func="mean")
assert means.data == {
'2017-09-25T11:37:00Z':
[[(0 + 0 + 0 + 0 + 1 + 1 + 1 + 1 + 2 + 2 + 2 + 2) / 12]]
}
def download_masked(self, tmp_path, format):
input = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands')
imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','',''))
polygon = geometry.Polygon([[0, 0], [1.9, 0], [1.9, 1.9], [0, 1.9]])
imagecollection = imagecollection.mask_polygon(mask=polygon)
filename = str(tmp_path / "test_download_masked_result.") + format
res = imagecollection.save_result(filename, format=format)
print(res)
def test_another_polygon_series(self):
input = self._create_spacetime_layer(no_data=-1.0)
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
polygon = Polygon(shell=[(2.0, 6.0), (6.0,
6.0), (6.0,
2.0), (2.0,
2.0), (2.0, 6.0)])
means = imagecollection.polygonal_mean_timeseries(polygon)
assert means == {
'2017-09-25T11:37:00':
[[(0 + 0 + 0 + 0 + 1 + 1 + 1 + 1 + 2 + 2 + 2 + 2) / 12]]
}
def test_point_series():
input = create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
transformed_collection = imagecollection.apply("cos")
for p in points[0:3]:
result = transformed_collection.timeseries(p.x, p.y)
print(result)
value = result.popitem()
assert math.cos(10) == value[1][0]
assert math.cos(5) == value[1][1]
def load_disk_data(self, format: str, glob_pattern: str, options: dict,
viewing_parameters: dict) -> object:
if format != 'GTiff':
raise NotImplementedError(
"The format is not supported by the backend: " + format)
date_regex = options['date_regex']
if glob_pattern.startswith("hdfs:"):
kerberos()
from_date = normalize_date(viewing_parameters.get("from", None))
to_date = normalize_date(viewing_parameters.get("to", None))
left = viewing_parameters.get("left", None)
right = viewing_parameters.get("right", None)
top = viewing_parameters.get("top", None)
bottom = viewing_parameters.get("bottom", None)
srs = viewing_parameters.get("srs", None)
band_indices = viewing_parameters.get("bands")
sc = gps.get_spark_context()
gateway = JavaGateway(
eager_load=True, gateway_parameters=sc._gateway.gateway_parameters)
jvm = gateway.jvm
extent = jvm.geotrellis.vector.Extent(float(left), float(bottom), float(right), float(top)) \
if left is not None and right is not None and top is not None and bottom is not None else None
pyramid = jvm.org.openeo.geotrellis.geotiff.PyramidFactory.from_disk(glob_pattern, date_regex) \
.pyramid_seq(extent, srs, from_date, to_date)
temporal_tiled_raster_layer = jvm.geopyspark.geotrellis.TemporalTiledRasterLayer
option = jvm.scala.Option
levels = {
pyramid.apply(index)._1(): TiledRasterLayer(
LayerType.SPACETIME,
temporal_tiled_raster_layer(
option.apply(pyramid.apply(index)._1()),
pyramid.apply(index)._2()))
for index in range(0, pyramid.size())
}
image_collection = GeotrellisTimeSeriesImageCollection(
pyramid=gps.Pyramid(levels),
service_registry=self._service_registry,
metadata={})
return image_collection.band_filter(
band_indices) if band_indices else image_collection
def test_reduce_bands():
input = create_spacetime_layer()
input = gps.Pyramid({0: input})
collection_metadata = GeopysparkCubeMetadata({
"cube:dimensions": {
"my_bands": {
"type": "bands",
"values": ["B04", "B08"]
},
}
})
imagecollection = GeopysparkDataCube(pyramid=input,
metadata=collection_metadata)
visitor = GeotrellisTileProcessGraphVisitor()
graph = {
"sum": {
"arguments": {
"data": {
"from_argument": "dimension_data"
},
"ignore_nodata": True
},
"process_id": "sum"
},
"subtract": {
"arguments": {
"data": {
"from_argument": "dimension_data"
}
},
"process_id": "subtract"
},
"divide": {
"arguments": {
"data": [{
"from_node": "sum"
}, {
"from_node": "subtract"
}]
},
"process_id": "divide",
"result": True
}
}
visitor.accept_process_graph(graph)
stitched = imagecollection.reduce_dimension(
dimension='my_bands', reducer=visitor,
env=EvalEnv()).pyramid.levels[0].to_spatial_layer().stitch()
print(stitched)
assert 3.0 == stitched.cells[0][0][0]
def test_merge_cubes(self):
red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
layer1 = self._create_spacetime_layer(cells=np.array([[red_ramp]]))
layer2 = self._create_spacetime_layer(cells=np.array([[nir_ramp]]))
metadata = CollectionMetadata(
{"properties": {
"eo:bands": [{
"name": "the_band"
}]
}})
cube1 = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer1}),
InMemoryServiceRegistry(),
metadata=metadata)
cube2 = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer2}),
InMemoryServiceRegistry(),
metadata=metadata)
sum = cube1.merge(cube2, 'sum')
stitched = sum.pyramid.levels[0].to_spatial_layer().stitch()
np.testing.assert_array_equal(red_ramp + nir_ramp,
stitched.cells[0, 0:4, 0:4])
def test_point_series(self):
input = self.create_spacetime_layer()
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
transformed_collection = imagecollection.apply("cos")
for p in self.points[0:3]:
result = transformed_collection.timeseries(p.x, p.y)
print(result)
value = result.popitem()
self.assertEqual(math.cos(10), value[1][0])
self.assertEqual(math.cos(5), value[1][1])
def _test_merge_cubes_subtract_spatial(left_spatial=False,
right_spatial=False):
# TODO: this would be cleaner with @pytest.mark.parameterize but that's not supported on TestCase methods
red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
layer1 = _create_spacetime_layer(cells=np.array([[red_ramp]]))
if left_spatial:
layer1 = layer1.to_spatial_layer()
layer2 = _create_spacetime_layer(cells=np.array([[nir_ramp]]))
if right_spatial:
layer2 = layer2.to_spatial_layer()
metadata = _build_metadata()
cube1 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer1}),
metadata=metadata)
cube2 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer2}),
metadata=metadata)
res = cube1.merge_cubes(cube2, 'subtract')
layer = res.pyramid.levels[0]
if layer.layer_type != LayerType.SPATIAL:
layer = layer.to_spatial_layer()
actual = layer.stitch().cells[0, 0:4, 0:4]
expected = red_ramp - nir_ramp
np.testing.assert_array_equal(expected, actual)
def test_zonal_statistics_datacube(self):
layer = self.create_spacetime_layer()
imagecollection = GeotrellisTimeSeriesImageCollection(gps.Pyramid({0: layer}), InMemoryServiceRegistry())
polygon = Polygon(shell=[
(0.0, 0.0),
(1.0, 0.0),
(1.0, 1.0),
(0.0, 1.0),
(0.0, 0.0)
])
polygon2 = Polygon(shell=[
(2.0, 2.0),
(3.0, 2.0),
(3.0, 3.0),
(2.0, 3.0),
(2.0, 2.0)
])
regions = GeometryCollection([polygon, MultiPolygon([polygon2])])
for use_file in [True,False]:
with self.subTest():
if use_file:
with NamedTemporaryFile(delete=False,suffix='.json',mode='r+') as fp:
json.dump(mapping(regions),fp)
regions_serialized = fp.name
else:
regions_serialized = regions
result = imagecollection.zonal_statistics(regions_serialized, "mean")
assert result.data == {
'2017-09-25T11:37:00Z': [[1.0, 2.0], [1.0, 2.0]]
}
result._regions = regions
covjson = result.to_covjson()
assert covjson["ranges"] == {
"band0": {
"type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"],
"shape": (1, 2),
"values": [1.0, 1.0]
},
"band1": {
"type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"],
"shape": (1, 2),
"values": [2.0, 2.0]
},
}
def _build_cube():
openeo_metadata = {
"cube:dimensions": {
"x": {"type": "spatial", "axis": "x"},
"y": {"type": "spatial", "axis": "y"},
"bands": {"type": "bands", "values": ["red", "nir"]},
"t": {"type": "temporal"}
},
"bands": [
{
"band_id": "red",
"name": "red",
"offset": 0,
"res_m": 10,
"scale": 0.0001,
"type": "int16",
"unit": "1",
"wavelength_nm": 664.5
},
{
"band_id": "nir",
"name": "nir",
"offset": 0,
"res_m": 10,
"scale": 0.0001,
"type": "int16",
"unit": "1",
"wavelength_nm": 835.1
}
],
"description": "Sentinel 2 Level-2: Bottom-of-atmosphere reflectances in cartographic geometry",
"extent": {
"bottom": 39,
"crs": "EPSG:4326",
"left": -34,
"right": 35,
"top": 71
},
"product_id": "CGS_SENTINEL2_RADIOMETRY_V101",
"time": {
"from": "2016-01-01",
"to": "2019-10-01"
}
}
# TODO: avoid instantiating TestTimeSeries? e.g. use pytest fixtures or simple builder functions.
layer = TestTimeSeries().create_spacetime_layer()
cube = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer}),metadata=openeo_metadata)
return cube
def test_point_series(self):
def custom_function(cells: np.ndarray, nd):
return cells[0] + cells[1]
input = self.create_spacetime_layer()
imagecollection = GeotrellisTimeSeriesImageCollection(
gps.Pyramid({0: input}), InMemoryServiceRegistry())
transformed_collection = imagecollection.apply_pixel([0, 1],
custom_function)
for p in self.points[0:3]:
result = transformed_collection.timeseries(p.x, p.y)
print(result)
value = result.popitem()
self.assertEqual(3.0, value[1][0])
