def test_apply_if():
input = create_spacetime_layer_singleband()
input = gps.Pyramid({0: input})
imagecollection = GeopysparkDataCube(pyramid=input)
graph = {
"6": {
"arguments": {
"reject": {
"from_parameter": "x"
},
"value": {
"from_node": "10"
},
"accept": 2.0
},
"process_id": "if",
"result": True
},
"10": {
"process_id": "gt",
"arguments": {
"x": {
"from_parameter": "x"
},
"y": 7.0
}
}
}
stitched = imagecollection.apply(
graph).pyramid.levels[0].to_spatial_layer().stitch()
print(stitched)
assert 2.0 == stitched.cells[0][0][0]
def test_zonal_statistics(self):
layer = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer}))
polygon = Polygon(shell=[
(0.0, 0.0),
(1.0, 0.0),
(1.0, 1.0),
(0.0, 1.0),
(0.0, 0.0)
])
result = imagecollection.zonal_statistics(polygon, "mean")
assert result.data == {'2017-09-25T11:37:00Z': [[1.0, 2.0]]}
covjson = result.to_covjson()
assert covjson["ranges"] == {
"band0": {
"type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"],
"shape": (1, 1),
"values": [1.0]
},
"band1": {
"type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"],
"shape": (1, 1),
"values": [2.0]
},
}
def test_apply_cos():
input = create_spacetime_layer()
cube = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
res = cube.apply("cos")
data = res.pyramid.levels[0].to_spatial_layer().stitch().cells
np.testing.assert_array_almost_equal(data[0, 2:6, 2:6], np.cos(first[0]))
np.testing.assert_array_almost_equal(data[1, 2:6, 2:6], np.cos(second[0]))
def test_reduce_bands_logical_ops():
input = create_spacetime_layer_singleband()
input = gps.Pyramid({0: input})
imagecollection = GeopysparkDataCube(pyramid=input)
visitor = GeotrellisTileProcessGraphVisitor()
graph = {
"eq": {
"arguments": {
"x": {
"from_argument": "data"
},
"y": 10
},
"process_id": "eq",
},
"not": {
"arguments": {
"expression": {
"from_node": "eq"
}
},
"process_id": "not",
"result": True
}
}
visitor.accept_process_graph(graph)
stitched = imagecollection.reduce_bands(
visitor).pyramid.levels[0].to_spatial_layer().stitch()
print(stitched)
assert 0 == stitched.cells[0][0][0]
def test_min_time(self):
input = Pyramid({0: self.tiled_raster_rdd})
cube = GeopysparkDataCube(pyramid=input,
metadata=self.collection_metadata)
env = EvalEnv()
min_time = cube.reduce_dimension(reducer=reducer('min'),
dimension='t',
env=env)
max_time = cube.reduce_dimension(reducer=reducer('max'),
dimension='t',
env=env)
stitched = min_time.pyramid.levels[0].stitch()
print(stitched)
self.assertEquals(2.0, stitched.cells[0][0][0])
for p in self.points[1:3]:
result = min_time.timeseries(p.x, p.y, srs="EPSG:3857")
print(result)
print(cube.timeseries(p.x, p.y, srs="EPSG:3857"))
max_result = max_time.timeseries(p.x, p.y, srs="EPSG:3857")
self.assertEqual(1.0, result['NoDate'])
self.assertEqual(2.0, max_result['NoDate'])
def test_reproject_spatial(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeopysparkDataCube(pyramid=input,
metadata=self.collection_metadata)
ref_path = str(self.temp_folder / "reproj_ref.tiff")
imagecollection.reduce('max',
dimension="t").save_result(ref_path,
format="GTIFF")
resampled = imagecollection.resample_spatial(resolution=0,
projection="EPSG:3395",
method="max")
metadata = resampled.pyramid.levels[0].layer_metadata
print(metadata)
self.assertTrue("proj=merc" in metadata.crs)
path = str(self.temp_folder / "reprojected.tiff")
res = resampled.reduce('max', dimension="t")
res.save_result(path, format="GTIFF")
with rasterio.open(ref_path) as ref_ds:
with rasterio.open(path) as ds:
print(ds.profile)
#this reprojection does not change the shape, so we can compare
assert ds.read().shape == ref_ds.read().shape
assert (ds.crs.to_epsg() == 3395)
def test_apply_complex_graph():
graph = {
"sin": {
"arguments": {
"x": {
"from_argument": "data"
}
},
"process_id": "sin",
"result": False
},
"multiply": {
"arguments": {
"x": {
"from_node": "sin"
},
"y": 5.0
},
"process_id": "multiply",
"result": True
}
}
input = create_spacetime_layer()
cube = GeopysparkDataCube(gps.Pyramid({0: input}),
InMemoryServiceRegistry())
res = cube.apply(graph)
data = res.pyramid.levels[0].to_spatial_layer().stitch().cells
np.testing.assert_array_almost_equal(data[0, 2:6, 2:6],
5.0 * np.sin(first[0]))
np.testing.assert_array_almost_equal(data[1, 2:6, 2:6],
5.0 * np.sin(second[0]))
def test_rename_dimension(self):
imagecollection = GeopysparkDataCube(pyramid=Pyramid(
{0: self.tiled_raster_rdd}),
metadata=self.collection_metadata)
dim_renamed = imagecollection.rename_dimension('t', 'myNewTimeDim')
dim_renamed.metadata.assert_valid_dimension('myNewTimeDim')
def download_no_args(self, tmp_path, format, format_options={}):
input = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
imagecollection.metadata=imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands')
imagecollection.metadata=imagecollection.metadata.append_band(Band('band_two','',''))
res = imagecollection.save_result(str(tmp_path / "test_download_result.") + format, format=format, format_options=format_options)
print(res)
return res
def test_reduce_nontemporal(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeopysparkDataCube(pyramid=input,
metadata=self.collection_metadata)
with self.assertRaises(FeatureUnsupportedException) as context:
imagecollection.reduce(
"max", dimension="gender").pyramid.levels[0].stitch()
print(context.exception)
def test_aggregate_max_time(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeopysparkDataCube(pyramid=input,
metadata=self.collection_metadata)
layer = imagecollection.reduce('max', dimension='t').pyramid.levels[0]
stitched = layer.stitch()
assert CellType.FLOAT32.value == layer.layer_metadata.cell_type
print(stitched)
self.assertEqual(2.0, stitched.cells[0][0][0])
def _test_aggregate_temporal(self, interval_list):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeopysparkDataCube(pyramid=input,
metadata=self.collection_metadata)
stitched = (imagecollection.aggregate_temporal(
interval_list, ["2017-01-03"], "min",
dimension="t").pyramid.levels[0].to_spatial_layer().stitch())
print(stitched)
expected_max = np.min([self.tile2.cells, self.tile.cells], axis=0)
assert_array_almost_equal(stitched.cells[0, 0:5, 0:5], expected_max)
def test_merge_cubes_exception_if_levels_do_not_match():
red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
layer1 = _create_spacetime_layer(cells=np.array([[red_ramp]]))
layer2 = _create_spacetime_layer(cells=np.array([[nir_ramp]]))
metadata = _build_metadata(bands=["the_band"])
cube1 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer1}),
metadata=metadata)
cube2 = GeopysparkDataCube(pyramid=gps.Pyramid({14: layer2}),
metadata=metadata)
with pytest.raises(OpenEOApiException) as excinfo:
res = cube1.merge_cubes(cube2, 'sum')
def test_apply_kernel_int(self):
kernel = np.array([[0, 1, 0], [1, 1, 1], [0, 1, 0]])
input = Pyramid({0: self.tiled_raster_rdd})
img = GeopysparkDataCube(pyramid=input,
metadata=self.collection_metadata)
stitched = img.apply_kernel(kernel).reduce(
'max', dimension="t").pyramid.levels[0].stitch()
assert stitched.cells[0][0][0] == 6.0
assert stitched.cells[0][0][1] == 8.0
assert stitched.cells[0][1][1] == 10.0
def test_another_polygon_series(self):
input = self._create_spacetime_layer(no_data=-1.0)
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
polygon = Polygon(shell=[(2.0, 6.0), (6.0,
6.0), (6.0,
2.0), (2.0,
2.0), (2.0, 6.0)])
means = imagecollection.zonal_statistics(regions=polygon, func="mean")
assert means.data == {
'2017-09-25T11:37:00Z':
[[(0 + 0 + 0 + 0 + 1 + 1 + 1 + 1 + 2 + 2 + 2 + 2) / 12]]
}
def test_apply_dimension_spatiotemporal(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeopysparkDataCube(
pyramid=input,
metadata=GeopysparkCubeMetadata({
"cube:dimensions": {
# TODO: also specify other dimensions?
"bands": {
"type": "bands",
"values": ["2"]
}
},
"summaries": {
"eo:bands": [{
"name": "2",
"common_name": "blue",
"wavelength_nm": 496.6,
"res_m": 10,
"scale": 0.0001,
"offset": 0,
"type": "int16",
"unit": "1"
}]
}
}))
udf_code = """
def rct_savitzky_golay(udf_data:UdfData):
from scipy.signal import savgol_filter
print(udf_data.get_datacube_list())
return udf_data
"""
result = imagecollection.apply_tiles_spatiotemporal(udf_code)
local_tiles = result.pyramid.levels[0].to_numpy_rdd().collect()
print(local_tiles)
self.assertEquals(len(TestMultipleDates.layer), len(local_tiles))
ref_dict = {
e[0]: e[1]
for e in imagecollection.pyramid.levels[0].convert_data_type(
CellType.FLOAT64).to_numpy_rdd().collect()
}
result_dict = {e[0]: e[1] for e in local_tiles}
for k, v in ref_dict.items():
tile = result_dict[k]
assert_array_almost_equal(np.squeeze(v.cells),
np.squeeze(tile.cells),
decimal=2)
def test_point_series():
input = create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
transformed_collection = imagecollection.apply("cos")
for p in points[0:3]:
result = transformed_collection.timeseries(p.x, p.y)
print(result)
value = result.popitem()
assert math.cos(10) == value[1][0]
assert math.cos(5) == value[1][1]
def test_reduce_bands():
input = create_spacetime_layer()
input = gps.Pyramid({0: input})
collection_metadata = GeopysparkCubeMetadata({
"cube:dimensions": {
"my_bands": {
"type": "bands",
"values": ["B04", "B08"]
},
}
})
imagecollection = GeopysparkDataCube(pyramid=input,
metadata=collection_metadata)
visitor = GeotrellisTileProcessGraphVisitor()
graph = {
"sum": {
"arguments": {
"data": {
"from_argument": "dimension_data"
},
"ignore_nodata": True
},
"process_id": "sum"
},
"subtract": {
"arguments": {
"data": {
"from_argument": "dimension_data"
}
},
"process_id": "subtract"
},
"divide": {
"arguments": {
"data": [{
"from_node": "sum"
}, {
"from_node": "subtract"
}]
},
"process_id": "divide",
"result": True
}
}
visitor.accept_process_graph(graph)
stitched = imagecollection.reduce_dimension(
dimension='my_bands', reducer=visitor,
env=EvalEnv()).pyramid.levels[0].to_spatial_layer().stitch()
print(stitched)
assert 3.0 == stitched.cells[0][0][0]
def test_aggregate_temporal_median(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeopysparkDataCube(pyramid=input,
metadata=self.collection_metadata)
stitched = (imagecollection.aggregate_temporal(
["2015-01-01", "2018-01-01"], ["2017-01-03"],
self._median_reducer(),
dimension="t").pyramid.levels[0].to_spatial_layer().stitch())
print(stitched)
expected_median = np.median(
[self.tile.cells, self.tile2.cells, self.tile.cells], axis=0)
#TODO nodata handling??
assert_array_almost_equal(stitched.cells[0, 1:2, 1:2],
expected_median[1:2, 1:2])
def test_merge_cubes_into_single_band():
red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
layer1 = _create_spacetime_layer(cells=np.array([[red_ramp]]))
layer2 = _create_spacetime_layer(cells=np.array([[nir_ramp]]))
metadata = _build_metadata(bands=["the_band"])
cube1 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer1}),
metadata=metadata)
cube2 = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer2}),
metadata=metadata)
res = cube1.merge_cubes(cube2, 'sum')
stitched = res.pyramid.levels[0].to_spatial_layer().stitch()
assert stitched.cells.shape[0] == 1
np.testing.assert_array_equal(red_ramp + nir_ramp, stitched.cells[0, 0:4,
0:4])
def test_zonal_statistics_datacube(self):
layer = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: layer}))
polygon = Polygon(shell=[
(0.0, 0.0),
(1.0, 0.0),
(1.0, 1.0),
(0.0, 1.0),
(0.0, 0.0)
])
polygon2 = Polygon(shell=[
(2.0, 2.0),
(3.0, 2.0),
(3.0, 3.0),
(2.0, 3.0),
(2.0, 2.0)
])
regions = GeometryCollection([polygon, MultiPolygon([polygon2])])
for use_file in [True,False]:
with self.subTest():
if use_file:
with NamedTemporaryFile(delete=False,suffix='.json',mode='r+') as fp:
json.dump(mapping(regions),fp)
regions_serialized = fp.name
else:
regions_serialized = regions
result = imagecollection.zonal_statistics(regions_serialized, "mean")
assert result.data == {
'2017-09-25T11:37:00Z': [[1.0, 2.0], [1.0, 2.0]]
}
result._regions = regions
covjson = result.to_covjson()
assert covjson["ranges"] == {
"band0": {
"type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"],
"shape": (1, 2),
"values": [1.0, 1.0]
},
"band1": {
"type": "NdArray", "dataType": "float", "axisNames": ["t", "composite"],
"shape": (1, 2),
"values": [2.0, 2.0]
},
}
def imagecollection_with_two_bands_and_three_dates_webmerc(request):
from geopyspark.geotrellis import (SpaceTimeKey, Tile, _convert_to_unix_time)
from geopyspark.geotrellis.constants import LayerType
from geopyspark.geotrellis.layer import TiledRasterLayer
import geopyspark as gps
from openeogeotrellis.geopysparkdatacube import GeopysparkDataCube,GeopysparkCubeMetadata
date1, date3, rdd = numpy_rdd_two_bands_and_three_dates()
metadata = {'cellType': 'int32ud-1',
'extent': extent_webmerc,
'crs': '+proj=merc +a=6378137 +b=6378137 +lat_ts=0 +lon_0=0 +x_0=0 +y_0=0 +k=1 +units=m +no_defs ',
'bounds': {
'minKey': {'col': 0, 'row': 0, 'instant': _convert_to_unix_time(date1)},
'maxKey': {'col': 1, 'row': 1, 'instant': _convert_to_unix_time(date3)}
},
'layoutDefinition': {
'extent': extent_webmerc,
'tileLayout': layout
}
}
geopyspark_layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
datacube = GeopysparkDataCube(pyramid=gps.Pyramid({0: geopyspark_layer}), metadata=GeopysparkCubeMetadata(openeo_metadata))
if request.instance:
request.instance.imagecollection_with_two_bands_and_three_dates = datacube
return datacube
def create_red_nir_layer():
red_ramp, nir_ramp = np.mgrid[0:4, 0:4]
layer = _create_spacetime_layer(cells=np.array([[red_ramp], [nir_ramp]]))
pyramid = gps.Pyramid({0: layer})
metadata = GeopysparkCubeMetadata({
"cube:dimensions": {
"x": {
"type": "spatial",
"axis": "x"
},
"y": {
"type": "spatial",
"axis": "y"
},
"bands": {
"type": "bands",
"values": ["B04", "B08"]
}
},
"summaries": {
"eo:bands": [
{
"name": "B04",
"common_name": "red"
},
{
"name": "B08",
"common_name": "nir"
},
]
}
})
imagecollection = GeopysparkDataCube(pyramid=pyramid, metadata=metadata)
return imagecollection
def test_resample_spatial(self):
input = Pyramid({0: self.tiled_raster_rdd})
imagecollection = GeopysparkDataCube(pyramid=input,
metadata=self.collection_metadata)
resampled = imagecollection.resample_spatial(resolution=0.05)
path = str(self.temp_folder / "resampled.tiff")
res = resampled.reduce('max', dimension="t")
res.save_result(path, format="GTIFF")
import rasterio
with rasterio.open(path) as ds:
print(ds.profile)
self.assertAlmostEqual(0.05, ds.res[0], 3)
def test_mask_raster_replacement_int(self):
def createMask(tile):
tile.cells[0][0][0] = 0.0
return tile
input = Pyramid({0: self.tiled_raster_rdd})
mask_layer = self.tiled_raster_rdd.map_tiles(createMask)
mask = Pyramid({0: mask_layer})
cube = GeopysparkDataCube(pyramid=input,
metadata=self.collection_metadata)
mask_cube = GeopysparkDataCube(pyramid=mask)
stitched = cube.mask(mask=mask_cube, replacement=10).reduce(
'max', dimension="t").pyramid.levels[0].stitch()
print(stitched)
assert stitched.cells[0][0][0] == 2.0
assert stitched.cells[0][0][1] == 10.0
def test_multiband_with_tile_coordinates(self):
metadata = GeopysparkCubeMetadata({
"cube:dimensions": {
# TODO: also specify other dimensions?
"bands": {
"type": "bands",
"values": ["2", "3", "4"]
}
},
"summaries": {
"eo:bands": [{
'name': '2',
'common_name': 'blue',
'wavelength_nm': 496.6,
'res_m': 10,
'scale': 0.0001,
'offset': 0,
'type': 'int16',
'unit': '1'
}, {
'name': '3',
'common_name': 'green',
'wavelength_nm': 560,
'res_m': 10,
'scale': 0.0001,
'offset': 0,
'type': 'int16',
'unit': '1'
}, {
'name': '4',
'common_name': 'red',
'wavelength_nm': 664.5,
'res_m': 10,
'scale': 0.0001,
'offset': 0,
'type': 'int16',
'unit': '1'
}]
}
})
datacube = GeopysparkDataCube._numpy_to_xarraydatacube(
TestMultiBandUDF.tile.cells,
SpatialExtent(bottom=100,
top=110,
left=200,
right=220,
height=1,
width=2),
band_coordinates=metadata.band_dimension.band_names)
the_array = datacube.get_array()
assert the_array is not None
np.testing.assert_almost_equal(
the_array.x.values,
np.array([185.0, 195.0, 205.0, 215.0, 225.0, 235.0]), 5)
np.testing.assert_almost_equal(the_array.y.values,
np.array([120.0, 110.0, 100.0, 90.0]),
5)
print(the_array)
def test_write_assets(self, tmp_path):
input = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
imagecollection.metadata = imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands')
imagecollection.metadata = imagecollection.metadata.append_band(Band('band_two', '', ''))
format = 'GTiff'
res = imagecollection.write_assets(str(tmp_path / "test_download_result.") + format, format=format,format_options={
"multidate":True,
"batch_mode":True
})
assert 1 == len(res)
name, asset = res.popitem()
assert Path(asset['href']).parent == tmp_path
assert asset['nodata'] == -1
assert asset['roles'] == ['data']
assert 2 == len(asset['bands'])
assert 'image/tiff; application=geotiff' == asset['type']
assert asset['datetime'] == "2017-09-25T11:37:00Z"
def test_reprojection():
"""
It is important that reprojection in Python and Geotrellis give the same result, for alignment of bounding boxes!
@return:
"""
reprojected = GeopysparkDataCube._reproject_extent("EPSG:4326","EPSG:32631",5.071, 51.21,5.1028,51.23)
print(reprojected)
assert reprojected.xmin == 644594.8230399278
assert reprojected.ymin == 5675216.271413178
assert reprojected.xmax == 646878.5028127492
assert reprojected.ymax == 5677503.191395153
def test_write_assets_samples_netcdf(self, tmp_path):
input = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(pyramid=gps.Pyramid({0: input}))
imagecollection.metadata = imagecollection.metadata.add_dimension('band_one', 'band_one', 'bands')
imagecollection.metadata = imagecollection.metadata.append_band(Band('band_two', '', ''))
format = 'netCDF'
res = imagecollection.write_assets(str(tmp_path / "test_download_result.") + format, format=format,format_options={
"batch_mode":True,
"geometries":geojson_to_geometry(self.features),
"sample_by_feature": True,
"feature_id_property": 'id'
})
assert len(res) == 3
name,asset = res.popitem()
file = asset['href']
assert asset['nodata'] == -1
assert asset['roles'] == ['data']
assert 2 == len(asset['bands'])
assert 'application/x-netcdf' == asset['type']
def test_viewing(self):
geotrellis_layer = self.create_spacetime_layer()
imagecollection = GeopysparkDataCube(
pyramid=gps.Pyramid({0: geotrellis_layer}))
metadata = imagecollection.tiled_viewing_service(
user_id='u9876',
service_id='s1234',
service_type="TMS",
api_version='0.4.0',
process_graph={}).service_metadata
print(metadata)
assert metadata.type == "TMS"
assert isinstance(metadata.attributes["bounds"], dict)
tileresponse = requests.get(metadata.url.format(x=0, y=0, z=0),
timeout=2)
assert tileresponse.status_code == 200
assert tileresponse.content.startswith(PNG_SIGNATURE)
tileresponse = requests.get(metadata.url.format(x=1, y=1, z=0),
timeout=2)
assert tileresponse.status_code == 200
assert tileresponse.content.startswith(PNG_SIGNATURE)
