def test_aggregate_polygon_result_empty_ts_data(tmp_path):
timeseries = {
"2019-01-01T12:34:56Z": [[1, 2], [3, 4]],
"2019-02-01T12:34:56Z": [[5, 6], []],
"2019-03-01T12:34:56Z": [[], []],
"2019-04-01T12:34:56Z": [[], [7, 8]],
"2019-05-01T12:34:56Z": [[9, 10], [11, 12]],
}
regions = GeometryCollection([
Polygon([(0, 0), (5, 1), (1, 4)]),
Polygon([(6, 1), (1, 7), (9, 9)])
])
result = AggregatePolygonResult(timeseries, regions=regions)
result.set_format("netcdf")
filename = result.to_netcdf(tmp_path / 'timeseries_xarray_empty.nc')
timeseries_ds = xr.open_dataset(filename)
assert_array_equal(timeseries_ds.time.data, [
np.datetime64("2019-01-01T12:34:56Z"),
np.datetime64("2019-02-01T12:34:56Z"),
np.datetime64("2019-04-01T12:34:56Z"),
np.datetime64("2019-05-01T12:34:56Z"),
])
assert_array_equal( timeseries_ds.band_0.sel(feature=0).data, [1, 5, np.nan, 9])
assert_array_equal( timeseries_ds.band_1.sel(feature=0).data, [2, 6, np.nan, 10])
def test_aggregate_polygon_result_basic(tmp_path):
time_series = {
"2019-10-15T08:15:45Z": [[1, 2, 3], [4, 5, 6]],
"2019-11-11T01:11:11Z": [[7, 8, 9], [10, 11, 12]],
}
regions = GeometryCollection(
[Polygon([(0, 0), (5, 1), (1, 4)]),
Polygon([(6, 1), (1, 7), (9, 9)])])
result = AggregatePolygonResult(time_series, regions=regions)
result.set_format("csv")
filename = result.to_csv(tmp_path / 'timeseries.csv')
expected_results = {
'2019-10-15T08:15:45Z__01': [1, 4],
'2019-10-15T08:15:45Z__02': [2, 5],
'2019-10-15T08:15:45Z__03': [3, 6],
'2019-11-11T01:11:11Z__01': [7, 10],
'2019-11-11T01:11:11Z__02': [8, 11],
'2019-11-11T01:11:11Z__03': [9, 12]
}
assert pd.DataFrame.from_dict(expected_results).equals(
pd.read_csv(filename))
def test_aggregate_polygon_result_empty_single_polygon_single_band(tmp_path):
"""https://github.com/Open-EO/openeo-python-driver/issues/70"""
time_series = {
"2019-01-01T12:34:56Z": [[]],
"2019-02-01T12:34:56Z": [[2]],
"2019-03-01T12:34:56Z": [[]],
"2019-04-01T12:34:56Z": [[8]],
"2019-05-01T12:34:56Z": [[9]],
}
regions = GeometryCollection([
Polygon([(0, 0), (5, 1), (1, 4)]),
])
result = AggregatePolygonResult(time_series, regions=regions)
result.set_format("csv")
filename = result.to_csv(tmp_path / 'timeseries_empty.csv')
expected_results = {
'2019-01-01T12:34:56Z': [nan],
'2019-02-01T12:34:56Z': [2],
'2019-03-01T12:34:56Z': [nan],
'2019-04-01T12:34:56Z': [8],
'2019-05-01T12:34:56Z': [9],
}
assert pd.DataFrame.from_dict(expected_results).equals(
pd.read_csv(filename))
def test_aggregate_polygon_result_empty_ts_data(tmp_path):
time_series = {
"2019-01-01T12:34:56Z": [[], [3, 4]],
"2019-02-01T12:34:56Z": [[5, 6], []],
"2019-03-01T12:34:56Z": [[], []],
"2019-04-01T12:34:56Z": [[], [7, 8]],
"2019-05-01T12:34:56Z": [[9, 10], [11, 12]],
}
regions = GeometryCollection(
[Polygon([(0, 0), (5, 1), (1, 4)]),
Polygon([(6, 1), (1, 7), (9, 9)])])
result = AggregatePolygonResult(time_series, regions=regions)
result.set_format("csv")
filename = result.to_csv(tmp_path / 'timeseries_empty.csv')
expected_results = {
'2019-01-01T12:34:56Z__01': [nan, 3],
'2019-01-01T12:34:56Z__02': [nan, 4],
'2019-02-01T12:34:56Z__01': [5, nan],
'2019-02-01T12:34:56Z__02': [6, nan],
'2019-03-01T12:34:56Z__01': [nan, nan],
'2019-03-01T12:34:56Z__02': [nan, nan],
'2019-04-01T12:34:56Z__01': [nan, 7],
'2019-04-01T12:34:56Z__02': [nan, 8],
'2019-05-01T12:34:56Z__01': [9, 11],
'2019-05-01T12:34:56Z__02': [10, 12]
}
assert pd.DataFrame.from_dict(expected_results).equals(
pd.read_csv(filename))
def test_aggregate_polygon_result_inconsistent_bands(tmp_path):
timeseries = {
"2019-01-01T12:34:56Z": [[1, 2], [3, 4, 5]],
"2019-02-01T12:34:56Z": [[6], []],
}
regions = GeometryCollection(
[Polygon([(0, 0), (5, 1), (1, 4)]),
Polygon([(6, 1), (1, 7), (9, 9)])])
result = AggregatePolygonResult(timeseries, regions=regions)
result.set_format("netcdf")
with pytest.raises(ValueError):
filename = result.to_netcdf(tmp_path / 'timeseries_xarray_invalid.nc')
def test_aggregate_polygon_result_basic():
timeseries = {
"2019-10-15T08:15:45Z": [[1, 2, 3], [4, 5, 6]],
"2019-11-11T01:11:11Z": [[7, 8, 9], [10, 11, 12]],
}
regions = GeometryCollection(
[Polygon([(0, 0), (5, 1), (1, 4)]),
Polygon([(6, 1), (1, 7), (9, 9)])])
result = AggregatePolygonResult(timeseries, regions=regions)
result.set_format("covjson")
data = result.get_data()
assert json_normalize(data) == load_json(
"aggregate_polygon_result_basic_covjson.json")
def test_aggregate_polygon_result_nan_values():
timeseries = {
"2019-10-15T08:15:45Z": [[1, 2, 3], [4, np.nan, 6]],
"2019-11-11T01:11:11Z": [[7, 8, 9], [np.nan, np.nan, np.nan]],
}
regions = GeometryCollection(
[Polygon([(0, 0), (5, 1), (1, 4)]),
Polygon([(6, 1), (1, 7), (9, 9)])])
result = AggregatePolygonResult(timeseries, regions=regions)
result.set_format("covjson")
data = json_normalize(result.prepare_for_json())
assert data["ranges"]["band0"]["values"] == [1, 4, 7, None]
assert data["ranges"]["band1"]["values"] == [2, None, 8, None]
assert data["ranges"]["band2"]["values"] == [3, 6, 9, None]
def test_aggregate_polygon_result_nan_values(tmp_path):
timeseries = {
"2019-10-15T08:15:45Z": [[1, 2, 3], [4, np.nan, 6]],
"2019-11-11T01:11:11Z": [[7, 8, 9], [np.nan, np.nan, np.nan]],
}
regions = GeometryCollection(
[Polygon([(0, 0), (5, 1), (1, 4)]),
Polygon([(6, 1), (1, 7), (9, 9)])])
result = AggregatePolygonResult(timeseries, regions=regions)
filename = result.to_netcdf(tmp_path / 'timeseries_xarray_nodata.nc')
timeseries_ds = xr.open_dataset(filename)
assert_array_equal(timeseries_ds.band_0.sel(feature=0).data, [1, 7])
assert_array_equal(timeseries_ds.band_2.sel(feature=0).data, [3, 9])
assert_array_equal(timeseries_ds.band_2.sel(feature=1).data, [6, np.nan])
def test_aggregate_polygon_result_basic(tmp_path):
timeseries = {
"2019-11-11T01:11:11Z": [[7, 8, 9], [10, 11, 12]],
"2019-10-15T08:15:45Z": [[1, 2, 3], [4, 5, 6]],
}
regions = GeometryCollection(
[Polygon([(0, 0), (5, 1), (1, 4)]),
Polygon([(6, 1), (1, 7), (9, 9)])])
metadata = CollectionMetadata({
"cube:dimensions": {
"x": {
"type": "spatial"
},
"b": {
"type": "bands",
"values": ["red", "green", "blue"]
}
}
})
result = AggregatePolygonResult(timeseries,
regions=regions,
metadata=metadata)
result.set_format("netcdf")
assets = result.write_assets(tmp_path)
theAsset = assets.popitem()[1]
filename = theAsset['href']
assert 'application/x-netcdf' == theAsset['type']
assert ["red", "green", "blue"] == [b['name'] for b in theAsset['bands']]
timeseries_ds = xr.open_dataset(filename)
print(timeseries_ds)
assert_array_equal(
timeseries_ds.band_0.coords['t'].data,
np.asarray([
np.datetime64('2019-10-15T08:15:45'),
np.datetime64('2019-11-11T01:11:11')
]))
timeseries_ds.band_0.sel(feature=1)
timeseries_ds.band_0.sel(t='2019-10-16')
print(timeseries_ds)
assert_array_equal(
4,
timeseries_ds.band_0.sel(feature=1).sel(t="2019-10-15T08:15:45Z").data)
def test_aggregate_polygon_result_basic(tmp_path):
timeseries = {
"2019-10-15T08:15:45Z": [[1, 2, 3], [4, 5, 6]],
"2019-11-11T01:11:11Z": [[7, 8, 9], [10, 11, 12]],
}
regions = GeometryCollection([
Polygon([(0, 0), (5, 1), (1, 4)]),
Polygon([(6, 1), (1, 7), (9, 9)])
])
result = AggregatePolygonResult(timeseries, regions=regions)
result.set_format("netcdf")
filename = result.to_netcdf(tmp_path / 'timeseries_xarray.nc')
timeseries_ds = xr.open_dataset(filename)
print(timeseries_ds)
assert_array_equal(timeseries_ds.band_0.coords['time'].data, np.asarray([ np.datetime64('2019-10-15T08:15:45'),np.datetime64('2019-11-11T01:11:11')]))
timeseries_ds.band_0.sel(feature=1)
timeseries_ds.band_0.sel( time='2019-10-16')
print(timeseries_ds)
def test_aggregate_polygon_result_empty_ts_data():
timeseries = {
"2019-01-01T12:34:56Z": [[1, 2], [3, 4]],
"2019-02-01T12:34:56Z": [[5, 6], []],
"2019-03-01T12:34:56Z": [[], []],
"2019-04-01T12:34:56Z": [[], [7, 8]],
"2019-05-01T12:34:56Z": [[9, 10], [11, 12]],
}
regions = GeometryCollection(
[Polygon([(0, 0), (5, 1), (1, 4)]),
Polygon([(6, 1), (1, 7), (9, 9)])])
result = AggregatePolygonResult(timeseries, regions=regions)
result.set_format("covjson")
data = json_normalize(result.prepare_for_json())
assert data["domain"]["axes"]["t"]["values"] == [
"2019-01-01T12:34:56Z",
"2019-02-01T12:34:56Z",
"2019-04-01T12:34:56Z",
"2019-05-01T12:34:56Z",
]
assert data["ranges"] == {
"band0": {
"type": "NdArray",
"dataType": "float",
"axisNames": ["t", "composite"],
"shape": [4, 2],
"values": [1, 3, 5, None, None, 7, 9, 11],
},
"band1": {
"type": "NdArray",
"dataType": "float",
"axisNames": ["t", "composite"],
"shape": [4, 2],
"values": [2, 4, 6, None, None, 8, 10, 12],
},
}
def aggregate_spatial(
self,
geometries: Union[BaseGeometry, str, DriverVectorCube],
reducer: dict,
target_dimension: str = "result",
) -> Union[AggregatePolygonResult, AggregatePolygonSpatialResult]:
# TODO #71 #114 EP-3981 normalize to vector cube instead of GeometryCollection
geometries, bbox = self._normalize_geometry(geometries)
cube = self.filter_bbox(**bbox, operation="_weak_spatial_extent")
cube._process(operation="aggregate_spatial",
arguments={"geometries": geometries})
if isinstance(geometries, (Polygon, MultiPolygon)):
geometries = GeometryCollection([geometries])
return AggregatePolygonResult(timeseries={}, regions=geometries)
def test_aggregate_polygon_result_inconsistent_bands():
timeseries = {
"2019-01-01T12:34:56Z": [[1, 2], [3, 4, 5]],
"2019-02-01T12:34:56Z": [[6], []],
}
regions = GeometryCollection(
[Polygon([(0, 0), (5, 1), (1, 4)]),
Polygon([(6, 1), (1, 7), (9, 9)])])
result = AggregatePolygonResult(timeseries, regions=regions)
result.set_format("covjson")
with pytest.raises(ValueError):
result.prepare_for_json()
def test_aggregate_polygon_result_inconsistent_bands(tmp_path):
time_series = {
"2019-01-01T12:34:56Z": [[1, 2], [3, 4, 5]],
"2019-02-01T12:34:56Z": [[6], []],
}
regions = GeometryCollection(
[Polygon([(0, 0), (5, 1), (1, 4)]),
Polygon([(6, 1), (1, 7), (9, 9)])])
result = AggregatePolygonResult(time_series, regions=regions)
result.set_format("csv")
with pytest.raises(IndexError):
result.to_csv(tmp_path / 'timeseries_invalid.csv')
def aggregate_spatial(
self,
geometries: Union[BaseGeometry, str, DriverVectorCube],
reducer: dict,
target_dimension: str = "result",
) -> Union[AggregatePolygonResult, AggregatePolygonSpatialResult,
DriverVectorCube]:
# TODO: support more advanced reducers too
assert isinstance(reducer, dict) and len(reducer) == 1
reducer = next(iter(reducer.values()))["process_id"]
assert reducer == 'mean' or reducer == 'avg'
def assert_polygon_sequence(geometries: Union[Sequence,
BaseMultipartGeometry]):
assert len(geometries) > 0
for g in geometries:
assert isinstance(g, Polygon) or isinstance(g, MultiPolygon)
# TODO #114 EP-3981 normalize to vector cube and preserve original properties
if isinstance(geometries, DriverVectorCube):
# Build dummy aggregation data cube
dims, coords = geometries.get_xarray_cube_basics()
if self.metadata.has_temporal_dimension():
dims += (self.metadata.temporal_dimension.name, )
coords[self.metadata.temporal_dimension.name] = [
"2015-07-06T00:00:00", "2015-08-22T00:00:00"
]
if self.metadata.has_band_dimension():
dims += (self.metadata.band_dimension.name, )
coords[self.metadata.band_dimension.
name] = self.metadata.band_names
shape = [len(coords[d]) for d in dims]
data = numpy.arange(numpy.prod(shape)).reshape(shape)
cube = xarray.DataArray(data=data,
dims=dims,
coords=coords,
name="aggregate_spatial")
return geometries.with_cube(cube=cube, flatten_prefix="agg")
elif isinstance(geometries, str):
geometries = [
geometry for geometry in DelayedVector(geometries).geometries
]
assert_polygon_sequence(geometries)
elif isinstance(geometries, GeometryCollection):
# TODO #71 #114 EP-3981: GeometryCollection is deprecated
assert_polygon_sequence(geometries)
elif isinstance(geometries, BaseGeometry):
assert_polygon_sequence([geometries])
else:
assert_polygon_sequence(geometries)
if self.metadata.has_temporal_dimension():
return AggregatePolygonResult(timeseries={
"2015-07-06T00:00:00": [2.345],
"2015-08-22T00:00:00": [float('nan')]
},
regions=geometries)
else:
return DummyAggregatePolygonSpatialResult(cube=self,
geometries=geometries)
def raster_to_vector(self):
return AggregatePolygonResult(timeseries={}, regions=None)