def test_merge(self):
graph1 = {
"sum1": {
"arguments": {
"data1": {
"from_node": "node1"
},
"data2": {
"from_node": "node3"
}
},
"process_id": "sum",
"result": True
}
}
graph2 = {
"sum1": {
"arguments": {
"data": {
"from_node": "node4"
},
"data2": [{
"from_node": "node4"
}]
},
"process_id": "sum",
},
"sum2": {
"arguments": {
"data": {
"from_node": "sum1"
},
"data2": [{
"from_node": "sum1"
}]
},
"process_id": "sum",
}
}
builder1 = GraphBuilder(graph1)
builder2 = GraphBuilder(graph2)
merged = builder1.merge(builder2).processes
import json
print(json.dumps(merged, indent=2))
self.assertIn("sum1", merged)
self.assertIn("sum4", merged)
self.assertIn("sum5", merged)
self.assertEqual("sum4",
merged["sum5"]["arguments"]["data"]["from_node"])
self.assertEqual("sum4",
merged["sum5"]["arguments"]["data2"][0]["from_node"])
def _reduce_bands_binary_xy(self, operator, other: Union[ImageCollection,
Union[int,
float]]):
"""
Pixelwise comparison of this data cube with another cube or constant.
:param other: Another data cube, or a constant
:return:
"""
if isinstance(other, int) or isinstance(other, float):
my_builder = self._get_band_graph_builder()
new_builder = None
extend_previous_callback_graph = my_builder is not None
if not extend_previous_callback_graph:
new_builder = GraphBuilder()
# TODO merge both process graphs?
new_builder.add_process(operator,
x={'from_argument': 'data'},
y=other,
result=True)
else:
new_builder = my_builder.shallow_copy()
current_result = new_builder.find_result_node_id()
new_builder.processes[current_result]['result'] = False
new_builder.add_process(operator,
x={'from_node': current_result},
y=other,
result=True)
return self._create_reduced_collection(
new_builder, extend_previous_callback_graph)
elif isinstance(other, ImageCollection):
return self._reduce_bands_binary(operator, other)
else:
raise ValueError("Unsupported right-hand operand: " + str(other))
def __invert__(self):
"""
:return:
"""
operator = 'not'
my_builder = self._get_band_graph_builder()
new_builder = None
extend_previous_callback_graph = my_builder is not None
# TODO: why does these `add_process` calls use "expression" instead of "data" like the other cases?
if not extend_previous_callback_graph:
new_builder = GraphBuilder()
# TODO merge both process graphs?
new_builder.add_process(operator,
expression={'from_argument': 'data'},
result=True)
else:
new_builder = my_builder.copy()
current_result = new_builder.find_result_node_id()
new_builder.processes[current_result]['result'] = False
new_builder.add_process(operator,
expression={'from_node': current_result},
result=True)
return self._create_reduced_collection(new_builder,
extend_previous_callback_graph)
def test_create_from_existing(self):
graph = {
"sum_01": {
"arguments": {
"data1": {
"from_node": "node1"
},
"data2": {
"from_node": "node3"
}
},
"process_id": "sum",
"result": True
},
"sum_02": {
"arguments": {
"data": {
"from_node": "node4"
}
},
"process_id": "sum",
}
}
builder = GraphBuilder(graph)
print(builder.processes)
self.assertEqual(2, builder.id_counter["sum"])
def _get_band_graph_builder(self):
current_node = self.graph[self.node_id]
if current_node["process_id"] == "reduce":
# TODO: check "dimension" of "reduce" in some way?
callback_graph = current_node["arguments"]["reducer"]["callback"]
return GraphBuilder.from_process_graph(callback_graph)
return None
def load_collection(cls,
collection_id: str,
session: 'Connection' = None,
spatial_extent: Union[Dict[str, float], None] = None,
temporal_extent: Union[List[Union[str,
datetime.datetime,
datetime.date]],
None] = None,
bands: Union[List[str], None] = None,
fetch_metadata=True):
"""
Create a new Image Collection/Raster Data cube.
:param collection_id: A collection id, should exist in the backend.
:param session: The session to use to connect with the backend.
:param spatial_extent: limit data to specified bounding box or polygons
:param temporal_extent: limit data to specified temporal interval
:param bands: only add the specified bands
:return:
"""
# TODO: rename function to load_collection for better similarity with corresponding process id?
builder = GraphBuilder()
process_id = 'load_collection'
normalized_temporal_extent = list(
get_temporal_extent(extent=temporal_extent)
) if temporal_extent is not None else None
arguments = {
'id': collection_id,
'spatial_extent': spatial_extent,
'temporal_extent': normalized_temporal_extent,
}
metadata = session.collection_metadata(
collection_id) if fetch_metadata else None
if bands:
if isinstance(bands, str):
bands = [bands]
if metadata:
bands = [
metadata.band_dimension.band_name(b, allow_common=False)
for b in bands
]
arguments['bands'] = bands
node_id = builder.process(process_id, arguments)
if bands:
metadata = metadata.filter_bands(bands)
return cls(node_id, builder, session, metadata=metadata)
def test_merge_issue50(self):
"""https://github.com/Open-EO/openeo-python-client/issues/50"""
graph = {
'op3': {
'process_id': 'op',
'arguments': {
'data': {
'from_node': 'op1',
'ref': 'A'
}
}
},
'op2': {
'process_id': 'op',
'arguments': {
'data': {
'from_node': 'src',
'ref': 'B'
}
}
},
'op1': {
'process_id': 'op',
'arguments': {
'data': {
'from_node': 'op2',
'ref': 'C'
}
}
},
'op4': {
'process_id': 'op',
'arguments': {
'data': {
'from_node': 'op3',
'ref': 'D'
}
}
},
}
builder = GraphBuilder(graph)
assert builder.processes['op1']['arguments']['data'] == {
'from_node': 'op2',
'ref': 'C'
}
assert builder.processes['op2']['arguments']['data'] == {
'from_node': 'src',
'ref': 'B'
}
assert builder.processes['op3']['arguments']['data'] == {
'from_node': 'op1',
'ref': 'A'
}
assert builder.processes['op4']['arguments']['data'] == {
'from_node': 'op3',
'ref': 'D'
}
def _graph_merge(self, other_graph: Dict):
newbuilder = self.builder.shallow_copy()
merged = newbuilder.merge(GraphBuilder.from_process_graph(other_graph))
# TODO: properly update metadata as well?
newCollection = ImageCollectionClient(self.node_id,
merged,
self.session,
metadata=self.metadata)
return newCollection
def test_empty_mask():
from shapely import geometry
polygon = geometry.Polygon([[1.0, 1.0], [2.0, 1.0], [2.0, 1.0], [1.0,
1.0]])
client = ImageCollectionClient(node_id=None,
builder=GraphBuilder(),
session=None)
with pytest.raises(ValueError, match=r"Mask .+ has an area of 0.0"):
client.mask(polygon)
def load_disk_collection(cls, session: 'Connection', file_format: str,
glob_pattern: str,
**options) -> 'ImageCollection':
"""
Loads image data from disk as an ImageCollection.
:param session: The session to use to connect with the backend.
:param file_format: the file format, e.g. 'GTiff'
:param glob_pattern: a glob pattern that matches the files to load from disk
:param options: options specific to the file format
:return: the data as an ImageCollection
"""
builder = GraphBuilder()
process_id = 'load_disk_data'
arguments = {
'format': file_format,
'glob_pattern': glob_pattern,
'options': options
}
node_id = builder.process(process_id, arguments)
return cls(node_id, builder, session, metadata={})
def _reduce_bands_binary_const(self, operator, other: Union[int, float]):
my_builder = self._get_band_graph_builder()
new_builder = None
extend_previous_callback_graph = my_builder is not None
if not extend_previous_callback_graph:
new_builder = GraphBuilder()
# TODO merge both process graphs?
new_builder.add_process(operator,
data=[{
'from_argument': 'data'
}, other],
result=True)
else:
current_result = my_builder.find_result_node_id()
new_builder = my_builder.shallow_copy()
new_builder.processes[current_result]['result'] = False
new_builder.add_process(operator,
data=[{
'from_node': current_result
}, other],
result=True)
return self._create_reduced_collection(new_builder,
extend_previous_callback_graph)
def test_create_empty(self):
builder = GraphBuilder()
builder.process("sum", {})
self.assertEqual(1, len(builder.processes))
def _reduce_bands_binary(self,
operator,
other: 'ImageCollectionClient',
arg_name='data'):
# first we create the callback
my_builder = self._get_band_graph_builder()
other_builder = other._get_band_graph_builder()
merged = GraphBuilder.combine(operator=operator,
first=my_builder
or {'from_argument': 'data'},
second=other_builder
or {'from_argument': 'data'},
arg_name=arg_name)
# callback is ready, now we need to properly set up the reduce process that will invoke it
if my_builder is None and other_builder is None:
# there was no previous reduce step, perhaps this is a cube merge?
# cube merge is happening when node id's differ, otherwise we can use regular reduce
if (self.node_id != other.node_id):
# we're combining data from two different datacubes: http://api.openeo.org/v/0.4.0/processreference/#merge_cubes
# set result node id's first, to keep track
my_builder = self.builder
my_builder.processes[self.node_id]['result'] = True
other_builder = other.builder
other_builder.processes[other.node_id]['result'] = True
cubes_merged = GraphBuilder.combine(operator="merge_cubes",
first=my_builder,
second=other_builder,
arg_name="cubes")
node_id = cubes_merged.find_result_node_id()
the_node = cubes_merged.processes[node_id]
the_node["result"] = False
cubes = the_node["arguments"]["cubes"]
the_node["arguments"]["cube1"] = cubes[0]
the_node["arguments"]["cube2"] = cubes[1]
del the_node["arguments"]["cubes"]
#there can be only one process for now
cube_list = list(
merged.processes.values())[0]["arguments"][arg_name]
assert len(cube_list) == 2
# it is really not clear if this is the agreed way to go
cube_list[0]["from_argument"] = "x"
cube_list[1]["from_argument"] = "y"
the_node["arguments"]["overlap_resolver"] = {
'callback': merged.processes
}
the_node["arguments"]["binary"] = True
return ImageCollectionClient(node_id,
cubes_merged,
self.session,
metadata=self.metadata)
else:
args = {
'data': {
'from_node': self.node_id
},
'reducer': {
'callback': merged.processes
}
}
return self.graph_add_process("reduce", args)
else:
left_data_arg = self.builder.processes[
self.node_id]["arguments"]["data"]
right_data_arg = other.builder.processes[
other.node_id]["arguments"]["data"]
if left_data_arg != right_data_arg:
raise BandMathException(
"'Band math' between bands of different image collections is not supported yet."
)
node_id = self.node_id
reducing_graph = self
if reducing_graph.graph[node_id]["process_id"] != "reduce":
node_id = other.node_id
reducing_graph = other
new_builder = reducing_graph.builder.shallow_copy()
new_builder.processes[node_id]['arguments']['reducer'][
'callback'] = merged.processes
# now current_node should be a reduce node, let's modify it
# TODO: properly update metadata of reduced cube? #metadatareducedimension
return ImageCollectionClient(node_id,
new_builder,
reducing_graph.session,
metadata=self.metadata)
