def _graph_merge(self, other_graph: Dict):
newbuilder = GraphBuilder(self.builder.processes)
merged = newbuilder.merge(GraphBuilder(other_graph))
newCollection = ImageCollectionClient(self.node_id, merged,
self.session)
newCollection.bands = self.bands
return newCollection
def create_collection(cls,
collection_id: str,
session: Connection = None,
spatial_extent: Union[Dict, None] = None,
temporal_extent: Union[List, None] = None,
bands: Union[List, None] = None):
"""
Create a new Image Collection/Raster Data cube.
:param collection_id: A collection id, should exist in the backend.
:param session: The session to use to connect with the backend.
:param spatial_extent: limit data to specified bounding box or polygons
:param temporal_extent: limit data to specified temporal interval
:param bands: only add the specified bands
:return:
"""
# TODO: rename function to load_collection for better similarity with corresponding process id?
builder = GraphBuilder()
if session.capabilities().api_version_check.at_least('0.4.0'):
process_id = 'load_collection'
arguments = {
'id': collection_id,
'spatial_extent': spatial_extent,
'temporal_extent': temporal_extent,
}
if bands:
arguments['bands'] = bands
else:
process_id = 'get_collection'
arguments = {'name': collection_id}
id = builder.process(process_id, arguments)
return ImageCollectionClient(id, builder, session)
def load_disk_collection(cls, session: 'Connection', file_format: str,
glob_pattern: str,
**options) -> 'ImageCollection':
"""
Loads image data from disk as an ImageCollection.
:param session: The session to use to connect with the backend.
:param file_format: the file format, e.g. 'GTiff'
:param glob_pattern: a glob pattern that matches the files to load from disk
:param options: options specific to the file format
:return: the data as an ImageCollection
"""
assert session.capabilities().api_version_check.at_least('0.4.0')
builder = GraphBuilder()
process_id = 'load_disk_data'
arguments = {
'format': file_format,
'glob_pattern': glob_pattern,
'options': options
}
node_id = builder.process(process_id, arguments)
return cls(node_id, builder, session, metadata={})
def load_collection(cls,
collection_id: str,
session: 'Connection' = None,
spatial_extent: Union[Dict[str, float], None] = None,
temporal_extent: Union[List[str], None] = None,
bands: Union[List[str], None] = None,
fetch_metadata=True):
"""
Create a new Image Collection/Raster Data cube.
:param collection_id: A collection id, should exist in the backend.
:param session: The session to use to connect with the backend.
:param spatial_extent: limit data to specified bounding box or polygons
:param temporal_extent: limit data to specified temporal interval
:param bands: only add the specified bands
:return:
"""
# TODO: rename function to load_collection for better similarity with corresponding process id?
assert session.capabilities().api_version_check.at_least('0.4.0')
builder = GraphBuilder()
process_id = 'load_collection'
arguments = {
'id': collection_id,
'spatial_extent': spatial_extent,
'temporal_extent': temporal_extent,
}
if bands:
arguments['bands'] = bands
node_id = builder.process(process_id, arguments)
metadata = session.collection_metadata(
collection_id) if fetch_metadata else None
return cls(node_id, builder, session, metadata=metadata)
def _graph_merge(self, other_graph: Dict):
newbuilder = GraphBuilder(self.builder.processes)
merged = newbuilder.merge(GraphBuilder(other_graph))
# TODO: properly update metadata as well?
newCollection = ImageCollectionClient(self.node_id,
merged,
self.session,
metadata=self.metadata)
return newCollection
def image_collection():
builder = GraphBuilder()
id = builder.process("get_collection", {'name': 'S1'})
connection = MagicMock(spec=Connection)
capabilities = MagicMock(spec=Capabilities)
connection.capabilities.return_value = capabilities
capabilities.version.return_value = "0.4.0"
return ImageCollectionClient(id, builder, connection)
def test_merge(self):
graph1 = {
"sum1": {
"arguments": {
"data1": {
"from_node": "node1"
},
"data2": {
"from_node": "node3"
}
},
"process_id": "sum",
"result": True
}
}
graph2 = {
"sum1": {
"arguments": {
"data": {
"from_node": "node4"
},
"data2": [{
"from_node": "node4"
}]
},
"process_id": "sum",
},
"sum2": {
"arguments": {
"data": {
"from_node": "sum1"
},
"data2": [{
"from_node": "sum1"
}]
},
"process_id": "sum",
}
}
builder1 = GraphBuilder(graph1)
builder2 = GraphBuilder(graph2)
merged = builder1.merge(builder2).processes
import json
print(json.dumps(merged, indent=2))
self.assertIn("sum1", merged)
self.assertIn("sum2", merged)
self.assertIn("sum3", merged)
self.assertEqual("sum2",
merged["sum3"]["arguments"]["data"]["from_node"])
self.assertEqual("sum2",
merged["sum3"]["arguments"]["data2"][0]["from_node"])
def graph_add_process(self, process_id, args) -> 'ImageCollection':
"""
Returns a new restimagery with an added process with the given process
id and a dictionary of arguments
:param process_id: String, Process Id of the added process.
:param args: Dict, Arguments of the process.
:return: imagery: Instance of the RestImagery class
"""
#don't modify in place, return new builder
newbuilder = GraphBuilder(self.builder.processes)
id = newbuilder.process(process_id, args)
newCollection = ImageCollectionClient(id, newbuilder, self.session)
newCollection.bands = self.bands
return newCollection
def test_create_from_existing(self):
graph = {
"sum_01": {
"arguments": {
"data1": {
"from_node": "node1"
},
"data2": {
"from_node": "node3"
}
},
"process_id": "sum",
"result": True
},
"sum_02": {
"arguments": {
"data": {
"from_node": "node4"
}
},
"process_id": "sum",
}
}
builder = GraphBuilder(graph)
print(builder.processes)
self.assertEqual(2, builder.id_counter["sum"])
def _get_band_graph_builder(self):
current_node = self.graph[self.node_id]
if current_node["process_id"] == "reduce":
if current_node["arguments"]["dimension"] == "spectral_bands":
callback_graph = current_node["arguments"]["reducer"]["callback"]
return GraphBuilder(graph=callback_graph)
return None
def _reduce_bands_binary(self, operator, other: 'ImageCollectionClient'):
# first we create the callback
fallback_node = {'from_argument': 'data'}
my_builder = self._get_band_graph_builder()
other_builder = other._get_band_graph_builder()
merged = GraphBuilder.combine(operator=operator,
first=my_builder or fallback_node,
second=other_builder or fallback_node)
# callback is ready, now we need to properly set up the reduce process that will invoke it
if my_builder is None and other_builder is None:
# there was no previous reduce step
args = {
'data': {'from_node': self.node_id},
'process': {
'callback': merged.processes
}
}
return self.graph_add_process("reduce", args)
else:
node_id = self.node_id
reducing_graph = self
if reducing_graph.graph[node_id]["process_id"] != "reduce":
node_id = other.node_id
reducing_graph = other
new_builder = reducing_graph.builder.copy()
new_builder.processes[node_id]['arguments']['reducer']['callback'] = merged.processes
# now current_node should be a reduce node, let's modify it
return ImageCollectionClient(node_id, new_builder, reducing_graph.session)
def _reduce_bands_binary_xy(self, operator, other: Union[ImageCollection,
Union[int,
float]]):
"""
Pixelwise comparison of this data cube with another cube or constant.
:param other: Another data cube, or a constant
:return:
"""
if isinstance(other, int) or isinstance(other, float):
my_builder = self._get_band_graph_builder()
new_builder = None
extend_previous_callback_graph = my_builder is not None
if not extend_previous_callback_graph:
new_builder = GraphBuilder()
# TODO merge both process graphs?
new_builder.add_process(operator,
x={'from_argument': 'data'},
y=other,
result=True)
else:
current_result = my_builder.find_result_node_id()
new_builder = my_builder.copy()
new_builder.processes[current_result]['result'] = False
new_builder.add_process(operator,
x={'from_node': current_result},
y=other,
result=True)
return self._create_reduced_collection(
new_builder, extend_previous_callback_graph)
elif isinstance(other, ImageCollection):
return self._reduce_bands_binary(operator, other)
else:
raise ValueError("Unsupported right-hand operand: " + str(other))
def __invert__(self):
"""
:return:
"""
operator = 'not'
my_builder = self._get_band_graph_builder()
new_builder = None
extend_previous_callback_graph = my_builder is not None
# TODO: why does these `add_process` calls use "expression" instead of "data" like the other cases?
if not extend_previous_callback_graph:
new_builder = GraphBuilder()
# TODO merge both process graphs?
new_builder.add_process(operator,
expression={'from_argument': 'data'},
result=True)
else:
current_result = my_builder.find_result_node_id()
new_builder = my_builder.copy()
new_builder.processes[current_result]['result'] = False
new_builder.add_process(operator,
expression={'from_node': current_result},
result=True)
return self._create_reduced_collection(new_builder,
extend_previous_callback_graph)
def test_merge_issue50(self):
"""https://github.com/Open-EO/openeo-python-client/issues/50"""
graph = {
'op3': {
'process_id': 'op',
'arguments': {
'data': {
'from_node': 'op1',
'ref': 'A'
}
}
},
'op2': {
'process_id': 'op',
'arguments': {
'data': {
'from_node': 'src',
'ref': 'B'
}
}
},
'op1': {
'process_id': 'op',
'arguments': {
'data': {
'from_node': 'op2',
'ref': 'C'
}
}
},
'op4': {
'process_id': 'op',
'arguments': {
'data': {
'from_node': 'op3',
'ref': 'D'
}
}
},
}
builder = GraphBuilder(graph)
assert builder.processes['op1']['arguments']['data'] == {
'from_node': 'op2',
'ref': 'C'
}
assert builder.processes['op2']['arguments']['data'] == {
'from_node': 'src',
'ref': 'B'
}
assert builder.processes['op3']['arguments']['data'] == {
'from_node': 'op1',
'ref': 'A'
}
assert builder.processes['op4']['arguments']['data'] == {
'from_node': 'op3',
'ref': 'D'
}
def test_empty_mask():
from shapely import geometry
polygon = geometry.Polygon([[1.0, 1.0], [2.0, 1.0], [2.0, 1.0], [1.0,
1.0]])
client = ImageCollectionClient(node_id=None,
builder=GraphBuilder(),
session=None)
with pytest.raises(ValueError, match=r"Mask .+ has an area of 0.0"):
client.mask(polygon)
def _reduce_bands_binary_const(self, operator, other:Union[int,float]):
my_builder = self._get_band_graph_builder()
new_builder = None
extend_previous_callback_graph = my_builder is not None
if not extend_previous_callback_graph:
new_builder = GraphBuilder()
# TODO merge both process graphs?
new_builder.add_process(operator, data=[{'from_argument': 'data'}, other], result=True)
else:
current_result = my_builder.find_result_node_id()
new_builder = my_builder.copy()
new_builder.processes[current_result]['result'] = False
new_builder.add_process(operator, data=[{'from_node': current_result}, other], result=True)
return self._create_reduced_collection(new_builder,extend_previous_callback_graph)
def setUp(self):
builder = GraphBuilder()
id = builder.process("get_collection", {'name': 'S1'})
connection = MagicMock(spec=Connection)
capabilities = MagicMock(spec=Capabilities)
connection.capabilities.return_value = capabilities
capabilities.version.return_value = "0.4.0"
self.img = ImageCollectionClient(id, builder, connection)
builder = GraphBuilder()
mask_id = builder.process("get_collection", {'name': 'S1_Mask'})
self.mask = ImageCollectionClient(mask_id, builder, connection)
def test_create_empty(self):
builder = GraphBuilder()
builder.process("sum", {})
self.assertEqual(1, len(builder.processes))
