def array_element(agraph, nodeid, jim):
verbose = True
node = agraph[nodeid]
parent_node = node.parent_process.content
data = jim[parent_node['arguments']['data']['from_node']]
if 'index' in node.content['arguments']:
bandindex = node.content['arguments']['index']
bandname = data.dimension['band'][bandindex]
elif 'label' in node.content['arguments']:
bandname = node.content['arguments']['label']
if verbose:
print("array_element with label {}".format(bandname))
bandindex = data.dimension['band'].index(bandname)
else:
raise AttributeError(
"Error: only index or label is supported for array_element")
if verbose:
print("array_element with index {}".format(bandindex))
if data is None:
jim[node.id] = None
elif isinstance(data, pj.Jim):
result = Cube(pj.geometry.cropBand(data, bandindex))
result.dimension['band'] = [bandname]
jim[node.id] = result
elif isinstance(data, pj.JimVect):
raise TypeError(
"Error: {} array_element not implemented for JimVect".format(
type(jim[node.id])))
elif isinstance(data, Collection):
raise TypeError(
"Error: {} array element not implemented for Collection".format(
type(jim[node.id])))
return jim[node.id]
def filter_temporal(agraph, nodeid, jim):
verbose = True
node = agraph[nodeid]
extent = node.content['arguments'].get('extent')
data = jim[node.content['arguments']['data']['from_node']]
if data is None:
jim[node.id] = None
return jim[node.id]
if not isinstance(data, Cube):
raise TypeError(
"Error: filter_temporal only implemented for Cube, not {}".format(
type(jim[node.content['arguments']['data']['from_node']])))
print("times in data: {}".format(data.getDimension('temporal')))
jim[node.id] = Cube(data)
#todo: should be solved in __init__ of Cube
jim[node.id].setDimension('temporal', data.getDimension('temporal'))
jim[node.id].setDimension('band', data.getDimension('band'))
jim[node.id].setResolution('temporal', data.getResolution('temporal'))
jim[node.id].setResolution('spatial', data.getResolution('spatial'))
if len(extent) == 0:
raise ValueError(
"extent should contain at least one date element, but got empty list: "
+ extent)
dateFrom = extent[0]
if len(extent) > 1:
dateTo = extent[1]
else:
dateTo = None
#todo: support datetime resolution finer than day
if not isinstance(dateFrom, datetime):
dateFrom = datetime.strptime(dateFrom, '%Y-%m-%d')
if dateTo:
if not isinstance(dateTo, datetime):
dateTo = datetime.strptime(dateTo, '%Y-%m-%d')
else:
dateTo = dateFrom + jim[node.id].resolution['temporal']
print("dateFrom: {}".format(dateFrom))
print("dateTo: {}".format(dateTo))
print("times in cube: {}".format(jim[node.id].getDimension('temporal')))
filtered_temporal = [
d for d in jim[node.id].getDimension('temporal')
if d >= dateFrom and d < dateTo
]
print("filtered times: {}".format(filtered_temporal))
planeindices = [
jim[node.id].getDimension('temporal').index(d)
for d in filtered_temporal
]
if len(planeindices) < 1:
raise ValueError("Error: filter temporal found no match")
# planeindices=[self.dimension['temporal'].index(d) for d in self.dimension['temporal'] if d >= dateFrom and d < dateTo]
jim[node.id] = Cube(pj.geometry.cropPlane(jim[node.id],
plane=planeindices))
jim[node.id].setDimension('temporal', filtered_temporal)
return jim[node.id]
def filter_bands(agraph, nodeid, jim):
verbose = True
node = agraph[nodeid]
bandindexes = []
if 'bands' in node.content['arguments']:
bandnames = node.content['arguments'].get('bands')
for bandname in bandnames:
bandindex = jim[node.content['arguments']['data']
['from_node']].dimension['band'].index(bandname)
if verbose:
print("array_element with label {}".format(bandname))
print("array_element with index {}".format(bandindex))
bandindexes.append(bandindex)
if jim[node.content['arguments']['data']['from_node']] is None:
jim[node.id] = None
elif isinstance(jim[node.content['arguments']['data']['from_node']],
pj.Jim):
jim[node.id] = Cube(
pj.geometry.cropBand(
jim[node.content['arguments']['data']['from_node']],
bandindexes))
jim[node.id].dimension['band'] = bandnames
return jim[node.id]
elif isinstance(jim[node.content['arguments']['data']['from_node']],
pj.JimVect):
raise TypeError(
"Error: {} array_element not implemented for JimVect".format(
type(jim[node.id])))
elif isinstance(jim[node.content['arguments']['data']['from_node']],
Collection):
raise TypeError(
"Error: {} array element not implemented for Collection".format(
type(jim[node.id])))
else:
raise AttributeError("Error: only bands is supported for filter_bands")
def apply_unary(agraph, nodeid, jim):
verbose = True
node = agraph[nodeid]
if verbose:
print(node)
print("apply {}".format(node.content['description']))
if jim[node.content['arguments']['data']['from_node']] is None:
jim[node.id] = None
return [node.id]
process_node = agraph[node.content['arguments']['process']['from_node']]
if verbose:
print("node is: {}".format(node.content))
print("process node is: {}".format(process_node))
if jim[process_node.id] is None:
if process_node.content['process_id'] == 'linear_scale_range':
inputMin = process_node.content['arguments'].get('inputMin', 0)
inputMax = process_node.content['arguments'].get('inputMax')
if inputMax is None:
raise ValueError("Error: inputMax not set")
outputMin = process_node.content['arguments'].get('outputMin', 0)
outputMax = process_node.content['arguments'].get('outputMax', 255)
#todo: handle data types
if jim[process_node.content['arguments']['x']
['from_node']] is None:
jim[node.id] = None
return [node.id]
jim[process_node.id] = Cube(
jim[process_node.content['arguments']['x']['from_node']])
jim[process_node.id] -= inputMin
jim[process_node.id] /= (inputMax - inputMin)
jim[process_node.id] *= (outputMax - outputMin)
jim[process_node.id] += outputMin
elif process_node.content['process_id'] == 'abs':
if jim[process_node.content['arguments']['x']
['from_node']] is None:
jim[node.id] = None
return [node.id]
jim[process_node.id] = Cube(
abs(jim[process_node.content['arguments']['x']['from_node']]))
else:
raise TypeError(
"Error: {} not implemented for process apply".format(
process_node.content['process_id']))
jim[node.id] = jim[process_node.id]
return jim[node.id]
def save_result(agraph, nodeid, jim, pathname):
verbose = True
node = agraph[nodeid]
if node.content['arguments']['data']['from_node'] not in jim:
print("cannot save result yet")
jim[node.id] = None
return jim[node.id]
if jim[node.content['arguments']['data']['from_node']]:
print("saving result")
jim[node.id] = jim[node.content['arguments']['data']['from_node']]
if isinstance(jim[node.id], pj.Jim):
#to save as multi-spectral GeoTIFF, 1 file per acquisition time
print("jim has {} planes".format(
jim[node.id].properties.nrOfPlane()))
plane2band = False
if jim[node.id].properties.nrOfPlane() > 1:
if jim[node.id].properties.nrOfBand() > 1:
for iplane, theDate in enumerate(
jim[node.id].dimension['temporal']):
print("cropPlane {}".format(iplane))
jimplane = Cube(
pj.geometry.cropPlane(jim[node.id], iplane))
jimplane.properties.setNoDataVals(0)
jimplane.io.write(os.path.join(
pathname,
theDate.strftime('%Y%m%d') + '.tif'),
co=['COMPRESS=LZW', 'TILED=YES'])
return jim[node.id]
else:
plane2band = True
jim[node.id].geometry.plane2band()
jim[node.id].io.write(pathname + '.tif',
co=['COMPRESS=LZW', 'TILED=YES'])
if plane2band:
jim[node.id].geometry.band2plane()
return jim[node.id]
elif isinstance(jim[node.id], pj.JimVect):
# print("saved result: {}".format(jim[node.id].np()))
jim[node.id].io.write(pathname + '.sqlite')
elif isinstance(jim[node.id], Collection):
raise TypeError(
"Error: {} virtual cube not implemented yet".format(
type(jim[node.id])))
else:
raise TypeError("Error: {} type not supported for writing".format(
type(jim[node.id])))
else:
print("cannot save result yet")
jim[node.id] = None
return jim[node.id]
def apply_binary(agraph, nodeid, jim):
verbose = True
node = agraph[nodeid]
if verbose:
print(node)
print("eq {}".format(node.content.get('description')))
jim[node.id] = None
# arguments = [item for sublist in node.content['arguments'].values() for item in sublist]
arguments = node.content['arguments']
x = arguments['x']
y = arguments['y']
if isinstance(x, dict):
if verbose:
print("type of jim is {}".format(type(jim[x['from_node']])))
if jim[x['from_node']] is None:
jim[node.id] = None
return jim[node.id]
else:
jim[node.id] = jim[x['from_node']]
else:
jim[node.id] = x
value = 0
if isinstance(y, dict):
if verbose:
print("type of jim is {}".format(type(jim[y['from_node']])))
if jim[y['from_node']] is None:
jim[node.id] = None
return jim[node.id]
else:
value = jim[y['from_node']]
else:
value = y
if node.content['process_id'] == 'add':
jim[node.id] += value
elif node.content['process_id'] == 'divide':
jim[node.id] /= value
elif node.content['process_id'] == 'eq':
jim[node.id] = Cube(jim[node.id] == value)
elif node.content['process_id'] == 'gt':
jim[node.id] = Cube(jim[node.id] > value)
elif node.content['process_id'] == 'gte':
jim[node.id] = Cube(jim[node.id] >= value)
elif node.content['process_id'] == 'lt':
jim[node.id] = Cube(jim[node.id] < value)
elif node.content['process_id'] == 'lte':
jim[node.id] = Cube(jim[node.id] <= value)
elif node.content['process_id'] == 'multiply':
jim[node.id] *= value
elif node.content['process_id'] == 'neq':
jim[node.id] = Cube(jim[node.id] != value)
elif node.content['process_id'] == 'subtract':
jim[node.id] -= value
else:
raise TypeError("Error: arithmetic {} not implemented".format(
node.content['process_id']))
return jim[node.id]
def aggregate_temporal(agraph, nodeid, jim):
verbose = True
node = agraph[nodeid]
#todo: not tested yet in openeo API v1.0
if node.content['arguments']['dimension'] == 'temporal':
if jim[node.content['arguments']['data']['from_node']] is None:
jim[node.id] = None
return [node.id]
reducer_node = agraph[node.content['arguments']['reducer']
['from_node']]
if verbose:
print("node is: {}".format(node.content))
print("reducer node is: {}".format(reducer_node))
if jim[reducer_node.id] is None:
rule = reducer_node.content['process_id']
if verbose:
print("reducer graph is: {}".format(reducer_node))
print("rule: {}".format(rule))
if isinstance(jim[node.content['arguments']['data']['from_node']],
pj.Jim):
jim[reducer_node.id] = Cube(
jim[node.content['arguments']['data']['from_node']])
jim[reducer_node.id].geometry.reducePlane(rule)
if jim[reducer_node.id] is not None:
jim[node.id] = jim[reducer_node.id]
return jim[node.id]
else:
raise ValueError("Error: jim_reduced is False")
elif isinstance(
jim[node.content['arguments']['data']['from_node']],
pj.JimVect):
raise TypeError("Error: reduce not implemented for JimVect")
elif isinstance(
jim[node.content['arguments']['data']['from_node']],
Collection):
raise TypeError("Error: reduce not implemented for Collection")
else:
#test
jim[node.id] = jim[reducer_node.id]
#jim[node.id]=jim[node.content['arguments']['data']['from_node']]
return jim[node.id]
else:
raise TypeError(
"Error: reduce not implemented for dimension different than temporal"
)
def normalized_difference(agraph, nodeid, jim):
verbose = True
node = agraph[nodeid]
nir = jim[node.content['arguments']['x']['from_node']]
if nir is None:
jim[node.id] = None
return [node.id]
red = jim[node.content['arguments']['y']['from_node']]
if nir is None:
jim[node.id] = None
return [node.id]
jim[node.id] = Cube(pj.pixops.convert(nir, 'GDT_Float32'))
nirnp = nir.np().astype(np.float)
rednp = red.np().astype(np.float)
ndvi = (nirnp - rednp) / (nirnp + rednp)
ndvi[np.isnan(ndvi)] = 0
jim[node.id].np()[:] = ndvi
jim[node.id].dimension['band'] = ['nd']
jim[node.id].dimension['temporal'] = nir.dimension['temporal']
return jim[node.id]
def ndvi(agraph, nodeid, jim):
verbose = True
node = agraph[nodeid]
data = jim[node.content['arguments']['data']['from_node']]
nir = node.content['arguments']['nir']
red = node.content['arguments']['red']
if nir is None:
jim[node.id] = None
return [node.id]
red = jim[node.content['arguments']['y']['from_node']]
if nir is None:
jim[node.id] = None
return [node.id]
jim[node.id] = Cube(pj.pixops.convert(nir, 'GDT_Float32'))
nirnp = jim2np(jim[node.id], nir).astype(np.float)
rednp = jim2np(jim[node.id], red).astype(np.float)
ndvi = (nirnp - rednp) / (nirnp + rednp)
ndvi[np.isnan(ndvi)] = 0
jim[node.id].np()[:] = ndvi
jim[node.id].dimension['band'] = ['nd']
jim[node.id].dimension['temporal'] = nir.dimension['temporal']
return jim[node.id]
def processGraph(self,
agraph,
tileindex=None,
tiletotal=None,
virtual=False):
jim = {}
verbose = True
if verbose:
print("initialize")
print("agraph.nodes: {}".format(agraph.nodes))
#for node in agraph.nodes.values():
for node in agraph.nodes:
jim[node.id] = None
finished = False
if verbose:
print("starting process")
while not finished:
if verbose:
print("finished is: {}".format(finished))
finished = True
#for node in agraph.nodes.values():
for node in agraph.nodes:
print("processing node {}".format(node.id))
if jim[node.id] is not None:
if verbose:
print("skipping node {} that was already "
"calculated".format(node.id))
continue
else:
self.processNode(agraph, node.id, jim, tileindex,
tiletotal, virtual)
if jim[node.id] is not None:
if verbose:
print("calculated result for {}".format(node.id))
print("type of jim returned: {}".format(
type(jim[node.id])))
if isinstance(jim[node.id], Cube):
if verbose:
print("number of planes returned: {}".format(
jim[node.id].properties.nrOfPlane()))
print("number of bands returned: {}".format(
jim[node.id].properties.nrOfBand()))
elif isinstance(jim[node.id], pj.JimVect):
if verbose:
print("number of features calculated: {}".format(
jim[node.id].properties.getFeatureCount()))
elif isinstance(jim[node.id], Collection):
if verbose:
print("Node is collection not loaded in memory")
elif isinstance(jim[node.id], bool):
if verbose:
print("Node is intermediate result")
elif isinstance(jim[node.id], pj.Jim):
if verbose:
print("Node is a Jim, converting to Cube")
jim[node.id] = Cube(jim[node.id])
else:
raise TypeError(
"Error: result should either be Jim or "
"JimVect")
for ancestor in node.ancestors().nodes:
collectGarbage = self.gc
for descendant in ancestor.descendants().nodes:
if jim[descendant.id] is None:
collectGarbage = False
print(
"cannot collect garbage for ancestor node "
"{} yet, found descendant {}".format(
ancestor.id, descendant.id))
break
if collectGarbage and not isinstance(
jim[ancestor.id], bool):
print("collecting garbage for node {}".format(
ancestor.id))
jim[ancestor.id] = True
gc.collect()
else:
if verbose:
print("could not calculate result for node {}".format(
node.id))
continue
ntodo = 0
#for node in agraph.nodes.values():
for node in agraph.nodes:
if jim[node.id] is None:
if not ntodo:
if verbose:
print("nodes to do:")
if verbose:
print(node.id)
ntodo += 1
if ntodo:
finished = False
elif verbose:
print("All nodes processed")
def merge_cubes(agraph, nodeid, jim):
verbose = True
node = agraph[nodeid]
if verbose:
print(node)
cube1 = jim[node.content['arguments']['cube1'].get('from_node')]
cube2 = jim[node.content['arguments']['cube2'].get('from_node')]
if verbose:
print("cube1 is: {}".format(cube1))
print("cube2 is: {}".format(cube2))
if cube1 is None or cube2 is None:
jim[node.id] = None
return jim[node.id]
overlap_resolver = node.content['arguments'].get('overlap_resolver')
if overlap_resolver is not None:
overlap_resolver_node = agraph[overlap_resolver]
x = jim[overlap_resolver_node.content['arguments']['x']['from_node']]
y = jim[overlap_resolver_node.content['arguments']['y']['from_node']]
if x is None or y is None:
jim[node.id] = None
return jim[node.id]
else:
if cube1.properties.nrOfCol() == cube2.properties.nrOfCol():
if cube1.properties.nrOfRow() == cube2.properties.nrOfRow():
if cube1.getDimension('temporal') == cube1.getDimension(
'temporal'):
if cube1.properties.nrOfPlane(
) != cube2.properties.nrOfPlane():
raise ValueError(
'Error: mismatch in temporal dimension ')
bandname1 = cube1.getDimension('band')
bandname2 = cube2.getDimension('band')
temporalOverlap = not set(bandname1).isdisjoint(bandname2)
bandnames = [
band for band in bandname1 + bandname2
if band not in list(set(bandname1) & set(bandname2))
]
bandOverlap = not set(bandname1).isdisjoint(bandname2)
if bandOverlap:
if overlap_resolver is None:
raise ValueError(
'Error: no overlap resolver defined in merge_cube'
)
else:
#todo: reduce
raise ValueError(
'Error: band overlap not yet supported in merge_cube'
)
else:
jim[node.id] = Cube(pj.geometry.stackBand(
cube1, cube2))
jim[node.id].setDimension('band', bandnames)
elif cube1.getDimension('band') == cube1.getDimension('band'):
if cube1.properties.nrOfBand(
) != cube2.properties.nrOfBand():
raise ValueError('Error: mismatch in band dimension ')
dimension1 = cube1.getDimension('temporal')
dimension2 = cube2.getDimension('temporal')
temporalOverlap = not set(dimension1).isdisjoint(
dimension2)
dimension = [
dim for dim in dimension1 + dimension2
if dim not in list(set(dimension1) & set(dimension2))
]
if temporalOverlap:
if overlap_resolver is None:
raise ValueError(
'Error: no overlap resolver defined in merge_cube'
)
else:
#todo: reduce
raise ValueError(
'Error: temporal overlap not yet supported in merge_cube'
)
else:
jim[node.id] = Cube(
pj.geometry.stackPlane(cube1, cube2))
jim[node.id].setDimension('temporal', dimension)
else:
raise ValueError(
'Error: merge_cube not supported if number of rows do not match'
)
else:
raise ValueError(
'Error: merge_cube not supported if number of cols do not match'
)
return jim[node.id]
def reduce_dimension(agraph, nodeid, jim):
verbose = True
node = agraph[nodeid]
if verbose:
print(node)
print("reducing {}".format(node.content['arguments']['dimension']))
if jim[node.content['arguments']['data']['from_node']] is None:
jim[node.id] = None
return [node.id]
reducer_node = agraph[node.content['arguments']['reducer']['from_node']]
if verbose:
print("node is: {}".format(node.content))
print("reducer node is: {}".format(reducer_node))
if jim[reducer_node.id] is None:
reducer_parent = reducer_node.parent_process.content
print("parent of reducer_node: {}".format(reducer_parent))
reducer_data = jim[reducer_parent['arguments']['data']['from_node']]
# assert reducer_data.properties.isEqual(jim[node.content['arguments']['data']['from_node']]), \
# "Warning: data from parent from reducer and argument data from node of current node id are not equal"
if node.content['arguments']['dimension'] in ['temporal', 'time', 't']:
# cube=Cube(jim[reducer_node.content['arguments']['data']['from_node']])
# jim[reducer_node.id]=Cube(jim[reducer_node.content['arguments']['data']['from_node']])
jim[reducer_node.id] = Cube(reducer_data)
if jim[reducer_node.id] is None:
jim[node.id] = jim[reducer_node.id]
return [node.id]
rule = reducer_node.content['process_id']
if rule in ['max', 'mean', 'median', 'min']:
# jim[reducer_node.id] = pj.geometry.reducePlane(cube,rule=reducer_node.content['process_id'])
# jim[reducer_node.id]=Cube(jim[node.content['arguments']['data']['from_node']])
jim[reducer_node.id].geometry.reducePlane(rule=rule)
elif reducer_node.content['process_id'] == 'first':
jim[reducer_node.id].geometry.cropPlane(0)
elif reducer_node.content['process_id'] == 'last':
jim[reducer_node.id].geometry.cropPlane(-1)
jim[reducer_node.id].setDimension('temporal', [])
# jim[reducer_node.id].setDimension('band',jim[reducer_node.content['arguments']['data']['from_node']].getDimension('band'))
jim[reducer_node.id].setDimension(
'band', reducer_data.getDimension('band'))
elif node.content['arguments']['dimension'] in [
'spectral', 'bands', 'b'
]:
if reducer_node.content['process_id'] == 'first':
# jim[reducer_node.id]=Cube(reducer_node.content['arguments']['data']['from_node'])
jim[reducer_node.id] = Cube(reducer_data)
# cube=jim[reducer_node.content['arguments']['data']['from_node']]
if jim[reducer_node.id] is None:
jim[node.id] = None
return [node.id]
elif isinstance(cube, pj.JimVect):
raise TypeError(
"Error: reduce not implemented for JimVect")
elif isinstance(cube, Collection):
raise TypeError(
"Error: reduce not implemented for Collection")
jim[reducer_node.id].geometry.cropBand(0)
jim[reducer_node.id].setDimension(
'band', jim[reducer_node.id].getDimension('band')[0:1])
elif reducer_node.content['process_id'] == 'last':
# cube=jim[reducer_node.content['arguments']['data']['from_node']]
jim[reducer_node.id].geometry.cropBand(-1)
jim[reducer_node.id].setDimension(
'band', jim[reducer_node.id].getDimension('band')[-1:])
jim[node.id] = jim[reducer_node.id]
return jim[node.id]
def run_udf(agraph, nodeid, jim):
verbose = True
node = agraph[nodeid]
if node.parent_process is not None:
parent_node = node.parent_process.content
if parent_node['arguments']['data']['from_node'] not in jim:
print("cannot run udf yet")
jim[node.id] = None
return jim[node.id]
data = jim[parent_node['arguments']['data']['from_node']]
else:
data = jim[node.content['arguments']['data']['from_node']]
if data is None:
jim[node.id] = None
return jim[node.id]
else:
print("running udf")
#find function name
udfDefinition = node.content['arguments']['udf']
udf_name = udfDefinition[udfDefinition.index('def ') +
4:udfDefinition.index('(')].strip()
print("udf_name is {}".format(udf_name))
exec(udfDefinition)
sig = inspect.signature(eval(udf_name))
if len(sig.parameters) != 1:
raise TypeError("Error: udf definition must have single parameter")
for key in sig.parameters:
imgname = key
params = sig.parameters.keys()
if 'import' in udfDefinition:
raise TypeError(
"Error: No import allowed in server-side execution functions")
if 'os.path' in udfDefinition or 'os.system' in udfDefinition or 'os.popen' in udfDefinition:
raise TypeError(
"No os module functions allowed in server-side execution functions!"
)
if 'sys' in udfDefinition:
raise TypeError(
"No sys module functions allowed in server-side execution functions!"
)
if 'subprocess' in udfDefinition:
raise TypeError(
"No subprocess call allowed in server-side execution functions!"
)
if 'eval' in udfDefinition:
raise TypeError(
"No eval call allowed in server-side execution functions!")
if 'exec' in udfDefinition or 'execfile' in udfDefinition:
raise TypeError(
"No exec call allowed in server-side execution functions!")
if 'array' in imgname:
jim[node.id] = eval(udf_name)(data.np())
elif 'jim' in imgname:
jim[node.id] = eval(udf_name)(data)
else:
raise TypeError(
"Error: name of first parameter should either be jim (for pyjeo Jim) or array (for Numpy array)"
.format(type(jim[node.id])))
if not isinstance(jim[node.id], Cube) or not isinstance(
jim[node.id], pj.JimVect):
if isinstance(jim[node.id], pj.Jim):
jim[node.id] = Cube(jim[node.id])
jim[node.id].dimension = data.dimension
elif isinstance(jim[node.id], np.ndarray):
aCube = Cube(data)
aCube.np()[:] = jim[node.id]
jim[node.id] = aCube
else:
raise TypeError(
"Error: udf returns {}, must be of type Jim/Cube, numpy.ndarray, or JimVect"
.format(type(jim[node.id])))