def array_element(agraph, nodeid, jim): verbose = True node = agraph[nodeid] parent_node = node.parent_process.content data = jim[parent_node['arguments']['data']['from_node']] if 'index' in node.content['arguments']: bandindex = node.content['arguments']['index'] bandname = data.dimension['band'][bandindex] elif 'label' in node.content['arguments']: bandname = node.content['arguments']['label'] if verbose: print("array_element with label {}".format(bandname)) bandindex = data.dimension['band'].index(bandname) else: raise AttributeError( "Error: only index or label is supported for array_element") if verbose: print("array_element with index {}".format(bandindex)) if data is None: jim[node.id] = None elif isinstance(data, pj.Jim): result = Cube(pj.geometry.cropBand(data, bandindex)) result.dimension['band'] = [bandname] jim[node.id] = result elif isinstance(data, pj.JimVect): raise TypeError( "Error: {} array_element not implemented for JimVect".format( type(jim[node.id]))) elif isinstance(data, Collection): raise TypeError( "Error: {} array element not implemented for Collection".format( type(jim[node.id]))) return jim[node.id]
def filter_temporal(agraph, nodeid, jim): verbose = True node = agraph[nodeid] extent = node.content['arguments'].get('extent') data = jim[node.content['arguments']['data']['from_node']] if data is None: jim[node.id] = None return jim[node.id] if not isinstance(data, Cube): raise TypeError( "Error: filter_temporal only implemented for Cube, not {}".format( type(jim[node.content['arguments']['data']['from_node']]))) print("times in data: {}".format(data.getDimension('temporal'))) jim[node.id] = Cube(data) #todo: should be solved in __init__ of Cube jim[node.id].setDimension('temporal', data.getDimension('temporal')) jim[node.id].setDimension('band', data.getDimension('band')) jim[node.id].setResolution('temporal', data.getResolution('temporal')) jim[node.id].setResolution('spatial', data.getResolution('spatial')) if len(extent) == 0: raise ValueError( "extent should contain at least one date element, but got empty list: " + extent) dateFrom = extent[0] if len(extent) > 1: dateTo = extent[1] else: dateTo = None #todo: support datetime resolution finer than day if not isinstance(dateFrom, datetime): dateFrom = datetime.strptime(dateFrom, '%Y-%m-%d') if dateTo: if not isinstance(dateTo, datetime): dateTo = datetime.strptime(dateTo, '%Y-%m-%d') else: dateTo = dateFrom + jim[node.id].resolution['temporal'] print("dateFrom: {}".format(dateFrom)) print("dateTo: {}".format(dateTo)) print("times in cube: {}".format(jim[node.id].getDimension('temporal'))) filtered_temporal = [ d for d in jim[node.id].getDimension('temporal') if d >= dateFrom and d < dateTo ] print("filtered times: {}".format(filtered_temporal)) planeindices = [ jim[node.id].getDimension('temporal').index(d) for d in filtered_temporal ] if len(planeindices) < 1: raise ValueError("Error: filter temporal found no match") # planeindices=[self.dimension['temporal'].index(d) for d in self.dimension['temporal'] if d >= dateFrom and d < dateTo] jim[node.id] = Cube(pj.geometry.cropPlane(jim[node.id], plane=planeindices)) jim[node.id].setDimension('temporal', filtered_temporal) return jim[node.id]
def filter_bands(agraph, nodeid, jim): verbose = True node = agraph[nodeid] bandindexes = [] if 'bands' in node.content['arguments']: bandnames = node.content['arguments'].get('bands') for bandname in bandnames: bandindex = jim[node.content['arguments']['data'] ['from_node']].dimension['band'].index(bandname) if verbose: print("array_element with label {}".format(bandname)) print("array_element with index {}".format(bandindex)) bandindexes.append(bandindex) if jim[node.content['arguments']['data']['from_node']] is None: jim[node.id] = None elif isinstance(jim[node.content['arguments']['data']['from_node']], pj.Jim): jim[node.id] = Cube( pj.geometry.cropBand( jim[node.content['arguments']['data']['from_node']], bandindexes)) jim[node.id].dimension['band'] = bandnames return jim[node.id] elif isinstance(jim[node.content['arguments']['data']['from_node']], pj.JimVect): raise TypeError( "Error: {} array_element not implemented for JimVect".format( type(jim[node.id]))) elif isinstance(jim[node.content['arguments']['data']['from_node']], Collection): raise TypeError( "Error: {} array element not implemented for Collection".format( type(jim[node.id]))) else: raise AttributeError("Error: only bands is supported for filter_bands")
def apply_unary(agraph, nodeid, jim): verbose = True node = agraph[nodeid] if verbose: print(node) print("apply {}".format(node.content['description'])) if jim[node.content['arguments']['data']['from_node']] is None: jim[node.id] = None return [node.id] process_node = agraph[node.content['arguments']['process']['from_node']] if verbose: print("node is: {}".format(node.content)) print("process node is: {}".format(process_node)) if jim[process_node.id] is None: if process_node.content['process_id'] == 'linear_scale_range': inputMin = process_node.content['arguments'].get('inputMin', 0) inputMax = process_node.content['arguments'].get('inputMax') if inputMax is None: raise ValueError("Error: inputMax not set") outputMin = process_node.content['arguments'].get('outputMin', 0) outputMax = process_node.content['arguments'].get('outputMax', 255) #todo: handle data types if jim[process_node.content['arguments']['x'] ['from_node']] is None: jim[node.id] = None return [node.id] jim[process_node.id] = Cube( jim[process_node.content['arguments']['x']['from_node']]) jim[process_node.id] -= inputMin jim[process_node.id] /= (inputMax - inputMin) jim[process_node.id] *= (outputMax - outputMin) jim[process_node.id] += outputMin elif process_node.content['process_id'] == 'abs': if jim[process_node.content['arguments']['x'] ['from_node']] is None: jim[node.id] = None return [node.id] jim[process_node.id] = Cube( abs(jim[process_node.content['arguments']['x']['from_node']])) else: raise TypeError( "Error: {} not implemented for process apply".format( process_node.content['process_id'])) jim[node.id] = jim[process_node.id] return jim[node.id]
def save_result(agraph, nodeid, jim, pathname): verbose = True node = agraph[nodeid] if node.content['arguments']['data']['from_node'] not in jim: print("cannot save result yet") jim[node.id] = None return jim[node.id] if jim[node.content['arguments']['data']['from_node']]: print("saving result") jim[node.id] = jim[node.content['arguments']['data']['from_node']] if isinstance(jim[node.id], pj.Jim): #to save as multi-spectral GeoTIFF, 1 file per acquisition time print("jim has {} planes".format( jim[node.id].properties.nrOfPlane())) plane2band = False if jim[node.id].properties.nrOfPlane() > 1: if jim[node.id].properties.nrOfBand() > 1: for iplane, theDate in enumerate( jim[node.id].dimension['temporal']): print("cropPlane {}".format(iplane)) jimplane = Cube( pj.geometry.cropPlane(jim[node.id], iplane)) jimplane.properties.setNoDataVals(0) jimplane.io.write(os.path.join( pathname, theDate.strftime('%Y%m%d') + '.tif'), co=['COMPRESS=LZW', 'TILED=YES']) return jim[node.id] else: plane2band = True jim[node.id].geometry.plane2band() jim[node.id].io.write(pathname + '.tif', co=['COMPRESS=LZW', 'TILED=YES']) if plane2band: jim[node.id].geometry.band2plane() return jim[node.id] elif isinstance(jim[node.id], pj.JimVect): # print("saved result: {}".format(jim[node.id].np())) jim[node.id].io.write(pathname + '.sqlite') elif isinstance(jim[node.id], Collection): raise TypeError( "Error: {} virtual cube not implemented yet".format( type(jim[node.id]))) else: raise TypeError("Error: {} type not supported for writing".format( type(jim[node.id]))) else: print("cannot save result yet") jim[node.id] = None return jim[node.id]
def apply_binary(agraph, nodeid, jim): verbose = True node = agraph[nodeid] if verbose: print(node) print("eq {}".format(node.content.get('description'))) jim[node.id] = None # arguments = [item for sublist in node.content['arguments'].values() for item in sublist] arguments = node.content['arguments'] x = arguments['x'] y = arguments['y'] if isinstance(x, dict): if verbose: print("type of jim is {}".format(type(jim[x['from_node']]))) if jim[x['from_node']] is None: jim[node.id] = None return jim[node.id] else: jim[node.id] = jim[x['from_node']] else: jim[node.id] = x value = 0 if isinstance(y, dict): if verbose: print("type of jim is {}".format(type(jim[y['from_node']]))) if jim[y['from_node']] is None: jim[node.id] = None return jim[node.id] else: value = jim[y['from_node']] else: value = y if node.content['process_id'] == 'add': jim[node.id] += value elif node.content['process_id'] == 'divide': jim[node.id] /= value elif node.content['process_id'] == 'eq': jim[node.id] = Cube(jim[node.id] == value) elif node.content['process_id'] == 'gt': jim[node.id] = Cube(jim[node.id] > value) elif node.content['process_id'] == 'gte': jim[node.id] = Cube(jim[node.id] >= value) elif node.content['process_id'] == 'lt': jim[node.id] = Cube(jim[node.id] < value) elif node.content['process_id'] == 'lte': jim[node.id] = Cube(jim[node.id] <= value) elif node.content['process_id'] == 'multiply': jim[node.id] *= value elif node.content['process_id'] == 'neq': jim[node.id] = Cube(jim[node.id] != value) elif node.content['process_id'] == 'subtract': jim[node.id] -= value else: raise TypeError("Error: arithmetic {} not implemented".format( node.content['process_id'])) return jim[node.id]
def aggregate_temporal(agraph, nodeid, jim): verbose = True node = agraph[nodeid] #todo: not tested yet in openeo API v1.0 if node.content['arguments']['dimension'] == 'temporal': if jim[node.content['arguments']['data']['from_node']] is None: jim[node.id] = None return [node.id] reducer_node = agraph[node.content['arguments']['reducer'] ['from_node']] if verbose: print("node is: {}".format(node.content)) print("reducer node is: {}".format(reducer_node)) if jim[reducer_node.id] is None: rule = reducer_node.content['process_id'] if verbose: print("reducer graph is: {}".format(reducer_node)) print("rule: {}".format(rule)) if isinstance(jim[node.content['arguments']['data']['from_node']], pj.Jim): jim[reducer_node.id] = Cube( jim[node.content['arguments']['data']['from_node']]) jim[reducer_node.id].geometry.reducePlane(rule) if jim[reducer_node.id] is not None: jim[node.id] = jim[reducer_node.id] return jim[node.id] else: raise ValueError("Error: jim_reduced is False") elif isinstance( jim[node.content['arguments']['data']['from_node']], pj.JimVect): raise TypeError("Error: reduce not implemented for JimVect") elif isinstance( jim[node.content['arguments']['data']['from_node']], Collection): raise TypeError("Error: reduce not implemented for Collection") else: #test jim[node.id] = jim[reducer_node.id] #jim[node.id]=jim[node.content['arguments']['data']['from_node']] return jim[node.id] else: raise TypeError( "Error: reduce not implemented for dimension different than temporal" )
def normalized_difference(agraph, nodeid, jim): verbose = True node = agraph[nodeid] nir = jim[node.content['arguments']['x']['from_node']] if nir is None: jim[node.id] = None return [node.id] red = jim[node.content['arguments']['y']['from_node']] if nir is None: jim[node.id] = None return [node.id] jim[node.id] = Cube(pj.pixops.convert(nir, 'GDT_Float32')) nirnp = nir.np().astype(np.float) rednp = red.np().astype(np.float) ndvi = (nirnp - rednp) / (nirnp + rednp) ndvi[np.isnan(ndvi)] = 0 jim[node.id].np()[:] = ndvi jim[node.id].dimension['band'] = ['nd'] jim[node.id].dimension['temporal'] = nir.dimension['temporal'] return jim[node.id]
def ndvi(agraph, nodeid, jim): verbose = True node = agraph[nodeid] data = jim[node.content['arguments']['data']['from_node']] nir = node.content['arguments']['nir'] red = node.content['arguments']['red'] if nir is None: jim[node.id] = None return [node.id] red = jim[node.content['arguments']['y']['from_node']] if nir is None: jim[node.id] = None return [node.id] jim[node.id] = Cube(pj.pixops.convert(nir, 'GDT_Float32')) nirnp = jim2np(jim[node.id], nir).astype(np.float) rednp = jim2np(jim[node.id], red).astype(np.float) ndvi = (nirnp - rednp) / (nirnp + rednp) ndvi[np.isnan(ndvi)] = 0 jim[node.id].np()[:] = ndvi jim[node.id].dimension['band'] = ['nd'] jim[node.id].dimension['temporal'] = nir.dimension['temporal'] return jim[node.id]
def processGraph(self, agraph, tileindex=None, tiletotal=None, virtual=False): jim = {} verbose = True if verbose: print("initialize") print("agraph.nodes: {}".format(agraph.nodes)) #for node in agraph.nodes.values(): for node in agraph.nodes: jim[node.id] = None finished = False if verbose: print("starting process") while not finished: if verbose: print("finished is: {}".format(finished)) finished = True #for node in agraph.nodes.values(): for node in agraph.nodes: print("processing node {}".format(node.id)) if jim[node.id] is not None: if verbose: print("skipping node {} that was already " "calculated".format(node.id)) continue else: self.processNode(agraph, node.id, jim, tileindex, tiletotal, virtual) if jim[node.id] is not None: if verbose: print("calculated result for {}".format(node.id)) print("type of jim returned: {}".format( type(jim[node.id]))) if isinstance(jim[node.id], Cube): if verbose: print("number of planes returned: {}".format( jim[node.id].properties.nrOfPlane())) print("number of bands returned: {}".format( jim[node.id].properties.nrOfBand())) elif isinstance(jim[node.id], pj.JimVect): if verbose: print("number of features calculated: {}".format( jim[node.id].properties.getFeatureCount())) elif isinstance(jim[node.id], Collection): if verbose: print("Node is collection not loaded in memory") elif isinstance(jim[node.id], bool): if verbose: print("Node is intermediate result") elif isinstance(jim[node.id], pj.Jim): if verbose: print("Node is a Jim, converting to Cube") jim[node.id] = Cube(jim[node.id]) else: raise TypeError( "Error: result should either be Jim or " "JimVect") for ancestor in node.ancestors().nodes: collectGarbage = self.gc for descendant in ancestor.descendants().nodes: if jim[descendant.id] is None: collectGarbage = False print( "cannot collect garbage for ancestor node " "{} yet, found descendant {}".format( ancestor.id, descendant.id)) break if collectGarbage and not isinstance( jim[ancestor.id], bool): print("collecting garbage for node {}".format( ancestor.id)) jim[ancestor.id] = True gc.collect() else: if verbose: print("could not calculate result for node {}".format( node.id)) continue ntodo = 0 #for node in agraph.nodes.values(): for node in agraph.nodes: if jim[node.id] is None: if not ntodo: if verbose: print("nodes to do:") if verbose: print(node.id) ntodo += 1 if ntodo: finished = False elif verbose: print("All nodes processed")
def merge_cubes(agraph, nodeid, jim): verbose = True node = agraph[nodeid] if verbose: print(node) cube1 = jim[node.content['arguments']['cube1'].get('from_node')] cube2 = jim[node.content['arguments']['cube2'].get('from_node')] if verbose: print("cube1 is: {}".format(cube1)) print("cube2 is: {}".format(cube2)) if cube1 is None or cube2 is None: jim[node.id] = None return jim[node.id] overlap_resolver = node.content['arguments'].get('overlap_resolver') if overlap_resolver is not None: overlap_resolver_node = agraph[overlap_resolver] x = jim[overlap_resolver_node.content['arguments']['x']['from_node']] y = jim[overlap_resolver_node.content['arguments']['y']['from_node']] if x is None or y is None: jim[node.id] = None return jim[node.id] else: if cube1.properties.nrOfCol() == cube2.properties.nrOfCol(): if cube1.properties.nrOfRow() == cube2.properties.nrOfRow(): if cube1.getDimension('temporal') == cube1.getDimension( 'temporal'): if cube1.properties.nrOfPlane( ) != cube2.properties.nrOfPlane(): raise ValueError( 'Error: mismatch in temporal dimension ') bandname1 = cube1.getDimension('band') bandname2 = cube2.getDimension('band') temporalOverlap = not set(bandname1).isdisjoint(bandname2) bandnames = [ band for band in bandname1 + bandname2 if band not in list(set(bandname1) & set(bandname2)) ] bandOverlap = not set(bandname1).isdisjoint(bandname2) if bandOverlap: if overlap_resolver is None: raise ValueError( 'Error: no overlap resolver defined in merge_cube' ) else: #todo: reduce raise ValueError( 'Error: band overlap not yet supported in merge_cube' ) else: jim[node.id] = Cube(pj.geometry.stackBand( cube1, cube2)) jim[node.id].setDimension('band', bandnames) elif cube1.getDimension('band') == cube1.getDimension('band'): if cube1.properties.nrOfBand( ) != cube2.properties.nrOfBand(): raise ValueError('Error: mismatch in band dimension ') dimension1 = cube1.getDimension('temporal') dimension2 = cube2.getDimension('temporal') temporalOverlap = not set(dimension1).isdisjoint( dimension2) dimension = [ dim for dim in dimension1 + dimension2 if dim not in list(set(dimension1) & set(dimension2)) ] if temporalOverlap: if overlap_resolver is None: raise ValueError( 'Error: no overlap resolver defined in merge_cube' ) else: #todo: reduce raise ValueError( 'Error: temporal overlap not yet supported in merge_cube' ) else: jim[node.id] = Cube( pj.geometry.stackPlane(cube1, cube2)) jim[node.id].setDimension('temporal', dimension) else: raise ValueError( 'Error: merge_cube not supported if number of rows do not match' ) else: raise ValueError( 'Error: merge_cube not supported if number of cols do not match' ) return jim[node.id]
def reduce_dimension(agraph, nodeid, jim): verbose = True node = agraph[nodeid] if verbose: print(node) print("reducing {}".format(node.content['arguments']['dimension'])) if jim[node.content['arguments']['data']['from_node']] is None: jim[node.id] = None return [node.id] reducer_node = agraph[node.content['arguments']['reducer']['from_node']] if verbose: print("node is: {}".format(node.content)) print("reducer node is: {}".format(reducer_node)) if jim[reducer_node.id] is None: reducer_parent = reducer_node.parent_process.content print("parent of reducer_node: {}".format(reducer_parent)) reducer_data = jim[reducer_parent['arguments']['data']['from_node']] # assert reducer_data.properties.isEqual(jim[node.content['arguments']['data']['from_node']]), \ # "Warning: data from parent from reducer and argument data from node of current node id are not equal" if node.content['arguments']['dimension'] in ['temporal', 'time', 't']: # cube=Cube(jim[reducer_node.content['arguments']['data']['from_node']]) # jim[reducer_node.id]=Cube(jim[reducer_node.content['arguments']['data']['from_node']]) jim[reducer_node.id] = Cube(reducer_data) if jim[reducer_node.id] is None: jim[node.id] = jim[reducer_node.id] return [node.id] rule = reducer_node.content['process_id'] if rule in ['max', 'mean', 'median', 'min']: # jim[reducer_node.id] = pj.geometry.reducePlane(cube,rule=reducer_node.content['process_id']) # jim[reducer_node.id]=Cube(jim[node.content['arguments']['data']['from_node']]) jim[reducer_node.id].geometry.reducePlane(rule=rule) elif reducer_node.content['process_id'] == 'first': jim[reducer_node.id].geometry.cropPlane(0) elif reducer_node.content['process_id'] == 'last': jim[reducer_node.id].geometry.cropPlane(-1) jim[reducer_node.id].setDimension('temporal', []) # jim[reducer_node.id].setDimension('band',jim[reducer_node.content['arguments']['data']['from_node']].getDimension('band')) jim[reducer_node.id].setDimension( 'band', reducer_data.getDimension('band')) elif node.content['arguments']['dimension'] in [ 'spectral', 'bands', 'b' ]: if reducer_node.content['process_id'] == 'first': # jim[reducer_node.id]=Cube(reducer_node.content['arguments']['data']['from_node']) jim[reducer_node.id] = Cube(reducer_data) # cube=jim[reducer_node.content['arguments']['data']['from_node']] if jim[reducer_node.id] is None: jim[node.id] = None return [node.id] elif isinstance(cube, pj.JimVect): raise TypeError( "Error: reduce not implemented for JimVect") elif isinstance(cube, Collection): raise TypeError( "Error: reduce not implemented for Collection") jim[reducer_node.id].geometry.cropBand(0) jim[reducer_node.id].setDimension( 'band', jim[reducer_node.id].getDimension('band')[0:1]) elif reducer_node.content['process_id'] == 'last': # cube=jim[reducer_node.content['arguments']['data']['from_node']] jim[reducer_node.id].geometry.cropBand(-1) jim[reducer_node.id].setDimension( 'band', jim[reducer_node.id].getDimension('band')[-1:]) jim[node.id] = jim[reducer_node.id] return jim[node.id]
def run_udf(agraph, nodeid, jim): verbose = True node = agraph[nodeid] if node.parent_process is not None: parent_node = node.parent_process.content if parent_node['arguments']['data']['from_node'] not in jim: print("cannot run udf yet") jim[node.id] = None return jim[node.id] data = jim[parent_node['arguments']['data']['from_node']] else: data = jim[node.content['arguments']['data']['from_node']] if data is None: jim[node.id] = None return jim[node.id] else: print("running udf") #find function name udfDefinition = node.content['arguments']['udf'] udf_name = udfDefinition[udfDefinition.index('def ') + 4:udfDefinition.index('(')].strip() print("udf_name is {}".format(udf_name)) exec(udfDefinition) sig = inspect.signature(eval(udf_name)) if len(sig.parameters) != 1: raise TypeError("Error: udf definition must have single parameter") for key in sig.parameters: imgname = key params = sig.parameters.keys() if 'import' in udfDefinition: raise TypeError( "Error: No import allowed in server-side execution functions") if 'os.path' in udfDefinition or 'os.system' in udfDefinition or 'os.popen' in udfDefinition: raise TypeError( "No os module functions allowed in server-side execution functions!" ) if 'sys' in udfDefinition: raise TypeError( "No sys module functions allowed in server-side execution functions!" ) if 'subprocess' in udfDefinition: raise TypeError( "No subprocess call allowed in server-side execution functions!" ) if 'eval' in udfDefinition: raise TypeError( "No eval call allowed in server-side execution functions!") if 'exec' in udfDefinition or 'execfile' in udfDefinition: raise TypeError( "No exec call allowed in server-side execution functions!") if 'array' in imgname: jim[node.id] = eval(udf_name)(data.np()) elif 'jim' in imgname: jim[node.id] = eval(udf_name)(data) else: raise TypeError( "Error: name of first parameter should either be jim (for pyjeo Jim) or array (for Numpy array)" .format(type(jim[node.id]))) if not isinstance(jim[node.id], Cube) or not isinstance( jim[node.id], pj.JimVect): if isinstance(jim[node.id], pj.Jim): jim[node.id] = Cube(jim[node.id]) jim[node.id].dimension = data.dimension elif isinstance(jim[node.id], np.ndarray): aCube = Cube(data) aCube.np()[:] = jim[node.id] jim[node.id] = aCube else: raise TypeError( "Error: udf returns {}, must be of type Jim/Cube, numpy.ndarray, or JimVect" .format(type(jim[node.id])))