def buildLoop(preProcessChain):
webhookUrl = APP.url + "/resources/processes/operations/update"
log.debug("Registering own webhook endpoint: " + str(webhookUrl))
tpl = tplEnv.get_template('actiniaCore/pc_loop.json')
class PCInputClass():
param = ''
value = ''
pcInputs = []
# Feature: transform input table information to grass connection string
# try:
# preProcessChainTableFeat = preProcessChain.feature_source.table
# except AttributeError as e:
# log.error(e)
# log.error("Feature has no data source")
# return
#
# if preProcessChainTableFeat is None:
# log.error("Feature has no data source")
# return
#
# connString = buildPCConnectionString(preProcessChainTableFeat)
# feat_db = connString[0]
# feat_layer = connString[1]
proc = preProcessChain.procs[0]
for input in proc.input:
inputType = getattr(input, "type", None)
if inputType == 'PARAMETER':
pcInputObject = PCInputClass()
pcInputObject.param = input.name
path = ACTINIACORE.filestorage + '/' + input.value[0]
pcInputObject.value = path
pcInputs.append(pcInputObject)
elif input.table:
if input.name in ['a', 'b', 'c']:
pcInputObject = PCInputClass()
pcInputObject.param = input.name
pcInputObject.value = input.value
pcInputs.append(pcInputObject)
else:
log.error("Don't know what to do with input.")
if (pcInputs is None or webhookUrl is None):
log.error('Could not set all variables to replace in template.')
return None
postbody = tpl.render(inputs=pcInputs,
webhookUrl=webhookUrl).replace('\n', '')
return postbody
def getRecordsByTags(tags):
""" Method to get records by tags from geonetwork
Attention: The tags received are case sensitive!
PropertyIsLike is sensitive, matchCase not possible here.
See https://trac.osgeo.org/geonetwork/wiki/CSW202Improvements
This method can be called by @app.route('/metadata/raw/tags/<tags>')
"""
url = GEONETWORK.csw_url
tpl = tplEnv.get_template('geonetwork/post_records_by_tags.xml')
tags = tags.split(',')
postbody = tpl.render(tags=tags).replace('\n', '')
headers = {'content-type': 'application/xml; charset=utf-8'}
try:
gnosresp = requests.post(
url,
data=postbody,
headers=headers,
auth=auth(GEONETWORK)
)
except requests.exceptions.ConnectionError:
return None
try:
parsedresp = xmltodict.parse(gnosresp.content)
records = json.dumps(parsedresp)
return records
except Exception:
return None
def buildPCS1Grd(preProcessChain):
processType = preProcessChain.get('processType')
if processType != 'preprocessing':
log.error('process type is unknown')
return None
webhookUrl = APP.url + "/resources/processes/operations/update"
log.debug("Registering own webhook endpoint: " + str(webhookUrl))
tpl = tplEnv.get_template('actiniaCore/pc_r.s1.grd_template.json')
user = ACTINIACORE.esa_apihub_user
pw = ACTINIACORE.esa_apihub_pw
S1A_name = preProcessChain.get('title')
raw_path = ACTINIACORE.filestorage + '/' + 'sentinel1/raw/'
preprocessing_path = (ACTINIACORE.filestorage + '/' +
'sentinel1/preprocessing/')
if (user is None or pw is None or S1A_name is None or raw_path is None
or preprocessing_path is None or webhookUrl is None):
log.error('Could not set all variables to replace in template.')
return None
postbody = tpl.render(user=user,
pw=pw,
S1A_name=S1A_name,
raw_path=raw_path,
preprocessing_path=preprocessing_path,
webhookUrl=webhookUrl).replace('\n', '')
return postbody
def getRecordsByTags(tags):
""" Method to get records by tags from geonetwork
Attention: The tags received are case sensitive!
PropertyIsLike is sensitive, matchCase not possible here.
See https://trac.osgeo.org/geonetwork/wiki/CSW202Improvements
This method can be called by @app.route('/metadata/raw/tags/<tags>')
"""
log.debug('looking for tags ' + str(tags))
try:
url = GEONETWORK.csw_url
tpl = tplEnv.get_template('geonetwork/post_records_by_tags.xml')
tags = tags.split(',')
postbody = tpl.render(tags=tags).replace('\n', '')
headers = {'content-type': 'application/xml; charset=utf-8'}
except Exception as e:
log.error('Could not set needed variable')
log.error(e)
return None
try:
gnosresp = requests.post(
url,
data=postbody,
headers=headers,
auth=auth(GEONETWORK)
)
return gnosresp.content
except requests.exceptions.ConnectionError:
log.error('Could not connect to gnos')
return None
def create(filename):
""" Method to get create metadata records in geonetwork
"""
url = GEONETWORK.csw_publication
recordtpl = tplEnv.get_template('geonetwork/template_metadaten.xml')
record = recordtpl.render(title=filename).replace('\n', '')
recordfs = recordtpl.render(title=filename)
postbodytpl = tplEnv.get_template('geonetwork/post_create_record.xml')
postbody = postbodytpl.render(metadata_record=record).replace('\n', '')
postbodyfs = postbodytpl.render(metadata_record=recordfs)
headers = {'content-type': 'application/xml; charset=utf-8'}
log.info('Creating metadata record')
try:
gnosresp = requests.post(url,
data=postbody,
headers=headers,
auth=auth(GEONETWORK))
except requests.exceptions.ConnectionError:
return None
if not os.path.isdir(FILEUPLOAD.templates):
os.makedirs(FILEUPLOAD.templates)
with open(FILEUPLOAD.templates + '/' + filename + '_template.xml',
'x') as file:
file.write(postbodyfs)
log.info('Received binary file, saved to ' + FILEUPLOAD.templates + '/' +
filename + '_template.xml')
try:
parsedresp = xmltodict.parse(gnosresp.content)
insertRes = parsedresp['csw:TransactionResponse']['csw:InsertResult']
uuid = insertRes['csw:BriefRecord']['identifier']
log.info('GNOS response is: ' + str(parsedresp))
return uuid
except Exception:
return None
def update(uuid, utcnow):
""" Method to update record in geonetwork
"""
connection = checkConnectionWithoutResponse('geonetwork')
if connection is None:
log.error('Not updating metadata for uuid ' + uuid)
return None
try:
response = getRecordByUUID(uuid)
doc = parseString(response.decode('utf-8'))
recordNode = doc.getElementsByTagName('gmd:MD_Metadata')[0]
log.debug('Found metadata to update for ' + uuid + ', and ' +
str(recordNode))
except Exception:
log.error('Could not find metadata record to update for uuid ' + uuid)
return None
record = updateXml(response, utcnow)
if record is None:
return None
try:
url = GEONETWORK.csw_pub
postbodytpl = tplEnv.get_template('geonetwork/post_update_record.xml')
postbody = postbodytpl.render(metadata_record=record,
uuid=uuid).replace('\n', '')
headers = {'content-type': 'application/xml; charset=utf-8'}
except Exception as e:
log.error('Could not set needed variable')
log.error(e)
return None
try:
log.info('Updating metadata record')
gnosresp = requests.post(url,
data=bytes(postbody, 'utf-8'),
headers=headers,
auth=auth(GEONETWORK))
if '<html>' in gnosresp.content.decode('utf-8'):
log.error('update error')
else:
log.info('update success')
return gnosresp
except requests.exceptions.ConnectionError:
log.error('Could not connect to gnos')
return None
except Exception as e:
log.error(e)
return None
def buildPCDummy(preProcessChain):
webhookUrl = APP.url + "/resources/processes/operations/update"
log.debug("Registering own webhook endpoint: " + str(webhookUrl))
tpl = tplEnv.get_template('actiniaCore/pc_point_in_polygon.json')
point = preProcessChain.get('point')
polygon = preProcessChain.get('polygon')
if polygon is None or point is None or webhookUrl is None:
log.error('Could not set all variables to replace in template.')
return None
postbody = tpl.render(point=point, polygon=polygon,
webhookUrl=webhookUrl).replace('\n', '')
return postbody
def getRecordByUUID(uuid):
""" Method to get record by uuid from geonetwork
This method can be called by @app.route('/metadata/raw/uuids/<uuid>')
"""
url = GEONETWORK.csw_url
tpl = tplEnv.get_template('geonetwork/get_record_by_uuid_kvp.json')
kvps = json.loads(tpl.render(uuid=uuid))
try:
gnosresp = requests.get(url, params=kvps, auth=auth(GEONETWORK))
except requests.exceptions.ConnectionError:
return None
try:
parsedresp = xmltodict.parse(gnosresp.content)
records = json.dumps(parsedresp)
return records
except Exception:
return None
def getRecordByUUID(uuid):
""" Method to get record by uuid from geonetwork
This method can be called by @app.route('/metadata/raw/uuids/<uuid>')
"""
try:
url = GEONETWORK.csw_url
tpl = tplEnv.get_template('geonetwork/get_record_by_uuid_kvp.json')
kvps = json.loads(tpl.render(uuid=uuid))
except Exception as e:
log.error('Could not set needed variable')
log.error(e)
return None
try:
gnosresp = requests.get(url, params=kvps, auth=auth(GEONETWORK))
return gnosresp.content
except requests.exceptions.ConnectionError:
log.error('Could not connect to gnos')
return None
def getRecordsByCategory(category):
""" Method to get records by category from geonetwork
This method can be called by
@app.route('/metadata/raw/categories/<category>')
"""
url = (GEONETWORK.csw_url + '-' + category)
tpl = tplEnv.get_template('geonetwork/get_records_by_category_kvp.json')
kvps = json.loads(tpl.render())
try:
gnosresp = requests.get(url, params=kvps, auth=auth(GEONETWORK))
except requests.exceptions.ConnectionError:
return None
try:
parsedresp = xmltodict.parse(gnosresp.content)
records = json.dumps(parsedresp)
return records
except Exception:
return None
def getRecordsByCategory(category):
""" Method to get records by category from geonetwork
This method can be called by
@app.route('/metadata/raw/categories/<category>')
"""
try:
url = (GEONETWORK.csw_url + '-' + category)
tpl = tplEnv.get_template('geonetwork/get_records_by_category_kvp.json')
kvps = json.loads(tpl.render())
except Exception as e:
log.error('Could not set needed variable')
log.error(e)
return None
try:
gnosresp = requests.get(url, params=kvps, auth=auth(GEONETWORK))
return gnosresp.content
except requests.exceptions.ConnectionError:
log.error('Could not connect to gnos')
return None
def ParseInterfaceDescription(xml_string, keys=None):
"""Parses output of GRASS interface-description
and returns openEO process object
"""
gm_dict = xmltodict.parse(xml_string)['task']
module_id = gm_dict['@name']
description = gm_dict['description']
categories = gm_dict['keywords'].replace(' ', '').split(',')
categories.append('grass-module')
parameters = {}
returns = {}
extrakwargs = dict()
try:
grass_params = gm_dict['parameter']
except KeyError:
logstring(module_id, "", "has no parameter")
grass_params = []
try:
flags = gm_dict['flag']
except KeyError:
logstring(module_id, "", "has no flags")
flags = []
for parameter in grass_params:
kwargs = dict()
schema_kwargs = dict()
if keys:
# case for actinia modules
key = setVirtualParameterKey(module_id, parameter)
if key not in keys:
continue
else:
# case for GRASS modules
key = setParameterKey(module_id, parameter)
schema_kwargs = setParamType(module_id, key, parameter, schema_kwargs)
kwargs = setParameterDescription(module_id, key, parameter, kwargs)
kwargs = setParameterRequired(parameter, kwargs)
schema_kwargs = setParameterDefault(parameter, schema_kwargs)
schema_kwargs = setParameterEnum(parameter, schema_kwargs)
param_object = ModuleParameter(**kwargs,
schema=ModuleParameterSchema(
**schema_kwargs))
if isOutput(parameter):
returns[key] = param_object
else:
parameters[key] = param_object
del kwargs
del schema_kwargs
for parameter in flags:
# not possible to specify flag values via template at the moment
if keys:
continue
kwargs = dict()
schema_kwargs = dict()
schema_kwargs['type'] = 'boolean'
schema_kwargs['default'] = 'False'
key = setParameterKey(module_id, parameter)
kwargs = setParameterDescription(module_id, key, parameter, kwargs)
kwargs = setParameterRequired(parameter, kwargs)
param_object = ModuleParameter(**kwargs,
schema=ModuleParameterSchema(
**schema_kwargs))
parameters[key] = param_object
del kwargs
del schema_kwargs
# custom extention for importer + exporter from actinia_core
try:
tpl = tplEnv.get_template('gmodules/' + module_id + '.json')
pc_template = json.loads(tpl.render().replace('\n', ''))
for key in [*pc_template]:
extrakwargs[key] = {}
for param in pc_template[key]:
extrakwargs[key][param] = ModuleParameter(
**pc_template[key][param])
except Exception as e:
# if no template for module exist, use as is (default)
log.debug('template %s does not exist.', e)
grass_module = Module(id=module_id,
description=description,
categories=sorted(categories),
parameters=parameters,
returns=returns,
**extrakwargs)
return grass_module