def checkConnection(name):
""" Method to test connection
Args:
name (string): resource to test. Can be 'actinia-core' or 'geonetwork'
Returns:
response (Response): of type SimpleResponseModel telling
connection success or failure
"""
if name == 'actinia-core':
url = ACTINIACORE.url + "version"
name = 'actinia-core'
type = 'json'
elif name == 'geonetwork':
url = GEONETWORK.csw_url
name = 'geonetwork'
type = 'xml'
try:
records = common.checkConnection(url, name, type)
except Exception:
log.error("Don't know which connection to test")
if records is not None:
res = jsonify(SimpleResponseModel(status=200, message="success"))
return make_response(res, 200)
elif records is None:
res = jsonify(SimpleResponseModel(status=404, message="failure"))
return make_response(res, 200)
def makeItXml(jsonDict):
try:
records = xmltodict.unparse(jsonDict)
return records
except Exception:
log.error('Error converting json back to xml')
return None
def buildLoop(preProcessChain):
webhookUrl = APP.url + "/resources/processes/operations/update"
log.debug("Registering own webhook endpoint: " + str(webhookUrl))
tpl = tplEnv.get_template('actiniaCore/pc_loop.json')
class PCInputClass():
param = ''
value = ''
pcInputs = []
# Feature: transform input table information to grass connection string
# try:
# preProcessChainTableFeat = preProcessChain.feature_source.table
# except AttributeError as e:
# log.error(e)
# log.error("Feature has no data source")
# return
#
# if preProcessChainTableFeat is None:
# log.error("Feature has no data source")
# return
#
# connString = buildPCConnectionString(preProcessChainTableFeat)
# feat_db = connString[0]
# feat_layer = connString[1]
proc = preProcessChain.procs[0]
for input in proc.input:
inputType = getattr(input, "type", None)
if inputType == 'PARAMETER':
pcInputObject = PCInputClass()
pcInputObject.param = input.name
path = ACTINIACORE.filestorage + '/' + input.value[0]
pcInputObject.value = path
pcInputs.append(pcInputObject)
elif input.table:
if input.name in ['a', 'b', 'c']:
pcInputObject = PCInputClass()
pcInputObject.param = input.name
pcInputObject.value = input.value
pcInputs.append(pcInputObject)
else:
log.error("Don't know what to do with input.")
if (pcInputs is None or webhookUrl is None):
log.error('Could not set all variables to replace in template.')
return None
postbody = tpl.render(inputs=pcInputs,
webhookUrl=webhookUrl).replace('\n', '')
return postbody
def checkConnection(url, name, expectedFormat):
""" Method to test connection
Args:
url (string): url of resource to test.
name (string): name of resource to test. Only used for logging
expectedFormat (string): Format in which resource will respond. Can be
'xml' or 'json'
"""
# can be called by e.g.
# checkConnection(GEONETWORK.csw_url, 'geonetwork', 'xml')
log.debug('Testing connection to ' + url)
try:
resp = requests.get(url)
except requests.exceptions.ConnectionError:
log.error('Connection Error to ' + name)
return None
try:
if expectedFormat == 'json':
parsedresp = json.loads(resp.text)
log.debug('Connection successfull to ' + name)
return True
except Exception:
log.error('Connection Error to ' + name)
return None
def buildPCS1Grd(preProcessChain):
processType = preProcessChain.get('processType')
if processType != 'preprocessing':
log.error('process type is unknown')
return None
webhookUrl = APP.url + "/resources/processes/operations/update"
log.debug("Registering own webhook endpoint: " + str(webhookUrl))
tpl = tplEnv.get_template('actiniaCore/pc_r.s1.grd_template.json')
user = ACTINIACORE.esa_apihub_user
pw = ACTINIACORE.esa_apihub_pw
S1A_name = preProcessChain.get('title')
raw_path = ACTINIACORE.filestorage + '/' + 'sentinel1/raw/'
preprocessing_path = (ACTINIACORE.filestorage + '/' +
'sentinel1/preprocessing/')
if (user is None or pw is None or S1A_name is None or raw_path is None
or preprocessing_path is None or webhookUrl is None):
log.error('Could not set all variables to replace in template.')
return None
postbody = tpl.render(user=user,
pw=pw,
S1A_name=S1A_name,
raw_path=raw_path,
preprocessing_path=preprocessing_path,
webhookUrl=webhookUrl).replace('\n', '')
return postbody
def createProcessChainTemplateList():
'''
list all stored templates and return as actinia-module list
'''
pc_list = []
tpl_list = pcTplEnv.list_templates(filter_func=filter_func)
for tpl_string in tpl_list:
tpl = pcTplEnv.get_template(tpl_string)
try:
pc_template = json.loads(tpl.render().replace('\n', ''))
except:
log.error('Error parsing template ' + tpl_string)
tpl_id = pc_template['id']
description = pc_template['description']
categories = ['actinia-module']
pc_response = (Module(id=tpl_id,
description=description,
categories=categories))
pc_list.append(pc_response)
return pc_list
def getAllJobs(filters, process):
""" Method to read all jobs from jobtabelle with filter
Args: filters (ImmutableMultiDict): the args from the HTTP call
Returns:
jobs (list): the records matching the filter
"""
log.debug('Received query for jobs')
if process == 'test':
query = Expression('a', '=', 'a')
else:
query = Expression(getattr(Job, 'process'), '=', process)
if filters:
log.debug("Found filters: " + str(filters))
keys = [key for key in filters]
for key in keys:
try:
getattr(Job, key)
except Exception as e:
log.warning(str(e))
continue
log.debug("Filter " + str(key) + " with value " +
str(filters[key]))
if isinstance(getattr(Job, key), AutoField):
try:
int(filters[key])
except Exception as e:
log.error(str(e))
jobdb.close()
return
try:
# even though operators are listed as == and & in peewee docs,
# for Expression creation use '=' and 'AND'.
exp = Expression(getattr(Job, key), '=', filters[key])
query = Expression(query, 'AND', exp)
except AttributeError as e:
log.error(str(e))
with jobdb:
queryResult = Job.select().where(query).dicts()
jobs = []
# iterating reopens db connection!!
for i in queryResult:
jobs.append(i)
log.info("Found " + str(len(jobs)) + " results for query.")
jobdb.close()
return jobs
def makeItJson(xml):
try:
parsedresp = xmltodict.parse(xml)
records = json.dumps(parsedresp)
return records
except Exception:
log.error('Error parsing XML response from gnos')
return None
def get(self, uuid):
try:
utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
gnosresp = update(uuid, utcnow)
if gnosresp is None:
raise Exception
res = make_response(gnosresp.content, 200)
return res
except Exception as e:
log.error('error parsing gnos response')
log.error(e)
res = (jsonify(
SimpleResponseModel(status=404,
message='Error looking for uuid "' + uuid +
'".')))
return make_response(res, 404)
def fillTemplateFromProcessChain(module):
""" This method receives a process chain for an actinia module and loads
the according process chain template. The received values will be
replaced to be passed to actinia. In case the template has more
placeholder values than it receives, the missing attribute is returned
as string.
"""
kwargs = {}
inOrOutputs = []
if module.get('inputs') is not None:
inOrOutputs += module.get('inputs')
if module.get('outputs') is not None:
inOrOutputs += module.get('outputs')
for item in inOrOutputs:
if (item.get('param') is None) or (item.get('value') is None):
return None
key = item['param']
val = item['value']
kwargs[key] = val
tpl_file = module["module"] + '.json'
# change path to template if in subdir
for i in pcTplEnv.list_templates(filter_func=filter_func):
if i.split('/')[-1] == tpl_file:
tpl_file = i
# find variables from processchain
tpl_source = pcTplEnv.loader.get_source(pcTplEnv, tpl_file)[0]
parsed_content = pcTplEnv.parse(tpl_source)
undef = meta.find_undeclared_variables(parsed_content)
for i in undef:
if i not in kwargs.keys():
log.error('Required parameter "' + i + '" not in process chain!')
return i
# fill process chain template
tpl = pcTplEnv.get_template(tpl_file)
pc_template = json.loads(tpl.render(**kwargs).replace('\n', ''))
return (pc_template['template']['list'])
def createJob(jsonDict, process):
""" Method to parse prePC including fetching information from
geonetwork and writing information to Jobtable
as well as starting job in actinia-core
This method can be called by HTTP POST
@app.route('/processes/test/jobs')
"""
prePC_orig = json.dumps(jsonDict)
# TODO: define prePC (pre processchain) model if differs from pc
# prePC = prePC(**jsonDict)
# as we don't hava a model yet
# prePC = json.dumps(jsonDict)
connection = checkConnectionWithoutResponse('actinia-core')
if connection is not None:
actiniaCoreResp = postActiniaCore(process, jsonDict)
log.debug(actiniaCoreResp)
# try:
# prePCDict = prePC.to_struct()
# except Exception as e:
# log.error('prePC is invalid!')
# log.error(e)
# return None
job = insertNewJob(
jsonDict,
jsonDict, # as we don't hava a model yet
process,
jsonDict.get(
'feature_type'), # empty at the moment (polygon later)
actiniaCoreResp)
if actiniaCoreResp['status'] == 'error':
log.error("Error start processing in actinia-core")
resourceId = parseActiniaIdFromUrl(actiniaCoreResp['resource_id'])
job = updateJob(resourceId, actiniaCoreResp)
return job
else:
return None
def buildPCDummy(preProcessChain):
webhookUrl = APP.url + "/resources/processes/operations/update"
log.debug("Registering own webhook endpoint: " + str(webhookUrl))
tpl = tplEnv.get_template('actiniaCore/pc_point_in_polygon.json')
point = preProcessChain.get('point')
polygon = preProcessChain.get('polygon')
if polygon is None or point is None or webhookUrl is None:
log.error('Could not set all variables to replace in template.')
return None
postbody = tpl.render(point=point, polygon=polygon,
webhookUrl=webhookUrl).replace('\n', '')
return postbody
def checkConnectionWithoutResponse(name):
""" Method to test connection
Args:
name (string): resource to test. Can be 'actinia-core' or 'geonetwork'
"""
if name == 'actinia-core':
url = ACTINIACORE.url + "version"
name = 'actinia-core'
type = 'json'
try:
connectionTest = common.checkConnection(url, name, type)
return connectionTest
except Exception:
log.error("Don't know which connection to test")
return None
def get(self, category):
try:
records = getRecordsByCategory(category)
res = make_response(records, 200)
res.headers['Content-Type'] = 'application/json'
return res
# except TemplateNotFound as e:
# print('ERROR: ' + repr(e) + " - " + e.message)
# return make_response('Error looking for category "' + category +
# '".', 404)
except Exception as e:
log.error('ERROR: ' + repr(e) + " - " + str(e))
res = (jsonify(
SimpleResponseModel(status=404,
message='Category "' + category +
'" not found.')))
return make_response(res, 404)
def cancelJobById(jobid):
""" Method to change the status of a job to 'TERMINATED' in the jobtabelle
by using its jobid
Args:
jobid (int): id of job
Returns:
record (dict): the record matching the id
"""
utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
try:
with jobdb:
queryResult = Job.select().where(
getattr(Job, JOBTABLE.id_field) == jobid).get()
record = model_to_dict(queryResult)
log.debug("Information read from jobtable.")
except Job.DoesNotExist:
record = None
try:
query = Job.update(
status='TERMINATED',
time_ended=utcnow).where(getattr(Job, JOBTABLE.id_field) == jobid)
with jobdb:
query.execute()
queryResult2 = Job.select().where(
getattr(Job, JOBTABLE.id_field) == jobid).get()
record = model_to_dict(queryResult2)
except Exception as e:
log.error('Could not set the status to "TERMINATED" ' +
'and the time_ended in the jobtable.')
log.error(str(e))
record = None
log.info("Information updated in jobtable for job with id " +
str(record['idpk_jobs']) + ".")
jobdb.close()
return record
def getRecordsByTags(tags):
""" Method to get records by tags from geonetwork
Attention: The tags received are case sensitive!
PropertyIsLike is sensitive, matchCase not possible here.
See https://trac.osgeo.org/geonetwork/wiki/CSW202Improvements
This method can be called by @app.route('/metadata/raw/tags/<tags>')
"""
log.debug('looking for tags ' + str(tags))
try:
url = GEONETWORK.csw_url
tpl = tplEnv.get_template('geonetwork/post_records_by_tags.xml')
tags = tags.split(',')
postbody = tpl.render(tags=tags).replace('\n', '')
headers = {'content-type': 'application/xml; charset=utf-8'}
except Exception as e:
log.error('Could not set needed variable')
log.error(e)
return None
try:
gnosresp = requests.post(
url,
data=postbody,
headers=headers,
auth=auth(GEONETWORK)
)
return gnosresp.content
except requests.exceptions.ConnectionError:
log.error('Could not connect to gnos')
return None
def updateXml(response, utcnow):
try:
doc = parseString(response.decode('utf-8'))
recordNode = doc.getElementsByTagName('gmd:MD_Metadata')[0]
dateStampEl = recordNode.getElementsByTagName('gmd:dateStamp')[0]
dateEl = dateStampEl.getElementsByTagName('gco:Date')
if len(dateEl) == 0:
dateEl = dateStampEl.getElementsByTagName('gco:DateTime')
if len(dateEl) == 0:
log.error('Could not find date element')
return None
else:
dateEl = dateEl[0]
else:
dateEl = dateEl[0]
utcnow = utcnow.split('T')[0]
if dateEl.firstChild.nodeType != dateEl.TEXT_NODE:
raise Exception("node does not contain text")
dateEl.firstChild.replaceWholeText(utcnow)
record = recordNode.toxml().replace('\n', '')
except Exception as e:
log.error('Could not set date in metadata record')
log.error(e)
return None
return record
def post(self):
jsonFile = request.form.get('jsonFile')
geodataFile = request.files.get('geodataFile')
if not os.path.isdir(FILEUPLOAD.geodata):
os.makedirs(FILEUPLOAD.geodata)
# See file operations here:
# https://www.programiz.com/python-programming/file-operation
if geodataFile:
# will be stored under original filename with uuid
filename = (str(uuid.uuid4()) + '.'
+ secure_filename(geodataFile.filename))
geodataFile.save(FILEUPLOAD.geodata + '/' + filename)
log.info('Received json file, saved to '
+ FILEUPLOAD.geodata + '/' + filename)
elif jsonFile:
# will be stored under random name, e.g.
# 5f82730c-f804-4e34-9c24-faaed902fab5.json
filename = str(uuid.uuid4()) + '.json'
with open(FILEUPLOAD.geodata + '/' + filename, 'x') as file:
file.write(jsonFile)
log.info('Received binary file, saved to '
+ FILEUPLOAD.geodata + '/' + filename)
else:
log.error('No file found in postbody')
metadatarecord = create(filename)
res = jsonify(FileUploadResponseModel(
status=200,
message="Upload success",
name=filename,
record=metadatarecord
))
res.headers['Access-Control-Allow-Origin'] = '*'
return make_response(res, 200)
def getRecordByUUID(uuid):
""" Method to get record by uuid from geonetwork
This method can be called by @app.route('/metadata/raw/uuids/<uuid>')
"""
try:
url = GEONETWORK.csw_url
tpl = tplEnv.get_template('geonetwork/get_record_by_uuid_kvp.json')
kvps = json.loads(tpl.render(uuid=uuid))
except Exception as e:
log.error('Could not set needed variable')
log.error(e)
return None
try:
gnosresp = requests.get(url, params=kvps, auth=auth(GEONETWORK))
return gnosresp.content
except requests.exceptions.ConnectionError:
log.error('Could not connect to gnos')
return None
def cancelJob(jobid):
""" Method to cancel job from Jobtable by id
This method can be called by HTTP POST
@app.route('/processes/test/jobs/<jobid>/operations/cancel')
"""
job = getJobById(jobid)
if not job == None:
log.debug('The job with jobid ' + str(jobid) + ' exists')
status = job['status']
resourceId = job['actinia_core_jobid']
if not status or not resourceId:
log.error('Job status or resourceId is not set!')
return None
else:
log.debug('Job status is ' + status + ' and resourceId is: ' +
resourceId)
connection = checkConnectionWithoutResponse('actinia-core')
if connection is not None:
if status in ['PENDING', 'RUNNING']:
log.debug('Status is in PENDING or RUNNING, will cancel')
res = cancelActiniaCore(resourceId)
if res:
log.debug('Actinia-Core response TRUE')
job = cancelJobById(jobid)
log.debug('Job in jobtable is ' + job['status'])
return job
else:
log.debug('Actinia-Core response is None')
return None
else:
log.debug('Status not in PENDING or RUNNING, pass')
return job
else:
log.error('There is no connection to actinia-core')
return None
else:
log.error('The job with jobid ' + str(jobid) + 'does not exist')
return None
def getRecordsByCategory(category):
""" Method to get records by category from geonetwork
This method can be called by
@app.route('/metadata/raw/categories/<category>')
"""
try:
url = (GEONETWORK.csw_url + '-' + category)
tpl = tplEnv.get_template('geonetwork/get_records_by_category_kvp.json')
kvps = json.loads(tpl.render())
except Exception as e:
log.error('Could not set needed variable')
log.error(e)
return None
try:
gnosresp = requests.get(url, params=kvps, auth=auth(GEONETWORK))
return gnosresp.content
except requests.exceptions.ConnectionError:
log.error('Could not connect to gnos')
return None
def updateJobByResourceID(resourceId, resp, status):
""" Method to update job in jobtabelle when processing status changed
Args:
resourceId (str): actinia-core resourceId
resp (dict): actinia-core response
status (string): actinia-core processing status
Returns:
updatedRecord (TODO): the updated record
"""
utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
try:
with jobdb:
queryResult = Job.select().where(
getattr(Job, 'actinia_core_jobid') == resourceId).get()
record = model_to_dict(queryResult)
log.debug("Information read from jobtable for job with id " +
str(record['idpk_jobs']) + ".")
except Job.DoesNotExist:
log.warning("Job does not exist and can therefore not be updated")
return None, None, None
# actinia-gdi_["PENDING", "RUNNING", "SUCCEES", "ERROR", "TERMINATED"]
# actinia-core [accepted, running, finished, error, terminated]
try:
log.debug("Update status to " + status + " for job with id " +
str(record['idpk_jobs']) + ".")
gdiStatus = record['status']
try:
gnosUuid = record['job_description']['feature_uuid']
except Exception:
log.warning('Feature has no uuid')
gnosUuid = None
if current_app.debug is False:
smallRes = dict()
smallRes['message'] = resp.get('message', None)
smallRes['process_results'] = resp.get('process_results', None)
resp = smallRes
# resp = smallifyResp(resp)
# TODO: test in debug, then remove
# resp = smallifyResp(resp)
if status == 'accepted':
log.debug('Status already set to "PENDING"')
return record, gnosUuid, utcnow
elif status == 'running':
if gdiStatus == 'RUNNING':
log.debug('Status already set to "RUNNING"')
return record, gnosUuid, utcnow
query = Job.update(
status='RUNNING',
actinia_core_response=resp,
time_started=utcnow
# TODO: check if time_estimated can be set
# time_estimated=
).where(getattr(Job, 'actinia_core_jobid') == resourceId)
elif status in ['finished', 'error', 'terminated']:
if status == 'finished':
gdiStatus = 'SUCCESS'
elif status == 'error':
gdiStatus = 'ERROR'
elif status == 'terminated':
gdiStatus = 'TERMINATED'
query = Job.update(
status=gdiStatus,
actinia_core_response=resp,
time_ended=utcnow).where(
getattr(Job, 'actinia_core_jobid') == resourceId)
else:
log.error('Could not set the status to actinia-core status:' +
status + '(Status not found.)')
return None, None, None
with jobdb:
query.execute()
queryResult = Job.select().where(
getattr(Job, 'actinia_core_jobid') == resourceId).get()
record = model_to_dict(queryResult)
except Exception as e:
log.error('Could not set the status to actinia-core status: ' + status)
log.error(str(e))
return None, None, None
log.info("Updated status to " + status + " for job with id " +
str(record['idpk_jobs']) + ".")
jobdb.close()
return record, gnosUuid, utcnow
def update(uuid, utcnow):
""" Method to update record in geonetwork
"""
connection = checkConnectionWithoutResponse('geonetwork')
if connection is None:
log.error('Not updating metadata for uuid ' + uuid)
return None
try:
response = getRecordByUUID(uuid)
doc = parseString(response.decode('utf-8'))
recordNode = doc.getElementsByTagName('gmd:MD_Metadata')[0]
log.debug('Found metadata to update for ' + uuid + ', and ' +
str(recordNode))
except Exception:
log.error('Could not find metadata record to update for uuid ' + uuid)
return None
record = updateXml(response, utcnow)
if record is None:
return None
try:
url = GEONETWORK.csw_pub
postbodytpl = tplEnv.get_template('geonetwork/post_update_record.xml')
postbody = postbodytpl.render(metadata_record=record,
uuid=uuid).replace('\n', '')
headers = {'content-type': 'application/xml; charset=utf-8'}
except Exception as e:
log.error('Could not set needed variable')
log.error(e)
return None
try:
log.info('Updating metadata record')
gnosresp = requests.post(url,
data=bytes(postbody, 'utf-8'),
headers=headers,
auth=auth(GEONETWORK))
if '<html>' in gnosresp.content.decode('utf-8'):
log.error('update error')
else:
log.info('update success')
return gnosresp
except requests.exceptions.ConnectionError:
log.error('Could not connect to gnos')
return None
except Exception as e:
log.error(e)
return None
