def post(self):
""" Update a job
This method is called by HTTP POST actinia-core webhook
@app.route('/resources/processes/operations/update')
This method is calling core method updateJob
"""
postbody = request.get_json(force=True)
if type(postbody) is dict:
postbody = json.dumps(postbody)
elif type(postbody) != 'str':
postbody = str(postbody)
actiniaCoreResp = json.loads(postbody)
resourceID = actiniaCoreResp['resource_id']
log.info("\n Received webhook call for " + resourceID)
job = updateJob(resourceID, actiniaCoreResp)
if job is not None:
return make_response(jsonify(job), 200)
else:
res = (jsonify(SimpleResponseModel(
status=404,
message='Error'
)))
return make_response(res, 404)
def getRecordRoot(record):
""" Method to get one record out of the geonetwork response. Valid for
both schemata 'gmd:MD_Metadata' and 'csw:Record'
TODO: support more than one tag in response
This method can handle GetRecordByIdResponse and GetRecordsResponse
"""
record = json.loads(record)
if 'csw:GetRecordByIdResponse' in record:
log.info("Found 1 record")
if 'gmd:MD_Metadata' in record["csw:GetRecordByIdResponse"]:
recordRoot = (
record["csw:GetRecordByIdResponse"]["gmd:MD_Metadata"]
)
elif 'csw:Record' in record["csw:GetRecordByIdResponse"]:
recordRoot = (
record["csw:GetRecordByIdResponse"]["csw:Record"]
)
else:
log.info("...But record is empty")
return
elif 'csw:GetRecordsResponse' in record:
if 'csw:SearchResults' in record["csw:GetRecordsResponse"]:
searchResults = (
record["csw:GetRecordsResponse"]["csw:SearchResults"]
)
else:
log.info("...But record is empty")
return
numberOfRecords = int(searchResults["@numberOfRecordsReturned"])
recordRoot = dict()
if numberOfRecords == 0:
log.warning("No records found")
return
elif numberOfRecords == 1:
log.info("Found 1 record")
recordRoot = searchResults["gmd:MD_Metadata"]
if not recordRoot:
recordRoot = searchResults["csw:Record"]
else:
log.info("Found " + str(numberOfRecords) + " records")
recordRoot = searchResults["gmd:MD_Metadata"][0]
if not recordRoot:
recordRoot = searchResults["csw:Record"][0]
else:
print("Could not parse GNOS response")
return
return recordRoot
def getAllJobs(filters, process):
""" Method to read all jobs from jobtabelle with filter
Args: filters (ImmutableMultiDict): the args from the HTTP call
Returns:
jobs (list): the records matching the filter
"""
log.debug('Received query for jobs')
if process == 'test':
query = Expression('a', '=', 'a')
else:
query = Expression(getattr(Job, 'process'), '=', process)
if filters:
log.debug("Found filters: " + str(filters))
keys = [key for key in filters]
for key in keys:
try:
getattr(Job, key)
except Exception as e:
log.warning(str(e))
continue
log.debug("Filter " + str(key) + " with value " +
str(filters[key]))
if isinstance(getattr(Job, key), AutoField):
try:
int(filters[key])
except Exception as e:
log.error(str(e))
jobdb.close()
return
try:
# even though operators are listed as == and & in peewee docs,
# for Expression creation use '=' and 'AND'.
exp = Expression(getattr(Job, key), '=', filters[key])
query = Expression(query, 'AND', exp)
except AttributeError as e:
log.error(str(e))
with jobdb:
queryResult = Job.select().where(query).dicts()
jobs = []
# iterating reopens db connection!!
for i in queryResult:
jobs.append(i)
log.info("Found " + str(len(jobs)) + " results for query.")
jobdb.close()
return jobs
def shortenActiniaCoreResp(fullResp):
try:
status = parseActiniaAsyncStatusResponse(fullResp, "status")
message = parseActiniaAsyncStatusResponse(fullResp, "message")
url = parseActiniaAsyncStatusResponse(fullResp, "urls.status")
except Exception:
return None
log.info('Status is "' + status + " - " + message + '"')
return json.loads('{"status": "' + url + '"}')
def update(uuid, utcnow):
""" Method to update record in geonetwork
"""
connection = checkConnectionWithoutResponse('geonetwork')
if connection is None:
log.error('Not updating metadata for uuid ' + uuid)
return None
try:
response = getRecordByUUID(uuid)
doc = parseString(response.decode('utf-8'))
recordNode = doc.getElementsByTagName('gmd:MD_Metadata')[0]
log.debug('Found metadata to update for ' + uuid + ', and ' +
str(recordNode))
except Exception:
log.error('Could not find metadata record to update for uuid ' + uuid)
return None
record = updateXml(response, utcnow)
if record is None:
return None
try:
url = GEONETWORK.csw_pub
postbodytpl = tplEnv.get_template('geonetwork/post_update_record.xml')
postbody = postbodytpl.render(metadata_record=record,
uuid=uuid).replace('\n', '')
headers = {'content-type': 'application/xml; charset=utf-8'}
except Exception as e:
log.error('Could not set needed variable')
log.error(e)
return None
try:
log.info('Updating metadata record')
gnosresp = requests.post(url,
data=bytes(postbody, 'utf-8'),
headers=headers,
auth=auth(GEONETWORK))
if '<html>' in gnosresp.content.decode('utf-8'):
log.error('update error')
else:
log.info('update success')
return gnosresp
except requests.exceptions.ConnectionError:
log.error('Could not connect to gnos')
return None
except Exception as e:
log.error(e)
return None
def insertNewJob(rule_configuration, job_description, process, feature_type,
actiniaCoreResp):
"""Insert new job into jobtabelle.
Args:
rule_configuration (dict): original preProcessChain
job_description (TODO): enriched preProcessChain with geometadata
Returns:
record (dict): the new record
"""
utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
actiniaCoreJobUrl = parseActiniaAsyncStatusResponse(
actiniaCoreResp, "urls.status")
actiniaCoreJobID = parseActiniaIdFromUrl(actiniaCoreJobUrl)
if current_app.debug is False:
smallRes = dict()
smallRes['message'] = actiniaCoreResp.get('message', None)
smallRes['process_results'] = actiniaCoreResp.get(
'process_results', None)
actiniaCoreResp = smallRes
# actiniaCoreResp = smallifyResp(actiniaCoreResp)
# TODO: test in debug, then remove
# actiniaCoreResp = smallifyResp(actiniaCoreResp)
job = Job(
**{
'rule_configuration': rule_configuration,
'job_description': job_description,
'status': 'PENDING',
'time_created': utcnow,
'process': process,
'feature_type': feature_type,
'actinia_core_response': actiniaCoreResp,
'actinia_core_jobid': actiniaCoreJobID
})
with jobdb:
job.save()
queryResult = Job.select().where(Job.time_created == utcnow).get()
record = model_to_dict(queryResult)
log.info("Created new job with id " + str(record['idpk_jobs']) + ".")
jobdb.close()
return record
def create(filename):
""" Method to get create metadata records in geonetwork
"""
url = GEONETWORK.csw_publication
recordtpl = tplEnv.get_template('geonetwork/template_metadaten.xml')
record = recordtpl.render(title=filename).replace('\n', '')
# recordfs = recordtpl.render(title=filename)
postbodytpl = tplEnv.get_template('geonetwork/post_create_record.xml')
postbody = postbodytpl.render(metadata_record=record).replace('\n', '')
# postbodyfs = postbodytpl.render(metadata_record=recordfs)
headers = {'content-type': 'application/xml; charset=utf-8'}
log.info('Creating metadata record')
try:
gnosresp = requests.post(url,
data=postbody,
headers=headers,
auth=auth(GEONETWORK))
except requests.exceptions.ConnectionError:
return None
# if not os.path.isdir(FILEUPLOAD.templates):
# os.makedirs(FILEUPLOAD.templates)
#
# with open(FILEUPLOAD.templates + '/' + filename + '_template.xml',
# 'x') as file:
# file.write(postbodyfs)
# log.info('Received binary file, saved to '
# + FILEUPLOAD.templates + '/' + filename + '_template.xml')
try:
parsedresp = xmltodict.parse(gnosresp.content)
insertRes = parsedresp['csw:TransactionResponse']['csw:InsertResult']
uuid = insertRes['csw:BriefRecord']['identifier']
log.info('GNOS response is: ' + str(parsedresp))
return uuid
except Exception:
return None
def cancelJobById(jobid):
""" Method to change the status of a job to 'TERMINATED' in the jobtabelle
by using its jobid
Args:
jobid (int): id of job
Returns:
record (dict): the record matching the id
"""
utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
try:
with jobdb:
queryResult = Job.select().where(
getattr(Job, JOBTABLE.id_field) == jobid).get()
record = model_to_dict(queryResult)
log.debug("Information read from jobtable.")
except Job.DoesNotExist:
record = None
try:
query = Job.update(
status='TERMINATED',
time_ended=utcnow).where(getattr(Job, JOBTABLE.id_field) == jobid)
with jobdb:
query.execute()
queryResult2 = Job.select().where(
getattr(Job, JOBTABLE.id_field) == jobid).get()
record = model_to_dict(queryResult2)
except Exception as e:
log.error('Could not set the status to "TERMINATED" ' +
'and the time_ended in the jobtable.')
log.error(str(e))
record = None
log.info("Information updated in jobtable for job with id " +
str(record['idpk_jobs']) + ".")
jobdb.close()
return record
def postActiniaCore(process, preProcessChain):
""" Method to start new job in actinia-core
Args:
process: (str): the process which is triggered
preProcessChain: (dict): the enriched preProcessChain object
Returns:
TODO: (dict): status url of actinia-core job
"""
url = ACTINIACORE.url + PROCESSING_ENDPOINT
headers = {'content-type': 'application/json; charset=utf-8'}
if process == 'test':
postbody = buildPCDummy(preProcessChain)
elif process == 'sentinel1':
postbody = buildPCS1Grd(preProcessChain)
else:
postbody = buildPCDummy(preProcessChain)
try:
# TODO: for now if error above, here an empty postbody is send to
# actinia-core. This leads to correct status "ERROR" at the moment.
# Make smarter (not call actinia-core, return correct error msg)
log.info('Posting to ' + url)
actiniaResp = requests.post(url,
data=postbody,
headers=headers,
auth=auth(ACTINIACORE))
# TODO: remove GET request here. It is a workaround for a bug in
# actinia-core which does not start a job if no request follows
actiniaCoreId = json.loads(actiniaResp.text)['resource_id']
url = ACTINIACORE.url + "resources/actinia-gdi/" + actiniaCoreId
actiniaResp2 = requests.get(url, auth=auth(ACTINIACORE))
except requests.exceptions.ConnectionError:
return None
return json.loads(actiniaResp2.text)
def post(self):
""" Create new job
This method is called by HTTP POST
@app.route('/processes/test/jobs')
This method is calling core method createJob
"""
log.info("\n Received HTTP POST with job:")
print(request.get_json(force=True))
# We determine the process type here, depending on which endpoint
# was called
process = request.path.split('/')[2]
job = createJob(request.get_json(force=True), process)
if job is not None:
return make_response(jsonify(job), 201)
else:
res = (jsonify(SimpleResponseModel(status=404, message='Error')))
return make_response(res, 404)
def get(self, jobid):
""" Wrapper method to receive HTTP call and pass it to function
This method is called by HTTP GET
@app.route('/processes/test/jobs/<jobid>')
This method is calling core method readJob
"""
if jobid is None:
return make_response("Not found", 404)
log.info("\n Received HTTP GET request for job with id " + str(jobid))
job = getJob(jobid)
if job is not None:
return make_response(jsonify(job), 200)
else:
res = (jsonify(
SimpleResponseModel(status=404,
message='Not Found: ' + request.url)))
return make_response(res, 404)
def post(self):
jsonFile = request.form.get('jsonFile')
geodataFile = request.files.get('geodataFile')
if not os.path.isdir(FILEUPLOAD.geodata):
os.makedirs(FILEUPLOAD.geodata)
# See file operations here:
# https://www.programiz.com/python-programming/file-operation
if geodataFile:
# will be stored under original filename with uuid
filename = (str(uuid.uuid4()) + '.'
+ secure_filename(geodataFile.filename))
geodataFile.save(FILEUPLOAD.geodata + '/' + filename)
log.info('Received json file, saved to '
+ FILEUPLOAD.geodata + '/' + filename)
elif jsonFile:
# will be stored under random name, e.g.
# 5f82730c-f804-4e34-9c24-faaed902fab5.json
filename = str(uuid.uuid4()) + '.json'
with open(FILEUPLOAD.geodata + '/' + filename, 'x') as file:
file.write(jsonFile)
log.info('Received binary file, saved to '
+ FILEUPLOAD.geodata + '/' + filename)
else:
log.error('No file found in postbody')
metadatarecord = create(filename)
res = jsonify(FileUploadResponseModel(
status=200,
message="Upload success",
name=filename,
record=metadatarecord
))
res.headers['Access-Control-Allow-Origin'] = '*'
return make_response(res, 200)
def post(self, jobid):
""" Wrapper method to cancel a job by using the jobId
This method is called by HTTP POST
@app.route('/processes/test/jobs/<jobid>/operations/cancel')
This method is calling core method readJob
"""
if jobid is None:
return make_response("Not found", 404)
log.info("\n Received HTTP POST request for job with id " +
str(jobid) + "to cancel the job")
job = cancelJob(jobid)
if job is not None:
return make_response(jsonify(job), 200)
else:
res = (jsonify(
SimpleResponseModel(status=404,
message='Not Found: ' + request.url)))
return make_response(res, 404)
def getJobById(jobid):
""" Method to read job from jobtabelle by id
Args:
jobid (int): id of job
Returns:
record (dict): the record matching the id
"""
try:
with jobdb:
queryResult = Job.select().where(
getattr(Job, JOBTABLE.id_field) == jobid).get()
record = model_to_dict(queryResult)
log.info("Information read from jobtable for job with id " +
str(record['idpk_jobs']) + ".")
except Job.DoesNotExist:
record = None
jobdb.close()
return record
def cancelActiniaCore(resourceId):
""" Wrapper method to cancel a job by using the jobId
This method is called by HTTP POST
@app.route('/processes/test/jobs/<jobid>/operations/cancel')
This method is calling core method readJob
Args:
resourceId: (string): actinia-core job id
"""
if resourceId is None:
return make_response("Not found", 404)
print("\n Received HTTP DELETE request for job with actinia-core jobID " +
resourceId)
DELETING_ENDPOINT = 'resources/'
url = (ACTINIACORE.url + DELETING_ENDPOINT + ACTINIACORE.user + '/' +
resourceId)
log.info('Requesting from ' + url)
try:
actiniaResp = requests.delete(url, auth=auth(ACTINIACORE))
except requests.exceptions.ConnectionError:
return None
try:
status = parseActiniaAsyncStatusResponse(json.loads(actiniaResp.text),
"status")
message = parseActiniaAsyncStatusResponse(json.loads(actiniaResp.text),
"message")
except Exception:
return None
log.info('Status is "' + status + " - " + message + '"')
return True
def parseMeta(record):
""" Method to parse record from geonetwork with model
TODO: support more than one tag in response
TODO: better error handling when attribute not found
This method can handle GetRecordByIdResponse and GetRecordsResponse
"""
if 'csw:GetRecordByIdResponse' in record:
log.info("Found 1 record")
if 'csw:Record' in json.loads(record)["csw:GetRecordByIdResponse"]:
recordRoot = (
json.loads(record)["csw:GetRecordByIdResponse"]["csw:Record"]
)
else:
log.info("...But record is empty")
return
elif 'csw:GetRecordsResponse' in record:
if 'csw:SearchResults' in json.loads(record)["csw:GetRecordsResponse"]:
searchResults = (
json.loads(record)["csw:GetRecordsResponse"]["csw:SearchResults"]
)
else:
log.info("...But record is empty")
return
numberOfRecords = int(searchResults["@numberOfRecordsReturned"])
recordRoot = dict()
if numberOfRecords == 0:
log.warning("No records found")
return
elif numberOfRecords == 1:
log.info("Found 1 record")
recordRoot = searchResults["csw:Record"]
else:
log.info("Found " + str(numberOfRecords) + " records")
recordRoot = searchResults["csw:Record"][0]
else:
print("Could not parse GNOS response")
return
if 'dc:identifier' in recordRoot:
uuid = recordRoot["dc:identifier"]
else:
uuid = 'null'
if 'ows:BoundingBox' in recordRoot:
recordBbox = recordRoot["ows:BoundingBox"]
if 'ows:LowerCorner' in recordBbox:
bbox_lower = recordBbox["ows:LowerCorner"]
if 'ows:UpperCorner' in recordBbox:
bbox_upper = recordBbox["ows:UpperCorner"]
bbox_a = float(bbox_lower.split(" ")[0])
bbox_b = float(bbox_lower.split(" ")[1])
bbox_c = float(bbox_upper.split(" ")[0])
bbox_d = float(bbox_upper.split(" ")[1])
bbox = [bbox_a, bbox_b, bbox_c, bbox_d]
if '@crs' in recordBbox:
crs = recordBbox["@crs"]
else:
bbox = []
crs = 'null'
if 'dc:URI' in recordRoot:
recordUri = recordRoot["dc:URI"]
if type(recordUri) is dict:
if "#text" in recordUri:
table = recordUri["#text"]
else:
table = recordUri[0]["#text"]
else:
table = 'null'
geodata_meta = GeodataMeta(
uuid=uuid,
bbox=bbox,
crs=crs,
table=table
)
return geodata_meta
def updateJobByResourceID(resourceId, resp, status):
""" Method to update job in jobtabelle when processing status changed
Args:
resourceId (str): actinia-core resourceId
resp (dict): actinia-core response
status (string): actinia-core processing status
Returns:
updatedRecord (TODO): the updated record
"""
utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
try:
with jobdb:
queryResult = Job.select().where(
getattr(Job, 'actinia_core_jobid') == resourceId).get()
record = model_to_dict(queryResult)
log.debug("Information read from jobtable for job with id " +
str(record['idpk_jobs']) + ".")
except Job.DoesNotExist:
log.warning("Job does not exist and can therefore not be updated")
return None, None, None
# actinia-gdi_["PENDING", "RUNNING", "SUCCEES", "ERROR", "TERMINATED"]
# actinia-core [accepted, running, finished, error, terminated]
try:
log.debug("Update status to " + status + " for job with id " +
str(record['idpk_jobs']) + ".")
gdiStatus = record['status']
try:
gnosUuid = record['job_description']['feature_uuid']
except Exception:
log.warning('Feature has no uuid')
gnosUuid = None
if current_app.debug is False:
smallRes = dict()
smallRes['message'] = resp.get('message', None)
smallRes['process_results'] = resp.get('process_results', None)
resp = smallRes
# resp = smallifyResp(resp)
# TODO: test in debug, then remove
# resp = smallifyResp(resp)
if status == 'accepted':
log.debug('Status already set to "PENDING"')
return record, gnosUuid, utcnow
elif status == 'running':
if gdiStatus == 'RUNNING':
log.debug('Status already set to "RUNNING"')
return record, gnosUuid, utcnow
query = Job.update(
status='RUNNING',
actinia_core_response=resp,
time_started=utcnow
# TODO: check if time_estimated can be set
# time_estimated=
).where(getattr(Job, 'actinia_core_jobid') == resourceId)
elif status in ['finished', 'error', 'terminated']:
if status == 'finished':
gdiStatus = 'SUCCESS'
elif status == 'error':
gdiStatus = 'ERROR'
elif status == 'terminated':
gdiStatus = 'TERMINATED'
query = Job.update(
status=gdiStatus,
actinia_core_response=resp,
time_ended=utcnow).where(
getattr(Job, 'actinia_core_jobid') == resourceId)
else:
log.error('Could not set the status to actinia-core status:' +
status + '(Status not found.)')
return None, None, None
with jobdb:
query.execute()
queryResult = Job.select().where(
getattr(Job, 'actinia_core_jobid') == resourceId).get()
record = model_to_dict(queryResult)
except Exception as e:
log.error('Could not set the status to actinia-core status: ' + status)
log.error(str(e))
return None, None, None
log.info("Updated status to " + status + " for job with id " +
str(record['idpk_jobs']) + ".")
jobdb.close()
return record, gnosUuid, utcnow