def post(self): """ Update a job This method is called by HTTP POST actinia-core webhook @app.route('/resources/processes/operations/update') This method is calling core method updateJob """ postbody = request.get_json(force=True) if type(postbody) is dict: postbody = json.dumps(postbody) elif type(postbody) != 'str': postbody = str(postbody) actiniaCoreResp = json.loads(postbody) resourceID = actiniaCoreResp['resource_id'] log.info("\n Received webhook call for " + resourceID) job = updateJob(resourceID, actiniaCoreResp) if job is not None: return make_response(jsonify(job), 200) else: res = (jsonify(SimpleResponseModel( status=404, message='Error' ))) return make_response(res, 404)
def getRecordRoot(record): """ Method to get one record out of the geonetwork response. Valid for both schemata 'gmd:MD_Metadata' and 'csw:Record' TODO: support more than one tag in response This method can handle GetRecordByIdResponse and GetRecordsResponse """ record = json.loads(record) if 'csw:GetRecordByIdResponse' in record: log.info("Found 1 record") if 'gmd:MD_Metadata' in record["csw:GetRecordByIdResponse"]: recordRoot = ( record["csw:GetRecordByIdResponse"]["gmd:MD_Metadata"] ) elif 'csw:Record' in record["csw:GetRecordByIdResponse"]: recordRoot = ( record["csw:GetRecordByIdResponse"]["csw:Record"] ) else: log.info("...But record is empty") return elif 'csw:GetRecordsResponse' in record: if 'csw:SearchResults' in record["csw:GetRecordsResponse"]: searchResults = ( record["csw:GetRecordsResponse"]["csw:SearchResults"] ) else: log.info("...But record is empty") return numberOfRecords = int(searchResults["@numberOfRecordsReturned"]) recordRoot = dict() if numberOfRecords == 0: log.warning("No records found") return elif numberOfRecords == 1: log.info("Found 1 record") recordRoot = searchResults["gmd:MD_Metadata"] if not recordRoot: recordRoot = searchResults["csw:Record"] else: log.info("Found " + str(numberOfRecords) + " records") recordRoot = searchResults["gmd:MD_Metadata"][0] if not recordRoot: recordRoot = searchResults["csw:Record"][0] else: print("Could not parse GNOS response") return return recordRoot
def getAllJobs(filters, process): """ Method to read all jobs from jobtabelle with filter Args: filters (ImmutableMultiDict): the args from the HTTP call Returns: jobs (list): the records matching the filter """ log.debug('Received query for jobs') if process == 'test': query = Expression('a', '=', 'a') else: query = Expression(getattr(Job, 'process'), '=', process) if filters: log.debug("Found filters: " + str(filters)) keys = [key for key in filters] for key in keys: try: getattr(Job, key) except Exception as e: log.warning(str(e)) continue log.debug("Filter " + str(key) + " with value " + str(filters[key])) if isinstance(getattr(Job, key), AutoField): try: int(filters[key]) except Exception as e: log.error(str(e)) jobdb.close() return try: # even though operators are listed as == and & in peewee docs, # for Expression creation use '=' and 'AND'. exp = Expression(getattr(Job, key), '=', filters[key]) query = Expression(query, 'AND', exp) except AttributeError as e: log.error(str(e)) with jobdb: queryResult = Job.select().where(query).dicts() jobs = [] # iterating reopens db connection!! for i in queryResult: jobs.append(i) log.info("Found " + str(len(jobs)) + " results for query.") jobdb.close() return jobs
def shortenActiniaCoreResp(fullResp): try: status = parseActiniaAsyncStatusResponse(fullResp, "status") message = parseActiniaAsyncStatusResponse(fullResp, "message") url = parseActiniaAsyncStatusResponse(fullResp, "urls.status") except Exception: return None log.info('Status is "' + status + " - " + message + '"') return json.loads('{"status": "' + url + '"}')
def update(uuid, utcnow): """ Method to update record in geonetwork """ connection = checkConnectionWithoutResponse('geonetwork') if connection is None: log.error('Not updating metadata for uuid ' + uuid) return None try: response = getRecordByUUID(uuid) doc = parseString(response.decode('utf-8')) recordNode = doc.getElementsByTagName('gmd:MD_Metadata')[0] log.debug('Found metadata to update for ' + uuid + ', and ' + str(recordNode)) except Exception: log.error('Could not find metadata record to update for uuid ' + uuid) return None record = updateXml(response, utcnow) if record is None: return None try: url = GEONETWORK.csw_pub postbodytpl = tplEnv.get_template('geonetwork/post_update_record.xml') postbody = postbodytpl.render(metadata_record=record, uuid=uuid).replace('\n', '') headers = {'content-type': 'application/xml; charset=utf-8'} except Exception as e: log.error('Could not set needed variable') log.error(e) return None try: log.info('Updating metadata record') gnosresp = requests.post(url, data=bytes(postbody, 'utf-8'), headers=headers, auth=auth(GEONETWORK)) if '<html>' in gnosresp.content.decode('utf-8'): log.error('update error') else: log.info('update success') return gnosresp except requests.exceptions.ConnectionError: log.error('Could not connect to gnos') return None except Exception as e: log.error(e) return None
def insertNewJob(rule_configuration, job_description, process, feature_type, actiniaCoreResp): """Insert new job into jobtabelle. Args: rule_configuration (dict): original preProcessChain job_description (TODO): enriched preProcessChain with geometadata Returns: record (dict): the new record """ utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') actiniaCoreJobUrl = parseActiniaAsyncStatusResponse( actiniaCoreResp, "urls.status") actiniaCoreJobID = parseActiniaIdFromUrl(actiniaCoreJobUrl) if current_app.debug is False: smallRes = dict() smallRes['message'] = actiniaCoreResp.get('message', None) smallRes['process_results'] = actiniaCoreResp.get( 'process_results', None) actiniaCoreResp = smallRes # actiniaCoreResp = smallifyResp(actiniaCoreResp) # TODO: test in debug, then remove # actiniaCoreResp = smallifyResp(actiniaCoreResp) job = Job( **{ 'rule_configuration': rule_configuration, 'job_description': job_description, 'status': 'PENDING', 'time_created': utcnow, 'process': process, 'feature_type': feature_type, 'actinia_core_response': actiniaCoreResp, 'actinia_core_jobid': actiniaCoreJobID }) with jobdb: job.save() queryResult = Job.select().where(Job.time_created == utcnow).get() record = model_to_dict(queryResult) log.info("Created new job with id " + str(record['idpk_jobs']) + ".") jobdb.close() return record
def create(filename): """ Method to get create metadata records in geonetwork """ url = GEONETWORK.csw_publication recordtpl = tplEnv.get_template('geonetwork/template_metadaten.xml') record = recordtpl.render(title=filename).replace('\n', '') # recordfs = recordtpl.render(title=filename) postbodytpl = tplEnv.get_template('geonetwork/post_create_record.xml') postbody = postbodytpl.render(metadata_record=record).replace('\n', '') # postbodyfs = postbodytpl.render(metadata_record=recordfs) headers = {'content-type': 'application/xml; charset=utf-8'} log.info('Creating metadata record') try: gnosresp = requests.post(url, data=postbody, headers=headers, auth=auth(GEONETWORK)) except requests.exceptions.ConnectionError: return None # if not os.path.isdir(FILEUPLOAD.templates): # os.makedirs(FILEUPLOAD.templates) # # with open(FILEUPLOAD.templates + '/' + filename + '_template.xml', # 'x') as file: # file.write(postbodyfs) # log.info('Received binary file, saved to ' # + FILEUPLOAD.templates + '/' + filename + '_template.xml') try: parsedresp = xmltodict.parse(gnosresp.content) insertRes = parsedresp['csw:TransactionResponse']['csw:InsertResult'] uuid = insertRes['csw:BriefRecord']['identifier'] log.info('GNOS response is: ' + str(parsedresp)) return uuid except Exception: return None
def cancelJobById(jobid): """ Method to change the status of a job to 'TERMINATED' in the jobtabelle by using its jobid Args: jobid (int): id of job Returns: record (dict): the record matching the id """ utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') try: with jobdb: queryResult = Job.select().where( getattr(Job, JOBTABLE.id_field) == jobid).get() record = model_to_dict(queryResult) log.debug("Information read from jobtable.") except Job.DoesNotExist: record = None try: query = Job.update( status='TERMINATED', time_ended=utcnow).where(getattr(Job, JOBTABLE.id_field) == jobid) with jobdb: query.execute() queryResult2 = Job.select().where( getattr(Job, JOBTABLE.id_field) == jobid).get() record = model_to_dict(queryResult2) except Exception as e: log.error('Could not set the status to "TERMINATED" ' + 'and the time_ended in the jobtable.') log.error(str(e)) record = None log.info("Information updated in jobtable for job with id " + str(record['idpk_jobs']) + ".") jobdb.close() return record
def postActiniaCore(process, preProcessChain): """ Method to start new job in actinia-core Args: process: (str): the process which is triggered preProcessChain: (dict): the enriched preProcessChain object Returns: TODO: (dict): status url of actinia-core job """ url = ACTINIACORE.url + PROCESSING_ENDPOINT headers = {'content-type': 'application/json; charset=utf-8'} if process == 'test': postbody = buildPCDummy(preProcessChain) elif process == 'sentinel1': postbody = buildPCS1Grd(preProcessChain) else: postbody = buildPCDummy(preProcessChain) try: # TODO: for now if error above, here an empty postbody is send to # actinia-core. This leads to correct status "ERROR" at the moment. # Make smarter (not call actinia-core, return correct error msg) log.info('Posting to ' + url) actiniaResp = requests.post(url, data=postbody, headers=headers, auth=auth(ACTINIACORE)) # TODO: remove GET request here. It is a workaround for a bug in # actinia-core which does not start a job if no request follows actiniaCoreId = json.loads(actiniaResp.text)['resource_id'] url = ACTINIACORE.url + "resources/actinia-gdi/" + actiniaCoreId actiniaResp2 = requests.get(url, auth=auth(ACTINIACORE)) except requests.exceptions.ConnectionError: return None return json.loads(actiniaResp2.text)
def post(self): """ Create new job This method is called by HTTP POST @app.route('/processes/test/jobs') This method is calling core method createJob """ log.info("\n Received HTTP POST with job:") print(request.get_json(force=True)) # We determine the process type here, depending on which endpoint # was called process = request.path.split('/')[2] job = createJob(request.get_json(force=True), process) if job is not None: return make_response(jsonify(job), 201) else: res = (jsonify(SimpleResponseModel(status=404, message='Error'))) return make_response(res, 404)
def get(self, jobid): """ Wrapper method to receive HTTP call and pass it to function This method is called by HTTP GET @app.route('/processes/test/jobs/<jobid>') This method is calling core method readJob """ if jobid is None: return make_response("Not found", 404) log.info("\n Received HTTP GET request for job with id " + str(jobid)) job = getJob(jobid) if job is not None: return make_response(jsonify(job), 200) else: res = (jsonify( SimpleResponseModel(status=404, message='Not Found: ' + request.url))) return make_response(res, 404)
def post(self): jsonFile = request.form.get('jsonFile') geodataFile = request.files.get('geodataFile') if not os.path.isdir(FILEUPLOAD.geodata): os.makedirs(FILEUPLOAD.geodata) # See file operations here: # https://www.programiz.com/python-programming/file-operation if geodataFile: # will be stored under original filename with uuid filename = (str(uuid.uuid4()) + '.' + secure_filename(geodataFile.filename)) geodataFile.save(FILEUPLOAD.geodata + '/' + filename) log.info('Received json file, saved to ' + FILEUPLOAD.geodata + '/' + filename) elif jsonFile: # will be stored under random name, e.g. # 5f82730c-f804-4e34-9c24-faaed902fab5.json filename = str(uuid.uuid4()) + '.json' with open(FILEUPLOAD.geodata + '/' + filename, 'x') as file: file.write(jsonFile) log.info('Received binary file, saved to ' + FILEUPLOAD.geodata + '/' + filename) else: log.error('No file found in postbody') metadatarecord = create(filename) res = jsonify(FileUploadResponseModel( status=200, message="Upload success", name=filename, record=metadatarecord )) res.headers['Access-Control-Allow-Origin'] = '*' return make_response(res, 200)
def post(self, jobid): """ Wrapper method to cancel a job by using the jobId This method is called by HTTP POST @app.route('/processes/test/jobs/<jobid>/operations/cancel') This method is calling core method readJob """ if jobid is None: return make_response("Not found", 404) log.info("\n Received HTTP POST request for job with id " + str(jobid) + "to cancel the job") job = cancelJob(jobid) if job is not None: return make_response(jsonify(job), 200) else: res = (jsonify( SimpleResponseModel(status=404, message='Not Found: ' + request.url))) return make_response(res, 404)
def getJobById(jobid): """ Method to read job from jobtabelle by id Args: jobid (int): id of job Returns: record (dict): the record matching the id """ try: with jobdb: queryResult = Job.select().where( getattr(Job, JOBTABLE.id_field) == jobid).get() record = model_to_dict(queryResult) log.info("Information read from jobtable for job with id " + str(record['idpk_jobs']) + ".") except Job.DoesNotExist: record = None jobdb.close() return record
def cancelActiniaCore(resourceId): """ Wrapper method to cancel a job by using the jobId This method is called by HTTP POST @app.route('/processes/test/jobs/<jobid>/operations/cancel') This method is calling core method readJob Args: resourceId: (string): actinia-core job id """ if resourceId is None: return make_response("Not found", 404) print("\n Received HTTP DELETE request for job with actinia-core jobID " + resourceId) DELETING_ENDPOINT = 'resources/' url = (ACTINIACORE.url + DELETING_ENDPOINT + ACTINIACORE.user + '/' + resourceId) log.info('Requesting from ' + url) try: actiniaResp = requests.delete(url, auth=auth(ACTINIACORE)) except requests.exceptions.ConnectionError: return None try: status = parseActiniaAsyncStatusResponse(json.loads(actiniaResp.text), "status") message = parseActiniaAsyncStatusResponse(json.loads(actiniaResp.text), "message") except Exception: return None log.info('Status is "' + status + " - " + message + '"') return True
def parseMeta(record): """ Method to parse record from geonetwork with model TODO: support more than one tag in response TODO: better error handling when attribute not found This method can handle GetRecordByIdResponse and GetRecordsResponse """ if 'csw:GetRecordByIdResponse' in record: log.info("Found 1 record") if 'csw:Record' in json.loads(record)["csw:GetRecordByIdResponse"]: recordRoot = ( json.loads(record)["csw:GetRecordByIdResponse"]["csw:Record"] ) else: log.info("...But record is empty") return elif 'csw:GetRecordsResponse' in record: if 'csw:SearchResults' in json.loads(record)["csw:GetRecordsResponse"]: searchResults = ( json.loads(record)["csw:GetRecordsResponse"]["csw:SearchResults"] ) else: log.info("...But record is empty") return numberOfRecords = int(searchResults["@numberOfRecordsReturned"]) recordRoot = dict() if numberOfRecords == 0: log.warning("No records found") return elif numberOfRecords == 1: log.info("Found 1 record") recordRoot = searchResults["csw:Record"] else: log.info("Found " + str(numberOfRecords) + " records") recordRoot = searchResults["csw:Record"][0] else: print("Could not parse GNOS response") return if 'dc:identifier' in recordRoot: uuid = recordRoot["dc:identifier"] else: uuid = 'null' if 'ows:BoundingBox' in recordRoot: recordBbox = recordRoot["ows:BoundingBox"] if 'ows:LowerCorner' in recordBbox: bbox_lower = recordBbox["ows:LowerCorner"] if 'ows:UpperCorner' in recordBbox: bbox_upper = recordBbox["ows:UpperCorner"] bbox_a = float(bbox_lower.split(" ")[0]) bbox_b = float(bbox_lower.split(" ")[1]) bbox_c = float(bbox_upper.split(" ")[0]) bbox_d = float(bbox_upper.split(" ")[1]) bbox = [bbox_a, bbox_b, bbox_c, bbox_d] if '@crs' in recordBbox: crs = recordBbox["@crs"] else: bbox = [] crs = 'null' if 'dc:URI' in recordRoot: recordUri = recordRoot["dc:URI"] if type(recordUri) is dict: if "#text" in recordUri: table = recordUri["#text"] else: table = recordUri[0]["#text"] else: table = 'null' geodata_meta = GeodataMeta( uuid=uuid, bbox=bbox, crs=crs, table=table ) return geodata_meta
def updateJobByResourceID(resourceId, resp, status): """ Method to update job in jobtabelle when processing status changed Args: resourceId (str): actinia-core resourceId resp (dict): actinia-core response status (string): actinia-core processing status Returns: updatedRecord (TODO): the updated record """ utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') try: with jobdb: queryResult = Job.select().where( getattr(Job, 'actinia_core_jobid') == resourceId).get() record = model_to_dict(queryResult) log.debug("Information read from jobtable for job with id " + str(record['idpk_jobs']) + ".") except Job.DoesNotExist: log.warning("Job does not exist and can therefore not be updated") return None, None, None # actinia-gdi_["PENDING", "RUNNING", "SUCCEES", "ERROR", "TERMINATED"] # actinia-core [accepted, running, finished, error, terminated] try: log.debug("Update status to " + status + " for job with id " + str(record['idpk_jobs']) + ".") gdiStatus = record['status'] try: gnosUuid = record['job_description']['feature_uuid'] except Exception: log.warning('Feature has no uuid') gnosUuid = None if current_app.debug is False: smallRes = dict() smallRes['message'] = resp.get('message', None) smallRes['process_results'] = resp.get('process_results', None) resp = smallRes # resp = smallifyResp(resp) # TODO: test in debug, then remove # resp = smallifyResp(resp) if status == 'accepted': log.debug('Status already set to "PENDING"') return record, gnosUuid, utcnow elif status == 'running': if gdiStatus == 'RUNNING': log.debug('Status already set to "RUNNING"') return record, gnosUuid, utcnow query = Job.update( status='RUNNING', actinia_core_response=resp, time_started=utcnow # TODO: check if time_estimated can be set # time_estimated= ).where(getattr(Job, 'actinia_core_jobid') == resourceId) elif status in ['finished', 'error', 'terminated']: if status == 'finished': gdiStatus = 'SUCCESS' elif status == 'error': gdiStatus = 'ERROR' elif status == 'terminated': gdiStatus = 'TERMINATED' query = Job.update( status=gdiStatus, actinia_core_response=resp, time_ended=utcnow).where( getattr(Job, 'actinia_core_jobid') == resourceId) else: log.error('Could not set the status to actinia-core status:' + status + '(Status not found.)') return None, None, None with jobdb: query.execute() queryResult = Job.select().where( getattr(Job, 'actinia_core_jobid') == resourceId).get() record = model_to_dict(queryResult) except Exception as e: log.error('Could not set the status to actinia-core status: ' + status) log.error(str(e)) return None, None, None log.info("Updated status to " + status + " for job with id " + str(record['idpk_jobs']) + ".") jobdb.close() return record, gnosUuid, utcnow