def checkConnection(name): """ Method to test connection Args: name (string): resource to test. Can be 'actinia-core' or 'geonetwork' Returns: response (Response): of type SimpleResponseModel telling connection success or failure """ if name == 'actinia-core': url = ACTINIACORE.url + "version" name = 'actinia-core' type = 'json' elif name == 'geonetwork': url = GEONETWORK.csw_url name = 'geonetwork' type = 'xml' try: records = common.checkConnection(url, name, type) except Exception: log.error("Don't know which connection to test") if records is not None: res = jsonify(SimpleResponseModel(status=200, message="success")) return make_response(res, 200) elif records is None: res = jsonify(SimpleResponseModel(status=404, message="failure")) return make_response(res, 200)
def makeItXml(jsonDict): try: records = xmltodict.unparse(jsonDict) return records except Exception: log.error('Error converting json back to xml') return None
def buildLoop(preProcessChain): webhookUrl = APP.url + "/resources/processes/operations/update" log.debug("Registering own webhook endpoint: " + str(webhookUrl)) tpl = tplEnv.get_template('actiniaCore/pc_loop.json') class PCInputClass(): param = '' value = '' pcInputs = [] # Feature: transform input table information to grass connection string # try: # preProcessChainTableFeat = preProcessChain.feature_source.table # except AttributeError as e: # log.error(e) # log.error("Feature has no data source") # return # # if preProcessChainTableFeat is None: # log.error("Feature has no data source") # return # # connString = buildPCConnectionString(preProcessChainTableFeat) # feat_db = connString[0] # feat_layer = connString[1] proc = preProcessChain.procs[0] for input in proc.input: inputType = getattr(input, "type", None) if inputType == 'PARAMETER': pcInputObject = PCInputClass() pcInputObject.param = input.name path = ACTINIACORE.filestorage + '/' + input.value[0] pcInputObject.value = path pcInputs.append(pcInputObject) elif input.table: if input.name in ['a', 'b', 'c']: pcInputObject = PCInputClass() pcInputObject.param = input.name pcInputObject.value = input.value pcInputs.append(pcInputObject) else: log.error("Don't know what to do with input.") if (pcInputs is None or webhookUrl is None): log.error('Could not set all variables to replace in template.') return None postbody = tpl.render(inputs=pcInputs, webhookUrl=webhookUrl).replace('\n', '') return postbody
def checkConnection(url, name, expectedFormat): """ Method to test connection Args: url (string): url of resource to test. name (string): name of resource to test. Only used for logging expectedFormat (string): Format in which resource will respond. Can be 'xml' or 'json' """ # can be called by e.g. # checkConnection(GEONETWORK.csw_url, 'geonetwork', 'xml') log.debug('Testing connection to ' + url) try: resp = requests.get(url) except requests.exceptions.ConnectionError: log.error('Connection Error to ' + name) return None try: if expectedFormat == 'json': parsedresp = json.loads(resp.text) log.debug('Connection successfull to ' + name) return True except Exception: log.error('Connection Error to ' + name) return None
def buildPCS1Grd(preProcessChain): processType = preProcessChain.get('processType') if processType != 'preprocessing': log.error('process type is unknown') return None webhookUrl = APP.url + "/resources/processes/operations/update" log.debug("Registering own webhook endpoint: " + str(webhookUrl)) tpl = tplEnv.get_template('actiniaCore/pc_r.s1.grd_template.json') user = ACTINIACORE.esa_apihub_user pw = ACTINIACORE.esa_apihub_pw S1A_name = preProcessChain.get('title') raw_path = ACTINIACORE.filestorage + '/' + 'sentinel1/raw/' preprocessing_path = (ACTINIACORE.filestorage + '/' + 'sentinel1/preprocessing/') if (user is None or pw is None or S1A_name is None or raw_path is None or preprocessing_path is None or webhookUrl is None): log.error('Could not set all variables to replace in template.') return None postbody = tpl.render(user=user, pw=pw, S1A_name=S1A_name, raw_path=raw_path, preprocessing_path=preprocessing_path, webhookUrl=webhookUrl).replace('\n', '') return postbody
def createProcessChainTemplateList(): ''' list all stored templates and return as actinia-module list ''' pc_list = [] tpl_list = pcTplEnv.list_templates(filter_func=filter_func) for tpl_string in tpl_list: tpl = pcTplEnv.get_template(tpl_string) try: pc_template = json.loads(tpl.render().replace('\n', '')) except: log.error('Error parsing template ' + tpl_string) tpl_id = pc_template['id'] description = pc_template['description'] categories = ['actinia-module'] pc_response = (Module(id=tpl_id, description=description, categories=categories)) pc_list.append(pc_response) return pc_list
def getAllJobs(filters, process): """ Method to read all jobs from jobtabelle with filter Args: filters (ImmutableMultiDict): the args from the HTTP call Returns: jobs (list): the records matching the filter """ log.debug('Received query for jobs') if process == 'test': query = Expression('a', '=', 'a') else: query = Expression(getattr(Job, 'process'), '=', process) if filters: log.debug("Found filters: " + str(filters)) keys = [key for key in filters] for key in keys: try: getattr(Job, key) except Exception as e: log.warning(str(e)) continue log.debug("Filter " + str(key) + " with value " + str(filters[key])) if isinstance(getattr(Job, key), AutoField): try: int(filters[key]) except Exception as e: log.error(str(e)) jobdb.close() return try: # even though operators are listed as == and & in peewee docs, # for Expression creation use '=' and 'AND'. exp = Expression(getattr(Job, key), '=', filters[key]) query = Expression(query, 'AND', exp) except AttributeError as e: log.error(str(e)) with jobdb: queryResult = Job.select().where(query).dicts() jobs = [] # iterating reopens db connection!! for i in queryResult: jobs.append(i) log.info("Found " + str(len(jobs)) + " results for query.") jobdb.close() return jobs
def makeItJson(xml): try: parsedresp = xmltodict.parse(xml) records = json.dumps(parsedresp) return records except Exception: log.error('Error parsing XML response from gnos') return None
def get(self, uuid): try: utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') gnosresp = update(uuid, utcnow) if gnosresp is None: raise Exception res = make_response(gnosresp.content, 200) return res except Exception as e: log.error('error parsing gnos response') log.error(e) res = (jsonify( SimpleResponseModel(status=404, message='Error looking for uuid "' + uuid + '".'))) return make_response(res, 404)
def fillTemplateFromProcessChain(module): """ This method receives a process chain for an actinia module and loads the according process chain template. The received values will be replaced to be passed to actinia. In case the template has more placeholder values than it receives, the missing attribute is returned as string. """ kwargs = {} inOrOutputs = [] if module.get('inputs') is not None: inOrOutputs += module.get('inputs') if module.get('outputs') is not None: inOrOutputs += module.get('outputs') for item in inOrOutputs: if (item.get('param') is None) or (item.get('value') is None): return None key = item['param'] val = item['value'] kwargs[key] = val tpl_file = module["module"] + '.json' # change path to template if in subdir for i in pcTplEnv.list_templates(filter_func=filter_func): if i.split('/')[-1] == tpl_file: tpl_file = i # find variables from processchain tpl_source = pcTplEnv.loader.get_source(pcTplEnv, tpl_file)[0] parsed_content = pcTplEnv.parse(tpl_source) undef = meta.find_undeclared_variables(parsed_content) for i in undef: if i not in kwargs.keys(): log.error('Required parameter "' + i + '" not in process chain!') return i # fill process chain template tpl = pcTplEnv.get_template(tpl_file) pc_template = json.loads(tpl.render(**kwargs).replace('\n', '')) return (pc_template['template']['list'])
def createJob(jsonDict, process): """ Method to parse prePC including fetching information from geonetwork and writing information to Jobtable as well as starting job in actinia-core This method can be called by HTTP POST @app.route('/processes/test/jobs') """ prePC_orig = json.dumps(jsonDict) # TODO: define prePC (pre processchain) model if differs from pc # prePC = prePC(**jsonDict) # as we don't hava a model yet # prePC = json.dumps(jsonDict) connection = checkConnectionWithoutResponse('actinia-core') if connection is not None: actiniaCoreResp = postActiniaCore(process, jsonDict) log.debug(actiniaCoreResp) # try: # prePCDict = prePC.to_struct() # except Exception as e: # log.error('prePC is invalid!') # log.error(e) # return None job = insertNewJob( jsonDict, jsonDict, # as we don't hava a model yet process, jsonDict.get( 'feature_type'), # empty at the moment (polygon later) actiniaCoreResp) if actiniaCoreResp['status'] == 'error': log.error("Error start processing in actinia-core") resourceId = parseActiniaIdFromUrl(actiniaCoreResp['resource_id']) job = updateJob(resourceId, actiniaCoreResp) return job else: return None
def buildPCDummy(preProcessChain): webhookUrl = APP.url + "/resources/processes/operations/update" log.debug("Registering own webhook endpoint: " + str(webhookUrl)) tpl = tplEnv.get_template('actiniaCore/pc_point_in_polygon.json') point = preProcessChain.get('point') polygon = preProcessChain.get('polygon') if polygon is None or point is None or webhookUrl is None: log.error('Could not set all variables to replace in template.') return None postbody = tpl.render(point=point, polygon=polygon, webhookUrl=webhookUrl).replace('\n', '') return postbody
def checkConnectionWithoutResponse(name): """ Method to test connection Args: name (string): resource to test. Can be 'actinia-core' or 'geonetwork' """ if name == 'actinia-core': url = ACTINIACORE.url + "version" name = 'actinia-core' type = 'json' try: connectionTest = common.checkConnection(url, name, type) return connectionTest except Exception: log.error("Don't know which connection to test") return None
def get(self, category): try: records = getRecordsByCategory(category) res = make_response(records, 200) res.headers['Content-Type'] = 'application/json' return res # except TemplateNotFound as e: # print('ERROR: ' + repr(e) + " - " + e.message) # return make_response('Error looking for category "' + category + # '".', 404) except Exception as e: log.error('ERROR: ' + repr(e) + " - " + str(e)) res = (jsonify( SimpleResponseModel(status=404, message='Category "' + category + '" not found.'))) return make_response(res, 404)
def cancelJobById(jobid): """ Method to change the status of a job to 'TERMINATED' in the jobtabelle by using its jobid Args: jobid (int): id of job Returns: record (dict): the record matching the id """ utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') try: with jobdb: queryResult = Job.select().where( getattr(Job, JOBTABLE.id_field) == jobid).get() record = model_to_dict(queryResult) log.debug("Information read from jobtable.") except Job.DoesNotExist: record = None try: query = Job.update( status='TERMINATED', time_ended=utcnow).where(getattr(Job, JOBTABLE.id_field) == jobid) with jobdb: query.execute() queryResult2 = Job.select().where( getattr(Job, JOBTABLE.id_field) == jobid).get() record = model_to_dict(queryResult2) except Exception as e: log.error('Could not set the status to "TERMINATED" ' + 'and the time_ended in the jobtable.') log.error(str(e)) record = None log.info("Information updated in jobtable for job with id " + str(record['idpk_jobs']) + ".") jobdb.close() return record
def getRecordsByTags(tags): """ Method to get records by tags from geonetwork Attention: The tags received are case sensitive! PropertyIsLike is sensitive, matchCase not possible here. See https://trac.osgeo.org/geonetwork/wiki/CSW202Improvements This method can be called by @app.route('/metadata/raw/tags/<tags>') """ log.debug('looking for tags ' + str(tags)) try: url = GEONETWORK.csw_url tpl = tplEnv.get_template('geonetwork/post_records_by_tags.xml') tags = tags.split(',') postbody = tpl.render(tags=tags).replace('\n', '') headers = {'content-type': 'application/xml; charset=utf-8'} except Exception as e: log.error('Could not set needed variable') log.error(e) return None try: gnosresp = requests.post( url, data=postbody, headers=headers, auth=auth(GEONETWORK) ) return gnosresp.content except requests.exceptions.ConnectionError: log.error('Could not connect to gnos') return None
def updateXml(response, utcnow): try: doc = parseString(response.decode('utf-8')) recordNode = doc.getElementsByTagName('gmd:MD_Metadata')[0] dateStampEl = recordNode.getElementsByTagName('gmd:dateStamp')[0] dateEl = dateStampEl.getElementsByTagName('gco:Date') if len(dateEl) == 0: dateEl = dateStampEl.getElementsByTagName('gco:DateTime') if len(dateEl) == 0: log.error('Could not find date element') return None else: dateEl = dateEl[0] else: dateEl = dateEl[0] utcnow = utcnow.split('T')[0] if dateEl.firstChild.nodeType != dateEl.TEXT_NODE: raise Exception("node does not contain text") dateEl.firstChild.replaceWholeText(utcnow) record = recordNode.toxml().replace('\n', '') except Exception as e: log.error('Could not set date in metadata record') log.error(e) return None return record
def post(self): jsonFile = request.form.get('jsonFile') geodataFile = request.files.get('geodataFile') if not os.path.isdir(FILEUPLOAD.geodata): os.makedirs(FILEUPLOAD.geodata) # See file operations here: # https://www.programiz.com/python-programming/file-operation if geodataFile: # will be stored under original filename with uuid filename = (str(uuid.uuid4()) + '.' + secure_filename(geodataFile.filename)) geodataFile.save(FILEUPLOAD.geodata + '/' + filename) log.info('Received json file, saved to ' + FILEUPLOAD.geodata + '/' + filename) elif jsonFile: # will be stored under random name, e.g. # 5f82730c-f804-4e34-9c24-faaed902fab5.json filename = str(uuid.uuid4()) + '.json' with open(FILEUPLOAD.geodata + '/' + filename, 'x') as file: file.write(jsonFile) log.info('Received binary file, saved to ' + FILEUPLOAD.geodata + '/' + filename) else: log.error('No file found in postbody') metadatarecord = create(filename) res = jsonify(FileUploadResponseModel( status=200, message="Upload success", name=filename, record=metadatarecord )) res.headers['Access-Control-Allow-Origin'] = '*' return make_response(res, 200)
def getRecordByUUID(uuid): """ Method to get record by uuid from geonetwork This method can be called by @app.route('/metadata/raw/uuids/<uuid>') """ try: url = GEONETWORK.csw_url tpl = tplEnv.get_template('geonetwork/get_record_by_uuid_kvp.json') kvps = json.loads(tpl.render(uuid=uuid)) except Exception as e: log.error('Could not set needed variable') log.error(e) return None try: gnosresp = requests.get(url, params=kvps, auth=auth(GEONETWORK)) return gnosresp.content except requests.exceptions.ConnectionError: log.error('Could not connect to gnos') return None
def cancelJob(jobid): """ Method to cancel job from Jobtable by id This method can be called by HTTP POST @app.route('/processes/test/jobs/<jobid>/operations/cancel') """ job = getJobById(jobid) if not job == None: log.debug('The job with jobid ' + str(jobid) + ' exists') status = job['status'] resourceId = job['actinia_core_jobid'] if not status or not resourceId: log.error('Job status or resourceId is not set!') return None else: log.debug('Job status is ' + status + ' and resourceId is: ' + resourceId) connection = checkConnectionWithoutResponse('actinia-core') if connection is not None: if status in ['PENDING', 'RUNNING']: log.debug('Status is in PENDING or RUNNING, will cancel') res = cancelActiniaCore(resourceId) if res: log.debug('Actinia-Core response TRUE') job = cancelJobById(jobid) log.debug('Job in jobtable is ' + job['status']) return job else: log.debug('Actinia-Core response is None') return None else: log.debug('Status not in PENDING or RUNNING, pass') return job else: log.error('There is no connection to actinia-core') return None else: log.error('The job with jobid ' + str(jobid) + 'does not exist') return None
def getRecordsByCategory(category): """ Method to get records by category from geonetwork This method can be called by @app.route('/metadata/raw/categories/<category>') """ try: url = (GEONETWORK.csw_url + '-' + category) tpl = tplEnv.get_template('geonetwork/get_records_by_category_kvp.json') kvps = json.loads(tpl.render()) except Exception as e: log.error('Could not set needed variable') log.error(e) return None try: gnosresp = requests.get(url, params=kvps, auth=auth(GEONETWORK)) return gnosresp.content except requests.exceptions.ConnectionError: log.error('Could not connect to gnos') return None
def updateJobByResourceID(resourceId, resp, status): """ Method to update job in jobtabelle when processing status changed Args: resourceId (str): actinia-core resourceId resp (dict): actinia-core response status (string): actinia-core processing status Returns: updatedRecord (TODO): the updated record """ utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') try: with jobdb: queryResult = Job.select().where( getattr(Job, 'actinia_core_jobid') == resourceId).get() record = model_to_dict(queryResult) log.debug("Information read from jobtable for job with id " + str(record['idpk_jobs']) + ".") except Job.DoesNotExist: log.warning("Job does not exist and can therefore not be updated") return None, None, None # actinia-gdi_["PENDING", "RUNNING", "SUCCEES", "ERROR", "TERMINATED"] # actinia-core [accepted, running, finished, error, terminated] try: log.debug("Update status to " + status + " for job with id " + str(record['idpk_jobs']) + ".") gdiStatus = record['status'] try: gnosUuid = record['job_description']['feature_uuid'] except Exception: log.warning('Feature has no uuid') gnosUuid = None if current_app.debug is False: smallRes = dict() smallRes['message'] = resp.get('message', None) smallRes['process_results'] = resp.get('process_results', None) resp = smallRes # resp = smallifyResp(resp) # TODO: test in debug, then remove # resp = smallifyResp(resp) if status == 'accepted': log.debug('Status already set to "PENDING"') return record, gnosUuid, utcnow elif status == 'running': if gdiStatus == 'RUNNING': log.debug('Status already set to "RUNNING"') return record, gnosUuid, utcnow query = Job.update( status='RUNNING', actinia_core_response=resp, time_started=utcnow # TODO: check if time_estimated can be set # time_estimated= ).where(getattr(Job, 'actinia_core_jobid') == resourceId) elif status in ['finished', 'error', 'terminated']: if status == 'finished': gdiStatus = 'SUCCESS' elif status == 'error': gdiStatus = 'ERROR' elif status == 'terminated': gdiStatus = 'TERMINATED' query = Job.update( status=gdiStatus, actinia_core_response=resp, time_ended=utcnow).where( getattr(Job, 'actinia_core_jobid') == resourceId) else: log.error('Could not set the status to actinia-core status:' + status + '(Status not found.)') return None, None, None with jobdb: query.execute() queryResult = Job.select().where( getattr(Job, 'actinia_core_jobid') == resourceId).get() record = model_to_dict(queryResult) except Exception as e: log.error('Could not set the status to actinia-core status: ' + status) log.error(str(e)) return None, None, None log.info("Updated status to " + status + " for job with id " + str(record['idpk_jobs']) + ".") jobdb.close() return record, gnosUuid, utcnow
def update(uuid, utcnow): """ Method to update record in geonetwork """ connection = checkConnectionWithoutResponse('geonetwork') if connection is None: log.error('Not updating metadata for uuid ' + uuid) return None try: response = getRecordByUUID(uuid) doc = parseString(response.decode('utf-8')) recordNode = doc.getElementsByTagName('gmd:MD_Metadata')[0] log.debug('Found metadata to update for ' + uuid + ', and ' + str(recordNode)) except Exception: log.error('Could not find metadata record to update for uuid ' + uuid) return None record = updateXml(response, utcnow) if record is None: return None try: url = GEONETWORK.csw_pub postbodytpl = tplEnv.get_template('geonetwork/post_update_record.xml') postbody = postbodytpl.render(metadata_record=record, uuid=uuid).replace('\n', '') headers = {'content-type': 'application/xml; charset=utf-8'} except Exception as e: log.error('Could not set needed variable') log.error(e) return None try: log.info('Updating metadata record') gnosresp = requests.post(url, data=bytes(postbody, 'utf-8'), headers=headers, auth=auth(GEONETWORK)) if '<html>' in gnosresp.content.decode('utf-8'): log.error('update error') else: log.info('update success') return gnosresp except requests.exceptions.ConnectionError: log.error('Could not connect to gnos') return None except Exception as e: log.error(e) return None