def getRecordRoot(record): """ Method to get one record out of the geonetwork response. Valid for both schemata 'gmd:MD_Metadata' and 'csw:Record' TODO: support more than one tag in response This method can handle GetRecordByIdResponse and GetRecordsResponse """ record = json.loads(record) if 'csw:GetRecordByIdResponse' in record: log.info("Found 1 record") if 'gmd:MD_Metadata' in record["csw:GetRecordByIdResponse"]: recordRoot = ( record["csw:GetRecordByIdResponse"]["gmd:MD_Metadata"] ) elif 'csw:Record' in record["csw:GetRecordByIdResponse"]: recordRoot = ( record["csw:GetRecordByIdResponse"]["csw:Record"] ) else: log.info("...But record is empty") return elif 'csw:GetRecordsResponse' in record: if 'csw:SearchResults' in record["csw:GetRecordsResponse"]: searchResults = ( record["csw:GetRecordsResponse"]["csw:SearchResults"] ) else: log.info("...But record is empty") return numberOfRecords = int(searchResults["@numberOfRecordsReturned"]) recordRoot = dict() if numberOfRecords == 0: log.warning("No records found") return elif numberOfRecords == 1: log.info("Found 1 record") recordRoot = searchResults["gmd:MD_Metadata"] if not recordRoot: recordRoot = searchResults["csw:Record"] else: log.info("Found " + str(numberOfRecords) + " records") recordRoot = searchResults["gmd:MD_Metadata"][0] if not recordRoot: recordRoot = searchResults["csw:Record"][0] else: print("Could not parse GNOS response") return return recordRoot
def getAllJobs(filters, process): """ Method to read all jobs from jobtabelle with filter Args: filters (ImmutableMultiDict): the args from the HTTP call Returns: jobs (list): the records matching the filter """ log.debug('Received query for jobs') if process == 'test': query = Expression('a', '=', 'a') else: query = Expression(getattr(Job, 'process'), '=', process) if filters: log.debug("Found filters: " + str(filters)) keys = [key for key in filters] for key in keys: try: getattr(Job, key) except Exception as e: log.warning(str(e)) continue log.debug("Filter " + str(key) + " with value " + str(filters[key])) if isinstance(getattr(Job, key), AutoField): try: int(filters[key]) except Exception as e: log.error(str(e)) jobdb.close() return try: # even though operators are listed as == and & in peewee docs, # for Expression creation use '=' and 'AND'. exp = Expression(getattr(Job, key), '=', filters[key]) query = Expression(query, 'AND', exp) except AttributeError as e: log.error(str(e)) with jobdb: queryResult = Job.select().where(query).dicts() jobs = [] # iterating reopens db connection!! for i in queryResult: jobs.append(i) log.info("Found " + str(len(jobs)) + " results for query.") jobdb.close() return jobs
def getUuidByRecordRoot(recordRoot): try: uuid = recordRoot["gmd:fileIdentifier"]["gco:CharacterString"] if not uuid: uuid = recordRoot["dc:identifier"] except Exception: log.warning('Could not set uuid') uuid = 'null' return uuid
def getBboxByRecordRoot(recordRoot): try: recordExtent = ( recordRoot["gmd:identificationInfo"] ["gmd:MD_DataIdentification"]["gmd:extent"] ) def parseExtent(bboxRoot): bbox_a = bboxRoot["gmd:eastBoundLongitude"]["gco:Decimal"] bbox_b = bboxRoot["gmd:southBoundLatitude"]["gco:Decimal"] bbox_c = bboxRoot["gmd:westBoundLongitude"]["gco:Decimal"] bbox_d = bboxRoot["gmd:northBoundLatitude"]["gco:Decimal"] bbox = [ float(bbox_a), float(bbox_b), float(bbox_c), float(bbox_d) ] return bbox if type(recordExtent) is list: for i in recordExtent: if 'gmd:geographicElement' in i["gmd:EX_Extent"]: geoEl = i["gmd:EX_Extent"]["gmd:geographicElement"] if type(geoEl) is list: bbox = [] for i in geoEl: if 'gmd:EX_GeographicBoundingBox' in i: bboxRoot = i["gmd:EX_GeographicBoundingBox"] bbox = parseExtent(bboxRoot) else: bboxRoot = geoEl["gmd:EX_GeographicBoundingBox"] bbox = parseExtent(bboxRoot) else: bbox = [] except Exception: log.warning('Could not set bbox') bbox = [] return bbox
def updateJob(resourceId, actiniaCoreResp): """ Method to update job in Jobtable This method is called by webhook endpoint """ status = actiniaCoreResp['status'] job, uuid, utcnow = updateJobByResourceID(resourceId, actiniaCoreResp, status) # TODO: for now if multiple records need to be updated, this # can be told by specifying multiple uuids comma-separated in the # "feature_uuid" field of the preprocesschain. This might change later... if status == 'finished': try: uuids = uuid.split(',') for uuid in uuids: update(uuid, utcnow) except Exception: log.warning('Could not update geonetwork record') return job
def updateJobByResourceID(resourceId, resp, status): """ Method to update job in jobtabelle when processing status changed Args: resourceId (str): actinia-core resourceId resp (dict): actinia-core response status (string): actinia-core processing status Returns: updatedRecord (TODO): the updated record """ utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') try: with jobdb: queryResult = Job.select().where( getattr(Job, 'actinia_core_jobid') == resourceId).get() record = model_to_dict(queryResult) log.debug("Information read from jobtable for job with id " + str(record['idpk_jobs']) + ".") except Job.DoesNotExist: log.warning("Job does not exist and can therefore not be updated") return None, None, None # actinia-gdi_["PENDING", "RUNNING", "SUCCEES", "ERROR", "TERMINATED"] # actinia-core [accepted, running, finished, error, terminated] try: log.debug("Update status to " + status + " for job with id " + str(record['idpk_jobs']) + ".") gdiStatus = record['status'] try: gnosUuid = record['job_description']['feature_uuid'] except Exception: log.warning('Feature has no uuid') gnosUuid = None if current_app.debug is False: smallRes = dict() smallRes['message'] = resp.get('message', None) smallRes['process_results'] = resp.get('process_results', None) resp = smallRes # resp = smallifyResp(resp) # TODO: test in debug, then remove # resp = smallifyResp(resp) if status == 'accepted': log.debug('Status already set to "PENDING"') return record, gnosUuid, utcnow elif status == 'running': if gdiStatus == 'RUNNING': log.debug('Status already set to "RUNNING"') return record, gnosUuid, utcnow query = Job.update( status='RUNNING', actinia_core_response=resp, time_started=utcnow # TODO: check if time_estimated can be set # time_estimated= ).where(getattr(Job, 'actinia_core_jobid') == resourceId) elif status in ['finished', 'error', 'terminated']: if status == 'finished': gdiStatus = 'SUCCESS' elif status == 'error': gdiStatus = 'ERROR' elif status == 'terminated': gdiStatus = 'TERMINATED' query = Job.update( status=gdiStatus, actinia_core_response=resp, time_ended=utcnow).where( getattr(Job, 'actinia_core_jobid') == resourceId) else: log.error('Could not set the status to actinia-core status:' + status + '(Status not found.)') return None, None, None with jobdb: query.execute() queryResult = Job.select().where( getattr(Job, 'actinia_core_jobid') == resourceId).get() record = model_to_dict(queryResult) except Exception as e: log.error('Could not set the status to actinia-core status: ' + status) log.error(str(e)) return None, None, None log.info("Updated status to " + status + " for job with id " + str(record['idpk_jobs']) + ".") jobdb.close() return record, gnosUuid, utcnow
def parseMeta(record): """ Method to parse record from geonetwork with model TODO: support more than one tag in response TODO: better error handling when attribute not found This method can handle GetRecordByIdResponse and GetRecordsResponse """ if 'csw:GetRecordByIdResponse' in record: log.info("Found 1 record") if 'csw:Record' in json.loads(record)["csw:GetRecordByIdResponse"]: recordRoot = ( json.loads(record)["csw:GetRecordByIdResponse"]["csw:Record"] ) else: log.info("...But record is empty") return elif 'csw:GetRecordsResponse' in record: if 'csw:SearchResults' in json.loads(record)["csw:GetRecordsResponse"]: searchResults = ( json.loads(record)["csw:GetRecordsResponse"]["csw:SearchResults"] ) else: log.info("...But record is empty") return numberOfRecords = int(searchResults["@numberOfRecordsReturned"]) recordRoot = dict() if numberOfRecords == 0: log.warning("No records found") return elif numberOfRecords == 1: log.info("Found 1 record") recordRoot = searchResults["csw:Record"] else: log.info("Found " + str(numberOfRecords) + " records") recordRoot = searchResults["csw:Record"][0] else: print("Could not parse GNOS response") return if 'dc:identifier' in recordRoot: uuid = recordRoot["dc:identifier"] else: uuid = 'null' if 'ows:BoundingBox' in recordRoot: recordBbox = recordRoot["ows:BoundingBox"] if 'ows:LowerCorner' in recordBbox: bbox_lower = recordBbox["ows:LowerCorner"] if 'ows:UpperCorner' in recordBbox: bbox_upper = recordBbox["ows:UpperCorner"] bbox_a = float(bbox_lower.split(" ")[0]) bbox_b = float(bbox_lower.split(" ")[1]) bbox_c = float(bbox_upper.split(" ")[0]) bbox_d = float(bbox_upper.split(" ")[1]) bbox = [bbox_a, bbox_b, bbox_c, bbox_d] if '@crs' in recordBbox: crs = recordBbox["@crs"] else: bbox = [] crs = 'null' if 'dc:URI' in recordRoot: recordUri = recordRoot["dc:URI"] if type(recordUri) is dict: if "#text" in recordUri: table = recordUri["#text"] else: table = recordUri[0]["#text"] else: table = 'null' geodata_meta = GeodataMeta( uuid=uuid, bbox=bbox, crs=crs, table=table ) return geodata_meta
def parseMeta(recordXml): """ Method to parse record from geonetwork with schema 'gmd' with model This method can handle GetRecordByIdResponse and GetRecordsResponse """ record = makeItJson(recordXml) if record is None: return None recordRoot = getRecordRoot(record) uuid = getUuidByRecordRoot(recordRoot) bbox = getBboxByRecordRoot(recordRoot) try: table = ( recordRoot["gmd:distributionInfo"]["gmd:MD_Distribution"] ["gmd:transferOptions"]["gmd:MD_DigitalTransferOptions"] ["gmd:onLine"]["gmd:CI_OnlineResource"]["gmd:linkage"]["gmd:URL"] ) except Exception: log.warning('Could not set table') table = 'null' try: format = ( recordRoot["gmd:distributionInfo"]["gmd:MD_Distribution"] ["gmd:distributionFormat"]["gmd:MD_Format"]["gmd:name"] ["gco:CharacterString"] ) except Exception: log.warning('Could not set format') format = 'null' try: featureCatalogUuid = ( recordRoot["gmd:contentInfo"]["gmd:MD_FeatureCatalogueDescription"] ["gmd:featureCatalogueCitation"]["@uuidref"] ) except Exception: log.warning('Could not set featureCatalogUuid') featureCatalogUuid = 'null' try: crs = ( recordRoot['gmd:referenceSystemInfo']['gmd:MD_ReferenceSystem'] ['gmd:referenceSystemIdentifier']['gmd:RS_Identifier']['gmd:code'] ['gco:CharacterString'] ) except Exception: log.warning('Could not set crs') crs = 'null' # TODO: write parsing function if we need to find id column for processing # of materialized views try: log.warning('Feature Catalog UUID is: ' + featureCatalogUuid) # featRec = getRecordByUUID(featureCatalogUuid) # featRecRoot = json.loads(featRec)['csw:GetRecordByIdResponse']['csw:Record'] # featRecAttr = featRecRoot['dc:subject'] # for i in featRecAttr: # if 'idpk' in i: # featureCatalogIdColumn = i # log.warning('Feature Catalog ID column is: ' + featureCatalogIdColumn) except Exception: log.warning('Could not set featureCatalogIdColumn from CatalogUuid') geodata_meta = GeodataMeta( uuid=uuid, bbox=bbox, crs=crs, table=table, format=format ) return geodata_meta