def getRecordRoot(record):
""" Method to get one record out of the geonetwork response. Valid for
both schemata 'gmd:MD_Metadata' and 'csw:Record'
TODO: support more than one tag in response
This method can handle GetRecordByIdResponse and GetRecordsResponse
"""
record = json.loads(record)
if 'csw:GetRecordByIdResponse' in record:
log.info("Found 1 record")
if 'gmd:MD_Metadata' in record["csw:GetRecordByIdResponse"]:
recordRoot = (
record["csw:GetRecordByIdResponse"]["gmd:MD_Metadata"]
)
elif 'csw:Record' in record["csw:GetRecordByIdResponse"]:
recordRoot = (
record["csw:GetRecordByIdResponse"]["csw:Record"]
)
else:
log.info("...But record is empty")
return
elif 'csw:GetRecordsResponse' in record:
if 'csw:SearchResults' in record["csw:GetRecordsResponse"]:
searchResults = (
record["csw:GetRecordsResponse"]["csw:SearchResults"]
)
else:
log.info("...But record is empty")
return
numberOfRecords = int(searchResults["@numberOfRecordsReturned"])
recordRoot = dict()
if numberOfRecords == 0:
log.warning("No records found")
return
elif numberOfRecords == 1:
log.info("Found 1 record")
recordRoot = searchResults["gmd:MD_Metadata"]
if not recordRoot:
recordRoot = searchResults["csw:Record"]
else:
log.info("Found " + str(numberOfRecords) + " records")
recordRoot = searchResults["gmd:MD_Metadata"][0]
if not recordRoot:
recordRoot = searchResults["csw:Record"][0]
else:
print("Could not parse GNOS response")
return
return recordRoot
def getAllJobs(filters, process):
""" Method to read all jobs from jobtabelle with filter
Args: filters (ImmutableMultiDict): the args from the HTTP call
Returns:
jobs (list): the records matching the filter
"""
log.debug('Received query for jobs')
if process == 'test':
query = Expression('a', '=', 'a')
else:
query = Expression(getattr(Job, 'process'), '=', process)
if filters:
log.debug("Found filters: " + str(filters))
keys = [key for key in filters]
for key in keys:
try:
getattr(Job, key)
except Exception as e:
log.warning(str(e))
continue
log.debug("Filter " + str(key) + " with value " +
str(filters[key]))
if isinstance(getattr(Job, key), AutoField):
try:
int(filters[key])
except Exception as e:
log.error(str(e))
jobdb.close()
return
try:
# even though operators are listed as == and & in peewee docs,
# for Expression creation use '=' and 'AND'.
exp = Expression(getattr(Job, key), '=', filters[key])
query = Expression(query, 'AND', exp)
except AttributeError as e:
log.error(str(e))
with jobdb:
queryResult = Job.select().where(query).dicts()
jobs = []
# iterating reopens db connection!!
for i in queryResult:
jobs.append(i)
log.info("Found " + str(len(jobs)) + " results for query.")
jobdb.close()
return jobs
def getUuidByRecordRoot(recordRoot):
try:
uuid = recordRoot["gmd:fileIdentifier"]["gco:CharacterString"]
if not uuid:
uuid = recordRoot["dc:identifier"]
except Exception:
log.warning('Could not set uuid')
uuid = 'null'
return uuid
def getBboxByRecordRoot(recordRoot):
try:
recordExtent = (
recordRoot["gmd:identificationInfo"]
["gmd:MD_DataIdentification"]["gmd:extent"]
)
def parseExtent(bboxRoot):
bbox_a = bboxRoot["gmd:eastBoundLongitude"]["gco:Decimal"]
bbox_b = bboxRoot["gmd:southBoundLatitude"]["gco:Decimal"]
bbox_c = bboxRoot["gmd:westBoundLongitude"]["gco:Decimal"]
bbox_d = bboxRoot["gmd:northBoundLatitude"]["gco:Decimal"]
bbox = [
float(bbox_a),
float(bbox_b),
float(bbox_c),
float(bbox_d)
]
return bbox
if type(recordExtent) is list:
for i in recordExtent:
if 'gmd:geographicElement' in i["gmd:EX_Extent"]:
geoEl = i["gmd:EX_Extent"]["gmd:geographicElement"]
if type(geoEl) is list:
bbox = []
for i in geoEl:
if 'gmd:EX_GeographicBoundingBox' in i:
bboxRoot = i["gmd:EX_GeographicBoundingBox"]
bbox = parseExtent(bboxRoot)
else:
bboxRoot = geoEl["gmd:EX_GeographicBoundingBox"]
bbox = parseExtent(bboxRoot)
else:
bbox = []
except Exception:
log.warning('Could not set bbox')
bbox = []
return bbox
def updateJob(resourceId, actiniaCoreResp):
""" Method to update job in Jobtable
This method is called by webhook endpoint
"""
status = actiniaCoreResp['status']
job, uuid, utcnow = updateJobByResourceID(resourceId, actiniaCoreResp,
status)
# TODO: for now if multiple records need to be updated, this
# can be told by specifying multiple uuids comma-separated in the
# "feature_uuid" field of the preprocesschain. This might change later...
if status == 'finished':
try:
uuids = uuid.split(',')
for uuid in uuids:
update(uuid, utcnow)
except Exception:
log.warning('Could not update geonetwork record')
return job
def updateJobByResourceID(resourceId, resp, status):
""" Method to update job in jobtabelle when processing status changed
Args:
resourceId (str): actinia-core resourceId
resp (dict): actinia-core response
status (string): actinia-core processing status
Returns:
updatedRecord (TODO): the updated record
"""
utcnow = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
try:
with jobdb:
queryResult = Job.select().where(
getattr(Job, 'actinia_core_jobid') == resourceId).get()
record = model_to_dict(queryResult)
log.debug("Information read from jobtable for job with id " +
str(record['idpk_jobs']) + ".")
except Job.DoesNotExist:
log.warning("Job does not exist and can therefore not be updated")
return None, None, None
# actinia-gdi_["PENDING", "RUNNING", "SUCCEES", "ERROR", "TERMINATED"]
# actinia-core [accepted, running, finished, error, terminated]
try:
log.debug("Update status to " + status + " for job with id " +
str(record['idpk_jobs']) + ".")
gdiStatus = record['status']
try:
gnosUuid = record['job_description']['feature_uuid']
except Exception:
log.warning('Feature has no uuid')
gnosUuid = None
if current_app.debug is False:
smallRes = dict()
smallRes['message'] = resp.get('message', None)
smallRes['process_results'] = resp.get('process_results', None)
resp = smallRes
# resp = smallifyResp(resp)
# TODO: test in debug, then remove
# resp = smallifyResp(resp)
if status == 'accepted':
log.debug('Status already set to "PENDING"')
return record, gnosUuid, utcnow
elif status == 'running':
if gdiStatus == 'RUNNING':
log.debug('Status already set to "RUNNING"')
return record, gnosUuid, utcnow
query = Job.update(
status='RUNNING',
actinia_core_response=resp,
time_started=utcnow
# TODO: check if time_estimated can be set
# time_estimated=
).where(getattr(Job, 'actinia_core_jobid') == resourceId)
elif status in ['finished', 'error', 'terminated']:
if status == 'finished':
gdiStatus = 'SUCCESS'
elif status == 'error':
gdiStatus = 'ERROR'
elif status == 'terminated':
gdiStatus = 'TERMINATED'
query = Job.update(
status=gdiStatus,
actinia_core_response=resp,
time_ended=utcnow).where(
getattr(Job, 'actinia_core_jobid') == resourceId)
else:
log.error('Could not set the status to actinia-core status:' +
status + '(Status not found.)')
return None, None, None
with jobdb:
query.execute()
queryResult = Job.select().where(
getattr(Job, 'actinia_core_jobid') == resourceId).get()
record = model_to_dict(queryResult)
except Exception as e:
log.error('Could not set the status to actinia-core status: ' + status)
log.error(str(e))
return None, None, None
log.info("Updated status to " + status + " for job with id " +
str(record['idpk_jobs']) + ".")
jobdb.close()
return record, gnosUuid, utcnow
def parseMeta(record):
""" Method to parse record from geonetwork with model
TODO: support more than one tag in response
TODO: better error handling when attribute not found
This method can handle GetRecordByIdResponse and GetRecordsResponse
"""
if 'csw:GetRecordByIdResponse' in record:
log.info("Found 1 record")
if 'csw:Record' in json.loads(record)["csw:GetRecordByIdResponse"]:
recordRoot = (
json.loads(record)["csw:GetRecordByIdResponse"]["csw:Record"]
)
else:
log.info("...But record is empty")
return
elif 'csw:GetRecordsResponse' in record:
if 'csw:SearchResults' in json.loads(record)["csw:GetRecordsResponse"]:
searchResults = (
json.loads(record)["csw:GetRecordsResponse"]["csw:SearchResults"]
)
else:
log.info("...But record is empty")
return
numberOfRecords = int(searchResults["@numberOfRecordsReturned"])
recordRoot = dict()
if numberOfRecords == 0:
log.warning("No records found")
return
elif numberOfRecords == 1:
log.info("Found 1 record")
recordRoot = searchResults["csw:Record"]
else:
log.info("Found " + str(numberOfRecords) + " records")
recordRoot = searchResults["csw:Record"][0]
else:
print("Could not parse GNOS response")
return
if 'dc:identifier' in recordRoot:
uuid = recordRoot["dc:identifier"]
else:
uuid = 'null'
if 'ows:BoundingBox' in recordRoot:
recordBbox = recordRoot["ows:BoundingBox"]
if 'ows:LowerCorner' in recordBbox:
bbox_lower = recordBbox["ows:LowerCorner"]
if 'ows:UpperCorner' in recordBbox:
bbox_upper = recordBbox["ows:UpperCorner"]
bbox_a = float(bbox_lower.split(" ")[0])
bbox_b = float(bbox_lower.split(" ")[1])
bbox_c = float(bbox_upper.split(" ")[0])
bbox_d = float(bbox_upper.split(" ")[1])
bbox = [bbox_a, bbox_b, bbox_c, bbox_d]
if '@crs' in recordBbox:
crs = recordBbox["@crs"]
else:
bbox = []
crs = 'null'
if 'dc:URI' in recordRoot:
recordUri = recordRoot["dc:URI"]
if type(recordUri) is dict:
if "#text" in recordUri:
table = recordUri["#text"]
else:
table = recordUri[0]["#text"]
else:
table = 'null'
geodata_meta = GeodataMeta(
uuid=uuid,
bbox=bbox,
crs=crs,
table=table
)
return geodata_meta
def parseMeta(recordXml):
""" Method to parse record from geonetwork with schema 'gmd' with model
This method can handle GetRecordByIdResponse and GetRecordsResponse
"""
record = makeItJson(recordXml)
if record is None:
return None
recordRoot = getRecordRoot(record)
uuid = getUuidByRecordRoot(recordRoot)
bbox = getBboxByRecordRoot(recordRoot)
try:
table = (
recordRoot["gmd:distributionInfo"]["gmd:MD_Distribution"]
["gmd:transferOptions"]["gmd:MD_DigitalTransferOptions"]
["gmd:onLine"]["gmd:CI_OnlineResource"]["gmd:linkage"]["gmd:URL"]
)
except Exception:
log.warning('Could not set table')
table = 'null'
try:
format = (
recordRoot["gmd:distributionInfo"]["gmd:MD_Distribution"]
["gmd:distributionFormat"]["gmd:MD_Format"]["gmd:name"]
["gco:CharacterString"]
)
except Exception:
log.warning('Could not set format')
format = 'null'
try:
featureCatalogUuid = (
recordRoot["gmd:contentInfo"]["gmd:MD_FeatureCatalogueDescription"]
["gmd:featureCatalogueCitation"]["@uuidref"]
)
except Exception:
log.warning('Could not set featureCatalogUuid')
featureCatalogUuid = 'null'
try:
crs = (
recordRoot['gmd:referenceSystemInfo']['gmd:MD_ReferenceSystem']
['gmd:referenceSystemIdentifier']['gmd:RS_Identifier']['gmd:code']
['gco:CharacterString']
)
except Exception:
log.warning('Could not set crs')
crs = 'null'
# TODO: write parsing function if we need to find id column for processing
# of materialized views
try:
log.warning('Feature Catalog UUID is: ' + featureCatalogUuid)
# featRec = getRecordByUUID(featureCatalogUuid)
# featRecRoot = json.loads(featRec)['csw:GetRecordByIdResponse']['csw:Record']
# featRecAttr = featRecRoot['dc:subject']
# for i in featRecAttr:
# if 'idpk' in i:
# featureCatalogIdColumn = i
# log.warning('Feature Catalog ID column is: ' + featureCatalogIdColumn)
except Exception:
log.warning('Could not set featureCatalogIdColumn from CatalogUuid')
geodata_meta = GeodataMeta(
uuid=uuid,
bbox=bbox,
crs=crs,
table=table,
format=format
)
return geodata_meta