def runningUpdateJobs(dbsession, logger):
""" Runs the persistent georeference job for update georeference jobs
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:return: int Number of processed jobs
"""
logger.info('Check for unprocessed update georeference jobs ...')
unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeUpdate(dbsession)
counter = 0
for job in unprocessedJobs:
logger.info('Start processing of a "update" georeference process with id - %s'%job.id)
georefObj = Georeferenzierungsprozess.clearRaceConditions(job, dbsession)
# get active georeference process and deactive him, if exist
activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(georefObj.mapid, dbsession)
mapObj = Map.by_id(georefObj.mapid, dbsession)
if activeGeorefProcess:
logger.info('Deactivate georeference processes with id %s ...'%activeGeorefProcess.id)
deactivate(activeGeorefProcess, mapObj, dbsession, logger)
logger.info('Activate georeference processes with id %s ...'%georefObj.id)
activate(georefObj, mapObj, dbsession, logger)
logger.info('Finish processing of a "update" georeference process with id - %s'%job.id)
counter += 1
return counter
def runningNewJobs(dbsession, logger):
""" Runs the persistent georeference job for new georeference jobs
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:return: int Number of processed jobs
"""
logger.info('Check for unprocessed new georeference jobs ...')
unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeNew(
dbsession)
counter = 0
for job in unprocessedJobs:
logger.info(
'Start processing of a "new" georeference process with id - %s' %
job.id)
georefObj = Georeferenzierungsprozess.clearRaceConditions(
job, dbsession)
mapObj = Map.by_id(georefObj.mapid, dbsession)
activate(georefObj, mapObj, dbsession, logger)
logger.info(
'Finish processing of a "new" georeference process with id - %s' %
job.id)
counter += 1
return counter
def generateGeoreferenceProcessForSpecificGeoreferenceId(georeferenceId, request, log):
""" Function generates a process process for a given map object id
:type georeferenceId: int
:type request: pyramid.request
:type log: logging.Logger
:return: dict """
georefProcessObj = Georeferenzierungsprozess.by_id(georeferenceId, request.db)
mapObj = Map.by_id(georefProcessObj.mapid, request.db)
# get general metadata
log.debug('Get general process process information ...')
generalMetadata = getGeneralMetadata(mapObj, request)
# check if there exist already a activate process process for
# this mapObj
log.debug('Get specific process process information ...')
georeferenceData = getSpecificGeoreferenceData(georefProcessObj, mapObj, 4326, request.db)
log.debug('Check if there are pending processes in the database')
warnMsg = {}
if checkIfPendingProcessesExist(mapObj, request):
warnMsg["warn"] = 'Right now another users is working on the georeferencing of this map sheet. For preventing information losses please try again in 15 minutes.'
# now merge dictionaries and create response
response = {}
response.update(generalMetadata)
response.update(georeferenceData)
response.update(warnMsg)
return response
def setIsValide(job, dbsession, logger, testing = False):
""" This function sets a georeference process as 'isvalide' and if there is no other
newer georeference process for this map its activate this georeference process for this
map.
:type georeference.models.vkdb.adminjobs.AdminJobs: job
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:type boolean: testing (Default: False) """
logger.debug('Set georeference process for id %s to isvalide ...'%(job.georefid))
# set georeferenceprocess to isvalide
georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession)
# check if there is an other process which is newer and more up to date
activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(georefProcess.mapid, dbsession)
mapObj = Map.by_id(georefProcess.mapid, dbsession)
if activeGeorefProcess and activeGeorefProcess.id >= georefProcess.id:
logger.info('The georeference process with the id %s or younger process is already active for this map object.'%georefProcess.id)
pass
elif activeGeorefProcess and activeGeorefProcess.id < georefProcess.id:
logger.info('Activate the is valide georeference process and deactive old one ...')
deactivate(activeGeorefProcess, mapObj, dbsession, logger)
activate(georefProcess, mapObj, dbsession, logger)
else:
logger.info('Activate georeference process %s for the map object %s ...'%(georefProcess.id, georefProcess.mapid))
activate(georefProcess, mapObj, dbsession, logger)
if georefProcess.adminvalidation == 'invalide':
mapObj.hasgeorefparams = mapObj.hasgeorefparams + 1
georefProcess.adminvalidation = 'isvalide'
def testGetDataRecord(self):
mapObj = Map.by_id(10000023, self.dbsession)
response = createSearchRecord(mapObj, self.dbsession, self.logger, self.georefProcess)
print '====================='
print 'Test if testGetDataRecord ...'
print 'Response: %s'%response
print '====================='
self.assertTrue('oai:de:slub-dresden:vk:id-10000023' in response, 'Missing key in response')
self.assertEqual(response['oai:de:slub-dresden:vk:id-10000023']['dataid'], 'df_dk_0010001_4630_1928', 'Dataid has not expected value')
def setInValide(job, dbsession, logger, testing=False):
""" This function sets a georeference process as 'isvalide' and if there is no other
newer georeference process for this map its activate this georeference process for this
map.
:type georeference.models.vkdb.adminjobs.AdminJobs: job
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:type boolean: testing (Default: False) """
logger.debug('Set georeference process for id %s to isvalide ...' %
(job.georefid))
# set georeferenceprocess to isvalide
georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession)
# update map object and datasource
mapObj = Map.by_id(georefProcess.mapid, dbsession)
if georefProcess.isactive == True and georefProcess.overwrites > 0:
logger.info(
'Deactive georeference process and activate georeference process with id %s ...'
% georefProcess.overwrites)
# deactive the georeference process
deactivate(georefProcess, mapObj, dbsession, logger)
# look if there is a valid overwrite id and if yes activate the matching
# process
newGeorefProcess = getLastValidGeoreferenceProcess(
georefProcess.overwrites, dbsession, logger)
if newGeorefProcess:
activate(newGeorefProcess, mapObj, dbsession, logger)
elif georefProcess.isactive == True and georefProcess.overwrites == 0:
logger.info('Deactive georeference process %s ...' %
georefProcess.overwrites)
# deactive the georeference process
deactivate(georefProcess, mapObj, dbsession, logger)
logger.debug('Set georeference process with id %s to inactive ...' %
georefProcess.overwrites)
if georefProcess.adminvalidation != 'invalide':
georefProcess.adminvalidation = 'invalide'
mapObj.hasgeorefparams = 0 if mapObj.hasgeorefparams - 1 < 0 else mapObj.hasgeorefparams - 1
def testPushRecordToEs(self):
mapObj = Map.by_id(10000023, self.dbsession)
datarecord = createSearchRecord(mapObj, self.dbsession, self.logger)
key = pushRecordToEs(datarecord, ELASTICSEARCH_INDEX, self.logger)
print '====================='
print 'Test if testPushRecordToEs ...'
print 'Response: %s'%key
print '====================='
# check if the record was insert correctly
response = getRecordFromEsById(key, ELASTICSEARCH_INDEX)
self.assertEqual(response['found'], True, 'Could not find expected record')
self.assertEqual(response['_id'], key, 'Key is not like expected')
# clear up
response = deleteRecordFromEsById(key, ELASTICSEARCH_INDEX)
if response['found'] is not True:
raise Exception("Problems while trying to clean up elasticsearch test index")
def runningNewJobs(dbsession, logger):
""" Runs the persistent georeference job for new georeference jobs
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:return: int Number of processed jobs
"""
logger.info('Check for unprocessed new georeference jobs ...')
unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeNew(dbsession)
counter = 0
for job in unprocessedJobs:
logger.info('Start processing of a "new" georeference process with id - %s'%job.id)
georefObj = Georeferenzierungsprozess.clearRaceConditions(job, dbsession)
mapObj = Map.by_id(georefObj.mapid, dbsession)
activate(georefObj, mapObj, dbsession, logger)
logger.info('Finish processing of a "new" georeference process with id - %s'%job.id)
counter += 1
return counter
def parseMapObjForId(requestParams, name, dbsession):
"""
This functions parses a map objectid from an objectid
:param requestParams: dict
:param name: str
:param dbsession: sqlalchemy.orm.session.Session
:return: process.models.vkdb.map.Map
:raise: process.utils.exceptions.ParameterException
"""
if name in requestParams:
validateId(requestParams[name])
# @deprecated
# do mapping for support of new name schema
mapObj = Map.by_id(int(requestParams[name]), dbsession)
if mapObj is None:
raise ParameterException('Missing or wrong objectid parameter.')
else:
return mapObj
raise ParameterException('Missing or wrong objectid parameter.')
def parseMapObjForId(requestParams, name, dbsession):
"""
This functions parses a map objectid from an objectid
:param requestParams: dict
:param name: str
:param dbsession: sqlalchemy.orm.session.Session
:return: process.models.vkdb.map.Map
:raise: process.utils.exceptions.ParameterException
"""
if name in requestParams:
validateId(requestParams[name])
# @deprecated
# do mapping for support of new name schema
mapObj = Map.by_id(int(requestParams[name]), dbsession)
if mapObj is None:
raise ParameterException('Missing or wrong objectid parameter.')
else:
return mapObj
raise ParameterException('Missing or wrong objectid parameter.')
def testPushRecordToEs(self):
mapObj = Map.by_id(10000023, self.dbsession)
datarecord = createSearchRecord(mapObj, self.dbsession, self.logger)
key = pushRecordToEs(datarecord, ELASTICSEARCH_INDEX, self.logger)
print '====================='
print 'Test if testPushRecordToEs ...'
print 'Response: %s' % key
print '====================='
# check if the record was insert correctly
response = getRecordFromEsById(key, ELASTICSEARCH_INDEX)
self.assertEqual(response['found'], True,
'Could not find expected record')
self.assertEqual(response['_id'], key, 'Key is not like expected')
# clear up
response = deleteRecordFromEsById(key, ELASTICSEARCH_INDEX)
if response['found'] is not True:
raise Exception(
"Problems while trying to clean up elasticsearch test index")
def setIsValide(job, dbsession, logger, testing=False):
""" This function sets a georeference process as 'isvalide' and if there is no other
newer georeference process for this map its activate this georeference process for this
map.
:type georeference.models.vkdb.adminjobs.AdminJobs: job
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:type boolean: testing (Default: False) """
logger.debug('Set georeference process for id %s to isvalide ...' %
(job.georefid))
# set georeferenceprocess to isvalide
georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession)
# check if there is an other process which is newer and more up to date
activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(
georefProcess.mapid, dbsession)
mapObj = Map.by_id(georefProcess.mapid, dbsession)
if activeGeorefProcess and activeGeorefProcess.id >= georefProcess.id:
logger.info(
'The georeference process with the id %s or younger process is already active for this map object.'
% georefProcess.id)
pass
elif activeGeorefProcess and activeGeorefProcess.id < georefProcess.id:
logger.info(
'Activate the is valide georeference process and deactive old one ...'
)
deactivate(activeGeorefProcess, mapObj, dbsession, logger)
activate(georefProcess, mapObj, dbsession, logger)
else:
logger.info(
'Activate georeference process %s for the map object %s ...' %
(georefProcess.id, georefProcess.mapid))
activate(georefProcess, mapObj, dbsession, logger)
if georefProcess.adminvalidation == 'invalide':
mapObj.hasgeorefparams = mapObj.hasgeorefparams + 1
georefProcess.adminvalidation = 'isvalide'
def runningUpdateJobs(dbsession, logger):
""" Runs the persistent georeference job for update georeference jobs
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:return: int Number of processed jobs
"""
logger.info('Check for unprocessed update georeference jobs ...')
unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeUpdate(
dbsession)
counter = 0
for job in unprocessedJobs:
logger.info(
'Start processing of a "update" georeference process with id - %s'
% job.id)
georefObj = Georeferenzierungsprozess.clearRaceConditions(
job, dbsession)
# get active georeference process and deactive him, if exist
activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(
georefObj.mapid, dbsession)
mapObj = Map.by_id(georefObj.mapid, dbsession)
if activeGeorefProcess:
logger.info('Deactivate georeference processes with id %s ...' %
activeGeorefProcess.id)
deactivate(activeGeorefProcess, mapObj, dbsession, logger)
logger.info('Activate georeference processes with id %s ...' %
georefObj.id)
activate(georefObj, mapObj, dbsession, logger)
logger.info(
'Finish processing of a "update" georeference process with id - %s'
% job.id)
counter += 1
return counter
def setInValide(job, dbsession, logger, testing = False):
""" This function sets a georeference process as 'isvalide' and if there is no other
newer georeference process for this map its activate this georeference process for this
map.
:type georeference.models.vkdb.adminjobs.AdminJobs: job
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:type boolean: testing (Default: False) """
logger.debug('Set georeference process for id %s to isvalide ...'%(job.georefid))
# set georeferenceprocess to isvalide
georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession)
# update map object and datasource
mapObj = Map.by_id(georefProcess.mapid, dbsession)
if georefProcess.isactive == True and georefProcess.overwrites > 0:
logger.info('Deactive georeference process and activate georeference process with id %s ...'%georefProcess.overwrites)
# deactive the georeference process
deactivate(georefProcess, mapObj, dbsession, logger)
# look if there is a valid overwrite id and if yes activate the matching
# process
newGeorefProcess = getLastValidGeoreferenceProcess(georefProcess.overwrites, dbsession, logger)
if newGeorefProcess:
activate(newGeorefProcess, mapObj, dbsession, logger)
elif georefProcess.isactive == True and georefProcess.overwrites == 0:
logger.info('Deactive georeference process %s ...'%georefProcess.overwrites)
# deactive the georeference process
deactivate(georefProcess, mapObj, dbsession, logger)
logger.debug('Set georeference process with id %s to inactive ...'%georefProcess.overwrites)
if georefProcess.adminvalidation != 'invalide':
georefProcess.adminvalidation = 'invalide'
mapObj.hasgeorefparams = 0 if mapObj.hasgeorefparams - 1 < 0 else mapObj.hasgeorefparams - 1
def generateGeoreferenceProcessForMapObj(mapObjId, request, log):
""" Function generates a process process for a given map object id
:type mapObjId: int
:type request: pyramid.request
:type log: logging.Logger
:return: dict """
def getMtbGLSpecificGeoreferenceInformation(mapObj, request):
""" Query the specific process base data for a messtischblatt/geological map
:type mapObj: georeference.models.vkdb.map.Map
:type request: pyramid.request
:return: dict """
srid = mapObj.getSRID(request.db)
extent = mapObj.getExtent(request.db, srid)
return {
'georeference': {
'source':'pixel',
'target':'EPSG:%s'%srid,
'gcps': [
{"source":[], "target":[extent[0],extent[1]]},
{"source":[], "target":[extent[0],extent[3]]},
{"source":[], "target":[extent[2],extent[1]]},
{"source":[], "target":[extent[2],extent[3]]}
],
'algorithm': 'affine'
},
'extent':extent
}
mapObj = Map.by_id(mapObjId, request.db)
# get general metadata
log.debug('Get general process process information ...')
generalMetadata = getGeneralMetadata(mapObj, request)
# check if there exist already a activate process process for
# this mapObj
log.debug('Get specific process process information ...')
if Georeferenzierungsprozess.isGeoreferenced(mapObj.id, request.db):
# there does exist a process process for this mapObj
georefProcessObj = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(mapObj.id, request.db)
georeferenceData = getSpecificGeoreferenceData(georefProcessObj, mapObj, 4326, request.db)
else:
# there does not exist a process process for this mapObj
georeferenceData = {
"timestamp": "",
"type": "new"
}
# This blog could be used for attaching information regarding the boundingbox and the extent to the
# response. In case no georeference information is registered yet this could be used by the client for
# doing a matching of corner point coordinates and the extent corners.
#
# log.debug('Check if there is special behavior needed in case of messtischblatt')
# mtbGeorefBaseData = {}
# if mapObj.maptype == 'M' and 'georeference' not in georeferenceData and mapObj.boundingbox is not None:
# mtbGeorefBaseData = getMtbGLSpecificGeoreferenceInformation(mapObj, request)
# response.update(mtbGeorefBaseData)
log.debug('Check if there are pending processes in the database')
warnMsg = {}
if checkIfPendingProcessesExist(mapObj, request):
warnMsg["warn"] = 'Right now another users is working on the georeferencing of this map sheet. For preventing information losses please try again in 15 minutes.'
# now merge dictionaries and create response
response = {
"recommendedsrid": mapObj.recommendedsrid
}
response.update(generalMetadata)
response.update(georeferenceData)
response.update(warnMsg)
return response