def runningUpdateJobs(dbsession, logger): """ Runs the persistent georeference job for update georeference jobs :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :return: int Number of processed jobs """ logger.info('Check for unprocessed update georeference jobs ...') unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeUpdate(dbsession) counter = 0 for job in unprocessedJobs: logger.info('Start processing of a "update" georeference process with id - %s'%job.id) georefObj = Georeferenzierungsprozess.clearRaceConditions(job, dbsession) # get active georeference process and deactive him, if exist activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(georefObj.mapid, dbsession) mapObj = Map.by_id(georefObj.mapid, dbsession) if activeGeorefProcess: logger.info('Deactivate georeference processes with id %s ...'%activeGeorefProcess.id) deactivate(activeGeorefProcess, mapObj, dbsession, logger) logger.info('Activate georeference processes with id %s ...'%georefObj.id) activate(georefObj, mapObj, dbsession, logger) logger.info('Finish processing of a "update" georeference process with id - %s'%job.id) counter += 1 return counter
def setIsValide(job, dbsession, logger, testing = False): """ This function sets a georeference process as 'isvalide' and if there is no other newer georeference process for this map its activate this georeference process for this map. :type georeference.models.vkdb.adminjobs.AdminJobs: job :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :type boolean: testing (Default: False) """ logger.debug('Set georeference process for id %s to isvalide ...'%(job.georefid)) # set georeferenceprocess to isvalide georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession) # check if there is an other process which is newer and more up to date activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(georefProcess.mapid, dbsession) mapObj = Map.by_id(georefProcess.mapid, dbsession) if activeGeorefProcess and activeGeorefProcess.id >= georefProcess.id: logger.info('The georeference process with the id %s or younger process is already active for this map object.'%georefProcess.id) pass elif activeGeorefProcess and activeGeorefProcess.id < georefProcess.id: logger.info('Activate the is valide georeference process and deactive old one ...') deactivate(activeGeorefProcess, mapObj, dbsession, logger) activate(georefProcess, mapObj, dbsession, logger) else: logger.info('Activate georeference process %s for the map object %s ...'%(georefProcess.id, georefProcess.mapid)) activate(georefProcess, mapObj, dbsession, logger) if georefProcess.adminvalidation == 'invalide': mapObj.hasgeorefparams = mapObj.hasgeorefparams + 1 georefProcess.adminvalidation = 'isvalide'
def setIsValide(job, dbsession, logger, testing=False): """ This function sets a georeference process as 'isvalide' and if there is no other newer georeference process for this map its activate this georeference process for this map. :type georeference.models.vkdb.adminjobs.AdminJobs: job :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :type boolean: testing (Default: False) """ logger.debug('Set georeference process for id %s to isvalide ...' % (job.georefid)) # set georeferenceprocess to isvalide georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession) # check if there is an other process which is newer and more up to date activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId( georefProcess.mapid, dbsession) mapObj = Map.by_id(georefProcess.mapid, dbsession) if activeGeorefProcess and activeGeorefProcess.id >= georefProcess.id: logger.info( 'The georeference process with the id %s or younger process is already active for this map object.' % georefProcess.id) pass elif activeGeorefProcess and activeGeorefProcess.id < georefProcess.id: logger.info( 'Activate the is valide georeference process and deactive old one ...' ) deactivate(activeGeorefProcess, mapObj, dbsession, logger) activate(georefProcess, mapObj, dbsession, logger) else: logger.info( 'Activate georeference process %s for the map object %s ...' % (georefProcess.id, georefProcess.mapid)) activate(georefProcess, mapObj, dbsession, logger) if georefProcess.adminvalidation == 'invalide': mapObj.hasgeorefparams = mapObj.hasgeorefparams + 1 georefProcess.adminvalidation = 'isvalide'
def runningUpdateJobs(dbsession, logger): """ Runs the persistent georeference job for update georeference jobs :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :return: int Number of processed jobs """ logger.info('Check for unprocessed update georeference jobs ...') unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeUpdate( dbsession) counter = 0 for job in unprocessedJobs: logger.info( 'Start processing of a "update" georeference process with id - %s' % job.id) georefObj = Georeferenzierungsprozess.clearRaceConditions( job, dbsession) # get active georeference process and deactive him, if exist activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId( georefObj.mapid, dbsession) mapObj = Map.by_id(georefObj.mapid, dbsession) if activeGeorefProcess: logger.info('Deactivate georeference processes with id %s ...' % activeGeorefProcess.id) deactivate(activeGeorefProcess, mapObj, dbsession, logger) logger.info('Activate georeference processes with id %s ...' % georefObj.id) activate(georefObj, mapObj, dbsession, logger) logger.info( 'Finish processing of a "update" georeference process with id - %s' % job.id) counter += 1 return counter
def generateGeoreferenceProcessForMapObj(mapObjId, request, log): """ Function generates a process process for a given map object id :type mapObjId: int :type request: pyramid.request :type log: logging.Logger :return: dict """ def getMtbGLSpecificGeoreferenceInformation(mapObj, request): """ Query the specific process base data for a messtischblatt/geological map :type mapObj: georeference.models.vkdb.map.Map :type request: pyramid.request :return: dict """ srid = mapObj.getSRID(request.db) extent = mapObj.getExtent(request.db, srid) return { 'georeference': { 'source':'pixel', 'target':'EPSG:%s'%srid, 'gcps': [ {"source":[], "target":[extent[0],extent[1]]}, {"source":[], "target":[extent[0],extent[3]]}, {"source":[], "target":[extent[2],extent[1]]}, {"source":[], "target":[extent[2],extent[3]]} ], 'algorithm': 'affine' }, 'extent':extent } mapObj = Map.by_id(mapObjId, request.db) # get general metadata log.debug('Get general process process information ...') generalMetadata = getGeneralMetadata(mapObj, request) # check if there exist already a activate process process for # this mapObj log.debug('Get specific process process information ...') if Georeferenzierungsprozess.isGeoreferenced(mapObj.id, request.db): # there does exist a process process for this mapObj georefProcessObj = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(mapObj.id, request.db) georeferenceData = getSpecificGeoreferenceData(georefProcessObj, mapObj, 4326, request.db) else: # there does not exist a process process for this mapObj georeferenceData = { "timestamp": "", "type": "new" } # This blog could be used for attaching information regarding the boundingbox and the extent to the # response. In case no georeference information is registered yet this could be used by the client for # doing a matching of corner point coordinates and the extent corners. # # log.debug('Check if there is special behavior needed in case of messtischblatt') # mtbGeorefBaseData = {} # if mapObj.maptype == 'M' and 'georeference' not in georeferenceData and mapObj.boundingbox is not None: # mtbGeorefBaseData = getMtbGLSpecificGeoreferenceInformation(mapObj, request) # response.update(mtbGeorefBaseData) log.debug('Check if there are pending processes in the database') warnMsg = {} if checkIfPendingProcessesExist(mapObj, request): warnMsg["warn"] = 'Right now another users is working on the georeferencing of this map sheet. For preventing information losses please try again in 15 minutes.' # now merge dictionaries and create response response = { "recommendedsrid": mapObj.recommendedsrid } response.update(generalMetadata) response.update(georeferenceData) response.update(warnMsg) return response
Copyright (c) 2015 Jacob Mendt Created on 02.10.15 @author: mendt @description: The following scripts pushs all database records to elasticsearch ''' import logging from georeference.settings import DBCONFIG_PARAMS from georeference.models.meta import getPostgresEngineString from georeference.models.meta import initializeDb from georeference.models.vkdb.georeferenzierungsprozess import Georeferenzierungsprozess from georeference.models.vkdb.map import Map from georeference.persistent.jobs.genericjobs import pushRecordToSearchIndex from georeference.persistent.jobs.genericjobs import removeRecordFromSearchIndex if __name__ == '__main__': logging.basicConfig() logger = logging.getLogger('Push recrords to ES') dbsession = initializeDb(getPostgresEngineString(DBCONFIG_PARAMS), False) maps = Map.all(dbsession) for mapObj in maps: if mapObj.istaktiv == True: print 'Push map record %s to elastic search ...' % mapObj.id georefObj = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(mapObj.id, dbsession) pushRecordToSearchIndex(mapObj, dbsession, logger, georefObj) else: print 'Remove map record %s from elastic search ...' % mapObj.id removeRecordFromSearchIndex(mapObj)
Created on 02.10.15 @author: mendt @description: The following scripts pushs all database records to elasticsearch ''' import logging from georeference.settings import DBCONFIG_PARAMS from georeference.models.meta import getPostgresEngineString from georeference.models.meta import initializeDb from georeference.models.vkdb.georeferenzierungsprozess import Georeferenzierungsprozess from georeference.models.vkdb.map import Map from georeference.persistent.jobs.genericjobs import pushRecordToSearchIndex from georeference.persistent.jobs.genericjobs import removeRecordFromSearchIndex if __name__ == '__main__': logging.basicConfig() logger = logging.getLogger('Push recrords to ES') dbsession = initializeDb(getPostgresEngineString(DBCONFIG_PARAMS), False) maps = Map.all(dbsession) for mapObj in maps: if mapObj.istaktiv == True: print 'Push map record %s to elastic search ...' % mapObj.id georefObj = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId( mapObj.id, dbsession) pushRecordToSearchIndex(mapObj, dbsession, logger, georefObj) else: print 'Remove map record %s from elastic search ...' % mapObj.id removeRecordFromSearchIndex(mapObj)
def georeferenceConfirm(request): """ The function persistent saves a georeference confirmation request :type request: pyramid.request :return: dict :raise: HTTPBadRequest :raise: HTTPInternalServerError """ LOGGER.info('Receive georeference validation request with parameters: %s' % request.json_body) try: # extract validation data LOGGER.debug('Parse request params ...') requestData = parseGeoreferenceParamsFromRequest(request) # in case of type new, # check if there already exist an actual georeference process for this object with this type if requestData[ 'type'] == 'new' and Georeferenzierungsprozess.isGeoreferenced( requestData['mapObj'].id, request.db): msg = 'There exists already an active georeference process for this map id. It is therefore not possible to save a georeference process of type "new".' LOGGER.debug(msg) georeferenceid = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId( requestData['mapObj'].id, request.db).id return {'text': msg, 'georeferenceid': georeferenceid} LOGGER.debug('Save georeference parameter in datase ...') timestamp = convertTimestampToPostgisString(datetime.now()) georefParam = str(convertUnicodeDictToUtf(requestData['georeference'])) overwrites = requestData[ 'overwrites'] if 'overwrites' in requestData else 0 georefProcess = Georeferenzierungsprozess( messtischblattid=requestData['mapObj'].apsobjectid, nutzerid=requestData['userid'], georefparams=ast.literal_eval(georefParam), timestamp=timestamp, isactive=False, type=requestData['type'], overwrites=overwrites, adminvalidation='', processed=False, mapid=requestData['mapObj'].id, algorithm=requestData['georeference']['algorithm']) request.db.add(georefProcess) request.db.flush() # add polygon if 'clip' in requestData: clipParam = convertUnicodeDictToUtf(requestData['clip']) polygonAsString = convertListToPostgisString( clipParam['polygon'], 'POLYGON') georefProcess.setClip(polygonAsString, stripSRIDFromEPSG(clipParam['source']), request.db) LOGGER.debug('Create response ...') # @TODO has to be updated points = int(len(requestData['georeference']['gcps']) * 5) return { 'text': 'Georeference result saved. It will soon be ready for use.', 'georeferenceid': georefProcess.id, 'points': points } except ParameterException as e: LOGGER.error(e) LOGGER.error(traceback.format_exc()) raise HTTPBadRequest(ERROR_MSG) except Exception as e: LOGGER.error(e) LOGGER.error(traceback.format_exc()) raise HTTPInternalServerError(ERROR_MSG)