def setIsValide(job, dbsession, logger, testing = False): """ This function sets a georeference process as 'isvalide' and if there is no other newer georeference process for this map its activate this georeference process for this map. :type georeference.models.vkdb.adminjobs.AdminJobs: job :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :type boolean: testing (Default: False) """ logger.debug('Set georeference process for id %s to isvalide ...'%(job.georefid)) # set georeferenceprocess to isvalide georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession) # check if there is an other process which is newer and more up to date activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(georefProcess.mapid, dbsession) mapObj = Map.by_id(georefProcess.mapid, dbsession) if activeGeorefProcess and activeGeorefProcess.id >= georefProcess.id: logger.info('The georeference process with the id %s or younger process is already active for this map object.'%georefProcess.id) pass elif activeGeorefProcess and activeGeorefProcess.id < georefProcess.id: logger.info('Activate the is valide georeference process and deactive old one ...') deactivate(activeGeorefProcess, mapObj, dbsession, logger) activate(georefProcess, mapObj, dbsession, logger) else: logger.info('Activate georeference process %s for the map object %s ...'%(georefProcess.id, georefProcess.mapid)) activate(georefProcess, mapObj, dbsession, logger) if georefProcess.adminvalidation == 'invalide': mapObj.hasgeorefparams = mapObj.hasgeorefparams + 1 georefProcess.adminvalidation = 'isvalide'
def runningNewJobs(dbsession, logger): """ Runs the persistent georeference job for new georeference jobs :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :return: int Number of processed jobs """ logger.info('Check for unprocessed new georeference jobs ...') unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeNew( dbsession) counter = 0 for job in unprocessedJobs: logger.info( 'Start processing of a "new" georeference process with id - %s' % job.id) georefObj = Georeferenzierungsprozess.clearRaceConditions( job, dbsession) mapObj = Map.by_id(georefObj.mapid, dbsession) activate(georefObj, mapObj, dbsession, logger) logger.info( 'Finish processing of a "new" georeference process with id - %s' % job.id) counter += 1 return counter
def runningUpdateJobs(dbsession, logger): """ Runs the persistent georeference job for update georeference jobs :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :return: int Number of processed jobs """ logger.info('Check for unprocessed update georeference jobs ...') unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeUpdate(dbsession) counter = 0 for job in unprocessedJobs: logger.info('Start processing of a "update" georeference process with id - %s'%job.id) georefObj = Georeferenzierungsprozess.clearRaceConditions(job, dbsession) # get active georeference process and deactive him, if exist activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(georefObj.mapid, dbsession) mapObj = Map.by_id(georefObj.mapid, dbsession) if activeGeorefProcess: logger.info('Deactivate georeference processes with id %s ...'%activeGeorefProcess.id) deactivate(activeGeorefProcess, mapObj, dbsession, logger) logger.info('Activate georeference processes with id %s ...'%georefObj.id) activate(georefObj, mapObj, dbsession, logger) logger.info('Finish processing of a "update" georeference process with id - %s'%job.id) counter += 1 return counter
def setProcessToIsValide(request): LOGGER.info( 'Request - Set admin evaluation of georeference process to isvalide ....' ) try: # remove georeference process if 'georeferenceid' in request.params: georeferenceid = request.params['georeferenceid'] comment = '' if 'comment' not in request.params else request.params[ 'comment'] # check if georeference id exist georeferenceprocess = Georeferenzierungsprozess.by_id( georeferenceid, request.db) if georeferenceprocess: newJob = createNewAdminJob(georeferenceprocess, 'isvalide', request.params['userid'], comment) request.db.add(newJob) return { 'message': 'The georeference process has been set to isvalide.' } else: raise Exception('Missing parameter (georeferenceid) ...') except Exception as e: LOGGER.error(e) LOGGER.error(traceback.format_exc()) return HTTPBadRequest(GENERAL_ERROR_MESSAGE)
def setUp(self): self.process = Georeferenzierungsprozess( mapid = 10002567, nutzerid = TEST_LOGIN, clippolygon = { 'source':'pixel', 'polygon': [[467, 923],[7281, 999],[7224, 7432],[258, 7471],[467, 923]]}, georefparams = { 'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [ {'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789]}, {'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789]}, {'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605]}, {'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605]} ], "algorithm": "affine", }, timestamp = "2014-08-09 12:20:26", type = 'new', algorithm = 'affine', isactive = True, processed = False, overwrites = 0, adminvalidation = '') self.map = Map(id = 10002567, apsobjectid=90015724, apsdateiname = "df_dk_0010001_4630_1928", boundingbox = "POLYGON((10.6666660308838 51.2999992370605,10.6666660308838 51.4000015258789,10.8333339691162 51.4000015258789,10.8333339691162 51.2999992370605,10.6666660308838 51.2999992370605))", originalimage = os.path.join(TEST_DATA_DIR, "df_dk_0010001_4630_1928.tif"))
def generateGeoreferenceProcessForSpecificGeoreferenceId(georeferenceId, request, log): """ Function generates a process process for a given map object id :type georeferenceId: int :type request: pyramid.request :type log: logging.Logger :return: dict """ georefProcessObj = Georeferenzierungsprozess.by_id(georeferenceId, request.db) mapObj = Map.by_id(georefProcessObj.mapid, request.db) # get general metadata log.debug('Get general process process information ...') generalMetadata = getGeneralMetadata(mapObj, request) # check if there exist already a activate process process for # this mapObj log.debug('Get specific process process information ...') georeferenceData = getSpecificGeoreferenceData(georefProcessObj, mapObj, 4326, request.db) log.debug('Check if there are pending processes in the database') warnMsg = {} if checkIfPendingProcessesExist(mapObj, request): warnMsg["warn"] = 'Right now another users is working on the georeferencing of this map sheet. For preventing information losses please try again in 15 minutes.' # now merge dictionaries and create response response = {} response.update(generalMetadata) response.update(georeferenceData) response.update(warnMsg) return response
def setUp(self): # create and insert test data to database self.testData = [ Map( id = 10000023, apsobjectid=90015724, apsdateiname = "df_dk_0010001_4630_1928", boundingbox = "POLYGON((16.9999980926514 51.7999992370605,16.9999980926514 51.9000015258789,17.1666679382324 51.9000015258789,17.1666679382324 51.7999992370605,16.9999980926514 51.7999992370605))", maptype="M", originalimage = os.path.join(self.testDataDir, "df_dk_0010001_4630_1928.tif"), georefimage = os.path.join(self.testDataDir, "df_dk_0010001_4630_1928.tif"), isttransformiert = True, recommendedsrid = 4314 ), Metadata( mapid = 10000023, imagezoomify = 'http://fotothek.slub-dresden.de/zooms/df/dk/0010000/df_dk_0010001_3352_1918/ImageProperties.xml', title = '', titleshort = '', scale = '1:25000', timepublish = datetime.now(), imagejpg = 'http://fotothek.slub-dresden.de/fotos/df/dk/0010000/df_dk_0010001_2655.jpg', thumbssmall = 'http://fotothek.slub-dresden.de/thumbs/df/dk/0010000/df_dk_0010001_6817.jpg', description = 'Ars an der Mosel. - Aufn. 1880, hrsg. 1882, Aufldr. 1916. - 1:25000. - [Berlin]: Kgl. Preuss. Landesaufnahme, 1916. - 1 Kt.', technic = 'Lithografie & Umdruck', type = 'Druckgraphik' ) ] self.georefProcess = Georeferenzierungsprozess( mapid = 10000023, messtischblattid = 90015724, nutzerid = self.user, clipparameter = {'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [\ {'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789]}, \ {'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789]}, \ {'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605]},\ {'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605]}]}, georefparams = {'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [\ {'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789]}, \ {'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789]}, \ {'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605]},\ {'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605]}]}, clippolygon = {'source': 'pixel', 'polygon': [[7813, 7517], [1652, 7523], [1677, 1666], [7830, 1661], [7813, 7517]]}, timestamp = "2014-08-09 12:20:26", type = 'update', isactive = False, processed = False, overwrites = 0, adminvalidation = '' ) try: for obj in self.testData: self.dbsession.add(obj) self.dbsession.flush() except Exception: raise
def testGetProcessWithTwoParam(self): print "--------------------------------------------------------------------------------------------" print "Testing correct working of georeferenceGetProcess for an objectid/georeferenceid for a existing process ..." # First test if it correctly response in case of existing georef process # Create dummy process georefProc = Georeferenzierungsprozess( mapid=10000023, messtischblattid=90015724, nutzerid=self.user, georefparams=ast.literal_eval( "{'new': { 'source': 'pixel', 'target': 'EPSG:4314',\ 'gcps': [{'source': [8681, 1013], 'target': [8.50000095367432, 54.7000007629395]},\ {'source': [8576, 7372], 'target': [8.50000095367432, 54.5999984741211]},\ {'source': [2358, 7260], 'target': [8.33333301544189, 54.5999984741211]},\ {'source': [2465, 888], 'target': [8.33333301544189, 54.7000007629395]}], 'algorithm':'affine'},\ 'remove': {'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [\ {'source': [483, 7227], 'target': [8.33333301544189, 54.5999984741211]}, {'source': [464, 840], 'target': [8.33333301544189, 54.7000007629395]}]}}" ), clip= "POLYGON((16.9999980926514 51.7999992370605,16.9999980926514 51.9000015258789,17.1666679382324 51.9000015258789,17.1666679382324 51.7999992370605,16.9999980926514 51.7999992370605))", timestamp="2014-08-09 12:20:26", type='new', isactive=True, processed=True, overwrites=0, adminvalidation='') self.dbsession.add(georefProc) self.dbsession.flush() # object id and georeferenceid are different for testing that the georeferenceid beat the objectid params = {'objectid': 'oai:de:slub-dresden:vk:id-10000023'} request = self.getPOSTRequest(params) response = getByObjectId(request) print "Response - %s" % response self.assertEqual(response['extent'], [ 16.9999980926514, 51.7999992370605, 17.1666679382324, 51.9000015258789 ], 'Wrong or missing parameter in response ...') self.assertEqual(response['objectid'], 10000023, 'Wrong or missing parameter in response ...') self.assertEqual(response['timestamp'], "2014-08-09 12:20:26", 'Wrong or missing parameter in response ...') self.assertEqual(response['georeferenceid'], georefProc.id, 'Wrong or missing parameter in response ...') self.assertEqual(response['type'], "update", 'Wrong or missing parameter in response ...') self.assertEqual( response['zoomify'], "http://fotothek.slub-dresden.de/zooms/df/dk/0010000/df_dk_0010001_3352_1918/ImageProperties.xml", 'Wrong or missing parameter in response ...') self.dbsession.delete(georefProc)
def runningNewJobs(dbsession, logger): """ Runs the persistent georeference job for new georeference jobs :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :return: int Number of processed jobs """ logger.info('Check for unprocessed new georeference jobs ...') unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeNew(dbsession) counter = 0 for job in unprocessedJobs: logger.info('Start processing of a "new" georeference process with id - %s'%job.id) georefObj = Georeferenzierungsprozess.clearRaceConditions(job, dbsession) mapObj = Map.by_id(georefObj.mapid, dbsession) activate(georefObj, mapObj, dbsession, logger) logger.info('Finish processing of a "new" georeference process with id - %s'%job.id) counter += 1 return counter
def setIsValide(job, dbsession, logger, testing=False): """ This function sets a georeference process as 'isvalide' and if there is no other newer georeference process for this map its activate this georeference process for this map. :type georeference.models.vkdb.adminjobs.AdminJobs: job :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :type boolean: testing (Default: False) """ logger.debug('Set georeference process for id %s to isvalide ...' % (job.georefid)) # set georeferenceprocess to isvalide georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession) # check if there is an other process which is newer and more up to date activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId( georefProcess.mapid, dbsession) mapObj = Map.by_id(georefProcess.mapid, dbsession) if activeGeorefProcess and activeGeorefProcess.id >= georefProcess.id: logger.info( 'The georeference process with the id %s or younger process is already active for this map object.' % georefProcess.id) pass elif activeGeorefProcess and activeGeorefProcess.id < georefProcess.id: logger.info( 'Activate the is valide georeference process and deactive old one ...' ) deactivate(activeGeorefProcess, mapObj, dbsession, logger) activate(georefProcess, mapObj, dbsession, logger) else: logger.info( 'Activate georeference process %s for the map object %s ...' % (georefProcess.id, georefProcess.mapid)) activate(georefProcess, mapObj, dbsession, logger) if georefProcess.adminvalidation == 'invalide': mapObj.hasgeorefparams = mapObj.hasgeorefparams + 1 georefProcess.adminvalidation = 'isvalide'
def runningUpdateJobs(dbsession, logger): """ Runs the persistent georeference job for update georeference jobs :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :return: int Number of processed jobs """ logger.info('Check for unprocessed update georeference jobs ...') unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeUpdate( dbsession) counter = 0 for job in unprocessedJobs: logger.info( 'Start processing of a "update" georeference process with id - %s' % job.id) georefObj = Georeferenzierungsprozess.clearRaceConditions( job, dbsession) # get active georeference process and deactive him, if exist activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId( georefObj.mapid, dbsession) mapObj = Map.by_id(georefObj.mapid, dbsession) if activeGeorefProcess: logger.info('Deactivate georeference processes with id %s ...' % activeGeorefProcess.id) deactivate(activeGeorefProcess, mapObj, dbsession, logger) logger.info('Activate georeference processes with id %s ...' % georefObj.id) activate(georefObj, mapObj, dbsession, logger) logger.info( 'Finish processing of a "update" georeference process with id - %s' % job.id) counter += 1 return counter
def getLastValidGeoreferenceProcess(overwritesId, dbsession, logger): """ This function goes down the overwrite chain and looks for the last valid georeference process :type Integer: job :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :return georeference.models.vkdb.georeferenzierungsprozess.Georeferenzierungsprozess """ georefProcess = Georeferenzierungsprozess.by_id(overwritesId, dbsession) if georefProcess.adminvalidation == 'isvalide' or georefProcess.adminvalidation == '': return georefProcess elif georefProcess.overwrites > 0: return getLastValidGeoreferenceProcess(georefProcess.overwrites, dbsession, logger) else: return None
def checkIfPendingProcessesExist(mapObj, request): """ Checks if there are process in the database which are not yet processed and through this race condition exists. :type mapObj: vkviewer.python.models.vkdb.Map :type request: pyramid.request :return: bool """ thereArePendingProcesses = False try: if Georeferenzierungsprozess.arePendingProcessForMapId(mapObj.id, request.db): thereArePendingProcesses = True except ProcessIsInvalideException as e: # This exception catch the case that are right now no process process registered # in the database for the given mapObj id pass return thereArePendingProcesses
def setUp(self): self.config = testing.setUp() self.config.registry.dbmaker = self.Session # create dummy georefprocess self.notReferencedObjId = 10000023 self.dummyProcess = Georeferenzierungsprozess( mapid = 10000023, messtischblattid = 90015724, nutzerid = TEST_LOGIN, clipparameter = "{'Test':'Test'}", georefparams = "{}", timestamp = "2014-08-09 12:20:26", type = 'new', clippolygon = '', algorithm = '', isactive = True, processed = False, overwrites = 0, adminvalidation = '') self.dummyParams = { 'georeference': { 'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [ {'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789]}, {'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789]}, {'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605]}, {'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605]} ] }, 'id': 10000023, "clip": { "source":"EPSG:4314", "polygon": [[10.6666660308838, 51.4000015258789],[10.8333339691162, 51.4000015258789],[10.8333339691162, 51.2999992370605], [10.6666660308838, 51.2999992370605],[10.6666660308838, 51.4000015258789]] }, "algorithm": "affine", "userid": TEST_LOGIN } # add test data self.dbsession.add(self.testData['mapObj']) self.dbsession.flush()
def setInValide(job, dbsession, logger, testing=False): """ This function sets a georeference process as 'isvalide' and if there is no other newer georeference process for this map its activate this georeference process for this map. :type georeference.models.vkdb.adminjobs.AdminJobs: job :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :type boolean: testing (Default: False) """ logger.debug('Set georeference process for id %s to isvalide ...' % (job.georefid)) # set georeferenceprocess to isvalide georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession) # update map object and datasource mapObj = Map.by_id(georefProcess.mapid, dbsession) if georefProcess.isactive == True and georefProcess.overwrites > 0: logger.info( 'Deactive georeference process and activate georeference process with id %s ...' % georefProcess.overwrites) # deactive the georeference process deactivate(georefProcess, mapObj, dbsession, logger) # look if there is a valid overwrite id and if yes activate the matching # process newGeorefProcess = getLastValidGeoreferenceProcess( georefProcess.overwrites, dbsession, logger) if newGeorefProcess: activate(newGeorefProcess, mapObj, dbsession, logger) elif georefProcess.isactive == True and georefProcess.overwrites == 0: logger.info('Deactive georeference process %s ...' % georefProcess.overwrites) # deactive the georeference process deactivate(georefProcess, mapObj, dbsession, logger) logger.debug('Set georeference process with id %s to inactive ...' % georefProcess.overwrites) if georefProcess.adminvalidation != 'invalide': georefProcess.adminvalidation = 'invalide' mapObj.hasgeorefparams = 0 if mapObj.hasgeorefparams - 1 < 0 else mapObj.hasgeorefparams - 1
def setProcessToIsValide(request): LOGGER.info('Request - Set admin evaluation of georeference process to isvalide ....') try: # remove georeference process if 'georeferenceid' in request.params: georeferenceid = request.params['georeferenceid'] comment = '' if 'comment' not in request.params else request.params['comment'] # check if georeference id exist georeferenceprocess = Georeferenzierungsprozess.by_id(georeferenceid, request.db) if georeferenceprocess: newJob = createNewAdminJob(georeferenceprocess, 'isvalide', request.params['userid'], comment) request.db.add(newJob) return {'message':'The georeference process has been set to isvalide.'} else: raise Exception('Missing parameter (georeferenceid) ...') except Exception as e: LOGGER.error(e) LOGGER.error(traceback.format_exc()) return HTTPBadRequest(GENERAL_ERROR_MESSAGE);
def setInValide(job, dbsession, logger, testing = False): """ This function sets a georeference process as 'isvalide' and if there is no other newer georeference process for this map its activate this georeference process for this map. :type georeference.models.vkdb.adminjobs.AdminJobs: job :type sqlalchemy.orm.session.Session: dbsession :type logging.Logger: logger :type boolean: testing (Default: False) """ logger.debug('Set georeference process for id %s to isvalide ...'%(job.georefid)) # set georeferenceprocess to isvalide georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession) # update map object and datasource mapObj = Map.by_id(georefProcess.mapid, dbsession) if georefProcess.isactive == True and georefProcess.overwrites > 0: logger.info('Deactive georeference process and activate georeference process with id %s ...'%georefProcess.overwrites) # deactive the georeference process deactivate(georefProcess, mapObj, dbsession, logger) # look if there is a valid overwrite id and if yes activate the matching # process newGeorefProcess = getLastValidGeoreferenceProcess(georefProcess.overwrites, dbsession, logger) if newGeorefProcess: activate(newGeorefProcess, mapObj, dbsession, logger) elif georefProcess.isactive == True and georefProcess.overwrites == 0: logger.info('Deactive georeference process %s ...'%georefProcess.overwrites) # deactive the georeference process deactivate(georefProcess, mapObj, dbsession, logger) logger.debug('Set georeference process with id %s to inactive ...'%georefProcess.overwrites) if georefProcess.adminvalidation != 'invalide': georefProcess.adminvalidation = 'invalide' mapObj.hasgeorefparams = 0 if mapObj.hasgeorefparams - 1 < 0 else mapObj.hasgeorefparams - 1
def generateGeoreferenceProcessForMapObj(mapObjId, request, log): """ Function generates a process process for a given map object id :type mapObjId: int :type request: pyramid.request :type log: logging.Logger :return: dict """ def getMtbGLSpecificGeoreferenceInformation(mapObj, request): """ Query the specific process base data for a messtischblatt/geological map :type mapObj: georeference.models.vkdb.map.Map :type request: pyramid.request :return: dict """ srid = mapObj.getSRID(request.db) extent = mapObj.getExtent(request.db, srid) return { 'georeference': { 'source':'pixel', 'target':'EPSG:%s'%srid, 'gcps': [ {"source":[], "target":[extent[0],extent[1]]}, {"source":[], "target":[extent[0],extent[3]]}, {"source":[], "target":[extent[2],extent[1]]}, {"source":[], "target":[extent[2],extent[3]]} ], 'algorithm': 'affine' }, 'extent':extent } mapObj = Map.by_id(mapObjId, request.db) # get general metadata log.debug('Get general process process information ...') generalMetadata = getGeneralMetadata(mapObj, request) # check if there exist already a activate process process for # this mapObj log.debug('Get specific process process information ...') if Georeferenzierungsprozess.isGeoreferenced(mapObj.id, request.db): # there does exist a process process for this mapObj georefProcessObj = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(mapObj.id, request.db) georeferenceData = getSpecificGeoreferenceData(georefProcessObj, mapObj, 4326, request.db) else: # there does not exist a process process for this mapObj georeferenceData = { "timestamp": "", "type": "new" } # This blog could be used for attaching information regarding the boundingbox and the extent to the # response. In case no georeference information is registered yet this could be used by the client for # doing a matching of corner point coordinates and the extent corners. # # log.debug('Check if there is special behavior needed in case of messtischblatt') # mtbGeorefBaseData = {} # if mapObj.maptype == 'M' and 'georeference' not in georeferenceData and mapObj.boundingbox is not None: # mtbGeorefBaseData = getMtbGLSpecificGeoreferenceInformation(mapObj, request) # response.update(mtbGeorefBaseData) log.debug('Check if there are pending processes in the database') warnMsg = {} if checkIfPendingProcessesExist(mapObj, request): warnMsg["warn"] = 'Right now another users is working on the georeferencing of this map sheet. For preventing information losses please try again in 15 minutes.' # now merge dictionaries and create response response = { "recommendedsrid": mapObj.recommendedsrid } response.update(generalMetadata) response.update(georeferenceData) response.update(warnMsg) return response
def setUp(self): self.config = testing.setUp() self.config.registry.dbmaker = self.Session # create dummy georefprocess self.notReferencedObjId = 10000023 self.dummyProcess = Georeferenzierungsprozess( mapid=10000023, messtischblattid=90015724, nutzerid=self.user, clipparameter="{'Test':'Test'}", georefparams="{'Test':'Test'}", timestamp="2014-08-09 12:20:26", type='new', isactive=True, processed=False, overwrites=0, adminvalidation='') self.dummyProcessUpdate = Georeferenzierungsprozess( mapid=10000023, messtischblattid=90015724, nutzerid=self.user, clipparameter= "{'new': {'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [\ {'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789]}, \ {'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789]}, \ {'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605]},\ {'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605]}]},\ 'remove':{'source': 'pixel', 'target': 'EPSG:4314', 'gcps':[]}}", georefparams= "{'new': {'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [\ {'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789]}, \ {'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789]}, \ {'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605]},\ {'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605]}]},\ 'remove':{'source': 'pixel', 'target': 'EPSG:4314', 'gcps':[]}}", timestamp="2014-08-09 12:20:26", type='update', isactive=False, processed=False, overwrites=0, adminvalidation='') # create and insert test data to database self.testData = [ Map(id=10000023, apsobjectid=90015724, apsdateiname="df_dk_0010001_3352_1890", boundingbox= "POLYGON((16.9999980926514 51.7999992370605,16.9999980926514 51.9000015258789,17.1666679382324 51.9000015258789,17.1666679382324 51.7999992370605,16.9999980926514 51.7999992370605))", maptype="M"), Map(id=10000024, apsobjectid=90015725, apsdateiname="df_dk_0010001_3352_18901", maptype="GL"), Metadata( mapid=10000023, imagezoomify= 'http://fotothek.slub-dresden.de/zooms/df/dk/0010000/df_dk_0010001_3352_1918/ImageProperties.xml', title='', titleshort=''), Metadata( mapid=10000024, imagezoomify= 'http://fotothek.slub-dresden.de/zooms/df/dk/0010000/df_dk_0010001_3352_1918/ImageProperties.xml', title='', titleshort='') ] try: for obj in self.testData: self.dbsession.add(obj) self.dbsession.flush() except Exception: raise
Copyright (c) 2015 Jacob Mendt Created on 02.10.15 @author: mendt @description: The following scripts pushs all database records to elasticsearch ''' import logging from georeference.settings import DBCONFIG_PARAMS from georeference.models.meta import getPostgresEngineString from georeference.models.meta import initializeDb from georeference.models.vkdb.georeferenzierungsprozess import Georeferenzierungsprozess from georeference.models.vkdb.map import Map from georeference.persistent.jobs.genericjobs import pushRecordToSearchIndex from georeference.persistent.jobs.genericjobs import removeRecordFromSearchIndex if __name__ == '__main__': logging.basicConfig() logger = logging.getLogger('Push recrords to ES') dbsession = initializeDb(getPostgresEngineString(DBCONFIG_PARAMS), False) maps = Map.all(dbsession) for mapObj in maps: if mapObj.istaktiv == True: print 'Push map record %s to elastic search ...' % mapObj.id georefObj = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(mapObj.id, dbsession) pushRecordToSearchIndex(mapObj, dbsession, logger, georefObj) else: print 'Remove map record %s from elastic search ...' % mapObj.id removeRecordFromSearchIndex(mapObj)
# -*- coding: utf-8 -*- ''' Copyright (c) 2015 Jacob Mendt Created on 02.10.15 @author: mendt @description: The following scripts updates the database path for all map files. ''' import ast from georeference.settings import DBCONFIG_PARAMS from georeference.models.meta import getPostgresEngineString from georeference.models.meta import initializeDb from georeference.models.vkdb.georeferenzierungsprozess import Georeferenzierungsprozess DATA_DIRECTORY = '/srv/vk/data/original' if __name__ == '__main__': dbsession = initializeDb(getPostgresEngineString(DBCONFIG_PARAMS)) georefObjs = Georeferenzierungsprozess.all(dbsession) for georefObj in georefObjs: clipparams = ast.literal_eval(georefObj.clipparameter) if georefObj.type == 'update' and 'new' in clipparams: newParams = clipparams['new'] georefObj.clipparameter = str(newParams) georefObj.georefparams = newParams print "Update georeference process with id %s"%georefObj.id dbsession.commit()
:return: """ print 'Update clip for georeference process with id %s ...' % georefObj.id # generate clip polygon polygon = georefObj.clippolygon['polygon'] gcps = parseGcps(georefObj.georefparams['gcps']) geoTransform = gdal.GCPsToGeoTransform(gcps) transformedClipPolygon = transformClipPolygon(polygon, geoTransform) correctedPolygon = sortPolygonCoordinates(transformedClipPolygon) # generate clip polygon string polygonAsStr = 'POLYGON((' for coordinate in correctedPolygon: polygonAsStr += str(coordinate[0]) + ' ' + str(coordinate[1]) + ',' polygonAsStr = polygonAsStr[:-1] + '))' # get srid srid = georefObj.georefparams['target'].split(':')[1] # update database if len(correctedPolygon) > 4: georefObj.setClip(polygonAsStr, srid, dbsession) if __name__ == '__main__': dbsession = initializeDb(getPostgresEngineString(DBCONFIG_PARAMS), False) processes = Georeferenzierungsprozess.all(dbsession) for georefObj in processes: if georefObj.clippolygon is not None and len(georefObj.georefparams['gcps']) == 4 and georefObj.id < 6934: updateClipPolygonFromOldPolygon(georefObj) dbsession.commit()
def georeferenceConfirm(request): """ The function persistent saves a georeference confirmation request :type request: pyramid.request :return: dict :raise: HTTPBadRequest :raise: HTTPInternalServerError """ LOGGER.info('Receive georeference validation request with parameters: %s' % request.json_body) try: # extract validation data LOGGER.debug('Parse request params ...') requestData = parseGeoreferenceParamsFromRequest(request) # in case of type new, # check if there already exist an actual georeference process for this object with this type if requestData[ 'type'] == 'new' and Georeferenzierungsprozess.isGeoreferenced( requestData['mapObj'].id, request.db): msg = 'There exists already an active georeference process for this map id. It is therefore not possible to save a georeference process of type "new".' LOGGER.debug(msg) georeferenceid = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId( requestData['mapObj'].id, request.db).id return {'text': msg, 'georeferenceid': georeferenceid} LOGGER.debug('Save georeference parameter in datase ...') timestamp = convertTimestampToPostgisString(datetime.now()) georefParam = str(convertUnicodeDictToUtf(requestData['georeference'])) overwrites = requestData[ 'overwrites'] if 'overwrites' in requestData else 0 georefProcess = Georeferenzierungsprozess( messtischblattid=requestData['mapObj'].apsobjectid, nutzerid=requestData['userid'], georefparams=ast.literal_eval(georefParam), timestamp=timestamp, isactive=False, type=requestData['type'], overwrites=overwrites, adminvalidation='', processed=False, mapid=requestData['mapObj'].id, algorithm=requestData['georeference']['algorithm']) request.db.add(georefProcess) request.db.flush() # add polygon if 'clip' in requestData: clipParam = convertUnicodeDictToUtf(requestData['clip']) polygonAsString = convertListToPostgisString( clipParam['polygon'], 'POLYGON') georefProcess.setClip(polygonAsString, stripSRIDFromEPSG(clipParam['source']), request.db) LOGGER.debug('Create response ...') # @TODO has to be updated points = int(len(requestData['georeference']['gcps']) * 5) return { 'text': 'Georeference result saved. It will soon be ready for use.', 'georeferenceid': georefProcess.id, 'points': points } except ParameterException as e: LOGGER.error(e) LOGGER.error(traceback.format_exc()) raise HTTPBadRequest(ERROR_MSG) except Exception as e: LOGGER.error(e) LOGGER.error(traceback.format_exc()) raise HTTPInternalServerError(ERROR_MSG)
def setUp(self): self.process = Georeferenzierungsprozess( mapid=10002567, nutzerid=TEST_LOGIN, clippolygon={ 'source': 'pixel', 'polygon': [[467, 923], [7281, 999], [7224, 7432], [258, 7471], [467, 923]] }, georefparams={ 'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [{ 'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789] }, { 'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789] }, { 'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605] }, { 'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605] }], "algorithm": "affine", }, timestamp="2014-08-09 12:20:26", type='new', algorithm='affine', isactive=True, processed=False, overwrites=0, adminvalidation='') self.map = Map( id=10002567, apsobjectid=90015724, apsdateiname="df_dk_0010001_4630_1928", boundingbox= "POLYGON((10.6666660308838 51.2999992370605,10.6666660308838 51.4000015258789,10.8333339691162 51.4000015258789,10.8333339691162 51.2999992370605,10.6666660308838 51.2999992370605))", originalimage=os.path.join(TEST_DATA_DIR, "df_dk_0010001_4630_1928.tif")) self.metadata = Metadata( mapid=10002567, imagezoomify= 'http://fotothek.slub-dresden.de/zooms/df/dk/0010000/df_dk_0010001_3352_1918/ImageProperties.xml', title='', titleshort='', scale='1:25000', timepublish=datetime.now(), imagejpg= 'http://fotothek.slub-dresden.de/fotos/df/dk/0010000/df_dk_0010001_2655.jpg', thumbssmall= 'http://fotothek.slub-dresden.de/thumbs/df/dk/0010000/df_dk_0010001_6817.jpg', description= 'Ars an der Mosel. - Aufn. 1880, hrsg. 1882, Aufldr. 1916. - 1:25000. - [Berlin]: Kgl. Preuss. Landesaufnahme, 1916. - 1 Kt.', technic='Lithografie & Umdruck', type='Druckgraphik') try: self.dbsession.add(self.metadata) self.dbsession.add(self.map) self.dbsession.flush() except Exception: raise
Created on 02.10.15 @author: mendt @description: The following scripts pushs all database records to elasticsearch ''' import logging from georeference.settings import DBCONFIG_PARAMS from georeference.models.meta import getPostgresEngineString from georeference.models.meta import initializeDb from georeference.models.vkdb.georeferenzierungsprozess import Georeferenzierungsprozess from georeference.models.vkdb.map import Map from georeference.persistent.jobs.genericjobs import pushRecordToSearchIndex from georeference.persistent.jobs.genericjobs import removeRecordFromSearchIndex if __name__ == '__main__': logging.basicConfig() logger = logging.getLogger('Push recrords to ES') dbsession = initializeDb(getPostgresEngineString(DBCONFIG_PARAMS), False) maps = Map.all(dbsession) for mapObj in maps: if mapObj.istaktiv == True: print 'Push map record %s to elastic search ...' % mapObj.id georefObj = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId( mapObj.id, dbsession) pushRecordToSearchIndex(mapObj, dbsession, logger, georefObj) else: print 'Remove map record %s from elastic search ...' % mapObj.id removeRecordFromSearchIndex(mapObj)