def setIsValide(job, dbsession, logger, testing = False):
""" This function sets a georeference process as 'isvalide' and if there is no other
newer georeference process for this map its activate this georeference process for this
map.
:type georeference.models.vkdb.adminjobs.AdminJobs: job
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:type boolean: testing (Default: False) """
logger.debug('Set georeference process for id %s to isvalide ...'%(job.georefid))
# set georeferenceprocess to isvalide
georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession)
# check if there is an other process which is newer and more up to date
activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(georefProcess.mapid, dbsession)
mapObj = Map.by_id(georefProcess.mapid, dbsession)
if activeGeorefProcess and activeGeorefProcess.id >= georefProcess.id:
logger.info('The georeference process with the id %s or younger process is already active for this map object.'%georefProcess.id)
pass
elif activeGeorefProcess and activeGeorefProcess.id < georefProcess.id:
logger.info('Activate the is valide georeference process and deactive old one ...')
deactivate(activeGeorefProcess, mapObj, dbsession, logger)
activate(georefProcess, mapObj, dbsession, logger)
else:
logger.info('Activate georeference process %s for the map object %s ...'%(georefProcess.id, georefProcess.mapid))
activate(georefProcess, mapObj, dbsession, logger)
if georefProcess.adminvalidation == 'invalide':
mapObj.hasgeorefparams = mapObj.hasgeorefparams + 1
georefProcess.adminvalidation = 'isvalide'
def runningNewJobs(dbsession, logger):
""" Runs the persistent georeference job for new georeference jobs
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:return: int Number of processed jobs
"""
logger.info('Check for unprocessed new georeference jobs ...')
unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeNew(
dbsession)
counter = 0
for job in unprocessedJobs:
logger.info(
'Start processing of a "new" georeference process with id - %s' %
job.id)
georefObj = Georeferenzierungsprozess.clearRaceConditions(
job, dbsession)
mapObj = Map.by_id(georefObj.mapid, dbsession)
activate(georefObj, mapObj, dbsession, logger)
logger.info(
'Finish processing of a "new" georeference process with id - %s' %
job.id)
counter += 1
return counter
def runningUpdateJobs(dbsession, logger):
""" Runs the persistent georeference job for update georeference jobs
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:return: int Number of processed jobs
"""
logger.info('Check for unprocessed update georeference jobs ...')
unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeUpdate(dbsession)
counter = 0
for job in unprocessedJobs:
logger.info('Start processing of a "update" georeference process with id - %s'%job.id)
georefObj = Georeferenzierungsprozess.clearRaceConditions(job, dbsession)
# get active georeference process and deactive him, if exist
activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(georefObj.mapid, dbsession)
mapObj = Map.by_id(georefObj.mapid, dbsession)
if activeGeorefProcess:
logger.info('Deactivate georeference processes with id %s ...'%activeGeorefProcess.id)
deactivate(activeGeorefProcess, mapObj, dbsession, logger)
logger.info('Activate georeference processes with id %s ...'%georefObj.id)
activate(georefObj, mapObj, dbsession, logger)
logger.info('Finish processing of a "update" georeference process with id - %s'%job.id)
counter += 1
return counter
def setProcessToIsValide(request):
LOGGER.info(
'Request - Set admin evaluation of georeference process to isvalide ....'
)
try:
# remove georeference process
if 'georeferenceid' in request.params:
georeferenceid = request.params['georeferenceid']
comment = '' if 'comment' not in request.params else request.params[
'comment']
# check if georeference id exist
georeferenceprocess = Georeferenzierungsprozess.by_id(
georeferenceid, request.db)
if georeferenceprocess:
newJob = createNewAdminJob(georeferenceprocess, 'isvalide',
request.params['userid'], comment)
request.db.add(newJob)
return {
'message': 'The georeference process has been set to isvalide.'
}
else:
raise Exception('Missing parameter (georeferenceid) ...')
except Exception as e:
LOGGER.error(e)
LOGGER.error(traceback.format_exc())
return HTTPBadRequest(GENERAL_ERROR_MESSAGE)
def setUp(self):
self.process = Georeferenzierungsprozess(
mapid = 10002567,
nutzerid = TEST_LOGIN,
clippolygon = {
'source':'pixel',
'polygon': [[467, 923],[7281, 999],[7224, 7432],[258, 7471],[467, 923]]},
georefparams = {
'source': 'pixel',
'target': 'EPSG:4314',
'gcps': [
{'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789]},
{'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789]},
{'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605]},
{'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605]}
],
"algorithm": "affine",
},
timestamp = "2014-08-09 12:20:26",
type = 'new',
algorithm = 'affine',
isactive = True,
processed = False,
overwrites = 0,
adminvalidation = '')
self.map = Map(id = 10002567, apsobjectid=90015724, apsdateiname = "df_dk_0010001_4630_1928",
boundingbox = "POLYGON((10.6666660308838 51.2999992370605,10.6666660308838 51.4000015258789,10.8333339691162 51.4000015258789,10.8333339691162 51.2999992370605,10.6666660308838 51.2999992370605))",
originalimage = os.path.join(TEST_DATA_DIR, "df_dk_0010001_4630_1928.tif"))
def generateGeoreferenceProcessForSpecificGeoreferenceId(georeferenceId, request, log):
""" Function generates a process process for a given map object id
:type georeferenceId: int
:type request: pyramid.request
:type log: logging.Logger
:return: dict """
georefProcessObj = Georeferenzierungsprozess.by_id(georeferenceId, request.db)
mapObj = Map.by_id(georefProcessObj.mapid, request.db)
# get general metadata
log.debug('Get general process process information ...')
generalMetadata = getGeneralMetadata(mapObj, request)
# check if there exist already a activate process process for
# this mapObj
log.debug('Get specific process process information ...')
georeferenceData = getSpecificGeoreferenceData(georefProcessObj, mapObj, 4326, request.db)
log.debug('Check if there are pending processes in the database')
warnMsg = {}
if checkIfPendingProcessesExist(mapObj, request):
warnMsg["warn"] = 'Right now another users is working on the georeferencing of this map sheet. For preventing information losses please try again in 15 minutes.'
# now merge dictionaries and create response
response = {}
response.update(generalMetadata)
response.update(georeferenceData)
response.update(warnMsg)
return response
def setUp(self):
# create and insert test data to database
self.testData = [
Map(
id = 10000023,
apsobjectid=90015724,
apsdateiname = "df_dk_0010001_4630_1928",
boundingbox = "POLYGON((16.9999980926514 51.7999992370605,16.9999980926514 51.9000015258789,17.1666679382324 51.9000015258789,17.1666679382324 51.7999992370605,16.9999980926514 51.7999992370605))",
maptype="M",
originalimage = os.path.join(self.testDataDir, "df_dk_0010001_4630_1928.tif"),
georefimage = os.path.join(self.testDataDir, "df_dk_0010001_4630_1928.tif"),
isttransformiert = True,
recommendedsrid = 4314
),
Metadata(
mapid = 10000023,
imagezoomify = 'http://fotothek.slub-dresden.de/zooms/df/dk/0010000/df_dk_0010001_3352_1918/ImageProperties.xml',
title = '',
titleshort = '',
scale = '1:25000',
timepublish = datetime.now(),
imagejpg = 'http://fotothek.slub-dresden.de/fotos/df/dk/0010000/df_dk_0010001_2655.jpg',
thumbssmall = 'http://fotothek.slub-dresden.de/thumbs/df/dk/0010000/df_dk_0010001_6817.jpg',
description = 'Ars an der Mosel. - Aufn. 1880, hrsg. 1882, Aufldr. 1916. - 1:25000. - [Berlin]: Kgl. Preuss. Landesaufnahme, 1916. - 1 Kt.',
technic = 'Lithografie & Umdruck',
type = 'Druckgraphik'
)
]
self.georefProcess = Georeferenzierungsprozess(
mapid = 10000023,
messtischblattid = 90015724,
nutzerid = self.user,
clipparameter = {'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [\
{'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789]}, \
{'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789]}, \
{'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605]},\
{'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605]}]},
georefparams = {'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [\
{'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789]}, \
{'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789]}, \
{'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605]},\
{'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605]}]},
clippolygon = {'source': 'pixel', 'polygon': [[7813, 7517], [1652, 7523], [1677, 1666], [7830, 1661], [7813, 7517]]},
timestamp = "2014-08-09 12:20:26",
type = 'update',
isactive = False,
processed = False,
overwrites = 0,
adminvalidation = ''
)
try:
for obj in self.testData:
self.dbsession.add(obj)
self.dbsession.flush()
except Exception:
raise
def testGetProcessWithTwoParam(self):
print "--------------------------------------------------------------------------------------------"
print "Testing correct working of georeferenceGetProcess for an objectid/georeferenceid for a existing process ..."
# First test if it correctly response in case of existing georef process
# Create dummy process
georefProc = Georeferenzierungsprozess(
mapid=10000023,
messtischblattid=90015724,
nutzerid=self.user,
georefparams=ast.literal_eval(
"{'new': { 'source': 'pixel', 'target': 'EPSG:4314',\
'gcps': [{'source': [8681, 1013], 'target': [8.50000095367432, 54.7000007629395]},\
{'source': [8576, 7372], 'target': [8.50000095367432, 54.5999984741211]},\
{'source': [2358, 7260], 'target': [8.33333301544189, 54.5999984741211]},\
{'source': [2465, 888], 'target': [8.33333301544189, 54.7000007629395]}], 'algorithm':'affine'},\
'remove': {'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [\
{'source': [483, 7227], 'target': [8.33333301544189, 54.5999984741211]}, {'source': [464, 840], 'target': [8.33333301544189, 54.7000007629395]}]}}"
),
clip=
"POLYGON((16.9999980926514 51.7999992370605,16.9999980926514 51.9000015258789,17.1666679382324 51.9000015258789,17.1666679382324 51.7999992370605,16.9999980926514 51.7999992370605))",
timestamp="2014-08-09 12:20:26",
type='new',
isactive=True,
processed=True,
overwrites=0,
adminvalidation='')
self.dbsession.add(georefProc)
self.dbsession.flush()
# object id and georeferenceid are different for testing that the georeferenceid beat the objectid
params = {'objectid': 'oai:de:slub-dresden:vk:id-10000023'}
request = self.getPOSTRequest(params)
response = getByObjectId(request)
print "Response - %s" % response
self.assertEqual(response['extent'], [
16.9999980926514, 51.7999992370605, 17.1666679382324,
51.9000015258789
], 'Wrong or missing parameter in response ...')
self.assertEqual(response['objectid'], 10000023,
'Wrong or missing parameter in response ...')
self.assertEqual(response['timestamp'], "2014-08-09 12:20:26",
'Wrong or missing parameter in response ...')
self.assertEqual(response['georeferenceid'], georefProc.id,
'Wrong or missing parameter in response ...')
self.assertEqual(response['type'], "update",
'Wrong or missing parameter in response ...')
self.assertEqual(
response['zoomify'],
"http://fotothek.slub-dresden.de/zooms/df/dk/0010000/df_dk_0010001_3352_1918/ImageProperties.xml",
'Wrong or missing parameter in response ...')
self.dbsession.delete(georefProc)
def runningNewJobs(dbsession, logger):
""" Runs the persistent georeference job for new georeference jobs
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:return: int Number of processed jobs
"""
logger.info('Check for unprocessed new georeference jobs ...')
unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeNew(dbsession)
counter = 0
for job in unprocessedJobs:
logger.info('Start processing of a "new" georeference process with id - %s'%job.id)
georefObj = Georeferenzierungsprozess.clearRaceConditions(job, dbsession)
mapObj = Map.by_id(georefObj.mapid, dbsession)
activate(georefObj, mapObj, dbsession, logger)
logger.info('Finish processing of a "new" georeference process with id - %s'%job.id)
counter += 1
return counter
def setIsValide(job, dbsession, logger, testing=False):
""" This function sets a georeference process as 'isvalide' and if there is no other
newer georeference process for this map its activate this georeference process for this
map.
:type georeference.models.vkdb.adminjobs.AdminJobs: job
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:type boolean: testing (Default: False) """
logger.debug('Set georeference process for id %s to isvalide ...' %
(job.georefid))
# set georeferenceprocess to isvalide
georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession)
# check if there is an other process which is newer and more up to date
activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(
georefProcess.mapid, dbsession)
mapObj = Map.by_id(georefProcess.mapid, dbsession)
if activeGeorefProcess and activeGeorefProcess.id >= georefProcess.id:
logger.info(
'The georeference process with the id %s or younger process is already active for this map object.'
% georefProcess.id)
pass
elif activeGeorefProcess and activeGeorefProcess.id < georefProcess.id:
logger.info(
'Activate the is valide georeference process and deactive old one ...'
)
deactivate(activeGeorefProcess, mapObj, dbsession, logger)
activate(georefProcess, mapObj, dbsession, logger)
else:
logger.info(
'Activate georeference process %s for the map object %s ...' %
(georefProcess.id, georefProcess.mapid))
activate(georefProcess, mapObj, dbsession, logger)
if georefProcess.adminvalidation == 'invalide':
mapObj.hasgeorefparams = mapObj.hasgeorefparams + 1
georefProcess.adminvalidation = 'isvalide'
def runningUpdateJobs(dbsession, logger):
""" Runs the persistent georeference job for update georeference jobs
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:return: int Number of processed jobs
"""
logger.info('Check for unprocessed update georeference jobs ...')
unprocessedJobs = Georeferenzierungsprozess.getUnprocessedObjectsOfTypeUpdate(
dbsession)
counter = 0
for job in unprocessedJobs:
logger.info(
'Start processing of a "update" georeference process with id - %s'
% job.id)
georefObj = Georeferenzierungsprozess.clearRaceConditions(
job, dbsession)
# get active georeference process and deactive him, if exist
activeGeorefProcess = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(
georefObj.mapid, dbsession)
mapObj = Map.by_id(georefObj.mapid, dbsession)
if activeGeorefProcess:
logger.info('Deactivate georeference processes with id %s ...' %
activeGeorefProcess.id)
deactivate(activeGeorefProcess, mapObj, dbsession, logger)
logger.info('Activate georeference processes with id %s ...' %
georefObj.id)
activate(georefObj, mapObj, dbsession, logger)
logger.info(
'Finish processing of a "update" georeference process with id - %s'
% job.id)
counter += 1
return counter
def getLastValidGeoreferenceProcess(overwritesId, dbsession, logger):
""" This function goes down the overwrite chain and looks for the last valid
georeference process
:type Integer: job
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:return georeference.models.vkdb.georeferenzierungsprozess.Georeferenzierungsprozess """
georefProcess = Georeferenzierungsprozess.by_id(overwritesId, dbsession)
if georefProcess.adminvalidation == 'isvalide' or georefProcess.adminvalidation == '':
return georefProcess
elif georefProcess.overwrites > 0:
return getLastValidGeoreferenceProcess(georefProcess.overwrites, dbsession, logger)
else:
return None
def getLastValidGeoreferenceProcess(overwritesId, dbsession, logger):
""" This function goes down the overwrite chain and looks for the last valid
georeference process
:type Integer: job
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:return georeference.models.vkdb.georeferenzierungsprozess.Georeferenzierungsprozess """
georefProcess = Georeferenzierungsprozess.by_id(overwritesId, dbsession)
if georefProcess.adminvalidation == 'isvalide' or georefProcess.adminvalidation == '':
return georefProcess
elif georefProcess.overwrites > 0:
return getLastValidGeoreferenceProcess(georefProcess.overwrites,
dbsession, logger)
else:
return None
def checkIfPendingProcessesExist(mapObj, request):
""" Checks if there are process in the database which are not yet processed and
through this race condition exists.
:type mapObj: vkviewer.python.models.vkdb.Map
:type request: pyramid.request
:return: bool """
thereArePendingProcesses = False
try:
if Georeferenzierungsprozess.arePendingProcessForMapId(mapObj.id, request.db):
thereArePendingProcesses = True
except ProcessIsInvalideException as e:
# This exception catch the case that are right now no process process registered
# in the database for the given mapObj id
pass
return thereArePendingProcesses
def checkIfPendingProcessesExist(mapObj, request):
""" Checks if there are process in the database which are not yet processed and
through this race condition exists.
:type mapObj: vkviewer.python.models.vkdb.Map
:type request: pyramid.request
:return: bool """
thereArePendingProcesses = False
try:
if Georeferenzierungsprozess.arePendingProcessForMapId(mapObj.id, request.db):
thereArePendingProcesses = True
except ProcessIsInvalideException as e:
# This exception catch the case that are right now no process process registered
# in the database for the given mapObj id
pass
return thereArePendingProcesses
def setUp(self):
self.config = testing.setUp()
self.config.registry.dbmaker = self.Session
# create dummy georefprocess
self.notReferencedObjId = 10000023
self.dummyProcess = Georeferenzierungsprozess(
mapid = 10000023,
messtischblattid = 90015724,
nutzerid = TEST_LOGIN,
clipparameter = "{'Test':'Test'}",
georefparams = "{}",
timestamp = "2014-08-09 12:20:26",
type = 'new',
clippolygon = '',
algorithm = '',
isactive = True,
processed = False,
overwrites = 0,
adminvalidation = '')
self.dummyParams = {
'georeference': {
'source': 'pixel',
'target': 'EPSG:4314',
'gcps': [
{'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789]},
{'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789]},
{'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605]},
{'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605]}
]
},
'id': 10000023,
"clip": {
"source":"EPSG:4314",
"polygon": [[10.6666660308838, 51.4000015258789],[10.8333339691162, 51.4000015258789],[10.8333339691162, 51.2999992370605],
[10.6666660308838, 51.2999992370605],[10.6666660308838, 51.4000015258789]]
},
"algorithm": "affine",
"userid": TEST_LOGIN
}
# add test data
self.dbsession.add(self.testData['mapObj'])
self.dbsession.flush()
def setInValide(job, dbsession, logger, testing=False):
""" This function sets a georeference process as 'isvalide' and if there is no other
newer georeference process for this map its activate this georeference process for this
map.
:type georeference.models.vkdb.adminjobs.AdminJobs: job
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:type boolean: testing (Default: False) """
logger.debug('Set georeference process for id %s to isvalide ...' %
(job.georefid))
# set georeferenceprocess to isvalide
georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession)
# update map object and datasource
mapObj = Map.by_id(georefProcess.mapid, dbsession)
if georefProcess.isactive == True and georefProcess.overwrites > 0:
logger.info(
'Deactive georeference process and activate georeference process with id %s ...'
% georefProcess.overwrites)
# deactive the georeference process
deactivate(georefProcess, mapObj, dbsession, logger)
# look if there is a valid overwrite id and if yes activate the matching
# process
newGeorefProcess = getLastValidGeoreferenceProcess(
georefProcess.overwrites, dbsession, logger)
if newGeorefProcess:
activate(newGeorefProcess, mapObj, dbsession, logger)
elif georefProcess.isactive == True and georefProcess.overwrites == 0:
logger.info('Deactive georeference process %s ...' %
georefProcess.overwrites)
# deactive the georeference process
deactivate(georefProcess, mapObj, dbsession, logger)
logger.debug('Set georeference process with id %s to inactive ...' %
georefProcess.overwrites)
if georefProcess.adminvalidation != 'invalide':
georefProcess.adminvalidation = 'invalide'
mapObj.hasgeorefparams = 0 if mapObj.hasgeorefparams - 1 < 0 else mapObj.hasgeorefparams - 1
def setProcessToIsValide(request):
LOGGER.info('Request - Set admin evaluation of georeference process to isvalide ....')
try:
# remove georeference process
if 'georeferenceid' in request.params:
georeferenceid = request.params['georeferenceid']
comment = '' if 'comment' not in request.params else request.params['comment']
# check if georeference id exist
georeferenceprocess = Georeferenzierungsprozess.by_id(georeferenceid, request.db)
if georeferenceprocess:
newJob = createNewAdminJob(georeferenceprocess, 'isvalide', request.params['userid'], comment)
request.db.add(newJob)
return {'message':'The georeference process has been set to isvalide.'}
else:
raise Exception('Missing parameter (georeferenceid) ...')
except Exception as e:
LOGGER.error(e)
LOGGER.error(traceback.format_exc())
return HTTPBadRequest(GENERAL_ERROR_MESSAGE);
def setInValide(job, dbsession, logger, testing = False):
""" This function sets a georeference process as 'isvalide' and if there is no other
newer georeference process for this map its activate this georeference process for this
map.
:type georeference.models.vkdb.adminjobs.AdminJobs: job
:type sqlalchemy.orm.session.Session: dbsession
:type logging.Logger: logger
:type boolean: testing (Default: False) """
logger.debug('Set georeference process for id %s to isvalide ...'%(job.georefid))
# set georeferenceprocess to isvalide
georefProcess = Georeferenzierungsprozess.by_id(job.georefid, dbsession)
# update map object and datasource
mapObj = Map.by_id(georefProcess.mapid, dbsession)
if georefProcess.isactive == True and georefProcess.overwrites > 0:
logger.info('Deactive georeference process and activate georeference process with id %s ...'%georefProcess.overwrites)
# deactive the georeference process
deactivate(georefProcess, mapObj, dbsession, logger)
# look if there is a valid overwrite id and if yes activate the matching
# process
newGeorefProcess = getLastValidGeoreferenceProcess(georefProcess.overwrites, dbsession, logger)
if newGeorefProcess:
activate(newGeorefProcess, mapObj, dbsession, logger)
elif georefProcess.isactive == True and georefProcess.overwrites == 0:
logger.info('Deactive georeference process %s ...'%georefProcess.overwrites)
# deactive the georeference process
deactivate(georefProcess, mapObj, dbsession, logger)
logger.debug('Set georeference process with id %s to inactive ...'%georefProcess.overwrites)
if georefProcess.adminvalidation != 'invalide':
georefProcess.adminvalidation = 'invalide'
mapObj.hasgeorefparams = 0 if mapObj.hasgeorefparams - 1 < 0 else mapObj.hasgeorefparams - 1
def generateGeoreferenceProcessForMapObj(mapObjId, request, log):
""" Function generates a process process for a given map object id
:type mapObjId: int
:type request: pyramid.request
:type log: logging.Logger
:return: dict """
def getMtbGLSpecificGeoreferenceInformation(mapObj, request):
""" Query the specific process base data for a messtischblatt/geological map
:type mapObj: georeference.models.vkdb.map.Map
:type request: pyramid.request
:return: dict """
srid = mapObj.getSRID(request.db)
extent = mapObj.getExtent(request.db, srid)
return {
'georeference': {
'source':'pixel',
'target':'EPSG:%s'%srid,
'gcps': [
{"source":[], "target":[extent[0],extent[1]]},
{"source":[], "target":[extent[0],extent[3]]},
{"source":[], "target":[extent[2],extent[1]]},
{"source":[], "target":[extent[2],extent[3]]}
],
'algorithm': 'affine'
},
'extent':extent
}
mapObj = Map.by_id(mapObjId, request.db)
# get general metadata
log.debug('Get general process process information ...')
generalMetadata = getGeneralMetadata(mapObj, request)
# check if there exist already a activate process process for
# this mapObj
log.debug('Get specific process process information ...')
if Georeferenzierungsprozess.isGeoreferenced(mapObj.id, request.db):
# there does exist a process process for this mapObj
georefProcessObj = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(mapObj.id, request.db)
georeferenceData = getSpecificGeoreferenceData(georefProcessObj, mapObj, 4326, request.db)
else:
# there does not exist a process process for this mapObj
georeferenceData = {
"timestamp": "",
"type": "new"
}
# This blog could be used for attaching information regarding the boundingbox and the extent to the
# response. In case no georeference information is registered yet this could be used by the client for
# doing a matching of corner point coordinates and the extent corners.
#
# log.debug('Check if there is special behavior needed in case of messtischblatt')
# mtbGeorefBaseData = {}
# if mapObj.maptype == 'M' and 'georeference' not in georeferenceData and mapObj.boundingbox is not None:
# mtbGeorefBaseData = getMtbGLSpecificGeoreferenceInformation(mapObj, request)
# response.update(mtbGeorefBaseData)
log.debug('Check if there are pending processes in the database')
warnMsg = {}
if checkIfPendingProcessesExist(mapObj, request):
warnMsg["warn"] = 'Right now another users is working on the georeferencing of this map sheet. For preventing information losses please try again in 15 minutes.'
# now merge dictionaries and create response
response = {
"recommendedsrid": mapObj.recommendedsrid
}
response.update(generalMetadata)
response.update(georeferenceData)
response.update(warnMsg)
return response
def setUp(self):
self.config = testing.setUp()
self.config.registry.dbmaker = self.Session
# create dummy georefprocess
self.notReferencedObjId = 10000023
self.dummyProcess = Georeferenzierungsprozess(
mapid=10000023,
messtischblattid=90015724,
nutzerid=self.user,
clipparameter="{'Test':'Test'}",
georefparams="{'Test':'Test'}",
timestamp="2014-08-09 12:20:26",
type='new',
isactive=True,
processed=False,
overwrites=0,
adminvalidation='')
self.dummyProcessUpdate = Georeferenzierungsprozess(
mapid=10000023,
messtischblattid=90015724,
nutzerid=self.user,
clipparameter=
"{'new': {'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [\
{'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789]}, \
{'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789]}, \
{'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605]},\
{'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605]}]},\
'remove':{'source': 'pixel', 'target': 'EPSG:4314', 'gcps':[]}}",
georefparams=
"{'new': {'source': 'pixel', 'target': 'EPSG:4314', 'gcps': [\
{'source': [467, 923], 'target': [10.6666660308838, 51.4000015258789]}, \
{'source': [7281, 999], 'target': [10.8333339691162, 51.4000015258789]}, \
{'source': [7224, 7432], 'target': [10.8333339691162, 51.2999992370605]},\
{'source': [258, 7471], 'target': [10.6666660308838, 51.2999992370605]}]},\
'remove':{'source': 'pixel', 'target': 'EPSG:4314', 'gcps':[]}}",
timestamp="2014-08-09 12:20:26",
type='update',
isactive=False,
processed=False,
overwrites=0,
adminvalidation='')
# create and insert test data to database
self.testData = [
Map(id=10000023,
apsobjectid=90015724,
apsdateiname="df_dk_0010001_3352_1890",
boundingbox=
"POLYGON((16.9999980926514 51.7999992370605,16.9999980926514 51.9000015258789,17.1666679382324 51.9000015258789,17.1666679382324 51.7999992370605,16.9999980926514 51.7999992370605))",
maptype="M"),
Map(id=10000024,
apsobjectid=90015725,
apsdateiname="df_dk_0010001_3352_18901",
maptype="GL"),
Metadata(
mapid=10000023,
imagezoomify=
'http://fotothek.slub-dresden.de/zooms/df/dk/0010000/df_dk_0010001_3352_1918/ImageProperties.xml',
title='',
titleshort=''),
Metadata(
mapid=10000024,
imagezoomify=
'http://fotothek.slub-dresden.de/zooms/df/dk/0010000/df_dk_0010001_3352_1918/ImageProperties.xml',
title='',
titleshort='')
]
try:
for obj in self.testData:
self.dbsession.add(obj)
self.dbsession.flush()
except Exception:
raise
Copyright (c) 2015 Jacob Mendt
Created on 02.10.15
@author: mendt
@description:
The following scripts pushs all database records to elasticsearch
'''
import logging
from georeference.settings import DBCONFIG_PARAMS
from georeference.models.meta import getPostgresEngineString
from georeference.models.meta import initializeDb
from georeference.models.vkdb.georeferenzierungsprozess import Georeferenzierungsprozess
from georeference.models.vkdb.map import Map
from georeference.persistent.jobs.genericjobs import pushRecordToSearchIndex
from georeference.persistent.jobs.genericjobs import removeRecordFromSearchIndex
if __name__ == '__main__':
logging.basicConfig()
logger = logging.getLogger('Push recrords to ES')
dbsession = initializeDb(getPostgresEngineString(DBCONFIG_PARAMS), False)
maps = Map.all(dbsession)
for mapObj in maps:
if mapObj.istaktiv == True:
print 'Push map record %s to elastic search ...' % mapObj.id
georefObj = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(mapObj.id, dbsession)
pushRecordToSearchIndex(mapObj, dbsession, logger, georefObj)
else:
print 'Remove map record %s from elastic search ...' % mapObj.id
removeRecordFromSearchIndex(mapObj)
# -*- coding: utf-8 -*-
'''
Copyright (c) 2015 Jacob Mendt
Created on 02.10.15
@author: mendt
@description:
The following scripts updates the database path for all map files.
'''
import ast
from georeference.settings import DBCONFIG_PARAMS
from georeference.models.meta import getPostgresEngineString
from georeference.models.meta import initializeDb
from georeference.models.vkdb.georeferenzierungsprozess import Georeferenzierungsprozess
DATA_DIRECTORY = '/srv/vk/data/original'
if __name__ == '__main__':
dbsession = initializeDb(getPostgresEngineString(DBCONFIG_PARAMS))
georefObjs = Georeferenzierungsprozess.all(dbsession)
for georefObj in georefObjs:
clipparams = ast.literal_eval(georefObj.clipparameter)
if georefObj.type == 'update' and 'new' in clipparams:
newParams = clipparams['new']
georefObj.clipparameter = str(newParams)
georefObj.georefparams = newParams
print "Update georeference process with id %s"%georefObj.id
dbsession.commit()
:return:
"""
print 'Update clip for georeference process with id %s ...' % georefObj.id
# generate clip polygon
polygon = georefObj.clippolygon['polygon']
gcps = parseGcps(georefObj.georefparams['gcps'])
geoTransform = gdal.GCPsToGeoTransform(gcps)
transformedClipPolygon = transformClipPolygon(polygon, geoTransform)
correctedPolygon = sortPolygonCoordinates(transformedClipPolygon)
# generate clip polygon string
polygonAsStr = 'POLYGON(('
for coordinate in correctedPolygon:
polygonAsStr += str(coordinate[0]) + ' ' + str(coordinate[1]) + ','
polygonAsStr = polygonAsStr[:-1] + '))'
# get srid
srid = georefObj.georefparams['target'].split(':')[1]
# update database
if len(correctedPolygon) > 4:
georefObj.setClip(polygonAsStr, srid, dbsession)
if __name__ == '__main__':
dbsession = initializeDb(getPostgresEngineString(DBCONFIG_PARAMS), False)
processes = Georeferenzierungsprozess.all(dbsession)
for georefObj in processes:
if georefObj.clippolygon is not None and len(georefObj.georefparams['gcps']) == 4 and georefObj.id < 6934:
updateClipPolygonFromOldPolygon(georefObj)
dbsession.commit()
def georeferenceConfirm(request):
""" The function persistent saves a georeference confirmation request
:type request: pyramid.request
:return: dict
:raise: HTTPBadRequest
:raise: HTTPInternalServerError """
LOGGER.info('Receive georeference validation request with parameters: %s' %
request.json_body)
try:
# extract validation data
LOGGER.debug('Parse request params ...')
requestData = parseGeoreferenceParamsFromRequest(request)
# in case of type new,
# check if there already exist an actual georeference process for this object with this type
if requestData[
'type'] == 'new' and Georeferenzierungsprozess.isGeoreferenced(
requestData['mapObj'].id, request.db):
msg = 'There exists already an active georeference process for this map id. It is therefore not possible to save a georeference process of type "new".'
LOGGER.debug(msg)
georeferenceid = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(
requestData['mapObj'].id, request.db).id
return {'text': msg, 'georeferenceid': georeferenceid}
LOGGER.debug('Save georeference parameter in datase ...')
timestamp = convertTimestampToPostgisString(datetime.now())
georefParam = str(convertUnicodeDictToUtf(requestData['georeference']))
overwrites = requestData[
'overwrites'] if 'overwrites' in requestData else 0
georefProcess = Georeferenzierungsprozess(
messtischblattid=requestData['mapObj'].apsobjectid,
nutzerid=requestData['userid'],
georefparams=ast.literal_eval(georefParam),
timestamp=timestamp,
isactive=False,
type=requestData['type'],
overwrites=overwrites,
adminvalidation='',
processed=False,
mapid=requestData['mapObj'].id,
algorithm=requestData['georeference']['algorithm'])
request.db.add(georefProcess)
request.db.flush()
# add polygon
if 'clip' in requestData:
clipParam = convertUnicodeDictToUtf(requestData['clip'])
polygonAsString = convertListToPostgisString(
clipParam['polygon'], 'POLYGON')
georefProcess.setClip(polygonAsString,
stripSRIDFromEPSG(clipParam['source']),
request.db)
LOGGER.debug('Create response ...')
# @TODO has to be updated
points = int(len(requestData['georeference']['gcps']) * 5)
return {
'text':
'Georeference result saved. It will soon be ready for use.',
'georeferenceid': georefProcess.id,
'points': points
}
except ParameterException as e:
LOGGER.error(e)
LOGGER.error(traceback.format_exc())
raise HTTPBadRequest(ERROR_MSG)
except Exception as e:
LOGGER.error(e)
LOGGER.error(traceback.format_exc())
raise HTTPInternalServerError(ERROR_MSG)
def setUp(self):
self.process = Georeferenzierungsprozess(
mapid=10002567,
nutzerid=TEST_LOGIN,
clippolygon={
'source':
'pixel',
'polygon': [[467, 923], [7281, 999], [7224, 7432], [258, 7471],
[467, 923]]
},
georefparams={
'source':
'pixel',
'target':
'EPSG:4314',
'gcps': [{
'source': [467, 923],
'target': [10.6666660308838, 51.4000015258789]
}, {
'source': [7281, 999],
'target': [10.8333339691162, 51.4000015258789]
}, {
'source': [7224, 7432],
'target': [10.8333339691162, 51.2999992370605]
}, {
'source': [258, 7471],
'target': [10.6666660308838, 51.2999992370605]
}],
"algorithm":
"affine",
},
timestamp="2014-08-09 12:20:26",
type='new',
algorithm='affine',
isactive=True,
processed=False,
overwrites=0,
adminvalidation='')
self.map = Map(
id=10002567,
apsobjectid=90015724,
apsdateiname="df_dk_0010001_4630_1928",
boundingbox=
"POLYGON((10.6666660308838 51.2999992370605,10.6666660308838 51.4000015258789,10.8333339691162 51.4000015258789,10.8333339691162 51.2999992370605,10.6666660308838 51.2999992370605))",
originalimage=os.path.join(TEST_DATA_DIR,
"df_dk_0010001_4630_1928.tif"))
self.metadata = Metadata(
mapid=10002567,
imagezoomify=
'http://fotothek.slub-dresden.de/zooms/df/dk/0010000/df_dk_0010001_3352_1918/ImageProperties.xml',
title='',
titleshort='',
scale='1:25000',
timepublish=datetime.now(),
imagejpg=
'http://fotothek.slub-dresden.de/fotos/df/dk/0010000/df_dk_0010001_2655.jpg',
thumbssmall=
'http://fotothek.slub-dresden.de/thumbs/df/dk/0010000/df_dk_0010001_6817.jpg',
description=
'Ars an der Mosel. - Aufn. 1880, hrsg. 1882, Aufldr. 1916. - 1:25000. - [Berlin]: Kgl. Preuss. Landesaufnahme, 1916. - 1 Kt.',
technic='Lithografie & Umdruck',
type='Druckgraphik')
try:
self.dbsession.add(self.metadata)
self.dbsession.add(self.map)
self.dbsession.flush()
except Exception:
raise
Created on 02.10.15
@author: mendt
@description:
The following scripts pushs all database records to elasticsearch
'''
import logging
from georeference.settings import DBCONFIG_PARAMS
from georeference.models.meta import getPostgresEngineString
from georeference.models.meta import initializeDb
from georeference.models.vkdb.georeferenzierungsprozess import Georeferenzierungsprozess
from georeference.models.vkdb.map import Map
from georeference.persistent.jobs.genericjobs import pushRecordToSearchIndex
from georeference.persistent.jobs.genericjobs import removeRecordFromSearchIndex
if __name__ == '__main__':
logging.basicConfig()
logger = logging.getLogger('Push recrords to ES')
dbsession = initializeDb(getPostgresEngineString(DBCONFIG_PARAMS), False)
maps = Map.all(dbsession)
for mapObj in maps:
if mapObj.istaktiv == True:
print 'Push map record %s to elastic search ...' % mapObj.id
georefObj = Georeferenzierungsprozess.getActualGeoreferenceProcessForMapId(
mapObj.id, dbsession)
pushRecordToSearchIndex(mapObj, dbsession, logger, georefObj)
else:
print 'Remove map record %s from elastic search ...' % mapObj.id
removeRecordFromSearchIndex(mapObj)