def createQueue(self):
queueName = self.tmsConfig.get("General", "sqsqueue")
maxChunks = int(self.tmsConfig.get("General", "maxChunks"))
self.t0 = time.time()
if len(queueName) <= 0:
logger.error("Missing queueName")
return
try:
sqs = getSQS()
q = sqs.get_queue(queueName)
if q is not None:
logger.error("Queue already exists. Can't overwrite existing queue. [%s]" % (queueName))
return
# queue with default message visiblity time of 3600. So each message is blocked
# for other users for 3600 seconds. Default would be 30 seconds. It's that high
# because each message contains maxChunks tiles
q = sqs.create_queue(queueName, visibility_timeout=visibility_timeout)
# Assure queue is kept for maximum of 14 weeks (aws limit). default would be 4 days.
sqs.set_queue_attribute(q, "MessageRetentionPeriod", 1209600)
except Exception as e:
logger.error("Error during creation of queue:\n" + str(e))
return
if q.count() > 0:
logger.error("Queue already contains messages. Use a different queue or delete this queue first")
return
logger.info("Queue " + queueName + " has been created")
tiles = Tiles(self.dbConfigFile, self.tmsConfig, self.t0)
nbTiles = self.numOfTiles()
try:
logger.info("Starting creation of SQS queue with approx. %s tiles)" % (nbTiles))
totalcount = 0
tcount = 0
messagecount = 0
msg = ""
for tile in tiles:
(bounds, tileXYZ, t0, dbConfigFile) = tile
if not msg:
msg += ","
msg += "%s,%s,%s" % (str(tileXYZ[0]), str(tileXYZ[1]), str(tileXYZ[2]))
tcount += 1
if tcount >= maxChunks:
messagecount += 1
writeSQSMessage(q, msg)
tcount = 0
msg = ""
totalcount = totalcount + 1
if not msg:
messagecount += 1
writeSQSMessage(q, msg)
except Exception as e:
logger.error("Error during writing of sqs message:\n" + str(e))
tend = time.time()
logger.info(
"It took %s to create %s message in SQS gueue representing %s tiles"
% (str(datetime.timedelta(seconds=tend - self.t0)), messagecount, totalcount)
)
def createTileFromQueue(tq):
pid = os.getpid()
try:
(qName, t0, dbConfigFile, bucketBasePath, hasLighting, hasWatermask) = tq
sqs = getSQS()
q = sqs.get_queue(qName)
geodetic = getTileGrid(4326)(tmsCompatible=True)
# we do this as long as we are finding messages in the queue
while True:
parseOk = True
try:
# 20 is maximum wait time
m = q.read(
visibility_timeout=visibility_timeout,
wait_time_seconds=20
)
if m is None:
logger.info(
'[%s] No more messages found. Closing process' % pid)
break
body = m.get_body()
tiles = map(int, body.split(','))
except Exception as e:
parseOk = False
if not parseOk or len(tiles) % 3 != 0:
msgBody = m.get_body()
logger.warning(
'[%s] Unparsable message received.'
'Skipping...and removing message [%s]' % (pid, msgBody)
)
q.delete_message(m)
continue
for i in range(0, len(tiles), 3):
try:
tileXYZ = [tiles[i], tiles[i + 1], tiles[i + 2]]
tilebounds = geodetic.tileBounds(
tileXYZ[2], tileXYZ[0], tileXYZ[1]
)
createTile(
(tilebounds, tileXYZ, t0, dbConfigFile,
bucketBasePath, hasLighting, hasWatermask)
)
except Exception as e:
logger.error(
'[%s] Error while processing '
'specific tile %s' % (pid, str(e)), exc_info=True)
# when successfull, we delete the message from the queue
logger.info('[%s] Successfully treated an SQS message: %s' % (
pid, body))
q.delete_message(m)
except Exception as e:
logger.error(
'[%s] Error occured during processing. '
'Halting process ' % str(e), exc_info=True)
def createTileFromQueue(tq):
pid = os.getpid()
try:
(qName, t0, dbConfigFile, hasLighting, hasWatermask) = tq
sqs = getSQS()
q = sqs.get_queue(qName)
geodetic = GlobalGeodetic(True)
# we do this as long as we are finding messages in the queue
while True:
parseOk = True
try:
# 20 is maximum wait time
m = q.read(
visibility_timeout = visibility_timeout,
wait_time_seconds = 20
)
if m is None:
logger.info(
'[%s] No more messages found. Closing process' % pid)
break
body = m.get_body()
tiles = map(int, body.split(','))
except Exception as e:
parseOk = False
if not parseOk or len(tiles) % 3 != 0:
logger.warning(
'[%s] Unparsable message received.'
'Skipping...and removing message [%s]' % (pid, m.get_body())
)
q.delete_message(m)
continue
for i in range(0, len(tiles), 3):
try:
tileXYZ = [tiles[i], tiles[i + 1], tiles[i + 2]]
tilebounds = geodetic.TileBounds(
tileXYZ[0], tileXYZ[1], tileXYZ[2]
)
createTile(
(tilebounds, tileXYZ, t0, dbConfigFile,
hasLighting, hasWatermask)
)
except Exception as e:
logger.error('[%s] Error while processing '
'specific tile %s' % (pid, str(e)), exc_info=True)
# when successfull, we delete the message from the queue
logger.info('[%s] Successfully treated an SQS message: %s' % (
pid, body))
q.delete_message(m)
except Exception as e:
logger.error('[%s] Error occured during processing. '
'Halting process ' % str(e), exc_info=True)
def deleteQueue(self):
queueName = self.tmsConfig.get('General', 'sqsqueue')
if len(queueName) <= 0:
logger.error('Missing queueName')
return
try:
sqs = getSQS()
q = sqs.get_queue(queueName)
sqs.delete_queue(q)
except Exception as e:
logger.error('Error during deletion of queue:\n' + str(e), exc_info=True)
return
def deleteQueue(self):
queueName = self.tmsConfig.get("General", "sqsqueue")
if len(queueName) <= 0:
logger.error("Missing queueName")
return
try:
sqs = getSQS()
q = sqs.get_queue(queueName)
sqs.delete_queue(q)
except Exception as e:
logger.error("Error during deletion of queue:\n" + str(e))
return
def deleteQueue(self):
queueName = self.tmsConfig.get('General', 'sqsqueue')
if len(queueName) <= 0:
logger.error('Missing queueName')
return
try:
sqs = getSQS()
q = sqs.get_queue(queueName)
sqs.delete_queue(q)
except Exception as e:
logger.error('Error during deletion of queue:\n' + str(e),
exc_info=True)
return
def queueStats(self):
queueName = self.tmsConfig.get('General', 'sqsqueue')
if len(queueName) <= 0:
logger.error('Missing queueName')
return
try:
sqs = getSQS()
q = sqs.get_queue(queueName)
attrs = sqs.get_queue_attributes(q)
except Exception as e:
logger.error('Error during statistics collection:\n' + str(e), exc_info=True)
return
logger.info(attrs)
def queueStats(self):
queueName = self.tmsConfig.get("General", "sqsqueue")
if len(queueName) <= 0:
logger.error("Missing queueName")
return
try:
sqs = getSQS()
q = sqs.get_queue(queueName)
attrs = sqs.get_queue_attributes(q)
except Exception as e:
logger.error("Error during statistics collection:\n" + str(e))
return
logger.info(attrs)
def queueStats(self):
queueName = self.tmsConfig.get('General', 'sqsqueue')
if len(queueName) <= 0:
logger.error('Missing queueName')
return
try:
sqs = getSQS()
q = sqs.get_queue(queueName)
attrs = sqs.get_queue_attributes(q)
except Exception as e:
logger.error('Error during statistics collection:\n' + str(e),
exc_info=True)
return
logger.info(attrs)
def createQueue(self):
queueName = self.tmsConfig.get('General', 'sqsqueue')
maxChunks = int(self.tmsConfig.get('General', 'maxChunks'))
self.t0 = time.time()
if len(queueName) <= 0:
logger.error('Missing queueName')
return
try:
sqs = getSQS()
q = sqs.get_queue(queueName)
if q is not None:
logger.error('Queue already exists. Can\'t overwrite'
' existing queue. [%s]' % (queueName))
return
# queue with default message visiblity time of 3600.
# So each message is blocked for other users for 3600 seconds.
# Default would be 30 seconds.
# It's that high because each message contains maxChunks tiles
q = sqs.create_queue(queueName,
visibility_timeout=visibility_timeout)
# Assure queue is kept for maximum of 14 weeks (aws limit).
# default would be 4 days.
sqs.set_queue_attribute(q, 'MessageRetentionPeriod', 1209600)
except Exception as e:
logger.error('Error during creation of queue:\n' + str(e),
exc_info=True)
return
if q.count() > 0:
logger.error('Queue already contains messages. '
'Use a different queue or delete this queue first')
return
logger.info('Queue ' + queueName + ' has been created')
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
nbTiles = self.numOfTiles()
try:
logger.info('Starting creation of SQS queue with approx. '
'%s tiles)' % (nbTiles))
totalcount = 0
tcount = 0
messagecount = 0
msg = ''
for tile in tiles:
(bounds, tileXYZ, t0, dbConfigFile, hasLighting,
hasWatermask) = tile
if msg:
msg += ','
msg += ('%s,%s,%s' %
(str(tileXYZ[0]), str(tileXYZ[1]), str(tileXYZ[2])))
tcount += 1
if tcount >= maxChunks:
messagecount += 1
writeSQSMessage(q, msg)
tcount = 0
msg = ''
totalcount = totalcount + 1
if msg:
messagecount += 1
writeSQSMessage(q, msg)
except Exception as e:
logger.error('Error during writing of sqs message:\n' + str(e),
exc_info=True)
tend = time.time()
logger.info('It took %s to create %s message in SQS gueue '
'representing %s tiles' %
(str(datetime.timedelta(seconds=tend - self.t0)),
messagecount, totalcount))
def createQueue(self):
queueName = self.tmsConfig.get('General', 'sqsqueue')
maxChunks = int(self.tmsConfig.get('General', 'maxChunks'))
self.t0 = time.time()
if len(queueName) <= 0:
logger.error('Missing queueName')
return
try:
sqs = getSQS()
q = sqs.get_queue(queueName)
if q is not None:
logger.error('Queue already exists. Can\'t overwrite'
' existing queue. [%s]' % (queueName))
return
# queue with default message visiblity time of 3600. So each message is
# blocked for other users for 3600 seconds. Default would be 30 seconds.
# It's that high because each message contains maxChunks tiles
q = sqs.create_queue(queueName, visibility_timeout = visibility_timeout)
# Assure queue is kept for maximum of 14 weeks (aws limit).
# default would be 4 days.
sqs.set_queue_attribute(q, 'MessageRetentionPeriod', 1209600)
except Exception as e:
logger.error('Error during creation of queue:\n' + str(e), exc_info=True)
return
if q.count() > 0:
logger.error('Queue already contains messages. '
'Use a different queue or delete this queue first')
return
logger.info('Queue ' + queueName + ' has been created')
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
nbTiles = self.numOfTiles()
try:
logger.info('Starting creation of SQS queue with approx. '
'%s tiles)' % (nbTiles))
totalcount = 0
tcount = 0
messagecount = 0
msg = ''
for tile in tiles:
(bounds, tileXYZ, t0, dbConfigFile,
hasLighting, hasWatermask) = tile
if msg:
msg += ','
msg += ('%s,%s,%s' % (
str(tileXYZ[0]), str(tileXYZ[1]), str(tileXYZ[2])
))
tcount += 1
if tcount >= maxChunks:
messagecount += 1
writeSQSMessage(q, msg)
tcount = 0
msg = ''
totalcount = totalcount + 1
if msg:
messagecount += 1
writeSQSMessage(q, msg)
except Exception as e:
logger.error('Error during writing of sqs message:\n' + str(e),
exc_info=True)
tend = time.time()
logger.info('It took %s to create %s message in SQS gueue '
'representing %s tiles' % (
str(datetime.timedelta(seconds=tend - self.t0)),
messagecount, totalcount
)
)
