def create(self):
def callback(counter, result):
if not counter % 10000:
logger.info('counter: %s' % counter)
logger.info('result: %s' % result)
self.t0 = time.time()
tilecount.value = 0
skipcount.value = 0
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
procfactor = int(self.tmsConfig.get('General', 'procfactor'))
pm = PoolManager(factor=procfactor)
maxChunks = int(self.tmsConfig.get('General', 'maxChunks'))
nbTiles = self.numOfTiles()
tilesPerProc = int(nbTiles / pm.nbOfProcesses)
if tilesPerProc < maxChunks:
maxChunks = tilesPerProc
if maxChunks < 1:
maxChunks = 1
logger.info('Starting creation of %s tiles (%s per chunk)' %
(nbTiles, maxChunks))
pm.imap_unordered(createTile, tiles, maxChunks, callback=callback)
tend = time.time()
logger.info('It took %s to create %s tiles (%s were skipped)' %
(str(datetime.timedelta(seconds=tend - self.t0)),
tilecount.value, skipcount.value))
def create(self):
self.t0 = time.time()
tilecount.value = 0
skipcount.value = 0
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
procfactor = int(self.tmsConfig.get('General', 'procfactor'))
pm = PoolManager(logger=logger, factor=procfactor)
maxChunks = int(self.tmsConfig.get('General', 'maxChunks'))
nbTiles = self.numOfTiles()
tilesPerProc = int(nbTiles / pm.numOfProcesses())
if tilesPerProc < maxChunks:
maxChunks = tilesPerProc
if maxChunks < 1:
maxChunks = 1
logger.info('Starting creation of %s tiles (%s per chunk)' %
(nbTiles, maxChunks))
pm.process(tiles, createTile, maxChunks)
tend = time.time()
logger.info('It took %s to create %s tiles (%s were skipped)' %
(str(datetime.timedelta(seconds=tend - self.t0)),
tilecount.value, skipcount.value))
def _stats(self, withDb=True):
self.t0 = time.time()
total = 0
msg = '\n'
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
geodetic = getTileGrid(4326)(tmsCompatible=True)
bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)
db = DB('configs/terrain/database.cfg')
try:
with db.userSession() as session:
for i in xrange(0, len(zooms)):
zoom = zooms[i]
model = modelsPyramid.getModelByZoom(zoom)
nbObjects = None
if withDb:
nbObjects = session.query(model).filter(
model.bboxIntersects(bounds)).count()
# top letf corner
tileMinX, tileMinY = geodetic.tileAddress(
zoom, [bounds[0], bounds[3]])
# bottom right
tileMaxX, tileMaxY = geodetic.tileAddress(
zoom, [bounds[2], bounds[1]])
tileBounds = geodetic.tileBounds(zoom, tileMinX, tileMinY)
xCount = tileMaxX - tileMinX + 1
yCount = tileMaxY - tileMinY + 1
nbTiles = xCount * yCount
total += nbTiles
pointA = transformCoordinate(
'POINT(%s %s)' % (tileBounds[0], tileBounds[1]), 4326,
21781).GetPoints()[0]
pointB = transformCoordinate(
'POINT(%s %s)' % (tileBounds[2], tileBounds[3]), 4326,
21781).GetPoints()[0]
length = int(round(c2d.distance(pointA, pointB)))
msg += 'At zoom %s:\n' % zoom
msg += 'We expect %s tiles overall\n' % nbTiles
msg += 'Min X is %s, Max X is %s\n' % (tileMinX, tileMaxX)
msg += '%s columns over X\n' % xCount
msg += 'Min Y is %s, Max Y is %s\n' % (tileMinY, tileMaxY)
msg += '%s rows over Y\n' % yCount
msg += '\n'
msg += 'A tile side is around %s meters' % length
if nbTiles > 0 and nbObjects is not None:
msg += 'We have an average of about %s triangles ' \
'per tile\n' % int(round(nbObjects / nbTiles))
msg += '\n\n'
msg += '%s tiles in total.' % total
except Exception as e:
logger.error('An error occured during statistics collection')
logger.error('%s' % e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return (total, msg)
def populateLakes(self):
self.setupDatabase()
logger.info('Action: populateLakes()')
# For now we never reproject lakes
with self.userSession() as session:
shpFile = self.config.get('Data', 'lakes')
if not os.path.exists(shpFile):
logger.error('Shapefile %s does not exists' % (shpFile))
sys.exit(1)
count = 1
shp = ShpToGDALFeatures(shpFile)
logger.info('Processing %s' % (shpFile))
bulk = BulkInsert(Lakes, session, withAutoCommit=1000)
for feature in shp.getFeatures():
polygon = feature.GetGeometryRef()
# Force 2D for lakes
polygon.FlattenTo2D()
# add shapefile path to dict
# self.shpFilePath
bulk.add(
dict(the_geom=WKTElement(polygon.ExportToWkt(), 4326)))
count += 1
bulk.commit()
logger.info('Commit %s features for %s.' % (count, shpFile))
# Once all features have been commited, start creating all
# the simplified versions of the lakes
logger.info('Simplifying lakes')
tiles = TerrainTiles(self.dbConfigFile, tmsConfig, time.time())
geodetic = GlobalGeodetic(True)
bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)
for i in xrange(0, len(zooms)):
zoom = zooms[i]
tablename = 'lakes_%s' % zoom
tileMinX, tileMinY = geodetic.LonLatToTile(
bounds[0], bounds[1], zoom)
tileMaxX, tileMaxY = geodetic.LonLatToTile(
bounds[2], bounds[3], zoom)
tileBounds = geodetic.TileBounds(tileMinX, tileMinY, zoom)
pointA = transformCoordinate(
'POINT(%s %s)' % (tileBounds[0], tileBounds[1]), 4326,
21781).GetPoints()[0]
pointB = transformCoordinate(
'POINT(%s %s)' % (tileBounds[2], tileBounds[3]), 4326,
21781).GetPoints()[0]
length = c2d.distance(pointA, pointB)
pixelArea = pow(length, 2) / pow(256.0, 2)
pixelLength = math.sqrt(pixelArea)
session.execute(
create_simplified_geom_table(tablename, pixelLength))
session.commit()
logger.info('Commit table public.%s with %s meters '
'tolerance' % (tablename, pixelLength))
def metadata(self):
t0 = time.time()
basePath = self.tmsConfig.get('General', 'bucketpath')
baseUrls = []
url = 'https://terrain.dev.bgdi.ch'
url += '/%s{z}/{x}/{y}.terrain?v={version}' % basePath
baseUrls.append(url)
# TODO
#for i in range(0, 5):
# url = 'https://terrain10%s.geo.admin.ch' % i
# url = 'https://terrain10%s.geo.admin.ch' % i
# url += '/%s{z}/{x}/{y}.terrain?v={version}' % basePath
# baseUrls.append(url)
db = DB('configs/terrain/database.cfg')
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, t0)
tMeta = TerrainMetadata(
bounds=tiles.bounds,
minzoom=tiles.tileMinZ,
maxzoom=tiles.tileMaxZ,
useGlobalTiles=True,
hasLighting=tiles.hasLighting,
hasWatermask=tiles.hasWatermask,
baseUrls=baseUrls)
try:
with db.userSession() as session:
tilecount = 1
for tile in tiles:
tMeta = scanTerrain(tMeta, tile, session, tilecount)
tilecount += 1
tend = time.time()
logger.info('It took %s to scan %s tiles' % (
str(datetime.timedelta(seconds=tend - t0)), tilecount))
except Exception as e:
logger.error('An error occured during layer.json creation')
logger.error('%s' % e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
with open('.tmp/layer.json', 'w') as f:
f.write(tMeta.toJSON())
def metadata(self):
t0 = time.time()
serverAddress = self.tmsConfig.get('General', 'serveraddress')
basePath = self.tmsConfig.get('General', 'bucketpath')
baseUrls = [
serverAddress + basePath + "{z}/{x}/{y}.terrain?v={version}"
]
db = DB('configs/terrain/database.cfg')
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, t0)
tMeta = TerrainMetadata(bounds=tiles.bounds,
minzoom=tiles.tileMinZ,
maxzoom=tiles.tileMaxZ,
useGlobalTiles=True,
hasLighting=tiles.hasLighting,
hasWatermask=tiles.hasWatermask,
baseUrls=baseUrls)
try:
with db.userSession() as session:
tilecount = 1
for tile in tiles:
tMeta = scanTerrain(tMeta, tile, session, tilecount)
tilecount += 1
tend = time.time()
logger.info(
'It took %s to scan %s tiles' %
(str(datetime.timedelta(seconds=tend - t0)), tilecount))
except Exception as e:
logger.error('An error occured during layer.json creation')
logger.error('%s' % e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
with open('.tmp/layer.json', 'w') as f:
f.write(tMeta.toJSON())
def createQueue(self):
queueName = self.tmsConfig.get('General', 'sqsqueue')
maxChunks = int(self.tmsConfig.get('General', 'maxChunks'))
self.t0 = time.time()
if len(queueName) <= 0:
logger.error('Missing queueName')
return
try:
sqs = getSQS()
q = sqs.get_queue(queueName)
if q is not None:
logger.error('Queue already exists. Can\'t overwrite'
' existing queue. [%s]' % (queueName))
return
# queue with default message visiblity time of 3600.
# So each message is blocked for other users for 3600 seconds.
# Default would be 30 seconds.
# It's that high because each message contains maxChunks tiles
q = sqs.create_queue(queueName,
visibility_timeout=visibility_timeout)
# Assure queue is kept for maximum of 14 weeks (aws limit).
# default would be 4 days.
sqs.set_queue_attribute(q, 'MessageRetentionPeriod', 1209600)
except Exception as e:
logger.error('Error during creation of queue:\n' + str(e),
exc_info=True)
return
if q.count() > 0:
logger.error('Queue already contains messages. '
'Use a different queue or delete this queue first')
return
logger.info('Queue ' + queueName + ' has been created')
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
nbTiles = self.numOfTiles()
try:
logger.info('Starting creation of SQS queue with approx. '
'%s tiles)' % (nbTiles))
totalcount = 0
tcount = 0
messagecount = 0
msg = ''
for tile in tiles:
(bounds, tileXYZ, t0, dbConfigFile, hasLighting,
hasWatermask) = tile
if msg:
msg += ','
msg += ('%s,%s,%s' %
(str(tileXYZ[0]), str(tileXYZ[1]), str(tileXYZ[2])))
tcount += 1
if tcount >= maxChunks:
messagecount += 1
writeSQSMessage(q, msg)
tcount = 0
msg = ''
totalcount = totalcount + 1
if msg:
messagecount += 1
writeSQSMessage(q, msg)
except Exception as e:
logger.error('Error during writing of sqs message:\n' + str(e),
exc_info=True)
tend = time.time()
logger.info('It took %s to create %s message in SQS gueue '
'representing %s tiles' %
(str(datetime.timedelta(seconds=tend - self.t0)),
messagecount, totalcount))
def _stats(self, withDb=True):
self.t0 = time.time()
total = 0
msg = '\n'
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
geodetic = GlobalGeodetic(True)
bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)
db = DB('configs/terrain/database.cfg')
try:
with db.userSession() as session:
for i in xrange(0, len(zooms)):
zoom = zooms[i]
model = modelsPyramid.getModelByZoom(zoom)
nbObjects = None
if withDb:
nbObjects = session.query(model).filter(
model.bboxIntersects(bounds)).count()
tileMinX, tileMinY = geodetic.LonLatToTile(
bounds[0], bounds[1], zoom)
tileMaxX, tileMaxY = geodetic.LonLatToTile(
bounds[2], bounds[3], zoom)
# Fast approach, but might not be fully correct
if tiles.fullonly == 1:
tileMinX += 1
tileMinY += 1
tileMaxX -= 1
tileMaxY -= 1
tileBounds = geodetic.TileBounds(tileMinX, tileMinY, zoom)
xCount = tileMaxX - tileMinX + 1
yCount = tileMaxY - tileMinY + 1
nbTiles = xCount * yCount
total += nbTiles
pointA = transformCoordinate(
'POINT(%s %s)' % (tileBounds[0], tileBounds[1]), 4326,
21781).GetPoints()[0]
pointB = transformCoordinate(
'POINT(%s %s)' % (tileBounds[2], tileBounds[3]), 4326,
21781).GetPoints()[0]
length = c2d.distance(pointA, pointB)
if tiles.fullonly == 1:
msg += 'WARNING: stats are approximative because ' \
'fullonly is activated!\n'
msg += 'At zoom %s:\n' % zoom
msg += 'We expect %s tiles overall\n' % nbTiles
msg += 'Min X is %s, Max X is %s\n' % (tileMinX, tileMaxX)
msg += '%s columns over X\n' % xCount
msg += 'Min Y is %s, Max Y is %s\n' % (tileMinY, tileMaxY)
msg += '%s rows over Y\n' % yCount
msg += '\n'
msg += 'A tile side is around %s meters' % int(
round(length))
if nbTiles > 0 and nbObjects is not None:
msg += 'We have an average of about %s triangles ' \
'per tile\n' % int(round(nbObjects / nbTiles))
msg += '\n\n'
msg += '%s tiles in total.' % total
except Exception as e:
logger.error('An error occured during statistics collection')
logger.error('%s' % e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return (total, msg)