def createTerrainBasedTileJSON(params):
baseUrls = getBaseUrls(params)
with open('configs/terrain/layer.json') as f:
terrainConfig = json.loads(f.read())
# Delete unrelevant fields
if 'extensions' in terrainConfig:
del terrainConfig['extensions']
# Overwrite
terrainConfig['minzoom'] = params.minZoom
terrainConfig['maxzoom'] = params.maxZoom
terrainConfig['name'] = params.name
terrainConfig['format'] = params.format
terrainConfig['description'] = params.description
terrainConfig['attribution'] = params.attribution
terrainConfig['tiles'] = baseUrls
geodetic = GlobalGeodetic(True)
# Fix with empty ranges if we start with a biggeer min zoom
for i in range(0, params.maxZoom + 1):
if i < params.minZoom:
terrainConfig['available'][i] = []
# Max zoom is heigher than max terrain zoom level
# In that case we include the full range within the bounds
if i >= len(terrainConfig['available']):
tileMinX, tileMinY = geodetic.LonLatToTile(params.bounds[0],
params.bounds[1], i)
tileMaxX, tileMaxY = geodetic.LonLatToTile(params.bounds[2],
params.bounds[3], i)
terrainConfig['available'].append(
dict(startX=tileMinX,
endX=tileMaxX,
startY=tileMinY,
endY=tileMaxY))
return json.dumps(terrainConfig)
def testWatermaskOnlyReader(self):
z = 9
x = 769
y = 319
geodetic = GlobalGeodetic(True)
ter = TerrainTile()
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
ter.fromFile('forge/data/quantized-mesh/%s_%s_%s_watermask.terrain' %
(z, x, y),
minx,
miny,
maxx,
maxy,
hasWatermask=True)
self.assertEqual(len(ter.watermask), 256)
for row in ter.watermask:
self.assertEqual(len(row), 256)
for val in row:
self.assertTrue(val >= 0)
self.assertTrue(val <= 255)
ter.toFile(self.tmpfile)
ter2 = TerrainTile()
ter2.fromFile(self.tmpfile, minx, miny, maxx, maxy, hasWatermask=True)
self.assertEqual(len(ter2.watermask), 256)
for i in range(0, len(ter.watermask)):
for j in range(0, len(ter.watermask[i])):
self.assertEqual(ter.watermask[i][j], ter2.watermask[i][j])
def populateLakes(self):
self.setupDatabase()
logger.info('Action: populateLakes()')
# For now we never reproject lakes
with self.userSession() as session:
shpFile = self.config.get('Data', 'lakes')
if not os.path.exists(shpFile):
logger.error('Shapefile %s does not exists' % (shpFile))
sys.exit(1)
count = 1
shp = ShpToGDALFeatures(shpFile)
logger.info('Processing %s' % (shpFile))
bulk = BulkInsert(Lakes, session, withAutoCommit=1000)
for feature in shp.getFeatures():
polygon = feature.GetGeometryRef()
# Force 2D for lakes
polygon.FlattenTo2D()
# add shapefile path to dict
# self.shpFilePath
bulk.add(
dict(the_geom=WKTElement(polygon.ExportToWkt(), 4326)))
count += 1
bulk.commit()
logger.info('Commit %s features for %s.' % (count, shpFile))
# Once all features have been commited, start creating all
# the simplified versions of the lakes
logger.info('Simplifying lakes')
tiles = TerrainTiles(self.dbConfigFile, tmsConfig, time.time())
geodetic = GlobalGeodetic(True)
bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)
for i in xrange(0, len(zooms)):
zoom = zooms[i]
tablename = 'lakes_%s' % zoom
tileMinX, tileMinY = geodetic.LonLatToTile(
bounds[0], bounds[1], zoom)
tileMaxX, tileMaxY = geodetic.LonLatToTile(
bounds[2], bounds[3], zoom)
tileBounds = geodetic.TileBounds(tileMinX, tileMinY, zoom)
pointA = transformCoordinate(
'POINT(%s %s)' % (tileBounds[0], tileBounds[1]), 4326,
21781).GetPoints()[0]
pointB = transformCoordinate(
'POINT(%s %s)' % (tileBounds[2], tileBounds[3]), 4326,
21781).GetPoints()[0]
length = c2d.distance(pointA, pointB)
pixelArea = pow(length, 2) / pow(256.0, 2)
pixelLength = math.sqrt(pixelArea)
session.execute(
create_simplified_geom_table(tablename, pixelLength))
session.commit()
logger.info('Commit table public.%s with %s meters '
'tolerance' % (tablename, pixelLength))
def testExtensionsReader(self):
z = 10
x = 1563
y = 590
geodetic = GlobalGeodetic(True)
ter = TerrainTile()
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
ter.fromFile(
'forge/data/quantized-mesh/%s_%s_%s_light_watermask.terrain' %
(z, x, y),
minx,
miny,
maxx,
maxy,
hasLighting=True,
hasWatermask=True)
# check indices
self.assertTrue(len(ter.indices) > 0)
# check edges
self.assertTrue(len(ter.westI) > 0)
self.assertTrue(len(ter.southI) > 0)
self.assertTrue(len(ter.eastI) > 0)
self.assertTrue(len(ter.northI) > 0)
# check extensions
self.assertEqual(len(ter.watermask), 1)
self.assertEqual(len(ter.watermask[0]), 1)
# Water only -> 255
self.assertEqual(ter.watermask[0][0], 255)
ter.toFile(self.tmpfile)
ter2 = TerrainTile()
ter2.fromFile(self.tmpfile,
minx,
miny,
maxx,
maxy,
hasLighting=True,
hasWatermask=True)
self.assertEqual(len(ter.watermask), len(ter2.watermask))
self.assertEqual(len(ter.watermask[0]), len(ter2.watermask[0]))
sign = lambda a: 1 if a > 0 else -1 if a < 0 else 0
for i in range(0, len(ter.vLight)):
for j in range(0, 3):
# We cannot have an exact equality with successive
# oct encoding and decoding
# Thus we only check the sign
self.assertEqual(sign(ter.vLight[i][j]),
sign(ter2.vLight[i][j]))
def grid(bounds, zoomLevels, fullonly):
geodetic = GlobalGeodetic(True)
for tileZ in zoomLevels:
tileMinX, tileMinY = geodetic.LonLatToTile(bounds[0], bounds[1], tileZ)
tileMaxX, tileMaxY = geodetic.LonLatToTile(bounds[2], bounds[3], tileZ)
for tileX in xrange(tileMinX, tileMaxX + 1):
for tileY in xrange(tileMinY, tileMaxY + 1):
tilebounds = geodetic.TileBounds(tileX, tileY, tileZ)
if fullonly == 0 or isInside(tilebounds, bounds):
yield (tilebounds, (tileX, tileY, tileZ))
def createTileFromQueue(tq):
pid = os.getpid()
try:
(qName, t0, dbConfigFile, hasLighting, hasWatermask) = tq
sqs = getSQS()
q = sqs.get_queue(qName)
geodetic = GlobalGeodetic(True)
# we do this as long as we are finding messages in the queue
while True:
parseOk = True
try:
# 20 is maximum wait time
m = q.read(visibility_timeout=visibility_timeout,
wait_time_seconds=20)
if m is None:
logger.info(
'[%s] No more messages found. Closing process' % pid)
break
body = m.get_body()
tiles = map(int, body.split(','))
except Exception as e:
parseOk = False
if not parseOk or len(tiles) % 3 != 0:
logger.warning('[%s] Unparsable message received.'
'Skipping...and removing message [%s]' %
(pid, m.get_body()))
q.delete_message(m)
continue
for i in range(0, len(tiles), 3):
try:
tileXYZ = [tiles[i], tiles[i + 1], tiles[i + 2]]
tilebounds = geodetic.TileBounds(tileXYZ[0], tileXYZ[1],
tileXYZ[2])
createTile((tilebounds, tileXYZ, t0, dbConfigFile,
hasLighting, hasWatermask))
except Exception as e:
logger.error('[%s] Error while processing '
'specific tile %s' % (pid, str(e)),
exc_info=True)
# when successfull, we delete the message from the queue
logger.info('[%s] Successfully treated an SQS message: %s' %
(pid, body))
q.delete_message(m)
except Exception as e:
logger.error('[%s] Error occured during processing. '
'Halting process ' % str(e),
exc_info=True)
def _initPyramidMetadata(self):
# It keeps track of the starting and ending tiles
# and the missing tiles in between
self.metadata = {}
self.ranges = {}
geodetic = GlobalGeodetic(True)
bounds = self.meta['bounds']
# Assume the whole extent is available
for z in range(self.tileMinZoom, self.tileMaxZoom + 1):
tileMinX, tileMinY = geodetic.LonLatToTile(bounds[0], bounds[1], z)
tileMaxX, tileMaxY = geodetic.LonLatToTile(bounds[2], bounds[3], z)
self.metadata[z] = dict(
x=[tileMinX, tileMaxX],
y=[tileMinY, tileMaxY]
)
self.ranges[z] = {}
def tileNotExists(tile):
h = {'Referer': 'http://geo.admin.ch'}
(bounds, tileXYZ, t0, basePath, tFormat, gridOrigin, tilesURLs) = tile
# Only native tiles for now
entryPoint = 'http:%s' % (random.choice(tilesURLs))
# Account for a different origin
if gridOrigin == 'topLeft':
geodetic = GlobalGeodetic(True)
nbYTiles = geodetic.GetNumberOfYTilesAtZoom(tileXYZ[2])
tilexyz = (tileXYZ[0], nbYTiles - tileXYZ[1] - 1, tileXYZ[2])
tileAdress = '/'.join(
(str(tilexyz[2]), str(tilexyz[1]), str(tilexyz[0])))
else:
tileAdress = '/'.join(
(str(tileXYZ[2]), str(tileXYZ[1]), str(tileXYZ[0])))
url = '%s%s%s.%s' % (entryPoint, basePath, tileAdress, tFormat)
tilecount.value += 1
try:
exists = resourceExists(url, headers=h)
except Exception as e:
logger.error('Connection Error', exc_info=True)
logger.error('%s was skipped' % url, exc_info=True)
raise Exception(e)
if not exists:
tileskipped.value += 1
if tilecount.value % 1000 == 0:
tend = time.time()
logger.info('It took %s to (HEAD) request %s tiles. %s skipped' %
(str(datetime.timedelta(seconds=tend - t0)),
tilecount.value, tileskipped.value))
logger.info('Last tile checked:')
logger.info(url)
if not exists:
# Return everything in terrain coordinates
# e.g. starting at the bottom left (Transformation is performed in Cesium)
# https://github.com/camptocamp/cesium/blob/c2c_patches/Source/
# Scene/UrlTemplateImageryProvider.js#L500
return tileXYZ
from forge.terrain import TerrainTile
from forge.terrain.topology import TerrainTopology
from forge.lib.shapefile_utils import ShpToGDALFeatures
from forge.lib.global_geodetic import GlobalGeodetic
basename = '7_133_98'
directory = '.tmp'
extension = '.terrain'
curDir = os.getcwd()
filePathSource = '%s/forge/data/shapefile-features/%s.shp' % (curDir, basename)
filePathTarget = '%s/%s%s' % (directory, basename, extension)
shapefile = ShpToGDALFeatures(shpFilePath=filePathSource)
features = shapefile.__read__()
terrainTopo = TerrainTopology(features=features)
terrainTopo.fromGDALFeatures()
terrainFormat = TerrainTile()
geodetic = GlobalGeodetic(True)
zxy = basename.split('_')
bounds = geodetic.TileBounds(float(zxy[1]), float(zxy[2]), float(zxy[0]))
terrainFormat.fromTerrainTopology(terrainTopo, bounds)
terrainFormat.toFile(filePathTarget)
# Display SwissCoordinates
terrainFormat.computeVerticesCoordinates(epsg=21781)
print terrainFormat
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], 'f:t:', ['from=', 'to='])
except getopt.GetoptError as err:
error(str(err), 2, usage=usage)
ffrom = None
to = None
ffile = None
for o, a in opts:
if o in ('-f', '--from'):
ffrom = a
if o in ('-t', '--to'):
to = a
if ffrom is None or to is None:
if len(args) != 1:
error("Please specify a file.", 4, usage=usage)
ffile = args[0]
tiles = []
# We have file, so we get the tiles from the file
if ffile is not None:
with open(ffile) as f:
for line in f:
line = line.rstrip()
if len(line) > 2:
tiles.append(map(int, line.split('/')))
# If we have from to, we catch layers.json from poc and use from to for levels
if ffrom is not None and to is not None:
req = urlopen(poc_base_url + 'layer.json')
layers = {}
for line in req:
layers = json.loads(line)
for zoom in range(int(ffrom), int(to) + 1):
level = layers['available'][zoom][0]
for x in range(int(level['startX']), int(level['endX']) + 1):
for y in range(int(level['startY']), int(level['endY']) + 1):
tiles.append([zoom, x, y])
bucket = getBucket()
g = GlobalGeodetic(tmscompatible=True)
if not os.path.isdir('tmp'):
os.mkdir('tmp')
for tile in tiles:
req = None
tilebounds = g.TileBounds(tile[1], tile[2], tile[0])
tilestring = str(tile[0]) + "/" + str(tile[1]) + "/" + str(tile[2])
try:
url = poc_base_url + tilestring + ".terrain?v=2.0.0"
req = urlopen(url)
with open(gzip_file_name, 'wb') as fp:
fp.write(req.read())
ff = gzip.open(gzip_file_name)
with open(temp_file_name, 'wb') as fp:
fp.write(ff.read())
ter = TerrainTile()
ter.fromFile(
temp_file_name, tilebounds[0],
tilebounds[2], tilebounds[1], tilebounds[3]
)
'''
if os.path.isfile(tms_file_name):
os.remove(tms_file_name)
if os.path.isfile(shape_file_name):
os.remove(shape_file_name)
if os.path.isfile(tms_from_shape_file_name):
os.remove(tms_from_shape_file_name)
ter.toFile(tms_file_name)
ter.toShapefile(shape_file_name)
shapefile = ShpToGDALFeatures(shpFilePath=shape_file_name)
features = shapefile.__read__()
topology = TerrainTopology(features=features)
topology.fromGDALFeatures()
terFromPoc = TerrainTile()
terFromPoc.fromFile(
tms_file_name, tilebounds[0], tilebounds[2], tilebounds[1], tilebounds[3]
)
terFromShape = TerrainTile()
terFromShape.fromTerrainTopology(topology)
terFromShape.toFile(tms_from_shape_file_name)
# Use this to select what is written to s3
ter2 = terFromShape
'''
ter2 = ter
fileObject = ter2.toStringIO()
compressedFile = gzipFileObject(fileObject)
bucketKey = tilestring + '.terrain'
print 'Uploading %s to S3' % bucketKey
writeToS3(bucket, bucketKey, compressedFile, 'POC Tiles copy',
contentType=ter.getContentType())
except Exception as e:
print "error with " + line + " " + str(e)
finally:
if req:
req.close()
return 0
minWKT = 'POINT (%f %f)' % (MINX, MINY)
maxWKT = 'POINT (%f %f)' % (MAXX, MAXY)
minPoint = transformCoordinate(minWKT, 21781, 4326)
maxPoint = transformCoordinate(maxWKT, 21781, 4326)
MINX = minPoint.GetX()
MINY = minPoint.GetY()
MAXX = maxPoint.GetX()
MAXY = maxPoint.GetY()
print 'Extent :'
print[MINX, MINY, MAXX, MAXY]
MINZOOM = 3
MAXZOOM = 17
geodetic = GlobalGeodetic(True)
drv = ogr.GetDriverByName('ESRI Shapefile')
directory = '.tmp/'
extension = '.shp'
# Generate table with min max tile coordinates for all zoomlevels
for tz in range(MINZOOM, MAXZOOM + 1):
filePathTarget = '%s%s%s' % (directory, tz, extension)
tminx, tminy = geodetic.LonLatToTile(MINX, MINY, tz)
tmaxx, tmaxy = geodetic.LonLatToTile(MAXX, MAXY, tz)
dataSource = drv.CreateDataSource(filePathTarget)
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
layer = dataSource.CreateLayer('%s' % tz, srs, ogr.wkbPolygon)
fieldKey = ogr.FieldDefn('Key', ogr.OFTString)
def createModelBasedTileJSON(params):
tilecount = 0
t0 = time.time()
baseUrls = getBaseUrls(params)
engine = getEngine(params)
metadata = sqlalchemy.MetaData(bind=engine, schema=params.dbSchema)
metadata.reflect()
try:
table = metadata.tables[params.dbSchema + '.' + params.tableName]
except KeyError:
raise ValueError('Table %s in schema %s could not be found' %
(params.tableName, params.dbSchema))
# We assume a pkey is defined by only one column for now.
pkColumn = table.primary_key.columns.items()[0]
pkColumnName = pkColumn[0].__str__()
pkColumnType = pkColumn[1].type.__class__
model = getOrmModel(pkColumnName, pkColumnType, params)
# Bounds generated from DB
try:
conn = engine.connect()
bounds = conn.execute(
tableExtentLiteral(params.dbSchema, params.tableName,
params.sridFrom)).fetchone()
strBounds = tuple(['{:2f}'.format(i) for i in bounds])
# Tuple to list for json converison
bounds = [b for b in bounds]
logger.info('Bounds are %s, %s, %s, %s' % strBounds)
except Exception as e:
logger.error('An error occured while determining the bounds',
exc_info=True)
raise Exception(e)
finally:
conn.close()
# pre-calculate the maximazed buffers in degrees
if params.pxTolerance:
geodetic = GlobalGeodetic(True)
buffers = {}
for z in range(params.minZoom, params.maxZoom):
buffers[z] = degreesToMeters(
geodetic.Resolution(z) * float(params.pxTolerance))
try:
session = scoped_session(sessionmaker(bind=engine))
# We usually don't scan the last levels
tiles = Tiles(bounds, params.minZoom, params.maxScanZoom, t0,
params.fullonly)
tMeta = LayerMetadata(bounds=bounds,
minzoom=params.minZoom,
maxzoom=params.maxZoom,
baseUrls=baseUrls,
description=params.description,
attribution=params.attribution,
name=params.name)
for tile in tiles:
metaBuffer = 0.
tilecount += 1
if params.pxTolerance:
xyz = tile[1]
metaBuffer = buffers[xyz[2]]
noGeom = scanLayer(tile, session, model, params.sridFrom,
params.sridTo, metaBuffer, tilecount)
if noGeom:
tMeta.removeTile(noGeom[0], noGeom[1], noGeom[2])
finally:
session.close()
engine.dispose()
return (tMeta.toJSON(), tilecount)
def _stats(self, withDb=True):
self.t0 = time.time()
total = 0
msg = '\n'
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
geodetic = GlobalGeodetic(True)
bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)
db = DB('configs/terrain/database.cfg')
try:
with db.userSession() as session:
for i in xrange(0, len(zooms)):
zoom = zooms[i]
model = modelsPyramid.getModelByZoom(zoom)
nbObjects = None
if withDb:
nbObjects = session.query(model).filter(
model.bboxIntersects(bounds)).count()
tileMinX, tileMinY = geodetic.LonLatToTile(
bounds[0], bounds[1], zoom)
tileMaxX, tileMaxY = geodetic.LonLatToTile(
bounds[2], bounds[3], zoom)
# Fast approach, but might not be fully correct
if tiles.fullonly == 1:
tileMinX += 1
tileMinY += 1
tileMaxX -= 1
tileMaxY -= 1
tileBounds = geodetic.TileBounds(tileMinX, tileMinY, zoom)
xCount = tileMaxX - tileMinX + 1
yCount = tileMaxY - tileMinY + 1
nbTiles = xCount * yCount
total += nbTiles
pointA = transformCoordinate(
'POINT(%s %s)' % (tileBounds[0], tileBounds[1]), 4326,
21781).GetPoints()[0]
pointB = transformCoordinate(
'POINT(%s %s)' % (tileBounds[2], tileBounds[3]), 4326,
21781).GetPoints()[0]
length = c2d.distance(pointA, pointB)
if tiles.fullonly == 1:
msg += 'WARNING: stats are approximative because ' \
'fullonly is activated!\n'
msg += 'At zoom %s:\n' % zoom
msg += 'We expect %s tiles overall\n' % nbTiles
msg += 'Min X is %s, Max X is %s\n' % (tileMinX, tileMaxX)
msg += '%s columns over X\n' % xCount
msg += 'Min Y is %s, Max Y is %s\n' % (tileMinY, tileMaxY)
msg += '%s rows over Y\n' % yCount
msg += '\n'
msg += 'A tile side is around %s meters' % int(
round(length))
if nbTiles > 0 and nbObjects is not None:
msg += 'We have an average of about %s triangles ' \
'per tile\n' % int(round(nbObjects / nbTiles))
msg += '\n\n'
msg += '%s tiles in total.' % total
except Exception as e:
logger.error('An error occured during statistics collection')
logger.error('%s' % e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return (total, msg)