def testWatermaskOnlyReader(self):
z = 9
x = 769
y = 319
geodetic = GlobalGeodetic(True)
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
ter = TerrainTile(west=minx, south=miny, east=maxx, north=maxy)
ter.fromFile('tests/data/%s_%s_%s_watermask.terrain' % (z, x, y),
hasWatermask=True)
self.assertEqual(len(ter.watermask), 256)
for row in ter.watermask:
self.assertEqual(len(row), 256)
for val in row:
self.assertGreaterEqual(val, 0)
self.assertLessEqual(val, 255)
ter.toFile(self.tmpfile)
ter2 = TerrainTile(west=minx, south=miny, east=maxx, north=maxy)
ter2.fromFile(self.tmpfile, hasWatermask=True)
self.assertEqual(len(ter2.watermask), 256)
for i in range(0, len(ter.watermask)):
for j in range(0, len(ter.watermask[i])):
self.assertEqual(ter.watermask[i][j], ter2.watermask[i][j])
self.assertEqual(ter2.getContentType(),
'application/vnd.quantized-mesh;extensions=watermask')
def testFromStringIO(self):
z = 10
x = 1563
y = 590
geodetic = GlobalGeodetic(True)
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
# Regular file not gzip compressed
ter = TerrainTile()
ter = TerrainTile(west=minx, south=miny, east=maxx, north=maxy)
with open('tests/data/%s_%s_%s_light_watermask.terrain' % (z, x, y)) as f:
content = cStringIO.StringIO(f.read())
ter.fromStringIO(content, hasLighting=True, hasWatermask=True)
# check indices
self.assertGreater(len(ter.indices), 0)
# check edges
self.assertGreater(len(ter.westI), 0)
self.assertGreater(len(ter.southI), 0)
self.assertGreater(len(ter.eastI), 0)
self.assertGreater(len(ter.northI), 0)
# check extensions
self.assertEqual(len(ter.watermask), 1)
self.assertEqual(len(ter.watermask[0]), 1)
# Water only -> 255
self.assertEqual(ter.watermask[0][0], 255)
def populateLakes(self):
self.setupDatabase()
logger.info('Action: populateLakes()')
# For now we never reproject lakes
with self.userSession() as session:
shpFile = self.config.get('Data', 'lakes')
if not os.path.exists(shpFile):
logger.error('Shapefile %s does not exists' % (shpFile))
sys.exit(1)
count = 1
shp = ShpToGDALFeatures(shpFile)
logger.info('Processing %s' % (shpFile))
bulk = BulkInsert(Lakes, session, withAutoCommit=1000)
for feature in shp.getFeatures():
polygon = feature.GetGeometryRef()
# Force 2D for lakes
polygon.FlattenTo2D()
# add shapefile path to dict
# self.shpFilePath
bulk.add(dict(the_geom='SRID=4326;' + polygon.ExportToWkt()))
count += 1
bulk.commit()
logger.info('Commit %s features for %s.' % (count, shpFile))
# Once all features have been commited, start creating all
# the simplified versions of the lakes
logger.info('Simplifying lakes')
tiles = TerrainTiles(self.dbConfigFile, tmsConfig, time.time())
geodetic = GlobalGeodetic(True)
bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)
for i in xrange(0, len(zooms)):
zoom = zooms[i]
tablename = 'lakes_%s' % zoom
tileMinX, tileMinY = geodetic.LonLatToTile(
bounds[0], bounds[1], zoom)
tileMaxX, tileMaxY = geodetic.LonLatToTile(
bounds[2], bounds[3], zoom)
tileBounds = geodetic.TileBounds(tileMinX, tileMinY, zoom)
pointA = transformCoordinate(
'POINT(%s %s)' % (tileBounds[0], tileBounds[1]), 4326,
21781).GetPoints()[0]
pointB = transformCoordinate(
'POINT(%s %s)' % (tileBounds[2], tileBounds[3]), 4326,
21781).GetPoints()[0]
length = c2d.distance(pointA, pointB)
pixelArea = pow(length, 2) / pow(256.0, 2)
pixelLength = math.sqrt(pixelArea)
session.execute(
create_simplified_geom_table(tablename, pixelLength))
session.commit()
logger.info('Commit table public.%s with %s meters '
'tolerance' % (tablename, pixelLength))
def load_tile(terrain_path, x, y, z):
"""
:rtype: EditableTerrainTile
"""
geodetic = GlobalGeodetic(True)
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
tile = EditableTerrainTile(west=minx, south=miny, east=maxx, north=maxy)
tile.fromFile(terrain_path, has_lighting=True)
return tile
def testEncodeDecode(self):
z = 0
x = 0
y = 0
globalGeodetic = GlobalGeodetic(True)
bounds = globalGeodetic.TileBounds(x, y, z)
ter = encode(geometries, bounds=bounds)
ter.toFile(self.tmpfile)
ter2 = decode(self.tmpfile, bounds)
self.assert_tile(ter, ter2)
# Partial tile nothing on the east edge
self.assertEqual(len(ter.eastI), 0)
self.assertEqual(len(ter.eastI), len(ter2.eastI))
def _initPyramidMetadata(self):
# It keeps track of the starting and ending tiles
# and the missing tiles in between
self.metadata = {}
self.ranges = {}
geodetic = GlobalGeodetic(True)
bounds = self.meta['bounds']
# Assume the whole extent is available
for z in range(self.tileMinZoom, self.tileMaxZoom + 1):
tileMinX, tileMinY = geodetic.LonLatToTile(bounds[0], bounds[1], z)
tileMaxX, tileMaxY = geodetic.LonLatToTile(bounds[2], bounds[3], z)
self.metadata[z] = dict(x=[tileMinX, tileMaxX],
y=[tileMinY, tileMaxY])
self.ranges[z] = {}
def testExtensionsReader(self):
z = 10
x = 1563
y = 590
geodetic = GlobalGeodetic(True)
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
ter = TerrainTile()
ter = TerrainTile(west=minx, south=miny, east=maxx, north=maxy)
ter.fromFile(
'tests/data/%s_%s_%s_light_watermask.terrain' % (z, x, y),
hasLighting=True, hasWatermask=True
)
# check indices
self.assertGreater(len(ter.indices), 0)
# check edges
self.assertGreater(len(ter.westI), 0)
self.assertGreater(len(ter.southI), 0)
self.assertGreater(len(ter.eastI), 0)
self.assertGreater(len(ter.northI), 0)
# check extensions
self.assertEqual(len(ter.watermask), 1)
self.assertEqual(len(ter.watermask[0]), 1)
# Water only -> 255
self.assertEqual(ter.watermask[0][0], 255)
ter.toFile(self.tmpfile)
ter2 = TerrainTile(west=minx, south=miny, east=maxx, north=maxy)
ter2.fromFile(self.tmpfile,
hasLighting=True, hasWatermask=True)
self.assertEqual(len(ter.watermask), len(ter2.watermask))
self.assertEqual(len(ter.watermask[0]), len(ter2.watermask[0]))
def sign(a):
return 1 if a > 0 else -1 if a < 0 else 0
for i in range(0, len(ter.vLight)):
for j in range(0, 3):
# We cannot have an exact equality with successive
# oct encoding and decoding
# Thus we only check the sign
self.assertEqual(
sign(ter.vLight[i][j]), sign(ter2.vLight[i][j]))
self.assertEqual(ter2.getContentType(),
'application/vnd.quantized-mesh;' +
'extensions=octvertexnormals-watermask')
def testEncodeDecode(self):
z = 0
x = 0
y = 0
globalGeodetic = GlobalGeodetic(True)
bounds = globalGeodetic.TileBounds(x, y, z)
ter = encode(geometries, bounds=bounds)
ter.toFile(self.tmpfile)
ter2 = decode(self.tmpfile, bounds)
self.assertIsInstance(ter.__repr__(), str)
self.assertIsInstance(ter2.__repr__(), str)
# check headers
self.assertGreater(len(ter.header), 0)
self.assertEqual(len(ter.header), len(ter2.header))
# check vertices
self.assertGreater(len(ter.u), 0)
self.assertGreater(len(ter.v), 0)
self.assertGreater(len(ter.h), 0)
self.assertEqual(len(ter.u), len(ter2.u))
self.assertEqual(len(ter.v), len(ter2.v))
self.assertEqual(len(ter.h), len(ter2.h))
for i, v in enumerate(ter.u):
self.assertEqual(v, ter2.u[i])
for i, v in enumerate(ter.v):
self.assertEqual(v, ter2.v[i])
for i, v in enumerate(ter.h):
self.assertEqual(v, ter2.h[i])
self.assertEqual(
len(ter.getVerticesCoordinates()),
len(ter2.getVerticesCoordinates())
)
# check indices
self.assertGreater(len(ter.indices), 0)
self.assertEqual(len(ter.indices), len(ter2.indices))
for i, v in enumerate(ter.indices):
self.assertEqual(v, ter2.indices[i], i)
# check edges
self.assertGreater(len(ter.westI), 0)
self.assertEqual(len(ter.westI), len(ter2.westI))
for i, v in enumerate(ter.westI):
self.assertEqual(v, ter2.westI[i], i)
self.assertEqual(len(ter.eastI), 0)
self.assertEqual(len(ter.eastI), len(ter2.eastI))
def testBoundingSpherePrecision(self):
x = 533
y = 383
z = 9
geodetic = GlobalGeodetic(True)
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
ter = TerrainTile(west=minx, south=miny, east=maxx, north=maxy)
ter.fromFile('tests/data/%s_%s_%s.terrain' % (z, x, y))
coords = [LLH2ECEF(*coord) for coord in ter.getVerticesCoordinates()]
sphere = BoundingSphere()
sphere.fromPoints(coords)
for coord in coords:
distance = c3d.distance(sphere.center, coord)
self.assertLessEqual(distance, sphere.radius)
def tileNotExists(tile):
h = {'Referer': 'http://geo.admin.ch'}
(bounds, tileXYZ, t0, basePath, tFormat, gridOrigin, tilesURLs) = tile
# Only native tiles for now
entryPoint = 'http:%s' % (random.choice(tilesURLs))
# Account for a different origin
if gridOrigin == 'topLeft':
geodetic = GlobalGeodetic(True)
nbYTiles = geodetic.GetNumberOfYTilesAtZoom(tileXYZ[2])
tilexyz = (tileXYZ[0], nbYTiles - tileXYZ[1] - 1, tileXYZ[2])
tileAdress = '/'.join(
(str(tilexyz[2]), str(tilexyz[1]), str(tilexyz[0])))
else:
tileAdress = '/'.join(
(str(tileXYZ[2]), str(tileXYZ[1]), str(tileXYZ[0])))
url = '%s%s%s.%s' % (entryPoint, basePath, tileAdress, tFormat)
tilecount.value += 1
try:
exists = resourceExists(url, headers=h)
except Exception as e:
logger.error('Connection Error', exc_info=True)
logger.error('%s was skipped' % url, exc_info=True)
raise Exception(e)
if not exists:
tileskipped.value += 1
if tilecount.value % 1000 == 0:
tend = time.time()
logger.info('It took %s to (HEAD) request %s tiles. %s skipped' % (
str(datetime.timedelta(seconds=tend - t0)),
tilecount.value, tileskipped.value)
)
logger.info('Last tile checked:')
logger.info(url)
if not exists:
# Return everything in terrain coordinates
# e.g. starting at the bottom left
# (Transformation is performed in Cesium)
# https://github.com/camptocamp/cesium/blob/c2c_patches/Source/
# Scene/UrlTemplateImageryProvider.js#L500
return tileXYZ
def testExtentionsReaderWriterGzipped(self):
z = 10
x = 1563
y = 590
geodetic = GlobalGeodetic(True)
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
# Regular file not gzip compressed
ter = TerrainTile()
ter = TerrainTile(west=minx, south=miny, east=maxx, north=maxy)
ter.fromFile(
'tests/data/%s_%s_%s_light_watermask.terrain' % (z, x, y),
hasLighting=True, hasWatermask=True
)
# Same file but gzipped this time
terG = TerrainTile()
terG = TerrainTile(west=minx, south=miny, east=maxx, north=maxy)
terG.fromFile(
'tests/data/%s_%s_%s_light_watermask.terrain.gz' % (z, x, y),
hasLighting=True, hasWatermask=True, gzipped=True
)
# check indices
self.assertEqual(len(terG.indices), len(ter.indices))
self.assertEqual(terG.indices[0], ter.indices[0])
# check edges
self.assertEqual(len(terG.westI), len(ter.westI))
self.assertEqual(len(terG.southI), len(ter.southI))
self.assertEqual(len(terG.eastI), len(ter.eastI))
self.assertEqual(len(terG.northI), len(ter.northI))
self.assertEqual(terG.westI[0], ter.westI[0])
self.assertEqual(terG.southI[0], ter.southI[0])
self.assertEqual(terG.eastI[0], ter.eastI[0])
self.assertEqual(terG.northI[0], ter.northI[0])
self.assertEqual(len(terG.watermask), len(ter.watermask))
self.assertEqual(len(terG.watermask[0]), len(ter.watermask[0]))
# Water only -> 255
self.assertEqual(terG.watermask[0][0], 255)
# To gzipped file
terG.toFile(self.tmpfile, gzipped=True)
def createTerrainBasedTileJSON(params):
baseUrls = getBaseUrls(params)
with open('configs/terrain/layer.json') as f:
terrainConfig = json.loads(f.read())
# Delete unrelevant fields
if 'extensions' in terrainConfig:
del terrainConfig['extensions']
# Overwrite
terrainConfig['minzoom'] = params.minZoom
terrainConfig['maxzoom'] = params.maxZoom
terrainConfig['name'] = params.name
terrainConfig['format'] = params.format
terrainConfig['description'] = params.description
terrainConfig['attribution'] = params.attribution
terrainConfig['tiles'] = baseUrls
geodetic = GlobalGeodetic(True)
# Fix with empty ranges if we start with a biggeer min zoom
for i in range(0, params.maxZoom + 1):
if i < params.minZoom:
terrainConfig['available'][i] = []
# Max zoom is heigher than max terrain zoom level
# In that case we include the full range within the bounds
if i >= len(terrainConfig['available']):
tileMinX, tileMinY = geodetic.LonLatToTile(
params.bounds[0], params.bounds[1], i
)
tileMaxX, tileMaxY = geodetic.LonLatToTile(
params.bounds[2], params.bounds[3], i
)
terrainConfig['available'].append(dict(
startX=tileMinX,
endX=tileMaxX,
startY=tileMinY,
endY=tileMaxY
))
return json.dumps(terrainConfig)
# import cStringIO
# import requests
# from quantized_mesh_tile.terrain import TerrainTile
# from quantized_mesh_tile.global_geodetic import GlobalGeodetic
# [z, x, y] = [14, 24297, 10735]
# geodetic = GlobalGeodetic(True)
# [west, south, east, north] = bounds = geodetic.TileBounds(x, y, z)
# url = 'http://assets.agi.com/stk-terrain/world/%s/%s/%s.terrain?v=1.16389.0' % (z, x, y)
# response = requests.get(url)
# content = cStringIO.StringIO(response.content)
# print west, south, east, north
# ter = TerrainTile(west=west, south=south, east=east, north=north)
# ter.fromStringIO(content)
# print ter.header
# print ter.getVerticesCoordinates()
# path = "9_533_383.terrain"
from quantized_mesh_tile.global_geodetic import GlobalGeodetic
from quantized_mesh_tile.terrain import TerrainTile
geodetic = GlobalGeodetic(True)
# [z, x, y] = [16,67465,51617]
[z, x, y] = [0, 0, 0]
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
ter = TerrainTile(west=minx, south=miny, east=maxx, north=maxy)
print geodetic.TileBounds(0, 0, 0)
# ter.fromFile('ahn_416656.terrain')
ter.fromFile('51617.terrain')
print ter.getTrianglesCoordinates()
def createModelBasedTileJSON(params):
tilecount = 0
t0 = time.time()
baseUrls = getBaseUrls(params)
engine = getEngine(params)
metadata = sqlalchemy.MetaData(bind=engine, schema=params.dbSchema)
metadata.reflect()
try:
table = metadata.tables[params.dbSchema + '.' + params.tableName]
except KeyError:
raise ValueError(
'Table %s in schema %s could not be found' % (
params.tableName, params.dbSchema))
# We assume a pkey is defined by only one column for now.
pkColumn = table.primary_key.columns.items()[0]
pkColumnName = pkColumn[0].__str__()
pkColumnType = pkColumn[1].type.__class__
model = getOrmModel(pkColumnName, pkColumnType, params)
# Bounds generated from DB
try:
conn = engine.connect()
bounds = conn.execute(
tableExtentLiteral(
params.dbSchema, params.tableName, params.sridFrom)
).fetchone()
strBounds = tuple(['{:2f}'.format(i) for i in bounds])
# Tuple to list for json converison
bounds = [b for b in bounds]
logger.info('Bounds are %s, %s, %s, %s' % strBounds)
except Exception as e:
logger.error(
'An error occured while determining the bounds', exc_info=True)
raise Exception(e)
finally:
conn.close()
# pre-calculate the maximazed buffers in degrees
if params.pxTolerance:
geodetic = GlobalGeodetic(True)
buffers = {}
for z in range(params.minZoom, params.maxZoom):
buffers[z] = degreesToMeters(
geodetic.Resolution(z) * float(params.pxTolerance)
)
try:
session = scoped_session(sessionmaker(bind=engine))
# We usually don't scan the last levels
tiles = Tiles(
bounds, params.minZoom, params.maxScanZoom, t0
)
tMeta = LayerMetadata(
bounds=bounds, minzoom=params.minZoom,
maxzoom=params.maxZoom, baseUrls=baseUrls,
description=params.description, attribution=params.attribution,
name=params.name
)
for tile in tiles:
metaBuffer = 0.
tilecount += 1
if params.pxTolerance:
xyz = tile[1]
metaBuffer = buffers[xyz[2]]
noGeom = scanLayer(
tile, session, model, params.sridFrom,
params.sridTo, metaBuffer, tilecount
)
if noGeom:
tMeta.removeTile(noGeom[0], noGeom[1], noGeom[2])
finally:
session.close()
engine.dispose()
return (tMeta.toJSON(), tilecount)
def testReaderWriter(self):
'''
Circle jerk testing.
We read the file with our reader
We write this data with our writer to a temporary file
We read this temporary file
We compare the results
'''
x = 533
y = 383
z = 9
geodetic = GlobalGeodetic(True)
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
ter = TerrainTile(west=minx, south=miny, east=maxx, north=maxy)
ter.fromFile('tests/data/%s_%s_%s.terrain' % (z, x, y))
ter.toFile(self.tmpfile)
self.assertIsInstance(ter.__repr__(), str)
ter2 = TerrainTile(west=minx, south=miny, east=maxx, north=maxy)
ter2.fromFile(self.tmpfile)
self.assertIsInstance(ter2.__repr__(), str)
# check headers
self.assertGreater(len(ter.header), 0)
self.assertEqual(len(ter.header), len(ter2.header))
self.assertEqual(len(ter.header), len(TerrainTile.quantizedMeshHeader))
for k, v in ter.header.iteritems():
self.assertEqual(v, ter2.header[k], 'For k = ' + k)
# check vertices
self.assertGreater(len(ter.u), 0)
self.assertGreater(len(ter.v), 0)
self.assertGreater(len(ter.h), 0)
self.assertEqual(len(ter.u), len(ter2.u))
self.assertEqual(len(ter.v), len(ter2.v))
self.assertEqual(len(ter.h), len(ter2.h))
for i, v in enumerate(ter.u):
self.assertEqual(v, ter2.u[i])
for i, v in enumerate(ter.v):
self.assertEqual(v, ter2.v[i])
for i, v in enumerate(ter.h):
self.assertEqual(v, ter2.h[i])
self.assertEqual(
len(ter.getVerticesCoordinates()),
len(ter2.getVerticesCoordinates())
)
# check indices
self.assertGreater(len(ter.indices), 0)
self.assertEqual(len(ter.indices), len(ter2.indices))
for i, v in enumerate(ter.indices):
self.assertEqual(v, ter2.indices[i], i)
# check edges
self.assertGreater(len(ter.westI), 0)
self.assertEqual(len(ter.westI), len(ter2.westI))
for i, v in enumerate(ter.westI):
self.assertEqual(v, ter2.westI[i], i)
self.assertGreater(len(ter.southI), 0)
self.assertEqual(len(ter.southI), len(ter2.southI))
for i, v in enumerate(ter.southI):
self.assertEqual(v, ter2.southI[i], i)
self.assertGreater(len(ter.eastI), 0)
self.assertEqual(len(ter.eastI), len(ter2.eastI))
for i, v in enumerate(ter.eastI):
self.assertEqual(v, ter2.eastI[i], i)
self.assertGreater(len(ter.northI), 0)
self.assertEqual(len(ter.northI), len(ter2.northI))
for i, v in enumerate(ter.northI):
self.assertEqual(v, ter2.northI[i], i)
self.assertEqual(ter2.getContentType(),
'application/vnd.quantized-mesh')
minWKT = 'POINT (%f %f)' % (MINX, MINY)
maxWKT = 'POINT (%f %f)' % (MAXX, MAXY)
minPoint = transformCoordinate(minWKT, 21781, 4326)
maxPoint = transformCoordinate(maxWKT, 21781, 4326)
MINX = minPoint.GetX()
MINY = minPoint.GetY()
MAXX = maxPoint.GetX()
MAXY = maxPoint.GetY()
print 'Extent :'
print[MINX, MINY, MAXX, MAXY]
MINZOOM = 3
MAXZOOM = 17
geodetic = GlobalGeodetic(True)
drv = ogr.GetDriverByName('ESRI Shapefile')
directory = '.tmp/'
extension = '.shp'
# Generate table with min max tile coordinates for all zoomlevels
for tz in range(MINZOOM, MAXZOOM + 1):
filePathTarget = '%s%s%s' % (directory, tz, extension)
tminx, tminy = geodetic.LonLatToTile(MINX, MINY, tz)
tmaxx, tmaxy = geodetic.LonLatToTile(MAXX, MAXY, tz)
dataSource = drv.CreateDataSource(filePathTarget)
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
layer = dataSource.CreateLayer('%s' % tz, srs, ogr.wkbPolygon)
fieldKey = ogr.FieldDefn('Key', ogr.OFTString)