def testWatermaskOnlyReader(self):
z = 9
x = 769
y = 319
geodetic = GlobalGeodetic(True)
ter = TerrainTile()
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
ter.fromFile('forge/data/quantized-mesh/%s_%s_%s_watermask.terrain' %
(z, x, y),
minx,
miny,
maxx,
maxy,
hasWatermask=True)
self.assertEqual(len(ter.watermask), 256)
for row in ter.watermask:
self.assertEqual(len(row), 256)
for val in row:
self.assertTrue(val >= 0)
self.assertTrue(val <= 255)
ter.toFile(self.tmpfile)
ter2 = TerrainTile()
ter2.fromFile(self.tmpfile, minx, miny, maxx, maxy, hasWatermask=True)
self.assertEqual(len(ter2.watermask), 256)
for i in range(0, len(ter.watermask)):
for j in range(0, len(ter.watermask[i])):
self.assertEqual(ter.watermask[i][j], ter2.watermask[i][j])
def testWatermaskOnlyReader(self):
z = 9
x = 769
y = 319
geodetic = GlobalGeodetic(True)
ter = TerrainTile()
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
ter.fromFile('forge/data/quantized-mesh/%s_%s_%s_watermask.terrain' % (z, x, y),
minx, miny, maxx, maxy, hasWatermask=True)
self.assertEqual(len(ter.watermask), 256)
for row in ter.watermask:
self.assertEqual(len(row), 256)
for val in row:
self.assertTrue(val >= 0)
self.assertTrue(val <= 255)
ter.toFile(self.tmpfile)
ter2 = TerrainTile()
ter2.fromFile(self.tmpfile, minx, miny, maxx, maxy, hasWatermask=True)
self.assertEqual(len(ter2.watermask), 256)
for i in range(0, len(ter.watermask)):
for j in range(0, len(ter.watermask[i])):
self.assertEqual(ter.watermask[i][j], ter2.watermask[i][j])
def testExtensionsReader(self):
z = 10
x = 1563
y = 590
geodetic = GlobalGeodetic(True)
ter = TerrainTile()
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
ter.fromFile(
'forge/data/quantized-mesh/%s_%s_%s_light_watermask.terrain' %
(z, x, y),
minx,
miny,
maxx,
maxy,
hasLighting=True,
hasWatermask=True)
# check indices
self.assertTrue(len(ter.indices) > 0)
# check edges
self.assertTrue(len(ter.westI) > 0)
self.assertTrue(len(ter.southI) > 0)
self.assertTrue(len(ter.eastI) > 0)
self.assertTrue(len(ter.northI) > 0)
# check extensions
self.assertEqual(len(ter.watermask), 1)
self.assertEqual(len(ter.watermask[0]), 1)
# Water only -> 255
self.assertEqual(ter.watermask[0][0], 255)
ter.toFile(self.tmpfile)
ter2 = TerrainTile()
ter2.fromFile(self.tmpfile,
minx,
miny,
maxx,
maxy,
hasLighting=True,
hasWatermask=True)
self.assertEqual(len(ter.watermask), len(ter2.watermask))
self.assertEqual(len(ter.watermask[0]), len(ter2.watermask[0]))
sign = lambda a: 1 if a > 0 else -1 if a < 0 else 0
for i in range(0, len(ter.vLight)):
for j in range(0, 3):
# We cannot have an exact equality with successive
# oct encoding and decoding
# Thus we only check the sign
self.assertEqual(sign(ter.vLight[i][j]),
sign(ter2.vLight[i][j]))
def testExtensionsReader(self):
z = 10
x = 1563
y = 590
geodetic = GlobalGeodetic(True)
ter = TerrainTile()
[minx, miny, maxx, maxy] = geodetic.TileBounds(x, y, z)
ter.fromFile(
'forge/data/quantized-mesh/%s_%s_%s_light_watermask.terrain' % (z, x, y),
minx, miny, maxx, maxy, hasLighting=True, hasWatermask=True
)
# check indices
self.assertTrue(len(ter.indices) > 0)
# check edges
self.assertTrue(len(ter.westI) > 0)
self.assertTrue(len(ter.southI) > 0)
self.assertTrue(len(ter.eastI) > 0)
self.assertTrue(len(ter.northI) > 0)
# check extensions
self.assertEqual(len(ter.watermask), 1)
self.assertEqual(len(ter.watermask[0]), 1)
# Water only -> 255
self.assertEqual(ter.watermask[0][0], 255)
ter.toFile(self.tmpfile)
ter2 = TerrainTile()
ter2.fromFile(self.tmpfile,
minx, miny, maxx, maxy, hasLighting=True, hasWatermask=True)
self.assertEqual(len(ter.watermask), len(ter2.watermask))
self.assertEqual(len(ter.watermask[0]), len(ter2.watermask[0]))
sign = lambda a: 1 if a > 0 else -1 if a < 0 else 0
for i in range(0, len(ter.vLight)):
for j in range(0, 3):
# We cannot have an exact equality with successive
# oct encoding and decoding
# Thus we only check the sign
self.assertEqual(sign(ter.vLight[i][j]), sign(ter2.vLight[i][j]))
from forge.terrain import TerrainTile
from forge.terrain.topology import TerrainTopology
from forge.lib.shapefile_utils import ShpToGDALFeatures
from forge.lib.global_geodetic import GlobalGeodetic
basename = '7_133_98'
directory = '.tmp'
extension = '.terrain'
curDir = os.getcwd()
filePathSource = '%s/forge/data/shapefile-features/%s.shp' % (curDir, basename)
filePathTarget = '%s/%s%s' % (directory, basename, extension)
shapefile = ShpToGDALFeatures(shpFilePath=filePathSource)
features = shapefile.__read__()
terrainTopo = TerrainTopology(features=features)
terrainTopo.fromGDALFeatures()
terrainFormat = TerrainTile()
geodetic = GlobalGeodetic(True)
zxy = basename.split('_')
bounds = geodetic.TileBounds(float(zxy[1]), float(zxy[2]), float(zxy[0]))
terrainFormat.fromTerrainTopology(terrainTopo, bounds)
terrainFormat.toFile(filePathTarget)
# Display SwissCoordinates
terrainFormat.computeVerticesCoordinates(epsg=21781)
print terrainFormat
from forge.terrain.topology import TerrainTopology
from forge.lib.shapefile_utils import ShpToGDALFeatures
from forge.lib.global_geodetic import GlobalGeodetic
basename = '7_133_98'
directory = '.tmp'
extension = '.terrain'
curDir = os.getcwd()
filePathSource = '%s/forge/data/shapefile-features/%s.shp' % (curDir, basename)
filePathTarget = '%s/%s%s' % (directory, basename, extension)
shapefile = ShpToGDALFeatures(shpFilePath=filePathSource)
features = shapefile.__read__()
terrainTopo = TerrainTopology(features=features)
terrainTopo.fromGDALFeatures()
terrainFormat = TerrainTile()
geodetic = GlobalGeodetic(True)
zxy = basename.split('_')
bounds = geodetic.TileBounds(float(zxy[1]), float(zxy[2]), float(zxy[0]))
terrainFormat.fromTerrainTopology(terrainTopo, bounds)
terrainFormat.toFile(filePathTarget)
# Display SwissCoordinates
terrainFormat.computeVerticesCoordinates(epsg=21781)
print terrainFormat
def testReaderWriter(self):
'''
Circle jerk testing.
We read the file with our reader
We write this data with our writer to a temporary file
We read this temporary file
We compare the results
'''
ter = TerrainTile()
# ter.fromFile(
# 'forge/data/quantized-mesh/0.terrain', 7.80938, 7.81773, 46.30261, 46.3079)
# ter.fromFile(
# 'forge/data/quantized-mesh/raron.flat.1.terrain',
# 7.80938, 7.81773, 46.30261, 46.30790)
ter.fromFile(
'forge/data/quantized-mesh/goms.mountains.1.terrain',
7.80938, 7.81773, 46.30261, 46.30799
)
ter.toFile(self.tmpfile)
ter2 = TerrainTile()
ter2.fromFile(self.tmpfile, 7.80938, 7.81773, 46.30261, 46.30799)
# check headers
self.assertTrue(len(ter.header) > 0)
self.assertEqual(len(ter.header), len(ter2.header))
self.assertEqual(len(ter.header), len(TerrainTile.quantizedMeshHeader))
for k, v in ter.header.iteritems():
self.assertEqual(v, ter2.header[k], 'For k = ' + k)
# check vertices
self.assertTrue(len(ter.u) > 0)
self.assertTrue(len(ter.v) > 0)
self.assertTrue(len(ter.h) > 0)
self.assertEqual(len(ter.u), len(ter2.u))
self.assertEqual(len(ter.v), len(ter2.v))
self.assertEqual(len(ter.h), len(ter2.h))
for i, v in enumerate(ter.u):
self.assertEqual(v, ter2.u[i])
for i, v in enumerate(ter.v):
self.assertEqual(v, ter2.v[i])
for i, v in enumerate(ter.h):
self.assertEqual(v, ter2.h[i])
# check indices
self.assertTrue(len(ter.indices) > 0)
self.assertEqual(len(ter.indices), len(ter2.indices))
for i, v in enumerate(ter.indices):
self.assertEqual(v, ter2.indices[i], i)
# check edges
self.assertTrue(len(ter.westI) > 0)
self.assertEqual(len(ter.westI), len(ter2.westI))
for i, v in enumerate(ter.westI):
self.assertEqual(v, ter2.westI[i], i)
self.assertTrue(len(ter.southI) > 0)
self.assertEqual(len(ter.southI), len(ter2.southI))
for i, v in enumerate(ter.southI):
self.assertEqual(v, ter2.southI[i], i)
self.assertTrue(len(ter.eastI) > 0)
self.assertEqual(len(ter.eastI), len(ter2.eastI))
for i, v in enumerate(ter.eastI):
self.assertEqual(v, ter2.eastI[i], i)
self.assertTrue(len(ter.northI) > 0)
self.assertEqual(len(ter.northI), len(ter2.northI))
for i, v in enumerate(ter.northI):
self.assertEqual(v, ter2.northI[i], i)
def testReaderWriter(self):
'''
Circle jerk testing.
We read the file with our reader
We write this data with our writer to a temporary file
We read this temporary file
We compare the results
'''
ter = TerrainTile()
# ter.fromFile(
# 'forge/data/quantized-mesh/0.terrain', 7.80938, 7.81773, 46.30261, 46.3079)
# ter.fromFile(
# 'forge/data/quantized-mesh/raron.flat.1.terrain',
# 7.80938, 7.81773, 46.30261, 46.30790)
ter.fromFile('forge/data/quantized-mesh/goms.mountains.1.terrain',
7.80938, 7.81773, 46.30261, 46.30799)
ter.toFile(self.tmpfile)
ter2 = TerrainTile()
ter2.fromFile(self.tmpfile, 7.80938, 7.81773, 46.30261, 46.30799)
# check headers
self.assertTrue(len(ter.header) > 0)
self.assertEqual(len(ter.header), len(ter2.header))
self.assertEqual(len(ter.header), len(TerrainTile.quantizedMeshHeader))
for k, v in ter.header.iteritems():
self.assertEqual(v, ter2.header[k], 'For k = ' + k)
# check vertices
self.assertTrue(len(ter.u) > 0)
self.assertTrue(len(ter.v) > 0)
self.assertTrue(len(ter.h) > 0)
self.assertEqual(len(ter.u), len(ter2.u))
self.assertEqual(len(ter.v), len(ter2.v))
self.assertEqual(len(ter.h), len(ter2.h))
for i, v in enumerate(ter.u):
self.assertEqual(v, ter2.u[i])
for i, v in enumerate(ter.v):
self.assertEqual(v, ter2.v[i])
for i, v in enumerate(ter.h):
self.assertEqual(v, ter2.h[i])
# check indices
self.assertTrue(len(ter.indices) > 0)
self.assertEqual(len(ter.indices), len(ter2.indices))
for i, v in enumerate(ter.indices):
self.assertEqual(v, ter2.indices[i], i)
# check edges
self.assertTrue(len(ter.westI) > 0)
self.assertEqual(len(ter.westI), len(ter2.westI))
for i, v in enumerate(ter.westI):
self.assertEqual(v, ter2.westI[i], i)
self.assertTrue(len(ter.southI) > 0)
self.assertEqual(len(ter.southI), len(ter2.southI))
for i, v in enumerate(ter.southI):
self.assertEqual(v, ter2.southI[i], i)
self.assertTrue(len(ter.eastI) > 0)
self.assertEqual(len(ter.eastI), len(ter2.eastI))
for i, v in enumerate(ter.eastI):
self.assertEqual(v, ter2.eastI[i], i)
self.assertTrue(len(ter.northI) > 0)
self.assertEqual(len(ter.northI), len(ter2.northI))
for i, v in enumerate(ter.northI):
self.assertEqual(v, ter2.northI[i], i)
def createTile(tile):
session = None
pid = os.getpid()
try:
(bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting,
hasWatermask) = tile
db = DB(dbConfigFile)
with db.userSession() as session:
# bucket = getBucket()
# Get the model according to the zoom level
model = modelsPyramid.getModelByZoom(tileXYZ[2])
watermask = []
if hasWatermask:
lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
query = session.query(
lakeModel.watermaskRasterize(bounds).label('watermask'))
for q in query:
watermask = q.watermask
# Get the interpolated point at the 4 corners
# 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY)
pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0),
(bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)]
def toSubQuery(x):
return session.query(
model.id, model.interpolateHeightOnPlane(pts[x])).filter(
and_(model.bboxIntersects(createBBox(pts[x], 0.01)),
model.pointIntersects(pts[x]))).subquery('p%s' %
x)
subqueries = [toSubQuery(i) for i in range(0, len(pts))]
# Get the height of the corner points as postgis cannot properly
# clip a polygon
cornerPts = {}
step = 2
j = step
query = session.query(*subqueries)
for q in query:
for i in range(0, len(q), step):
sub = q[i:j]
j += step
cornerPts[sub[0]] = list(
to_shape(WKBElement(sub[1])).coords)
# Clip using the bounds
clippedGeometry = model.bboxClippedGeom(bounds)
query = session.query(model.id,
clippedGeometry.label('clip')).filter(
model.bboxIntersects(bounds))
terrainTopo = TerrainTopology(hasLighting=hasLighting)
for q in query:
if to_shape(q.clip).geom_type == "GeometryCollection":
continue
if to_shape(q.clip).geom_type == "LineString" or to_shape(
q.clip).geom_type == "Point":
continue
coords = list(to_shape(q.clip).exterior.coords)
if q.id in cornerPts:
pt = cornerPts[q.id][0]
for i in range(0, len(coords)):
c = coords[i]
if c[0] == pt[0] and c[1] == pt[1]:
coords[i] = [c[0], c[1], pt[2]]
try:
rings = processRingCoordinates(coords)
except Exception as e:
msg = '[%s] --------- ERROR ------- occured while ' \
'collapsing non triangular shapes\n' % pid
msg += '[%s]: %s' % (pid, e)
logger.error(msg, exc_info=True)
raise Exception(e)
# Redundant coord has been remove already
for vertices in rings:
terrainTopo.addVertices(vertices)
baseDir = '/Users/liberostelios/3d_forge_geodata/output'
terrainDir = '%s/%s/%s' % (baseDir, tileXYZ[2], tileXYZ[0])
bucketKey = '%s/%s.terrain' % (terrainDir, tileXYZ[1])
verticesLength = len(terrainTopo.vertices)
if verticesLength > 0:
terrainTopo.create()
# Prepare terrain tile
terrainFormat = TerrainTile(watermask=watermask)
terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds)
# STELIOS: We want this in a file, instead
if not os.path.isdir(terrainDir):
os.makedirs(terrainDir)
terrainFormat.toFile(bucketKey)
logger.info('[%s] Just did %s %s' % (pid, bucketKey, bounds))
# Bytes manipulation and compression
# fileObject = terrainFormat.toStringIO()
# compressedFile = gzipFileObject(fileObject)
# writeToS3(
# bucket, bucketKey, compressedFile, model.__tablename__,
# bucketBasePath, contentType=terrainFormat.getContentType()
# )
tend = time.time()
tilecount.value += 1
val = tilecount.value
total = val + skipcount.value
if val % 10 == 0:
logger.info('[%s] Last tile %s (%s rings). '
'%s to write %s tiles. (total processed: %s)' %
(pid, bucketKey, verticesLength,
str(datetime.timedelta(seconds=tend - t0)),
val, total))
else:
skipcount.value += 1
val = skipcount.value
total = val + tilecount.value
# One should write an empyt tile
logger.info('[%s] Skipping %s %s because no features found '
'for this tile (%s skipped from %s total)' %
(pid, bucketKey, bounds, val, total))
except Exception as e:
logger.error(e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return 0
