def _stats(self, withDb=True):
self.t0 = time.time()
total = 0
msg = '\n'
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
geodetic = getTileGrid(4326)(tmsCompatible=True)
bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)
db = DB('configs/terrain/database.cfg')
try:
with db.userSession() as session:
for i in xrange(0, len(zooms)):
zoom = zooms[i]
model = modelsPyramid.getModelByZoom(zoom)
nbObjects = None
if withDb:
nbObjects = session.query(model).filter(
model.bboxIntersects(bounds)).count()
# top letf corner
tileMinX, tileMinY = geodetic.tileAddress(
zoom, [bounds[0], bounds[3]])
# bottom right
tileMaxX, tileMaxY = geodetic.tileAddress(
zoom, [bounds[2], bounds[1]])
tileBounds = geodetic.tileBounds(zoom, tileMinX, tileMinY)
xCount = tileMaxX - tileMinX + 1
yCount = tileMaxY - tileMinY + 1
nbTiles = xCount * yCount
total += nbTiles
pointA = transformCoordinate(
'POINT(%s %s)' % (tileBounds[0], tileBounds[1]), 4326,
21781).GetPoints()[0]
pointB = transformCoordinate(
'POINT(%s %s)' % (tileBounds[2], tileBounds[3]), 4326,
21781).GetPoints()[0]
length = int(round(c2d.distance(pointA, pointB)))
msg += 'At zoom %s:\n' % zoom
msg += 'We expect %s tiles overall\n' % nbTiles
msg += 'Min X is %s, Max X is %s\n' % (tileMinX, tileMaxX)
msg += '%s columns over X\n' % xCount
msg += 'Min Y is %s, Max Y is %s\n' % (tileMinY, tileMaxY)
msg += '%s rows over Y\n' % yCount
msg += '\n'
msg += 'A tile side is around %s meters' % length
if nbTiles > 0 and nbObjects is not None:
msg += 'We have an average of about %s triangles ' \
'per tile\n' % int(round(nbObjects / nbTiles))
msg += '\n\n'
msg += '%s tiles in total.' % total
except Exception as e:
logger.error('An error occured during statistics collection')
logger.error('%s' % e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return (total, msg)
def _stats(self, withDb=True):
self.t0 = time.time()
total = 0
msg = "\n"
tiles = Tiles(self.dbConfigFile, self.tmsConfig, self.t0)
geodetic = GlobalGeodetic(True)
bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)
db = DB("database.cfg")
self.DBSession = scoped_session(sessionmaker(bind=db.userEngine))
for i in xrange(0, len(zooms)):
zoom = zooms[i]
model = modelsPyramid.getModelByZoom(zoom)
nbObjects = None
if withDb:
nbObjects = self.DBSession.query(model).filter(model.bboxIntersects(bounds)).count()
tileMinX, tileMinY = geodetic.LonLatToTile(bounds[0], bounds[1], zoom)
tileMaxX, tileMaxY = geodetic.LonLatToTile(bounds[2], bounds[3], zoom)
# Fast approach, but might not be fully correct
if tiles.fullonly == 1:
tileMinX += 1
tileMinY += 1
tileMaxX -= 1
tileMaxY -= 1
tileBounds = geodetic.TileBounds(tileMinX, tileMinY, zoom)
xCount = tileMaxX - tileMinX + 1
yCount = tileMaxY - tileMinY + 1
nbTiles = xCount * yCount
total += nbTiles
pointA = transformCoordinate("POINT(%s %s)" % (tileBounds[0], tileBounds[1]), 4326, 21781).GetPoints()[0]
pointB = transformCoordinate("POINT(%s %s)" % (tileBounds[2], tileBounds[3]), 4326, 21781).GetPoints()[0]
length = c2d.distance(pointA, pointB)
if tiles.fullonly == 1:
msg += "WARNING: stats are approximative because fullonly is activated!\n"
msg += "At zoom %s:\n" % zoom
msg += "We expect %s tiles overall\n" % nbTiles
msg += "Min X is %s, Max X is %s\n" % (tileMinX, tileMaxX)
msg += "%s columns over X\n" % xCount
msg += "Min Y is %s, Max Y is %s\n" % (tileMinY, tileMaxY)
msg += "%s rows over Y\n" % yCount
msg += "\n"
msg += "A tile side is around %s meters long\n" % int(round(length))
if nbTiles > 0 and nbObjects is not None:
msg += "We have an average of about %s triangles per tile\n" % int(round(nbObjects / nbTiles))
msg += "\n"
return (total, msg)
def scanTerrain(tMeta, tile, session, tilecount):
try:
(bounds, tileXYZ, t0, dbConfigFile) = tile
# Get the model according to the zoom level
model = modelsPyramid.getModelByZoom(tileXYZ[2])
query = session.query(model.id).filter(model.bboxIntersects(bounds)).limit(1)
try:
query.one()
except NoResultFound as e:
tMeta.removeTile(tileXYZ[0], tileXYZ[1], tileXYZ[2])
except Exception as e:
logger.error(e)
raise Exception(e)
tend = time.time()
if tilecount % 1000 == 0:
logger.info("It took %s to scan %s tiles" % (str(datetime.timedelta(seconds=tend - t0)), tilecount))
return tMeta
def scanTerrain(tMeta, tile, session, tilecount):
try:
(bounds, tileXYZ, t0, dbConfigFile, hasLighting, hasWatermask) = tile
# Get the model according to the zoom level
model = modelsPyramid.getModelByZoom(tileXYZ[2])
query = session.query(model.id).filter(
model.bboxIntersects(bounds)).limit(1)
try:
query.one()
except NoResultFound as e:
tMeta.removeTile(tileXYZ[0], tileXYZ[1], tileXYZ[2])
except Exception as e:
logger.error(e, exc_info=True)
raise Exception(e)
tend = time.time()
if tilecount % 1000 == 0:
logger.info('It took %s to scan %s tiles' %
(str(datetime.timedelta(seconds=tend - t0)), tilecount))
return tMeta
def createTile(tile):
session = None
pid = os.getpid()
try:
(bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting,
hasWatermask) = tile
db = DB(dbConfigFile)
bucketKey = '%s/%s/%s.terrain' % (tileXYZ[2], tileXYZ[0], tileXYZ[1])
with db.userSession() as session:
bucket = getBucket()
# Get the model according to the zoom level
model = modelsPyramid.getModelByZoom(tileXYZ[2])
watermask = []
if hasWatermask:
lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
query = session.query(
lakeModel.watermaskRasterize(bounds).label('watermask'))
for q in query:
watermask = q.watermask
# Get the interpolated point at the 4 corners
# 0: (minX, minY), 1: (minX, maxY),
# 2: (maxX, maxY), 3: (maxX, minY)
pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0),
(bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)]
def toSubQuery(x):
return session.query(
model.id, model.interpolateHeightOnPlane(pts[x])).filter(
and_(model.bboxIntersects(createBBox(pts[x], 0.01)),
model.pointIntersects(pts[x]))).subquery('p%s' %
x)
subqueries = [toSubQuery(i) for i in range(0, len(pts))]
# Get the height of the corner points as postgis cannot properly
# clip a polygon
cornerPts = {}
step = 2
j = step
query = session.query(*subqueries)
for q in query:
for i in range(0, len(q), step):
sub = q[i:j]
j += step
cornerPts[sub[0]] = list(
to_shape(WKBElement(sub[1])).coords)
# Clip using the bounds
clippedGeometry = model.bboxClippedGeom(bounds)
query = session.query(model.id,
clippedGeometry.label('clip')).filter(
model.bboxIntersects(bounds))
geomCoords = []
for q in query:
coords = list(to_shape(q.clip).exterior.coords)
if q.id in cornerPts:
pt = cornerPts[q.id][0]
for i in range(0, len(coords)):
c = coords[i]
if c[0] == pt[0] and c[1] == pt[1]:
coords[i] = [c[0], c[1], pt[2]]
geomCoords.append(coords)
nbGeoms = len(geomCoords)
if nbGeoms > 0:
try:
terrainTile = encode(geomCoords,
bounds=bounds,
autocorrectGeometries=True,
hasLighting=hasLighting,
watermask=watermask)
except Exception as e:
msg = '[%s] --------- ERROR ------- occured while ' % pid
msg += 'encoding terrain tile\n'
msg += '[%s]: %s' % (pid, e)
logger.error(msg, exc_info=True)
raise Exception(e)
writeToS3(bucket,
bucketKey,
terrainTile.toBytesIO(gzipped=True),
model.__tablename__,
bucketBasePath,
contentType=terrainTile.getContentType())
tend = time.time()
tilecount.value += 1
tilesCreated = tilecount.value
total = tilesCreated + skipcount.value
if tilesCreated % 1000 == 0:
logger.info('[%s] Last tile %s (%s rings). '
'%s to write %s tiles. (total processed: %s)' %
(pid, bucketKey, nbGeoms,
str(datetime.timedelta(seconds=tend - t0)),
tilesCreated, total))
else:
skipcount.value += 1
val = skipcount.value
total = val + tilecount.value
# One should write an empyt tile
logger.info('[%s] Skipping %s %s because no features found '
'for this tile (%s skipped from %s total)' %
(pid, bucketKey, bounds, val, total))
except Exception as e:
logger.error(e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return 0
def createTile(tile):
session = None
pid = os.getpid()
try:
(bounds, tileXYZ, t0, dbConfigFile, bucketBasePath,
hasLighting, hasWatermask) = tile
db = DB(dbConfigFile)
with db.userSession() as session:
bucket = getBucket()
# Get the model according to the zoom level
model = modelsPyramid.getModelByZoom(tileXYZ[2])
watermask = []
if hasWatermask:
lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
query = session.query(
lakeModel.watermaskRasterize(bounds).label('watermask')
)
for q in query:
watermask = q.watermask
# Get the interpolated point at the 4 corners
# 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY)
pts = [
(bounds[0], bounds[1], 0),
(bounds[0], bounds[3], 0),
(bounds[2], bounds[3], 0),
(bounds[2], bounds[1], 0)
]
def toSubQuery(x):
return session.query(
model.id, model.interpolateHeightOnPlane(pts[x])
).filter(
and_(
model.bboxIntersects(createBBox(pts[x], 0.01)),
model.pointIntersects(pts[x])
)
).subquery('p%s' % x)
subqueries = [toSubQuery(i) for i in range(0, len(pts))]
# Get the height of the corner points as postgis cannot properly
# clip a polygon
cornerPts = {}
step = 2
j = step
query = session.query(*subqueries)
for q in query:
for i in range(0, len(q), step):
sub = q[i:j]
j += step
cornerPts[sub[0]] = list(
to_shape(WKBElement(sub[1])).coords
)
# Clip using the bounds
clippedGeometry = model.bboxClippedGeom(bounds)
query = session.query(
model.id,
clippedGeometry.label('clip')
).filter(model.bboxIntersects(bounds))
terrainTopo = TerrainTopology(hasLighting=hasLighting)
for q in query:
coords = list(to_shape(q.clip).exterior.coords)
if q.id in cornerPts:
pt = cornerPts[q.id][0]
for i in range(0, len(coords)):
c = coords[i]
if c[0] == pt[0] and c[1] == pt[1]:
coords[i] = [c[0], c[1], pt[2]]
try:
rings = processRingCoordinates(coords)
except Exception as e:
msg = '[%s] --------- ERROR ------- occured while ' \
'collapsing non triangular shapes\n' % pid
msg += '[%s]: %s' % (pid, e)
logger.error(msg, exc_info=True)
raise Exception(e)
# Redundant coord has been remove already
for vertices in rings:
terrainTopo.addVertices(vertices)
bucketKey = '%s/%s/%s.terrain' % (
tileXYZ[2], tileXYZ[0], tileXYZ[1])
verticesLength = len(terrainTopo.vertices)
if verticesLength > 0:
terrainTopo.create()
# Prepare terrain tile
terrainFormat = TerrainTile(watermask=watermask)
terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds)
# Bytes manipulation and compression
fileObject = terrainFormat.toStringIO()
compressedFile = gzipFileObject(fileObject)
writeToS3(
bucket, bucketKey, compressedFile, model.__tablename__,
bucketBasePath, contentType=terrainFormat.getContentType()
)
tend = time.time()
tilecount.value += 1
val = tilecount.value
total = val + skipcount.value
if val % 10 == 0:
logger.info('[%s] Last tile %s (%s rings). '
'%s to write %s tiles. (total processed: %s)' % (
pid, bucketKey, verticesLength,
str(datetime.timedelta(seconds=tend - t0)),
val, total
)
)
else:
skipcount.value += 1
val = skipcount.value
total = val + tilecount.value
# One should write an empyt tile
logger.info('[%s] Skipping %s %s because no features found '
'for this tile (%s skipped from %s total)' % (
pid, bucketKey, bounds, val, total
)
)
except Exception as e:
logger.error(e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return 0
def _stats(self, withDb=True):
self.t0 = time.time()
total = 0
msg = '\n'
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
geodetic = GlobalGeodetic(True)
bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)
db = DB('configs/terrain/database.cfg')
try:
with db.userSession() as session:
for i in xrange(0, len(zooms)):
zoom = zooms[i]
model = modelsPyramid.getModelByZoom(zoom)
nbObjects = None
if withDb:
nbObjects = session.query(model).filter(
model.bboxIntersects(bounds)
).count()
tileMinX, tileMinY = geodetic.LonLatToTile(
bounds[0], bounds[1], zoom
)
tileMaxX, tileMaxY = geodetic.LonLatToTile(
bounds[2], bounds[3], zoom
)
# Fast approach, but might not be fully correct
if tiles.fullonly == 1:
tileMinX += 1
tileMinY += 1
tileMaxX -= 1
tileMaxY -= 1
tileBounds = geodetic.TileBounds(tileMinX, tileMinY, zoom)
xCount = tileMaxX - tileMinX + 1
yCount = tileMaxY - tileMinY + 1
nbTiles = xCount * yCount
total += nbTiles
pointA = transformCoordinate('POINT(%s %s)' % (
tileBounds[0], tileBounds[1]), 4326, 21781
).GetPoints()[0]
pointB = transformCoordinate('POINT(%s %s)' % (
tileBounds[2], tileBounds[3]), 4326, 21781
).GetPoints()[0]
length = c2d.distance(pointA, pointB)
if tiles.fullonly == 1:
msg += 'WARNING: stats are approximative because ' \
'fullonly is activated!\n'
msg += 'At zoom %s:\n' % zoom
msg += 'We expect %s tiles overall\n' % nbTiles
msg += 'Min X is %s, Max X is %s\n' % (tileMinX, tileMaxX)
msg += '%s columns over X\n' % xCount
msg += 'Min Y is %s, Max Y is %s\n' % (tileMinY, tileMaxY)
msg += '%s rows over Y\n' % yCount
msg += '\n'
msg += 'A tile side is around %s meters' % int(round(length))
if nbTiles > 0 and nbObjects is not None:
msg += 'We have an average of about %s triangles ' \
'per tile\n' % int(round(nbObjects / nbTiles))
msg += '\n\n'
msg += '%s tiles in total.' % total
except Exception as e:
logger.error('An error occured during statistics collection')
logger.error('%s' % e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return (total, msg)
def createTile(tile):
session = None
pid = os.getpid()
try:
(bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting,
hasWatermask) = tile
db = DB(dbConfigFile)
with db.userSession() as session:
# bucket = getBucket()
# Get the model according to the zoom level
model = modelsPyramid.getModelByZoom(tileXYZ[2])
watermask = []
if hasWatermask:
lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
query = session.query(
lakeModel.watermaskRasterize(bounds).label('watermask'))
for q in query:
watermask = q.watermask
# Get the interpolated point at the 4 corners
# 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY)
pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0),
(bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)]
def toSubQuery(x):
return session.query(
model.id, model.interpolateHeightOnPlane(pts[x])).filter(
and_(model.bboxIntersects(createBBox(pts[x], 0.01)),
model.pointIntersects(pts[x]))).subquery('p%s' %
x)
subqueries = [toSubQuery(i) for i in range(0, len(pts))]
# Get the height of the corner points as postgis cannot properly
# clip a polygon
cornerPts = {}
step = 2
j = step
query = session.query(*subqueries)
for q in query:
for i in range(0, len(q), step):
sub = q[i:j]
j += step
cornerPts[sub[0]] = list(
to_shape(WKBElement(sub[1])).coords)
# Clip using the bounds
clippedGeometry = model.bboxClippedGeom(bounds)
query = session.query(model.id,
clippedGeometry.label('clip')).filter(
model.bboxIntersects(bounds))
terrainTopo = TerrainTopology(hasLighting=hasLighting)
for q in query:
if to_shape(q.clip).geom_type == "GeometryCollection":
continue
if to_shape(q.clip).geom_type == "LineString" or to_shape(
q.clip).geom_type == "Point":
continue
coords = list(to_shape(q.clip).exterior.coords)
if q.id in cornerPts:
pt = cornerPts[q.id][0]
for i in range(0, len(coords)):
c = coords[i]
if c[0] == pt[0] and c[1] == pt[1]:
coords[i] = [c[0], c[1], pt[2]]
try:
rings = processRingCoordinates(coords)
except Exception as e:
msg = '[%s] --------- ERROR ------- occured while ' \
'collapsing non triangular shapes\n' % pid
msg += '[%s]: %s' % (pid, e)
logger.error(msg, exc_info=True)
raise Exception(e)
# Redundant coord has been remove already
for vertices in rings:
terrainTopo.addVertices(vertices)
baseDir = '/Users/liberostelios/3d_forge_geodata/output'
terrainDir = '%s/%s/%s' % (baseDir, tileXYZ[2], tileXYZ[0])
bucketKey = '%s/%s.terrain' % (terrainDir, tileXYZ[1])
verticesLength = len(terrainTopo.vertices)
if verticesLength > 0:
terrainTopo.create()
# Prepare terrain tile
terrainFormat = TerrainTile(watermask=watermask)
terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds)
# STELIOS: We want this in a file, instead
if not os.path.isdir(terrainDir):
os.makedirs(terrainDir)
terrainFormat.toFile(bucketKey)
logger.info('[%s] Just did %s %s' % (pid, bucketKey, bounds))
# Bytes manipulation and compression
# fileObject = terrainFormat.toStringIO()
# compressedFile = gzipFileObject(fileObject)
# writeToS3(
# bucket, bucketKey, compressedFile, model.__tablename__,
# bucketBasePath, contentType=terrainFormat.getContentType()
# )
tend = time.time()
tilecount.value += 1
val = tilecount.value
total = val + skipcount.value
if val % 10 == 0:
logger.info('[%s] Last tile %s (%s rings). '
'%s to write %s tiles. (total processed: %s)' %
(pid, bucketKey, verticesLength,
str(datetime.timedelta(seconds=tend - t0)),
val, total))
else:
skipcount.value += 1
val = skipcount.value
total = val + tilecount.value
# One should write an empyt tile
logger.info('[%s] Skipping %s %s because no features found '
'for this tile (%s skipped from %s total)' %
(pid, bucketKey, bounds, val, total))
except Exception as e:
logger.error(e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return 0
def _stats(self, withDb=True):
self.t0 = time.time()
total = 0
msg = '\n'
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
geodetic = GlobalGeodetic(True)
bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)
db = DB('configs/terrain/database.cfg')
try:
with db.userSession() as session:
for i in xrange(0, len(zooms)):
zoom = zooms[i]
model = modelsPyramid.getModelByZoom(zoom)
nbObjects = None
if withDb:
nbObjects = session.query(model).filter(
model.bboxIntersects(bounds)).count()
tileMinX, tileMinY = geodetic.LonLatToTile(
bounds[0], bounds[1], zoom)
tileMaxX, tileMaxY = geodetic.LonLatToTile(
bounds[2], bounds[3], zoom)
# Fast approach, but might not be fully correct
if tiles.fullonly == 1:
tileMinX += 1
tileMinY += 1
tileMaxX -= 1
tileMaxY -= 1
tileBounds = geodetic.TileBounds(tileMinX, tileMinY, zoom)
xCount = tileMaxX - tileMinX + 1
yCount = tileMaxY - tileMinY + 1
nbTiles = xCount * yCount
total += nbTiles
pointA = transformCoordinate(
'POINT(%s %s)' % (tileBounds[0], tileBounds[1]), 4326,
21781).GetPoints()[0]
pointB = transformCoordinate(
'POINT(%s %s)' % (tileBounds[2], tileBounds[3]), 4326,
21781).GetPoints()[0]
length = c2d.distance(pointA, pointB)
if tiles.fullonly == 1:
msg += 'WARNING: stats are approximative because ' \
'fullonly is activated!\n'
msg += 'At zoom %s:\n' % zoom
msg += 'We expect %s tiles overall\n' % nbTiles
msg += 'Min X is %s, Max X is %s\n' % (tileMinX, tileMaxX)
msg += '%s columns over X\n' % xCount
msg += 'Min Y is %s, Max Y is %s\n' % (tileMinY, tileMaxY)
msg += '%s rows over Y\n' % yCount
msg += '\n'
msg += 'A tile side is around %s meters' % int(
round(length))
if nbTiles > 0 and nbObjects is not None:
msg += 'We have an average of about %s triangles ' \
'per tile\n' % int(round(nbObjects / nbTiles))
msg += '\n\n'
msg += '%s tiles in total.' % total
except Exception as e:
logger.error('An error occured during statistics collection')
logger.error('%s' % e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return (total, msg)