def _stats(self, withDb=True): self.t0 = time.time() total = 0 msg = '\n' tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0) geodetic = getTileGrid(4326)(tmsCompatible=True) bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat) zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1) db = DB('configs/terrain/database.cfg') try: with db.userSession() as session: for i in xrange(0, len(zooms)): zoom = zooms[i] model = modelsPyramid.getModelByZoom(zoom) nbObjects = None if withDb: nbObjects = session.query(model).filter( model.bboxIntersects(bounds)).count() # top letf corner tileMinX, tileMinY = geodetic.tileAddress( zoom, [bounds[0], bounds[3]]) # bottom right tileMaxX, tileMaxY = geodetic.tileAddress( zoom, [bounds[2], bounds[1]]) tileBounds = geodetic.tileBounds(zoom, tileMinX, tileMinY) xCount = tileMaxX - tileMinX + 1 yCount = tileMaxY - tileMinY + 1 nbTiles = xCount * yCount total += nbTiles pointA = transformCoordinate( 'POINT(%s %s)' % (tileBounds[0], tileBounds[1]), 4326, 21781).GetPoints()[0] pointB = transformCoordinate( 'POINT(%s %s)' % (tileBounds[2], tileBounds[3]), 4326, 21781).GetPoints()[0] length = int(round(c2d.distance(pointA, pointB))) msg += 'At zoom %s:\n' % zoom msg += 'We expect %s tiles overall\n' % nbTiles msg += 'Min X is %s, Max X is %s\n' % (tileMinX, tileMaxX) msg += '%s columns over X\n' % xCount msg += 'Min Y is %s, Max Y is %s\n' % (tileMinY, tileMaxY) msg += '%s rows over Y\n' % yCount msg += '\n' msg += 'A tile side is around %s meters' % length if nbTiles > 0 and nbObjects is not None: msg += 'We have an average of about %s triangles ' \ 'per tile\n' % int(round(nbObjects / nbTiles)) msg += '\n\n' msg += '%s tiles in total.' % total except Exception as e: logger.error('An error occured during statistics collection') logger.error('%s' % e, exc_info=True) raise Exception(e) finally: db.userEngine.dispose() return (total, msg)
def metadata(self): t0 = time.time() basePath = self.tmsConfig.get('General', 'bucketpath') baseUrls = [] url = 'https://terrain.dev.bgdi.ch' url += '/%s{z}/{x}/{y}.terrain?v={version}' % basePath baseUrls.append(url) # TODO #for i in range(0, 5): # url = 'https://terrain10%s.geo.admin.ch' % i # url = 'https://terrain10%s.geo.admin.ch' % i # url += '/%s{z}/{x}/{y}.terrain?v={version}' % basePath # baseUrls.append(url) db = DB('configs/terrain/database.cfg') tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, t0) tMeta = TerrainMetadata( bounds=tiles.bounds, minzoom=tiles.tileMinZ, maxzoom=tiles.tileMaxZ, useGlobalTiles=True, hasLighting=tiles.hasLighting, hasWatermask=tiles.hasWatermask, baseUrls=baseUrls) try: with db.userSession() as session: tilecount = 1 for tile in tiles: tMeta = scanTerrain(tMeta, tile, session, tilecount) tilecount += 1 tend = time.time() logger.info('It took %s to scan %s tiles' % ( str(datetime.timedelta(seconds=tend - t0)), tilecount)) except Exception as e: logger.error('An error occured during layer.json creation') logger.error('%s' % e, exc_info=True) raise Exception(e) finally: db.userEngine.dispose() with open('.tmp/layer.json', 'w') as f: f.write(tMeta.toJSON())
def metadata(self): t0 = time.time() serverAddress = self.tmsConfig.get('General', 'serveraddress') basePath = self.tmsConfig.get('General', 'bucketpath') baseUrls = [ serverAddress + basePath + "{z}/{x}/{y}.terrain?v={version}" ] db = DB('configs/terrain/database.cfg') tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, t0) tMeta = TerrainMetadata(bounds=tiles.bounds, minzoom=tiles.tileMinZ, maxzoom=tiles.tileMaxZ, useGlobalTiles=True, hasLighting=tiles.hasLighting, hasWatermask=tiles.hasWatermask, baseUrls=baseUrls) try: with db.userSession() as session: tilecount = 1 for tile in tiles: tMeta = scanTerrain(tMeta, tile, session, tilecount) tilecount += 1 tend = time.time() logger.info( 'It took %s to scan %s tiles' % (str(datetime.timedelta(seconds=tend - t0)), tilecount)) except Exception as e: logger.error('An error occured during layer.json creation') logger.error('%s' % e, exc_info=True) raise Exception(e) finally: db.userEngine.dispose() with open('.tmp/layer.json', 'w') as f: f.write(tMeta.toJSON())
def metadata(self): t0 = time.time() basePath = self.tmsConfig.get('General', 'bucketpath') baseUrls = [ "//terrain0.geo.admin.ch/" + basePath + "{z}/{x}/{y}.terrain?v={version}", "//terrain1.geo.admin.ch/" + basePath + "{z}/{x}/{y}.terrain?v={version}", "//terrain2.geo.admin.ch/" + basePath + "{z}/{x}/{y}.terrain?v={version}", "//terrain3.geo.admin.ch/" + basePath + "{z}/{x}/{y}.terrain?v={version}", "//terrain4.geo.admin.ch/" + basePath + "{z}/{x}/{y}.terrain?v={version}" ] db = DB('configs/terrain/database.cfg') tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, t0) tMeta = TerrainMetadata( bounds=tiles.bounds, minzoom=tiles.tileMinZ, maxzoom=tiles.tileMaxZ, useGlobalTiles=True, hasLighting=tiles.hasLighting, hasWatermask=tiles.hasWatermask, baseUrls=baseUrls) try: with db.userSession() as session: tilecount = 1 for tile in tiles: tMeta = scanTerrain(tMeta, tile, session, tilecount) tilecount += 1 tend = time.time() logger.info('It took %s to scan %s tiles' % ( str(datetime.timedelta(seconds=tend - t0)), tilecount)) except Exception as e: logger.error('An error occured during layer.json creation') logger.error('%s' % e, exc_info=True) raise Exception(e) finally: db.userEngine.dispose() with open('.tmp/layer.json', 'w') as f: f.write(tMeta.toJSON())
def main(): try: opts, args = getopt.getopt(sys.argv[1:], 'c:', ['config=']) except getopt.GetoptError as err: error(str(err), 2, usage=usage) dbConfigFile = 'configs/terrain/database.cfg' for o, a in opts: if o in ('-c', '--config'): dbConfigFile = a if len(args) < 1: error('you must specify a command', 3, usage=usage) db = DB(dbConfigFile) command = args[0] if command == 'console': db.console() elif command == 'create': db.create() elif command == 'createuser': db.createUser() elif command == 'createdb': db.createDB() elif command == 'setupfunctions': db.setupFunctions() elif command == 'populate': db.populate() elif command == 'populatelakes': db.populateLakes() elif command == 'dropuser': db.dropUser() elif command == 'dropdb': db.dropDatabase() elif command == 'destroy': db.destroy() else: error("unknown command '%(command)s'" % {'command': command}, 4, usage=usage)
def createTile(tile): session = None pid = os.getpid() try: (bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting, hasWatermask) = tile db = DB(dbConfigFile) bucketKey = '%s/%s/%s.terrain' % (tileXYZ[2], tileXYZ[0], tileXYZ[1]) with db.userSession() as session: bucket = getBucket() # Get the model according to the zoom level model = modelsPyramid.getModelByZoom(tileXYZ[2]) watermask = [] if hasWatermask: lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2]) query = session.query( lakeModel.watermaskRasterize(bounds).label('watermask')) for q in query: watermask = q.watermask # Get the interpolated point at the 4 corners # 0: (minX, minY), 1: (minX, maxY), # 2: (maxX, maxY), 3: (maxX, minY) pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0), (bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)] def toSubQuery(x): return session.query( model.id, model.interpolateHeightOnPlane(pts[x])).filter( and_(model.bboxIntersects(createBBox(pts[x], 0.01)), model.pointIntersects(pts[x]))).subquery('p%s' % x) subqueries = [toSubQuery(i) for i in range(0, len(pts))] # Get the height of the corner points as postgis cannot properly # clip a polygon cornerPts = {} step = 2 j = step query = session.query(*subqueries) for q in query: for i in range(0, len(q), step): sub = q[i:j] j += step cornerPts[sub[0]] = list( to_shape(WKBElement(sub[1])).coords) # Clip using the bounds clippedGeometry = model.bboxClippedGeom(bounds) query = session.query(model.id, clippedGeometry.label('clip')).filter( model.bboxIntersects(bounds)) geomCoords = [] for q in query: coords = list(to_shape(q.clip).exterior.coords) if q.id in cornerPts: pt = cornerPts[q.id][0] for i in range(0, len(coords)): c = coords[i] if c[0] == pt[0] and c[1] == pt[1]: coords[i] = [c[0], c[1], pt[2]] geomCoords.append(coords) nbGeoms = len(geomCoords) if nbGeoms > 0: try: terrainTile = encode(geomCoords, bounds=bounds, autocorrectGeometries=True, hasLighting=hasLighting, watermask=watermask) except Exception as e: msg = '[%s] --------- ERROR ------- occured while ' % pid msg += 'encoding terrain tile\n' msg += '[%s]: %s' % (pid, e) logger.error(msg, exc_info=True) raise Exception(e) writeToS3(bucket, bucketKey, terrainTile.toBytesIO(gzipped=True), model.__tablename__, bucketBasePath, contentType=terrainTile.getContentType()) tend = time.time() tilecount.value += 1 tilesCreated = tilecount.value total = tilesCreated + skipcount.value if tilesCreated % 1000 == 0: logger.info('[%s] Last tile %s (%s rings). ' '%s to write %s tiles. (total processed: %s)' % (pid, bucketKey, nbGeoms, str(datetime.timedelta(seconds=tend - t0)), tilesCreated, total)) else: skipcount.value += 1 val = skipcount.value total = val + tilecount.value # One should write an empyt tile logger.info('[%s] Skipping %s %s because no features found ' 'for this tile (%s skipped from %s total)' % (pid, bucketKey, bounds, val, total)) except Exception as e: logger.error(e, exc_info=True) raise Exception(e) finally: db.userEngine.dispose() return 0
def createTile(tile): session = None pid = os.getpid() try: (bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting, hasWatermask) = tile db = DB(dbConfigFile) with db.userSession() as session: bucket = getBucket() # Get the model according to the zoom level model = modelsPyramid.getModelByZoom(tileXYZ[2]) watermask = [] if hasWatermask: lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2]) query = session.query( lakeModel.watermaskRasterize(bounds).label('watermask') ) for q in query: watermask = q.watermask # Get the interpolated point at the 4 corners # 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY) pts = [ (bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0), (bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0) ] def toSubQuery(x): return session.query( model.id, model.interpolateHeightOnPlane(pts[x]) ).filter( and_( model.bboxIntersects(createBBox(pts[x], 0.01)), model.pointIntersects(pts[x]) ) ).subquery('p%s' % x) subqueries = [toSubQuery(i) for i in range(0, len(pts))] # Get the height of the corner points as postgis cannot properly # clip a polygon cornerPts = {} step = 2 j = step query = session.query(*subqueries) for q in query: for i in range(0, len(q), step): sub = q[i:j] j += step cornerPts[sub[0]] = list( to_shape(WKBElement(sub[1])).coords ) # Clip using the bounds clippedGeometry = model.bboxClippedGeom(bounds) query = session.query( model.id, clippedGeometry.label('clip') ).filter(model.bboxIntersects(bounds)) terrainTopo = TerrainTopology(hasLighting=hasLighting) for q in query: coords = list(to_shape(q.clip).exterior.coords) if q.id in cornerPts: pt = cornerPts[q.id][0] for i in range(0, len(coords)): c = coords[i] if c[0] == pt[0] and c[1] == pt[1]: coords[i] = [c[0], c[1], pt[2]] try: rings = processRingCoordinates(coords) except Exception as e: msg = '[%s] --------- ERROR ------- occured while ' \ 'collapsing non triangular shapes\n' % pid msg += '[%s]: %s' % (pid, e) logger.error(msg, exc_info=True) raise Exception(e) # Redundant coord has been remove already for vertices in rings: terrainTopo.addVertices(vertices) bucketKey = '%s/%s/%s.terrain' % ( tileXYZ[2], tileXYZ[0], tileXYZ[1]) verticesLength = len(terrainTopo.vertices) if verticesLength > 0: terrainTopo.create() # Prepare terrain tile terrainFormat = TerrainTile(watermask=watermask) terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds) # Bytes manipulation and compression fileObject = terrainFormat.toStringIO() compressedFile = gzipFileObject(fileObject) writeToS3( bucket, bucketKey, compressedFile, model.__tablename__, bucketBasePath, contentType=terrainFormat.getContentType() ) tend = time.time() tilecount.value += 1 val = tilecount.value total = val + skipcount.value if val % 10 == 0: logger.info('[%s] Last tile %s (%s rings). ' '%s to write %s tiles. (total processed: %s)' % ( pid, bucketKey, verticesLength, str(datetime.timedelta(seconds=tend - t0)), val, total ) ) else: skipcount.value += 1 val = skipcount.value total = val + tilecount.value # One should write an empyt tile logger.info('[%s] Skipping %s %s because no features found ' 'for this tile (%s skipped from %s total)' % ( pid, bucketKey, bounds, val, total ) ) except Exception as e: logger.error(e, exc_info=True) raise Exception(e) finally: db.userEngine.dispose() return 0
def _stats(self, withDb=True): self.t0 = time.time() total = 0 msg = '\n' tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0) geodetic = GlobalGeodetic(True) bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat) zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1) db = DB('configs/terrain/database.cfg') try: with db.userSession() as session: for i in xrange(0, len(zooms)): zoom = zooms[i] model = modelsPyramid.getModelByZoom(zoom) nbObjects = None if withDb: nbObjects = session.query(model).filter( model.bboxIntersects(bounds) ).count() tileMinX, tileMinY = geodetic.LonLatToTile( bounds[0], bounds[1], zoom ) tileMaxX, tileMaxY = geodetic.LonLatToTile( bounds[2], bounds[3], zoom ) # Fast approach, but might not be fully correct if tiles.fullonly == 1: tileMinX += 1 tileMinY += 1 tileMaxX -= 1 tileMaxY -= 1 tileBounds = geodetic.TileBounds(tileMinX, tileMinY, zoom) xCount = tileMaxX - tileMinX + 1 yCount = tileMaxY - tileMinY + 1 nbTiles = xCount * yCount total += nbTiles pointA = transformCoordinate('POINT(%s %s)' % ( tileBounds[0], tileBounds[1]), 4326, 21781 ).GetPoints()[0] pointB = transformCoordinate('POINT(%s %s)' % ( tileBounds[2], tileBounds[3]), 4326, 21781 ).GetPoints()[0] length = c2d.distance(pointA, pointB) if tiles.fullonly == 1: msg += 'WARNING: stats are approximative because ' \ 'fullonly is activated!\n' msg += 'At zoom %s:\n' % zoom msg += 'We expect %s tiles overall\n' % nbTiles msg += 'Min X is %s, Max X is %s\n' % (tileMinX, tileMaxX) msg += '%s columns over X\n' % xCount msg += 'Min Y is %s, Max Y is %s\n' % (tileMinY, tileMaxY) msg += '%s rows over Y\n' % yCount msg += '\n' msg += 'A tile side is around %s meters' % int(round(length)) if nbTiles > 0 and nbObjects is not None: msg += 'We have an average of about %s triangles ' \ 'per tile\n' % int(round(nbObjects / nbTiles)) msg += '\n\n' msg += '%s tiles in total.' % total except Exception as e: logger.error('An error occured during statistics collection') logger.error('%s' % e, exc_info=True) raise Exception(e) finally: db.userEngine.dispose() return (total, msg)
def createTile(tile): session = None pid = os.getpid() try: (bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting, hasWatermask) = tile db = DB(dbConfigFile) with db.userSession() as session: # bucket = getBucket() # Get the model according to the zoom level model = modelsPyramid.getModelByZoom(tileXYZ[2]) watermask = [] if hasWatermask: lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2]) query = session.query( lakeModel.watermaskRasterize(bounds).label('watermask')) for q in query: watermask = q.watermask # Get the interpolated point at the 4 corners # 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY) pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0), (bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)] def toSubQuery(x): return session.query( model.id, model.interpolateHeightOnPlane(pts[x])).filter( and_(model.bboxIntersects(createBBox(pts[x], 0.01)), model.pointIntersects(pts[x]))).subquery('p%s' % x) subqueries = [toSubQuery(i) for i in range(0, len(pts))] # Get the height of the corner points as postgis cannot properly # clip a polygon cornerPts = {} step = 2 j = step query = session.query(*subqueries) for q in query: for i in range(0, len(q), step): sub = q[i:j] j += step cornerPts[sub[0]] = list( to_shape(WKBElement(sub[1])).coords) # Clip using the bounds clippedGeometry = model.bboxClippedGeom(bounds) query = session.query(model.id, clippedGeometry.label('clip')).filter( model.bboxIntersects(bounds)) terrainTopo = TerrainTopology(hasLighting=hasLighting) for q in query: if to_shape(q.clip).geom_type == "GeometryCollection": continue if to_shape(q.clip).geom_type == "LineString" or to_shape( q.clip).geom_type == "Point": continue coords = list(to_shape(q.clip).exterior.coords) if q.id in cornerPts: pt = cornerPts[q.id][0] for i in range(0, len(coords)): c = coords[i] if c[0] == pt[0] and c[1] == pt[1]: coords[i] = [c[0], c[1], pt[2]] try: rings = processRingCoordinates(coords) except Exception as e: msg = '[%s] --------- ERROR ------- occured while ' \ 'collapsing non triangular shapes\n' % pid msg += '[%s]: %s' % (pid, e) logger.error(msg, exc_info=True) raise Exception(e) # Redundant coord has been remove already for vertices in rings: terrainTopo.addVertices(vertices) baseDir = '/Users/liberostelios/3d_forge_geodata/output' terrainDir = '%s/%s/%s' % (baseDir, tileXYZ[2], tileXYZ[0]) bucketKey = '%s/%s.terrain' % (terrainDir, tileXYZ[1]) verticesLength = len(terrainTopo.vertices) if verticesLength > 0: terrainTopo.create() # Prepare terrain tile terrainFormat = TerrainTile(watermask=watermask) terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds) # STELIOS: We want this in a file, instead if not os.path.isdir(terrainDir): os.makedirs(terrainDir) terrainFormat.toFile(bucketKey) logger.info('[%s] Just did %s %s' % (pid, bucketKey, bounds)) # Bytes manipulation and compression # fileObject = terrainFormat.toStringIO() # compressedFile = gzipFileObject(fileObject) # writeToS3( # bucket, bucketKey, compressedFile, model.__tablename__, # bucketBasePath, contentType=terrainFormat.getContentType() # ) tend = time.time() tilecount.value += 1 val = tilecount.value total = val + skipcount.value if val % 10 == 0: logger.info('[%s] Last tile %s (%s rings). ' '%s to write %s tiles. (total processed: %s)' % (pid, bucketKey, verticesLength, str(datetime.timedelta(seconds=tend - t0)), val, total)) else: skipcount.value += 1 val = skipcount.value total = val + tilecount.value # One should write an empyt tile logger.info('[%s] Skipping %s %s because no features found ' 'for this tile (%s skipped from %s total)' % (pid, bucketKey, bounds, val, total)) except Exception as e: logger.error(e, exc_info=True) raise Exception(e) finally: db.userEngine.dispose() return 0
def _stats(self, withDb=True): self.t0 = time.time() total = 0 msg = '\n' tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0) geodetic = GlobalGeodetic(True) bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat) zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1) db = DB('configs/terrain/database.cfg') try: with db.userSession() as session: for i in xrange(0, len(zooms)): zoom = zooms[i] model = modelsPyramid.getModelByZoom(zoom) nbObjects = None if withDb: nbObjects = session.query(model).filter( model.bboxIntersects(bounds)).count() tileMinX, tileMinY = geodetic.LonLatToTile( bounds[0], bounds[1], zoom) tileMaxX, tileMaxY = geodetic.LonLatToTile( bounds[2], bounds[3], zoom) # Fast approach, but might not be fully correct if tiles.fullonly == 1: tileMinX += 1 tileMinY += 1 tileMaxX -= 1 tileMaxY -= 1 tileBounds = geodetic.TileBounds(tileMinX, tileMinY, zoom) xCount = tileMaxX - tileMinX + 1 yCount = tileMaxY - tileMinY + 1 nbTiles = xCount * yCount total += nbTiles pointA = transformCoordinate( 'POINT(%s %s)' % (tileBounds[0], tileBounds[1]), 4326, 21781).GetPoints()[0] pointB = transformCoordinate( 'POINT(%s %s)' % (tileBounds[2], tileBounds[3]), 4326, 21781).GetPoints()[0] length = c2d.distance(pointA, pointB) if tiles.fullonly == 1: msg += 'WARNING: stats are approximative because ' \ 'fullonly is activated!\n' msg += 'At zoom %s:\n' % zoom msg += 'We expect %s tiles overall\n' % nbTiles msg += 'Min X is %s, Max X is %s\n' % (tileMinX, tileMaxX) msg += '%s columns over X\n' % xCount msg += 'Min Y is %s, Max Y is %s\n' % (tileMinY, tileMaxY) msg += '%s rows over Y\n' % yCount msg += '\n' msg += 'A tile side is around %s meters' % int( round(length)) if nbTiles > 0 and nbObjects is not None: msg += 'We have an average of about %s triangles ' \ 'per tile\n' % int(round(nbObjects / nbTiles)) msg += '\n\n' msg += '%s tiles in total.' % total except Exception as e: logger.error('An error occured during statistics collection') logger.error('%s' % e, exc_info=True) raise Exception(e) finally: db.userEngine.dispose() return (total, msg)