def main(template): t0 = time.time() tilecount = None layerConfig = ConfigParser.RawConfigParser() layerConfig.read(template) try: terrainBased = layerConfig.getboolean('Grid', 'terrainBased') except ConfigParser.NoSectionError as e: logger.error(e, exc_info=True) raise ValueError('The layer configuration file contains errors.') if terrainBased: # params = parseTerrainBasedLayer(layerConfig) # tileJSON = createTerrainBasedTileJSON(params) params = parseTerrainBasedLayer(layerConfig) tileJSON = createS3BasedTileJSON(params) else: dbConfig = ConfigParser.RawConfigParser() dbConfig.read('configs/raster/database.cfg') params = parseModelBasedLayer(dbConfig, layerConfig) (tileJSON, tilecount) = createModelBasedTileJSON(params) # Same bucket for now bucket = getBucket() fileObj = cStringIO.StringIO() fileObj.write(tileJSON) fileObj = gzipFileObject(fileObj) logger.info('Uploading %slayer.json to S3' % params.bucketBasePath) writeToS3(bucket, 'layer.json', fileObj, 'tilejson', params.bucketBasePath, contentType='application/json') logger.info('layer.json has been uploaded successfully') if tilecount: tend = time.time() logger.info('It took %s to scan %s tiles' % (str(datetime.timedelta(seconds=tend - t0)), tilecount))
def copyAGITiles(zooms, bounds, bucketBasePath): count = 0 fullonly = 0 headers = {'Accept': 'application/vnd.quantized-mesh;' + 'extensions=octvertexnormals-' + 'watermask,application/octet-stream;q=0.9,*/*;q=0.01'} baseURL = 'http://assets.agi.com/stk-terrain/world/' bucket = getBucket() for bxyz in grid(bounds, zooms, fullonly): f = cStringIO.StringIO() tilebounds, [x, y, z] = bxyz bucketKey = tilePathTemplate(x, y, z) f.write(loadTileContent(baseURL, bucketKey, headers)) f.seek(0) compressedFile = gzipFileObject(f) writeToS3(bucket, bucketKey, compressedFile, 'poc_watermask', bucketBasePath) count += 1 if count % 20 == 0: print 'Copying %s...' % bucketKey print '%s tiles have been copied so far.' % count
def copyAGITiles(zooms, bounds, bucketBasePath): count = 0 headers = { 'Accept': 'application/vnd.quantized-mesh;' + 'extensions=octvertexnormals-' + 'watermask,application/octet-stream;q=0.9,*/*;q=0.01'} baseURL = 'http://assets.agi.com/stk-terrain/world/' bucket = getBucket() for bxyz in grid(bounds, zooms[0], zooms[len(zooms) - 1]): f = cStringIO.StringIO() tilebounds, [x, y, z] = bxyz bucketKey = tilePathTemplate(x, y, z) f.write(loadTileContent(baseURL, bucketKey, headers)) f.seek(0) compressedFile = gzipFileObject(f) writeToS3( bucket, bucketKey, compressedFile, 'poc_watermask', bucketBasePath) count += 1 if count % 20 == 0: print 'Copying %s...' % bucketKey print '%s tiles have been copied so far.' % count
def main(template): t0 = time.time() tilecount = None layerConfig = ConfigParser.RawConfigParser() layerConfig.read(template) try: terrainBased = layerConfig.getboolean('Grid', 'terrainBased') except ConfigParser.NoSectionError as e: logger.error(e, exc_info=True) raise ValueError('The layer configuration file contains errors.') if terrainBased: # params = parseTerrainBasedLayer(layerConfig) # tileJSON = createTerrainBasedTileJSON(params) params = parseTerrainBasedLayer(layerConfig) tileJSON = createS3BasedTileJSON(params) else: dbConfig = ConfigParser.RawConfigParser() dbConfig.read('configs/raster/database.cfg') params = parseModelBasedLayer(dbConfig, layerConfig) (tileJSON, tilecount) = createModelBasedTileJSON(params) # Same bucket for now bucket = getBucket() fileObj = cStringIO.StringIO() fileObj.write(tileJSON) fileObj = gzipFileObject(fileObj) logger.info('Uploading %slayer.json to S3' % params.bucketBasePath) writeToS3(bucket, 'layer.json', fileObj, 'tilejson', params.bucketBasePath, contentType='application/json') logger.info('layer.json has been uploaded successfully') if tilecount: tend = time.time() logger.info('It took %s to scan %s tiles' % ( str(datetime.timedelta(seconds=tend - t0)), tilecount))
def main(): try: opts, args = getopt.getopt(sys.argv[1:], 'f:t:', ['from=', 'to=']) except getopt.GetoptError as err: error(str(err), 2, usage=usage) ffrom = None to = None ffile = None for o, a in opts: if o in ('-f', '--from'): ffrom = a if o in ('-t', '--to'): to = a if ffrom is None or to is None: if len(args) != 1: error("Please specify a file.", 4, usage=usage) ffile = args[0] tiles = [] # We have file, so we get the tiles from the file if ffile is not None: with open(ffile) as f: for line in f: line = line.rstrip() if len(line) > 2: tiles.append(map(int, line.split('/'))) # If we have from to, we catch layers.json from poc and use from to for levels if ffrom is not None and to is not None: req = urlopen(poc_base_url + 'layer.json') layers = {} for line in req: layers = json.loads(line) for zoom in range(int(ffrom), int(to) + 1): level = layers['available'][zoom][0] for x in range(int(level['startX']), int(level['endX']) + 1): for y in range(int(level['startY']), int(level['endY']) + 1): tiles.append([zoom, x, y]) bucket = getBucket() g = GlobalGeodetic(tmscompatible=True) if not os.path.isdir('tmp'): os.mkdir('tmp') for tile in tiles: req = None tilebounds = g.TileBounds(tile[1], tile[2], tile[0]) tilestring = str(tile[0]) + "/" + str(tile[1]) + "/" + str(tile[2]) try: url = poc_base_url + tilestring + ".terrain?v=2.0.0" req = urlopen(url) with open(gzip_file_name, 'wb') as fp: fp.write(req.read()) ff = gzip.open(gzip_file_name) with open(temp_file_name, 'wb') as fp: fp.write(ff.read()) ter = TerrainTile() ter.fromFile( temp_file_name, tilebounds[0], tilebounds[2], tilebounds[1], tilebounds[3] ) ''' if os.path.isfile(tms_file_name): os.remove(tms_file_name) if os.path.isfile(shape_file_name): os.remove(shape_file_name) if os.path.isfile(tms_from_shape_file_name): os.remove(tms_from_shape_file_name) ter.toFile(tms_file_name) ter.toShapefile(shape_file_name) shapefile = ShpToGDALFeatures(shpFilePath=shape_file_name) features = shapefile.__read__() topology = TerrainTopology(features=features) topology.fromGDALFeatures() terFromPoc = TerrainTile() terFromPoc.fromFile( tms_file_name, tilebounds[0], tilebounds[2], tilebounds[1], tilebounds[3] ) terFromShape = TerrainTile() terFromShape.fromTerrainTopology(topology) terFromShape.toFile(tms_from_shape_file_name) # Use this to select what is written to s3 ter2 = terFromShape ''' ter2 = ter fileObject = ter2.toStringIO() compressedFile = gzipFileObject(fileObject) bucketKey = tilestring + '.terrain' print 'Uploading %s to S3' % bucketKey writeToS3(bucket, bucketKey, compressedFile, 'POC Tiles copy', contentType=ter.getContentType()) except Exception as e: print "error with " + line + " " + str(e) finally: if req: req.close() return 0
from forge.configs import tmsConfig from forge.terrain import TerrainTile from forge.terrain.topology import TerrainTopology from forge.lib.shapefile_utils import ShpToGDALFeatures from forge.lib.helpers import gzipFileObject from forge.lib.boto_conn import getBucket, writeToS3 basePath = '/var/local/cartoweb/tmp/3dpoc/swissalti3d/Interlaken_Pyr17/' extension = '.terrain' i = 0 t0 = time.time() getShapefiles = lambda x: x.endswith('.shp') shapefilesNames = filter(getShapefiles, os.listdir(basePath)) bucket = getBucket() bucketBasePath = tmsConfig.get('General', 'bucketpath') for f in shapefilesNames: filePathSource = basePath + f shapefile = ShpToGDALFeatures(shpFilePath=filePathSource) features = shapefile.__read__() terrainTopo = TerrainTopology(features=features) terrainTopo.fromGDALFeatures() terrainFormat = TerrainTile() terrainFormat.fromTerrainTopology(terrainTopo) fileObject = terrainFormat.toStringIO() compressedFile = gzipFileObject(fileObject)
def createTile(tile): session = None pid = os.getpid() try: (bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting, hasWatermask) = tile db = DB(dbConfigFile) bucketKey = '%s/%s/%s.terrain' % (tileXYZ[2], tileXYZ[0], tileXYZ[1]) with db.userSession() as session: bucket = getBucket() # Get the model according to the zoom level model = modelsPyramid.getModelByZoom(tileXYZ[2]) watermask = [] if hasWatermask: lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2]) query = session.query( lakeModel.watermaskRasterize(bounds).label('watermask')) for q in query: watermask = q.watermask # Get the interpolated point at the 4 corners # 0: (minX, minY), 1: (minX, maxY), # 2: (maxX, maxY), 3: (maxX, minY) pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0), (bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)] def toSubQuery(x): return session.query( model.id, model.interpolateHeightOnPlane(pts[x])).filter( and_(model.bboxIntersects(createBBox(pts[x], 0.01)), model.pointIntersects(pts[x]))).subquery('p%s' % x) subqueries = [toSubQuery(i) for i in range(0, len(pts))] # Get the height of the corner points as postgis cannot properly # clip a polygon cornerPts = {} step = 2 j = step query = session.query(*subqueries) for q in query: for i in range(0, len(q), step): sub = q[i:j] j += step cornerPts[sub[0]] = list( to_shape(WKBElement(sub[1])).coords) # Clip using the bounds clippedGeometry = model.bboxClippedGeom(bounds) query = session.query(model.id, clippedGeometry.label('clip')).filter( model.bboxIntersects(bounds)) geomCoords = [] for q in query: coords = list(to_shape(q.clip).exterior.coords) if q.id in cornerPts: pt = cornerPts[q.id][0] for i in range(0, len(coords)): c = coords[i] if c[0] == pt[0] and c[1] == pt[1]: coords[i] = [c[0], c[1], pt[2]] geomCoords.append(coords) nbGeoms = len(geomCoords) if nbGeoms > 0: try: terrainTile = encode(geomCoords, bounds=bounds, autocorrectGeometries=True, hasLighting=hasLighting, watermask=watermask) except Exception as e: msg = '[%s] --------- ERROR ------- occured while ' % pid msg += 'encoding terrain tile\n' msg += '[%s]: %s' % (pid, e) logger.error(msg, exc_info=True) raise Exception(e) writeToS3(bucket, bucketKey, terrainTile.toBytesIO(gzipped=True), model.__tablename__, bucketBasePath, contentType=terrainTile.getContentType()) tend = time.time() tilecount.value += 1 tilesCreated = tilecount.value total = tilesCreated + skipcount.value if tilesCreated % 1000 == 0: logger.info('[%s] Last tile %s (%s rings). ' '%s to write %s tiles. (total processed: %s)' % (pid, bucketKey, nbGeoms, str(datetime.timedelta(seconds=tend - t0)), tilesCreated, total)) else: skipcount.value += 1 val = skipcount.value total = val + tilecount.value # One should write an empyt tile logger.info('[%s] Skipping %s %s because no features found ' 'for this tile (%s skipped from %s total)' % (pid, bucketKey, bounds, val, total)) except Exception as e: logger.error(e, exc_info=True) raise Exception(e) finally: db.userEngine.dispose() return 0
def createTile(tile): session = None pid = os.getpid() try: (bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting, hasWatermask) = tile db = DB(dbConfigFile) with db.userSession() as session: bucket = getBucket() # Get the model according to the zoom level model = modelsPyramid.getModelByZoom(tileXYZ[2]) watermask = [] if hasWatermask: lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2]) query = session.query( lakeModel.watermaskRasterize(bounds).label('watermask') ) for q in query: watermask = q.watermask # Get the interpolated point at the 4 corners # 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY) pts = [ (bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0), (bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0) ] def toSubQuery(x): return session.query( model.id, model.interpolateHeightOnPlane(pts[x]) ).filter( and_( model.bboxIntersects(createBBox(pts[x], 0.01)), model.pointIntersects(pts[x]) ) ).subquery('p%s' % x) subqueries = [toSubQuery(i) for i in range(0, len(pts))] # Get the height of the corner points as postgis cannot properly # clip a polygon cornerPts = {} step = 2 j = step query = session.query(*subqueries) for q in query: for i in range(0, len(q), step): sub = q[i:j] j += step cornerPts[sub[0]] = list( to_shape(WKBElement(sub[1])).coords ) # Clip using the bounds clippedGeometry = model.bboxClippedGeom(bounds) query = session.query( model.id, clippedGeometry.label('clip') ).filter(model.bboxIntersects(bounds)) terrainTopo = TerrainTopology(hasLighting=hasLighting) for q in query: coords = list(to_shape(q.clip).exterior.coords) if q.id in cornerPts: pt = cornerPts[q.id][0] for i in range(0, len(coords)): c = coords[i] if c[0] == pt[0] and c[1] == pt[1]: coords[i] = [c[0], c[1], pt[2]] try: rings = processRingCoordinates(coords) except Exception as e: msg = '[%s] --------- ERROR ------- occured while ' \ 'collapsing non triangular shapes\n' % pid msg += '[%s]: %s' % (pid, e) logger.error(msg, exc_info=True) raise Exception(e) # Redundant coord has been remove already for vertices in rings: terrainTopo.addVertices(vertices) bucketKey = '%s/%s/%s.terrain' % ( tileXYZ[2], tileXYZ[0], tileXYZ[1]) verticesLength = len(terrainTopo.vertices) if verticesLength > 0: terrainTopo.create() # Prepare terrain tile terrainFormat = TerrainTile(watermask=watermask) terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds) # Bytes manipulation and compression fileObject = terrainFormat.toStringIO() compressedFile = gzipFileObject(fileObject) writeToS3( bucket, bucketKey, compressedFile, model.__tablename__, bucketBasePath, contentType=terrainFormat.getContentType() ) tend = time.time() tilecount.value += 1 val = tilecount.value total = val + skipcount.value if val % 10 == 0: logger.info('[%s] Last tile %s (%s rings). ' '%s to write %s tiles. (total processed: %s)' % ( pid, bucketKey, verticesLength, str(datetime.timedelta(seconds=tend - t0)), val, total ) ) else: skipcount.value += 1 val = skipcount.value total = val + tilecount.value # One should write an empyt tile logger.info('[%s] Skipping %s %s because no features found ' 'for this tile (%s skipped from %s total)' % ( pid, bucketKey, bounds, val, total ) ) except Exception as e: logger.error(e, exc_info=True) raise Exception(e) finally: db.userEngine.dispose() return 0
import datetime from forge.configs import tmsConfig from forge.terrain import TerrainTile from forge.terrain.topology import TerrainTopology from forge.lib.shapefile_utils import ShpToGDALFeatures from forge.lib.helpers import gzipFileObject from forge.lib.boto_conn import getBucket, writeToS3 basePath = '/var/local/cartoweb/tmp/3dpoc/swissalti3d/Interlaken_Pyr17/' extension = '.terrain' i = 0 t0 = time.time() getShapefiles = lambda x: x.endswith('.shp') shapefilesNames = filter(getShapefiles, os.listdir(basePath)) bucket = getBucket() bucketBasePath = tmsConfig.get('General', 'bucketpath') for f in shapefilesNames: filePathSource = basePath + f shapefile = ShpToGDALFeatures(shpFilePath=filePathSource) features = shapefile.__read__() terrainTopo = TerrainTopology(features=features) terrainTopo.fromGDALFeatures() terrainFormat = TerrainTile() terrainFormat.fromTerrainTopology(terrainTopo) fileObject = terrainFormat.toStringIO() compressedFile = gzipFileObject(fileObject)
def createTile(tile): session = None pid = os.getpid() try: (bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting, hasWatermask) = tile db = DB(dbConfigFile) with db.userSession() as session: bucket = getBucket() # Get the model according to the zoom level model = modelsPyramid.getModelByZoom(tileXYZ[2]) watermask = [] if hasWatermask: lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2]) query = session.query( lakeModel.watermaskRasterize(bounds).label('watermask')) for q in query: watermask = q.watermask # Get the interpolated point at the 4 corners # 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY) pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0), (bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)] def toSubQuery(x): return session.query( model.id, model.interpolateHeightOnPlane(pts[x])).filter( and_(model.bboxIntersects(createBBox(pts[x], 0.01)), model.pointIntersects(pts[x]))).subquery('p%s' % x) subqueries = [toSubQuery(i) for i in range(0, len(pts))] # Get the height of the corner points as postgis cannot properly # clip a polygon cornerPts = {} step = 2 j = step query = session.query(*subqueries) for q in query: for i in range(0, len(q), step): sub = q[i:j] j += step cornerPts[sub[0]] = list( to_shape(WKBElement(sub[1])).coords) # Clip using the bounds clippedGeometry = model.bboxClippedGeom(bounds) query = session.query(model.id, clippedGeometry.label('clip')).filter( model.bboxIntersects(bounds)) terrainTopo = TerrainTopology(hasLighting=hasLighting) for q in query: coords = list(to_shape(q.clip).exterior.coords) if q.id in cornerPts: pt = cornerPts[q.id][0] for i in range(0, len(coords)): c = coords[i] if c[0] == pt[0] and c[1] == pt[1]: coords[i] = [c[0], c[1], pt[2]] try: rings = processRingCoordinates(coords) except Exception as e: msg = '[%s] --------- ERROR ------- occured while ' \ 'collapsing non triangular shapes\n' % pid msg += '[%s]: %s' % (pid, e) logger.error(msg, exc_info=True) raise Exception(e) # Redundant coord has been remove already for vertices in rings: terrainTopo.addVertices(vertices) bucketKey = '%s/%s/%s.terrain' % (tileXYZ[2], tileXYZ[0], tileXYZ[1]) verticesLength = len(terrainTopo.vertices) if verticesLength > 0: terrainTopo.create() # Prepare terrain tile terrainFormat = TerrainTile(watermask=watermask) terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds) # Bytes manipulation and compression fileObject = terrainFormat.toStringIO() compressedFile = gzipFileObject(fileObject) writeToS3(bucket, bucketKey, compressedFile, model.__tablename__, bucketBasePath, contentType=terrainFormat.getContentType()) tend = time.time() tilecount.value += 1 val = tilecount.value total = val + skipcount.value if val % 10 == 0: logger.info('[%s] Last tile %s (%s rings). ' '%s to write %s tiles. (total processed: %s)' % (pid, bucketKey, verticesLength, str(datetime.timedelta(seconds=tend - t0)), val, total)) else: skipcount.value += 1 val = skipcount.value total = val + tilecount.value # One should write an empyt tile logger.info('[%s] Skipping %s %s because no features found ' 'for this tile (%s skipped from %s total)' % (pid, bucketKey, bounds, val, total)) except Exception as e: logger.error(e, exc_info=True) raise Exception(e) finally: db.userEngine.dispose() return 0