def main(template):
t0 = time.time()
tilecount = None
layerConfig = ConfigParser.RawConfigParser()
layerConfig.read(template)
try:
terrainBased = layerConfig.getboolean('Grid', 'terrainBased')
except ConfigParser.NoSectionError as e:
logger.error(e, exc_info=True)
raise ValueError('The layer configuration file contains errors.')
if terrainBased:
# params = parseTerrainBasedLayer(layerConfig)
# tileJSON = createTerrainBasedTileJSON(params)
params = parseTerrainBasedLayer(layerConfig)
tileJSON = createS3BasedTileJSON(params)
else:
dbConfig = ConfigParser.RawConfigParser()
dbConfig.read('configs/raster/database.cfg')
params = parseModelBasedLayer(dbConfig, layerConfig)
(tileJSON, tilecount) = createModelBasedTileJSON(params)
# Same bucket for now
bucket = getBucket()
fileObj = cStringIO.StringIO()
fileObj.write(tileJSON)
fileObj = gzipFileObject(fileObj)
logger.info('Uploading %slayer.json to S3' % params.bucketBasePath)
writeToS3(bucket,
'layer.json',
fileObj,
'tilejson',
params.bucketBasePath,
contentType='application/json')
logger.info('layer.json has been uploaded successfully')
if tilecount:
tend = time.time()
logger.info('It took %s to scan %s tiles' %
(str(datetime.timedelta(seconds=tend - t0)), tilecount))
def copyAGITiles(zooms, bounds, bucketBasePath):
count = 0
fullonly = 0
headers = {'Accept': 'application/vnd.quantized-mesh;' +
'extensions=octvertexnormals-' +
'watermask,application/octet-stream;q=0.9,*/*;q=0.01'}
baseURL = 'http://assets.agi.com/stk-terrain/world/'
bucket = getBucket()
for bxyz in grid(bounds, zooms, fullonly):
f = cStringIO.StringIO()
tilebounds, [x, y, z] = bxyz
bucketKey = tilePathTemplate(x, y, z)
f.write(loadTileContent(baseURL, bucketKey, headers))
f.seek(0)
compressedFile = gzipFileObject(f)
writeToS3(bucket, bucketKey, compressedFile,
'poc_watermask', bucketBasePath)
count += 1
if count % 20 == 0:
print 'Copying %s...' % bucketKey
print '%s tiles have been copied so far.' % count
def copyAGITiles(zooms, bounds, bucketBasePath):
count = 0
headers = {
'Accept': 'application/vnd.quantized-mesh;' +
'extensions=octvertexnormals-' +
'watermask,application/octet-stream;q=0.9,*/*;q=0.01'}
baseURL = 'http://assets.agi.com/stk-terrain/world/'
bucket = getBucket()
for bxyz in grid(bounds, zooms[0], zooms[len(zooms) - 1]):
f = cStringIO.StringIO()
tilebounds, [x, y, z] = bxyz
bucketKey = tilePathTemplate(x, y, z)
f.write(loadTileContent(baseURL, bucketKey, headers))
f.seek(0)
compressedFile = gzipFileObject(f)
writeToS3(
bucket, bucketKey, compressedFile,
'poc_watermask', bucketBasePath)
count += 1
if count % 20 == 0:
print 'Copying %s...' % bucketKey
print '%s tiles have been copied so far.' % count
def main(template):
t0 = time.time()
tilecount = None
layerConfig = ConfigParser.RawConfigParser()
layerConfig.read(template)
try:
terrainBased = layerConfig.getboolean('Grid', 'terrainBased')
except ConfigParser.NoSectionError as e:
logger.error(e, exc_info=True)
raise ValueError('The layer configuration file contains errors.')
if terrainBased:
# params = parseTerrainBasedLayer(layerConfig)
# tileJSON = createTerrainBasedTileJSON(params)
params = parseTerrainBasedLayer(layerConfig)
tileJSON = createS3BasedTileJSON(params)
else:
dbConfig = ConfigParser.RawConfigParser()
dbConfig.read('configs/raster/database.cfg')
params = parseModelBasedLayer(dbConfig, layerConfig)
(tileJSON, tilecount) = createModelBasedTileJSON(params)
# Same bucket for now
bucket = getBucket()
fileObj = cStringIO.StringIO()
fileObj.write(tileJSON)
fileObj = gzipFileObject(fileObj)
logger.info('Uploading %slayer.json to S3' % params.bucketBasePath)
writeToS3(bucket, 'layer.json', fileObj, 'tilejson', params.bucketBasePath,
contentType='application/json')
logger.info('layer.json has been uploaded successfully')
if tilecount:
tend = time.time()
logger.info('It took %s to scan %s tiles' % (
str(datetime.timedelta(seconds=tend - t0)), tilecount))
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], 'f:t:', ['from=', 'to='])
except getopt.GetoptError as err:
error(str(err), 2, usage=usage)
ffrom = None
to = None
ffile = None
for o, a in opts:
if o in ('-f', '--from'):
ffrom = a
if o in ('-t', '--to'):
to = a
if ffrom is None or to is None:
if len(args) != 1:
error("Please specify a file.", 4, usage=usage)
ffile = args[0]
tiles = []
# We have file, so we get the tiles from the file
if ffile is not None:
with open(ffile) as f:
for line in f:
line = line.rstrip()
if len(line) > 2:
tiles.append(map(int, line.split('/')))
# If we have from to, we catch layers.json from poc and use from to for levels
if ffrom is not None and to is not None:
req = urlopen(poc_base_url + 'layer.json')
layers = {}
for line in req:
layers = json.loads(line)
for zoom in range(int(ffrom), int(to) + 1):
level = layers['available'][zoom][0]
for x in range(int(level['startX']), int(level['endX']) + 1):
for y in range(int(level['startY']), int(level['endY']) + 1):
tiles.append([zoom, x, y])
bucket = getBucket()
g = GlobalGeodetic(tmscompatible=True)
if not os.path.isdir('tmp'):
os.mkdir('tmp')
for tile in tiles:
req = None
tilebounds = g.TileBounds(tile[1], tile[2], tile[0])
tilestring = str(tile[0]) + "/" + str(tile[1]) + "/" + str(tile[2])
try:
url = poc_base_url + tilestring + ".terrain?v=2.0.0"
req = urlopen(url)
with open(gzip_file_name, 'wb') as fp:
fp.write(req.read())
ff = gzip.open(gzip_file_name)
with open(temp_file_name, 'wb') as fp:
fp.write(ff.read())
ter = TerrainTile()
ter.fromFile(
temp_file_name, tilebounds[0],
tilebounds[2], tilebounds[1], tilebounds[3]
)
'''
if os.path.isfile(tms_file_name):
os.remove(tms_file_name)
if os.path.isfile(shape_file_name):
os.remove(shape_file_name)
if os.path.isfile(tms_from_shape_file_name):
os.remove(tms_from_shape_file_name)
ter.toFile(tms_file_name)
ter.toShapefile(shape_file_name)
shapefile = ShpToGDALFeatures(shpFilePath=shape_file_name)
features = shapefile.__read__()
topology = TerrainTopology(features=features)
topology.fromGDALFeatures()
terFromPoc = TerrainTile()
terFromPoc.fromFile(
tms_file_name, tilebounds[0], tilebounds[2], tilebounds[1], tilebounds[3]
)
terFromShape = TerrainTile()
terFromShape.fromTerrainTopology(topology)
terFromShape.toFile(tms_from_shape_file_name)
# Use this to select what is written to s3
ter2 = terFromShape
'''
ter2 = ter
fileObject = ter2.toStringIO()
compressedFile = gzipFileObject(fileObject)
bucketKey = tilestring + '.terrain'
print 'Uploading %s to S3' % bucketKey
writeToS3(bucket, bucketKey, compressedFile, 'POC Tiles copy',
contentType=ter.getContentType())
except Exception as e:
print "error with " + line + " " + str(e)
finally:
if req:
req.close()
return 0
getShapefiles = lambda x: x.endswith('.shp')
shapefilesNames = filter(getShapefiles, os.listdir(basePath))
bucket = getBucket()
bucketBasePath = tmsConfig.get('General', 'bucketpath')
for f in shapefilesNames:
filePathSource = basePath + f
shapefile = ShpToGDALFeatures(shpFilePath=filePathSource)
features = shapefile.__read__()
terrainTopo = TerrainTopology(features=features)
terrainTopo.fromGDALFeatures()
terrainFormat = TerrainTile()
terrainFormat.fromTerrainTopology(terrainTopo)
fileObject = terrainFormat.toStringIO()
compressedFile = gzipFileObject(fileObject)
# Hardcoded scheme because we don't have any separators in the names
# for now
keyPath = f[1:3] + '/' + f[3:9] + '/' + f[9:14] + extension
print 'Writing %s to S3' % keyPath
writeToS3(bucket, keyPath, compressedFile, basePath, bucketBasePath,
contentType=terrainFormat.getContentType())
i += 1
t1 = time.time()
ti = t1 - t0
print 'It took %s secs to write %s/%s tiles' % (
str(datetime.timedelta(seconds=ti)), i, len(shapefilesNames)
)
# -*- coding: utf-8 -*-
import cStringIO
from forge.configs import tmsConfig
from forge.lib.helpers import gzipFileObject
from forge.lib.boto_conn import getBucket, writeToS3
bucket = getBucket()
layerJSONPath = 'forge/data/json-conf/layer.json'
bucketBasePath = tmsConfig.get('General', 'bucketpath')
with open(layerJSONPath) as f:
fileObj = cStringIO.StringIO()
fileObj.write(f.read())
fileObj = gzipFileObject(fileObj)
writeToS3(bucket, 'layer.json', fileObj, 'DB Scan',
bucketBasePath, 'application/json')
def createTile(tile):
session = None
pid = os.getpid()
try:
(bounds, tileXYZ, t0, dbConfigFile, bucketBasePath,
hasLighting, hasWatermask) = tile
db = DB(dbConfigFile)
with db.userSession() as session:
bucket = getBucket()
# Get the model according to the zoom level
model = modelsPyramid.getModelByZoom(tileXYZ[2])
watermask = []
if hasWatermask:
lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
query = session.query(
lakeModel.watermaskRasterize(bounds).label('watermask')
)
for q in query:
watermask = q.watermask
# Get the interpolated point at the 4 corners
# 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY)
pts = [
(bounds[0], bounds[1], 0),
(bounds[0], bounds[3], 0),
(bounds[2], bounds[3], 0),
(bounds[2], bounds[1], 0)
]
def toSubQuery(x):
return session.query(
model.id, model.interpolateHeightOnPlane(pts[x])
).filter(
and_(
model.bboxIntersects(createBBox(pts[x], 0.01)),
model.pointIntersects(pts[x])
)
).subquery('p%s' % x)
subqueries = [toSubQuery(i) for i in range(0, len(pts))]
# Get the height of the corner points as postgis cannot properly
# clip a polygon
cornerPts = {}
step = 2
j = step
query = session.query(*subqueries)
for q in query:
for i in range(0, len(q), step):
sub = q[i:j]
j += step
cornerPts[sub[0]] = list(
to_shape(WKBElement(sub[1])).coords
)
# Clip using the bounds
clippedGeometry = model.bboxClippedGeom(bounds)
query = session.query(
model.id,
clippedGeometry.label('clip')
).filter(model.bboxIntersects(bounds))
terrainTopo = TerrainTopology(hasLighting=hasLighting)
for q in query:
coords = list(to_shape(q.clip).exterior.coords)
if q.id in cornerPts:
pt = cornerPts[q.id][0]
for i in range(0, len(coords)):
c = coords[i]
if c[0] == pt[0] and c[1] == pt[1]:
coords[i] = [c[0], c[1], pt[2]]
try:
rings = processRingCoordinates(coords)
except Exception as e:
msg = '[%s] --------- ERROR ------- occured while ' \
'collapsing non triangular shapes\n' % pid
msg += '[%s]: %s' % (pid, e)
logger.error(msg, exc_info=True)
raise Exception(e)
# Redundant coord has been remove already
for vertices in rings:
terrainTopo.addVertices(vertices)
bucketKey = '%s/%s/%s.terrain' % (
tileXYZ[2], tileXYZ[0], tileXYZ[1])
verticesLength = len(terrainTopo.vertices)
if verticesLength > 0:
terrainTopo.create()
# Prepare terrain tile
terrainFormat = TerrainTile(watermask=watermask)
terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds)
# Bytes manipulation and compression
fileObject = terrainFormat.toStringIO()
compressedFile = gzipFileObject(fileObject)
writeToS3(
bucket, bucketKey, compressedFile, model.__tablename__,
bucketBasePath, contentType=terrainFormat.getContentType()
)
tend = time.time()
tilecount.value += 1
val = tilecount.value
total = val + skipcount.value
if val % 10 == 0:
logger.info('[%s] Last tile %s (%s rings). '
'%s to write %s tiles. (total processed: %s)' % (
pid, bucketKey, verticesLength,
str(datetime.timedelta(seconds=tend - t0)),
val, total
)
)
else:
skipcount.value += 1
val = skipcount.value
total = val + tilecount.value
# One should write an empyt tile
logger.info('[%s] Skipping %s %s because no features found '
'for this tile (%s skipped from %s total)' % (
pid, bucketKey, bounds, val, total
)
)
except Exception as e:
logger.error(e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return 0
shapefilesNames = filter(getShapefiles, os.listdir(basePath))
bucket = getBucket()
bucketBasePath = tmsConfig.get('General', 'bucketpath')
for f in shapefilesNames:
filePathSource = basePath + f
shapefile = ShpToGDALFeatures(shpFilePath=filePathSource)
features = shapefile.__read__()
terrainTopo = TerrainTopology(features=features)
terrainTopo.fromGDALFeatures()
terrainFormat = TerrainTile()
terrainFormat.fromTerrainTopology(terrainTopo)
fileObject = terrainFormat.toStringIO()
compressedFile = gzipFileObject(fileObject)
# Hardcoded scheme because we don't have any separators in the names
# for now
keyPath = f[1:3] + '/' + f[3:9] + '/' + f[9:14] + extension
print 'Writing %s to S3' % keyPath
writeToS3(bucket,
keyPath,
compressedFile,
basePath,
bucketBasePath,
contentType=terrainFormat.getContentType())
i += 1
t1 = time.time()
ti = t1 - t0
print 'It took %s secs to write %s/%s tiles' % (str(
# -*- coding: utf-8 -*-
import cStringIO
from forge.configs import tmsConfig
from forge.lib.helpers import gzipFileObject
from forge.lib.boto_conn import getBucket, writeToS3
bucket = getBucket()
layerJSONPath = 'forge/data/json-conf/layer.json'
bucketBasePath = tmsConfig.get('General', 'bucketpath')
with open(layerJSONPath) as f:
fileObj = cStringIO.StringIO()
fileObj.write(f.read())
fileObj = gzipFileObject(fileObj)
writeToS3(bucket, 'layer.json', fileObj, 'DB Scan', bucketBasePath,
'application/json')
def createTile(tile):
session = None
pid = os.getpid()
try:
(bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting,
hasWatermask) = tile
db = DB(dbConfigFile)
with db.userSession() as session:
bucket = getBucket()
# Get the model according to the zoom level
model = modelsPyramid.getModelByZoom(tileXYZ[2])
watermask = []
if hasWatermask:
lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
query = session.query(
lakeModel.watermaskRasterize(bounds).label('watermask'))
for q in query:
watermask = q.watermask
# Get the interpolated point at the 4 corners
# 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY)
pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0),
(bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)]
def toSubQuery(x):
return session.query(
model.id, model.interpolateHeightOnPlane(pts[x])).filter(
and_(model.bboxIntersects(createBBox(pts[x], 0.01)),
model.pointIntersects(pts[x]))).subquery('p%s' %
x)
subqueries = [toSubQuery(i) for i in range(0, len(pts))]
# Get the height of the corner points as postgis cannot properly
# clip a polygon
cornerPts = {}
step = 2
j = step
query = session.query(*subqueries)
for q in query:
for i in range(0, len(q), step):
sub = q[i:j]
j += step
cornerPts[sub[0]] = list(
to_shape(WKBElement(sub[1])).coords)
# Clip using the bounds
clippedGeometry = model.bboxClippedGeom(bounds)
query = session.query(model.id,
clippedGeometry.label('clip')).filter(
model.bboxIntersects(bounds))
terrainTopo = TerrainTopology(hasLighting=hasLighting)
for q in query:
coords = list(to_shape(q.clip).exterior.coords)
if q.id in cornerPts:
pt = cornerPts[q.id][0]
for i in range(0, len(coords)):
c = coords[i]
if c[0] == pt[0] and c[1] == pt[1]:
coords[i] = [c[0], c[1], pt[2]]
try:
rings = processRingCoordinates(coords)
except Exception as e:
msg = '[%s] --------- ERROR ------- occured while ' \
'collapsing non triangular shapes\n' % pid
msg += '[%s]: %s' % (pid, e)
logger.error(msg, exc_info=True)
raise Exception(e)
# Redundant coord has been remove already
for vertices in rings:
terrainTopo.addVertices(vertices)
bucketKey = '%s/%s/%s.terrain' % (tileXYZ[2], tileXYZ[0],
tileXYZ[1])
verticesLength = len(terrainTopo.vertices)
if verticesLength > 0:
terrainTopo.create()
# Prepare terrain tile
terrainFormat = TerrainTile(watermask=watermask)
terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds)
# Bytes manipulation and compression
fileObject = terrainFormat.toStringIO()
compressedFile = gzipFileObject(fileObject)
writeToS3(bucket,
bucketKey,
compressedFile,
model.__tablename__,
bucketBasePath,
contentType=terrainFormat.getContentType())
tend = time.time()
tilecount.value += 1
val = tilecount.value
total = val + skipcount.value
if val % 10 == 0:
logger.info('[%s] Last tile %s (%s rings). '
'%s to write %s tiles. (total processed: %s)' %
(pid, bucketKey, verticesLength,
str(datetime.timedelta(seconds=tend - t0)),
val, total))
else:
skipcount.value += 1
val = skipcount.value
total = val + tilecount.value
# One should write an empyt tile
logger.info('[%s] Skipping %s %s because no features found '
'for this tile (%s skipped from %s total)' %
(pid, bucketKey, bounds, val, total))
except Exception as e:
logger.error(e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return 0