Example #1
0
def main(template):
    t0 = time.time()
    tilecount = None
    layerConfig = ConfigParser.RawConfigParser()
    layerConfig.read(template)
    try:
        terrainBased = layerConfig.getboolean('Grid', 'terrainBased')
    except ConfigParser.NoSectionError as e:
        logger.error(e, exc_info=True)
        raise ValueError('The layer configuration file contains errors.')

    if terrainBased:
        # params = parseTerrainBasedLayer(layerConfig)
        # tileJSON = createTerrainBasedTileJSON(params)
        params = parseTerrainBasedLayer(layerConfig)
        tileJSON = createS3BasedTileJSON(params)
    else:
        dbConfig = ConfigParser.RawConfigParser()
        dbConfig.read('configs/raster/database.cfg')
        params = parseModelBasedLayer(dbConfig, layerConfig)
        (tileJSON, tilecount) = createModelBasedTileJSON(params)

    # Same bucket for now
    bucket = getBucket()
    fileObj = cStringIO.StringIO()
    fileObj.write(tileJSON)
    fileObj = gzipFileObject(fileObj)
    logger.info('Uploading %slayer.json to S3' % params.bucketBasePath)

    writeToS3(bucket,
              'layer.json',
              fileObj,
              'tilejson',
              params.bucketBasePath,
              contentType='application/json')
    logger.info('layer.json has been uploaded successfully')

    if tilecount:
        tend = time.time()
        logger.info('It took %s to scan %s tiles' %
                    (str(datetime.timedelta(seconds=tend - t0)), tilecount))
Example #2
0
def copyAGITiles(zooms, bounds, bucketBasePath):
    count = 0
    fullonly = 0
    headers = {'Accept': 'application/vnd.quantized-mesh;' +
        'extensions=octvertexnormals-' +
        'watermask,application/octet-stream;q=0.9,*/*;q=0.01'}
    baseURL = 'http://assets.agi.com/stk-terrain/world/'
    bucket = getBucket()
    for bxyz in grid(bounds, zooms, fullonly):
        f = cStringIO.StringIO()
        tilebounds, [x, y, z] = bxyz
        bucketKey = tilePathTemplate(x, y, z)
        f.write(loadTileContent(baseURL, bucketKey, headers))
        f.seek(0)
        compressedFile = gzipFileObject(f)
        writeToS3(bucket, bucketKey, compressedFile,
            'poc_watermask', bucketBasePath)
        count += 1
        if count % 20 == 0:
            print 'Copying %s...' % bucketKey
            print '%s tiles have been copied so far.' % count
Example #3
0
def copyAGITiles(zooms, bounds, bucketBasePath):
    count = 0
    headers = {
        'Accept': 'application/vnd.quantized-mesh;' +
        'extensions=octvertexnormals-' +
        'watermask,application/octet-stream;q=0.9,*/*;q=0.01'}
    baseURL = 'http://assets.agi.com/stk-terrain/world/'
    bucket = getBucket()
    for bxyz in grid(bounds, zooms[0], zooms[len(zooms) - 1]):
        f = cStringIO.StringIO()
        tilebounds, [x, y, z] = bxyz
        bucketKey = tilePathTemplate(x, y, z)
        f.write(loadTileContent(baseURL, bucketKey, headers))
        f.seek(0)
        compressedFile = gzipFileObject(f)
        writeToS3(
            bucket, bucketKey, compressedFile,
            'poc_watermask', bucketBasePath)
        count += 1
        if count % 20 == 0:
            print 'Copying %s...' % bucketKey
            print '%s tiles have been copied so far.' % count
Example #4
0
def main(template):
    t0 = time.time()
    tilecount = None
    layerConfig = ConfigParser.RawConfigParser()
    layerConfig.read(template)
    try:
        terrainBased = layerConfig.getboolean('Grid', 'terrainBased')
    except ConfigParser.NoSectionError as e:
        logger.error(e, exc_info=True)
        raise ValueError('The layer configuration file contains errors.')

    if terrainBased:
        # params = parseTerrainBasedLayer(layerConfig)
        # tileJSON = createTerrainBasedTileJSON(params)
        params = parseTerrainBasedLayer(layerConfig)
        tileJSON = createS3BasedTileJSON(params)
    else:
        dbConfig = ConfigParser.RawConfigParser()
        dbConfig.read('configs/raster/database.cfg')
        params = parseModelBasedLayer(dbConfig, layerConfig)
        (tileJSON, tilecount) = createModelBasedTileJSON(params)

    # Same bucket for now
    bucket = getBucket()
    fileObj = cStringIO.StringIO()
    fileObj.write(tileJSON)
    fileObj = gzipFileObject(fileObj)
    logger.info('Uploading %slayer.json to S3' % params.bucketBasePath)

    writeToS3(bucket, 'layer.json', fileObj, 'tilejson', params.bucketBasePath,
        contentType='application/json')
    logger.info('layer.json has been uploaded successfully')

    if tilecount:
        tend = time.time()
        logger.info('It took %s to scan %s tiles' % (
            str(datetime.timedelta(seconds=tend - t0)), tilecount))
def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'f:t:', ['from=', 'to='])
    except getopt.GetoptError as err:
        error(str(err), 2, usage=usage)

    ffrom = None
    to = None
    ffile = None

    for o, a in opts:
        if o in ('-f', '--from'):
            ffrom = a
        if o in ('-t', '--to'):
            to = a

    if ffrom is None or to is None:
        if len(args) != 1:
            error("Please specify a file.", 4, usage=usage)
        ffile = args[0]

    tiles = []

    # We have file, so we get the tiles from the file
    if ffile is not None:
        with open(ffile) as f:
            for line in f:
                line = line.rstrip()
                if len(line) > 2:
                    tiles.append(map(int, line.split('/')))

    # If we have from to, we catch layers.json from poc and use from to for levels
    if ffrom is not None and to is not None:
        req = urlopen(poc_base_url + 'layer.json')
        layers = {}
        for line in req:
            layers = json.loads(line)
        for zoom in range(int(ffrom), int(to) + 1):
            level = layers['available'][zoom][0]
            for x in range(int(level['startX']), int(level['endX']) + 1):
                for y in range(int(level['startY']), int(level['endY']) + 1):
                    tiles.append([zoom, x, y])

    bucket = getBucket()
    g = GlobalGeodetic(tmscompatible=True)
    if not os.path.isdir('tmp'):
        os.mkdir('tmp')
    for tile in tiles:
        req = None
        tilebounds = g.TileBounds(tile[1], tile[2], tile[0])
        tilestring = str(tile[0]) + "/" + str(tile[1]) + "/" + str(tile[2])
        try:
            url = poc_base_url + tilestring + ".terrain?v=2.0.0"
            req = urlopen(url)
            with open(gzip_file_name, 'wb') as fp:
                fp.write(req.read())
            ff = gzip.open(gzip_file_name)
            with open(temp_file_name, 'wb') as fp:
                fp.write(ff.read())

            ter = TerrainTile()
            ter.fromFile(
                temp_file_name, tilebounds[0],
                tilebounds[2], tilebounds[1], tilebounds[3]
            )
            '''
            if os.path.isfile(tms_file_name):
                os.remove(tms_file_name)

            if os.path.isfile(shape_file_name):
                os.remove(shape_file_name)

            if os.path.isfile(tms_from_shape_file_name):
                os.remove(tms_from_shape_file_name)

            ter.toFile(tms_file_name)
            ter.toShapefile(shape_file_name)

            shapefile = ShpToGDALFeatures(shpFilePath=shape_file_name)
            features = shapefile.__read__()
            topology = TerrainTopology(features=features)
            topology.fromGDALFeatures()

            terFromPoc = TerrainTile()
            terFromPoc.fromFile(
                tms_file_name, tilebounds[0], tilebounds[2], tilebounds[1], tilebounds[3]
            )

            terFromShape = TerrainTile()
            terFromShape.fromTerrainTopology(topology)
            terFromShape.toFile(tms_from_shape_file_name)

            # Use this to select what is written to s3
            ter2 = terFromShape
            '''
            ter2 = ter

            fileObject = ter2.toStringIO()
            compressedFile = gzipFileObject(fileObject)

            bucketKey = tilestring + '.terrain'
            print 'Uploading %s to S3' % bucketKey
            writeToS3(bucket, bucketKey, compressedFile, 'POC Tiles copy',
                contentType=ter.getContentType())

        except Exception as e:
            print "error with " + line + " " + str(e)
        finally:
            if req:
                req.close()
    return 0
Example #6
0
getShapefiles = lambda x: x.endswith('.shp')
shapefilesNames = filter(getShapefiles, os.listdir(basePath))
bucket = getBucket()
bucketBasePath = tmsConfig.get('General', 'bucketpath')

for f in shapefilesNames:
    filePathSource = basePath + f
    shapefile = ShpToGDALFeatures(shpFilePath=filePathSource)
    features = shapefile.__read__()

    terrainTopo = TerrainTopology(features=features)
    terrainTopo.fromGDALFeatures()
    terrainFormat = TerrainTile()
    terrainFormat.fromTerrainTopology(terrainTopo)

    fileObject = terrainFormat.toStringIO()
    compressedFile = gzipFileObject(fileObject)

    # Hardcoded scheme because we don't have any separators in the names
    # for now
    keyPath = f[1:3] + '/' + f[3:9] + '/' + f[9:14] + extension
    print 'Writing %s to S3' % keyPath
    writeToS3(bucket, keyPath, compressedFile, basePath, bucketBasePath,
        contentType=terrainFormat.getContentType())
    i += 1
    t1 = time.time()
    ti = t1 - t0
    print 'It took %s secs to write %s/%s tiles' % (
        str(datetime.timedelta(seconds=ti)), i, len(shapefilesNames)
    )
Example #7
0
# -*- coding: utf-8 -*-

import cStringIO
from forge.configs import tmsConfig
from forge.lib.helpers import gzipFileObject
from forge.lib.boto_conn import getBucket, writeToS3

bucket = getBucket()
layerJSONPath = 'forge/data/json-conf/layer.json'
bucketBasePath = tmsConfig.get('General', 'bucketpath')

with open(layerJSONPath) as f:
    fileObj = cStringIO.StringIO()
    fileObj.write(f.read())
    fileObj = gzipFileObject(fileObj)
    writeToS3(bucket, 'layer.json', fileObj, 'DB Scan',
        bucketBasePath, 'application/json')
Example #8
0
def createTile(tile):
    session = None
    pid = os.getpid()

    try:
        (bounds, tileXYZ, t0, dbConfigFile, bucketBasePath,
            hasLighting, hasWatermask) = tile

        db = DB(dbConfigFile)
        with db.userSession() as session:
            bucket = getBucket()

            # Get the model according to the zoom level
            model = modelsPyramid.getModelByZoom(tileXYZ[2])

            watermask = []
            if hasWatermask:
                lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
                query = session.query(
                    lakeModel.watermaskRasterize(bounds).label('watermask')
                )
                for q in query:
                    watermask = q.watermask

            # Get the interpolated point at the 4 corners
            # 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY)
            pts = [
                (bounds[0], bounds[1], 0),
                (bounds[0], bounds[3], 0),
                (bounds[2], bounds[3], 0),
                (bounds[2], bounds[1], 0)
            ]

            def toSubQuery(x):
                return session.query(
                    model.id, model.interpolateHeightOnPlane(pts[x])
                ).filter(
                    and_(
                        model.bboxIntersects(createBBox(pts[x], 0.01)),
                        model.pointIntersects(pts[x])
                    )
                ).subquery('p%s' % x)
            subqueries = [toSubQuery(i) for i in range(0, len(pts))]

            # Get the height of the corner points as postgis cannot properly
            # clip a polygon
            cornerPts = {}
            step = 2
            j = step
            query = session.query(*subqueries)
            for q in query:
                for i in range(0, len(q), step):
                    sub = q[i:j]
                    j += step
                    cornerPts[sub[0]] = list(
                        to_shape(WKBElement(sub[1])).coords
                    )

            # Clip using the bounds
            clippedGeometry = model.bboxClippedGeom(bounds)
            query = session.query(
                model.id,
                clippedGeometry.label('clip')
            ).filter(model.bboxIntersects(bounds))

            terrainTopo = TerrainTopology(hasLighting=hasLighting)
            for q in query:
                coords = list(to_shape(q.clip).exterior.coords)
                if q.id in cornerPts:
                    pt = cornerPts[q.id][0]
                    for i in range(0, len(coords)):
                        c = coords[i]
                        if c[0] == pt[0] and c[1] == pt[1]:
                            coords[i] = [c[0], c[1], pt[2]]

                try:
                    rings = processRingCoordinates(coords)
                except Exception as e:
                    msg = '[%s] --------- ERROR ------- occured while ' \
                        'collapsing non triangular shapes\n' % pid
                    msg += '[%s]: %s' % (pid, e)
                    logger.error(msg, exc_info=True)
                    raise Exception(e)
                # Redundant coord has been remove already
                for vertices in rings:
                    terrainTopo.addVertices(vertices)

            bucketKey = '%s/%s/%s.terrain' % (
                tileXYZ[2], tileXYZ[0], tileXYZ[1])
            verticesLength = len(terrainTopo.vertices)
            if verticesLength > 0:
                terrainTopo.create()
                # Prepare terrain tile
                terrainFormat = TerrainTile(watermask=watermask)
                terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds)

                # Bytes manipulation and compression
                fileObject = terrainFormat.toStringIO()
                compressedFile = gzipFileObject(fileObject)
                writeToS3(
                    bucket, bucketKey, compressedFile, model.__tablename__,
                    bucketBasePath, contentType=terrainFormat.getContentType()
                )
                tend = time.time()
                tilecount.value += 1
                val = tilecount.value
                total = val + skipcount.value
                if val % 10 == 0:
                    logger.info('[%s] Last tile %s (%s rings). '
                        '%s to write %s tiles. (total processed: %s)' % (
                            pid, bucketKey, verticesLength,
                            str(datetime.timedelta(seconds=tend - t0)),
                            val, total
                        )
                    )

            else:
                skipcount.value += 1
                val = skipcount.value
                total = val + tilecount.value
                # One should write an empyt tile
                logger.info('[%s] Skipping %s %s because no features found '
                    'for this tile (%s skipped from %s total)' % (
                        pid, bucketKey, bounds, val, total
                    )
                )
    except Exception as e:
        logger.error(e, exc_info=True)
        raise Exception(e)
    finally:
        db.userEngine.dispose()

    return 0
Example #9
0
shapefilesNames = filter(getShapefiles, os.listdir(basePath))
bucket = getBucket()
bucketBasePath = tmsConfig.get('General', 'bucketpath')

for f in shapefilesNames:
    filePathSource = basePath + f
    shapefile = ShpToGDALFeatures(shpFilePath=filePathSource)
    features = shapefile.__read__()

    terrainTopo = TerrainTopology(features=features)
    terrainTopo.fromGDALFeatures()
    terrainFormat = TerrainTile()
    terrainFormat.fromTerrainTopology(terrainTopo)

    fileObject = terrainFormat.toStringIO()
    compressedFile = gzipFileObject(fileObject)

    # Hardcoded scheme because we don't have any separators in the names
    # for now
    keyPath = f[1:3] + '/' + f[3:9] + '/' + f[9:14] + extension
    print 'Writing %s to S3' % keyPath
    writeToS3(bucket,
              keyPath,
              compressedFile,
              basePath,
              bucketBasePath,
              contentType=terrainFormat.getContentType())
    i += 1
    t1 = time.time()
    ti = t1 - t0
    print 'It took %s secs to write %s/%s tiles' % (str(
Example #10
0
# -*- coding: utf-8 -*-

import cStringIO
from forge.configs import tmsConfig
from forge.lib.helpers import gzipFileObject
from forge.lib.boto_conn import getBucket, writeToS3

bucket = getBucket()
layerJSONPath = 'forge/data/json-conf/layer.json'
bucketBasePath = tmsConfig.get('General', 'bucketpath')

with open(layerJSONPath) as f:
    fileObj = cStringIO.StringIO()
    fileObj.write(f.read())
    fileObj = gzipFileObject(fileObj)
    writeToS3(bucket, 'layer.json', fileObj, 'DB Scan', bucketBasePath,
              'application/json')
Example #11
0
def createTile(tile):
    session = None
    pid = os.getpid()

    try:
        (bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting,
         hasWatermask) = tile

        db = DB(dbConfigFile)
        with db.userSession() as session:
            bucket = getBucket()

            # Get the model according to the zoom level
            model = modelsPyramid.getModelByZoom(tileXYZ[2])

            watermask = []
            if hasWatermask:
                lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
                query = session.query(
                    lakeModel.watermaskRasterize(bounds).label('watermask'))
                for q in query:
                    watermask = q.watermask

            # Get the interpolated point at the 4 corners
            # 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY)
            pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0),
                   (bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)]

            def toSubQuery(x):
                return session.query(
                    model.id, model.interpolateHeightOnPlane(pts[x])).filter(
                        and_(model.bboxIntersects(createBBox(pts[x], 0.01)),
                             model.pointIntersects(pts[x]))).subquery('p%s' %
                                                                      x)

            subqueries = [toSubQuery(i) for i in range(0, len(pts))]

            # Get the height of the corner points as postgis cannot properly
            # clip a polygon
            cornerPts = {}
            step = 2
            j = step
            query = session.query(*subqueries)
            for q in query:
                for i in range(0, len(q), step):
                    sub = q[i:j]
                    j += step
                    cornerPts[sub[0]] = list(
                        to_shape(WKBElement(sub[1])).coords)

            # Clip using the bounds
            clippedGeometry = model.bboxClippedGeom(bounds)
            query = session.query(model.id,
                                  clippedGeometry.label('clip')).filter(
                                      model.bboxIntersects(bounds))

            terrainTopo = TerrainTopology(hasLighting=hasLighting)
            for q in query:
                coords = list(to_shape(q.clip).exterior.coords)
                if q.id in cornerPts:
                    pt = cornerPts[q.id][0]
                    for i in range(0, len(coords)):
                        c = coords[i]
                        if c[0] == pt[0] and c[1] == pt[1]:
                            coords[i] = [c[0], c[1], pt[2]]

                try:
                    rings = processRingCoordinates(coords)
                except Exception as e:
                    msg = '[%s] --------- ERROR ------- occured while ' \
                        'collapsing non triangular shapes\n' % pid
                    msg += '[%s]: %s' % (pid, e)
                    logger.error(msg, exc_info=True)
                    raise Exception(e)
                # Redundant coord has been remove already
                for vertices in rings:
                    terrainTopo.addVertices(vertices)

            bucketKey = '%s/%s/%s.terrain' % (tileXYZ[2], tileXYZ[0],
                                              tileXYZ[1])
            verticesLength = len(terrainTopo.vertices)
            if verticesLength > 0:
                terrainTopo.create()
                # Prepare terrain tile
                terrainFormat = TerrainTile(watermask=watermask)
                terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds)

                # Bytes manipulation and compression
                fileObject = terrainFormat.toStringIO()
                compressedFile = gzipFileObject(fileObject)
                writeToS3(bucket,
                          bucketKey,
                          compressedFile,
                          model.__tablename__,
                          bucketBasePath,
                          contentType=terrainFormat.getContentType())
                tend = time.time()
                tilecount.value += 1
                val = tilecount.value
                total = val + skipcount.value
                if val % 10 == 0:
                    logger.info('[%s] Last tile %s (%s rings). '
                                '%s to write %s tiles. (total processed: %s)' %
                                (pid, bucketKey, verticesLength,
                                 str(datetime.timedelta(seconds=tend - t0)),
                                 val, total))

            else:
                skipcount.value += 1
                val = skipcount.value
                total = val + tilecount.value
                # One should write an empyt tile
                logger.info('[%s] Skipping %s %s because no features found '
                            'for this tile (%s skipped from %s total)' %
                            (pid, bucketKey, bounds, val, total))
    except Exception as e:
        logger.error(e, exc_info=True)
        raise Exception(e)
    finally:
        db.userEngine.dispose()

    return 0