예제 #1
0
    def _stats(self, withDb=True):
        self.t0 = time.time()
        total = 0

        msg = '\n'
        tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
        geodetic = getTileGrid(4326)(tmsCompatible=True)
        bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
        zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)

        db = DB('configs/terrain/database.cfg')
        try:
            with db.userSession() as session:
                for i in xrange(0, len(zooms)):
                    zoom = zooms[i]
                    model = modelsPyramid.getModelByZoom(zoom)
                    nbObjects = None
                    if withDb:
                        nbObjects = session.query(model).filter(
                            model.bboxIntersects(bounds)).count()
                    # top letf corner
                    tileMinX, tileMinY = geodetic.tileAddress(
                        zoom, [bounds[0], bounds[3]])
                    # bottom right
                    tileMaxX, tileMaxY = geodetic.tileAddress(
                        zoom, [bounds[2], bounds[1]])
                    tileBounds = geodetic.tileBounds(zoom, tileMinX, tileMinY)
                    xCount = tileMaxX - tileMinX + 1
                    yCount = tileMaxY - tileMinY + 1
                    nbTiles = xCount * yCount
                    total += nbTiles
                    pointA = transformCoordinate(
                        'POINT(%s %s)' % (tileBounds[0], tileBounds[1]), 4326,
                        21781).GetPoints()[0]
                    pointB = transformCoordinate(
                        'POINT(%s %s)' % (tileBounds[2], tileBounds[3]), 4326,
                        21781).GetPoints()[0]
                    length = int(round(c2d.distance(pointA, pointB)))
                    msg += 'At zoom %s:\n' % zoom
                    msg += 'We expect %s tiles overall\n' % nbTiles
                    msg += 'Min X is %s, Max X is %s\n' % (tileMinX, tileMaxX)
                    msg += '%s columns over X\n' % xCount
                    msg += 'Min Y is %s, Max Y is %s\n' % (tileMinY, tileMaxY)
                    msg += '%s rows over Y\n' % yCount
                    msg += '\n'
                    msg += 'A tile side is around %s meters' % length
                    if nbTiles > 0 and nbObjects is not None:
                        msg += 'We have an average of about %s triangles ' \
                               'per tile\n' % int(round(nbObjects / nbTiles))
                    msg += '\n\n'
            msg += '%s tiles in total.' % total
        except Exception as e:
            logger.error('An error occured during statistics collection')
            logger.error('%s' % e, exc_info=True)
            raise Exception(e)
        finally:
            db.userEngine.dispose()

        return (total, msg)
예제 #2
0
    def metadata(self):
        t0 = time.time()
        basePath = self.tmsConfig.get('General', 'bucketpath')
        baseUrls = []
        url = 'https://terrain.dev.bgdi.ch'
        url += '/%s{z}/{x}/{y}.terrain?v={version}' % basePath
        baseUrls.append(url)
        # TODO
        #for i in range(0, 5):
        #    url = 'https://terrain10%s.geo.admin.ch' % i
        #    url = 'https://terrain10%s.geo.admin.ch' % i
        #    url += '/%s{z}/{x}/{y}.terrain?v={version}' % basePath
        #    baseUrls.append(url)

        db = DB('configs/terrain/database.cfg')
        tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, t0)
        tMeta = TerrainMetadata(
            bounds=tiles.bounds,
            minzoom=tiles.tileMinZ,
            maxzoom=tiles.tileMaxZ,
            useGlobalTiles=True,
            hasLighting=tiles.hasLighting,
            hasWatermask=tiles.hasWatermask,
            baseUrls=baseUrls)

        try:
            with db.userSession() as session:
                tilecount = 1
                for tile in tiles:
                    tMeta = scanTerrain(tMeta, tile, session, tilecount)
                    tilecount += 1

                tend = time.time()
                logger.info('It took %s to scan %s tiles' % (
                    str(datetime.timedelta(seconds=tend - t0)), tilecount))
        except Exception as e:
            logger.error('An error occured during layer.json creation')
            logger.error('%s' % e, exc_info=True)
            raise Exception(e)
        finally:
            db.userEngine.dispose()

        with open('.tmp/layer.json', 'w') as f:
            f.write(tMeta.toJSON())
예제 #3
0
    def metadata(self):
        t0 = time.time()
        serverAddress = self.tmsConfig.get('General', 'serveraddress')
        basePath = self.tmsConfig.get('General', 'bucketpath')
        baseUrls = [
            serverAddress + basePath + "{z}/{x}/{y}.terrain?v={version}"
        ]

        db = DB('configs/terrain/database.cfg')
        tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, t0)
        tMeta = TerrainMetadata(bounds=tiles.bounds,
                                minzoom=tiles.tileMinZ,
                                maxzoom=tiles.tileMaxZ,
                                useGlobalTiles=True,
                                hasLighting=tiles.hasLighting,
                                hasWatermask=tiles.hasWatermask,
                                baseUrls=baseUrls)

        try:
            with db.userSession() as session:
                tilecount = 1
                for tile in tiles:
                    tMeta = scanTerrain(tMeta, tile, session, tilecount)
                    tilecount += 1

                tend = time.time()
                logger.info(
                    'It took %s to scan %s tiles' %
                    (str(datetime.timedelta(seconds=tend - t0)), tilecount))
        except Exception as e:
            logger.error('An error occured during layer.json creation')
            logger.error('%s' % e, exc_info=True)
            raise Exception(e)
        finally:
            db.userEngine.dispose()

        with open('.tmp/layer.json', 'w') as f:
            f.write(tMeta.toJSON())
예제 #4
0
파일: tiler.py 프로젝트: escribano/3d-forge
    def metadata(self):
        t0 = time.time()
        basePath = self.tmsConfig.get('General', 'bucketpath')
        baseUrls = [
            "//terrain0.geo.admin.ch/" + basePath + "{z}/{x}/{y}.terrain?v={version}",
            "//terrain1.geo.admin.ch/" + basePath + "{z}/{x}/{y}.terrain?v={version}",
            "//terrain2.geo.admin.ch/" + basePath + "{z}/{x}/{y}.terrain?v={version}",
            "//terrain3.geo.admin.ch/" + basePath + "{z}/{x}/{y}.terrain?v={version}",
            "//terrain4.geo.admin.ch/" + basePath + "{z}/{x}/{y}.terrain?v={version}"
        ]

        db = DB('configs/terrain/database.cfg')
        tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, t0)
        tMeta = TerrainMetadata(
            bounds=tiles.bounds, minzoom=tiles.tileMinZ, maxzoom=tiles.tileMaxZ,
            useGlobalTiles=True, hasLighting=tiles.hasLighting,
            hasWatermask=tiles.hasWatermask, baseUrls=baseUrls)

        try:
            with db.userSession() as session:
                tilecount = 1
                for tile in tiles:
                    tMeta = scanTerrain(tMeta, tile, session, tilecount)
                    tilecount += 1

                tend = time.time()
                logger.info('It took %s to scan %s tiles' % (
                    str(datetime.timedelta(seconds=tend - t0)), tilecount))
        except Exception as e:
            logger.error('An error occured during layer.json creation')
            logger.error('%s' % e, exc_info=True)
            raise Exception(e)
        finally:
            db.userEngine.dispose()

        with open('.tmp/layer.json', 'w') as f:
            f.write(tMeta.toJSON())
예제 #5
0
def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'c:', ['config='])
    except getopt.GetoptError as err:
        error(str(err), 2, usage=usage)

    dbConfigFile = 'configs/terrain/database.cfg'
    for o, a in opts:
        if o in ('-c', '--config'):
            dbConfigFile = a

    if len(args) < 1:
        error('you must specify a command', 3, usage=usage)

    db = DB(dbConfigFile)

    command = args[0]
    if command == 'console':
        db.console()
    elif command == 'create':
        db.create()
    elif command == 'createuser':
        db.createUser()
    elif command == 'createdb':
        db.createDB()
    elif command == 'setupfunctions':
        db.setupFunctions()
    elif command == 'populate':
        db.populate()
    elif command == 'populatelakes':
        db.populateLakes()
    elif command == 'dropuser':
        db.dropUser()
    elif command == 'dropdb':
        db.dropDatabase()
    elif command == 'destroy':
        db.destroy()
    else:
        error("unknown command '%(command)s'" % {'command': command}, 4, usage=usage)
예제 #6
0
def createTile(tile):
    session = None
    pid = os.getpid()

    try:
        (bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting,
         hasWatermask) = tile

        db = DB(dbConfigFile)
        bucketKey = '%s/%s/%s.terrain' % (tileXYZ[2], tileXYZ[0], tileXYZ[1])
        with db.userSession() as session:
            bucket = getBucket()

            # Get the model according to the zoom level
            model = modelsPyramid.getModelByZoom(tileXYZ[2])

            watermask = []
            if hasWatermask:
                lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
                query = session.query(
                    lakeModel.watermaskRasterize(bounds).label('watermask'))
                for q in query:
                    watermask = q.watermask

            # Get the interpolated point at the 4 corners
            # 0: (minX, minY), 1: (minX, maxY),
            # 2: (maxX, maxY), 3: (maxX, minY)
            pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0),
                   (bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)]

            def toSubQuery(x):
                return session.query(
                    model.id, model.interpolateHeightOnPlane(pts[x])).filter(
                        and_(model.bboxIntersects(createBBox(pts[x], 0.01)),
                             model.pointIntersects(pts[x]))).subquery('p%s' %
                                                                      x)

            subqueries = [toSubQuery(i) for i in range(0, len(pts))]

            # Get the height of the corner points as postgis cannot properly
            # clip a polygon
            cornerPts = {}
            step = 2
            j = step
            query = session.query(*subqueries)
            for q in query:
                for i in range(0, len(q), step):
                    sub = q[i:j]
                    j += step
                    cornerPts[sub[0]] = list(
                        to_shape(WKBElement(sub[1])).coords)

            # Clip using the bounds
            clippedGeometry = model.bboxClippedGeom(bounds)
            query = session.query(model.id,
                                  clippedGeometry.label('clip')).filter(
                                      model.bboxIntersects(bounds))

            geomCoords = []
            for q in query:
                coords = list(to_shape(q.clip).exterior.coords)
                if q.id in cornerPts:
                    pt = cornerPts[q.id][0]
                    for i in range(0, len(coords)):
                        c = coords[i]
                        if c[0] == pt[0] and c[1] == pt[1]:
                            coords[i] = [c[0], c[1], pt[2]]
                geomCoords.append(coords)

            nbGeoms = len(geomCoords)
            if nbGeoms > 0:
                try:
                    terrainTile = encode(geomCoords,
                                         bounds=bounds,
                                         autocorrectGeometries=True,
                                         hasLighting=hasLighting,
                                         watermask=watermask)
                except Exception as e:
                    msg = '[%s] --------- ERROR ------- occured while ' % pid
                    msg += 'encoding terrain tile\n'
                    msg += '[%s]: %s' % (pid, e)
                    logger.error(msg, exc_info=True)
                    raise Exception(e)

                writeToS3(bucket,
                          bucketKey,
                          terrainTile.toBytesIO(gzipped=True),
                          model.__tablename__,
                          bucketBasePath,
                          contentType=terrainTile.getContentType())
                tend = time.time()
                tilecount.value += 1
                tilesCreated = tilecount.value
                total = tilesCreated + skipcount.value
                if tilesCreated % 1000 == 0:
                    logger.info('[%s] Last tile %s (%s rings). '
                                '%s to write %s tiles. (total processed: %s)' %
                                (pid, bucketKey, nbGeoms,
                                 str(datetime.timedelta(seconds=tend - t0)),
                                 tilesCreated, total))

            else:
                skipcount.value += 1
                val = skipcount.value
                total = val + tilecount.value
                # One should write an empyt tile
                logger.info('[%s] Skipping %s %s because no features found '
                            'for this tile (%s skipped from %s total)' %
                            (pid, bucketKey, bounds, val, total))
    except Exception as e:
        logger.error(e, exc_info=True)
        raise Exception(e)
    finally:
        db.userEngine.dispose()

    return 0
예제 #7
0
파일: tiler.py 프로젝트: escribano/3d-forge
def createTile(tile):
    session = None
    pid = os.getpid()

    try:
        (bounds, tileXYZ, t0, dbConfigFile, bucketBasePath,
            hasLighting, hasWatermask) = tile

        db = DB(dbConfigFile)
        with db.userSession() as session:
            bucket = getBucket()

            # Get the model according to the zoom level
            model = modelsPyramid.getModelByZoom(tileXYZ[2])

            watermask = []
            if hasWatermask:
                lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
                query = session.query(
                    lakeModel.watermaskRasterize(bounds).label('watermask')
                )
                for q in query:
                    watermask = q.watermask

            # Get the interpolated point at the 4 corners
            # 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY)
            pts = [
                (bounds[0], bounds[1], 0),
                (bounds[0], bounds[3], 0),
                (bounds[2], bounds[3], 0),
                (bounds[2], bounds[1], 0)
            ]

            def toSubQuery(x):
                return session.query(
                    model.id, model.interpolateHeightOnPlane(pts[x])
                ).filter(
                    and_(
                        model.bboxIntersects(createBBox(pts[x], 0.01)),
                        model.pointIntersects(pts[x])
                    )
                ).subquery('p%s' % x)
            subqueries = [toSubQuery(i) for i in range(0, len(pts))]

            # Get the height of the corner points as postgis cannot properly
            # clip a polygon
            cornerPts = {}
            step = 2
            j = step
            query = session.query(*subqueries)
            for q in query:
                for i in range(0, len(q), step):
                    sub = q[i:j]
                    j += step
                    cornerPts[sub[0]] = list(
                        to_shape(WKBElement(sub[1])).coords
                    )

            # Clip using the bounds
            clippedGeometry = model.bboxClippedGeom(bounds)
            query = session.query(
                model.id,
                clippedGeometry.label('clip')
            ).filter(model.bboxIntersects(bounds))

            terrainTopo = TerrainTopology(hasLighting=hasLighting)
            for q in query:
                coords = list(to_shape(q.clip).exterior.coords)
                if q.id in cornerPts:
                    pt = cornerPts[q.id][0]
                    for i in range(0, len(coords)):
                        c = coords[i]
                        if c[0] == pt[0] and c[1] == pt[1]:
                            coords[i] = [c[0], c[1], pt[2]]

                try:
                    rings = processRingCoordinates(coords)
                except Exception as e:
                    msg = '[%s] --------- ERROR ------- occured while ' \
                        'collapsing non triangular shapes\n' % pid
                    msg += '[%s]: %s' % (pid, e)
                    logger.error(msg, exc_info=True)
                    raise Exception(e)
                # Redundant coord has been remove already
                for vertices in rings:
                    terrainTopo.addVertices(vertices)

            bucketKey = '%s/%s/%s.terrain' % (
                tileXYZ[2], tileXYZ[0], tileXYZ[1])
            verticesLength = len(terrainTopo.vertices)
            if verticesLength > 0:
                terrainTopo.create()
                # Prepare terrain tile
                terrainFormat = TerrainTile(watermask=watermask)
                terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds)

                # Bytes manipulation and compression
                fileObject = terrainFormat.toStringIO()
                compressedFile = gzipFileObject(fileObject)
                writeToS3(
                    bucket, bucketKey, compressedFile, model.__tablename__,
                    bucketBasePath, contentType=terrainFormat.getContentType()
                )
                tend = time.time()
                tilecount.value += 1
                val = tilecount.value
                total = val + skipcount.value
                if val % 10 == 0:
                    logger.info('[%s] Last tile %s (%s rings). '
                        '%s to write %s tiles. (total processed: %s)' % (
                            pid, bucketKey, verticesLength,
                            str(datetime.timedelta(seconds=tend - t0)),
                            val, total
                        )
                    )

            else:
                skipcount.value += 1
                val = skipcount.value
                total = val + tilecount.value
                # One should write an empyt tile
                logger.info('[%s] Skipping %s %s because no features found '
                    'for this tile (%s skipped from %s total)' % (
                        pid, bucketKey, bounds, val, total
                    )
                )
    except Exception as e:
        logger.error(e, exc_info=True)
        raise Exception(e)
    finally:
        db.userEngine.dispose()

    return 0
예제 #8
0
파일: tiler.py 프로젝트: escribano/3d-forge
    def _stats(self, withDb=True):
        self.t0 = time.time()
        total = 0

        msg = '\n'
        tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
        geodetic = GlobalGeodetic(True)
        bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
        zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)

        db = DB('configs/terrain/database.cfg')
        try:
            with db.userSession() as session:
                for i in xrange(0, len(zooms)):
                    zoom = zooms[i]
                    model = modelsPyramid.getModelByZoom(zoom)
                    nbObjects = None
                    if withDb:
                        nbObjects = session.query(model).filter(
                            model.bboxIntersects(bounds)
                        ).count()
                    tileMinX, tileMinY = geodetic.LonLatToTile(
                        bounds[0], bounds[1], zoom
                    )
                    tileMaxX, tileMaxY = geodetic.LonLatToTile(
                        bounds[2], bounds[3], zoom
                    )
                    # Fast approach, but might not be fully correct
                    if tiles.fullonly == 1:
                        tileMinX += 1
                        tileMinY += 1
                        tileMaxX -= 1
                        tileMaxY -= 1
                    tileBounds = geodetic.TileBounds(tileMinX, tileMinY, zoom)
                    xCount = tileMaxX - tileMinX + 1
                    yCount = tileMaxY - tileMinY + 1
                    nbTiles = xCount * yCount
                    total += nbTiles
                    pointA = transformCoordinate('POINT(%s %s)' % (
                        tileBounds[0], tileBounds[1]), 4326, 21781
                    ).GetPoints()[0]
                    pointB = transformCoordinate('POINT(%s %s)' % (
                        tileBounds[2], tileBounds[3]), 4326, 21781
                    ).GetPoints()[0]
                    length = c2d.distance(pointA, pointB)
                    if tiles.fullonly == 1:
                        msg += 'WARNING: stats are approximative because ' \
                               'fullonly is activated!\n'
                    msg += 'At zoom %s:\n' % zoom
                    msg += 'We expect %s tiles overall\n' % nbTiles
                    msg += 'Min X is %s, Max X is %s\n' % (tileMinX, tileMaxX)
                    msg += '%s columns over X\n' % xCount
                    msg += 'Min Y is %s, Max Y is %s\n' % (tileMinY, tileMaxY)
                    msg += '%s rows over Y\n' % yCount
                    msg += '\n'
                    msg += 'A tile side is around %s meters' % int(round(length))
                    if nbTiles > 0 and nbObjects is not None:
                        msg += 'We have an average of about %s triangles ' \
                               'per tile\n' % int(round(nbObjects / nbTiles))
                    msg += '\n\n'
            msg += '%s tiles in total.' % total
        except Exception as e:
            logger.error('An error occured during statistics collection')
            logger.error('%s' % e, exc_info=True)
            raise Exception(e)
        finally:
            db.userEngine.dispose()

        return (total, msg)
예제 #9
0
def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'c:', ['config='])
    except getopt.GetoptError as err:
        error(str(err), 2, usage=usage)

    dbConfigFile = 'configs/terrain/database.cfg'
    for o, a in opts:
        if o in ('-c', '--config'):
            dbConfigFile = a

    if len(args) < 1:
        error('you must specify a command', 3, usage=usage)

    db = DB(dbConfigFile)

    command = args[0]
    if command == 'console':
        db.console()
    elif command == 'create':
        db.create()
    elif command == 'createuser':
        db.createUser()
    elif command == 'createdb':
        db.createDB()
    elif command == 'setupfunctions':
        db.setupFunctions()
    elif command == 'populate':
        db.populate()
    elif command == 'populatelakes':
        db.populateLakes()
    elif command == 'dropuser':
        db.dropUser()
    elif command == 'dropdb':
        db.dropDatabase()
    elif command == 'destroy':
        db.destroy()
    else:
        error("unknown command '%(command)s'" % {'command': command},
              4,
              usage=usage)
예제 #10
0
def createTile(tile):
    session = None
    pid = os.getpid()

    try:
        (bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting,
         hasWatermask) = tile

        db = DB(dbConfigFile)
        with db.userSession() as session:
            # bucket = getBucket()

            # Get the model according to the zoom level
            model = modelsPyramid.getModelByZoom(tileXYZ[2])

            watermask = []
            if hasWatermask:
                lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
                query = session.query(
                    lakeModel.watermaskRasterize(bounds).label('watermask'))
                for q in query:
                    watermask = q.watermask

            # Get the interpolated point at the 4 corners
            # 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY)
            pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0),
                   (bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)]

            def toSubQuery(x):
                return session.query(
                    model.id, model.interpolateHeightOnPlane(pts[x])).filter(
                        and_(model.bboxIntersects(createBBox(pts[x], 0.01)),
                             model.pointIntersects(pts[x]))).subquery('p%s' %
                                                                      x)

            subqueries = [toSubQuery(i) for i in range(0, len(pts))]

            # Get the height of the corner points as postgis cannot properly
            # clip a polygon
            cornerPts = {}
            step = 2
            j = step
            query = session.query(*subqueries)
            for q in query:
                for i in range(0, len(q), step):
                    sub = q[i:j]
                    j += step
                    cornerPts[sub[0]] = list(
                        to_shape(WKBElement(sub[1])).coords)

            # Clip using the bounds
            clippedGeometry = model.bboxClippedGeom(bounds)
            query = session.query(model.id,
                                  clippedGeometry.label('clip')).filter(
                                      model.bboxIntersects(bounds))

            terrainTopo = TerrainTopology(hasLighting=hasLighting)
            for q in query:
                if to_shape(q.clip).geom_type == "GeometryCollection":
                    continue
                if to_shape(q.clip).geom_type == "LineString" or to_shape(
                        q.clip).geom_type == "Point":
                    continue
                coords = list(to_shape(q.clip).exterior.coords)
                if q.id in cornerPts:
                    pt = cornerPts[q.id][0]
                    for i in range(0, len(coords)):
                        c = coords[i]
                        if c[0] == pt[0] and c[1] == pt[1]:
                            coords[i] = [c[0], c[1], pt[2]]

                try:
                    rings = processRingCoordinates(coords)
                except Exception as e:
                    msg = '[%s] --------- ERROR ------- occured while ' \
                        'collapsing non triangular shapes\n' % pid
                    msg += '[%s]: %s' % (pid, e)
                    logger.error(msg, exc_info=True)
                    raise Exception(e)
                # Redundant coord has been remove already
                for vertices in rings:
                    terrainTopo.addVertices(vertices)

            baseDir = '/Users/liberostelios/3d_forge_geodata/output'
            terrainDir = '%s/%s/%s' % (baseDir, tileXYZ[2], tileXYZ[0])
            bucketKey = '%s/%s.terrain' % (terrainDir, tileXYZ[1])
            verticesLength = len(terrainTopo.vertices)
            if verticesLength > 0:
                terrainTopo.create()
                # Prepare terrain tile
                terrainFormat = TerrainTile(watermask=watermask)
                terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds)

                # STELIOS: We want this in a file, instead

                if not os.path.isdir(terrainDir):
                    os.makedirs(terrainDir)
                terrainFormat.toFile(bucketKey)

                logger.info('[%s] Just did %s %s' % (pid, bucketKey, bounds))

                # Bytes manipulation and compression
                # fileObject = terrainFormat.toStringIO()
                # compressedFile = gzipFileObject(fileObject)
                # writeToS3(
                #     bucket, bucketKey, compressedFile, model.__tablename__,
                #     bucketBasePath, contentType=terrainFormat.getContentType()
                # )
                tend = time.time()
                tilecount.value += 1
                val = tilecount.value
                total = val + skipcount.value
                if val % 10 == 0:
                    logger.info('[%s] Last tile %s (%s rings). '
                                '%s to write %s tiles. (total processed: %s)' %
                                (pid, bucketKey, verticesLength,
                                 str(datetime.timedelta(seconds=tend - t0)),
                                 val, total))

            else:
                skipcount.value += 1
                val = skipcount.value
                total = val + tilecount.value
                # One should write an empyt tile
                logger.info('[%s] Skipping %s %s because no features found '
                            'for this tile (%s skipped from %s total)' %
                            (pid, bucketKey, bounds, val, total))
    except Exception as e:
        logger.error(e, exc_info=True)
        raise Exception(e)
    finally:
        db.userEngine.dispose()

    return 0
예제 #11
0
    def _stats(self, withDb=True):
        self.t0 = time.time()
        total = 0

        msg = '\n'
        tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
        geodetic = GlobalGeodetic(True)
        bounds = (tiles.minLon, tiles.minLat, tiles.maxLon, tiles.maxLat)
        zooms = range(tiles.tileMinZ, tiles.tileMaxZ + 1)

        db = DB('configs/terrain/database.cfg')
        try:
            with db.userSession() as session:
                for i in xrange(0, len(zooms)):
                    zoom = zooms[i]
                    model = modelsPyramid.getModelByZoom(zoom)
                    nbObjects = None
                    if withDb:
                        nbObjects = session.query(model).filter(
                            model.bboxIntersects(bounds)).count()
                    tileMinX, tileMinY = geodetic.LonLatToTile(
                        bounds[0], bounds[1], zoom)
                    tileMaxX, tileMaxY = geodetic.LonLatToTile(
                        bounds[2], bounds[3], zoom)
                    # Fast approach, but might not be fully correct
                    if tiles.fullonly == 1:
                        tileMinX += 1
                        tileMinY += 1
                        tileMaxX -= 1
                        tileMaxY -= 1
                    tileBounds = geodetic.TileBounds(tileMinX, tileMinY, zoom)
                    xCount = tileMaxX - tileMinX + 1
                    yCount = tileMaxY - tileMinY + 1
                    nbTiles = xCount * yCount
                    total += nbTiles
                    pointA = transformCoordinate(
                        'POINT(%s %s)' % (tileBounds[0], tileBounds[1]), 4326,
                        21781).GetPoints()[0]
                    pointB = transformCoordinate(
                        'POINT(%s %s)' % (tileBounds[2], tileBounds[3]), 4326,
                        21781).GetPoints()[0]
                    length = c2d.distance(pointA, pointB)
                    if tiles.fullonly == 1:
                        msg += 'WARNING: stats are approximative because ' \
                               'fullonly is activated!\n'
                    msg += 'At zoom %s:\n' % zoom
                    msg += 'We expect %s tiles overall\n' % nbTiles
                    msg += 'Min X is %s, Max X is %s\n' % (tileMinX, tileMaxX)
                    msg += '%s columns over X\n' % xCount
                    msg += 'Min Y is %s, Max Y is %s\n' % (tileMinY, tileMaxY)
                    msg += '%s rows over Y\n' % yCount
                    msg += '\n'
                    msg += 'A tile side is around %s meters' % int(
                        round(length))
                    if nbTiles > 0 and nbObjects is not None:
                        msg += 'We have an average of about %s triangles ' \
                               'per tile\n' % int(round(nbObjects / nbTiles))
                    msg += '\n\n'
            msg += '%s tiles in total.' % total
        except Exception as e:
            logger.error('An error occured during statistics collection')
            logger.error('%s' % e, exc_info=True)
            raise Exception(e)
        finally:
            db.userEngine.dispose()

        return (total, msg)