Beispiel #1
0
    def createTiles(self):
        tilecount.value = 0
        skipcount.value = 0
        queueName = self.tmsConfig.get('General', 'sqsqueue')
        self.t0 = time.time()
        if len(queueName) <= 0:
            logger.error('Missing queueName')
            return
        procfactor = int(self.tmsConfig.get('General', 'procfactor'))

        pm = PoolManager(factor=procfactor)
        qtiles = QueueTerrainTiles(
            queueName,
            self.dbConfigFile,
            self.tmsConfig,
            self.t0,
            pm.nbOfProcesses
        )

        logger.info('Starting creation of tiles from queue %s ' % (queueName))
        pm.imap_unordered(createTileFromQueue, qtiles, 1)
        tend = time.time()
        logger.info(
            'It took %s to create %s tiles (%s were skipped) from queue' % (
                str(datetime.timedelta(seconds=tend - self.t0)),
                tilecount.value,
                skipcount.value
            )
        )
Beispiel #2
0
    def create(self):
        def callback(counter, result):
            if not counter % 10000:
                logger.info('counter: %s' % counter)
                logger.info('result: %s' % result)

        self.t0 = time.time()

        tilecount.value = 0
        skipcount.value = 0

        tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
        procfactor = int(self.tmsConfig.get('General', 'procfactor'))

        pm = PoolManager(factor=procfactor)
        maxChunks = int(self.tmsConfig.get('General', 'maxChunks'))

        nbTiles = self.numOfTiles()
        tilesPerProc = int(nbTiles / pm.nbOfProcesses)
        if tilesPerProc < maxChunks:
            maxChunks = tilesPerProc
        if maxChunks < 1:
            maxChunks = 1

        logger.info('Starting creation of %s tiles (%s per chunk)' %
                    (nbTiles, maxChunks))
        pm.imap_unordered(createTile, tiles, maxChunks, callback=callback)

        tend = time.time()
        logger.info('It took %s to create %s tiles (%s were skipped)' %
                    (str(datetime.timedelta(seconds=tend - self.t0)),
                     tilecount.value, skipcount.value))
def setup_func_with_callback():
    global pm_c
    global callback

    def callback(counter, res):
        print counter
        print res
    pm_c = PoolManager(numProcs=2, store=True)
    pm_c.imap_unordered(add, xrange(0, 100), 2, callback=callback)
Beispiel #4
0
def setup_func_with_callback():
    global pm_c
    global callback

    def callback(counter, res):
        print counter
        print res

    pm_c = PoolManager(numProcs=2, store=True)
    pm_c.imap_unordered(add, xrange(0, 100), 2, callback=callback)
Beispiel #5
0
    def populateTables(self):
        logger.info('Action: populateTables()')

        reproject = self.config.get('Reprojection', 'reproject')
        keepfiles = self.config.get('Reprojection', 'keepfiles')
        outDirectory = self.config.get('Reprojection', 'outDirectory')
        geosuiteCmd = self.config.get('Reprojection', 'geosuiteCmd')
        fromPFrames = self.config.get('Reprojection', 'fromPFrames')
        toPFrames = self.config.get('Reprojection', 'toPFrames')
        fromAFrames = self.config.get('Reprojection', 'fromAFrames')
        toAFrames = self.config.get('Reprojection', 'toAFrames')
        logfile = self.config.get('Reprojection', 'logfile')
        errorfile = self.config.get('Reprojection', 'errorfile')

        if not os.path.exists(outDirectory):
            raise OSError('%s does not exist' % outDirectory)
        if not os.path.exists(geosuiteCmd):
            raise OSError('%s does not exist' % geosuiteCmd)

        tstart = time.time()
        models = modelsPyramid.models
        featuresArgs = []
        for i in range(0, len(models)):
            model = models[i]
            for shp in model.__shapefiles__:

                featuresArgs.append(
                    PopulateFeaturesArguments(
                        engineURL=self.userEngine.url,
                        modelIndex=i,
                        shpFile=shp,
                        reproject=True if reproject == '1' else False,
                        keepfiles=True if keepfiles == '1' else False,
                        outDirectory=outDirectory,
                        geosuiteCmd=geosuiteCmd,
                        fromPFrames=fromPFrames,
                        toPFrames=toPFrames,
                        fromAFrames=fromAFrames,
                        toAFrames=toAFrames,
                        logfile=logfile,
                        errorfile=errorfile))

        cpuCount = multiprocessing.cpu_count()
        numFiles = len(featuresArgs)
        numProcs = cpuCount if numFiles >= cpuCount else numFiles
        pm = PoolManager(numProcs=numProcs, factor=1)
        pm.imap_unordered(populateFeatures, featuresArgs, 1)

        tend = time.time()
        logger.info('All tables have been created. It took %s' %
                    str(datetime.timedelta(seconds=tend - tstart)))
Beispiel #6
0
def copyKeys(fromPrefix, toPrefix, zooms):
    t0 = time.time()
    copycount.value = 0
    for zoom in zooms:
        log.info('doing zoom ' + str(zoom))
        t0zoom = time.time()
        keys = S3KeyIterator(fromPrefix + str(zoom) + '/',
                             toPrefix + str(zoom) + '/', t0)
        cpuCount = multiprocessing.cpu_count()
        pm = PoolManager(numProcs=cpuCount, factor=1)
        pm.imap_unordered(copyKey, keys, 50)

        log.info('It took %s to copy this zoomlevel (total %s)' %
                 (str(datetime.timedelta(seconds=time.time() - t0zoom)),
                  copycount.value))
    log.info(
        'It took %s to copy for all zoomlevels (total %s)' %
        (str(datetime.timedelta(seconds=time.time() - t0)), copycount.value))
Beispiel #7
0
def createS3BasedTileJSON(params):
    t0 = time.time()
    maxChunks = 50
    baseUrls = getBaseUrls(params)
    tiles = Tiles(
        params.bounds, params.minZoom, params.maxScanZoom,
        t0, basePath=params.bucketBasePath,
        tFormat=params.format, gridOrigin=params.gridOrigin,
        tilesURLs=params.tilesURLs
    )
    pm = PoolManager(factor=1, store=True)
    tMeta = LayerMetadata(
        bounds=params.bounds, minzoom=params.minZoom,
        maxzoom=params.maxZoom, baseUrls=baseUrls,
        description=params.description, attribution=params.attribution,
        format=params.format, name=params.name
    )
    pm.imap_unordered(tileNotExists, tiles, maxChunks)
    for xyz in pm.results:
        tMeta.removeTile(xyz[0], xyz[1], xyz[2])
    return tMeta.toJSON()
def main():
    if len(sys.argv) != 2:
        print 'Please provide a json configuration. Exit.'
        sys.exit(1)
    try:
        conf = parseJsonConf(sys.argv[1])
    except Exception as e:
        print 'Error while parsing json file'
        raise Exception(e)

    global layerBodId
    global lods
    global filters
    global gutter
    global gagrid

    layerBodId = conf.get('layerBodId')
    lods = conf.get('lods')
    filters = conf.get('filters')
    gutter = float(conf.get('gutter', 10))

    extent = conf.get('extent')
    tileSizePx = conf.get('tileSizePx', 256.0)
    gagrid = GeoadminTileGrid(extent=extent, tileSizePx=float(tileSizePx))

    minZoom = conf.get('minZoom', 0)
    maxZoom = conf.get('maxZoom', 0)
    tileGrid = gagrid.iterGrid(minZoom, maxZoom)

    pm = PoolManager(factor=2)
    pm.imap_unordered(tileGrid, createTile, 50, callback=callback)

    # End of process
    t3 = time.time()
    tf = t3 - t0
    print 'Tile generation process ended/stopped.'
    print 'It took %s' % str(datetime.timedelta(seconds=tf))
Beispiel #9
0
def setup_func_error_within_subprocess():
    global pm_error_sub
    pm_error_sub = PoolManager(numProcs=2, store=True)
    pm_error_sub.imap_unordered(error_func, xrange(0, 100), 2)
Beispiel #10
0
def setup_func_error():
    global pm_error
    pm_error = PoolManager(numProcs=2, factor=2, store=True)
    pm_error.imap_unordered(add, 'notvalid', 2)
Beispiel #11
0
def setup_func_store():
    global pm_store
    pm_store = PoolManager(numProcs=2, store=True)
    pm_store.imap_unordered(add, xrange(0, 100), 2)
Beispiel #12
0
def setup_func_no_store():
    global pm
    pm = PoolManager(numProcs=2)
    pm.imap_unordered(add, xrange(0, 100), 2)
def setup_func_error_within_subprocess():
    global pm_error_sub
    pm_error_sub = PoolManager(numProcs=2, store=True)
    pm_error_sub.imap_unordered(error_func, xrange(0, 100), 2)
def setup_func_error():
    global pm_error
    pm_error = PoolManager(numProcs=2, factor=2, store=True)
    pm_error.imap_unordered(add, 'notvalid', 2)
def setup_func_store():
    global pm_store
    pm_store = PoolManager(numProcs=2, store=True)
    pm_store.imap_unordered(add, xrange(0, 100), 2)
def setup_func_no_store():
    global pm
    pm = PoolManager(numProcs=2)
    pm.imap_unordered(add, xrange(0, 100), 2)