def copyKeys(fromPrefix, toPrefix, zooms):
t0 = time.time()
copycount.value = 0
for zoom in zooms:
log.info('doing zoom ' + str(zoom))
t0zoom = time.time()
keys = S3KeyIterator(fromPrefix + str(zoom) + '/', toPrefix + str(zoom) + '/', t0)
pm = PoolManager(log)
pm.process(keys, copyKey, 50)
log.info(
'It took %s to copy this zoomlevel (total %s)' %
(str(
datetime.timedelta(
seconds=time.time() -
t0zoom)),
copycount.value))
log.info(
'It took %s to copy for all zoomlevels (total %s)' %
(str(
datetime.timedelta(
seconds=time.time() -
t0)),
copycount.value))
def createS3BasedTileJSON(params):
t0 = time.time()
maxChunks = 50
baseUrls = getBaseUrls(params)
tiles = Tiles(params.bounds,
params.minZoom,
params.maxScanZoom,
t0,
fullonly=params.fullonly,
basePath=params.bucketBasePath,
tFormat=params.format,
gridOrigin=params.gridOrigin,
tilesURLs=params.tilesURLs)
pm = PoolManager(logger=logger, factor=1, store=True)
tMeta = LayerMetadata(bounds=params.bounds,
minzoom=params.minZoom,
maxzoom=params.maxZoom,
baseUrls=baseUrls,
description=params.description,
attribution=params.attribution,
format=params.format,
name=params.name)
pm.process(tiles, tileNotExists, maxChunks)
for xyz in pm.results:
tMeta.removeTile(xyz[0], xyz[1], xyz[2])
return tMeta.toJSON()
def create(self):
self.t0 = time.time()
tilecount.value = 0
skipcount.value = 0
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
procfactor = int(self.tmsConfig.get('General', 'procfactor'))
pm = PoolManager(logger=logger, factor=procfactor)
maxChunks = int(self.tmsConfig.get('General', 'maxChunks'))
nbTiles = self.numOfTiles()
tilesPerProc = int(nbTiles / pm.numOfProcesses())
if tilesPerProc < maxChunks:
maxChunks = tilesPerProc
if maxChunks < 1:
maxChunks = 1
logger.info('Starting creation of %s tiles (%s per chunk)' % (
nbTiles, maxChunks))
pm.process(tiles, createTile, maxChunks)
tend = time.time()
logger.info('It took %s to create %s tiles (%s were skipped)' % (
str(datetime.timedelta(seconds=tend - self.t0)), tilecount.value,
skipcount.value
))
def create(self):
self.t0 = time.time()
tilecount.value = 0
skipcount.value = 0
tiles = TerrainTiles(self.dbConfigFile, self.tmsConfig, self.t0)
procfactor = int(self.tmsConfig.get('General', 'procfactor'))
pm = PoolManager(logger=logger, factor=procfactor)
maxChunks = int(self.tmsConfig.get('General', 'maxChunks'))
nbTiles = self.numOfTiles()
tilesPerProc = int(nbTiles / pm.numOfProcesses())
if tilesPerProc < maxChunks:
maxChunks = tilesPerProc
if maxChunks < 1:
maxChunks = 1
logger.info('Starting creation of %s tiles (%s per chunk)' %
(nbTiles, maxChunks))
pm.process(tiles, createTile, maxChunks)
tend = time.time()
logger.info('It took %s to create %s tiles (%s were skipped)' %
(str(datetime.timedelta(seconds=tend - self.t0)),
tilecount.value, skipcount.value))
def populateTables(self):
logger.info('Action: populateTables()')
reproject = self.config.get('Reprojection', 'reproject')
keepfiles = self.config.get('Reprojection', 'keepfiles')
outDirectory = self.config.get('Reprojection', 'outDirectory')
geosuiteCmd = self.config.get('Reprojection', 'geosuiteCmd')
fromPFrames = self.config.get('Reprojection', 'fromPFrames')
toPFrames = self.config.get('Reprojection', 'toPFrames')
fromAFrames = self.config.get('Reprojection', 'fromAFrames')
toAFrames = self.config.get('Reprojection', 'toAFrames')
logfile = self.config.get('Reprojection', 'logfile')
errorfile = self.config.get('Reprojection', 'errorfile')
if not os.path.exists(outDirectory):
raise OSError('%s does not exist' % outDirectory)
# if not os.path.exists(geosuiteCmd):
# raise OSError('%s does not exist' % geosuiteCmd)
tstart = time.time()
models = modelsPyramid.models
featuresArgs = []
for i in range(0, len(models)):
model = models[i]
for shp in model.__shapefiles__:
featuresArgs.append(
PopulateFeaturesArguments(
engineURL=self.userEngine.url,
modelIndex=i,
shpFile=shp,
reproject=True if reproject == '1' else False,
keepfiles=True if keepfiles == '1' else False,
outDirectory=outDirectory,
geosuiteCmd=geosuiteCmd,
fromPFrames=fromPFrames,
toPFrames=toPFrames,
fromAFrames=fromAFrames,
toAFrames=toAFrames,
logfile=logfile,
errorfile=errorfile))
if not len(featuresArgs):
raise OSError('The are no files')
cpuCount = multiprocessing.cpu_count()
numFiles = len(featuresArgs)
numProcs = cpuCount if numFiles >= cpuCount else numFiles
pm = PoolManager(logger=logger, numProcs=numProcs, factor=1)
pm.process(featuresArgs, populateFeatures, 1)
tend = time.time()
logger.info('All tables have been created. It took %s' %
str(datetime.timedelta(seconds=tend - tstart)))
def populateTables(self):
logger.info('Action: populateTables()')
reproject = self.config.get('Reprojection', 'reproject')
keepfiles = self.config.get('Reprojection', 'keepfiles')
outDirectory = self.config.get('Reprojection', 'outDirectory')
geosuiteCmd = self.config.get('Reprojection', 'geosuiteCmd')
fromPFrames = self.config.get('Reprojection', 'fromPFrames')
toPFrames = self.config.get('Reprojection', 'toPFrames')
fromAFrames = self.config.get('Reprojection', 'fromAFrames')
toAFrames = self.config.get('Reprojection', 'toAFrames')
logfile = self.config.get('Reprojection', 'logfile')
errorfile = self.config.get('Reprojection', 'errorfile')
if not os.path.exists(outDirectory):
raise OSError('%s does not exist' % outDirectory)
if not os.path.exists(geosuiteCmd):
raise OSError('%s does not exist' % geosuiteCmd)
tstart = time.time()
models = modelsPyramid.models
featuresArgs = []
for i in range(0, len(models)):
model = models[i]
for shp in model.__shapefiles__:
featuresArgs.append(PopulateFeaturesArguments(
engineURL = self.userEngine.url,
modelIndex = i,
shpFile = shp,
reproject = True if reproject == '1' else False,
keepfiles = True if keepfiles == '1' else False,
outDirectory = outDirectory,
geosuiteCmd = geosuiteCmd,
fromPFrames = fromPFrames,
toPFrames = toPFrames,
fromAFrames = fromAFrames,
toAFrames = toAFrames,
logfile = logfile,
errorfile = errorfile
))
cpuCount = multiprocessing.cpu_count()
numFiles = len(featuresArgs)
numProcs = cpuCount if numFiles >= cpuCount else numFiles
pm = PoolManager(logger=logger, numProcs=numProcs, factor=1)
pm.process(featuresArgs, populateFeatures, 1)
tend = time.time()
logger.info('All tables have been created. It took %s' % str(
datetime.timedelta(seconds=tend - tstart)))
def createTiles(self):
tilecount.value = 0
skipcount.value = 0
queueName = self.tmsConfig.get("General", "sqsqueue")
self.t0 = time.time()
if len(queueName) <= 0:
logger.error("Missing queueName")
return
procfactor = int(self.tmsConfig.get("General", "procfactor"))
pm = PoolManager(logger=logger, factor=procfactor)
qtiles = QueueTiles(queueName, self.dbConfigFile, self.t0, pm.numOfProcesses())
logger.info("Starting creation of tiles from queue %s " % (queueName))
pm.process(qtiles, createTileFromQueue, 1)
tend = time.time()
logger.info(
"It took %s to create %s tiles (%s were skipped) from queue"
% (str(datetime.timedelta(seconds=tend - self.t0)), tilecount.value, skipcount.value)
)
def createS3BasedTileJSON(params):
t0 = time.time()
maxChunks = 50
baseUrls = getBaseUrls(params)
tiles = Tiles(
params.bounds, params.minZoom, params.maxScanZoom,
t0, fullonly=params.fullonly, basePath=params.bucketBasePath,
tFormat=params.format, gridOrigin=params.gridOrigin,
tilesURLs=params.tilesURLs
)
pm = PoolManager(logger=logger, factor=1, store=True)
tMeta = LayerMetadata(
bounds=params.bounds, minzoom=params.minZoom,
maxzoom=params.maxZoom, baseUrls=baseUrls,
description=params.description, attribution=params.attribution,
format=params.format, name=params.name
)
pm.process(tiles, tileNotExists, maxChunks)
for xyz in pm.results:
tMeta.removeTile(xyz[0], xyz[1], xyz[2])
return tMeta.toJSON()
def createTiles(self):
tilecount.value = 0
skipcount.value = 0
queueName = self.tmsConfig.get('General', 'sqsqueue')
self.t0 = time.time()
if len(queueName) <= 0:
logger.error('Missing queueName')
return
procfactor = int(self.tmsConfig.get('General', 'procfactor'))
pm = PoolManager(logger=logger, factor=procfactor)
qtiles = QueueTerrainTiles(queueName,
self.dbConfigFile, self.tmsConfig, self.t0,
pm.numOfProcesses())
logger.info('Starting creation of tiles from queue %s ' % (queueName))
pm.process(qtiles, createTileFromQueue, 1)
tend = time.time()
logger.info(
'It took %s to create %s tiles (%s were skipped) from queue' %
(str(datetime.timedelta(seconds=tend - self.t0)), tilecount.value,
skipcount.value))
