def ConfigureRunningEnv(num_process, size_tile_cache):
"""Configures the running environment.
Optimal performance is obtained by increasing the number of process
and a geo cache big enough to avoid swapping. Each tile covers about
100kmx100km. Swapping statistics can be obtaines using the routine:
`CheckTerrainTileCacheOk()`.
Args:
num_process: Number of process. Special values: -1: #cpu/2 ; -2: #cpu-1
size_tile_cache: Geo cache size in number of tiles. The memory usage is
about: `num_process * size_tile_cache * 60 MB`
"""
# Configure the global pool of processes
mpool.Configure(num_process)
# Configure the geo drivers to avoid swap
# - for main process
drive.ConfigureTerrainDriver(cache_size=size_tile_cache)
drive.ConfigureNlcdDriver(cache_size=size_tile_cache)
# - for worker processes
mpool.RunOnEachWorkerProcess(drive.ConfigureTerrainDriver,
terrain_dir=None,
cache_size=size_tile_cache)
mpool.RunOnEachWorkerProcess(drive.ConfigureNlcdDriver,
nlcd_dir=None,
cache_size=size_tile_cache)
return mpool.GetNumWorkerProcesses()
def CheckTerrainTileCacheOk():
"""Check tiles cache well behaved."""
print('Check of tile cache swap:')
num_workers = mpool.GetNumWorkerProcesses()
if num_workers:
tile_stats = mpool.RunOnEachWorkerProcess(_getTileStats)
else:
tile_stats = [_getTileStats()]
num_active_tiles, cnt_per_tile = tile_stats[np.argmax(
[tile_stat[0] for tile_stat in tile_stats])]
if not num_active_tiles:
print('-- Cache ERROR: No active tiles read')
elif max(cnt_per_tile) > 1:
print(
'-- Cache WARNING: cache tile too small - tiles are swapping from cache.'
)
if num_workers:
pool = mpool.Pool()
pool.apply_async(_printTileStats)
else:
_printTileStats()
else:
print('-- Cache tile: OK (no swapping)')
# Process the commad line arguments.
options = parser.parse_args()
logging.getLogger().setLevel(_LOGGER_MAP[options.log_level.lower()])
# Configure the multiprocessing worker pool.
# Your options are:
# 0: single process (default if not called)
# -1: use half of the cpus
# -2: use all cpus (minus one)
# a specific number of cpus
# Or your own `pool`.
logging.info('Start Worker processes')
mpool.Configure(num_processes=NUM_PROCESSES)
num_workers = mpool.GetNumWorkerProcesses()
logging.info(' ... %d workers started' % num_workers)
# Configure geo drivers
logging.info('Configure geo drivers')
(num_tiles_master_ned, num_tiles_worker_ned, num_tiles_master_nlcd,
num_tiles_worker_nlcd) = GetGeoCacheSize(num_workers)
if num_tiles_master_ned < 16:
logging.warning('Required geo cache size %d (for master) is low'
'- too few memory or too many workers' %
num_tiles_master_ned)
logging.info(' ... NED: cache size: %d per master, %d for workers' %
(num_tiles_master_ned, num_tiles_worker_ned))
logging.info(' ... NLCD: cache size: %d per master, %d for workers' %
(num_tiles_master_ned, num_tiles_worker_ned))