def main():
# Initialize the MPI pool
pool = MPIPool()
# Make sure only we run map() on the master process
if not pool.is_master():
pool.wait()
sys.exit(0)
# create some random input data
x = np.random.uniform(size=10000)
y = np.random.uniform(size=10000)
tasks = np.vstack((x,y)).T
vals = pool.map(worker, tasks)
pool.close()
def get_pool(mpi=False, threads=None, **kwargs):
"""
Get a pool object to pass to emcee for parallel processing.
If mpi is False and threads is None, pool is None.
Parameters
----------
mpi : bool
Use MPI or not. If specified, ignores the threads kwarg.
threads : int (optional)
If mpi is False and threads is specified, use a Python
multiprocessing pool with the specified number of threads.
**kwargs
Any other keyword arguments are passed through to the pool
initializers.
"""
if mpi:
from mpipool import MPIPool
# Initialize the MPI pool
pool = MPIPool(**kwargs)
# Make sure the thread we're running on is the master
if not pool.is_master():
pool.wait()
sys.exit(0)
logger.debug("Running with MPI...")
elif threads > 1:
logger.debug("Running with multiprocessing on {} cores..."
.format(threads))
pool = multiprocessing.Pool(threads, **kwargs)
else:
logger.debug("Running serial...")
pool = SerialPool(**kwargs)
return pool
def get_pool(mpi=False, threads=None, **kwargs):
""" Get a pool object to pass to emcee for parallel processing.
If mpi is False and threads is None, pool is None.
Parameters
----------
mpi : bool
Use MPI or not. If specified, ignores the threads kwarg.
threads : int (optional)
If mpi is False and threads is specified, use a Python
multiprocessing pool with the specified number of threads.
**kwargs
Any other keyword arguments are passed through to the pool
initializers.
"""
if mpi:
from mpipool import MPIPool
# Initialize the MPI pool
pool = MPIPool(**kwargs)
# Make sure the thread we're running on is the master
if not pool.is_master():
pool.wait()
sys.exit(0)
logger.debug("Running with MPI...")
elif threads is not None and threads > 1:
logger.debug(
"Running with multiprocessing on {} cores...".format(threads))
pool = multiprocessing.Pool(threads, **kwargs)
else:
logger.debug("Running serial...")
pool = SerialPool(**kwargs)
return pool
import numpy as np
from mpipool import MPIPool
# define the function that will be applied to tasks
def worker(task):
x, y = task
return x**2 + 2 * y
# create the pool
pool = MPIPool()
# only run map() on the master process, all other processes wait for their work
if not pool.is_master():
pool.wait()
# worker processes exit after they have done their work
sys.exit(0)
# the following code is executed by the master process only
# create some random input data
x = np.random.uniform(size=10)
y = np.random.uniform(size=10)
tasks = list(zip(x, y))
# crate a callback function
def cb(x):
print x
stats_dict[stat].append([output[stat], os.path.basename(name1),
os.path.basename(name2)])
print("Starting pool at {}".format(datetime.now()))
if multiprocess:
use_mpi = False
if use_mpi:
from mpipool import MPIPool
pool = MPIPool(loadbalance=False)
if not pool.is_master():
# Wait for instructions from the master process.
pool.wait()
sys.exit(0)
else:
from multiprocessing import cpu_count, Pool
psize = cpu_count()
print("Found {} CPUs to run on.".format(psize))
pool = Pool(processes=psize)
print("Pool created at {}".format(datetime.now()))
# Run distance between comparisons
stats_dict = dict.fromkeys(stats)
for stat in stats_dict:
if multiprocess:
stats_dict[stat] = manager.list([])
