def get_pool(mpi=False, threads=1):
if mpi: # using MPI
from mpipool import MPIPool
pool = MPIPool()
pool.start()
if not pool.is_master():
sys.exit(0)
elif threads > 1: # using multiprocessing
from multiprocessing import Pool
pool = Pool(processes=threads)
else:
raise RuntimeError, "Wrong arguments: either mpi=True or threads>1."
return pool
def get_pool(mpi=False,threads=1):
if mpi: # using MPI
from mpipool import MPIPool
pool = MPIPool()
pool.start()
if not pool.is_master():
sys.exit(0)
elif threads>1: # using multiprocessing
from multiprocessing import Pool
pool = Pool(processes=threads)
else:
raise RuntimeError,"Wrong arguments: either mpi=True or threads>1."
return pool
def main():
# Initialize the MPI pool
pool = MPIPool()
# Make sure only we run map() on the master process
if not pool.is_master():
pool.wait()
sys.exit(0)
# create some random input data
x = np.random.uniform(size=10000)
y = np.random.uniform(size=10000)
tasks = np.vstack((x,y)).T
vals = pool.map(worker, tasks)
pool.close()
def get_pool(mpi=False, threads=1):
"""
Create a thread pool for paralleling DEM simulations within GrainLearning
:param mpi: bool, default=False
:param threads: int, default=1
"""
if mpi: # using MPI
from mpipool import MPIPool
pool = MPIPool()
pool.start()
if not pool.is_master():
sys.exit(0)
elif threads > 1: # using multiprocessing
from multiprocessing import Pool
pool = Pool(processes=threads, maxtasksperchild=10)
else:
raise RuntimeError("Wrong arguments: either mpi=True or threads>1.")
return pool
def get_pool(mpi=False, threads=None, **kwargs):
"""
Get a pool object to pass to emcee for parallel processing.
If mpi is False and threads is None, pool is None.
Parameters
----------
mpi : bool
Use MPI or not. If specified, ignores the threads kwarg.
threads : int (optional)
If mpi is False and threads is specified, use a Python
multiprocessing pool with the specified number of threads.
**kwargs
Any other keyword arguments are passed through to the pool
initializers.
"""
if mpi:
from mpipool import MPIPool
# Initialize the MPI pool
pool = MPIPool(**kwargs)
# Make sure the thread we're running on is the master
if not pool.is_master():
pool.wait()
sys.exit(0)
logger.debug("Running with MPI...")
elif threads > 1:
logger.debug("Running with multiprocessing on {} cores..."
.format(threads))
pool = multiprocessing.Pool(threads, **kwargs)
else:
logger.debug("Running serial...")
pool = SerialPool(**kwargs)
return pool
def get_pool(mpi=False, threads=None, **kwargs):
""" Get a pool object to pass to emcee for parallel processing.
If mpi is False and threads is None, pool is None.
Parameters
----------
mpi : bool
Use MPI or not. If specified, ignores the threads kwarg.
threads : int (optional)
If mpi is False and threads is specified, use a Python
multiprocessing pool with the specified number of threads.
**kwargs
Any other keyword arguments are passed through to the pool
initializers.
"""
if mpi:
from mpipool import MPIPool
# Initialize the MPI pool
pool = MPIPool(**kwargs)
# Make sure the thread we're running on is the master
if not pool.is_master():
pool.wait()
sys.exit(0)
logger.debug("Running with MPI...")
elif threads is not None and threads > 1:
logger.debug(
"Running with multiprocessing on {} cores...".format(threads))
pool = multiprocessing.Pool(threads, **kwargs)
else:
logger.debug("Running serial...")
pool = SerialPool(**kwargs)
return pool
import sys
import numpy as np
from mpipool import MPIPool
# define the function that will be applied to tasks
def worker(task):
x, y = task
return x**2 + 2 * y
# create the pool
pool = MPIPool()
# only run map() on the master process, all other processes wait for their work
if not pool.is_master():
pool.wait()
# worker processes exit after they have done their work
sys.exit(0)
# the following code is executed by the master process only
# create some random input data
x = np.random.uniform(size=10)
y = np.random.uniform(size=10)
tasks = list(zip(x, y))
# crate a callback function
def cb(x):
print x
for stat in output:
stats_dict[stat].append([output[stat], os.path.basename(name1),
os.path.basename(name2)])
print("Starting pool at {}".format(datetime.now()))
if multiprocess:
use_mpi = False
if use_mpi:
from mpipool import MPIPool
pool = MPIPool(loadbalance=False)
if not pool.is_master():
# Wait for instructions from the master process.
pool.wait()
sys.exit(0)
else:
from multiprocessing import cpu_count, Pool
psize = cpu_count()
print("Found {} CPUs to run on.".format(psize))
pool = Pool(processes=psize)
print("Pool created at {}".format(datetime.now()))
# Run distance between comparisons
stats_dict = dict.fromkeys(stats)
for stat in stats_dict:
