def parallel_map(
func,
iterable,
processes=1,
nslots=1,
method=None,
asynchronous=True,
callback=None,
preserve_order=True,
job_category="low",
):
"""
A wrapper function to call either drmaa or easy_mp to do a parallel map
calculation. This function is setup so that in each case we can select
the number of cores on a machine
"""
from dials.util.cluster_map import cluster_map as drmaa_parallel_map
warnings.warn(
"The dials.util.parallel_map function is deprecated",
UserWarning,
stacklevel=2,
)
if method == "drmaa":
return drmaa_parallel_map(
func=func,
iterable=iterable,
callback=callback,
nslots=nslots,
njobs=processes,
job_category=job_category,
)
else:
qsub_command = "qsub -pe smp %d" % nslots
return libtbx.easy_mp.parallel_map(
func=func,
iterable=iterable,
callback=callback,
method=method,
processes=processes,
qsub_command=qsub_command,
asynchronous=asynchronous,
preserve_order=preserve_order,
preserve_exception_message=True,
)
def parallel_map(func,
iterable,
processes=1,
nslots=1,
method=None,
asynchronous=True,
callback=None,
preserve_order=True,
preserve_exception_message=True,
job_category="low"):
'''
A wrapper function to call either drmaa or easy_mp to do a parallel map
calculation. This function is setup so that in each case we can select
the number of cores on a machine
'''
from dials.util.cluster_map import cluster_map as drmaa_parallel_map
from libtbx.easy_mp import parallel_map as easy_mp_parallel_map
if method == "drmaa":
return drmaa_parallel_map(func=func,
iterable=iterable,
callback=callback,
nslots=nslots,
njobs=processes,
job_category=job_category)
else:
qsub_command = 'qsub -pe smp %d' % nslots
return easy_mp_parallel_map(
func=func,
iterable=iterable,
callback=callback,
method=method,
processes=processes,
qsub_command=qsub_command,
asynchronous=asynchronous,
preserve_order=preserve_order,
preserve_exception_message=preserve_exception_message)
def multi_node_parallel_map(
func,
iterable,
njobs=1,
nproc=1,
cluster_method=None,
asynchronous=True,
callback=None,
preserve_order=True,
):
"""
A wrapper function to call a function using multiple cluster nodes and with
multiple processors on each node
"""
# The function to all on the cluster
cluster_func = __cluster_function_wrapper(
func=func,
nproc=nproc,
asynchronous=asynchronous,
preserve_order=preserve_order,
)
# Create the cluster iterable
cluster_iterable = _iterable_grouper(iterable, nproc)
# Create the cluster callback
if callback is not None:
cluster_callback = _iterable_wrapper(callback)
else:
cluster_callback = None
# Do the parallel map on the cluster
# Call either drmaa or easy_mp to do a parallel map calculation.
# This function is set up so that in each case we can select
# the number of cores on a machine
if cluster_method == "drmaa":
from dials.util.cluster_map import cluster_map as drmaa_parallel_map
result = drmaa_parallel_map(
func=cluster_func,
iterable=cluster_iterable,
callback=cluster_callback,
nslots=nproc,
njobs=njobs,
job_category="low",
)
else:
result = libtbx.easy_mp.parallel_map(
func=cluster_func,
iterable=cluster_iterable,
callback=cluster_callback,
method=cluster_method,
processes=njobs,
qsub_command=f"qsub -pe smp {nproc}",
asynchronous=asynchronous,
preserve_order=preserve_order,
preserve_exception_message=True,
)
# return result
return [item for rlist in result for item in rlist]