def process(self):
"""
Do all the processing tasks.
:return: The processing results
"""
start_time = time()
self.manager.initialize()
mp_method = self.manager.params.mp.method
mp_njobs = self.manager.params.mp.njobs
mp_nproc = self.manager.params.mp.nproc
if (mp_njobs * mp_nproc) > 1 and platform.system(
) == "Windows": # platform.system() forks which is bad for MPI, so don't use it unless nproc > 1
logger.warning(
"Multiprocessing is not available on windows. Setting nproc = 1\n"
)
mp_nproc = 1
mp_njobs = 1
assert mp_nproc > 0, "Invalid number of processors"
if mp_nproc * mp_njobs > len(self.manager):
mp_nproc = min(mp_nproc, len(self.manager))
mp_njobs = int(math.ceil(len(self.manager) / mp_nproc))
logger.info(self.manager.summary())
if mp_njobs > 1:
assert mp_method != "none" and mp_method is not None
logger.info(
" Using %s with %d parallel job(s) and %d processes per node\n",
mp_method,
mp_njobs,
mp_nproc,
)
else:
logger.info(" Using multiprocessing with %d parallel job(s)\n", )
if mp_njobs * mp_nproc > 1:
def process_output(result):
for message in result[1]:
logger.handle(message)
self.manager.accumulate(result[0])
multi_node_parallel_map(
func=execute_parallel_task,
iterable=list(self.manager.tasks()),
njobs=mp_njobs,
nproc=mp_nproc,
callback=process_output,
cluster_method=mp_method,
preserve_order=True,
preserve_exception_message=True,
)
else:
for task in self.manager.tasks():
self.manager.accumulate(task())
self.manager.finalize()
end_time = time()
self.manager.time.user_time = end_time - start_time
result1, result2 = self.manager.result()
return result1, result2, self.manager.time
def process(self):
"""
Do all the processing tasks.
:return: The processing results
"""
start_time = time()
self.manager.initialize()
mp_method = self.manager.params.integration.mp.method
mp_nproc = min(len(self.manager),
self.manager.params.integration.mp.nproc)
mp_njobs = self.manager.params.integration.mp.njobs
if (
mp_nproc > 1 and platform.system() == "Windows"
): # platform.system() forks which is bad for MPI, so don't use it unless nproc > 1
logger.warning(
"Multiprocessing is not available on windows. Setting nproc = 1\n"
)
mp_nproc = 1
assert mp_nproc > 0, "Invalid number of processors"
logger.info(self.manager.summary())
logger.info(" Using %s with %d parallel job(s)\n" %
(mp_method, mp_nproc))
if mp_nproc > 1:
def process_output(result):
for message in result[1]:
logger.log(message.levelno, message.msg)
self.manager.accumulate(result[0])
result[0].reflections = None
result[0].data = None
def execute_task(task):
log.config_simple_cached()
result = task()
handlers = logging.getLogger("dials").handlers
assert len(handlers) == 1, "Invalid number of logging handlers"
return result, handlers[0].messages()
multi_node_parallel_map(
func=execute_task,
iterable=list(self.manager.tasks()),
njobs=mp_njobs,
nproc=mp_nproc,
callback=process_output,
cluster_method=mp_method,
preserve_order=True,
preserve_exception_message=True,
)
else:
for task in self.manager.tasks():
self.manager.accumulate(task())
self.manager.finalize()
end_time = time()
self.manager.time.user_time = end_time - start_time
result = self.manager.result()
return result, self.manager.time
def process(self):
"""
Do all the processing tasks.
:return: The processing results
"""
start_time = time()
self.manager.initialize()
mp_method = self.manager.params.mp.method
mp_njobs = self.manager.params.mp.njobs
mp_nproc = self.manager.params.mp.nproc
assert mp_nproc > 0, "Invalid number of processors"
if mp_nproc * mp_njobs > len(self.manager):
mp_nproc = min(mp_nproc, len(self.manager))
mp_njobs = int(math.ceil(len(self.manager) / mp_nproc))
logger.info(self.manager.summary())
if mp_njobs > 1:
assert mp_method != "none" and mp_method is not None
logger.info(
" Using %s with %d parallel job(s) and %d processes per node\n",
mp_method,
mp_njobs,
mp_nproc,
)
else:
logger.info(" Using multiprocessing with %d parallel job(s)\n",
mp_nproc)
if mp_njobs * mp_nproc > 1:
def process_output(result):
rehandle_cached_records(result[1])
self.manager.accumulate(result[0])
multi_node_parallel_map(
func=execute_parallel_task,
iterable=list(self.manager.tasks()),
njobs=mp_njobs,
nproc=mp_nproc,
callback=process_output,
cluster_method=mp_method,
preserve_order=True,
)
else:
for task in self.manager.tasks():
self.manager.accumulate(task())
self.manager.finalize()
end_time = time()
self.manager.time.user_time = end_time - start_time
result1, result2 = self.manager.result()
return result1, result2, self.manager.time
def __init__(self, experiments, reflections, reference=None, params=None):
# Create the reference manager
integration_manager = IntegrationManager(
experiments, reflections, reference, params
)
# Print some output
logger.info(integration_manager.summary())
# Execute each task
if params.integration.mp.njobs > 1:
if params.integration.mp.method == "multiprocessing":
_assert_enough_memory(
params.integration.mp.njobs
* compute_required_memory(
experiments[0].imageset, params.integration.block.size
),
params.integration.block.max_memory_usage,
)
def process_output(result):
for message in result[1]:
logger.log(message.levelno, message.msg)
integration_manager.accumulate(result[0])
result[0].reflections = None
result[0].data = None
multi_node_parallel_map(
func=execute_parallel_task,
iterable=integration_manager.tasks(),
nproc=params.integration.mp.nproc,
njobs=params.integration.mp.njobs,
callback=process_output,
cluster_method=params.integration.mp.method,
preserve_order=True,
preserve_exception_message=True,
)
else:
for task in integration_manager.tasks():
result = task()
integration_manager.accumulate(result)
# Finalize the processing
integration_manager.finalize()
# Set the reflections and profiles
self._reflections = integration_manager.result()
def test_cached_log_records(caplog):
# Generate some cached log messages in easy_mp child processes.
results = multi_node_parallel_map(
log_something,
iterable=range(5),
njobs=2,
nproc=2,
cluster_method="multiprocessing",
)
# Get all the resulting log records in a single flattened list.
results = [record for records in results for record in records]
results_batch = batch_multi_node_parallel_map(
log_something,
range(5),
njobs=2,
nproc=2,
cluster_method="multiprocessing",
callback=lambda _: None,
)
# Get all the resulting log records in a single flattened list.
results_batch = [
record for batch in results_batch for records in batch
for record in records
]
# Check that re-handling the messages in a logger in this process with a
# threshold severity of WARNING results in no log records being emitted.
with dials.util.log.LoggingContext("dials", logging.WARNING):
dials.util.log.rehandle_cached_records(results)
assert not caplog.records
dials.util.log.rehandle_cached_records(results_batch)
assert not caplog.records
# Check that re-handling the messages in a logger in this process with a
# threshold severity of INFO results in all the log records being emitted.
with dials.util.log.LoggingContext("dials", logging.INFO):
dials.util.log.rehandle_cached_records(results)
assert caplog.records == results
caplog.clear()
dials.util.log.rehandle_cached_records(results_batch)
assert caplog.records == results_batch
caplog.clear()
def __init__(self, experiments, reflections, params=None):
from dials.util import pprint
# Create the reference manager
reference_manager = ReferenceCalculatorManager(experiments,
reflections, params)
# Print some output
logger.info(reference_manager.summary())
# Execute each task
if params.integration.mp.njobs > 1:
if params.integration.mp.method == "multiprocessing":
assert_enough_memory(
params.integration.mp.njobs *
compute_required_memory(experiments[0].imageset,
params.integration.block.size),
params.integration.block.max_memory_usage,
)
def process_output(result):
for message in result[1]:
logger.log(message.levelno, message.msg)
reference_manager.accumulate(result[0])
result[0].reflections = None
result[0].data = None
multi_node_parallel_map(
func=ExecuteParallelTask(),
iterable=reference_manager.tasks(),
nproc=params.integration.mp.nproc,
njobs=params.integration.mp.njobs,
callback=process_output,
cluster_method=params.integration.mp.method,
preserve_order=True,
preserve_exception_message=True,
)
else:
for task in reference_manager.tasks():
result = task()
reference_manager.accumulate(result)
# Finalize the processing
reference_manager.finalize()
# Set the reflections and profiles
self._reflections = reference_manager.result()
self._profiles = reference_manager.reference
# Write the profiles to file
if params.integration.debug.reference.output:
with open(params.integration.debug.reference.filename,
"wb") as outfile:
import six.moves.cPickle as pickle
pickle.dump(self._profiles, outfile)
# Print the profiles to the debug log
for i in range(len(self._profiles)):
logger.debug("")
logger.debug("Reference Profiles for experiment %d" % i)
logger.debug("")
reference = self._profiles[i].reference()
for j in range(len(reference)):
data = reference.data(j)
logger.debug("Profile %d" % j)
if len(data) > 0:
logger.debug(pprint.profile3d(data))
else:
logger.debug("** NO PROFILE **")
def process(self):
'''
Do all the processing tasks.
:return: The processing results
'''
from time import time
from dials.util.mp import multi_node_parallel_map
import platform
from math import ceil
start_time = time()
self.manager.initialize()
mp_method = self.manager.params.mp.method
mp_njobs = self.manager.params.mp.njobs
mp_nproc = self.manager.params.mp.nproc
if (mp_njobs * mp_nproc) > 1 and platform.system(
) == "Windows": # platform.system() forks which is bad for MPI, so don't use it unless nproc > 1
logger.warn("")
logger.warn("*" * 80)
logger.warn(
"Multiprocessing is not available on windows. Setting nproc = 1"
)
logger.warn("*" * 80)
logger.warn("")
mp_nproc = 1
mp_njobs = 1
assert mp_nproc > 0, "Invalid number of processors"
if mp_nproc * mp_njobs > len(self.manager):
mp_nproc = min(mp_nproc, len(self.manager))
mp_njobs = int(ceil(len(self.manager) / mp_nproc))
logger.info(self.manager.summary())
if mp_njobs > 1:
assert mp_method is not 'none' and mp_method is not None
logger.info(
' Using %s with %d parallel job(s) and %d processes per node\n'
% (mp_method, mp_njobs, mp_nproc))
else:
logger.info(' Using multiprocessing with %d parallel job(s)\n' %
(mp_nproc))
if mp_njobs * mp_nproc > 1:
def process_output(result):
for message in result[1]:
logger.log(message.levelno, message.msg)
self.manager.accumulate(result[0])
result[0].reflections = None
result[0].data = None
multi_node_parallel_map(func=ExecuteParallelTask(),
iterable=list(self.manager.tasks()),
njobs=mp_njobs,
nproc=mp_nproc,
callback=process_output,
cluster_method=mp_method,
preserve_order=True,
preserve_exception_message=True)
else:
for task in self.manager.tasks():
self.manager.accumulate(task())
self.manager.finalize()
end_time = time()
self.manager.time.user_time = end_time - start_time
result1, result2 = self.manager.result()
return result1, result2, self.manager.time