def process(self): """ Do all the processing tasks. :return: The processing results """ start_time = time() self.manager.initialize() mp_method = self.manager.params.mp.method mp_njobs = self.manager.params.mp.njobs mp_nproc = self.manager.params.mp.nproc if (mp_njobs * mp_nproc) > 1 and platform.system( ) == "Windows": # platform.system() forks which is bad for MPI, so don't use it unless nproc > 1 logger.warning( "Multiprocessing is not available on windows. Setting nproc = 1\n" ) mp_nproc = 1 mp_njobs = 1 assert mp_nproc > 0, "Invalid number of processors" if mp_nproc * mp_njobs > len(self.manager): mp_nproc = min(mp_nproc, len(self.manager)) mp_njobs = int(math.ceil(len(self.manager) / mp_nproc)) logger.info(self.manager.summary()) if mp_njobs > 1: assert mp_method != "none" and mp_method is not None logger.info( " Using %s with %d parallel job(s) and %d processes per node\n", mp_method, mp_njobs, mp_nproc, ) else: logger.info(" Using multiprocessing with %d parallel job(s)\n", ) if mp_njobs * mp_nproc > 1: def process_output(result): for message in result[1]: logger.handle(message) self.manager.accumulate(result[0]) multi_node_parallel_map( func=execute_parallel_task, iterable=list(self.manager.tasks()), njobs=mp_njobs, nproc=mp_nproc, callback=process_output, cluster_method=mp_method, preserve_order=True, preserve_exception_message=True, ) else: for task in self.manager.tasks(): self.manager.accumulate(task()) self.manager.finalize() end_time = time() self.manager.time.user_time = end_time - start_time result1, result2 = self.manager.result() return result1, result2, self.manager.time
def process(self): """ Do all the processing tasks. :return: The processing results """ start_time = time() self.manager.initialize() mp_method = self.manager.params.integration.mp.method mp_nproc = min(len(self.manager), self.manager.params.integration.mp.nproc) mp_njobs = self.manager.params.integration.mp.njobs if ( mp_nproc > 1 and platform.system() == "Windows" ): # platform.system() forks which is bad for MPI, so don't use it unless nproc > 1 logger.warning( "Multiprocessing is not available on windows. Setting nproc = 1\n" ) mp_nproc = 1 assert mp_nproc > 0, "Invalid number of processors" logger.info(self.manager.summary()) logger.info(" Using %s with %d parallel job(s)\n" % (mp_method, mp_nproc)) if mp_nproc > 1: def process_output(result): for message in result[1]: logger.log(message.levelno, message.msg) self.manager.accumulate(result[0]) result[0].reflections = None result[0].data = None def execute_task(task): log.config_simple_cached() result = task() handlers = logging.getLogger("dials").handlers assert len(handlers) == 1, "Invalid number of logging handlers" return result, handlers[0].messages() multi_node_parallel_map( func=execute_task, iterable=list(self.manager.tasks()), njobs=mp_njobs, nproc=mp_nproc, callback=process_output, cluster_method=mp_method, preserve_order=True, preserve_exception_message=True, ) else: for task in self.manager.tasks(): self.manager.accumulate(task()) self.manager.finalize() end_time = time() self.manager.time.user_time = end_time - start_time result = self.manager.result() return result, self.manager.time
def process(self): """ Do all the processing tasks. :return: The processing results """ start_time = time() self.manager.initialize() mp_method = self.manager.params.mp.method mp_njobs = self.manager.params.mp.njobs mp_nproc = self.manager.params.mp.nproc assert mp_nproc > 0, "Invalid number of processors" if mp_nproc * mp_njobs > len(self.manager): mp_nproc = min(mp_nproc, len(self.manager)) mp_njobs = int(math.ceil(len(self.manager) / mp_nproc)) logger.info(self.manager.summary()) if mp_njobs > 1: assert mp_method != "none" and mp_method is not None logger.info( " Using %s with %d parallel job(s) and %d processes per node\n", mp_method, mp_njobs, mp_nproc, ) else: logger.info(" Using multiprocessing with %d parallel job(s)\n", mp_nproc) if mp_njobs * mp_nproc > 1: def process_output(result): rehandle_cached_records(result[1]) self.manager.accumulate(result[0]) multi_node_parallel_map( func=execute_parallel_task, iterable=list(self.manager.tasks()), njobs=mp_njobs, nproc=mp_nproc, callback=process_output, cluster_method=mp_method, preserve_order=True, ) else: for task in self.manager.tasks(): self.manager.accumulate(task()) self.manager.finalize() end_time = time() self.manager.time.user_time = end_time - start_time result1, result2 = self.manager.result() return result1, result2, self.manager.time
def __init__(self, experiments, reflections, reference=None, params=None): # Create the reference manager integration_manager = IntegrationManager( experiments, reflections, reference, params ) # Print some output logger.info(integration_manager.summary()) # Execute each task if params.integration.mp.njobs > 1: if params.integration.mp.method == "multiprocessing": _assert_enough_memory( params.integration.mp.njobs * compute_required_memory( experiments[0].imageset, params.integration.block.size ), params.integration.block.max_memory_usage, ) def process_output(result): for message in result[1]: logger.log(message.levelno, message.msg) integration_manager.accumulate(result[0]) result[0].reflections = None result[0].data = None multi_node_parallel_map( func=execute_parallel_task, iterable=integration_manager.tasks(), nproc=params.integration.mp.nproc, njobs=params.integration.mp.njobs, callback=process_output, cluster_method=params.integration.mp.method, preserve_order=True, preserve_exception_message=True, ) else: for task in integration_manager.tasks(): result = task() integration_manager.accumulate(result) # Finalize the processing integration_manager.finalize() # Set the reflections and profiles self._reflections = integration_manager.result()
def test_cached_log_records(caplog): # Generate some cached log messages in easy_mp child processes. results = multi_node_parallel_map( log_something, iterable=range(5), njobs=2, nproc=2, cluster_method="multiprocessing", ) # Get all the resulting log records in a single flattened list. results = [record for records in results for record in records] results_batch = batch_multi_node_parallel_map( log_something, range(5), njobs=2, nproc=2, cluster_method="multiprocessing", callback=lambda _: None, ) # Get all the resulting log records in a single flattened list. results_batch = [ record for batch in results_batch for records in batch for record in records ] # Check that re-handling the messages in a logger in this process with a # threshold severity of WARNING results in no log records being emitted. with dials.util.log.LoggingContext("dials", logging.WARNING): dials.util.log.rehandle_cached_records(results) assert not caplog.records dials.util.log.rehandle_cached_records(results_batch) assert not caplog.records # Check that re-handling the messages in a logger in this process with a # threshold severity of INFO results in all the log records being emitted. with dials.util.log.LoggingContext("dials", logging.INFO): dials.util.log.rehandle_cached_records(results) assert caplog.records == results caplog.clear() dials.util.log.rehandle_cached_records(results_batch) assert caplog.records == results_batch caplog.clear()
def __init__(self, experiments, reflections, params=None): from dials.util import pprint # Create the reference manager reference_manager = ReferenceCalculatorManager(experiments, reflections, params) # Print some output logger.info(reference_manager.summary()) # Execute each task if params.integration.mp.njobs > 1: if params.integration.mp.method == "multiprocessing": assert_enough_memory( params.integration.mp.njobs * compute_required_memory(experiments[0].imageset, params.integration.block.size), params.integration.block.max_memory_usage, ) def process_output(result): for message in result[1]: logger.log(message.levelno, message.msg) reference_manager.accumulate(result[0]) result[0].reflections = None result[0].data = None multi_node_parallel_map( func=ExecuteParallelTask(), iterable=reference_manager.tasks(), nproc=params.integration.mp.nproc, njobs=params.integration.mp.njobs, callback=process_output, cluster_method=params.integration.mp.method, preserve_order=True, preserve_exception_message=True, ) else: for task in reference_manager.tasks(): result = task() reference_manager.accumulate(result) # Finalize the processing reference_manager.finalize() # Set the reflections and profiles self._reflections = reference_manager.result() self._profiles = reference_manager.reference # Write the profiles to file if params.integration.debug.reference.output: with open(params.integration.debug.reference.filename, "wb") as outfile: import six.moves.cPickle as pickle pickle.dump(self._profiles, outfile) # Print the profiles to the debug log for i in range(len(self._profiles)): logger.debug("") logger.debug("Reference Profiles for experiment %d" % i) logger.debug("") reference = self._profiles[i].reference() for j in range(len(reference)): data = reference.data(j) logger.debug("Profile %d" % j) if len(data) > 0: logger.debug(pprint.profile3d(data)) else: logger.debug("** NO PROFILE **")
def process(self): ''' Do all the processing tasks. :return: The processing results ''' from time import time from dials.util.mp import multi_node_parallel_map import platform from math import ceil start_time = time() self.manager.initialize() mp_method = self.manager.params.mp.method mp_njobs = self.manager.params.mp.njobs mp_nproc = self.manager.params.mp.nproc if (mp_njobs * mp_nproc) > 1 and platform.system( ) == "Windows": # platform.system() forks which is bad for MPI, so don't use it unless nproc > 1 logger.warn("") logger.warn("*" * 80) logger.warn( "Multiprocessing is not available on windows. Setting nproc = 1" ) logger.warn("*" * 80) logger.warn("") mp_nproc = 1 mp_njobs = 1 assert mp_nproc > 0, "Invalid number of processors" if mp_nproc * mp_njobs > len(self.manager): mp_nproc = min(mp_nproc, len(self.manager)) mp_njobs = int(ceil(len(self.manager) / mp_nproc)) logger.info(self.manager.summary()) if mp_njobs > 1: assert mp_method is not 'none' and mp_method is not None logger.info( ' Using %s with %d parallel job(s) and %d processes per node\n' % (mp_method, mp_njobs, mp_nproc)) else: logger.info(' Using multiprocessing with %d parallel job(s)\n' % (mp_nproc)) if mp_njobs * mp_nproc > 1: def process_output(result): for message in result[1]: logger.log(message.levelno, message.msg) self.manager.accumulate(result[0]) result[0].reflections = None result[0].data = None multi_node_parallel_map(func=ExecuteParallelTask(), iterable=list(self.manager.tasks()), njobs=mp_njobs, nproc=mp_nproc, callback=process_output, cluster_method=mp_method, preserve_order=True, preserve_exception_message=True) else: for task in self.manager.tasks(): self.manager.accumulate(task()) self.manager.finalize() end_time = time() self.manager.time.user_time = end_time - start_time result1, result2 = self.manager.result() return result1, result2, self.manager.time