def _run_multiprocess(cases, ncpu, **kwargs):
"""
Run multiprocessing jobs
Returns
-------
"""
logger.info('Processing {} jobs on {} CPUs'.format(len(cases), ncpu))
logger.handlers[0].setLevel(logging.WARNING)
# start processes
jobs = []
for idx, file in enumerate(cases):
job = Process(name=f'Process {idx:d}',
target=run_case,
args=(file, ),
kwargs=kwargs)
jobs.append(job)
job.start()
start_msg = f'Process {idx:d} <{file:s}> started.'
print(start_msg)
logger.debug(start_msg)
if (idx % ncpu == ncpu - 1) or (idx == len(cases) - 1):
sleep(0.1)
for job in jobs:
job.join()
jobs = []
# restore command line output when all jobs are done
logger.handlers[0].setLevel(logging.INFO)
def _run_multiprocess_proc(cases, ncpu=os.cpu_count(), **kwargs):
"""
Run multiprocessing with `Process`.
Return values from `run_case` are not preserved. Always return `True` when done.
"""
# start processes
jobs = []
for idx, file in enumerate(cases):
job = Process(name=f'Process {idx:d}',
target=run_case,
args=(file, ),
kwargs=kwargs)
jobs.append(job)
job.start()
start_msg = f'Process {idx:d} for "{file:s}" started.'
print(start_msg)
logger.debug(start_msg)
if (idx % ncpu == ncpu - 1) or (idx == len(cases) - 1):
sleep(0.1)
for job in jobs:
job.join()
jobs = []
return True
def _run_multiprocess_proc(cases, ncpu=os.cpu_count(), **kwargs):
# start processes
jobs = []
for idx, file in enumerate(cases):
job = Process(name=f'Process {idx:d}', target=run_case, args=(file,), kwargs=kwargs)
jobs.append(job)
job.start()
start_msg = f'Process {idx:d} for "{file:s}" started.'
print(start_msg)
logger.debug(start_msg)
if (idx % ncpu == ncpu - 1) or (idx == len(cases) - 1):
sleep(0.1)
for job in jobs:
job.join()
jobs = []
return True