def _generate_serial(func,
args_list,
prog=True,
verbose=True,
nTasks=None,
**kwargs):
""" internal serial generator """
if nTasks is None:
nTasks = len(args_list)
if verbose:
print(
'[util_parallel._generate_serial] executing %d %s tasks in serial'
% (nTasks, get_funcname(func)))
prog = prog and verbose and nTasks > 1
# Get iterator with or without progress
lbl = '(sergen) %s: ' % (get_funcname(func), )
args_iter = (util_progress.ProgressIter(args_list,
nTotal=nTasks,
lbl=lbl,
freq=kwargs.get('freq', None),
adjust=kwargs.get('adjust', False))
if prog else args_list)
if __TIME_GENERATE__:
tt = util_time.tic('_generate_serial func=' + get_funcname(func))
for args in args_iter:
result = func(args)
yield result
if __TIME_GENERATE__:
util_time.toc(tt)
def _generate_serial(func, args_list, prog=True, verbose=True, nTasks=None,
quiet=QUIET, **kwargs):
""" internal serial generator """
if nTasks is None:
nTasks = len(args_list)
if verbose and not quiet:
print('[util_parallel._generate_serial] executing %d %s tasks in serial' %
(nTasks, get_funcname(func)))
prog = prog and verbose and nTasks > 1
# Get iterator with or without progress
verbose = verbose or not quiet
lbl = '(sergen) %s: ' % (get_funcname(func),)
args_iter = (
util_progress.ProgressIter(args_list, nTotal=nTasks,
lbl=lbl,
freq=kwargs.get('freq', None),
adjust=kwargs.get('adjust', False),
verbose=verbose)
if prog else args_list
)
if __TIME_GENERATE__:
tt = util_time.tic('_generate_serial func=' + get_funcname(func))
for args in args_iter:
result = func(args)
yield result
if __TIME_GENERATE__:
util_time.toc(tt)
def end_progress(count_=nTotal, write_fn=write_fn, flush_fn=flush_fn):
write_fn(fmt_str % (count_))
write_fn('\n')
flush_fn()
if with_time:
util_time.toc(tt)
if pad_stdout:
write_fn('\n\n')
flush_fn()
def _generate_parallel(func, args_list, ordered=True, chunksize=1,
prog=True, verbose=True, nTasks=None, freq=None):
"""
Parallel process generator
"""
prog = prog and verbose
if nTasks is None:
nTasks = len(args_list)
if chunksize is None:
chunksize = max(1, nTasks // (__POOL__._processes ** 2))
if verbose:
prefix = '[util_parallel._generate_parallel]'
fmtstr = prefix + 'executing %d %s tasks using %d processes with chunksize=%r'
print(fmtstr % (nTasks, get_funcname(func), __POOL__._processes, chunksize))
pmap_func = __POOL__.imap if ordered else __POOL__.imap_unordered
raw_generator = pmap_func(func, args_list, chunksize)
# Get iterator with or without progress
result_generator = (
util_progress.ProgressIter(raw_generator, nTotal=nTasks, lbl=get_funcname(func) + ': ', freq=freq)
if prog else raw_generator
)
if __TIME_GENERATE__:
tt = util_time.tic('_generate_parallel func=' + get_funcname(func))
try:
for result in result_generator:
yield result
if __EAGER_JOIN__:
close_pool()
except Exception as ex:
util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True)
if __EAGER_JOIN__:
close_pool()
print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__)
if __SERIAL_FALLBACK__:
print('Trying to handle error by falling back to serial')
serial_generator = _generate_serial(
func, args_list, prog=prog, verbose=verbose, nTasks=nTasks, freq=freq)
for result in serial_generator:
yield result
else:
raise
if __TIME_GENERATE__:
util_time.toc(tt)
def _generate_serial(func, args_list, prog=True, verbose=True, nTasks=None, freq=None):
""" internal serial generator """
if nTasks is None:
nTasks = len(args_list)
if verbose:
print('[util_parallel._generate_serial] executing %d %s tasks in serial' %
(nTasks, get_funcname(func)))
prog = prog and verbose and nTasks > 1
# Get iterator with or without progress
args_iter = (
util_progress.ProgressIter(args_list, nTotal=nTasks, lbl=get_funcname(func) + ': ', freq=freq)
if prog else args_list
)
if __TIME_GENERATE__:
tt = util_time.tic('_generate_serial func=' + get_funcname(func))
for args in args_iter:
result = func(args)
yield result
if __TIME_GENERATE__:
util_time.toc(tt)
def _generate_parallel(func, args_list, ordered=True, chunksize=None,
prog=True, verbose=True, quiet=QUIET, nTasks=None,
**kwargs):
"""
Parallel process generator
"""
global __POOL__
if FUTURE_ON:
raise AssertionError('USE FUTURES')
if USE_GLOBAL_POOL:
global __POOL__
pool = __POOL__
else:
# Maybe global pools are bad?
pool = new_pool(num_procs=get_default_numprocs(),
init_worker=init_worker,
maxtasksperchild=None)
#pool = new_pool()
prog = prog and verbose
if nTasks is None:
nTasks = len(args_list)
if chunksize is None:
chunksize = max(min(4, nTasks), min(8, nTasks // (pool._processes ** 2)))
if verbose or VERBOSE_PARALLEL:
prefix = '[util_parallel._generate_parallel]'
fmtstr = (prefix +
'executing %d %s tasks using %d processes with chunksize=%r')
print(fmtstr % (nTasks, get_funcname(func), pool._processes, chunksize))
#import utool as ut
#buffered = ut.get_argflag('--buffered')
#buffered = False
#if buffered:
# # current tests indicate that normal pool.imap is faster than buffered
# # generation
# source_gen = (func(args) for args in args_list)
# raw_generator = buffered_generator(source_gen)
#else:
pmap_func = pool.imap if ordered else pool.imap_unordered
raw_generator = pmap_func(func, args_list, chunksize)
# Get iterator with or without progress
if prog:
verbose = not quiet
lbl = '(pargen) %s: ' % (get_funcname(func),)
result_generator = util_progress.ProgressIter(
raw_generator, nTotal=nTasks, lbl=lbl,
freq=kwargs.get('freq', None),
backspace=kwargs.get('backspace', True),
adjust=kwargs.get('adjust', False),
verbose=verbose
)
else:
result_generator = raw_generator
if __TIME_GENERATE__:
tt = util_time.tic('_generate_parallel func=' + get_funcname(func))
try:
# Start generating
for result in result_generator:
yield result
if __EAGER_JOIN__:
if USE_GLOBAL_POOL:
close_pool(quiet=quiet)
else:
pool.close()
pool.join()
except Exception as ex:
util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True)
if __EAGER_JOIN__:
if USE_GLOBAL_POOL:
close_pool(quiet=quiet)
else:
pool.close()
pool.join()
# DONT DO SERIAL FALLBACK IN GENERATOR CAN CAUSE ERRORS
raise
# print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__)
# if __SERIAL_FALLBACK__:
# print('Trying to handle error by falling back to serial')
# serial_generator = _generate_serial(
# func, args_list, prog=prog, verbose=verbose, nTasks=nTasks,
# **kwargs)
# for result in serial_generator:
# yield result
# else:
# raise
if __TIME_GENERATE__:
util_time.toc(tt)
def _generate_parallel(func,
args_list,
ordered=True,
chunksize=None,
prog=True,
verbose=True,
quiet=QUIET,
nTasks=None,
**kwargs):
"""
Parallel process generator
"""
global __POOL__
if USE_GLOBAL_POOL:
global __POOL__
pool = __POOL__
else:
# Maybe global pools are bad?
pool = new_pool(num_procs=get_default_numprocs(),
init_worker=init_worker,
maxtasksperchild=None)
#pool = new_pool()
prog = prog and verbose
if nTasks is None:
nTasks = len(args_list)
if chunksize is None:
chunksize = max(min(4, nTasks), min(8, nTasks // (pool._processes**2)))
if verbose or VERBOSE_PARALLEL:
prefix = '[util_parallel._generate_parallel]'
fmtstr = (prefix +
'executing %d %s tasks using %d processes with chunksize=%r')
print(fmtstr %
(nTasks, get_funcname(func), pool._processes, chunksize))
#import utool as ut
#buffered = ut.get_argflag('--buffered')
#buffered = False
#if buffered:
# # current tests indicate that normal pool.imap is faster than buffered
# # generation
# source_gen = (func(args) for args in args_list)
# raw_generator = buffered_generator(source_gen)
#else:
pmap_func = pool.imap if ordered else pool.imap_unordered
raw_generator = pmap_func(func, args_list, chunksize)
# Get iterator with or without progress
if prog:
lbl = '(pargen) %s: ' % (get_funcname(func), )
result_generator = util_progress.ProgressIter(
raw_generator,
nTotal=nTasks,
lbl=lbl,
freq=kwargs.get('freq', None),
backspace=kwargs.get('backspace', True),
adjust=kwargs.get('adjust', False))
else:
result_generator = raw_generator
if __TIME_GENERATE__:
tt = util_time.tic('_generate_parallel func=' + get_funcname(func))
try:
# Start generating
for result in result_generator:
yield result
if __EAGER_JOIN__:
if USE_GLOBAL_POOL:
close_pool(quiet=quiet)
else:
pool.close()
pool.join()
except Exception as ex:
util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True)
if __EAGER_JOIN__:
if USE_GLOBAL_POOL:
close_pool(quiet=quiet)
else:
pool.close()
pool.join()
print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__)
if __SERIAL_FALLBACK__:
print('Trying to handle error by falling back to serial')
serial_generator = _generate_serial(func,
args_list,
prog=prog,
verbose=verbose,
nTasks=nTasks,
**kwargs)
for result in serial_generator:
yield result
else:
raise
if __TIME_GENERATE__:
util_time.toc(tt)