def _generate_serial(func, args_list, prog=True, verbose=True, nTasks=None,
quiet=QUIET, **kwargs):
""" internal serial generator """
if nTasks is None:
nTasks = len(args_list)
if verbose and not quiet:
print('[util_parallel._generate_serial] executing %d %s tasks in serial' %
(nTasks, get_funcname(func)))
prog = prog and verbose and nTasks > 1
# Get iterator with or without progress
verbose = verbose or not quiet
lbl = '(sergen) %s: ' % (get_funcname(func),)
args_iter = (
util_progress.ProgressIter(args_list, nTotal=nTasks,
lbl=lbl,
freq=kwargs.get('freq', None),
adjust=kwargs.get('adjust', False),
verbose=verbose)
if prog else args_list
)
if __TIME_GENERATE__:
tt = util_time.tic('_generate_serial func=' + get_funcname(func))
for args in args_iter:
result = func(args)
yield result
if __TIME_GENERATE__:
util_time.toc(tt)
def _generate_serial(func,
args_list,
prog=True,
verbose=True,
nTasks=None,
**kwargs):
""" internal serial generator """
if nTasks is None:
nTasks = len(args_list)
if verbose:
print(
'[util_parallel._generate_serial] executing %d %s tasks in serial'
% (nTasks, get_funcname(func)))
prog = prog and verbose and nTasks > 1
# Get iterator with or without progress
lbl = '(sergen) %s: ' % (get_funcname(func), )
args_iter = (util_progress.ProgressIter(args_list,
nTotal=nTasks,
lbl=lbl,
freq=kwargs.get('freq', None),
adjust=kwargs.get('adjust', False))
if prog else args_list)
if __TIME_GENERATE__:
tt = util_time.tic('_generate_serial func=' + get_funcname(func))
for args in args_iter:
result = func(args)
yield result
if __TIME_GENERATE__:
util_time.toc(tt)
def _generate_parallel(func, args_list, ordered=True, chunksize=1,
prog=True, verbose=True, nTasks=None, freq=None):
"""
Parallel process generator
"""
prog = prog and verbose
if nTasks is None:
nTasks = len(args_list)
if chunksize is None:
chunksize = max(1, nTasks // (__POOL__._processes ** 2))
if verbose:
prefix = '[util_parallel._generate_parallel]'
fmtstr = prefix + 'executing %d %s tasks using %d processes with chunksize=%r'
print(fmtstr % (nTasks, get_funcname(func), __POOL__._processes, chunksize))
pmap_func = __POOL__.imap if ordered else __POOL__.imap_unordered
raw_generator = pmap_func(func, args_list, chunksize)
# Get iterator with or without progress
result_generator = (
util_progress.ProgressIter(raw_generator, nTotal=nTasks, lbl=get_funcname(func) + ': ', freq=freq)
if prog else raw_generator
)
if __TIME_GENERATE__:
tt = util_time.tic('_generate_parallel func=' + get_funcname(func))
try:
for result in result_generator:
yield result
if __EAGER_JOIN__:
close_pool()
except Exception as ex:
util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True)
if __EAGER_JOIN__:
close_pool()
print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__)
if __SERIAL_FALLBACK__:
print('Trying to handle error by falling back to serial')
serial_generator = _generate_serial(
func, args_list, prog=prog, verbose=verbose, nTasks=nTasks, freq=freq)
for result in serial_generator:
yield result
else:
raise
if __TIME_GENERATE__:
util_time.toc(tt)
def _generate_serial(func, args_list, prog=True, verbose=True, nTasks=None, freq=None):
""" internal serial generator """
if nTasks is None:
nTasks = len(args_list)
if verbose:
print('[util_parallel._generate_serial] executing %d %s tasks in serial' %
(nTasks, get_funcname(func)))
prog = prog and verbose and nTasks > 1
# Get iterator with or without progress
args_iter = (
util_progress.ProgressIter(args_list, nTotal=nTasks, lbl=get_funcname(func) + ': ', freq=freq)
if prog else args_list
)
if __TIME_GENERATE__:
tt = util_time.tic('_generate_serial func=' + get_funcname(func))
for args in args_iter:
result = func(args)
yield result
if __TIME_GENERATE__:
util_time.toc(tt)
def _generate_parallel(func, args_list, ordered=True, chunksize=None,
prog=True, verbose=True, quiet=QUIET, nTasks=None,
**kwargs):
"""
Parallel process generator
"""
global __POOL__
if FUTURE_ON:
raise AssertionError('USE FUTURES')
if USE_GLOBAL_POOL:
global __POOL__
pool = __POOL__
else:
# Maybe global pools are bad?
pool = new_pool(num_procs=get_default_numprocs(),
init_worker=init_worker,
maxtasksperchild=None)
#pool = new_pool()
prog = prog and verbose
if nTasks is None:
nTasks = len(args_list)
if chunksize is None:
chunksize = max(min(4, nTasks), min(8, nTasks // (pool._processes ** 2)))
if verbose or VERBOSE_PARALLEL:
prefix = '[util_parallel._generate_parallel]'
fmtstr = (prefix +
'executing %d %s tasks using %d processes with chunksize=%r')
print(fmtstr % (nTasks, get_funcname(func), pool._processes, chunksize))
#import utool as ut
#buffered = ut.get_argflag('--buffered')
#buffered = False
#if buffered:
# # current tests indicate that normal pool.imap is faster than buffered
# # generation
# source_gen = (func(args) for args in args_list)
# raw_generator = buffered_generator(source_gen)
#else:
pmap_func = pool.imap if ordered else pool.imap_unordered
raw_generator = pmap_func(func, args_list, chunksize)
# Get iterator with or without progress
if prog:
verbose = not quiet
lbl = '(pargen) %s: ' % (get_funcname(func),)
result_generator = util_progress.ProgressIter(
raw_generator, nTotal=nTasks, lbl=lbl,
freq=kwargs.get('freq', None),
backspace=kwargs.get('backspace', True),
adjust=kwargs.get('adjust', False),
verbose=verbose
)
else:
result_generator = raw_generator
if __TIME_GENERATE__:
tt = util_time.tic('_generate_parallel func=' + get_funcname(func))
try:
# Start generating
for result in result_generator:
yield result
if __EAGER_JOIN__:
if USE_GLOBAL_POOL:
close_pool(quiet=quiet)
else:
pool.close()
pool.join()
except Exception as ex:
util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True)
if __EAGER_JOIN__:
if USE_GLOBAL_POOL:
close_pool(quiet=quiet)
else:
pool.close()
pool.join()
# DONT DO SERIAL FALLBACK IN GENERATOR CAN CAUSE ERRORS
raise
# print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__)
# if __SERIAL_FALLBACK__:
# print('Trying to handle error by falling back to serial')
# serial_generator = _generate_serial(
# func, args_list, prog=prog, verbose=verbose, nTasks=nTasks,
# **kwargs)
# for result in serial_generator:
# yield result
# else:
# raise
if __TIME_GENERATE__:
util_time.toc(tt)
def log_progress(lbl='Progress: ', nTotal=0, flushfreq=4, startafter=-1,
start=True, repl=False, approx=False, disable=False,
writefreq=1, with_time=False, backspace=True,
pad_stdout=False, wfreq=None, ffreq=None, freq=None, total=None,
num=None, with_totaltime=None):
"""
DEPRICATE
FIXME: depricate for ProgressIter.
still used in util_dev
"""
global AGGROFLUSH
# Alias kwargs with simpler names
if num is not None:
nTotal = num
if total is not None:
nTotal = total
if wfreq is not None:
writefreq = wfreq
if ffreq is not None:
flushfreq = ffreq
if freq is not None:
writefreq = flushfreq = freq
if with_totaltime is not None:
with_time = with_totaltime
# flush frequency must be a multiple of write frequency
flushfreq = max(int(round(flushfreq / writefreq)), 1) * writefreq
if nTotal < startafter or disable:
# Do not mark progress if only executing a small number of tasks
def mark_progress(*args):
pass
def end_progress(*args):
pass
return mark_progress, end_progress
else:
write_fn = PROGRESS_WRITE
flush_fn = PROGRESS_FLUSH
# build format string for displaying progress
fmt_str = progress_str(nTotal, lbl=lbl, repl=repl, approx=approx,
backspace=backspace)
if AGGROFLUSH:
# Progress function which automatically flushes
def mark_progress(count, flush_fn=flush_fn):
count_ = count + 1
write_fn(fmt_str % (count_))
flush_fn()
else:
# Progress function flushes every <flushfreq> times
def mark_progress(count, fmt_str=fmt_str, flushfreq=flushfreq,
writefreq=writefreq, write_fn=write_fn,
flush_fn=flush_fn):
count_ = count + 1
if count_ % writefreq == 0:
write_fn(fmt_str % count_)
if count_ % flushfreq == 0:
flush_fn()
if pad_stdout:
write_fn('\n')
write_fn('\n')
flush_fn()
if with_time:
tt = util_time.tic(lbl)
def end_progress(count_=nTotal, write_fn=write_fn, flush_fn=flush_fn):
write_fn(fmt_str % (count_))
write_fn('\n')
flush_fn()
if with_time:
util_time.toc(tt)
if pad_stdout:
write_fn('\n\n')
flush_fn()
#mark_progress(0)
if start:
mark_progress(-1)
return mark_progress, end_progress
def _generate_parallel(func,
args_list,
ordered=True,
chunksize=None,
prog=True,
verbose=True,
quiet=QUIET,
nTasks=None,
**kwargs):
"""
Parallel process generator
"""
global __POOL__
if USE_GLOBAL_POOL:
global __POOL__
pool = __POOL__
else:
# Maybe global pools are bad?
pool = new_pool(num_procs=get_default_numprocs(),
init_worker=init_worker,
maxtasksperchild=None)
#pool = new_pool()
prog = prog and verbose
if nTasks is None:
nTasks = len(args_list)
if chunksize is None:
chunksize = max(min(4, nTasks), min(8, nTasks // (pool._processes**2)))
if verbose or VERBOSE_PARALLEL:
prefix = '[util_parallel._generate_parallel]'
fmtstr = (prefix +
'executing %d %s tasks using %d processes with chunksize=%r')
print(fmtstr %
(nTasks, get_funcname(func), pool._processes, chunksize))
#import utool as ut
#buffered = ut.get_argflag('--buffered')
#buffered = False
#if buffered:
# # current tests indicate that normal pool.imap is faster than buffered
# # generation
# source_gen = (func(args) for args in args_list)
# raw_generator = buffered_generator(source_gen)
#else:
pmap_func = pool.imap if ordered else pool.imap_unordered
raw_generator = pmap_func(func, args_list, chunksize)
# Get iterator with or without progress
if prog:
lbl = '(pargen) %s: ' % (get_funcname(func), )
result_generator = util_progress.ProgressIter(
raw_generator,
nTotal=nTasks,
lbl=lbl,
freq=kwargs.get('freq', None),
backspace=kwargs.get('backspace', True),
adjust=kwargs.get('adjust', False))
else:
result_generator = raw_generator
if __TIME_GENERATE__:
tt = util_time.tic('_generate_parallel func=' + get_funcname(func))
try:
# Start generating
for result in result_generator:
yield result
if __EAGER_JOIN__:
if USE_GLOBAL_POOL:
close_pool(quiet=quiet)
else:
pool.close()
pool.join()
except Exception as ex:
util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True)
if __EAGER_JOIN__:
if USE_GLOBAL_POOL:
close_pool(quiet=quiet)
else:
pool.close()
pool.join()
print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__)
if __SERIAL_FALLBACK__:
print('Trying to handle error by falling back to serial')
serial_generator = _generate_serial(func,
args_list,
prog=prog,
verbose=verbose,
nTasks=nTasks,
**kwargs)
for result in serial_generator:
yield result
else:
raise
if __TIME_GENERATE__:
util_time.toc(tt)
def log_progress(lbl='Progress: ', nTotal=0, flushfreq=4, startafter=-1,
start=True, repl=False, approx=False, disable=False,
writefreq=1, with_time=False, backspace=True,
pad_stdout=False, wfreq=None, ffreq=None, freq=None, total=None,
num=None, with_totaltime=None):
"""
Returns two functions (mark_progress, end_progress) which will handle
logging progress in a for loop.
flush frequency must be a multiple of write frequency
Args:
lbl (str): progress label
nTotal (int):
flushfreq (int):
startafter (int):
start (bool):
repl (bool):
approx (bool):
disable (bool):
writefreq (int):
with_totaltime (bool):
backspace (bool):
pad_stdout (bool):
wfreq (None): alias for write_freq
ffreq (None): alias for flush_freq
freq (None): alias for flush_freq and write_freq (prefered)
total (None): alias for nTotal
num (None): alias for nTotal
Example:
>>> import utool, time
>>> from six.moves import range
>>> # Define a dummy task
>>> spam = 42.0
>>> nTotal = 1000
>>> iter_ = (num for num in range(0, nTotal * 2, 2))
>>> # Create progress functions
... mark_, end_ = utool.log_progress('prog ', nTotal, flushfreq=17)
\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\bprog 0/1000
>>> for count, item in enumerate(iter_): #doctest: +ELLIPSIS
... # Call with enumerate to keep track of a count variable
... time.sleep(.001)
... spam += item + count
... mark_(count)
\b...prog 1000/1000
>>> # Mark completion
>>> end_()
<BLANKLINE>
"""
# utool.auto_docstr('utool.util_progress', 'log_progress')
# python -c "import utool; utool.print_auto_docstr('utool.util_progress', 'log_progress')"
#
# In reference to above docstr:
# I don't completely understand why some of the >>> and ... had to be where
# they are, but doctest gets very angry if its not in this format
# TODO: Option to display rate of progress
# TODO: Option to display eta
global AGGROFLUSH
# Alias kwargs with simpler names
if num is not None:
nTotal = num
if total is not None:
nTotal = total
if wfreq is not None:
writefreq = wfreq
if ffreq is not None:
flushfreq = ffreq
if freq is not None:
writefreq = flushfreq = freq
if with_totaltime is not None:
with_time = with_totaltime
# flush frequency must be a multiple of write frequency
flushfreq = max(int(round(flushfreq / writefreq)), 1) * writefreq
if nTotal < startafter or disable:
# Do not mark progress if only executing a small number of tasks
def mark_progress(*args):
pass
def end_progress(*args):
pass
return mark_progress, end_progress
else:
write_fn = PROGRESS_WRITE
flush_fn = PROGRESS_FLUSH
# build format string for displaying progress
fmt_str = progress_str(nTotal, lbl=lbl, repl=repl, approx=approx,
backspace=backspace)
if AGGROFLUSH:
# Progress function which automatically flushes
def mark_progress(count, flush_fn=flush_fn):
count_ = count + 1
write_fn(fmt_str % (count_))
flush_fn()
else:
# Progress function flushes every <flushfreq> times
def mark_progress(count, fmt_str=fmt_str, flushfreq=flushfreq,
writefreq=writefreq, write_fn=write_fn,
flush_fn=flush_fn):
count_ = count + 1
if count_ % writefreq == 0:
write_fn(fmt_str % count_)
if count_ % flushfreq == 0:
flush_fn()
if pad_stdout:
write_fn('\n')
write_fn('\n')
flush_fn()
if with_time:
tt = util_time.tic(lbl)
def end_progress(count_=nTotal, write_fn=write_fn, flush_fn=flush_fn):
write_fn(fmt_str % (count_))
write_fn('\n')
flush_fn()
if with_time:
util_time.toc(tt)
if pad_stdout:
write_fn('\n\n')
flush_fn()
#mark_progress(0)
if start:
mark_progress(-1)
return mark_progress, end_progress