def _generate_serial(func, args_list, prog=True, verbose=True, nTasks=None, quiet=QUIET, **kwargs): """ internal serial generator """ if nTasks is None: nTasks = len(args_list) if verbose and not quiet: print('[util_parallel._generate_serial] executing %d %s tasks in serial' % (nTasks, get_funcname(func))) prog = prog and verbose and nTasks > 1 # Get iterator with or without progress verbose = verbose or not quiet lbl = '(sergen) %s: ' % (get_funcname(func),) args_iter = ( util_progress.ProgressIter(args_list, nTotal=nTasks, lbl=lbl, freq=kwargs.get('freq', None), adjust=kwargs.get('adjust', False), verbose=verbose) if prog else args_list ) if __TIME_GENERATE__: tt = util_time.tic('_generate_serial func=' + get_funcname(func)) for args in args_iter: result = func(args) yield result if __TIME_GENERATE__: util_time.toc(tt)
def _generate_serial(func, args_list, prog=True, verbose=True, nTasks=None, **kwargs): """ internal serial generator """ if nTasks is None: nTasks = len(args_list) if verbose: print( '[util_parallel._generate_serial] executing %d %s tasks in serial' % (nTasks, get_funcname(func))) prog = prog and verbose and nTasks > 1 # Get iterator with or without progress lbl = '(sergen) %s: ' % (get_funcname(func), ) args_iter = (util_progress.ProgressIter(args_list, nTotal=nTasks, lbl=lbl, freq=kwargs.get('freq', None), adjust=kwargs.get('adjust', False)) if prog else args_list) if __TIME_GENERATE__: tt = util_time.tic('_generate_serial func=' + get_funcname(func)) for args in args_iter: result = func(args) yield result if __TIME_GENERATE__: util_time.toc(tt)
def _generate_parallel(func, args_list, ordered=True, chunksize=1, prog=True, verbose=True, nTasks=None, freq=None): """ Parallel process generator """ prog = prog and verbose if nTasks is None: nTasks = len(args_list) if chunksize is None: chunksize = max(1, nTasks // (__POOL__._processes ** 2)) if verbose: prefix = '[util_parallel._generate_parallel]' fmtstr = prefix + 'executing %d %s tasks using %d processes with chunksize=%r' print(fmtstr % (nTasks, get_funcname(func), __POOL__._processes, chunksize)) pmap_func = __POOL__.imap if ordered else __POOL__.imap_unordered raw_generator = pmap_func(func, args_list, chunksize) # Get iterator with or without progress result_generator = ( util_progress.ProgressIter(raw_generator, nTotal=nTasks, lbl=get_funcname(func) + ': ', freq=freq) if prog else raw_generator ) if __TIME_GENERATE__: tt = util_time.tic('_generate_parallel func=' + get_funcname(func)) try: for result in result_generator: yield result if __EAGER_JOIN__: close_pool() except Exception as ex: util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True) if __EAGER_JOIN__: close_pool() print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__) if __SERIAL_FALLBACK__: print('Trying to handle error by falling back to serial') serial_generator = _generate_serial( func, args_list, prog=prog, verbose=verbose, nTasks=nTasks, freq=freq) for result in serial_generator: yield result else: raise if __TIME_GENERATE__: util_time.toc(tt)
def _generate_serial(func, args_list, prog=True, verbose=True, nTasks=None, freq=None): """ internal serial generator """ if nTasks is None: nTasks = len(args_list) if verbose: print('[util_parallel._generate_serial] executing %d %s tasks in serial' % (nTasks, get_funcname(func))) prog = prog and verbose and nTasks > 1 # Get iterator with or without progress args_iter = ( util_progress.ProgressIter(args_list, nTotal=nTasks, lbl=get_funcname(func) + ': ', freq=freq) if prog else args_list ) if __TIME_GENERATE__: tt = util_time.tic('_generate_serial func=' + get_funcname(func)) for args in args_iter: result = func(args) yield result if __TIME_GENERATE__: util_time.toc(tt)
def _generate_parallel(func, args_list, ordered=True, chunksize=None, prog=True, verbose=True, quiet=QUIET, nTasks=None, **kwargs): """ Parallel process generator """ global __POOL__ if FUTURE_ON: raise AssertionError('USE FUTURES') if USE_GLOBAL_POOL: global __POOL__ pool = __POOL__ else: # Maybe global pools are bad? pool = new_pool(num_procs=get_default_numprocs(), init_worker=init_worker, maxtasksperchild=None) #pool = new_pool() prog = prog and verbose if nTasks is None: nTasks = len(args_list) if chunksize is None: chunksize = max(min(4, nTasks), min(8, nTasks // (pool._processes ** 2))) if verbose or VERBOSE_PARALLEL: prefix = '[util_parallel._generate_parallel]' fmtstr = (prefix + 'executing %d %s tasks using %d processes with chunksize=%r') print(fmtstr % (nTasks, get_funcname(func), pool._processes, chunksize)) #import utool as ut #buffered = ut.get_argflag('--buffered') #buffered = False #if buffered: # # current tests indicate that normal pool.imap is faster than buffered # # generation # source_gen = (func(args) for args in args_list) # raw_generator = buffered_generator(source_gen) #else: pmap_func = pool.imap if ordered else pool.imap_unordered raw_generator = pmap_func(func, args_list, chunksize) # Get iterator with or without progress if prog: verbose = not quiet lbl = '(pargen) %s: ' % (get_funcname(func),) result_generator = util_progress.ProgressIter( raw_generator, nTotal=nTasks, lbl=lbl, freq=kwargs.get('freq', None), backspace=kwargs.get('backspace', True), adjust=kwargs.get('adjust', False), verbose=verbose ) else: result_generator = raw_generator if __TIME_GENERATE__: tt = util_time.tic('_generate_parallel func=' + get_funcname(func)) try: # Start generating for result in result_generator: yield result if __EAGER_JOIN__: if USE_GLOBAL_POOL: close_pool(quiet=quiet) else: pool.close() pool.join() except Exception as ex: util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True) if __EAGER_JOIN__: if USE_GLOBAL_POOL: close_pool(quiet=quiet) else: pool.close() pool.join() # DONT DO SERIAL FALLBACK IN GENERATOR CAN CAUSE ERRORS raise # print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__) # if __SERIAL_FALLBACK__: # print('Trying to handle error by falling back to serial') # serial_generator = _generate_serial( # func, args_list, prog=prog, verbose=verbose, nTasks=nTasks, # **kwargs) # for result in serial_generator: # yield result # else: # raise if __TIME_GENERATE__: util_time.toc(tt)
def log_progress(lbl='Progress: ', nTotal=0, flushfreq=4, startafter=-1, start=True, repl=False, approx=False, disable=False, writefreq=1, with_time=False, backspace=True, pad_stdout=False, wfreq=None, ffreq=None, freq=None, total=None, num=None, with_totaltime=None): """ DEPRICATE FIXME: depricate for ProgressIter. still used in util_dev """ global AGGROFLUSH # Alias kwargs with simpler names if num is not None: nTotal = num if total is not None: nTotal = total if wfreq is not None: writefreq = wfreq if ffreq is not None: flushfreq = ffreq if freq is not None: writefreq = flushfreq = freq if with_totaltime is not None: with_time = with_totaltime # flush frequency must be a multiple of write frequency flushfreq = max(int(round(flushfreq / writefreq)), 1) * writefreq if nTotal < startafter or disable: # Do not mark progress if only executing a small number of tasks def mark_progress(*args): pass def end_progress(*args): pass return mark_progress, end_progress else: write_fn = PROGRESS_WRITE flush_fn = PROGRESS_FLUSH # build format string for displaying progress fmt_str = progress_str(nTotal, lbl=lbl, repl=repl, approx=approx, backspace=backspace) if AGGROFLUSH: # Progress function which automatically flushes def mark_progress(count, flush_fn=flush_fn): count_ = count + 1 write_fn(fmt_str % (count_)) flush_fn() else: # Progress function flushes every <flushfreq> times def mark_progress(count, fmt_str=fmt_str, flushfreq=flushfreq, writefreq=writefreq, write_fn=write_fn, flush_fn=flush_fn): count_ = count + 1 if count_ % writefreq == 0: write_fn(fmt_str % count_) if count_ % flushfreq == 0: flush_fn() if pad_stdout: write_fn('\n') write_fn('\n') flush_fn() if with_time: tt = util_time.tic(lbl) def end_progress(count_=nTotal, write_fn=write_fn, flush_fn=flush_fn): write_fn(fmt_str % (count_)) write_fn('\n') flush_fn() if with_time: util_time.toc(tt) if pad_stdout: write_fn('\n\n') flush_fn() #mark_progress(0) if start: mark_progress(-1) return mark_progress, end_progress
def _generate_parallel(func, args_list, ordered=True, chunksize=None, prog=True, verbose=True, quiet=QUIET, nTasks=None, **kwargs): """ Parallel process generator """ global __POOL__ if USE_GLOBAL_POOL: global __POOL__ pool = __POOL__ else: # Maybe global pools are bad? pool = new_pool(num_procs=get_default_numprocs(), init_worker=init_worker, maxtasksperchild=None) #pool = new_pool() prog = prog and verbose if nTasks is None: nTasks = len(args_list) if chunksize is None: chunksize = max(min(4, nTasks), min(8, nTasks // (pool._processes**2))) if verbose or VERBOSE_PARALLEL: prefix = '[util_parallel._generate_parallel]' fmtstr = (prefix + 'executing %d %s tasks using %d processes with chunksize=%r') print(fmtstr % (nTasks, get_funcname(func), pool._processes, chunksize)) #import utool as ut #buffered = ut.get_argflag('--buffered') #buffered = False #if buffered: # # current tests indicate that normal pool.imap is faster than buffered # # generation # source_gen = (func(args) for args in args_list) # raw_generator = buffered_generator(source_gen) #else: pmap_func = pool.imap if ordered else pool.imap_unordered raw_generator = pmap_func(func, args_list, chunksize) # Get iterator with or without progress if prog: lbl = '(pargen) %s: ' % (get_funcname(func), ) result_generator = util_progress.ProgressIter( raw_generator, nTotal=nTasks, lbl=lbl, freq=kwargs.get('freq', None), backspace=kwargs.get('backspace', True), adjust=kwargs.get('adjust', False)) else: result_generator = raw_generator if __TIME_GENERATE__: tt = util_time.tic('_generate_parallel func=' + get_funcname(func)) try: # Start generating for result in result_generator: yield result if __EAGER_JOIN__: if USE_GLOBAL_POOL: close_pool(quiet=quiet) else: pool.close() pool.join() except Exception as ex: util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True) if __EAGER_JOIN__: if USE_GLOBAL_POOL: close_pool(quiet=quiet) else: pool.close() pool.join() print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__) if __SERIAL_FALLBACK__: print('Trying to handle error by falling back to serial') serial_generator = _generate_serial(func, args_list, prog=prog, verbose=verbose, nTasks=nTasks, **kwargs) for result in serial_generator: yield result else: raise if __TIME_GENERATE__: util_time.toc(tt)
def log_progress(lbl='Progress: ', nTotal=0, flushfreq=4, startafter=-1, start=True, repl=False, approx=False, disable=False, writefreq=1, with_time=False, backspace=True, pad_stdout=False, wfreq=None, ffreq=None, freq=None, total=None, num=None, with_totaltime=None): """ Returns two functions (mark_progress, end_progress) which will handle logging progress in a for loop. flush frequency must be a multiple of write frequency Args: lbl (str): progress label nTotal (int): flushfreq (int): startafter (int): start (bool): repl (bool): approx (bool): disable (bool): writefreq (int): with_totaltime (bool): backspace (bool): pad_stdout (bool): wfreq (None): alias for write_freq ffreq (None): alias for flush_freq freq (None): alias for flush_freq and write_freq (prefered) total (None): alias for nTotal num (None): alias for nTotal Example: >>> import utool, time >>> from six.moves import range >>> # Define a dummy task >>> spam = 42.0 >>> nTotal = 1000 >>> iter_ = (num for num in range(0, nTotal * 2, 2)) >>> # Create progress functions ... mark_, end_ = utool.log_progress('prog ', nTotal, flushfreq=17) \b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\bprog 0/1000 >>> for count, item in enumerate(iter_): #doctest: +ELLIPSIS ... # Call with enumerate to keep track of a count variable ... time.sleep(.001) ... spam += item + count ... mark_(count) \b...prog 1000/1000 >>> # Mark completion >>> end_() <BLANKLINE> """ # utool.auto_docstr('utool.util_progress', 'log_progress') # python -c "import utool; utool.print_auto_docstr('utool.util_progress', 'log_progress')" # # In reference to above docstr: # I don't completely understand why some of the >>> and ... had to be where # they are, but doctest gets very angry if its not in this format # TODO: Option to display rate of progress # TODO: Option to display eta global AGGROFLUSH # Alias kwargs with simpler names if num is not None: nTotal = num if total is not None: nTotal = total if wfreq is not None: writefreq = wfreq if ffreq is not None: flushfreq = ffreq if freq is not None: writefreq = flushfreq = freq if with_totaltime is not None: with_time = with_totaltime # flush frequency must be a multiple of write frequency flushfreq = max(int(round(flushfreq / writefreq)), 1) * writefreq if nTotal < startafter or disable: # Do not mark progress if only executing a small number of tasks def mark_progress(*args): pass def end_progress(*args): pass return mark_progress, end_progress else: write_fn = PROGRESS_WRITE flush_fn = PROGRESS_FLUSH # build format string for displaying progress fmt_str = progress_str(nTotal, lbl=lbl, repl=repl, approx=approx, backspace=backspace) if AGGROFLUSH: # Progress function which automatically flushes def mark_progress(count, flush_fn=flush_fn): count_ = count + 1 write_fn(fmt_str % (count_)) flush_fn() else: # Progress function flushes every <flushfreq> times def mark_progress(count, fmt_str=fmt_str, flushfreq=flushfreq, writefreq=writefreq, write_fn=write_fn, flush_fn=flush_fn): count_ = count + 1 if count_ % writefreq == 0: write_fn(fmt_str % count_) if count_ % flushfreq == 0: flush_fn() if pad_stdout: write_fn('\n') write_fn('\n') flush_fn() if with_time: tt = util_time.tic(lbl) def end_progress(count_=nTotal, write_fn=write_fn, flush_fn=flush_fn): write_fn(fmt_str % (count_)) write_fn('\n') flush_fn() if with_time: util_time.toc(tt) if pad_stdout: write_fn('\n\n') flush_fn() #mark_progress(0) if start: mark_progress(-1) return mark_progress, end_progress