Пример #1
0
def _generate_serial(func,
                     args_list,
                     prog=True,
                     verbose=True,
                     nTasks=None,
                     **kwargs):
    """ internal serial generator  """
    if nTasks is None:
        nTasks = len(args_list)
    if verbose:
        print(
            '[util_parallel._generate_serial] executing %d %s tasks in serial'
            % (nTasks, get_funcname(func)))
    prog = prog and verbose and nTasks > 1
    # Get iterator with or without progress
    lbl = '(sergen) %s: ' % (get_funcname(func), )
    args_iter = (util_progress.ProgressIter(args_list,
                                            nTotal=nTasks,
                                            lbl=lbl,
                                            freq=kwargs.get('freq', None),
                                            adjust=kwargs.get('adjust', False))
                 if prog else args_list)
    if __TIME_GENERATE__:
        tt = util_time.tic('_generate_serial func=' + get_funcname(func))
    for args in args_iter:
        result = func(args)
        yield result
    if __TIME_GENERATE__:
        util_time.toc(tt)
Пример #2
0
def _generate_serial(func, args_list, prog=True, verbose=True, nTasks=None,
                     quiet=QUIET, **kwargs):
    """ internal serial generator  """
    if nTasks is None:
        nTasks = len(args_list)
    if verbose and not quiet:
        print('[util_parallel._generate_serial] executing %d %s tasks in serial' %
                (nTasks, get_funcname(func)))
    prog = prog and verbose and nTasks > 1
    # Get iterator with or without progress
    verbose = verbose or not quiet
    lbl = '(sergen) %s: ' % (get_funcname(func),)
    args_iter = (
        util_progress.ProgressIter(args_list, nTotal=nTasks,
                                   lbl=lbl,
                                   freq=kwargs.get('freq', None),
                                   adjust=kwargs.get('adjust', False),
                                   verbose=verbose)
        if prog else args_list
    )
    if __TIME_GENERATE__:
        tt = util_time.tic('_generate_serial func=' + get_funcname(func))
    for args in args_iter:
        result = func(args)
        yield result
    if __TIME_GENERATE__:
        util_time.toc(tt)
Пример #3
0
 def end_progress(count_=nTotal, write_fn=write_fn, flush_fn=flush_fn):
     write_fn(fmt_str % (count_))
     write_fn('\n')
     flush_fn()
     if with_time:
         util_time.toc(tt)
     if pad_stdout:
         write_fn('\n\n')
         flush_fn()
Пример #4
0
def _generate_parallel(func, args_list, ordered=True, chunksize=1,
                       prog=True, verbose=True, nTasks=None, freq=None):
    """
    Parallel process generator
    """
    prog = prog and verbose
    if nTasks is None:
        nTasks = len(args_list)
    if chunksize is None:
        chunksize = max(1, nTasks // (__POOL__._processes ** 2))
    if verbose:
        prefix = '[util_parallel._generate_parallel]'
        fmtstr = prefix + 'executing %d %s tasks using %d processes with chunksize=%r'
        print(fmtstr % (nTasks, get_funcname(func), __POOL__._processes, chunksize))
    pmap_func = __POOL__.imap if ordered else __POOL__.imap_unordered
    raw_generator = pmap_func(func, args_list, chunksize)
    # Get iterator with or without progress
    result_generator = (
        util_progress.ProgressIter(raw_generator, nTotal=nTasks, lbl=get_funcname(func) + ': ', freq=freq)
        if prog else raw_generator
    )
    if __TIME_GENERATE__:
        tt = util_time.tic('_generate_parallel func=' + get_funcname(func))
    try:
        for result in result_generator:
            yield result
        if __EAGER_JOIN__:
            close_pool()
    except Exception as ex:
        util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True)
        if __EAGER_JOIN__:
            close_pool()
        print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__)
        if __SERIAL_FALLBACK__:
            print('Trying to handle error by falling back to serial')
            serial_generator = _generate_serial(
                func, args_list, prog=prog, verbose=verbose, nTasks=nTasks, freq=freq)
            for result in serial_generator:
                yield result
        else:
            raise
    if __TIME_GENERATE__:
        util_time.toc(tt)
Пример #5
0
def _generate_serial(func, args_list, prog=True, verbose=True, nTasks=None, freq=None):
    """ internal serial generator  """
    if nTasks is None:
        nTasks = len(args_list)
    if verbose:
        print('[util_parallel._generate_serial] executing %d %s tasks in serial' %
                (nTasks, get_funcname(func)))
    prog = prog and verbose and nTasks > 1
    # Get iterator with or without progress
    args_iter = (
        util_progress.ProgressIter(args_list, nTotal=nTasks, lbl=get_funcname(func) + ': ', freq=freq)
        if prog else args_list
    )
    if __TIME_GENERATE__:
        tt = util_time.tic('_generate_serial func=' + get_funcname(func))
    for args in args_iter:
        result = func(args)
        yield result
    if __TIME_GENERATE__:
        util_time.toc(tt)
Пример #6
0
def _generate_parallel(func, args_list, ordered=True, chunksize=None,
                       prog=True, verbose=True, quiet=QUIET, nTasks=None,
                       **kwargs):
    """
    Parallel process generator
    """
    global __POOL__
    if FUTURE_ON:
        raise AssertionError('USE FUTURES')
    if USE_GLOBAL_POOL:
        global __POOL__
        pool = __POOL__
    else:
        # Maybe global pools are bad?
        pool = new_pool(num_procs=get_default_numprocs(),
                        init_worker=init_worker,
                        maxtasksperchild=None)
        #pool = new_pool()

    prog = prog and verbose
    if nTasks is None:
        nTasks = len(args_list)
    if chunksize is None:
        chunksize = max(min(4, nTasks), min(8, nTasks // (pool._processes ** 2)))
    if verbose or VERBOSE_PARALLEL:
        prefix = '[util_parallel._generate_parallel]'
        fmtstr = (prefix +
                  'executing %d %s tasks using %d processes with chunksize=%r')
        print(fmtstr % (nTasks, get_funcname(func), pool._processes, chunksize))

    #import utool as ut
    #buffered = ut.get_argflag('--buffered')
    #buffered = False
    #if buffered:
    #    # current tests indicate that normal pool.imap is faster than buffered
    #    # generation
    #    source_gen = (func(args) for args in args_list)
    #    raw_generator = buffered_generator(source_gen)
    #else:
    pmap_func = pool.imap if ordered else pool.imap_unordered
    raw_generator = pmap_func(func, args_list, chunksize)

    # Get iterator with or without progress
    if prog:
        verbose = not quiet
        lbl = '(pargen) %s: ' % (get_funcname(func),)
        result_generator = util_progress.ProgressIter(
            raw_generator, nTotal=nTasks, lbl=lbl,
            freq=kwargs.get('freq', None),
            backspace=kwargs.get('backspace', True),
            adjust=kwargs.get('adjust', False),
            verbose=verbose
        )

    else:
        result_generator = raw_generator

    if __TIME_GENERATE__:
        tt = util_time.tic('_generate_parallel func=' + get_funcname(func))
    try:
        # Start generating
        for result in result_generator:
            yield result
        if __EAGER_JOIN__:
            if USE_GLOBAL_POOL:
                close_pool(quiet=quiet)
            else:
                pool.close()
                pool.join()
    except Exception as ex:
        util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True)
        if __EAGER_JOIN__:
            if USE_GLOBAL_POOL:
                close_pool(quiet=quiet)
            else:
                pool.close()
                pool.join()
        # DONT DO SERIAL FALLBACK IN GENERATOR CAN CAUSE ERRORS
        raise
        # print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__)
        # if __SERIAL_FALLBACK__:
        #     print('Trying to handle error by falling back to serial')
        #     serial_generator = _generate_serial(
        #         func, args_list, prog=prog, verbose=verbose, nTasks=nTasks,
        #         **kwargs)
        #     for result in serial_generator:
        #         yield result
        # else:
        #     raise
    if __TIME_GENERATE__:
        util_time.toc(tt)
Пример #7
0
def _generate_parallel(func,
                       args_list,
                       ordered=True,
                       chunksize=None,
                       prog=True,
                       verbose=True,
                       quiet=QUIET,
                       nTasks=None,
                       **kwargs):
    """
    Parallel process generator
    """
    global __POOL__
    if USE_GLOBAL_POOL:
        global __POOL__
        pool = __POOL__
    else:
        # Maybe global pools are bad?
        pool = new_pool(num_procs=get_default_numprocs(),
                        init_worker=init_worker,
                        maxtasksperchild=None)
        #pool = new_pool()

    prog = prog and verbose
    if nTasks is None:
        nTasks = len(args_list)
    if chunksize is None:
        chunksize = max(min(4, nTasks), min(8, nTasks // (pool._processes**2)))
    if verbose or VERBOSE_PARALLEL:
        prefix = '[util_parallel._generate_parallel]'
        fmtstr = (prefix +
                  'executing %d %s tasks using %d processes with chunksize=%r')
        print(fmtstr %
              (nTasks, get_funcname(func), pool._processes, chunksize))

    #import utool as ut
    #buffered = ut.get_argflag('--buffered')
    #buffered = False
    #if buffered:
    #    # current tests indicate that normal pool.imap is faster than buffered
    #    # generation
    #    source_gen = (func(args) for args in args_list)
    #    raw_generator = buffered_generator(source_gen)
    #else:
    pmap_func = pool.imap if ordered else pool.imap_unordered
    raw_generator = pmap_func(func, args_list, chunksize)

    # Get iterator with or without progress
    if prog:
        lbl = '(pargen) %s: ' % (get_funcname(func), )
        result_generator = util_progress.ProgressIter(
            raw_generator,
            nTotal=nTasks,
            lbl=lbl,
            freq=kwargs.get('freq', None),
            backspace=kwargs.get('backspace', True),
            adjust=kwargs.get('adjust', False))

    else:
        result_generator = raw_generator

    if __TIME_GENERATE__:
        tt = util_time.tic('_generate_parallel func=' + get_funcname(func))
    try:
        # Start generating
        for result in result_generator:
            yield result
        if __EAGER_JOIN__:
            if USE_GLOBAL_POOL:
                close_pool(quiet=quiet)
            else:
                pool.close()
                pool.join()
    except Exception as ex:
        util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True)
        if __EAGER_JOIN__:
            if USE_GLOBAL_POOL:
                close_pool(quiet=quiet)
            else:
                pool.close()
                pool.join()
        print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__)
        if __SERIAL_FALLBACK__:
            print('Trying to handle error by falling back to serial')
            serial_generator = _generate_serial(func,
                                                args_list,
                                                prog=prog,
                                                verbose=verbose,
                                                nTasks=nTasks,
                                                **kwargs)
            for result in serial_generator:
                yield result
        else:
            raise
    if __TIME_GENERATE__:
        util_time.toc(tt)