def _generate_serial(func, args_list, prog=True, verbose=True, nTasks=None, quiet=QUIET, **kwargs): """ internal serial generator """ if nTasks is None: nTasks = len(args_list) if verbose and not quiet: print('[util_parallel._generate_serial] executing %d %s tasks in serial' % (nTasks, get_funcname(func))) prog = prog and verbose and nTasks > 1 # Get iterator with or without progress verbose = verbose or not quiet lbl = '(sergen) %s: ' % (get_funcname(func),) args_iter = ( util_progress.ProgressIter(args_list, nTotal=nTasks, lbl=lbl, freq=kwargs.get('freq', None), adjust=kwargs.get('adjust', False), verbose=verbose) if prog else args_list ) if __TIME_GENERATE__: tt = util_time.tic('_generate_serial func=' + get_funcname(func)) for args in args_iter: result = func(args) yield result if __TIME_GENERATE__: util_time.toc(tt)
def _generate_serial(func, args_list, prog=True, verbose=True, nTasks=None, **kwargs): """ internal serial generator """ if nTasks is None: nTasks = len(args_list) if verbose: print( '[util_parallel._generate_serial] executing %d %s tasks in serial' % (nTasks, get_funcname(func))) prog = prog and verbose and nTasks > 1 # Get iterator with or without progress lbl = '(sergen) %s: ' % (get_funcname(func), ) args_iter = (util_progress.ProgressIter(args_list, nTotal=nTasks, lbl=lbl, freq=kwargs.get('freq', None), adjust=kwargs.get('adjust', False)) if prog else args_list) if __TIME_GENERATE__: tt = util_time.tic('_generate_serial func=' + get_funcname(func)) for args in args_iter: result = func(args) yield result if __TIME_GENERATE__: util_time.toc(tt)
def __argv_flag_dec(func, default=False, quiet=QUIET): flag = meta_util_six.get_funcname(func) if flag.find('no') == 0: flag = flag[2:] flag = '--' + flag.replace('_', '-') def GaurdWrapper(*args, **kwargs): # FIXME: the --print-all is a hack default_ = kwargs.pop('default', default) alias_flags = kwargs.pop('alias_flags', []) is_flagged = (get_argflag(flag, default_) or get_argflag('--print-all') or any([get_argflag(_) for _ in alias_flags])) if is_flagged: indent_lbl = flag.replace('--', '').replace('print-', '') print('') print('\n+ --- ' + indent_lbl + ' ___') #with util_print.Indenter('[%s]' % indent_lbl): ret = func(*args, **kwargs) print('L ___ ' + indent_lbl + '___\n') return ret else: if not quiet: print('\n~~~ %s ~~~' % flag) meta_util_six.set_funcname(GaurdWrapper, meta_util_six.get_funcname(func)) return GaurdWrapper
def process(func, args_list, args_dict={}, force_serial=None, nTasks=None, quiet=QUIET): """ Use ut.generate rather than ut.process Args: func (func): args_list (list or iter): args_dict (dict): force_serial (bool): Returns: result of parallel map(func, args_list) CommandLine: python -m utool.util_parallel --test-process Example: >>> # SLOW_DOCTEST >>> import utool as ut >>> num = 8700 # parallel is slower for smaller numbers >>> flag_generator0 = ut.process(ut.is_prime, list(zip(range(0, num))), force_serial=True) >>> flag_list0 = list(flag_generator0) >>> flag_generator1 = ut.process(ut.is_prime, list(zip(range(0, num))), force_serial=False) >>> flag_list1 = list(flag_generator1) >>> assert flag_list0 == flag_list1 """ if __FORCE_SERIAL__: force_serial = __FORCE_SERIAL__ if FUTURE_ON: raise AssertionError('USE FUTURES') if USE_GLOBAL_POOL: ensure_pool(quiet=quiet) if nTasks is None: nTasks = len(args_list) if __POOL__ == 1 or force_serial: if not quiet: print('[util_parallel] executing %d %s tasks in serial' % (nTasks, get_funcname(func))) result_list = _process_serial(func, args_list, args_dict, nTasks=nTasks, quiet=quiet) else: if __POOL__ is None: pool = new_pool(num_procs=get_default_numprocs(), init_worker=init_worker, maxtasksperchild=None) else: pool = __POOL__ if not quiet: print('[util_parallel] executing %d %s tasks using %d processes' % (nTasks, get_funcname(func), pool._processes)) result_list = _process_parallel(func, args_list, args_dict, nTasks=nTasks, quiet=quiet, pool=pool) return result_list
def wrp_adder(*args, **kwargs): if DEBUG_ADDERS or VERB_CONTROL: print('+------') print('[ADD]: ' + get_funcname(func)) funccall_str = ut.func_str(func, args, kwargs, packed=True) print('\n' + funccall_str + '\n') print('L------') if VERB_CONTROL: print('[ADD]: ' + get_funcname(func)) builtins.print('\n' + ut.func_str(func, args, kwargs) + '\n') return func_(*args, **kwargs)
def wrp_adder(*args, **kwargs): if DEBUG_ADDERS or VERB_CONTROL: print("+------") print("[ADD]: " + get_funcname(func)) funccall_str = ut.func_str(func, args, kwargs, packed=True) print("\n" + funccall_str + "\n") print("L------") if VERB_CONTROL: print("[ADD]: " + get_funcname(func)) builtins.print("\n" + ut.func_str(func, args, kwargs) + "\n") return func_(*args, **kwargs)
def _process_parallel(func, args_list, args_dict={}, nTasks=None): """ Parallel process map Use generate instead """ # Define progress observers if nTasks is None: nTasks = len(args_list) num_tasks_returned_ptr = [0] mark_prog, end_prog = util_progress.progress_func( max_val=nTasks, lbl=get_funcname(func) + ': ') def _callback(result): mark_prog(num_tasks_returned_ptr[0]) sys.stdout.flush() num_tasks_returned_ptr[0] += 1 # Send all tasks to be executed asynconously apply_results = [__POOL__.apply_async(func, args, args_dict, _callback) for args in args_list] # Wait until all tasks have been processed while num_tasks_returned_ptr[0] < nTasks: #print('Waiting: ' + str(num_tasks_returned_ptr[0]) + '/' + str(nTasks)) pass end_prog() # Get the results result_list = [ap.get() for ap in apply_results] if __EAGER_JOIN__: close_pool() return result_list
def profile_withfuncname_filter(func): # Test to see if this function is specified if _profile_func_flag(meta_util_six.get_funcname(func)): if __DEBUG_PROF__: print('profile func %r' % (func,)) return PROFILE_FUNC(func) return func
def checkinfo_wrapper(*args, **kwargs): suggested_fix = '' funcname = get_funcname(func) packagename = funcname.replace('_version', '') pipname_ = pipname if pipname is not None else packagename try: infodict = func(*args, **kwargs) except ImportError as ex: infodict = module_stdinfo_dict(None, name=pipname_) suggested_fix = 'pip install ' + pipname_ if not sys.platform.startswith('win32'): suggested_fix = 'sudo ' + suggested_fix return False, 'None', target, infodict, ut.formatex(ex), suggested_fix except Exception as ex: infodict = module_stdinfo_dict(None, name=pipname_) return False, 'None', target, infodict, ut.formatex(ex), 'Some unknown error in ' + packagename current_version = infodict['__version__'] # Build status text msg = ut.dict_str(infodict, strvals=True) msg += '\n' + '%s: %r >= (target=%r)?' % (funcname, current_version, target) statustext = ut.msgblock(infodict['__name__'], msg) # Check if passed passed = version_ge_target(current_version, target) # Suggest possible fix if not passed: suggested_fix = 'pip install ' + infodict['__name__'] + ' --upgrade' if not sys.platform.startswith('win32'): suggested_fix = 'sudo ' + suggested_fix return passed, current_version, target, infodict, statustext, suggested_fix
def _process_parallel(func, args_list, args_dict={}, nTasks=None, quiet=QUIET, pool=None): """ Parallel process map Use generate instead """ # Define progress observers if nTasks is None: nTasks = len(args_list) lbl = '(parproc) %s: ' % (get_funcname(func),) _prog = util_progress.ProgressIter( range(nTasks), nTotal=nTasks, lbl=lbl, adjust=True) _prog_iter = iter(_prog) num_tasks_returned_ptr = [0] def _callback(result): six.next(_prog_iter) num_tasks_returned_ptr[0] += 1 # Send all tasks to be executed asynconously apply_results = [pool.apply_async(func, args, args_dict, _callback) for args in args_list] # Wait until all tasks have been processed while num_tasks_returned_ptr[0] < nTasks: #print('Waiting: ' + str(num_tasks_returned_ptr[0]) + '/' + str(nTasks)) pass # Get the results result_list = [ap.get() for ap in apply_results] if __EAGER_JOIN__: if USE_GLOBAL_POOL: close_pool(quiet=quiet) else: pool.close() pool.join() return result_list
def profile_withfuncname_filter(func): # Test to see if this function is specified if _profile_func_flag(meta_util_six.get_funcname(func)): if __DEBUG_PROF__: print('profile func %r' % (func, )) return PROFILE_FUNC(func) return func
def wrp_interested(*args, **kwargs): sys.stdout.write('#\n') sys.stdout.write('#\n') sys.stdout.write( '<!INTERESTED>: ' + meta_util_six.get_funcname(func) + '\n') print('INTERESTING... ' + (' ' * 30) + ' <----') return func(*args, **kwargs)
def __argv_flag_dec(func, default=False, quiet=QUIET, indent=False): """ Logic for controlling if a function gets called based on command line """ from utool import util_decor flagname = meta_util_six.get_funcname(func) if flagname.find('no') == 0: flagname = flagname[2:] flags = ( '--' + flagname.replace('_', '-'), '--' + flagname, ) @util_decor.ignores_exc_tb(outer_wrapper=False) def GaurdWrapper(*args, **kwargs): from utool import util_print # FIXME: the --print-all is a hack default_ = kwargs.pop('default', default) alias_flags = kwargs.pop('alias_flags', []) is_flagged = (get_argflag(flags, default_) or get_argflag('--print-all') or any([get_argflag(_) for _ in alias_flags])) if flagname in kwargs: is_flagged = kwargs.pop(flagname) if is_flagged: func_label = flags[0].replace('--', '').replace('print-', '') # print('') print('\n+ --- ' + func_label + ' ___') use_indent = indent is not False if indent is True: indent_ = '[%s]' % func_label else: indent_ = indent with util_print.Indenter(indent_, enabled=use_indent): ret = func(*args, **kwargs) print('L ___ ' + func_label + '___\n') return ret else: PRINT_DISABLED_FLAGDEC = not get_argflag( '--noinform', help_='does not print disabled flag decorators') if not quiet and PRINT_DISABLED_FLAGDEC: #print('\n~~~ %s ~~~' % flag) print('~~~ %s ~~~' % flags[0]) meta_util_six.set_funcname(GaurdWrapper, meta_util_six.get_funcname(func)) return GaurdWrapper
def _inject_funcs(module, *func_list): for func in func_list: if (module is not None and hasattr(module, '__name__') and module.__name__ not in __INJECT_BLACKLIST__ and not module.__name__.startswith('six') and not module.__name__.startswith('sys')): #print('setting: %s.%s = %r' % (module.__name__, meta_util_six.get_funcname(func), func)) setattr(module, meta_util_six.get_funcname(func), func)
def __argv_flag_dec(func, default=False, quiet=QUIET, indent=False): """ Logic for controlling if a function gets called based on command line """ from utool import util_decor flagname = meta_util_six.get_funcname(func) if flagname.find('no') == 0: flagname = flagname[2:] flags = ( '--' + flagname.replace('_', '-'), '--' + flagname, ) @util_decor.ignores_exc_tb(outer_wrapper=False) def GaurdWrapper(*args, **kwargs): from utool import util_print # FIXME: the --print-all is a hack default_ = kwargs.pop('default', default) alias_flags = kwargs.pop('alias_flags', []) is_flagged = (get_argflag(flags, default_) or get_argflag('--print-all') or any([get_argflag(_) for _ in alias_flags])) if is_flagged: func_label = flags[0].replace('--', '').replace('print-', '') print('') print('\n+ --- ' + func_label + ' ___') use_indent = indent is not False if indent is True: indent_ = '[%s]' % func_label else: indent_ = indent with util_print.Indenter(indent_, enabled=use_indent): ret = func(*args, **kwargs) print('L ___ ' + func_label + '___\n') return ret else: PRINT_DISABLED_FLAGDEC = not get_argflag( '--noinform', help_='does not print disabled flag decorators') if not quiet and PRINT_DISABLED_FLAGDEC: #print('\n~~~ %s ~~~' % flag) print('~~~ %s ~~~' % flags[0]) meta_util_six.set_funcname(GaurdWrapper, meta_util_six.get_funcname(func)) return GaurdWrapper
def checkqterr_wrapper(self, *args, **kwargs): try: result = func(self, *args, **kwargs) except Exception as ex: funcname = meta_util_six.get_funcname(func) msg = 'caught exception in %r' % (funcname,) ut.printex(ex, msg, tb=True, pad_stdout=True) raise return result
def tracefunc(func): lbl = '[trace.' + meta_util_six.get_funcname(func) + ']' def wrp_tracefunc(*args, **kwargs): print(lbl + ' +--- ENTER ---') with util_print.Indenter(lbl + ' |'): ret = func(*args, **kwargs) print(lbl + ' L___ EXIT ____') return ret return wrp_tracefunc
def __new__(cls, name, bases, dct): """ Args: cls (type): meta name (str): classname supers (list): bases dct (dict): class dictionary """ method_list = get_comparison_methods() for func in method_list: if get_funcname(func) not in dct: funcname = get_funcname(func) dct[funcname] = func else: funcname = get_funcname(func) dct['meta_' + funcname] = func #ut.inject_func_as_method(metaself, func) return type.__new__(cls, name, bases, dct)
def ex_wrapper(*args, **kwargs): try: return func(*args, **kwargs) except Exception as ex: import utool msg = ('[tools] ERROR: %s(%r, %r)' % (meta_util_six.get_funcname(func), args, kwargs)) #print(msg) utool.printex(ex, msg) #print('[tools] ERROR: %r' % ex) raise
def _generate_parallel(func, args_list, ordered=True, chunksize=1, prog=True, verbose=True, nTasks=None, freq=None): """ Parallel process generator """ prog = prog and verbose if nTasks is None: nTasks = len(args_list) if chunksize is None: chunksize = max(1, nTasks // (__POOL__._processes ** 2)) if verbose: prefix = '[util_parallel._generate_parallel]' fmtstr = prefix + 'executing %d %s tasks using %d processes with chunksize=%r' print(fmtstr % (nTasks, get_funcname(func), __POOL__._processes, chunksize)) pmap_func = __POOL__.imap if ordered else __POOL__.imap_unordered raw_generator = pmap_func(func, args_list, chunksize) # Get iterator with or without progress result_generator = ( util_progress.ProgressIter(raw_generator, nTotal=nTasks, lbl=get_funcname(func) + ': ', freq=freq) if prog else raw_generator ) if __TIME_GENERATE__: tt = util_time.tic('_generate_parallel func=' + get_funcname(func)) try: for result in result_generator: yield result if __EAGER_JOIN__: close_pool() except Exception as ex: util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True) if __EAGER_JOIN__: close_pool() print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__) if __SERIAL_FALLBACK__: print('Trying to handle error by falling back to serial') serial_generator = _generate_serial( func, args_list, prog=prog, verbose=verbose, nTasks=nTasks, freq=freq) for result in serial_generator: yield result else: raise if __TIME_GENERATE__: util_time.toc(tt)
def pyqtSlotWrapper(func): #printDBG('[GUITOOL._SLOT] Wrapping: %r' % func.__name__) funcname = meta_util_six.get_funcname(func) @QtCore.pyqtSlot(*types, name=funcname) @ut.ignores_exc_tb def slot_wrapper(self, *args, **kwargs): result = func(self, *args, **kwargs) return result slot_wrapper = functools.update_wrapper(slot_wrapper, func) return slot_wrapper
def __new__(cls, name, bases, dct): """ cls - meta name - classname supers - bases dct - class dictionary """ #assert 'get_cfgstr_list' in dct, ( # 'must have defined get_cfgstr_list. name=%r' % (name,)) # Inject registered function for func in methods_list: if get_funcname(func) not in dct: funcname = get_funcname(func) dct[funcname] = func else: funcname = get_funcname(func) dct['meta_' + funcname] = func #ut.inject_func_as_method(metaself, func) return type.__new__(cls, name, bases, dct)
def process(func, args_list, args_dict={}, force_serial=__FORCE_SERIAL__, nTasks=None): """ Use ut.generate rather than ut.process Args: func (func): args_list (list or iter): args_dict (dict): force_serial (bool): Returns: result of parallel map(func, args_list) CommandLine: python -m utool.util_parallel --test-process Example: >>> # SLOW_DOCTEST >>> import utool as ut >>> num = 8700 # parallel is slower for smaller numbers >>> flag_generator0 = ut.process(ut.is_prime, zip(range(0, num)), force_serial=True) >>> flag_list0 = list(flag_generator0) >>> flag_generator1 = ut.process(ut.is_prime, zip(range(0, num)), force_serial=False) >>> flag_list1 = list(flag_generator1) >>> assert flag_list0 == flag_list1 """ ensure_pool() if nTasks is None: nTasks = len(args_list) if __POOL__ == 1 or force_serial: if not QUIET: print('[util_parallel] executing %d %s tasks in serial' % (nTasks, get_funcname(func))) result_list = _process_serial(func, args_list, args_dict, nTasks=nTasks) else: if not QUIET: print('[util_parallel] executing %d %s tasks using %d processes' % (nTasks, get_funcname(func), __POOL__._processes)) result_list = _process_parallel(func, args_list, args_dict, nTasks=nTasks) return result_list
def wrp_setter(*args, **kwargs): if DEBUG_SETTERS or VERB_CONTROL: print('+------') print('[SET]: ' + get_funcname(func)) print('[SET]: called by: ' + ut.get_caller_name(range(1, 7))) funccall_str = ut.func_str(func, args, kwargs, packed=True) print('\n' + funccall_str + '\n') print('L------') #builtins.print('\n' + funccall_str + '\n') #print('set: funcname=%r, args=%r, kwargs=%r' % (get_funcname(func), args, kwargs)) return func_(*args, **kwargs)
def wrp_setter(*args, **kwargs): if DEBUG_SETTERS or VERB_CONTROL: print("+------") print("[SET]: " + get_funcname(func)) print("[SET]: called by: " + ut.get_caller_name(range(1, 7))) funccall_str = ut.func_str(func, args, kwargs, packed=True) print("\n" + funccall_str + "\n") print("L------") # builtins.print('\n' + funccall_str + '\n') # print('set: funcname=%r, args=%r, kwargs=%r' % (get_funcname(func), args, kwargs)) return func_(*args, **kwargs)
def wrp_getter(*args, **kwargs): #if ut.DEBUG: # print('[IN GETTER] args=%r' % (args,)) # print('[IN GETTER] kwargs=%r' % (kwargs,)) if DEBUG_GETTERS or VERB_CONTROL: print('+------') print('[GET]: ' + get_funcname(func)) funccall_str = ut.func_str(func, args, kwargs, packed=True) print('\n' + funccall_str + '\n') print('L------') return func_(*args, **kwargs)
def wrp_getter(*args, **kwargs): # if ut.DEBUG: # print('[IN GETTER] args=%r' % (args,)) # print('[IN GETTER] kwargs=%r' % (kwargs,)) if DEBUG_GETTERS or VERB_CONTROL: print("+------") print("[GET]: " + get_funcname(func)) funccall_str = ut.func_str(func, args, kwargs, packed=True) print("\n" + funccall_str + "\n") print("L------") return func_(*args, **kwargs)
def _generate_serial(func, args_list, prog=True, verbose=True, nTasks=None, freq=None): """ internal serial generator """ if nTasks is None: nTasks = len(args_list) if verbose: print('[util_parallel._generate_serial] executing %d %s tasks in serial' % (nTasks, get_funcname(func))) prog = prog and verbose and nTasks > 1 # Get iterator with or without progress args_iter = ( util_progress.ProgressIter(args_list, nTotal=nTasks, lbl=get_funcname(func) + ': ', freq=freq) if prog else args_list ) if __TIME_GENERATE__: tt = util_time.tic('_generate_serial func=' + get_funcname(func)) for args in args_iter: result = func(args) yield result if __TIME_GENERATE__: util_time.toc(tt)
def inject_func_as_method(self, func, method_name=None, class_=None, allow_override=False, allow_main=False): """ Injects a function into an object as a method Wraps func as a bound method of self. Then injects func into self It is preferable to use make_class_method_decorator and inject_instance Args: self (object): class instance func : some function whos first arugment is a class instance method_name (str) : default=func.__name__, if specified renames the method class_ (type) : if func is an unbound method of this class References: http://stackoverflow.com/questions/1015307/python-bind-an-unbound-method """ if method_name is None: method_name = get_funcname(func) #printDBG('Injecting method_name=%r' % method_name) old_method = getattr(self, method_name, None) #import utool as ut #ut.embed() # Bind function to the class instance #new_method = types.MethodType(func, self, self.__class__) new_method = func.__get__(self, self.__class__) #new_method = profile(func.__get__(self, self.__class__)) if old_method is not None: if not allow_main and ( old_method.im_func.func_globals['__name__'] != '__main__' and new_method.im_func.func_globals['__name__'] == '__main__'): if True or VERBOSE_CLASS: print('[util_class] skipping re-inject of %r from __main__' % method_name) return if old_method is new_method or old_method.im_func is new_method.im_func: print('WARNING: Injecting the same function twice: %r' % new_method) elif allow_override is False: raise AssertionError('Overrides are not allowed. Already have method_name=%r' % (method_name)) elif allow_override == 'warn': print('WARNING: Overrides are not allowed. Already have method_name=%r. Skipping' % (method_name)) return elif allow_override == 'override+warn': #import utool as ut #ut.embed() print('WARNING: Overrides are allowed, but dangerous. method_name=%r.' % (method_name)) print('old_method = %r, im_func=%s' % (old_method, str(old_method.im_func))) print('new_method = %r, im_func=%s' % (new_method, str(new_method.im_func))) print(old_method.im_func.func_globals['__name__']) print(new_method.im_func.func_globals['__name__']) # TODO: does this actually decrement the refcount enough? del old_method setattr(self, method_name, new_method)
def pyqtSlotWrapper(func): # printDBG('[GUITOOL._SLOT] Wrapping: %r' % func.__name__) funcname = meta_util_six.get_funcname(func) @QtCore.pyqtSlot(*types, name=funcname) @ut.ignores_exc_tb def slot_wrapper(self, *args, **kwargs): result = func(self, *args, **kwargs) return result slot_wrapper = functools.update_wrapper(slot_wrapper, func) return slot_wrapper
def inject_func_as_property(self, func, method_name=None, class_=None): """ WARNING: properties are more safely injected using metaclasses References: http://stackoverflow.com/questions/13850114/dynamically-adding-methods-with-or-without-metaclass-in-python """ if method_name is None: method_name = get_funcname(func) #new_method = func.__get__(self, self.__class__) new_property = property(func) setattr(self.__class__, method_name, new_property)
def inject_func_as_property(self, func, method_name=None, class_=None, allow_override=False, allow_main=False, verbose=True): """ WARNING: properties are more safely injected using metaclasses References: http://stackoverflow.com/questions/13850114/dynamically-adding-methods-with-or-without-metaclass-in-python """ if method_name is None: method_name = get_funcname(func) #new_method = func.__get__(self, self.__class__) new_property = property(func) setattr(self.__class__, method_name, new_property)
def wrp_onexceptreport(*args, **kwargs): try: #import utool #if utool.DEBUG: # print('[IN EXCPRPT] args=%r' % (args,)) # print('[IN EXCPRPT] kwargs=%r' % (kwargs,)) return func(*args, **kwargs) except Exception as ex: from utool import util_str arg_strs = ', '.join([repr(util_str.truncate_str(str(arg))) for arg in args]) kwarg_strs = ', '.join([util_str.truncate_str('%s=%r' % (key, val)) for key, val in six.iteritems(kwargs)]) msg = ('\nERROR: funcname=%r,\n * args=%s,\n * kwargs=%r\n' % (meta_util_six.get_funcname(func), arg_strs, kwarg_strs)) msg += ' * len(args) = %r\n' % len(args) msg += ' * len(kwargs) = %r\n' % len(kwargs) util_dbg.printex(ex, msg, pad_stdout=True) raise
def indent_func(input_): """ Takes either no arguments or an alias label """ if isinstance(input_, six.string_types): # A label was specified lbl = input_ return _indent_decor(lbl) elif isinstance(input_, (bool, tuple)): # Allow individually turning of of this decorator func = input_ return func else: # Use the function name as the label func = input_ lbl = '[' + meta_util_six.get_funcname(func) + ']' return _indent_decor(lbl)(func)
def _process_serial(func, args_list, args_dict={}, nTasks=None, quiet=QUIET): """ Serial process map Use generate instead """ if nTasks is None: nTasks = len(args_list) result_list = [] lbl = '(serproc) %s: ' % (get_funcname(func),) prog_iter = util_progress.ProgressIter( args_list, nTotal=nTasks, lbl=lbl, adjust=True) # Execute each task sequentially for args in prog_iter: result = func(*args, **args_dict) result_list.append(result) return result_list
def tracefunc_xml(func): """ Causes output of function to be printed in an XML style block """ funcname = meta_util_six.get_funcname(func) def wrp_tracefunc2(*args, **kwargs): verbose = kwargs.get('verbose', True) if verbose: print('<%s>' % (funcname,)) with util_print.Indenter(' '): ret = func(*args, **kwargs) if verbose: print('</%s>' % (funcname,)) return ret wrp_tracefunc2_ = ignores_exc_tb(wrp_tracefunc2) wrp_tracefunc2_ = preserve_sig(wrp_tracefunc2_, func) return wrp_tracefunc2_
def _run_benchmark(setup_, func_list, argstr, number=1000): import timeit setup = GLOBAL_SETUP + '\n' + setup_ print('----------') print('BENCHMARK: ' + utool.get_caller_name()) for func in func_list: funcname = func if isinstance(func, str) else get_funcname(func) print('Running: %s' % funcname) stmt = funcname + argstr try: total_time = timeit.timeit(stmt=stmt, setup=setup, number=number) except ImportError as ex: utool.printex(ex, iswarning=True) except Exception as ex: utool.printex(ex, iswarning=False) raise print(' * timed: %r seconds in %s' % (total_time, funcname)) return locals()
def _process_parallel(func, args_list, args_dict={}, nTasks=None, quiet=QUIET, pool=None): """ Parallel process map Use generate instead """ # Define progress observers if nTasks is None: nTasks = len(args_list) lbl = '(parproc) %s: ' % (get_funcname(func), ) _prog = util_progress.ProgressIter(range(nTasks), nTotal=nTasks, lbl=lbl, adjust=True) _prog_iter = iter(_prog) num_tasks_returned_ptr = [0] def _callback(result): six.next(_prog_iter) num_tasks_returned_ptr[0] += 1 # Send all tasks to be executed asynconously apply_results = [ pool.apply_async(func, args, args_dict, _callback) for args in args_list ] # Wait until all tasks have been processed while num_tasks_returned_ptr[0] < nTasks: #print('Waiting: ' + str(num_tasks_returned_ptr[0]) + '/' + str(nTasks)) pass # Get the results result_list = [ap.get() for ap in apply_results] if __EAGER_JOIN__: if USE_GLOBAL_POOL: close_pool(quiet=quiet) else: pool.close() pool.join() return result_list
def _process_serial(func, args_list, args_dict={}, nTasks=None, quiet=QUIET): """ Serial process map Use generate instead """ if nTasks is None: nTasks = len(args_list) result_list = [] lbl = '(serproc) %s: ' % (get_funcname(func), ) prog_iter = util_progress.ProgressIter(args_list, nTotal=nTasks, lbl=lbl, adjust=True) # Execute each task sequentially for args in prog_iter: result = func(*args, **args_dict) result_list.append(result) return result_list
def _run_benchmark(setup_, func_list, argstr, number=1000): import timeit setup = GLOBAL_SETUP + "\n" + setup_ print("----------") print("BENCHMARK: " + utool.get_caller_name()) for func in func_list: funcname = func if isinstance(func, str) else get_funcname(func) print("Running: %s" % funcname) stmt = funcname + argstr try: total_time = timeit.timeit(stmt=stmt, setup=setup, number=number) except ImportError as ex: utool.printex(ex, iswarning=True) except Exception as ex: utool.printex(ex, iswarning=False) raise print(" * timed: %r seconds in %s" % (total_time, funcname)) return locals()
def execstr_func(func): print(' ! Getting executable source for: ' + meta_util_six.get_funcname(func)) _src = inspect.getsource(func) execstr = textwrap.dedent(_src[_src.find(':') + 1:]) # Remove return statments while True: stmtx = execstr.find('return') # Find first 'return' if stmtx == -1: break # Fail condition # The characters which might make a return not have its own line stmt_endx = len(execstr) - 1 for stmt_break in '\n;': print(execstr) print('') print(stmtx) stmt_endx_new = execstr[stmtx:].find(stmt_break) if -1 < stmt_endx_new < stmt_endx: stmt_endx = stmt_endx_new # now have variables stmt_x, stmt_endx before = execstr[:stmtx] after = execstr[stmt_endx:] execstr = before + after return execstr
def make_redirect(func): # PRESERVES ALL SIGNATURES WITH EXECS src_fmt = r''' def {funcname}{defsig}: """ {orig_docstr}""" return {orig_funcname}{callsig} ''' from utool._internal import meta_util_six orig_docstr = meta_util_six.get_funcdoc(func) funcname = meta_util_six.get_funcname(func) orig_funcname = modname.split('.')[-1] + '.' + funcname orig_docstr = '' if orig_docstr is None else orig_docstr import textwrap # Put wrapped function into a scope import inspect argspec = inspect.getargspec(func) (args, varargs, varkw, defaults) = argspec defsig = inspect.formatargspec(*argspec) callsig = inspect.formatargspec(*argspec[0:3]) src_fmtdict = dict(funcname=funcname, orig_funcname=orig_funcname, defsig=defsig, callsig=callsig, orig_docstr=orig_docstr) src = textwrap.dedent(src_fmt).format(**src_fmtdict) return src
def inject_func_as_method(self, func, method_name=None, class_=None, allow_override=False, allow_main=False, verbose=True, override=None, force=False): """ Injects a function into an object as a method Wraps func as a bound method of self. Then injects func into self It is preferable to use make_class_method_decorator and inject_instance Args: self (object): class instance func : some function whos first arugment is a class instance method_name (str) : default=func.__name__, if specified renames the method class_ (type) : if func is an unbound method of this class References: http://stackoverflow.com/questions/1015307/python-bind-an-unbound-method """ if override is not None: # TODO depcirate allow_override allow_override = override if method_name is None: method_name = get_funcname(func) if force: allow_override = True allow_main = True old_method = getattr(self, method_name, None) # Bind function to the class instance #new_method = types.MethodType(func, self, self.__class__) new_method = func.__get__(self, self.__class__) #new_method = profile(func.__get__(self, self.__class__)) if old_method is not None: old_im_func = get_method_func(old_method) new_im_func = get_method_func(new_method) if not allow_main and old_im_func is not None and ( get_funcglobals(old_im_func)['__name__'] != '__main__' and get_funcglobals(new_im_func)['__name__'] == '__main__'): if True or VERBOSE_CLASS: print('[util_class] skipping re-inject of %r from __main__' % method_name) return if old_method is new_method or old_im_func is new_im_func: #if verbose and util_arg.NOT_QUIET: # print('WARNING: Skipping injecting the same function twice: %r' % new_method) #print('WARNING: Injecting the same function twice: %r' % new_method) return elif allow_override is False: raise AssertionError( 'Overrides are not allowed. Already have method_name=%r' % (method_name)) elif allow_override == 'warn': print( 'WARNING: Overrides are not allowed. Already have method_name=%r. Skipping' % (method_name)) return elif allow_override == 'override+warn': #import utool as ut #ut.embed() print('WARNING: Overrides are allowed, but dangerous. method_name=%r.' % (method_name)) print('old_method = %r, im_func=%s' % (old_method, str(old_im_func))) print('new_method = %r, im_func=%s' % (new_method, str(new_im_func))) print(get_funcglobals(old_im_func)['__name__']) print(get_funcglobals(new_im_func)['__name__']) # TODO: does this actually decrement the refcount enough? del old_method setattr(self, method_name, new_method)
def _generate_parallel(func, args_list, ordered=True, chunksize=None, prog=True, verbose=True, quiet=QUIET, nTasks=None, **kwargs): """ Parallel process generator """ global __POOL__ if USE_GLOBAL_POOL: global __POOL__ pool = __POOL__ else: # Maybe global pools are bad? pool = new_pool(num_procs=get_default_numprocs(), init_worker=init_worker, maxtasksperchild=None) #pool = new_pool() prog = prog and verbose if nTasks is None: nTasks = len(args_list) if chunksize is None: chunksize = max(min(4, nTasks), min(8, nTasks // (pool._processes**2))) if verbose or VERBOSE_PARALLEL: prefix = '[util_parallel._generate_parallel]' fmtstr = (prefix + 'executing %d %s tasks using %d processes with chunksize=%r') print(fmtstr % (nTasks, get_funcname(func), pool._processes, chunksize)) #import utool as ut #buffered = ut.get_argflag('--buffered') #buffered = False #if buffered: # # current tests indicate that normal pool.imap is faster than buffered # # generation # source_gen = (func(args) for args in args_list) # raw_generator = buffered_generator(source_gen) #else: pmap_func = pool.imap if ordered else pool.imap_unordered raw_generator = pmap_func(func, args_list, chunksize) # Get iterator with or without progress if prog: lbl = '(pargen) %s: ' % (get_funcname(func), ) result_generator = util_progress.ProgressIter( raw_generator, nTotal=nTasks, lbl=lbl, freq=kwargs.get('freq', None), backspace=kwargs.get('backspace', True), adjust=kwargs.get('adjust', False)) else: result_generator = raw_generator if __TIME_GENERATE__: tt = util_time.tic('_generate_parallel func=' + get_funcname(func)) try: # Start generating for result in result_generator: yield result if __EAGER_JOIN__: if USE_GLOBAL_POOL: close_pool(quiet=quiet) else: pool.close() pool.join() except Exception as ex: util_dbg.printex(ex, 'Parallel Generation Failed!', '[utool]', tb=True) if __EAGER_JOIN__: if USE_GLOBAL_POOL: close_pool(quiet=quiet) else: pool.close() pool.join() print('__SERIAL_FALLBACK__ = %r' % __SERIAL_FALLBACK__) if __SERIAL_FALLBACK__: print('Trying to handle error by falling back to serial') serial_generator = _generate_serial(func, args_list, prog=prog, verbose=verbose, nTasks=nTasks, **kwargs) for result in serial_generator: yield result else: raise if __TIME_GENERATE__: util_time.toc(tt)
def assert_modules(): """ checkinfo functions return info_dict checkinfo_func CommandLine: python -m ibeis.tests.assert_modules --test-assert_modules Example: >>> # DOCTEST_ENABLE >>> from ibeis.tests.assert_modules import * # NOQA >>> detailed_msg = assert_modules() >>> print(detailed_msg) """ MACHINE_NAME = ut.get_computer_name() machine_info_lines = [] machine_info_lines.append('sys.version = %r ' % (sys.version)) machine_info_lines.append('PATH = ' + ut.list_str(ut.get_path_dirs())) machine_info_lines.append('\n\n\n============================') machine_info_lines.append('Begining assert modules main') machine_info_lines.append('* MACHINE_NAME = %r' % MACHINE_NAME) machine_info_text = '\n'.join(machine_info_lines) print(machine_info_text) statustext_list = [] failed_list = [] fix_list = [] SHOW_STATUS = not ut.get_argflag(('--nostatus', '--nostat')) for checkinfo_wrapper in ASSERT_FUNCS: passed, current_version, target, infodict, statustext, suggested_fix = checkinfo_wrapper() funcname = get_funcname(checkinfo_wrapper) if SHOW_STATUS: statustext_list.append(statustext) if passed: statustext_list.append(funcname + ' ' + str(infodict['__version__']) + ' passed') #statustext_list.append('') else: failed_list.append(funcname + ' FAILED!!!') fix_list.append(suggested_fix) statustext_list.append(funcname + ' FAILED!!!') if SHOW_STATUS: statustext_list.append('') output_text = '\n'.join(statustext_list) failed_text = '\n'.join(failed_list) print(output_text) print(failed_text) check_exist_text = check_modules_exists() print(check_exist_text) fix_text = '' if len(fix_list) > 0: fix_text += ('suggested fixes:\n') fix_text += ('\n'.join(fix_list) + '\n') print(fix_text) detailed_msg = '\n'.join([ machine_info_text, output_text, failed_text, check_exist_text, fix_text, ]) return detailed_msg
def prof_wrapper(*args, **kwargs): import utool as ut with ut.Timer(meta_util_six.get_funcname(func)): return func(*args, **kwargs)
def wrp_deleter(*args, **kwargs): if VERB_CONTROL: print('[DELETE]: ' + get_funcname(func)) builtins.print('\n' + ut.func_str(func, args, kwargs) + '\n') return func_(*args, **kwargs)
def process(func, args_list, args_dict={}, force_serial=None, nTasks=None, quiet=QUIET): """ Use ut.generate rather than ut.process Args: func (func): args_list (list or iter): args_dict (dict): force_serial (bool): Returns: result of parallel map(func, args_list) CommandLine: python -m utool.util_parallel --test-process Example: >>> # SLOW_DOCTEST >>> import utool as ut >>> num = 8700 # parallel is slower for smaller numbers >>> flag_generator0 = ut.process(ut.is_prime, list(zip(range(0, num))), force_serial=True) >>> flag_list0 = list(flag_generator0) >>> flag_generator1 = ut.process(ut.is_prime, list(zip(range(0, num))), force_serial=False) >>> flag_list1 = list(flag_generator1) >>> assert flag_list0 == flag_list1 """ if force_serial is None: force_serial = __FORCE_SERIAL__ if USE_GLOBAL_POOL: ensure_pool(quiet=quiet) if nTasks is None: nTasks = len(args_list) if __POOL__ == 1 or force_serial: if not QUIET: print('[util_parallel] executing %d %s tasks in serial' % (nTasks, get_funcname(func))) result_list = _process_serial(func, args_list, args_dict, nTasks=nTasks, quiet=quiet) else: if __POOL__ is None: pool = new_pool(num_procs=get_default_numprocs(), init_worker=init_worker, maxtasksperchild=None) else: pool = __POOL__ if not QUIET: print('[util_parallel] executing %d %s tasks using %d processes' % (nTasks, get_funcname(func), pool._processes)) result_list = _process_parallel(func, args_list, args_dict, nTasks=nTasks, quiet=quiet, pool=pool) return result_list
uuid_ = get_image_uuid(img_bytes_) return uuid_ if __name__ == '__main__': multiprocessing.freeze_support() # win32 test_funcs = [ make_uuid_PIL_bytes, make_uuid_NUMPY_bytes, make_uuid_NUMPY_STRIDE_16_bytes, make_uuid_NUMPY_STRIDE_64_bytes, make_uuid_CONTIG_NUMPY_bytes, make_uuid_CONTIG_NUMPY_STRIDE_16_bytes, make_uuid_CONTIG_NUMPY_STRIDE_64_bytes, ] func_strs = ', '.join([get_funcname(func) for func in test_funcs]) # cool trick setup = 'from __main__ import (gpath, %s) ' % (func_strs, ) number = 10 for func in test_funcs: funcname = get_funcname(func) print('Running: %s' % funcname) if __LINE_PROFILE__: start = time.time() for _ in range(number): func(gpath) total_time = time.time() - start else: import timeit