def help2md(filepath, output='README.md', name='code'): """help2md Converts python help to a .md file. params: - filename - The full path of the input file - output - The full path of the output file ( defaults to README.md ) - name - The name of the file. It puts this at the top of the document. """ document = '#' + name + '\n' c = imp.load_source(name, filepath) doc = inspect.getdoc(c) if doc: document += doc + '\n' else: document += '\n' main_file = getattr(c, '__file__') modules = [] items = dir(c) for i in items: item = getattr(c, i) if inspect.isfunction(item): doc = inspect.getdoc(item) if doc == None: doc = 'No documentation' params = inspect.formatargspec(*inspect.getfullargspec(item)) document += ('\n\n##' + i + '\n```python\ndef ' + i + params + ':\n\t"""' + '\n\t'.join(doc.split('\n')) + '"""\n```') if inspect.isclass(item): doc = inspect.getdoc(item) if doc == None: doc = 'No documentation' document += ('\n\n##' + i + '\n```python\nclass ' + i + '():\n\t"""' + '\n\t'.join(doc.split('\n')) + '"""\n```') methods = dir(item) for m in methods: mitem = getattr(item, m) if inspect.isfunction(mitem): params = inspect.formatargspec( *inspect.getfullargspec(mitem)) doc = inspect.getdoc(mitem) if doc == None: doc = 'No documentation' document += ('\n\n###' + m + '\n```python\n\tdef ' + m + params + ':\n\t\t"""' + '\n\t\t'.join( doc.split('\n')) + '"""\n```') if inspect.ismodule(item): modules.append(i) document += '\n\n# Dependencies\n- ' + '\n- '.join(modules) document += '\n\n***Made with help2md***' with open(output, 'w') as ofile: ofile.write(document) return None
def build_message_from_report(code_builder_map, report_item, force_text=""): if report_item.code not in code_builder_map: return build_default_message_from_report(report_item, force_text) message = code_builder_map[report_item.code] #Sometimes report item info is not needed for message building. #In that case the message is a string. Otherwise the message is a callable. if not callable(message): return message + force_text try: # Object functools.partial cannot be used with inspect because it is not # regular python function. We have to use original function for that. if isinstance(message, partial): keywords = message.keywords if message.keywords is not None else {} args = inspect.getfullargspec(message.func).args del args[:len(message.args)] args = [arg for arg in args if arg not in keywords] else: args = inspect.getfullargspec(message).args if "force_text" in args: return message(report_item.info, force_text) return message(report_item.info) + force_text except(TypeError, KeyError): return build_default_message_from_report(report_item, force_text)
def __sig_from_func(self, func): """Extract function signature, default arguments, keyword-only arguments, and whether or not variable positional or keyword arguments are allowed. This also supports calling unbound instance methods by passing an object instance as the first argument; however, unbound classmethod and staticmethod objects are not callable, so we do not attempt to support them here.""" if isinstance(func, types.MethodType): # A bound instance or class method. argspec = getfullargspec(func.__func__) self.pargl = argspec[0][1:] else: # A regular function, an unbound instance method, or a # bound static method. argspec = getfullargspec(func) self.pargl = argspec[0][:] if argspec[3] is not None: def_offset = len(self.pargl) - len(argspec[3]) self.def_argv = dict((self.pargl[def_offset+i],argspec[3][i]) \ for i in range(len(argspec[3]))) else: self.def_argv = {} self.var_pargs = argspec[1] is not None self.var_kargs = argspec[2] is not None self.kargl = argspec[4] # We need keyword-only arguments' default values too. if argspec[5] is not None: self.def_argv.update(argspec[5])
def _check_methods(self, cls, interface, error_mode): """ Checks that a class implements the methods on an interface. """ cls_methods = self._get_public_methods(cls) interface_methods = self._get_public_methods(interface) for name in interface_methods: if name not in cls_methods: return self._handle_error( MISSING_METHOD % ( self._class_name(cls), name, self._class_name(interface), ), error_mode, ) # Check that the method signatures are the same: cls_argspec = getfullargspec(cls_methods[name]) interface_argspec = getfullargspec(interface_methods[name]) if cls_argspec != interface_argspec: return self._handle_error( BAD_SIGNATURE % ( self._class_name(cls), name, self._class_name(interface), ), error_mode, ) return True
def _function_signatures_match(func1:types.FunctionType, func2:types.FunctionType): """Return True if all args, kwargs, kw-only args, and defaults match""" spec1 = inspect.getfullargspec(func1) spec2 = inspect.getfullargspec(func2) return all(lambda: getattr(spec1, attr) != getattr(spec2, attr) for attr in spec_attrs_to_check)
def decorator(caller, _func=None): """decorator(caller) converts a caller function into a decorator""" if _func is not None: # return a decorated function # this is obsolete behavior; you should use decorate instead return decorate(_func, caller) # else return a decorator function if inspect.isclass(caller): name = caller.__name__.lower() callerfunc = get_init(caller) doc = 'decorator(%s) converts functions/generators into ' \ 'factories of %s objects' % (caller.__name__, caller.__name__) fun = getfullargspec(callerfunc).args[1] # second arg elif inspect.isfunction(caller): if caller.__name__ == '<lambda>': name = '_lambda_' else: name = caller.__name__ callerfunc = caller doc = caller.__doc__ fun = getfullargspec(callerfunc).args[0] # first arg else: # assume caller is an object with a __call__ method name = caller.__class__.__name__.lower() callerfunc = caller.__call__.__func__ doc = caller.__call__.__doc__ fun = getfullargspec(callerfunc).args[1] # second arg evaldict = callerfunc.__globals__.copy() evaldict['_call_'] = caller evaldict['_decorate_'] = decorate return FunctionMaker.create( '%s(%s)' % (name, fun), 'return _decorate_(%s, _call_)' % fun, evaldict, call=caller, doc=doc, module=caller.__module__, __wrapped__=caller)
def __init__(self, cids1=None, cids2=None, forwards=None, backwards=None, labels1=None, labels2=None, **kwargs): # NOTE: we explicitly specify ``cids1`` and ``cids2`` as the two first # arguments for backwards-compatibility with callers that use positional # arguments. if forwards is None and backwards is None: raise TypeError("Must supply either forwards or backwards") self.forwards = forwards self.backwards = backwards # NOTE: the getattr(forwards, 'func', forwards) in the following code # is to make sure that things work properly if the functions are # PartialResult objects. if labels1 is None: if forwards is not None: if isinstance(forwards, types.MethodType): labels1 = getfullargspec(getattr(forwards, 'func', forwards))[0][1:] else: labels1 = getfullargspec(getattr(forwards, 'func', forwards))[0] else: raise ValueError("labels1 needs to be specified if forwards isn't") if labels2 is None: if backwards is not None: if isinstance(backwards, types.MethodType): labels2 = getfullargspec(getattr(backwards, 'func', backwards))[0][1:] else: labels2 = getfullargspec(getattr(backwards, 'func', backwards))[0] else: raise ValueError("labels2 needs to be specified if backwards isn't") super(MultiLink, self).__init__(cids1=cids1, cids2=cids2, **kwargs)
def serve(self): """ Start serving the command registry to the outside world. Send hello and register event callbacks. """ for clazz in PluginRegistry.modules.values(): for mname, method in getmembers(clazz): if ismethod(method) and hasattr(method, "isCommand"): func = mname # Adjust documentation if not method.__help__: raise CommandInvalid(C.make_error("COMMAND_WITHOUT_DOCS", method=func)) doc = re.sub("(\s|\n)+", " ", method.__help__).strip() self.log.debug("registering %s" % func) info = { 'name': func, 'path': "%s.%s" % (clazz.__class__.__name__, mname), 'sig': [] if not getfullargspec(method).args else getfullargspec(method).args, 'target': clazz.get_target(), 'type': getattr(method, "type", NORMAL), 'doc': doc, } if 'self' in info['sig']: info['sig'].remove('self') self.commands[func] = info
def parameters(self): """ Returns list of parameters required for func """ if hasattr(self.func, "_original_"): return inspect.getfullargspec(self.func._original_) return inspect.getfullargspec(self.func)
def decorator(func): # If the function is decorated and uses @functools.wrapper then use the # __wrapped__ attribute to look at the original function signature. # # This allows us to see past the generic *args, **kwargs seen on most decorators. if hasattr(func, '__wrapped__'): args = inspect.getfullargspec(func.__wrapped__)[0] else: args = inspect.getfullargspec(func)[0] args.pop(0) if async_callbacks: if type(async_callbacks) != tuple: raise TypeError('async_callbacks must be a tuple of (keyword for return callback, keyword for error callback)') if len(async_callbacks) != 2: raise ValueError('async_callbacks must be a tuple of (keyword for return callback, keyword for error callback)') args.remove(async_callbacks[0]) args.remove(async_callbacks[1]) if sender_keyword: args.remove(sender_keyword) if rel_path_keyword: args.remove(rel_path_keyword) if path_keyword: args.remove(path_keyword) if destination_keyword: args.remove(destination_keyword) if message_keyword: args.remove(message_keyword) if connection_keyword: args.remove(connection_keyword) if in_signature: in_sig = tuple(Signature(in_signature)) if len(in_sig) > len(args): raise ValueError('input signature is longer than the number of arguments taken') elif len(in_sig) < len(args): raise ValueError('input signature is shorter than the number of arguments taken') func._dbus_is_method = True func._dbus_async_callbacks = async_callbacks func._dbus_interface = dbus_interface func._dbus_in_signature = in_signature func._dbus_out_signature = out_signature func._dbus_sender_keyword = sender_keyword func._dbus_path_keyword = path_keyword func._dbus_rel_path_keyword = rel_path_keyword func._dbus_destination_keyword = destination_keyword func._dbus_message_keyword = message_keyword func._dbus_connection_keyword = connection_keyword func._dbus_args = args func._dbus_get_args_options = dict(byte_arrays=byte_arrays) if is_py2: func._dbus_get_args_options['utf8_strings'] = kwargs.get( 'utf8_strings', False) elif 'utf8_strings' in kwargs: raise TypeError("unexpected keyword argument 'utf8_strings'") return func
def get_arg_spec(self, obj): if inspect.isfunction(obj): arg_spec = getfullargspec(obj) elif inspect.ismethod(obj): arg_spec = getfullargspec(obj) del arg_spec.args[0] # remove first argument else: raise ValueError('Argument is not function and not method') return arg_spec
def create(balancer_type, **kwargs): try: balancer = BALANCER_TYPES[balancer_type.lower()] args = inspect.getfullargspec(balancer.__init__).args params = {k:v for k,v in kwargs.items() if k in args} return balancer(**params) except Exception: logs.hmlog.debug(inspect.getfullargspec(balancer.__init__).args) logs.hmlog.error(format_exc()) return None
def __new__(cls, name, bases, attrs): import inspect if name != 'ExplorationTechniqueCompat': if 'step' in attrs and not inspect.getfullargspec(attrs['step']).defaults: attrs['step'] = cls._step_factory(attrs['step']) if 'filter' in attrs and inspect.getfullargspec(attrs['filter']).args[1] != 'simgr': attrs['filter'] = cls._filter_factory(attrs['filter']) if 'step_state' in attrs and inspect.getfullargspec(attrs['step_state']).args[1] != 'simgr': attrs['step_state'] = cls._step_state_factory(attrs['step_state']) return type.__new__(cls, name, bases, attrs)
def test_getfullargspec(self): # Test preservation of function argument specification. It # actually needs to match that of the adapter function the # prototype of which was supplied via the dummy function. def _adapter(arg1, arg2, *, arg3=None, **kwargs): pass function1a_argspec = inspect.getfullargspec(_adapter) function1d_argspec = inspect.getfullargspec(function1d) self.assertEqual(function1a_argspec, function1d_argspec)
def _get_argspec(f): if sys.version_info[:2] >= (3, 0): # inspect has deprecated getargspec since version 3.0 argspec = inspect.getfullargspec(f) else: try: argspec = inspect.getargspec(f) except ValueError: # this can happen in python 3.5 when # function has keyword-only arguments or annotations argspec = inspect.getfullargspec(f) return argspec
def test_spec_inspect_signature(self): def myfunc(x, y): pass mock = create_autospec(myfunc) mock(1, 2) mock(x=1, y=2) self.assertEqual(inspect.getfullargspec(mock), inspect.getfullargspec(myfunc)) self.assertEqual(mock.mock_calls, [call(1, 2), call(x=1, y=2)]) self.assertRaises(TypeError, mock, 1)
def _filter_baseanalysis_kwargs(function, kwargs): """ create two dictionaries with kwargs separated for function and AnalysisBase Parameters ---------- function : callable function to be called kwargs : dict keyword argument dictionary Returns ------- base_args : dict dictionary of AnalysisBase kwargs kwargs : dict kwargs without AnalysisBase kwargs Raises ------ ValueError : if ``function`` has the same kwargs as ``BaseAnalysis`` """ try: # pylint: disable=deprecated-method base_argspec = inspect.getfullargspec(AnalysisBase.__init__) except AttributeError: # pylint: disable=deprecated-method base_argspec = inspect.getargspec(AnalysisBase.__init__) n_base_defaults = len(base_argspec.defaults) base_kwargs = {name: val for name, val in zip(base_argspec.args[-n_base_defaults:], base_argspec.defaults)} try: # pylint: disable=deprecated-method argspec = inspect.getfullargspec(function) except AttributeError: # pylint: disable=deprecated-method argspec = inspect.getargspec(function) for base_kw in six.iterkeys(base_kwargs): if base_kw in argspec.args: raise ValueError( "argument name '{}' clashes with AnalysisBase argument." "Now allowed are: {}".format(base_kw, base_kwargs.keys())) base_args = {} for argname, default in six.iteritems(base_kwargs): base_args[argname] = kwargs.pop(argname, default) return base_args, kwargs
def test_spec_inspect_signature_annotations(self): def foo(a: int, b: int=10, *, c:int) -> int: return a + b + c mock = create_autospec(foo) mock(1, 2, c=3) mock(1, c=3) self.assertEqual(inspect.getfullargspec(mock), inspect.getfullargspec(foo)) self.assertEqual(mock.mock_calls, [call(1, 2, c=3), call(1, c=3)]) self.assertRaises(TypeError, mock, 1) self.assertRaises(TypeError, mock, 1, 2, 3, c=4)
def get_arg_spec(self, obj): if inspect.isclass(obj): if obj.__init__ is object.__init__: # to avoid an error arg_spec = getfullargspec(lambda self: None) else: arg_spec = getfullargspec(obj.__init__) del arg_spec.args[0] # remove first argument elif hasattr(obj, '__call__'): arg_spec = getfullargspec(obj.__call__) del arg_spec.args[0] # remove first argument else: raise TypeError('Could not determine the signature of ' + str(obj)) return arg_spec
def test_classmethod(): """Test that @decorator works on classmethods.""" @decorator def test_decorator(wrapped, instance, args, kwargs): global state state = 'run' valid_names = {'TestBeforeStaticmethod', 'TestAfterStaticmethod'} assert (instance is None and args[0].__name__ in valid_names or instance.__name__ in valid_names) return wrapped(*args, **kwargs) # --- Decorator before classmethod class TestBeforeStaticmethod: @test_decorator @classmethod def f(cls, a, b): """Return 1.""" return 1 _test_decorated(TestBeforeStaticmethod.f) inst = TestBeforeStaticmethod() _test_decorated(inst.f) # Make sure introspection works assert inspect.getfullargspec(inst.f).args == ['cls', 'a', 'b'] assert inspect.getsource(inst.f) == (' @test_decorator\n' ' @classmethod\n' ' def f(cls, a, b):\n' ' """Return 1."""\n' ' return 1\n') # --- Decorator after staticmethod class TestAfterStaticmethod: @classmethod @test_decorator def f(cls, a, b): """Return 1.""" return 1 _test_decorated(TestAfterStaticmethod.f) inst = TestAfterStaticmethod() _test_decorated(inst.f) # Make sure introspection works assert inspect.getfullargspec(inst.f).args == ['cls', 'a', 'b'] assert inspect.getsource(inst.f) == (' @classmethod\n' ' @test_decorator\n' ' def f(cls, a, b):\n' ' """Return 1."""\n' ' return 1\n')
def test_signature(self): try: from inspect import getfullargspec, getdoc self.assertEqual(getfullargspec(self.cached.__call__), getfullargspec(self.O.__call__)) self.assertEqual(getdoc(self.cached.__call__), getdoc(self.O.__call__)) except ImportError: try: from inspect import signature, getdoc print(signature(self.cached.__call__), signature(self.O.__call__)) self.assertEqual(signature(self.cached.__call__), signature(self.O.__call__)) self.assertEqual(getdoc(self.cached.__call__), getdoc(self.O.__call__)) except ImportError: from inspect import getargspec, getdoc self.assertEqual(getargspec(self.cached.__call__), getargspec(self.O.__call__)) self.assertEqual(getdoc(self.cached.__call__), getdoc(self.O.__call__))
def test_staticmethod(): """Test that @decorator works on staticmethods.""" @decorator def test_decorator(wrapped, instance, args, kwargs): global state state = 'run' assert instance is None return wrapped(*args, **kwargs) # --- Decorator before staticmethod class TestBeforeStaticmethod: @test_decorator @staticmethod def f(a, b): """Return 1.""" return 1 _test_decorated(TestBeforeStaticmethod.f) inst = TestBeforeStaticmethod() _test_decorated(inst.f) # Make sure introspection works assert inspect.getfullargspec(inst.f).args == ['a', 'b'] assert inspect.getsource(inst.f) == (' @test_decorator\n' ' @staticmethod\n' ' def f(a, b):\n' ' """Return 1."""\n' ' return 1\n') # --- Decorator after staticmethod class TestAfterStaticmethod: @staticmethod @test_decorator def f(a, b): """Return 1.""" return 1 _test_decorated(TestAfterStaticmethod.f) inst = TestAfterStaticmethod() _test_decorated(inst.f) # Make sure introspection works assert inspect.getfullargspec(inst.f).args == ['a', 'b'] assert inspect.getsource(inst.f) == (' @staticmethod\n' ' @test_decorator\n' ' def f(a, b):\n' ' """Return 1."""\n' ' return 1\n')
def _get_setters_and_targets(self): """ Get the attribute strings and a full path to where the setter is defined for all setters in an object. """ setters = [] for name in dir(self.o): if not name.startswith('set_'): continue o = getattr(self.o, name) if not six.callable(o): continue if six.PY2: nargs = len(inspect.getargspec(o)[0]) else: nargs = len(inspect.getfullargspec(o)[0]) if nargs < 2: continue func = o if self.is_alias(func): continue source_class = self.o.__module__ + "." + self.o.__name__ for cls in self.o.mro(): if name in cls.__dict__: source_class = cls.__module__ + "." + cls.__name__ break setters.append((name[4:], source_class + "." + name)) return setters
def IsEnabled(): # backwards compatibility until 2015 argspec = inspect.getfullargspec(internal.on_IsEnabled) if len(argspec.args) == 0 or (len(argspec.args) == 1 and argspec.args[0] == 'self'): return internal.on_IsEnabled() else: return internal.on_IsEnabled(GetClock())
def test_wraps(): """ Tests the compatibility replacement for functools.wraps which supports argument preservation across all supported Python versions. """ def foo(a, b, c=1, d=2, e=3, **kwargs): """A test function.""" return a, b, c, d, e, kwargs @wraps(foo) def bar(*args, **kwargs): return ('test',) + foo(*args, **kwargs) expected = ('test', 1, 2, 3, 4, 5, {'f': 6, 'g': 7}) assert bar(1, 2, 3, 4, 5, f=6, g=7) == expected assert bar.__name__ == 'foo' if foo.__doc__ is not None: # May happen if using optimized opcode assert bar.__doc__ == "A test function." if hasattr(foo, '__qualname__'): assert bar.__qualname__ == foo.__qualname__ if six.PY2: argspec = inspect.getargspec(bar) assert argspec.keywords == 'kwargs' else: argspec = inspect.getfullargspec(bar) assert argspec.varkw == 'kwargs' assert argspec.args == ['a', 'b', 'c', 'd', 'e'] assert argspec.defaults == (1, 2, 3)
def wrapper(*args, **kwargs): """ When called returns instance of AttributiveGenerator instead of generator. """ def __iter__(self): return self def send(self): raise NotImplementedError def throw(self): raise NotImplementedError tdict = { '__iter__': __iter__, 'send': send, 'throw': throw,} # use type to create dynamic instance of class AttributiveGenerator #spec = {'__iter__': lambda self: self, 'send': ,} AG = type("AttributiveGenerator", (Generator,), tdict) ag = AG() # create instance so we can inject it into genfunc fargs = inspect.getfullargspec(genfunc).args if fargs and fargs[0] == 'self': gen = genfunc(args[0], ag, *args[1:], **kwargs) else: gen = genfunc(ag, *args, **kwargs) # create generator insert ag ref # now add to class references to gen attributes "duckify" for attr in ('__next__', 'close', 'send', 'throw', 'gi_code', 'gi_frame', 'gi_running', 'gi_yieldfrom'): setattr(AG, attr, getattr(gen, attr)) functools.update_wrapper(wrapper=ag, wrapped=gen) return ag
def getinfo(func): """ Returns an info dictionary containing: - name (the name of the function : str) - argnames (the names of the arguments : list) - defaults (the values of the default arguments : tuple) - signature (the signature : str) - doc (the docstring : str) - module (the module name : str) - dict (the function __dict__ : str) >>> def f(self, x=1, y=2, *args, **kw): pass >>> info = getinfo(f) >>> info["name"] 'f' >>> info["argnames"] ['self', 'x', 'y', 'args', 'kw'] >>> info["defaults"] (1, 2) >>> info["signature"] 'self, x, y, *args, **kw' """ assert inspect.ismethod(func) or inspect.isfunction(func) if sys.version_info[0] >= 3: argspec = inspect.getfullargspec(func) else: argspec = inspect.getargspec(func) regargs, varargs, varkwargs, defaults = argspec[:4] argnames = list(regargs) if varargs: argnames.append(varargs) if varkwargs: argnames.append(varkwargs) signature = inspect.formatargspec( regargs, varargs, varkwargs, defaults, formatvalue=lambda value: "" )[1:-1] # pypy compatibility if hasattr(func, '__closure__'): _closure = func.__closure__ _globals = func.__globals__ else: _closure = func.func_closure _globals = func.func_globals return dict( name=func.__name__, argnames=argnames, signature=signature, defaults=func.__defaults__, doc=func.__doc__, module=func.__module__, dict=func.__dict__, globals=_globals, closure=_closure, )
def menu_decorator(f): """Decorator of a view function that should be included in the menu.""" if isinstance(app, Blueprint): endpoint = app.name + '.' + f.__name__ before_first_request = app.before_app_first_request else: endpoint = f.__name__ before_first_request = app.before_first_request expected = inspect.getfullargspec(f).args if PY3 else \ inspect.getargspec(f).args @before_first_request def _register_menu_item(): # str(path) allows path to be a string-convertible object # that may be useful for delayed evaluation of path item = current_menu.submenu(str(path)) item.register( endpoint, text, order, endpoint_arguments_constructor=endpoint_arguments_constructor, dynamic_list_constructor=dynamic_list_constructor, active_when=active_when, visible_when=visible_when, expected_args=expected, **kwargs) return f
def test_wraps_exclude_names(): """ Test the optional ``exclude_names`` argument to the wraps decorator. """ # This particular test demonstrates wrapping an instance method # as a function and excluding the "self" argument: class TestClass(object): def method(self, a, b, c=1, d=2, **kwargs): return (a, b, c, d, kwargs) test = TestClass() @wraps(test.method, exclude_args=('self',)) def func(*args, **kwargs): return test.method(*args, **kwargs) if six.PY2: argspec = inspect.getargspec(func) else: argspec = inspect.getfullargspec(func) assert argspec.args == ['a', 'b', 'c', 'd'] assert func('a', 'b', e=3) == ('a', 'b', 1, 2, {'e': 3})
def getargspec(func, f): # Check if it's a real bound method or if it's implicitly calling __init__ # (i.e. FooClass(...) and not FooClass.__init__(...) -- the former would # not take 'self', the latter would: try: func_name = getattr(f, '__name__', None) except TypeError: return None try: is_bound_method = ((inspect.ismethod(f) and f.im_self is not None) or (func_name == '__init__' and not func.endswith('.__init__'))) except: # if f is a method from a xmlrpclib.Server instance, func_name == # '__init__' throws xmlrpclib.Fault (see #202) return None try: if PY3: argspec = inspect.getfullargspec(f) else: argspec = inspect.getargspec(f) argspec = list(argspec) fixlongargs(f, argspec) argspec = [func, argspec, is_bound_method] except (TypeError, KeyError): with AttrCleaner(f): argspec = getpydocspec(f, func) if argspec is None: return None if inspect.ismethoddescriptor(f): argspec[1][0].insert(0, 'obj') argspec.append(is_bound_method) return argspec
__author__ = "Davide Micieli" __all__ = [ 'read_tiff', 'read_fits', 'read_fits_stack', 'read_tiff_stack', 'get_rect_coordinates_from_roi', 'read_image', 'read_stack_from_list', 'read_dataset', 'read_image_stack', 'write_fits', 'write_tiff', 'write_tiff_stack', 'write_fits_stack', 'get_rect_coordinates_from_roi', 'get_filename_pattern' ] #logging.basicConfig(level=logging.WARNING) logs = logging.getLogger(__name__) known_ext = ['.tif', '.tiff', '.fits'] # to ensure compatibility with astropy 1.X, clobber option is deprecated and replaced with overwrite if 'clobber' in inspect.getfullargspec(astropy.io.fits.writeto).args: arg_overwrite = 'clobber' else: arg_overwrite = 'overwrite' def get_file_extension(fname): """ This function returns the extension of a given file name or file path """ return os.path.splitext(fname)[1] def get_rect_coordinates_from_roi(fname): """ This function returns the coordinates from a rectangular region of interest
def transcribe_params(self, params=None, **kwargs): if params is None: params = self.params # reference params = params.copy() # don't contaminate DAI params, since we know we use lgbm-xgb as base has_eval_set = self.have_eval_set(kwargs) # only needs (and does) operate at fit-time from catboost import CatBoostClassifier, CatBoostRegressor, EFstrType fullspec_regression = inspect.getfullargspec(CatBoostRegressor) kwargs_regression = {k: v for k, v in zip(fullspec_regression.args, fullspec_regression.defaults)} fullspec_classification = inspect.getfullargspec(CatBoostClassifier) kwargs_classification = {k: v for k, v in zip(fullspec_classification.args, fullspec_classification.defaults)} if self.num_classes == 1: allowed_params = kwargs_regression else: allowed_params = kwargs_classification params_copy = copy.deepcopy(params) for k, v in params_copy.items(): if k not in allowed_params.keys(): del params[k] # now transcribe k = 'boosting_type' if k in params: params[k] = 'Plain' k = 'grow_policy' if k in params: params[k] = 'Depthwise' if params[k] == 'depthwise' else 'Lossguide' k = 'eval_metric' if k in params and params[k].upper() == 'AUC': params[k] = 'AUC' map = {'regression': 'RMSE', 'mse': 'RMSE', 'mae': 'MAE', "mape": 'MAPE', "huber": 'Huber', "fair": 'FairLoss', "rmse": "RMSE", "gamma": "RMSE", # unsupported by catboost "tweedie": "Tweedie", "poisson": "Poisson", "quantile": "Quantile", 'binary': 'Logloss', 'auc': 'AUC', "xentropy": 'CrossEntropy', 'multiclass': 'MultiClass'} k = 'objective' if k in params and params[k] in map.keys(): params[k] = map[params[k]] k = 'eval_metric' if k in params and params[k] in map.keys(): params[k] = map[params[k]] if 'objective' in params: # don't randomly choose these since then model not stable GA -> final # but backup shouldn't really be used AFAIK if params['objective'] == 'Huber': backup = float(config.huber_alpha_list[0]) params['delta'] = params.pop('alpha', backup) if params['objective'] == 'Quantile': backup = float(config.quantile_alpha[0]) params['delta'] = params.pop('alpha', backup) if params['objective'] == 'Tweedie': backup = float(config.tweedie_variance_power_list[0]) params['tweedie_variance_power'] = params.pop('tweedie_variance_power', backup) if params['objective'] == 'FairLoss': backup = float(config.fair_c_list[0]) params['smoothness'] = params.pop('fair_c', backup) params.pop('verbose', None) params.pop('verbose_eval', None) params.pop('logging_level', None) if 'grow_policy' in params: if params['grow_policy'] == 'Lossguide': params.pop('max_depth', None) if params['grow_policy'] == 'Depthwise': params.pop('num_leaves', None) else: params['grow_policy'] = 'SymmetricTree' uses_gpus, n_gpus = self.get_uses_gpus(params) if params['task_type'] == 'CPU': params.pop('grow_policy', None) params.pop('num_leaves', None) params.pop('max_leaves', None) params.pop('min_data_in_leaf', None) params.pop('min_child_samples', None) if params['task_type'] == 'GPU': params.pop('colsample_bylevel', None) # : 0.35 if 'grow_policy' in params and params['grow_policy'] in ['Depthwise', 'SymmetricTree']: if 'max_depth' in params and params['max_depth'] in [0, -1]: params['max_depth'] = max(2, int(np.log(params.get('num_leaves', 2 ** 6)))) else: params.pop('max_depth', None) params.pop('depth', None) if 'grow_policy' in params and params['grow_policy'] == 'Lossguide': # if 'num_leaves' in params and params['num_leaves'] == -1: # params['num_leaves'] = 2 ** params.get('max_depth', 6) if 'max_leaves' in params and params['max_leaves'] in [0, -1]: params['max_leaves'] = 2 ** params.get('max_depth', 6) else: params.pop('max_leaves', None) if 'num_leaves' in params and 'max_leaves' in params: params.pop('num_leaves', None) # apply limits if 'max_leaves' in params: params['max_leaves'] = min(params['max_leaves'], 65536) if 'max_depth' in params: params['max_depth'] = min(params['max_depth'], 16) params.update({'train_dir': user_dir(), 'allow_writing_files': False, 'thread_count': self.params_base.get('n_jobs', 4)}) if 'reg_lambda' in params and params['reg_lambda'] <= 0.0: params['reg_lambda'] = 3.0 # assume meant unset if self._can_handle_categorical: if 'max_cat_to_onehot' in params: params['one_hot_max_size'] = params['max_cat_to_onehot'] params.pop('max_cat_to_onehot', None) if uses_gpus: params['one_hot_max_size'] = min(params.get('one_hot_max_size', 255), 255) else: params['one_hot_max_size'] = min(params.get('one_hot_max_size', 65535), 65535) if 'one_hot_max_size' in params: params['one_hot_max_size'] = max(self._min_one_hot_max_size, params['one_hot_max_size']) params['max_bin'] = params.get('max_bin', 254) if params['task_type'] == 'CPU': params['max_bin'] = min(params['max_bin'], 254) # https://github.com/catboost/catboost/issues/1010 if params['task_type'] == 'GPU': params['max_bin'] = min(params['max_bin'], 127) # https://github.com/catboost/catboost/issues/1010 if uses_gpus: # https://catboost.ai/docs/features/training-on-gpu.html params['devices'] = "%d-%d" % ( self.params_base.get('gpu_id', 0), self.params_base.get('gpu_id', 0) + n_gpus - 1) params['gpu_ram_part'] = 0.3 # per-GPU, assumes GPU locking or no other experiments running if self.num_classes > 2: params.pop("eval_metric", None) params['train_dir'] = self.context.experiment_tmp_dir params['allow_writing_files'] = False # assume during fit self.params_base could have been updated assert 'n_estimators' in params assert 'learning_rate' in params params['n_estimators'] = self.params_base.get('n_estimators', 100) params['learning_rate'] = self.params_base.get('learning_rate', config.min_learning_rate) params['learning_rate'] = min(params['learning_rate'], 0.5) # 1.0 leads to illegal access on GPUs params['learning_rate'] = max(config.min_learning_rate, max(self._min_learning_rate_catboost, params['learning_rate'])) if 'early_stopping_rounds' not in params and has_eval_set: params['early_stopping_rounds'] = 150 # temp fix # assert 'early_stopping_rounds' in params if uses_gpus: params.pop('sampling_frequency', None) if not uses_gpus and params['bootstrap_type'] == 'Poisson': params['bootstrap_type'] = 'Bayesian' # revert to default if uses_gpus and params['bootstrap_type'] == 'MVS': params['bootstrap_type'] = 'Bayesian' # revert to default if 'bootstrap_type' not in params or params['bootstrap_type'] not in ['Poisson', 'Bernoulli']: params.pop('subsample', None) # only allowed for those 2 bootstrap_type settings if params['bootstrap_type'] not in ['Bayesian']: params.pop('bagging_temperature', None) if not (self.num_classes == 2 and params['objective'] == 'Logloss'): params.pop('scale_pos_weight', None) # go back to some default eval_metric if self.num_classes == 1: if 'eval_metric' not in params or params['eval_metric'] not in ['MAE', 'MAPE', 'Poisson', 'Quantile', 'RMSE', 'LogLinQuantile', 'Lq', 'Huber', 'Expectile', 'FairLoss', 'NumErrors', 'SMAPE', 'R2', 'MSLE', 'MedianAbsoluteError']: params['eval_metric'] = 'RMSE' elif self.num_classes == 2: if 'eval_metric' not in params or params['eval_metric'] not in ['Logloss', 'CrossEntropy', 'Precision', 'Recall', 'F1', 'BalancedAccuracy', 'BalancedErrorRate', 'MCC', 'Accuracy', 'CtrFactor', 'AUC', 'NormalizedGini', 'BrierScore', 'HingeLoss', 'HammingLoss', 'ZeroOneLoss', 'Kappa', 'WKappa', 'LogLikelihoodOfPrediction']: params['eval_metric'] = 'Logloss' else: if 'eval_metric' not in params or params['eval_metric'] not in ['MultiClass', 'MultiClassOneVsAll', 'Precision', 'Recall', 'F1', 'TotalF1', 'MCC', 'Accuracy', 'HingeLoss', 'HammingLoss', 'ZeroOneLoss', 'Kappa', 'WKappa', 'AUC']: params['eval_metric'] = 'MultiClass' # set system stuff here params['silent'] = self.params_base.get('silent', True) if config.debug_daimodel_level >= 1: params['silent'] = False # Can enable for tracking improvement in console/dai.log if have access params['random_state'] = self.params_base.get('random_state', 1234) params['thread_count'] = self.params_base.get('n_jobs', max(1, physical_cores_count)) # -1 is not supported return params
def ask_questions(): model_name_question = { 'type': 'input', 'name': 'value', 'message': 'your model name ():', 'input_type': 'name_identifier' } model_name = prompt(model_name_question) task_type_question = { 'type': 'rawlist', 'name': 'value', 'message': 'choose task type', 'choices': task_type_choices } task_type = prompt(task_type_question) network_name_question = { 'type': 'rawlist', 'name': 'value', 'message': 'choose network', 'choices': network_name_choices(task_type) } network_name = prompt(network_name_question) dataset_format_question = { 'type': 'rawlist', 'name': 'value', 'message': 'choose dataset format', 'choices': dataset_format_choices(task_type) } dataset_format = prompt(dataset_format_question) enable_data_augmentation = { 'type': 'confirm', 'name': 'value', 'message': 'enable data augmentation?', 'default': True } train_dataset_path_question = { 'type': 'input', 'name': 'value', 'message': 'training dataset path:', } train_path = prompt(train_dataset_path_question) enable_test_dataset_path_question = { 'type': 'rawlist', 'name': 'value', 'message': 'set validation dataset? \ (if answer no, the dataset will be separated for training and validation by 9:1 ratio.)', 'choices': ['yes', 'no'] } enable_test_dataset_path = prompt(enable_test_dataset_path_question) test_dataset_path_question = { 'type': 'input', 'name': 'value', 'message': 'validation dataset path:', } if enable_test_dataset_path == 'yes': test_path = prompt(test_dataset_path_question) else: test_path = '' batch_size_question = { 'type': 'input', 'name': 'value', 'message': 'batch size (integer):', 'input_type': 'integer', 'default': default_batch_size(task_type), } batch_size = prompt(batch_size_question) image_size_question = { 'type': 'input', 'name': 'value', 'message': 'image size (integer x integer):', 'default': '128x128', "filter": image_size_filter, "validate": generate_image_size_validate(network_name), } image_size = prompt(image_size_question) training_epochs_question = { 'type': 'input', 'name': 'value', 'message': 'how many epochs do you run training (integer):', 'input_type': 'integer', 'default': '100' } training_epochs = prompt(training_epochs_question) training_optimizer_question = { 'type': 'rawlist', 'name': 'value', 'message': 'select optimizer:', 'choices': ['Momentum', 'Adam'], 'default': 'Momentum' } training_optimizer = prompt(training_optimizer_question) initial_learning_rate_value_question = { 'type': 'input', 'name': 'value', 'message': 'initial learning rate:', 'default': '0.001' } initial_learning_rate_value = prompt(initial_learning_rate_value_question) # learning rate schedule learning_rate_schedule_question = { 'type': 'rawlist', 'name': 'value', 'message': 'choose learning rate schedule \ ({epochs} is the number of training epochs you entered before):', 'choices': list(learning_rate_schedule_map.values()), 'default': learning_rate_schedule_map["constant"], } _tmp_learning_rate_schedule = prompt(learning_rate_schedule_question) for key, value in learning_rate_schedule_map.items(): if value == _tmp_learning_rate_schedule: learning_rate_schedule = key if prompt(enable_data_augmentation): all_augmentor = {} checkboxes = [] for name, obj in inspect.getmembers(augmentor): if inspect.isclass(obj) and issubclass(obj, Processor): argspec = inspect.getfullargspec(obj) # ignore self args = argspec.args[1:] defaults = argspec.defaults if len(args) == len(defaults): default_val = [(arg, default) for arg, default in zip(args, defaults)] default_str = " (default: {})".format(", ".join( ["{}={}".format(a, d) for a, d in default_val])) else: defaults = ("# Please fill a value.", ) * ( len(args) - len(defaults)) + defaults default_val = [(arg, default) for arg, default in zip(args, defaults)] default_str = " (**caution**: No default value is provided, \ please modify manually after config exported.)" all_augmentor[name + default_str] = { "name": name, "defaults": default_val } checkboxes.append({"name": name + default_str, "value": name}) data_augmentation_question = { 'type': 'checkbox', 'name': 'value', 'message': 'Please choose augmentors:', 'choices': checkboxes } data_augmentation_res = prompt(data_augmentation_question) data_augmentation = {} if data_augmentation_res: for v in data_augmentation_res: data_augmentation[all_augmentor[v] ["name"]] = all_augmentor[v]["defaults"] quantize_first_convolution_question = { 'type': 'rawlist', 'name': 'value', 'message': 'apply quantization at the first layer?', 'choices': ['yes', 'no'] } quantize_first_convolution = prompt(quantize_first_convolution_question) r = {} for k, v in locals().items(): if k != 'r' and not k.endswith("question"): r[k] = v return r
def build_tower(self, network_def, x_image, y_ref, void_label, n_classes, tower_setup): use_dropout = not tower_setup.is_training gpu_str = "/gpu:" + str(tower_setup.gpu) if tower_setup.is_main_train_tower: print("inputs:", [x_image.get_shape().as_list()]) with tf.device(gpu_str), tf.name_scope("tower_gpu_" + str(tower_setup.gpu)): output_layer = None layers = {} for name, layer_def in network_def.items(): layer_def = layer_def.copy() layer_class = layer_def["class"] if layer_class == "GraphSection": if self.use_partialflow: if self.current_graph_section is not None: self.current_graph_section.__exit__( None, None, None) self.current_graph_section = self.graph_section_manager.new_section( ) self.graph_sections.append(self.current_graph_section) self.current_graph_section.__enter__() # else: # print >> log.v1, "warning, GraphSection defined, but use_partialflow is False. Ignoring sections" continue del layer_def["class"] class_ = get_layer_class(layer_class) spec = inspect.getfullargspec(class_.__init__) args = spec[0] if "from" in layer_def: inputs = sum( [layers[x].outputs for x in layer_def["from"]], []) del layer_def["from"] else: inputs = [x_image] if "concat" in layer_def: concat = sum( [layers[x].outputs for x in layer_def["concat"]], []) layer_def["concat"] = concat if "alternative_labels" in layer_def: layer_def["targets"] = sum([ layers[x].out_labels for x in layer_def["alternative_labels"] ]) layer_def["n_classes"] = 2 del layer_def["alternative_labels"] elif class_.output_layer: layer_def["targets"] = y_ref layer_def["n_classes"] = n_classes if "void_label" in args: layer_def["void_label"] = void_label elif "targets" in args: layer_def["targets"] = y_ref layer_def["name"] = name layer_def["inputs"] = inputs if "dropout" in args and not use_dropout: layer_def["dropout"] = 0.0 if "tower_setup" in args: layer_def["tower_setup"] = tower_setup # check if all args are specified defaults = spec[3] if defaults is None: defaults = [] n_non_default_args = len(args) - len(defaults) non_default_args = args[1:n_non_default_args] # without self for arg in non_default_args: assert arg in layer_def, (name, arg) layer = class_(**layer_def) if tower_setup.is_main_train_tower: print(name, "shape:", [l.get_shape().as_list() for l in layer.outputs]) layers[name] = layer if class_.output_layer: assert output_layer is None, "Currently only 1 output layer is supported" output_layer = layer assert output_layer is not None, "No output layer in network" n = tf.shape(y_ref)[0] assert len(output_layer.outputs) == 1, len(output_layer.outputs) loss, measures, y_softmax = output_layer.loss, output_layer.measures, output_layer.outputs[ 0] regularizers_tower = [] update_ops_tower = [] for l in layers.values(): self.summaries += l.summaries regularizers_tower += l.regularizers update_ops_tower += l.update_ops n_params = sum([l.n_params for l in layers.values()]) return loss, measures, y_softmax, n, n_params, regularizers_tower, update_ops_tower, layers
def test_overridden_by_assumptions_empty(self, **kwargs): func = decorators.overridden_by_assumptions()(self.func) assert func.overridden_by_assumptions == () assert func.__name__ == self.func_name assert getfullargspec(func) == self.func_argspec
def _function_argc(function): args, _, _, _, _, _, _ = inspect.getfullargspec(function) return len(args)
def _create_workflow( self, pipeline_func: Callable, pipeline_name: Text = None, pipeline_description: Text = None, params_list: List[dsl.PipelineParam] = None, pipeline_conf: dsl.PipelineConf = None, ) -> Dict[Text, Any]: """ Internal implementation of create_workflow.""" params_list = params_list or [] argspec = inspect.getfullargspec(pipeline_func) # Create the arg list with no default values and call pipeline function. # Assign type information to the PipelineParam pipeline_meta = _extract_pipeline_metadata(pipeline_func) pipeline_meta.name = pipeline_name or pipeline_meta.name pipeline_meta.description = pipeline_description or pipeline_meta.description pipeline_name = sanitize_k8s_name(pipeline_meta.name) # Need to first clear the default value of dsl.PipelineParams. Otherwise, it # will be resolved immediately in place when being to each component. default_param_values = {} for param in params_list: default_param_values[param.name] = param.value param.value = None # Currently only allow specifying pipeline params at one place. if params_list and pipeline_meta.inputs: raise ValueError( 'Either specify pipeline params in the pipeline function, or in "params_list", but not both.' ) args_list = [] for arg_name in argspec.args: arg_type = None for input in pipeline_meta.inputs or []: if arg_name == input.name: arg_type = input.type break args_list.append( dsl.PipelineParam(sanitize_k8s_name(arg_name, True), param_type=arg_type)) with dsl.Pipeline(pipeline_name) as dsl_pipeline: pipeline_func(*args_list) pipeline_conf = pipeline_conf or dsl_pipeline.conf # Configuration passed to the compiler is overriding. Unfortunately, it's not trivial to detect whether the dsl_pipeline.conf was ever modified. self._validate_exit_handler(dsl_pipeline) self._sanitize_and_inject_artifact(dsl_pipeline, pipeline_conf) # Fill in the default values. args_list_with_defaults = [] if pipeline_meta.inputs: args_list_with_defaults = [ dsl.PipelineParam(sanitize_k8s_name(arg_name, True)) for arg_name in argspec.args ] if argspec.defaults: for arg, default in zip(reversed(args_list_with_defaults), reversed(argspec.defaults)): arg.value = default.value if isinstance( default, dsl.PipelineParam) else default elif params_list: # Or, if args are provided by params_list, fill in pipeline_meta. for param in params_list: param.value = default_param_values[param.name] args_list_with_defaults = params_list pipeline_meta.inputs = [ InputSpec(name=param.name, type=param.param_type, default=param.value) for param in params_list ] op_transformers = [add_pod_env] op_transformers.extend(pipeline_conf.op_transformers) workflow = self._create_pipeline_workflow( args_list_with_defaults, dsl_pipeline, op_transformers, pipeline_conf, ) from ._data_passing_rewriter import fix_big_data_passing workflow = fix_big_data_passing(workflow) import json workflow.setdefault('metadata', {}).setdefault( 'annotations', {})['pipelines.kubeflow.org/pipeline_spec'] = json.dumps( pipeline_meta.to_dict(), sort_keys=True) return workflow
def test_gen_palindromic(self): assert gen_palindromic assert 'n' in getfullargspec(gen_palindromic).args
def test_represent(self): assert represent assert 'n' in getfullargspec(represent).args
def df_apply(df, func, axis=0, raw=False, result_type=None, args=(), dtypes=None, dtype=None, name=None, output_type=None, index=None, elementwise=None, **kwds): """ Apply a function along an axis of the DataFrame. Objects passed to the function are Series objects whose index is either the DataFrame's index (``axis=0``) or the DataFrame's columns (``axis=1``). By default (``result_type=None``), the final return type is inferred from the return type of the applied function. Otherwise, it depends on the `result_type` argument. Parameters ---------- func : function Function to apply to each column or row. axis : {0 or 'index', 1 or 'columns'}, default 0 Axis along which the function is applied: * 0 or 'index': apply function to each column. * 1 or 'columns': apply function to each row. raw : bool, default False Determines if row or column is passed as a Series or ndarray object: * ``False`` : passes each row or column as a Series to the function. * ``True`` : the passed function will receive ndarray objects instead. If you are just applying a NumPy reduction function this will achieve much better performance. result_type : {'expand', 'reduce', 'broadcast', None}, default None These only act when ``axis=1`` (columns): * 'expand' : list-like results will be turned into columns. * 'reduce' : returns a Series if possible rather than expanding list-like results. This is the opposite of 'expand'. * 'broadcast' : results will be broadcast to the original shape of the DataFrame, the original index and columns will be retained. The default behaviour (None) depends on the return value of the applied function: list-like results will be returned as a Series of those. However if the apply function returns a Series these are expanded to columns. output_type : {'dataframe', 'series'}, default None Specify type of returned object. See `Notes` for more details. dtypes : Series, default None Specify dtypes of returned DataFrames. See `Notes` for more details. dtype : numpy.dtype, default None Specify dtype of returned Series. See `Notes` for more details. name : str, default None Specify name of returned Series. See `Notes` for more details. index : Index, default None Specify index of returned object. See `Notes` for more details. elementwise : bool, default False Specify whether ``func`` is an elementwise function: * ``False`` : The function is not elementwise. Mars will try concatenating chunks in rows (when ``axis=0``) or in columns (when ``axis=1``) and then apply ``func`` onto the concatenated chunk. The concatenation step can cause extra latency. * ``True`` : The function is elementwise. Mars will apply ``func`` to original chunks. This will not introduce extra concatenation step and reduce overhead. args : tuple Positional arguments to pass to `func` in addition to the array/series. **kwds Additional keyword arguments to pass as keywords arguments to `func`. Returns ------- Series or DataFrame Result of applying ``func`` along the given axis of the DataFrame. See Also -------- DataFrame.applymap: For elementwise operations. DataFrame.aggregate: Only perform aggregating type operations. DataFrame.transform: Only perform transforming type operations. Notes ----- When deciding output dtypes and shape of the return value, Mars will try applying ``func`` onto a mock DataFrame, and the apply call may fail. When this happens, you need to specify the type of apply call (DataFrame or Series) in output_type. * For DataFrame output, you need to specify a list or a pandas Series as ``dtypes`` of output DataFrame. ``index`` of output can also be specified. * For Series output, you need to specify ``dtype`` and ``name`` of output Series. Examples -------- >>> import numpy as np >>> import mars.tensor as mt >>> import mars.dataframe as md >>> df = md.DataFrame([[4, 9]] * 3, columns=['A', 'B']) >>> df.execute() A B 0 4 9 1 4 9 2 4 9 Using a reducing function on either axis >>> df.apply(np.sum, axis=0).execute() A 12 B 27 dtype: int64 >>> df.apply(np.sum, axis=1).execute() 0 13 1 13 2 13 dtype: int64 Returning a list-like will result in a Series >>> df.apply(lambda x: [1, 2], axis=1).execute() 0 [1, 2] 1 [1, 2] 2 [1, 2] dtype: object Passing ``result_type='expand'`` will expand list-like results to columns of a Dataframe >>> df.apply(lambda x: [1, 2], axis=1, result_type='expand').execute() 0 1 0 1 2 1 1 2 2 1 2 Returning a Series inside the function is similar to passing ``result_type='expand'``. The resulting column names will be the Series index. >>> df.apply(lambda x: md.Series([1, 2], index=['foo', 'bar']), axis=1).execute() foo bar 0 1 2 1 1 2 2 1 2 Passing ``result_type='broadcast'`` will ensure the same shape result, whether list-like or scalar is returned by the function, and broadcast it along the axis. The resulting column names will be the originals. >>> df.apply(lambda x: [1, 2], axis=1, result_type='broadcast').execute() A B 0 1 2 1 1 2 2 1 2 """ if isinstance(func, (list, dict)): return df.aggregate(func) output_types = kwds.pop('output_types', None) object_type = kwds.pop('object_type', None) output_types = validate_output_types(output_type=output_type, output_types=output_types, object_type=object_type) output_type = output_types[0] if output_types else None # calling member function if isinstance(func, str): func = getattr(df, func) sig = inspect.getfullargspec(func) if "axis" in sig.args: kwds["axis"] = axis return func(*args, **kwds) op = ApplyOperand(func=func, axis=axis, raw=raw, result_type=result_type, args=args, kwds=kwds, output_type=output_type, elementwise=elementwise) return op(df, dtypes=dtypes, dtype=dtype, name=name, index=index)
def wrapper(*args): if PY3: arg_spec = inspect.getfullargspec(func) else: arg_spec = inspect.getargspec(func) return func(*args[0:len(arg_spec.args)])
def news_river_api_request(self, bool_operator='OR', language='EN', sortBy='_score', sortOrder='DESC', limit=100): valid_operators = ['AND', 'OR', 'NOT'] valid_country_codes = [] request_url = 'https://api.newsriver.io/v2/search' response = None """ if bool_operator not in valid_operators: pass # raise some exception else: if bool_operator is not 'AND': self.newsriver_keywords.replace('AND', bool_operator.center(len(bool_operator)+2*len(' '), ' ')) if countryCode not in valid_country_codes: pass # raise some exception else: if bool_operator is not 'AND': self.newsriver_keywords.replace('AND', bool_operator) """ payload = {} payload[ 'query'] = self.newsriver_keywords + ' AND ' + 'language:{}'.format( language) arg_spec = inspect.getfullargspec(self.news_river_api_request) for kwargs, vals in zip(arg_spec.args[2:], arg_spec.defaults[1:]): payload[kwargs] = vals params = urllib.parse.urlencode(payload, quote_via=urllib.parse.quote) headers = {} headers["Authorization"] = self.newsriver_token response = requests.get(request_url, headers=headers, params=params) # response.raise_for_status() custom_log(response.request.body) """ try: response = requests.get(request_url, headers=headers, params=params, timeout = (15, 30)) response.raise_for_status() custom_log(response) except ex.HTTPError: pass # handle exception except ex.Timeout: pass # handle exception """ custom_log(response.text) loaded = json.loads(response.text) na = news_agg(response) compiled = na.json_decodecompile() custom_log(compiled)
def test_word_count(self): assert word_count assert 'file' in getfullargspec(word_count).args
def callInvoke(self, obj, methodname, vals, location): methods = inspect.getmembers(obj, callable) if not methods: return ( False, PLambdaException(f'Object not invokable: {obj} {location}')) method = None if not isString(methodname): return (False, PLambdaException( f'Method name not a string: {methodname} {location}')) for (name, value) in methods: if name == methodname: method = value break if method is None: return ( False, PLambdaException(f'No such method: {methodname} {location}')) # Ugliness under python's hood: # Cannot get the argspec of a builtin # http://stackoverflow.com/questions/3276635/how-to-get-the-number-of-args-of-a-built-in-function-in-python # http://stackoverflow.com/questions/990016/how-to-find-out-the-arity-of-a-method-in-python # # Can avoid using inspect for simple cases: # def arity(obj, method): # return getattr(obj.__class__, method).func_code.co_argcount - 1 # remove self # if not isinstance(method, types.BuiltinFunctionType): # 05/08/20 flipped inspect.getargspec(method) to # inspect.getfullargspec(method) without having a clue what I was doing. # but the doc said: "Deprecated since version 3.0: Use # getfullargspec() for an updated API that is usually a drop-in replacement, # but also correctly handles function annotations and keyword-only parameters." argspec = inspect.getfullargspec(method) # string2error(f'argspec({method}) = {argspec}') # if it is an object we have to *not* count 'self', # but if it is a class we need to pass all the args! offset = 0 # if the thing has vargargs (e.g. decorators for example, then we better just try to apply) if (not inspect.ismodule(obj)) and ( not inspect.isclass(obj)) and argspec.varargs is None: offset = 1 ndefaults = len(argspec.defaults) if argspec.defaults else 0 nargs = len(argspec.args) - offset nvals = len(vals) # my guess is that we will need to revisit this. what do we # do when some of the defaults are used but not others? if (nvals < nargs - ndefaults) or (nargs < nvals): msg = f'Arity of {methodname} args {vals} does not match the argspec: {argspec.args[offset:]}. defaults: {argspec.defaults} varargs: {argspec.varargs}' return (False, PLambdaException(msg)) retval = None try: retval = method(*vals) return (True, retval) except Exception as e: return (False, PLambdaException(f'invoke {location} threw {str(e)}'))
readme_table_post = '**POST requests**\n\n' readme_table_post += '| Resource | Argument | Flags | Function |\n| ------ | ------ | ------ | ------ |\n' readme_table_post += '| `admin` | `password_hash` | | Sent a SHA256 hash of the admin password to obtain an admin session |\n' readme_table_get = '**GET requests**\n\n' readme_table_get += '| Resource | Flags | Function |\n| ------ | ------ | ------ |\n' for attr_name in dir(io_functions.InputHandler): attr = getattr(io_functions.InputHandler, attr_name) if hasattr(attr, 'is_api_method'): method_data_for_json = { 'name': attr_name, 'resource': attr_name.replace('_', '/'), 'description': attr.__doc__ } is_post = len(getfullargspec(attr)[0]) > 1 readme_tmp = '| `' + attr_name.replace('_', '/') + '` ' if len(getfullargspec(attr)[0]) > 1: method_arg = getfullargspec(attr)[0][1] readme_tmp += '| `' + method_arg + '` |' method_data_for_json['argument'] = method_arg else: method_arg = None readme_tmp += '|' method_data_for_json['argument'] = None api_list.append(method_data_for_json) for api_flag in api_flags:
def test_is_palindromic(self): assert is_palindromic assert 'n' in getfullargspec(is_palindromic).args
def register_handler( self, call: Callable[[Any, Message], Optional[Response]]) -> None: """Register a handler call. The message type handled by the call is determined by its type annotation. """ # TODO: can use types.GenericAlias in 3.9. from typing import _GenericAlias # type: ignore from typing import Union, get_type_hints, get_args sig = inspect.getfullargspec(call) # The provided callable should be a method taking one 'msg' arg. expectedsig = ['self', 'msg'] if sig.args != expectedsig: raise ValueError(f'Expected callable signature of {expectedsig};' f' got {sig.args}') # Check annotation types to determine what message types we handle. # Return-type annotation can be a Union, but we probably don't # have it available at runtime. Explicitly pull it in. anns = get_type_hints(call, localns={'Union': Union}) msgtype = anns.get('msg') if not isinstance(msgtype, type): raise TypeError( f'expected a type for "msg" annotation; got {type(msgtype)}.') assert issubclass(msgtype, Message) ret = anns.get('return') responsetypes: Tuple[Union[Type[Any], Type[None]], ...] # Return types can be a single type or a union of types. if isinstance(ret, _GenericAlias): targs = get_args(ret) if not all(isinstance(a, type) for a in targs): raise TypeError(f'expected only types for "return" annotation;' f' got {targs}.') responsetypes = targs else: if not isinstance(ret, type): raise TypeError(f'expected one or more types for' f' "return" annotation; got a {type(ret)}.') responsetypes = (ret, ) # Return type of None translates to EmptyResponse. responsetypes = tuple(EmptyResponse if r is type(None) else r for r in responsetypes) # Make sure our protocol has this message type registered and our # return types exactly match. (Technically we could return a subset # of the supported types; can allow this in the future if it makes # sense). registered_types = self._protocol.message_ids_by_type.keys() if msgtype not in registered_types: raise TypeError(f'Message type {msgtype} is not registered' f' in this Protocol.') if msgtype in self._handlers: raise TypeError(f'Message type {msgtype} already has a registered' f' handler.') # Make sure the responses exactly matches what the message expects. if set(responsetypes) != set(msgtype.get_response_types()): raise TypeError( f'Provided response types {responsetypes} do not' f' match the set expected by message type {msgtype}: ' f'({msgtype.get_response_types()})') # Ok; we're good! self._handlers[msgtype] = call
def generate_signature_schema(func, update_keys=None, exclude_keys=frozenset()): """Generate a dictionary to test function signatures with Cerberus' Schema. Parameters ---------- func : function The function used to build the schema. update_keys : dict Keys in here have priority over automatic generation. It can be used to make an argument mandatory, or to use a specific validator. exclude_keys : list-like Keys in here are ignored and not included in the schema. Returns ------- func_schema : dict The dictionary to be used as Cerberus Validator schema. Contains all keyword variables in the function signature as optional argument with the default type as validator. Unit bearing strings are converted. Argument with default None are always accepted. Camel case parameters in the function are converted to underscore style. Examples -------- >>> from cerberus import Validator >>> def f(a, b, camelCase=True, none=None, quantity=3.0*unit.angstroms): ... pass >>> f_dict = generate_signature_schema(f, exclude_keys=['quantity']) >>> print(isinstance(f_dict, dict)) True >>> f_validator = Validator(generate_signature_schema(f)) >>> f_validator.validated({'quantity': '1.0*nanometer'}) {'quantity': Quantity(value=1.0, unit=nanometer)} """ if update_keys is None: update_keys = {} func_schema = {} arg_spec = inspect.getfullargspec(unwrap_py2(func)) args = arg_spec.args defaults = arg_spec.defaults # Check keys that must be excluded from first pass exclude_keys = set(exclude_keys) exclude_keys.update(update_keys) # Transform camelCase to underscore args = [utils.camelcase_to_underscore(arg) for arg in args] # Build schema optional_validator = {'required': False} # Keys are always optional for this type for arg, default_value in zip(args[-len(defaults):], defaults): if arg not in exclude_keys: # User defined keys are added later if default_value is None: # None defaults are always accepted, and considered nullable validator = {'nullable': True} elif isinstance(default_value, unit.Quantity): # Convert unit strings validator = {'coerce': to_unit_coercer(default_value.unit)} else: validator = type_to_cerberus_map(type(default_value)) # Add the argument to the existing schema as a keyword # To the new keyword, add the optional flag and the "validator" flag # of either 'validator' or 'type' depending on how it was processed func_schema = {**func_schema, **{arg: {**optional_validator, **validator}}} # Add special user keys func_schema.update(update_keys) return func_schema
def test_overridden_by_assumptions_triple(self, **kwargs): func = decorators.overridden_by_assumptions( 'a1', 'a2', 'a3')(self.func) assert func.overridden_by_assumptions == ('a1', 'a2', 'a3',) assert func.__name__ == self.func_name assert getfullargspec(func) == self.func_argspec
def get_plugin_init_parameters(self, pluginname): required_parameters = inspect.getfullargspec( util.plugin(pluginname).__init__)
def test_assumes_single(self, **kwargs): func = decorators.assumes('a1')(self.func) assert func.assumptions == ('a1',) assert func.__name__ == self.func_name assert getfullargspec(func) == self.func_argspec
def test_divisors (self): assert divisors assert 'n' in getfullargspec(divisors).args
def run_test_as_given(test): if inspect.isclass(test): # Provide a meaningful error to users, instead of exceptions from # internals that assume we're dealing with a function. raise InvalidArgument("@given cannot be applied to a class.") given_arguments = tuple(_given_arguments) given_kwargs = dict(_given_kwargs) original_argspec = getfullargspec(test) check_invalid = is_invalid_test(test.__name__, original_argspec, given_arguments, given_kwargs) # If the argument check found problems, return a dummy test function # that will raise an error if it is actually called. if check_invalid is not None: return check_invalid # Because the argument check succeeded, we can convert @given's # positional arguments into keyword arguments for simplicity. if given_arguments: assert not given_kwargs for name, strategy in zip(reversed(original_argspec.args), reversed(given_arguments)): given_kwargs[name] = strategy # These have been converted, so delete them to prevent accidental use. del given_arguments argspec = new_given_argspec(original_argspec, given_kwargs) @impersonate(test) @define_function_signature(test.__name__, test.__doc__, argspec) def wrapped_test(*arguments, **kwargs): # Tell pytest to omit the body of this function from tracebacks __tracebackhide__ = True test = wrapped_test.hypothesis.inner_test if getattr(test, "is_hypothesis_test", False): raise InvalidArgument(( "You have applied @given to the test %s more than once, which " "wraps the test several times and is extremely slow. A " "similar effect can be gained by combining the arguments " "of the two calls to given. For example, instead of " "@given(booleans()) @given(integers()), you could write " "@given(booleans(), integers())") % (test.__name__, )) settings = wrapped_test._hypothesis_internal_use_settings random = get_random_for_wrapped_test(test, wrapped_test) # Use type information to convert "infer" arguments into appropriate # strategies. if infer in given_kwargs.values(): hints = get_type_hints(test) for name in [ name for name, value in given_kwargs.items() if value is infer ]: if name not in hints: raise InvalidArgument( "passed %s=infer for %s, but %s has no type annotation" % (name, test.__name__, name)) given_kwargs[name] = st.from_type(hints[name]) processed_args = process_arguments_to_given( wrapped_test, arguments, kwargs, given_kwargs, argspec, test, settings, ) arguments, kwargs, test_runner, search_strategy = processed_args runner = getattr(search_strategy, "runner", None) if isinstance(runner, TestCase) and test.__name__ in dir(TestCase): msg = ("You have applied @given to the method %s, which is " "used by the unittest runner but is not itself a test." " This is not useful in any way." % test.__name__) fail_health_check(settings, msg, HealthCheck.not_a_test_method) if bad_django_TestCase(runner): # pragma: no cover # Covered by the Django tests, but not the pytest coverage task raise InvalidArgument( "You have applied @given to a method on %s, but this " "class does not inherit from the supported versions in " "`hypothesis.extra.django`. Use the Hypothesis variants " "to ensure that each example is run in a separate " "database transaction." % qualname(type(runner))) state = StateForActualGivenExecution( test_runner, search_strategy, test, settings, random, wrapped_test, ) reproduce_failure = wrapped_test._hypothesis_internal_use_reproduce_failure # If there was a @reproduce_failure decorator, use it to reproduce # the error (or complain that we couldn't). Either way, this will # always raise some kind of error. if reproduce_failure is not None: expected_version, failure = reproduce_failure if expected_version != __version__: raise InvalidArgument( ("Attempting to reproduce a failure from a different " "version of Hypothesis. This failure is from %s, but " "you are currently running %r. Please change your " "Hypothesis version to a matching one.") % (expected_version, __version__)) try: state.execute_once( ConjectureData.for_buffer(decode_failure(failure)), print_example=True, is_final=True, ) raise DidNotReproduce( "Expected the test to raise an error, but it " "completed successfully.") except StopTest: raise DidNotReproduce( "The shape of the test data has changed in some way " "from where this blob was defined. Are you sure " "you're running the same test?") except UnsatisfiedAssumption: raise DidNotReproduce( "The test data failed to satisfy an assumption in the " "test. Have you added it since this blob was " "generated?") # There was no @reproduce_failure, so start by running any explicit # examples from @example decorators. execute_explicit_examples(state, wrapped_test, arguments, kwargs) # If there were any explicit examples, they all ran successfully. # The next step is to use the Conjecture engine to run the test on # many different inputs. if not (Phase.reuse in settings.phases or Phase.generate in settings.phases): return try: if isinstance(runner, TestCase) and hasattr(runner, "subTest"): subTest = runner.subTest try: runner.subTest = fake_subTest state.run_engine() finally: runner.subTest = subTest else: state.run_engine() except BaseException as e: # The exception caught here should either be an actual test # failure (or MultipleFailures), or some kind of fatal error # that caused the engine to stop. generated_seed = wrapped_test._hypothesis_internal_use_generated_seed with local_settings(settings): if not (state.failed_normally or generated_seed is None): if running_under_pytest: report( "You can add @seed(%(seed)d) to this test or " "run pytest with --hypothesis-seed=%(seed)d " "to reproduce this failure." % {"seed": generated_seed}) else: report("You can add @seed(%d) to this test to " "reproduce this failure." % (generated_seed, )) # The dance here is to avoid showing users long tracebacks # full of Hypothesis internals they don't care about. # We have to do this inline, to avoid adding another # internal stack frame just when we've removed the rest. # # Using a variable for our trimmed error ensures that the line # which will actually appear in tracebacks is as clear as # possible - "raise the_error_hypothesis_found". the_error_hypothesis_found = e.with_traceback( get_trimmed_traceback()) raise the_error_hypothesis_found # After having created the decorated test function, we need to copy # over some attributes to make the switch as seamless as possible. for attrib in dir(test): if not (attrib.startswith("_") or hasattr(wrapped_test, attrib)): setattr(wrapped_test, attrib, getattr(test, attrib)) wrapped_test.is_hypothesis_test = True if hasattr(test, "_hypothesis_internal_settings_applied"): # Used to check if @settings is applied twice. wrapped_test._hypothesis_internal_settings_applied = True wrapped_test._hypothesis_internal_use_seed = getattr( test, "_hypothesis_internal_use_seed", None) wrapped_test._hypothesis_internal_use_settings = (getattr( test, "_hypothesis_internal_use_settings", None) or Settings.default) wrapped_test._hypothesis_internal_use_reproduce_failure = getattr( test, "_hypothesis_internal_use_reproduce_failure", None) wrapped_test.hypothesis = HypothesisHandle(test) return wrapped_test
def test_leap(self): assert leap assert 'year' in getfullargspec(leap).args
def _parse_signature(func): """Return a signature object for the function. .. deprecated:: 2.0 Will be removed in 2.1 along with utils.bind/validate_arguments """ # if we have a cached validator for this function, return it parse = _signature_cache.get(func) if parse is not None: return parse # inspect the function signature and collect all the information tup = inspect.getfullargspec(func) positional, vararg_var, kwarg_var, defaults = tup[:4] defaults = defaults or () arg_count = len(positional) arguments = [] for idx, name in enumerate(positional): if isinstance(name, list): raise TypeError( "cannot parse functions that unpack tuples in the function signature" ) try: default = defaults[idx - arg_count] except IndexError: param = (name, False, None) else: param = (name, True, default) arguments.append(param) arguments = tuple(arguments) def parse(args, kwargs): new_args = [] missing = [] extra = {} # consume as many arguments as positional as possible for idx, (name, has_default, default) in enumerate(arguments): try: new_args.append(args[idx]) except IndexError: try: new_args.append(kwargs.pop(name)) except KeyError: if has_default: new_args.append(default) else: missing.append(name) else: if name in kwargs: extra[name] = kwargs.pop(name) # handle extra arguments extra_positional = args[arg_count:] if vararg_var is not None: new_args.extend(extra_positional) extra_positional = () if kwargs and kwarg_var is None: extra.update(kwargs) kwargs = {} return ( new_args, kwargs, missing, extra, extra_positional, arguments, vararg_var, kwarg_var, ) _signature_cache[func] = parse return parse
def run(data): # Set up dynamic context needed by a single test run. with local_settings(self.settings): with deterministic_PRNG(): with BuildContext(data, is_final=is_final): # Generate all arguments to the test function. args, kwargs = data.draw(self.search_strategy) if expected_failure is not None: text_repr[0] = arg_string(test, args, kwargs) if print_example or current_verbosity( ) >= Verbosity.verbose: output = StringIO() printer = RepresentationPrinter(output) if print_example: printer.text("Falsifying example:") else: printer.text("Trying example:") if self.print_given_args: printer.text(" ") printer.text(test.__name__) with printer.group(indent=4, open="(", close=""): printer.break_() for v in args: printer.pretty(v) # We add a comma unconditionally because # generated arguments will always be # kwargs, so there will always be more # to come. printer.text(",") printer.breakable() # We need to make sure to print these in the argument order for # Python 2 and older versionf of Python 3.5. In modern versions # this isn't an issue because kwargs is ordered. arg_order = { v: i for i, v in enumerate( getfullargspec(self.test).args) } for i, (k, v) in enumerate( sorted( kwargs.items(), key=lambda t: ( arg_order.get( t[0], float("inf")), t[0], ), )): printer.text(k) printer.text("=") printer.pretty(v) printer.text(",") if i + 1 < len(kwargs): printer.breakable() printer.break_() printer.text(")") printer.flush() report(output.getvalue()) return test(*args, **kwargs)
def export_longformer(model, onnx_model_path, export_padding): input_ids, attention_mask, global_attention_mask = get_dummy_inputs( model.config, export_padding, device=torch.device('cpu')) example_outputs = model(input_ids, attention_mask=attention_mask, global_attention_mask=global_attention_mask) if version.parse(transformers.__version__) < version.parse("4.0.0"): raise RuntimeError("This tool requires transformers 4.0.0 or later.") # Here we replace LongformerSelfAttention.forward using our implmentation for exporting ONNX model from transformers import LongformerSelfAttention import inspect key = ' '.join( inspect.getfullargspec(LongformerSelfAttention.forward).args) args_to_func = { 'self hidden_states attention_mask layer_head_mask is_index_masked is_index_global_attn is_global_attn output_attentions': my_longformer_self_attention_forward_4_3_2, 'self hidden_states attention_mask is_index_masked is_index_global_attn is_global_attn output_attentions': my_longformer_self_attention_forward_4_3, 'self hidden_states attention_mask is_index_masked is_index_global_attn is_global_attn': my_longformer_self_attention_forward_4, } if key not in args_to_func: print("Current arguments", inspect.getfullargspec(LongformerSelfAttention.forward).args) raise RuntimeError( "LongformerSelfAttention.forward arguments are different. Please install supported version (like transformers 4.3.0)." ) # Store for restoring later original_forward = LongformerSelfAttention.forward LongformerSelfAttention.forward = args_to_func[key] example_inputs = (input_ids, attention_mask, global_attention_mask) Path(onnx_model_path).parent.mkdir(parents=True, exist_ok=True) torch.onnx.export( model, example_inputs, onnx_model_path, opset_version=11, example_outputs=example_outputs, input_names=["input_ids", "attention_mask", "global_attention_mask"], output_names=["last_state", "pooler"], dynamic_axes={ 'input_ids': { 0: 'batch_size', 1: 'sequence_length' }, 'attention_mask': { 0: 'batch_size', 1: 'sequence_length' }, 'global_attention_mask': { 0: 'batch_size', 1: 'sequence_length' }, 'last_state': { 0: 'batch_size', 1: 'sequence_length' }, 'pooler': { 0: 'batch_size', 1: 'sequence_length' } }, custom_opsets={"com.microsoft": 1}) print(f"ONNX model exported to {onnx_model_path}") # Restore original implementaiton: LongformerSelfAttention.forward = original_forward
def test_args(self): arg = getfullargspec(q07_randomsearch_predict).args self.assertEqual(len(arg), 1, "Expected argument(s) %d, Given %d" % (1, len(arg)))
def test_nwords(self): assert nwords assert 's' in getfullargspec(nwords).args
def livepatch(old, new, modname=None, visit_stack=(), cache=None, assume_type=None, heed_hook=True): """ Livepatch C{old} with contents of C{new}. If C{old} can't be livepatched, then return C{new}. @param old: The object to be updated @param new: The object used as the source for the update. @type modname: C{str} @param modname: Only livepatch C{old} if it was defined in the given fully-qualified module name. If C{None}, then update regardless of module. @param assume_type: Update as if both C{old} and C{new} were of type C{assume_type}. If C{None}, then C{old} and C{new} must have the same type. For internal use. @param cache: Cache of already-updated objects. Map from (id(old), id(new)) to result. @param visit_stack: Ids of objects that are currently being updated. Used to deal with reference cycles. For internal use. @param heed_hook: If C{True}, heed the C{__livepatch__} hook on C{new}, if any. If C{False}, ignore any C{__livepatch__} hook on C{new}. @return: Either live-patched C{old}, or C{new}. """ if old is new: return new # If we're already visiting this object (due to a reference cycle), then # don't recurse again. if id(old) in visit_stack: return old if cache is None: cache = {} cachekey = (id(old), id(new)) try: return cache[cachekey] except KeyError: pass visit_stack += (id(old), ) def do_livepatch(): new_modname = _get_definition_module(new) if modname and new_modname and new_modname != modname: # Ignore objects that have been imported from another module. # Just update their references. return new if assume_type is not None: use_type = assume_type else: oldtype = type(old) newtype = type(new) if oldtype is newtype: # Easy, common case: Type didn't change. use_type = oldtype elif (oldtype.__name__ == newtype.__name__ and oldtype.__module__ == newtype.__module__ == modname and getattr(sys.modules[modname], newtype.__name__, None) is newtype and oldtype is livepatch(oldtype, newtype, modname=modname, visit_stack=visit_stack, cache=cache)): # Type of this object was defined in this module. This # includes metaclasses defined in the same module. use_type = oldtype else: # If the type changed, then give up. return new try: mro = type.mro(use_type) except TypeError: mro = [use_type, object] # old-style class # Dispatch on type. Include parent classes (in C3 linearized # method resolution order), in particular so that this works on # classes with custom metaclasses that subclass C{type}. for t in mro: try: update = _LIVEPATCH_DISPATCH_TABLE[t] break except KeyError: pass else: # We should have found at least C{object} raise AssertionError("unreachable") # Dispatch. return update(old, new, modname=modname, cache=cache, visit_stack=visit_stack) if heed_hook: hook = (getattr(new, "__livepatch__", None) or getattr(new, "__reload_update__", None)) # XXX if unbound method or a descriptor, then we should ignore it. # XXX test for that. else: hook = None if hook is None: # No hook is defined or the caller instructed us to ignore it. # Do the standard livepatch. result = do_livepatch() else: # Call a hook for updating. # Build dict of optional kwargs. avail_kwargs = dict(old=old, new=new, do_livepatch=do_livepatch, modname=modname, cache=cache, visit_stack=visit_stack) # Find out which optional kwargs the hook wants. kwargs = {} if PY2: argspec = inspect.getargspec(hook) else: argspec = inspect.getfullargspec(hook) argnames = argspec.args if hasattr(hook, "__func__"): # Skip 'self' arg. argnames = argnames[1:] # Pick kwargs that are wanted and available. args = [] kwargs = {} for n in argnames: try: kwargs[n] = avail_kwargs[n] if argspec.keywords if PY2 else argspec.varkw: break except KeyError: # For compatibility, allow first argument to be 'old' with any # name, as long as there's no other arg 'old'. # We intentionally allow this even if the user specified # **kwargs. if not args and not kwargs and 'old' not in argnames: args.append(old) else: # Rely on default being set. If a default isn't set, the # user will get a TypeError. pass if argspec.keywords if PY2 else argspec.varkw: # Use all available kwargs. kwargs = avail_kwargs # Call hook. result = hook(*args, **kwargs) cache[cachekey] = result return result