def test_keywordonly_inheritance(self): """ Tests if the decorator deals with inheritance properly. """ kinds_B = { 'self': inspect_sig.Parameter.POSITIONAL_OR_KEYWORD, 'args': inspect_sig.Parameter.VAR_POSITIONAL, 'kwargs': inspect_sig.Parameter.VAR_KEYWORD, 'e': inspect_sig.Parameter.KEYWORD_ONLY, } kinds_A = { 'self': inspect_sig.Parameter.POSITIONAL_OR_KEYWORD, 'a': inspect_sig.Parameter.POSITIONAL_OR_KEYWORD, 'b': inspect_sig.Parameter.POSITIONAL_OR_KEYWORD, 'kwargs': inspect_sig.Parameter.VAR_KEYWORD, 'c': inspect_sig.Parameter.KEYWORD_ONLY, 'd': inspect_sig.Parameter.KEYWORD_ONLY, } sig_B = inspect_sig.signature(self._B.__init__) for param in sig_B.parameters.values(): self.assertTrue(param.kind == kinds_B[param.name]) self.assertEqual(len(sig_B.parameters), len(kinds_B)) sig_A = inspect_sig.signature(self._A.__init__) for param in sig_A.parameters.values(): self.assertTrue(param.kind == kinds_A[param.name]) self.assertEqual(len(sig_A.parameters), len(kinds_A)) with self.assertRaises(TypeError): b = self._B(3, 5, 7, d=2, e=6)
def instantiate(self, definition): item = definition['item'] if hasattr(item, '__ioc_definition__'): definition.update(item.__ioc_definition__) args, kwargs = [], {} is_lambda = definition.get('call_type', None) == FUNCTION_TYPE sig = signature(item) if 'container' in sig.parameters: kwargs['container'] = self.container if 'init' in definition: init = definition['init'] updated_args, updated_kwargs = self.get_args_kwargs(init) args.extend(updated_args) kwargs.update(updated_kwargs) if isfunction(init): sig = signature(init) if 'container' in sig.parameters: kwargs['container'] = self.container init = init(*args, **kwargs) definition['init'] = init if not is_lambda: args, kwargs = self.get_args_kwargs(init) item = item(*args, **kwargs) if is_lambda and isinstance(item, str): # Special case for items that might be retrieved via lambda expressions with suppress(Exception): definition['item'] = self.container.load_item_from_string(item) item, args, kwargs = self.instantiate(definition) return item, args, kwargs
def band_plot(log_likelihood, x_min, x_max, y_min, y_max, n_sigma=1, steps=20, **kwargs): r"""This is an alias for `likelihood_contour` which is present for backward compatibility.""" warnings.warn("The `band_plot` function has been replaced " "by `likelihood_contour` (or " "`likelihood_contour_data` in conjunction with `contour`) " "and might be removed in the future. " "Please update your code.", FutureWarning) valid_args = inspect.signature(likelihood_contour_data).parameters.keys() data_kwargs = {k:v for k,v in kwargs.items() if k in valid_args} if 'pre_calculated_z' not in kwargs: contour_kwargs = likelihood_contour_data(log_likelihood, x_min, x_max, y_min, y_max, n_sigma, steps, **data_kwargs) else: contour_kwargs = {} nx, ny = kwargs['pre_calculated_z'].shape _x = np.linspace(x_min, x_max, nx) _y = np.linspace(y_min, y_max, ny) x, y = np.meshgrid(_x, _y) contour_kwargs['x'] = x contour_kwargs['y'] = y contour_kwargs['z'] = kwargs['pre_calculated_z'] if isinstance(n_sigma, Number): contour_kwargs['levels'] = [delta_chi2(n_sigma, dof=2)] else: contour_kwargs['levels'] = [delta_chi2(n, dof=2) for n in n_sigma] valid_args = inspect.signature(contour).parameters.keys() contour_kwargs.update({k:v for k,v in kwargs.items() if k in valid_args}) contour(**contour_kwargs) return contour_kwargs['x'], contour_kwargs['y'], contour_kwargs['z']
def _quote_rpc(self, function, loc): if isinstance(function, SpecializedFunction): host_function = function.host_function else: host_function = function ret_type = builtins.TNone() if isinstance(host_function, pytypes.BuiltinFunctionType): pass elif (isinstance(host_function, pytypes.FunctionType) or \ isinstance(host_function, pytypes.MethodType)): if isinstance(host_function, pytypes.FunctionType): signature = inspect.signature(host_function) else: # inspect bug? signature = inspect.signature(host_function.__func__) if signature.return_annotation is not inspect.Signature.empty: ret_type = self._extract_annot(host_function, signature.return_annotation, "return type", loc, is_syscall=False) else: assert False function_type = types.TRPC(ret_type, service=self.embedding_map.store_object(host_function)) self.functions[function] = function_type return function_type
def checkParam(self, params: list): if not hasattr(self.__class__, "begin") or not hasattr(self.__class__, "end"): return False sbegin = inspect.signature(self.begin) send = inspect.signature(self.end) idx = 0 for param in list(sbegin.parameters.values())[1:]: if idx >= len(params) and param.default is inspect.Parameter.empty: raise RuntimeError( "{}: No parameter given to begin" " method for argument {}, expected {}".format(self.__class__.__name__, idx, param.annotation) ) elif idx < len(params) and not isinstance(params[idx], param.annotation): raise TypeError( "{}: Wrong parameter in begin method parameter {} " "expected {} got {}".format(self.__class__.__name__, idx, type(params[idx]), param.annotation) ) idx += 1 idx = 0 for param in list(send.parameters.values())[1:]: if idx >= len(params) and param.default is inspect.Parameter.empty: raise RuntimeError( "{}: No parameter given to end" " method for argument {}, expected {}".format(self.__class__.__name__, idx, param.annotation) ) elif idx < len(params) and not isinstance(params[idx], param.annotation): raise TypeError( "{}: Wrong parameter in end method parameter {} " "expected {} got {}".format(self.__class__.__name__, idx, type(params[idx]), param.annotation) ) idx += 1 return True
def __init__(self, save_path: str = None, load_path: str = None, mode: str = None, **kwargs): # Calls parent constructor. Results in creation of save_folder if it doesn't exist super().__init__(save_path=save_path, load_path=load_path, mode=mode) # Dicts are mutable! To prevent changes in config dict outside this class # we use deepcopy opt = copy.deepcopy(kwargs) # Finds all input parameters of the network __init__ to pass them into network later network_parameter_names = list(inspect.signature(CharacterTagger.__init__).parameters) # Fills all provided parameters from opt (opt is a dictionary formed from the model # json config file, except the "name" field) network_parameters = {par: opt[par] for par in network_parameter_names if par in opt} self._net = CharacterTagger(**network_parameters) # Finds all parameters for network train to pass them into train method later train_parameters_names = list(inspect.signature(self._net.train_on_batch).parameters) # Fills all provided parameters from opt train_parameters = {par: opt[par] for par in train_parameters_names if par in opt} self.train_parameters = train_parameters self.opt = opt # Tries to load the model from model `load_path`, if it is available self.load()
def arg_checker(func, *args, **kwargs): try: signature(func).bind(*args, **kwargs) except TypeError: return False else: return True
def has_arg(fn, arg_name): """Checks if a callable accepts a given keyword argument. Args: fn: callable to inspect arg_name: string, keyword argument name to check Returns: bool, whether `fn` accepts a `arg_name` keyword argument. """ if sys.version_info < (3,): if isinstance(fn, types.FunctionType) or isinstance(fn, types.MethodType): arg_spec = inspect.getargspec(fn) else: try: arg_spec = inspect.getargspec(fn.__call__) except AttributeError: return False return (arg_name in arg_spec.args) elif sys.version_info < (3, 6): arg_spec = inspect.getfullargspec(fn) return (arg_name in arg_spec.args or arg_name in arg_spec.kwonlyargs) else: try: signature = inspect.signature(fn) except ValueError: # handling Cython signature = inspect.signature(fn.__call__) parameter = signature.parameters.get(arg_name) if parameter is None: return False return (parameter.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY))
def _explicitize_args(func): # Python 2 if hasattr(func, 'func_code'): varnames = func.func_code.co_varnames # Python 3 else: varnames = func.__code__.co_varnames def wrapper(*args, **kwargs): if '_explicit_args' in kwargs.keys(): raise Exception('Variable _explicit_args should not be set.') kwargs['_explicit_args'] = \ list( set( list(varnames[:len(args)]) + [k for k, _ in kwargs.items()] ) ) if 'self' in kwargs['_explicit_args']: kwargs['_explicit_args'].remove('self') return func(*args, **kwargs) # If Python 3, we can set the function signature to be correct if hasattr(inspect, 'signature'): # pylint: disable=no-member new_sig = inspect.signature(wrapper).replace( parameters=inspect.signature(func).parameters.values() ) wrapper.__signature__ = new_sig return wrapper
def dispatch(instance, *args, **kwargs): for fun in self: sig = inspect.signature(fun) try: # check if signatures match - number and names of arguments work bound = sig.bind(instance, *args, **kwargs) bound_with_annotation = { arg: [bound.arguments[arg], sig.parameters[arg].annotation] for arg in bound.arguments.keys() } try: # see if annotations are used, if so use them to convert arguments to the expected type for arg, (value, annotation) in bound_with_annotation.items(): if (callable(annotation) and not annotation is inspect.Signature.empty): bound.arguments[args] = annotation(value) # found a proper match! return fun(*bound.args, **bound.kwargs) except ValueError as e: # annotations used, argument types mismatch - try the next function pass except TypeError as e: # signatures don't match - try the next function pass raise ValueError( "no method found to handle passed arguments {} {} {}\n{}".format( instance, args, kwargs, "\n".join(str(inspect.signature(fun)) for fun in self) ) )
def method(self, command=None): """ Dump keyboard shortcuts to console. At some point, make this to use dialog window instead. :return: None """ if command: # command is probably KatajaAction's run_command -method. # then we want its 'method' -method's docstring. found = False for cls in inspect.getmro(command.__self__.__class__): if cls.__dict__.get('method', None): # print('----------------------------') print('<b>' + cls.k_action_uid + str(inspect.signature(cls.method)) + '</b>') print(f'<i>""" {cls.method.__doc__.replace(" ", " ")} """</i>') found = True break if not found: print(command.__doc__) else: d = ctrl.ui.actions keys = sorted(list(d.keys())) print('---------- Available actions ----------') for key in keys: my_class = d[key].__class__ command = getattr(my_class, "k_command", "") sig = str(inspect.signature(my_class.method)) if sig.startswith('(self, '): sig = '(' + sig[7:] elif sig == '(self)': sig = '()' print(f'<b>{key + sig:.<60}</b> : {command}') print('---------------------------------------') print('<b>help(method_name)</b> for more information.')
def decorated(*args, **kwargs): for param, arg in zip(inspect.signature(f).parameters.values(), args): if param.name not in kwargs: kwargs[param.name] = arg decoded_args = {} for param in inspect.signature(f).parameters.values(): arg = kwargs[param.name] if param.kind is not inspect.Parameter.POSITIONAL_OR_KEYWORD: raise ValueError('The decode_args function only works for POSITIONAL_OR_KEYWORD parameters, but a {} parameter was found'.format(param.kind)) if param.annotation is inspect.Parameter.empty or not isinstance(arg, str): # no annotation or a direct function call decoded_args[param.name] = arg elif param.annotation is Dimension: try: int(arg) except: decoded_args[param.name] = Dimension[arg] else: decoded_args[param.name] = Dimension(int(arg)) elif param.annotation is Player: decoded_args[param.name] = Player(arg) elif param.annotation is int: decoded_args[param.name] = int(arg) elif param.annotation is minecraft.World: decoded_args[param.name] = minecraft.World(arg) elif param.annotation == 'color': decoded_args[param.name] = (int(arg[:2], 16), int(arg[2:4], 16), int(arg[4:6], 16)) elif isinstance(param.annotation, range): if int(arg) not in param.annotation: bottle.abort(403, 'Parameter {} must be in {}'.format(param.name, param.annotation)) decoded_args[param.name] = int(arg) else: raise TypeError('The decode_args function is not implemented for the argument type {:?}'.format(param.annotation)) return f(**decoded_args)
def signature_argspec(f): from inspect import signature, Parameter, _empty try: if NO_ARGSPEC: sig = signature(f, follow_wrapped=False) else: sig = signature(f) except ValueError: raise TypeError("unsupported callable") args = list( k for k, v in sig.parameters.items() if v.kind in (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD) ) varargs = None keywords = None for k, v in sig.parameters.items(): if v.kind == Parameter.VAR_POSITIONAL: varargs = k elif v.kind == Parameter.VAR_KEYWORD: keywords = k defaults = [] for a in reversed(args): default = sig.parameters[a].default if default is _empty: break else: defaults.append(default) if defaults: defaults = tuple(reversed(defaults)) else: defaults = None return FakeArgSpec(args, varargs, keywords, defaults)
def _check_function_template(obj, template, ctx=""): raise ObiwanError("checking function templates is not supported yet") obj_ret = None if inspect.isclass(obj): # classes are callable too obj_ret = obj obj = obj.__init__ if not inspect.isfunction(obj): raise ObiwanException("%s %s is not a function" % (ctx, obj)) try: expect = inspect.signature(template) except TypeError: raise ObiwanException("%s template %s is not a function" % (ctx, template)) except ValueError: raise ObiwanException("%s cannot extract signature from template %s" % (ctx, template)) try: got = inspect.signature(obj) except (TypeError, ValueError): raise ObiwanException("%s cannot extract signature from %s" % (ctx, obj)) expect_return = expect.return_annotation if expect.return_annotation is not inspect.Signature.empty else None got_return = obj_ret or (got.return_annotation if got.return_annotation is not inspect.Signature.empty else None) if bool(expect_return) != bool(got_return) and expect_return is not any: raise ObiwanException("%s function %s does not return %s" % (ctx, got, expect_return)) ##### We need to check signatures not values if expect_return and expect_return is not any: duckable(got_return, expect_return, "%s return" % ctx) expect_params = expect.parameters.values() got_params = got.parameters.values() if len(expect_params) != len(got_params): raise ObiwanException("%s function %s mismatches %s" % (ctx, got, expect)) for i, (g, e) in enumerate(zip(got_params, expect_params)): if e.annotation is inspect.Parameter.empty or e.annotation is any: continue if g.annotation is inspect.Parameter.empty: raise ObiwanException("%s function %s parameter %s should be %s but is unannotated" % (ctx, got, i, e.annotation)) duckable(g.annotation, e.annotation, "%s function %s parameter %d" % (ctx, got, i))
def inspect_missing_functions(original_type, target_type): """ Find functions which exist in original_type but not in target_type, or the signature is modified. :return: the tuple of the missing function name and its signature, and the name of the functions the signature of which is different and its original and modified signature. """ missing = [] modified = [] for name, func in inspect.getmembers(original_type, inspect.isfunction): # Skip the private attributes if name.startswith('_'): continue original_signature = inspect.signature(func, follow_wrapped=True) if hasattr(target_type, name): f = getattr(target_type, name) if inspect.isfunction(f): target_signature = inspect.signature(f) if str(original_signature) != str(target_signature): modified.append((name, original_signature, target_signature)) continue missing.append((name, original_signature)) return missing, modified
def do_inspect(self, arg): obj = self._getval(arg) data = OrderedDict() data['Type'] = type(obj).__name__ data['String Form'] = str(obj).strip() if hasattr(obj, '__len__'): data['Length'] = len(obj) try: data['File'] = inspect.getabsfile(obj) except TypeError: pass if (isinstance(obj, type) and hasattr(obj, '__init__') and getattr(obj, '__module__') != '__builtin__'): # Class - show definition and docstring for constructor data['Docstring'] = obj.__doc__ data['Constructor information'] = '' try: data[' Definition'] = '%s%s' % (arg, signature(obj)) except ValueError: pass data[' Docstring'] = obj.__init__.__doc__ else: try: data['Definition'] = '%s%s' % (arg, signature(obj)) except (TypeError, ValueError): pass data['Docstring'] = obj.__doc__ for key, value in data.items(): formatted_key = Color.set(Color.red, key + ':') self.stdout.write('%-28s %s\n' % (formatted_key, value))
def test_get_final_value(self): mapping = self._settings_mapping v = mapping.get_final_value('enabled', fparams=None) self.assertEqual(v, True) mapping['enabled'] = 'enabled_kwd=' d = {'enabled_kwd': 17} v = mapping.get_final_value('enabled', d, fparams=None) self.assertEqual(v, 17) def f(a, enabled_kwd=3): pass fparams = inspect.signature(f).parameters v = mapping.get_final_value('enabled', fparams=fparams) self.assertEqual(v, 3) def g(a, wrong_kwd='nevermind'): pass gparams = inspect.signature(g).parameters v = mapping.get_final_value('enabled', fparams=gparams) self.assertEqual(v, False) def h(a, enabled_kwd=[]): pass hparams = inspect.signature(h).parameters v = mapping.get_final_value('enabled', fparams=hparams) self.assertEqual(v, False)
def __init__(self, *, obj, loop, whitelist=None, blacklist=None, executor=ThreadPoolExecutor, timeout=5): '''Initialize what methods are exposed. Also intialize an executor to run the object's methods in. THis is because they could be blocking and calling these directly would drastically affect the reactivity of the event loop. Args: obj (object): The object to expose loop (asyncio event loop): The event loop to use when scheduling a function Kwargs: whitelist (iterable): A list of methods to expose blacklist (iterable): A list of methods not to expose timeout (int): The timeout value to wait for the executed functions to return executor (ProcessPoolExecutor or ThreadPoolExecutor): The executor upon which to execute the objects functions. Whitelist and blacklist of mutually exclusive. Only use one of these! >>> loop = asyncio.get_event_loop() >>> class t(): ... def func(self): ... pass >>> a = ObjectWrapper(t, loop, whitelist=['func']) ''' self._obj = obj self._loop = loop self.__add_executor(loop, executor=executor) obj_methods = getmembers(obj, ismethod) self._funcs = {} self._func_sigs = {} if whitelist is not None: for func_name,func in obj_methods: if func_name in whitelist: self._funcs[func_name] = func_caller(func, loop, timeout) self._func_sigs[func_name] = signature(func) elif blacklist is not None: for func_name,func in obj_methods: if func_name in blacklist or func_name[0] == '_': continue self._funcs[func_name] = func_caller(func, loop, timeout) self._func_sigs[func_name] = signature(func) else: for func_name,func in obj_methods: if func_name[0] == '_': continue self._funcs[func_name] = func_caller(func, loop, timeout) self._func_sigs[func_name] = signature(func)
def test_order_of_blocks_arguments(): from inspect import signature # remove 'self': meth_args = list(signature(sf.SoundFile.blocks).parameters)[1:] meth_args[3:3] = ['start', 'stop'] func_args = list(signature(sf.blocks).parameters) assert func_args[:10] == ['file'] + meth_args
def call_and_catch_signature_error(_attr, *args, **kwargs): try: return _attr(*args, **kwargs) except TypeError as e: try: signature(_attr).bind(*args, **kwargs) # do sig test only in error case to preserve resources on successful calls raise except TypeError: raise RPCInvalidArguments(e)
def _check_arguments(func, std_arg_name, kwargs): """ Check kwargs passed match the signature of the function. This is a slight hack but is required since we can't catch argument mismatches without catching all TypeErrors which is worse. """ try: inspect.signature(func).bind(**{std_arg_name: None}, **kwargs) except TypeError as e: raise BadDataException('{}: {}'.format(func.__qualname__, e.args[0])) from e
def init_array(self, name, ary, value): # No defaults are supplied if value is None: ary.fill(0) # The array is defaulted with some function elif isinstance(value, types.MethodType): try: signature(value).bind(self, ary) except TypeError: raise TypeError(('The signature of the function supplied ' 'for setting the value on array %s is incorrect. ' 'The function signature has the form f(slvr, ary), ' 'where f is some function that will set values ' 'on the array, slvr is a Solver object which provides ' 'useful information to the function, ' 'and ary is the NumPy array which must be ' 'initialised with values.') % (name)) returned_ary = value(self, ary) if returned_ary is not None: ary[:] = returned_ary elif isinstance(value, types.LambdaType): try: signature(value).bind(self, ary) except TypeError: raise TypeError(('The signature of the lambda supplied ' 'for setting the value on array %s is incorrect. ' 'The function signature has the form lambda slvr, ary:, ' 'where lambda provides functionality for setting values ' 'on the array, slvr is a Solver object which provides ' 'useful information to the function, ' 'and ary is the NumPy array which must be ' 'initialised with values.') % (name)) returned_ary = value(self, ary) if returned_ary is not None: ary[:] = returned_ary # Got an ndarray, try set it equal elif isinstance(value, np.ndarray): try: ary[:] = value except BaseException as e: raise ValueError(('Tried to assign array %s with ' 'value NumPy array, but this failed ' 'with %s') % (name, repr(e))) # Assume some sort of value has been supplied # Give it to NumPy else: try: ary.fill(value) except BaseException as e: raise ValueError(('Tried to fill array %s with ' 'value value %s, but NumPy\'s fill function ' 'failed with %s') % (name, value, repr(e)))
def replace_method(the_class, class_method_name, new_method): class_method = getattr(the_class, class_method_name) old_signature = set(inspect.signature(class_method).parameters) new_signature = set(inspect.signature(new_method).parameters) if old_signature < new_signature: # only patch if SUBSET of parameters setattr(the_class, class_method_name, new_method) logger.info("{} replaced with {}".format( class_method_name, new_method.__name__))
def fix_signature(proxy_func, wrapped_func): """Tries to update proxy_func to have a signature that matches the wrapped func.""" try: proxy_func.__wrapped__.__signature__ = inspect.signature(wrapped_func) except AttributeError: # Non-critical error try: proxy_func.__signature__ = inspect.signature(wrapped_func) except AttributeError: pass return proxy_func
def test_signature(): # Define a test class and an event handler a = A() # signature is preserved: (!!) assert signature(a.on_blubb) == signature(on_blubb) # NOTE: we even got the exact object as default parameter, not only an # exact copy: assert a.on_blubb(with_kwonlyarg="xyz") is referenced
def __init__(self, name, bases, dict): super(_PythonSitePluginMeta, self).__init__(name, bases, dict) if bases == (): return # the original SitePlugin class for name in "name", "author", "rev": if name not in self.__dict__: raise TypeError("SitePlugin subclasses must define the 'name:str', 'author:str' and 'rev:int' properties") obj = self() for name in "forUrl", "video": inspect.signature(getattr(obj, name)).bind("singleArg") _state.register_site(obj)
def __init__(self): self.__signature__ = forwards( signature(self.__call__), merge( signature(self.do), signature(self.prepare), signature(self.status) ) ) self._sigtools__wrappers = ( self.do, self.prepare, self.status, self.__call__)
def has_request_args(fn): sig = inspect.signature(fn) params = inspect.signature(fn).parameters found = False for name, param in params.items(): if name == 'request': found = True continue if found and (param.kind != inspect.Parameter.VAR_POSITIONAL and param.kind != inspect.Parameter.KEYWORD_ONLY and param.kind != inspect.Parameter.VAR_KEYWORD): raise ValueError('request parameter must be the last named parameter in function: %s%s' % (fn.__name__, str(sig))) return found
def authenticate(self, request, remote_user, shib_meta): """ The username passed as ``remote_user`` is considered trusted. This method simply returns the ``User`` object with the given username, creating a new ``User`` object if ``create_unknown_user`` is ``True``. Returns None if ``create_unknown_user`` is ``False`` and a ``User`` object with the given username is not found in the database. """ if not remote_user: return User = get_user_model() username = self.clean_username(remote_user) field_names = [x.name for x in User._meta.get_fields()] shib_user_params = dict([(k, shib_meta[k]) for k in field_names if k in shib_meta]) # Note that this could be accomplished in one try-except clause, but # instead we use get_or_create when creating unknown users since it has # built-in safeguards for multiple threads. if self.create_unknown_user: user, created = User.objects.get_or_create(username=username, defaults=shib_user_params) if created: """ @note: setting password for user needs on initial creation of user instead of after auth.login() of middleware. because get_session_auth_hash() returns the salted_hmac value of salt and password. If it remains after the auth.login() it will return a different auth_hash than what's stored in session "request.session[HASH_SESSION_KEY]". Also we don't need to update the user's password everytime he logs in. """ user.set_unusable_password() user.save() args = (request, user) try: inspect.signature(self.configure_user).bind(request, user) except AttributeError: # getcallargs required for Python 2.7 support, deprecated after 3.5 try: inspect.getcallargs(self.configure_user, request, user) except TypeError: args = (user,) except TypeError: args = (user,) user = self.configure_user(*args) else: try: user = User.objects.get(username=username) except User.DoesNotExist: return # After receiving a valid user, we update the the user attributes according to the shibboleth # parameters. Otherwise the parameters (like mail address, sure_name or last_name) will always # be the initial values from the first login. Only save user object if there are any changes. if not min([getattr(user, k) == v for k, v in shib_user_params.items()]): user.__dict__.update(**shib_user_params) user.save() return user if self.user_can_authenticate(user) else None
def decorate(asyncfunc): if inspect.signature(syncfunc) != inspect.signature(asyncfunc): raise TypeError('%s and async %s have different signatures' % (syncfunc.__name__, asyncfunc.__name__)) @wraps(asyncfunc) def wrapper(*args, **kwargs): if _from_coroutine(): return asyncfunc(*args, **kwargs) else: return syncfunc(*args, **kwargs) return wrapper
def _convert_torch_model_loss_fn_to_onnx(self, inputs, device): # Dynamic axes dynamic_axes = {} for input in self.model_desc.inputs: symbolic_axis = {} for i, axis in enumerate(input.shape): if isinstance(axis, str): symbolic_axis[i] = axis if len(symbolic_axis): dynamic_axes[input.name] = symbolic_axis for output in self.model_desc.outputs: symbolic_axis = {} for i, axis in enumerate(output.shape): if isinstance(axis, str): symbolic_axis[i] = axis if len(symbolic_axis): dynamic_axes[output.name] = symbolic_axis if isinstance(inputs, torch.Tensor): inputs = [inputs] if isinstance(inputs, dict): sample_inputs = [ inputs[k.name_].to(device=device) for k in self.model_desc.inputs ] elif isinstance(inputs, (list, tuple)): sample_inputs = [ input.to(device=device) for i, input in enumerate(inputs) if i < len(self.model_desc.inputs) ] else: raise RuntimeError( "Unexpected input type. Only torch.Tensor, or dict/list/tuple of torch.Tensor is supported." ) # PyTorch ONNX exporter does not match argument names # This is an issue because the ONNX graph depends on all inputs to be specified # Validate loss_fn if self.loss_fn: sig_loss = signature(self.loss_fn) if len(sig_loss.parameters) != 2: raise RuntimeError( "loss function should take two arguments - predict and label." ) # Basic input names from model input_names = [input.name for input in self.model_desc.inputs] sig = signature(self._torch_model.forward) ordered_input_list = list(sig.parameters.keys()) # Label from loss_fn goes after model input if self.loss_fn: ordered_input_list = [ *ordered_input_list, list(sig_loss.parameters.keys())[1] ] class CombineTorchModelLossFnWrapInput(torch.nn.Module): def __init__(self, model, loss_fn, input_names): super().__init__() self.model = model self.loss_fn = loss_fn self.input_names = input_names def forward(self, *inputs): sig = signature(self.model.forward) input_dict = {} for key in sig.parameters.keys(): if key in self.input_names: input_dict[key] = inputs[self.input_names.index(key)] model_out = self.model(**input_dict) if self.loss_fn is None: return model_out label = inputs[-1] preds = model_out return self.loss_fn(preds, label), preds model = CombineTorchModelLossFnWrapInput(self._torch_model, self.loss_fn, input_names) # Do an inference to grab output types model.eval() with torch.no_grad(): # Deepcopy inputs, since input values may change after model run. sample_inputs_copy = copy.deepcopy(sample_inputs) try: # Deepcopy model, in case model is stateful and changes after model run. model_copy = copy.deepcopy(model) except Exception: model_copy = model warnings.warn( "This model cannot be deep copied (or pickled), which is a required step for stateful models to be properly exported to ONNX." " Compute will continue, but unexpected results may occur!" ) sample_outputs = model_copy(*sample_inputs_copy) self.torch_sample_outputs = sample_outputs model.train() if isinstance(sample_outputs, torch.Tensor): sample_outputs = [sample_outputs] # Append 'dtype' for model description's inputs/outputs for idx_i, sample_input in enumerate(sample_inputs): if idx_i < len(self.model_desc.inputs): self.model_desc.add_type_to_input_description( idx_i, sample_input.dtype) for idx_o, sample_output in enumerate(sample_outputs): if idx_o < len(self.model_desc.outputs): self.model_desc.add_type_to_output_description( idx_o, sample_output.dtype) # Export the model to ONNX f = io.BytesIO() # Deepcopy inputs, since input values may change after model run. sample_inputs_copy = copy.deepcopy(sample_inputs) # Handle contrib OPs support from onnxruntime.training import register_custom_ops_pytorch_exporter if self.options._internal_use.enable_onnx_contrib_ops: # Enable contrib ops export from PyTorch register_custom_ops_pytorch_exporter.register_custom_op() else: # Unregister contrib ops, if they were registered in previous calls register_custom_ops_pytorch_exporter.unregister_custom_op() # Export torch.nn.Module to ONNX torch.onnx._export( model, tuple(sample_inputs_copy), f, input_names=[input.name for input in self.model_desc.inputs], output_names=[output.name for output in self.model_desc.outputs], opset_version=self.options._internal_use.onnx_opset_version, dynamic_axes=dynamic_axes, _retain_param_name=True, example_outputs=tuple(sample_outputs), do_constant_folding=False, training=torch.onnx.TrainingMode.TRAINING) onnx_model = onnx.load_model_from_string(f.getvalue()) # Remove 'model.' prefix introduced by CombineTorchModelLossFn class if isinstance(model, CombineTorchModelLossFnWrapInput): replace_name_dict = {} for n in onnx_model.graph.initializer: if n.name.startswith('model.'): replace_name_dict[n.name] = n.name[len('model.'):] n.name = replace_name_dict[n.name] for n in onnx_model.graph.node: for i, name in enumerate(n.input): if name in replace_name_dict: n.input[i] = replace_name_dict[name] return onnx_model
# -*- coding: utf-8 -*- import inspect #Qdef a(a, b=0, *c, d, e=1, **f): def a(a, **f): pass aa = inspect.signature(a) print("inspect.signature(fn)是:%s" % aa) print("inspect.signature(fn)的类型:%s" % (type(aa))) print("\n")
def _nx_wrapper(nx_function, ax, **kwargs): """ This is a wrapper for NetworkX functions. NetworkX graphing functions have many parameters that change relatively often. This wrapper is an attempt at future-proofing. It uses `inspect` to figure out what parameters the function expects and then only passes parameters that will be recognized. If a parameter is not recognized, the function throws a warning which will hopefully alert the user to changes in the NetworkX API without causing our package to crash. This function throws a ValueError if a required arugment is not passed. Parameters ---------- nx_function : function a NetworkX function to wrap. ax : matplotlib.axes object axis on whichi to do plotting. this low-level function assumes you've already made the ax in some other context. **kwargs : kwargs arguments to pass to that function. returns ax, nx_fcn_return """ required_args = {} args_with_defaults = {} # Figure out what networkx function expects fcn_sig = inspect.signature(nx_function) for p in fcn_sig.parameters: if fcn_sig.parameters[p].default is fcn_sig.parameters[p].empty: required_args[p] = None else: args_with_defaults[p] = fcn_sig.parameters[p] # Go through the kwargs passed in and construct input_kwargs. input_kwargs = {} input_kwargs["ax"] = ax bad_keys = [] for k in kwargs: # First see if this is a required argument without a default. try: required_args.pop(k) input_kwargs[k] = kwargs[k] continue except KeyError: pass # Next see if this is an argument that has a default. try: args_with_defaults.pop(k) input_kwargs[k] = kwargs[k] continue except KeyError: pass bad_keys.append(k) # Make sure all required arguments were passed if len(required_args) > 0: err = f"Not all required arguments passed to '{nx_function}'\n" err += f"\nYour networkx version is {nx.__version__}\n" err += f"Missing required arguments are:\n" for a in required_args: err += f" {a}\n" err += "\n" raise ValueError(err) # Warn if arguments were passed that were not recognized. if len(bad_keys) > 0: w = f"Not all keyword arguments were recognized by '{nx_function}'\n" w += f"\nYour networkx version is {nx.__version__}\n" w += f"Unrecognized arguments are:\n" for a in bad_keys: w += f" {a}\n" w += "\n" if len(args_with_defaults) > 0: w += "\nNot all possible arguments were supplied. Did you mean to \n" w += "pass one of these (default given in parentheses)?\n" for a in args_with_defaults: w += f" {a} ({args_with_defaults[a].default})\n" w += "\n\n" warnings.warn(w) return ax, nx_function(**input_kwargs)
def _init_graph(self) -> None: # Collect inputs. self.input_names = [] for param in inspect.signature(self._build_func).parameters.values(): if param.kind == param.POSITIONAL_OR_KEYWORD and param.default is param.empty: self.input_names.append(param.name) self.num_inputs = len(self.input_names) assert self.num_inputs >= 1 # Choose name and scope. if self.name is None: self.name = self._build_func_name assert re.match("^[A-Za-z0-9_.\\-]*$", self.name) with tf.name_scope(None): self.scope = tf.get_default_graph().unique_name(self.name, mark_as_used=True) # Finalize build func kwargs. build_kwargs = dict(self.static_kwargs) build_kwargs["is_template_graph"] = True build_kwargs["components"] = self.components # Build template graph. with tfutil.absolute_variable_scope( self.scope, reuse=tf.AUTO_REUSE), tfutil.absolute_name_scope( self.scope): # ignore surrounding scopes assert tf.get_variable_scope().name == self.scope assert tf.get_default_graph().get_name_scope() == self.scope with tf.control_dependencies( None): # ignore surrounding control dependencies self.input_templates = [ tf.placeholder(tf.float32, name=name) for name in self.input_names ] out_expr = self._build_func(*self.input_templates, **build_kwargs) # Collect outputs. assert tfutil.is_tf_expression(out_expr) or isinstance(out_expr, tuple) self.output_templates = [ out_expr ] if tfutil.is_tf_expression(out_expr) else list(out_expr) self.num_outputs = len(self.output_templates) assert self.num_outputs >= 1 assert all(tfutil.is_tf_expression(t) for t in self.output_templates) # Perform sanity checks. if any(t.shape.ndims is None for t in self.input_templates): raise ValueError( "Network input shapes not defined. Please call x.set_shape() for each input." ) if any(t.shape.ndims is None for t in self.output_templates): raise ValueError( "Network output shapes not defined. Please call x.set_shape() where applicable." ) if any(not isinstance(comp, Network) for comp in self.components.values()): raise ValueError( "Components of a Network must be Networks themselves.") if len(self.components) != len( set(comp.name for comp in self.components.values())): raise ValueError("Components of a Network must have unique names.") # List inputs and outputs. self.input_shapes = [ tfutil.shape_to_list(t.shape) for t in self.input_templates ] self.output_shapes = [ tfutil.shape_to_list(t.shape) for t in self.output_templates ] self.input_shape = self.input_shapes[0] self.output_shape = self.output_shapes[0] self.output_names = [ t.name.split("/")[-1].split(":")[0] for t in self.output_templates ] # List variables. self.own_vars = OrderedDict( (var.name[len(self.scope) + 1:].split(":")[0], var) for var in tf.global_variables(self.scope + "/")) self.vars = OrderedDict(self.own_vars) self.vars.update((comp.name + "/" + name, var) for comp in self.components.values() for name, var in comp.vars.items()) self.trainables = OrderedDict( (name, var) for name, var in self.vars.items() if var.trainable) self.var_global_to_local = OrderedDict( (var.name.split(":")[0], name) for name, var in self.vars.items())
def handle_event(self, event: CommandRequestEvent, kafka_service: KafkaService): # Safety check payload = event.payload if not payload or not isinstance(payload, CommandRequestPayload): logger.info("Wrong payload passed") return command = event.payload.command args = event.payload.args logger.info(f"Command {command} called with {args}") if not hasattr(self.cmd_object, command): # May be another callback in the chain will be able to handle it logger.info("No command method found") return method = getattr(self.cmd_object, command) if callable(method): sig = signature(method) result = None code = KafkaCommandCallback.CODE_SUCCESS msg = self._get_message(KafkaCommandCallback.OK, command) pos_args = [] kw_args = {} if isinstance(args, list): pos_args = tuple(args) elif isinstance(args, dict): kw_args = dict(args) if "event" in sig.parameters: kw_args["event"] = event if "kafka_service" in sig.parameters: kw_args["kafka_service"] = kafka_service params = OrderedDict(sig.parameters) # Remove positionals orderparams = list(params) for i in range(0, min(len(pos_args), len(orderparams))): del params[orderparams[i]] # Remove named for k in kw_args: if k in params: del params[k] from_event = vars(event) for key in from_event.keys() & params.keys(): kw_args[key] = from_event[key] try: logger.info(f"executing {command}({pos_args}, {kw_args})") result = method(*pos_args, **kw_args) except TypeError as e: logger.exception("Method invocation failed") code = KafkaCommandCallback.CODE_CLIENT_ERROR msg = self._get_message(KafkaCommandCallback.INVALID_ARGS, command, str(e)) except Exception as e: logger.exception("Command failed") code = KafkaCommandCallback.CODE_COMMAND_ERROR msg = self._get_message(KafkaCommandCallback.UNKNOWN_ERROR, command, str(e)) response_payload = CommandResponsePayload(code, msg, event, result) response_event = create_from_object(CommandResponseEvent, event, payload=response_payload) kafka_service.send_event(response_event, self.cmd_return_topic) else: logger.info(f"Uncallable {command} member requested")
def create( cls, obj: Any, class_from_init: bool, ignore_dunder: bool, ignore_private: bool, ignore_inherited: bool, importing_module: Optional[str] = None, prefer_docstring: bool = True ) -> ClassDescriptor: """Create a class Args: obj (Any): The class class_from_init (bool): If True take the docstring from the init function ignore_dunder (bool): If True ignore <span>__</span>XXX<span>__</span> functions ignore_private (bool): If True ignore private methods (those prefixed <span>_</span>XXX) ignore_inherited (bool): If True ignore inherited methods importing_module (Optional[str], optional): The importing module, defaults to None prefer_docstring (bool): If true prefer the docstring. Returns: ClassDescriptor: The class descriptor """ is_named_tuple = is_named_tuple_type(obj) valid_names: List[str] = [] valid_names.extend(getattr(obj, '__dict__', {}).keys()) valid_names.extend(getattr(obj, '__slots__', [])) members: Dict[str, Any] = { name: value for name, value in inspect.getmembers(obj) if not ignore_inherited or name in valid_names } docstring = _get_docstring(obj, members, class_from_init, is_named_tuple) name = obj.__qualname__ if hasattr( obj, '__qualname__') else obj.__name__ summary = docstring.short_description if docstring else None description = docstring.long_description if docstring else None attributes: List[ArgumentDescriptor] = [] if docstring: attrs = [ (meta.args[1], meta.description) for meta in docstring.meta if 'attribute' in meta.args ] for attr_details, attr_desc in attrs: attr_name, _sep, attr_type = attr_details.partition(' ') attr_type = attr_type.strip('()') attributes.append( ArgumentDescriptor(attr_name, attr_type, attr_desc) ) properties: List[PropertyDescriptor] = [] methods: List[CallableDescriptor] = [] class_methods: List[CallableDescriptor] = [] for member_name, member in members.items(): if member_name == '__init__' or ( ignore_dunder and member_name.startswith('__') and member_name.endswith('__') ) or (ignore_private and member_name.startswith('_')): continue if member.__class__ is property: properties.append( PropertyDescriptor.create( member, obj, member_name ) ) elif inspect.isfunction(member): # Instance methods methods.append( CallableDescriptor.create( member, callable_type=CallableType.METHOD, prefer_docstring=prefer_docstring, qualifier=name ) ) elif inspect.ismethod(member): # Class methods class_methods.append( CallableDescriptor.create( member, callable_type=CallableType.CLASS_METHOD, prefer_docstring=prefer_docstring, qualifier=name ) ) examples: Optional[List[str]] = [ meta.description for meta in docstring.meta if 'examples' in meta.args ] if docstring is not None else None module_obj = inspect.getmodule(obj) module = importing_module or obj.__module__ package = module_obj.__package__ if module_obj else None file = make_file_relative( module_obj.__file__ if module_obj and hasattr(module_obj, '__file__') else None ) bases = [ ClassDescriptor.create( base, class_from_init=class_from_init, ignore_dunder=ignore_dunder, ignore_private=ignore_private, ignore_inherited=ignore_inherited, importing_module=importing_module, prefer_docstring=prefer_docstring ) for base in getattr(obj, '__bases__', []) if base is not object ] try: signature = inspect.signature(obj) constructor = CallableDescriptor.create( obj, signature, docstring, CallableType.CONSTRUCTOR, prefer_docstring=prefer_docstring ) except ValueError: constructor = None return ClassDescriptor( name, summary, description, constructor, attributes, properties, class_methods, methods, examples, module, package, file, bases )
def info(object=None, maxwidth=76, output=sys.stdout, toplevel='numpy'): """ Get help information for a function, class, or module. Parameters ---------- object : object or str, optional Input object or name to get information about. If `object` is a numpy object, its docstring is given. If it is a string, available modules are searched for matching objects. If None, information about `info` itself is returned. maxwidth : int, optional Printing width. output : file like object, optional File like object that the output is written to, default is ``stdout``. The object has to be opened in 'w' or 'a' mode. toplevel : str, optional Start search at this level. See Also -------- source, lookfor Notes ----- When used interactively with an object, ``np.info(obj)`` is equivalent to ``help(obj)`` on the Python prompt or ``obj?`` on the IPython prompt. Examples -------- >>> np.info(np.polyval) # doctest: +SKIP polyval(p, x) Evaluate the polynomial p at x. ... When using a string for `object` it is possible to get multiple results. >>> np.info('fft') # doctest: +SKIP *** Found in numpy *** Core FFT routines ... *** Found in numpy.fft *** fft(a, n=None, axis=-1) ... *** Repeat reference found in numpy.fft.fftpack *** *** Total of 3 references found. *** """ global _namedict, _dictlist # Local import to speed up numpy's import time. import pydoc import inspect if (hasattr(object, '_ppimport_importer') or hasattr(object, '_ppimport_module')): object = object._ppimport_module elif hasattr(object, '_ppimport_attr'): object = object._ppimport_attr if object is None: info(info) elif isinstance(object, ndarray): _info(object, output=output) elif isinstance(object, str): if _namedict is None: _namedict, _dictlist = _makenamedict(toplevel) numfound = 0 objlist = [] for namestr in _dictlist: try: obj = _namedict[namestr][object] if id(obj) in objlist: print("\n " "*** Repeat reference found in %s *** " % namestr, file=output ) else: objlist.append(id(obj)) print(" *** Found in %s ***" % namestr, file=output) info(obj) print("-"*maxwidth, file=output) numfound += 1 except KeyError: pass if numfound == 0: print("Help for %s not found." % object, file=output) else: print("\n " "*** Total of %d references found. ***" % numfound, file=output ) elif inspect.isfunction(object) or inspect.ismethod(object): name = object.__name__ try: arguments = str(inspect.signature(object)) except Exception: arguments = "()" if len(name+arguments) > maxwidth: argstr = _split_line(name, arguments, maxwidth) else: argstr = name + arguments print(" " + argstr + "\n", file=output) print(inspect.getdoc(object), file=output) elif inspect.isclass(object): name = object.__name__ try: arguments = str(inspect.signature(object)) except Exception: arguments = "()" if len(name+arguments) > maxwidth: argstr = _split_line(name, arguments, maxwidth) else: argstr = name + arguments print(" " + argstr + "\n", file=output) doc1 = inspect.getdoc(object) if doc1 is None: if hasattr(object, '__init__'): print(inspect.getdoc(object.__init__), file=output) else: print(inspect.getdoc(object), file=output) methods = pydoc.allmethods(object) public_methods = [meth for meth in methods if meth[0] != '_'] if public_methods: print("\n\nMethods:\n", file=output) for meth in public_methods: thisobj = getattr(object, meth, None) if thisobj is not None: methstr, other = pydoc.splitdoc( inspect.getdoc(thisobj) or "None" ) print(" %s -- %s" % (meth, methstr), file=output) elif hasattr(object, '__doc__'): print(inspect.getdoc(object), file=output)
def getfuncargnames( function: Callable[..., Any], *, name: str = "", is_method: bool = False, cls: Optional[type] = None ) -> Tuple[str, ...]: """Returns the names of a function's mandatory arguments. This should return the names of all function arguments that: * Aren't bound to an instance or type as in instance or class methods. * Don't have default values. * Aren't bound with functools.partial. * Aren't replaced with mocks. The is_method and cls arguments indicate that the function should be treated as a bound method even though it's not unless, only in the case of cls, the function is a static method. The name parameter should be the original name in which the function was collected. """ # TODO(RonnyPfannschmidt): This function should be refactored when we # revisit fixtures. The fixture mechanism should ask the node for # the fixture names, and not try to obtain directly from the # function object well after collection has occurred. # The parameters attribute of a Signature object contains an # ordered mapping of parameter names to Parameter instances. This # creates a tuple of the names of the parameters that don't have # defaults. try: parameters = signature(function).parameters except (ValueError, TypeError) as e: fail( "Could not determine arguments of {!r}: {}".format(function, e), pytrace=False, ) arg_names = tuple( p.name for p in parameters.values() if ( p.kind is Parameter.POSITIONAL_OR_KEYWORD or p.kind is Parameter.KEYWORD_ONLY ) and p.default is Parameter.empty ) if not name: name = function.__name__ # If this function should be treated as a bound method even though # it's passed as an unbound method or function, remove the first # parameter name. if is_method or ( cls and not isinstance(cls.__dict__.get(name, None), staticmethod) ): arg_names = arg_names[1:] # Remove any names that will be replaced with mocks. if hasattr(function, "__wrapped__"): arg_names = arg_names[num_mock_patch_args(function) :] return arg_names
def has_named_kw_args(fn): params = inspect.signature(fn).parameters for name, param in params.items(): if param.kind == inspect.Parameter.KEYWORD_ONLY: return True
def fit(self, X, y, sample_weight=None): """Fit the calibrated model Parameters ---------- X : array-like, shape (n_samples, n_features) Training data. y : array-like, shape (n_samples,) Target values. sample_weight : array-like of shape (n_samples,), default=None Sample weights. If None, then samples are equally weighted. Returns ------- self : object Returns an instance of self. """ X, y = self._validate_data(X, y, accept_sparse=['csc', 'csr', 'coo'], force_all_finite=False, allow_nd=True) X, y = indexable(X, y) le = LabelBinarizer().fit(y) self.classes_ = le.classes_ # Check that each cross-validation fold can have at least one # example per class n_folds = self.cv if isinstance(self.cv, int) \ else self.cv.n_folds if hasattr(self.cv, "n_folds") else None if n_folds and \ np.any([np.sum(y == class_) < n_folds for class_ in self.classes_]): raise ValueError("Requesting %d-fold cross-validation but provided" " less than %d examples for at least one class." % (n_folds, n_folds)) self.calibrated_classifiers_ = [] if self.base_estimator is None: # we want all classifiers that don't expose a random_state # to be deterministic (and we don't want to expose this one). base_estimator = LinearSVC(random_state=0) else: base_estimator = self.base_estimator if self.cv == "prefit": calibrated_classifier = _CalibratedClassifier(base_estimator, method=self.method) calibrated_classifier.fit(X, y, sample_weight) self.calibrated_classifiers_.append(calibrated_classifier) else: cv = check_cv(self.cv, y, classifier=True) fit_parameters = signature(base_estimator.fit).parameters base_estimator_supports_sw = "sample_weight" in fit_parameters if sample_weight is not None: sample_weight = _check_sample_weight(sample_weight, X) if not base_estimator_supports_sw: estimator_name = type(base_estimator).__name__ warnings.warn("Since %s does not support sample_weights, " "sample weights will only be used for the " "calibration itself." % estimator_name) for train, test in cv.split(X, y): this_estimator = clone(base_estimator) if sample_weight is not None and base_estimator_supports_sw: this_estimator.fit(X[train], y[train], sample_weight=sample_weight[train]) else: this_estimator.fit(X[train], y[train]) calibrated_classifier = _CalibratedClassifier( this_estimator, method=self.method, classes=self.classes_) sw = None if sample_weight is None else sample_weight[test] calibrated_classifier.fit(X[test], y[test], sample_weight=sw) self.calibrated_classifiers_.append(calibrated_classifier) return self
def test_inspect_sum(self): signature = inspect.signature(np.sum) assert_('axis' in signature.parameters)
def check_argument_types(self, function: Callable[..., HttpResponse], openapi_parameters: List[Dict[str, Any]]) -> None: """ We construct for both the OpenAPI data and the function's definition a set of tuples of the form (var_name, type) and then compare those sets to see if the OpenAPI data defines a different type than that actually accepted by the function. Otherwise, we print out the exact differences for convenient debugging and raise an AssertionError. """ openapi_params = set( ) # type: Set[Tuple[str, Union[type, Tuple[type, object]]]] for element in openapi_parameters: name = element["name"] # type: str schema = element["schema"] if 'oneOf' in schema: # Hack: Just use the type of the first value # Ideally, we'd turn this into a Union type. _type = VARMAP[schema['oneOf'][0]['type']] else: _type = VARMAP[schema["type"]] if _type == list: items = schema["items"] if "anyOf" in items.keys(): subtypes = [] for st in items["anyOf"]: st = st["type"] subtypes.append(VARMAP[st]) self.assertTrue(len(subtypes) > 1) sub_type = self.get_type_by_priority(subtypes) else: sub_type = VARMAP[element["schema"]["items"]["type"]] self.assertIsNotNone(sub_type) openapi_params.add((name, (_type, sub_type))) else: openapi_params.add((name, _type)) function_params = set( ) # type: Set[Tuple[str, Union[type, Tuple[type, object]]]] # Iterate through the decorators to find the original # function, wrapped by has_request_variables, so we can parse # its arguments. while getattr(function, "__wrapped__", None): function = getattr(function, "__wrapped__", None) # Tell mypy this is never None. assert function is not None # Now, we do inference mapping each REQ parameter's # declaration details to the Python/mypy types for the # arguments passed to it. # # Because the mypy types are the types used inside the inner # function (after the original data is processed by any # validators, converters, etc.), they will not always match # the API-level argument types. The main case where this # happens is when a `converter` is used that changes the types # of its parameters. for vname, defval in inspect.signature(function).parameters.items(): defval = defval.default if defval.__class__ is _REQ: # TODO: The below inference logic in cases where # there's a converter function declared is incorrect. # Theoretically, we could restructure the converter # function model so that we can check what type it # excepts to be passed to make validation here # possible. vtype = self.get_standardized_argument_type( function.__annotations__[vname]) vname = defval.post_var_name # type: ignore # See zerver/lib/request.py function_params.add((vname, vtype)) diff = openapi_params - function_params if diff: # nocoverage self.render_openapi_type_exception(function, openapi_params, function_params, diff)
def accepts_one_argument(function: Callable): return len(signature(function).parameters) == 1
def get_default_args(func: Callable) -> Dict[str, Any]: signature = inspect.signature(func) return [ v.default if v.default is not inspect.Parameter.empty else None for v in signature.parameters.values() ]
def __init__(self, model, model_desc, optim_config, loss_fn=None, options=None): # Basic validation assert model is not None, "'model' is required and must be either a 'torch.nn.Module' or ONNX model" assert isinstance(model_desc, dict), "'model_desc' must be a 'dict'" assert isinstance(optim_config, optim._OptimizerConfig),\ "'optim_config' is required and must be any of 'AdamConfig', 'LambConfig' or 'SGDConfig'" assert loss_fn is None or (callable(loss_fn) and len(signature(loss_fn).parameters) == 2),\ "'loss_fn' must be either 'None' or a callable with two parameters" assert options is None or isinstance(options, ORTTrainerOptions),\ "'options' must be either 'None' or 'ORTTrainerOptions'" # Model + Loss validation # Supported combinarios are # ---------------------------------------- # | | Model | Loss | # ---------------------------------------- # | 1 | torch.nn.Module | None | # | 2 | torch.nn.Module | torch.nn.Module | # | 3 | ONNX | None | # ---------------------------------------- self._torch_model = None self._onnx_model = None if isinstance(model, torch.nn.Module): assert loss_fn is None or isinstance(model, torch.nn.Module),\ "'loss_fn' must be either 'None' or 'torch.nn.Module'" self._torch_model = model self.loss_fn = loss_fn # TODO: Subject to change after checkpoint redesign self._torch_state_dict_keys = list(model.state_dict().keys()) elif isinstance(model, onnx.ModelProto): assert loss_fn is None, "'loss_fn' must not be specified when 'model' is an ONNX model" self._onnx_model = model self.loss_fn = None else: raise ValueError( "'model' must be either 'torch.nn.Module' or 'onnx.ModelProto'" ) self.model_desc = _ORTTrainerModelDesc(model_desc) self.optim_config = optim_config # ORTTrainerOptions if not options: options = ORTTrainerOptions() self.options = options if self.options.mixed_precision.enabled and not self.options.mixed_precision.loss_scaler: # TODO: Move this to model_desc_validation.py self.options.mixed_precision.loss_scaler = amp.loss_scaler.DynamicLossScaler( ) # Post processing ONNX model given as input if self._onnx_model: if self.options._internal_use.enable_internal_postprocess: self._onnx_model = postprocess.run_postprocess( self._onnx_model) if self.options._internal_use.extra_postprocess: self._onnx_model = self.options._internal_use.extra_postprocess( self._onnx_model) assert isinstance( self._onnx_model, onnx.ModelProto ), "'extra_postprocess' must return a ONNX model" # When input model is already ONNX (and not exported from Pytorch within ORTTrainer), # append 'dtype' from ONNX into model description's for idx_i, i_desc in enumerate(self.model_desc.inputs): dtype = None for onnx_input in self._onnx_model.graph.input: if onnx_input.name == i_desc.name: dtype = _utils.dtype_onnx_to_torch( onnx_input.type.tensor_type.elem_type) self.model_desc.add_type_to_input_description( idx_i, dtype) break assert dtype is not None, f"ONNX model with unknown input type ({i_desc.name})" for idx_o, o_desc in enumerate(self.model_desc.outputs): dtype = None for onnx_output in self._onnx_model.graph.output: if onnx_output.name == o_desc.name: dtype = _utils.dtype_onnx_to_torch( onnx_output.type.tensor_type.elem_type) self.model_desc.add_type_to_output_description( idx_o, dtype) break assert dtype is not None, f"ONNX model with unknown output type ({o_desc.name})" # Set GPU device and memory limit if 'cuda' in self.options.device.id.lower(): mem_limit = self.options.device.mem_limit if mem_limit > 0: ort.set_cuda_mem_limit(self.options.device.mem_limit) ort.set_cuda_device_id( _utils.get_device_index(self.options.device.id)) # TODO: Subject to change after checkpoint redesign self._state_dict = {} self._train_step_info = TrainStepInfo(self.optim_config) self._training_session = None self._init_session()
def decorator(func): import pytest os.makedirs(result_dir, exist_ok=True) old_sig = inspect.signature(func) @pytest.mark.parametrize("ext", extensions) def wrapper(*args, ext="png", request=None, **kwargs): if "ext" in old_sig.parameters: kwargs["ext"] = ext if "request" in old_sig.parameters: kwargs["request"] = request try: file_name = "".join(c for c in request.node.name if c in ALLOWED_CHARS) except AttributeError: # 'NoneType' object has no attribute 'node' file_name = func.__name__ try: fig_ref, fig_test = func(*args, **kwargs) ref_image_path = os.path.join(result_dir, f"{file_name}-expected.{ext}") test_image_path = os.path.join(result_dir, f"{file_name}.{ext}") fig_ref.savefig(ref_image_path) fig_test.savefig(test_image_path) # Code below is adapted for PyGMT, and is originally based on # matplotlib.testing.decorators._raise_on_image_difference err = compare_images( expected=ref_image_path, actual=test_image_path, tol=tol, in_decorator=True, ) if err is None: # Images are the same os.remove(ref_image_path) os.remove(test_image_path) else: # Images are not the same for key in ["actual", "expected", "diff"]: err[key] = os.path.relpath(err[key]) raise GMTImageComparisonFailure( "images not close (RMS %(rms).3f):\n\t%(actual)s\n\t%(expected)s " % err) finally: del fig_ref del fig_test parameters = [ param for param in old_sig.parameters.values() if param.name not in {"fig_test", "fig_ref"} ] if "ext" not in old_sig.parameters: parameters += [inspect.Parameter("ext", KEYWORD_ONLY)] if "request" not in old_sig.parameters: parameters += [inspect.Parameter("request", KEYWORD_ONLY)] new_sig = old_sig.replace(parameters=parameters) wrapper.__signature__ = new_sig # reach a bit into pytest internals to hoist the marks from # our wrapped function new_marks = getattr(func, "pytestmark", []) + wrapper.pytestmark wrapper.pytestmark = new_marks return wrapper
def has_var_kw_arg(fn): params = inspect.signature(fn).parameters for name, param in params.items(): if param.kind == inspect.Parameter.VAR_KEYWORD: return True
try: src, ext = build[obj] except KeyError: return self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) list( multiprocessing.pool.ThreadPool(multiprocessing.cpu_count()).imap( compile_one, objects)) return objects # Plant the parallel compile function. if _check_env_flag('COMPILE_PARALLEL', default='1'): try: if (inspect.signature(distutils.ccompiler.CCompiler.compile) == inspect.signature(_compile_parallel)): distutils.ccompiler.CCompiler.compile = _compile_parallel except: pass class Clean(distutils.command.clean.clean): def run(self): import glob import re with open('.gitignore', 'r') as f: ignores = f.read() pat = re.compile(r'^#( BEGIN NOT-CLEAN-FILES )?') for wildcard in filter(None, ignores.split('\n')): match = pat.match(wildcard)
def get_function_args(callable): all_parameters = frozenset(signature(callable).parameters) return list(all_parameters)
def fit(self, X, y, sample_weight=None): """Fit the calibrated model. Parameters ---------- X : array-like of shape (n_samples, n_features) Training data. y : array-like of shape (n_samples,) Target values. sample_weight : array-like of shape (n_samples,), default=None Sample weights. If None, then samples are equally weighted. Returns ------- self : object Returns an instance of self. """ check_classification_targets(y) X, y = indexable(X, y) if self.base_estimator is None: # we want all classifiers that don't expose a random_state # to be deterministic (and we don't want to expose this one). base_estimator = LinearSVC(random_state=0) else: base_estimator = self.base_estimator self.calibrated_classifiers_ = [] if self.cv == "prefit": # `classes_` and `n_features_in_` should be consistent with that # of base_estimator if isinstance(self.base_estimator, Pipeline): check_is_fitted(self.base_estimator[-1]) else: check_is_fitted(self.base_estimator) with suppress(AttributeError): self.n_features_in_ = base_estimator.n_features_in_ self.classes_ = self.base_estimator.classes_ pred_method = _get_prediction_method(base_estimator) n_classes = len(self.classes_) predictions = _compute_predictions(pred_method, X, n_classes) calibrated_classifier = _fit_calibrator( base_estimator, predictions, y, self.classes_, self.method, sample_weight ) self.calibrated_classifiers_.append(calibrated_classifier) else: X, y = self._validate_data( X, y, accept_sparse=['csc', 'csr', 'coo'], force_all_finite=False, allow_nd=True ) # Set `classes_` using all `y` label_encoder_ = LabelEncoder().fit(y) self.classes_ = label_encoder_.classes_ n_classes = len(self.classes_) # sample_weight checks fit_parameters = signature(base_estimator.fit).parameters supports_sw = "sample_weight" in fit_parameters if sample_weight is not None: sample_weight = _check_sample_weight(sample_weight, X) if not supports_sw: estimator_name = type(base_estimator).__name__ warnings.warn(f"Since {estimator_name} does not support " "sample_weights, sample weights will only be" " used for the calibration itself.") # Check that each cross-validation fold can have at least one # example per class if isinstance(self.cv, int): n_folds = self.cv elif hasattr(self.cv, "n_splits"): n_folds = self.cv.n_splits else: n_folds = None if n_folds and np.any([np.sum(y == class_) < n_folds for class_ in self.classes_]): raise ValueError(f"Requesting {n_folds}-fold " "cross-validation but provided less than " f"{n_folds} examples for at least one class.") cv = check_cv(self.cv, y, classifier=True) if self.ensemble: parallel = Parallel(n_jobs=self.n_jobs) self.calibrated_classifiers_ = parallel( delayed(_fit_classifier_calibrator_pair)( clone(base_estimator), X, y, train=train, test=test, method=self.method, classes=self.classes_, supports_sw=supports_sw, sample_weight=sample_weight) for train, test in cv.split(X, y) ) else: this_estimator = clone(base_estimator) method_name = _get_prediction_method(this_estimator).__name__ pred_method = partial( cross_val_predict, estimator=this_estimator, X=X, y=y, cv=cv, method=method_name, n_jobs=self.n_jobs ) predictions = _compute_predictions(pred_method, X, n_classes) if sample_weight is not None and supports_sw: this_estimator.fit(X, y, sample_weight) else: this_estimator.fit(X, y) calibrated_classifier = _fit_calibrator( this_estimator, predictions, y, self.classes_, self.method, sample_weight ) self.calibrated_classifiers_.append(calibrated_classifier) return self
def test_instance_method(): r"""Ensure instance methods' signature.""" assert hasattr(ResSAttnRNNBlock, 'forward') assert inspect.signature(ResSAttnRNNBlock.forward) == Signature( parameters=[ Parameter( name='self', kind=Parameter.POSITIONAL_OR_KEYWORD, default=Parameter.empty, ), Parameter( name='batch_tk_mask', kind=Parameter.POSITIONAL_OR_KEYWORD, annotation=torch.Tensor, default=Parameter.empty, ), Parameter( name='batch_tk_reps', kind=Parameter.POSITIONAL_OR_KEYWORD, annotation=torch.Tensor, default=Parameter.empty, ), ], return_annotation=torch.Tensor, ) assert hasattr(ResSAttnRNNModel, '__init__') assert inspect.signature(ResSAttnRNNModel.__init__) == Signature( parameters=[ Parameter( name='self', kind=Parameter.POSITIONAL_OR_KEYWORD, default=Parameter.empty, ), Parameter( name='d_emb', kind=Parameter.KEYWORD_ONLY, annotation=int, default=Parameter.empty, ), Parameter( name='d_hid', kind=Parameter.KEYWORD_ONLY, annotation=int, default=Parameter.empty, ), Parameter( name='n_hid_lyr', kind=Parameter.KEYWORD_ONLY, annotation=int, default=Parameter.empty, ), Parameter( name='n_post_hid_lyr', kind=Parameter.KEYWORD_ONLY, annotation=int, default=Parameter.empty, ), Parameter( name='n_pre_hid_lyr', kind=Parameter.KEYWORD_ONLY, annotation=int, default=Parameter.empty, ), Parameter( name='p_emb', kind=Parameter.KEYWORD_ONLY, annotation=float, default=Parameter.empty, ), Parameter( name='p_hid', kind=Parameter.KEYWORD_ONLY, annotation=float, default=Parameter.empty, ), Parameter( name='tknzr', kind=Parameter.KEYWORD_ONLY, annotation=BaseTknzr, default=Parameter.empty, ), Parameter( name='kwargs', kind=Parameter.VAR_KEYWORD, annotation=Optional[Dict], ), ], return_annotation=Signature.empty, )
def _create_workflow( self, pipeline_func: Callable, pipeline_name: Optional[Text] = None, pipeline_description: Optional[Text] = None, params_list: Optional[List[dsl.PipelineParam]] = None, pipeline_conf: Optional[dsl.PipelineConf] = None, ) -> Dict[Text, Any]: """ Internal implementation of create_workflow.""" params_list = params_list or [] # Create the arg list with no default values and call pipeline function. # Assign type information to the PipelineParam pipeline_meta = _extract_pipeline_metadata(pipeline_func) pipeline_meta.name = pipeline_name or pipeline_meta.name pipeline_meta.description = pipeline_description or pipeline_meta.description pipeline_name = sanitize_k8s_name(pipeline_meta.name) # Need to first clear the default value of dsl.PipelineParams. Otherwise, it # will be resolved immediately in place when being to each component. default_param_values = OrderedDict() if self._pipeline_root_param: params_list.append(self._pipeline_root_param) if self._pipeline_name_param: params_list.append(self._pipeline_name_param) for param in params_list: default_param_values[param.name] = param.value param.value = None args_list = [] kwargs_dict = dict() signature = inspect.signature(pipeline_func) for arg_name, arg in signature.parameters.items(): arg_type = None for input in pipeline_meta.inputs or []: if arg_name == input.name: arg_type = input.type break param = dsl.PipelineParam(sanitize_k8s_name(arg_name, True), param_type=arg_type) if arg.kind == inspect.Parameter.KEYWORD_ONLY: kwargs_dict[arg_name] = param else: args_list.append(param) with dsl.Pipeline(pipeline_name) as dsl_pipeline: pipeline_func(*args_list, **kwargs_dict) pipeline_conf = pipeline_conf or dsl_pipeline.conf # Configuration passed to the compiler is overriding. Unfortunately, it's not trivial to detect whether the dsl_pipeline.conf was ever modified. self._validate_exit_handler(dsl_pipeline) self._sanitize_and_inject_artifact(dsl_pipeline, pipeline_conf) # Fill in the default values by merging two param lists. args_list_with_defaults = OrderedDict() if pipeline_meta.inputs: args_list_with_defaults = OrderedDict([ (sanitize_k8s_name(input_spec.name, True), input_spec.default) for input_spec in pipeline_meta.inputs ]) if params_list: # Or, if args are provided by params_list, fill in pipeline_meta. for k, v in default_param_values.items(): args_list_with_defaults[k] = v pipeline_meta.inputs = pipeline_meta.inputs or [] for param in params_list: pipeline_meta.inputs.append( InputSpec( name=param.name, type=param.param_type, default=default_param_values[param.name])) op_transformers = [add_pod_env] pod_labels = {_SDK_VERSION_LABEL: kfp.__version__, _SDK_ENV_LABEL:_SDK_ENV_DEFAULT} op_transformers.append(add_pod_labels(pod_labels)) op_transformers.extend(pipeline_conf.op_transformers) if self._mode == dsl.PipelineExecutionMode.V2_COMPATIBLE: # Add self._pipeline_name_param and self._pipeline_root_param to ops inputs # if they don't exist already. for op in dsl_pipeline.ops.values(): insert_pipeline_name_param = True insert_pipeline_root_param = True for param in op.inputs: if param.name == self._pipeline_name_param.name: insert_pipeline_name_param = False elif param.name == self._pipeline_root_param.name: insert_pipeline_root_param = False if insert_pipeline_name_param: op.inputs.append(self._pipeline_name_param) if insert_pipeline_root_param: op.inputs.append(self._pipeline_root_param) workflow = self._create_pipeline_workflow( args_list_with_defaults, dsl_pipeline, op_transformers, pipeline_conf, ) from ._data_passing_rewriter import fix_big_data_passing workflow = fix_big_data_passing(workflow) workflow = _data_passing_rewriter.add_pod_name_passing( workflow, str(self._pipeline_root_param or None)) if pipeline_conf and pipeline_conf.data_passing_method != None: workflow = pipeline_conf.data_passing_method(workflow) metadata = workflow.setdefault('metadata', {}) annotations = metadata.setdefault('annotations', {}) labels = metadata.setdefault('labels', {}) annotations[_SDK_VERSION_LABEL] = kfp.__version__ annotations['pipelines.kubeflow.org/pipeline_compilation_time'] = datetime.datetime.now().isoformat() annotations['pipelines.kubeflow.org/pipeline_spec'] = json.dumps(pipeline_meta.to_dict(), sort_keys=True) if self._mode == dsl.PipelineExecutionMode.V2_COMPATIBLE: annotations['pipelines.kubeflow.org/v2_pipeline'] = "true" labels['pipelines.kubeflow.org/v2_pipeline'] = "true" # Labels might be logged better than annotations so adding some information here as well labels[_SDK_VERSION_LABEL] = kfp.__version__ return workflow
def udf( client, python_func, in_types, out_type, schema=None, replace=False, name=None, ): """Defines a UDF in the database Parameters ---------- client: PostgreSQLClient python_func: python function in_types: List[DataType] out_type : DataType schema: str - optionally specify the schema in which to define the UDF replace: bool - replace UDF in database if already exists name: str - name for the UDF to be defined in database Returns ------- Callable The ibis UDF object as a wrapped function """ if name is None: internal_name = python_func.__name__ else: internal_name = name signature = inspect.signature(python_func) parameter_names = signature.parameters.keys() replace_text = ' OR REPLACE ' if replace else '' schema_fragment = (schema + '.') if schema else '' template = """CREATE {replace} FUNCTION {schema_fragment}{name}({signature}) RETURNS {return_type} LANGUAGE plpythonu AS $$ {func_definition} return {internal_name}({args}) $$; """ postgres_signature = ', '.join( '{name} {type}'.format(name=name, type=ibis_to_postgres_str(type_)) for name, type_ in zip(parameter_names, in_types)) return_type = ibis_to_postgres_str(out_type) # If function definition is indented extra, # Postgres UDF will fail with indentation error. func_definition = dedent(inspect.getsource(python_func)) if func_definition.strip().startswith('@'): raise PostgresUDFError( 'Use of decorators on a function to be turned into Postgres UDF ' 'is not supported. The body of the UDF must be wholly ' 'self-contained. ') # Since the decorator syntax does not first bind # the function name to the wrapped function but instead includes # the decorator(s). Therefore, the decorators themselves will # be included in the string coming from `inspect.getsource()`. # Since the decorator objects are not defined, execution of the # UDF results in a NameError. formatted_sql = template.format( replace=replace_text, schema_fragment=schema_fragment, name=internal_name, signature=postgres_signature, return_type=return_type, func_definition=func_definition, # for internal_name, need to make sure this works if passing # name parameter internal_name=python_func.__name__, args=', '.join(parameter_names), ) client.con.execute(formatted_sql) return existing_udf( name=internal_name, input_types=in_types, output_type=out_type, schema=schema, parameters=parameter_names, )
def _create_component_spec_from_airflow_op( op_class: type, base_image: str = _default_airflow_base_image, result_output_name: str = 'Result', variables_dict_output_name: str = 'Variables', xcoms_dict_output_name: str = 'XComs', variables_to_output: List[str] = None, xcoms_to_output: List[str] = None, modules_to_capture: List[str] = None, ): variables_output_names = variables_to_output or [] xcoms_output_names = xcoms_to_output or [] modules_to_capture = modules_to_capture or [op_class.__module__] modules_to_capture.append(_run_airflow_op.__module__) output_names = [] if result_output_name is not None: output_names.append(result_output_name) if variables_dict_output_name is not None: output_names.append(variables_dict_output_name) if xcoms_dict_output_name is not None: output_names.append(xcoms_dict_output_name) output_names.extend(variables_output_names) output_names.extend(xcoms_output_names) from collections import namedtuple returnType = namedtuple('AirflowOpOutputs', output_names) def _run_airflow_op_closure(*op_args, **op_kwargs) -> returnType: (result, variables, xcoms) = _run_airflow_op(op_class, *op_args, **op_kwargs) output_values = {} import json if result_output_name is not None: output_values[result_output_name] = str(result) if variables_dict_output_name is not None: output_values[variables_dict_output_name] = json.dumps(variables) if xcoms_dict_output_name is not None: output_values[xcoms_dict_output_name] = json.dumps(xcoms) for name in variables_output_names: output_values[name] = variables[name] for name in xcoms_output_names: output_values[name] = xcoms[name] return returnType(**output_values) # Hacking the function signature so that correct component interface is generated import inspect parameters = inspect.signature(op_class).parameters.values() #Filtering out `*args` and `**kwargs` parameters that some operators have parameters = [ param for param in parameters if param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD ] sig = inspect.Signature( parameters=parameters, return_annotation=returnType, ) _run_airflow_op_closure.__signature__ = sig _run_airflow_op_closure.__name__ = op_class.__name__ return _func_to_component_spec(_run_airflow_op_closure, base_image=base_image, use_code_pickling=True, modules_to_capture=modules_to_capture)
def test_inherent_method(): r'''Ensure inherent methods' signature are same as base class.''' assert ( inspect.signature(SAttnRNNModel.forward) == inspect.signature(ResSAttnRNNModel.forward) ) assert ( inspect.signature(SAttnRNNModel.load) == inspect.signature(ResSAttnRNNModel.load) ) assert ( inspect.signature(SAttnRNNModel.loss_fn) == inspect.signature(ResSAttnRNNModel.loss_fn) ) assert ( inspect.signature(SAttnRNNModel.pred) == inspect.signature(ResSAttnRNNModel.pred) ) assert ( inspect.signature(SAttnRNNModel.ppl) == inspect.signature(ResSAttnRNNModel.ppl) ) assert ( inspect.signature(SAttnRNNModel.save) == inspect.signature(ResSAttnRNNModel.save) ) assert ( inspect.signature(SAttnRNNModel.train_parser) == inspect.signature(ResSAttnRNNModel.train_parser) )
# -*- coding: utf-8 -*- # @Time : 2019/7/10 23:38 # @Author : songxy # @Email : [email protected] # @File : 3.partial.py from functools import partial import inspect #### partial 函数对函数参数进行固定返回一个新函数 def add(x,y): return x+y newadd = partial(add,x=50) print(newadd(y=100,x=2222)) sig = inspect.signature(newadd) params = sig.parameters.items() print(params)
def _init_graph(self) -> None: assert self._var_inits is not None assert self._input_templates is None assert self._output_templates is None assert self._own_vars is None # Initialize components. if self._components is None: self._components = util.EasyDict() # Choose build func kwargs. build_kwargs = dict(self.static_kwargs) build_kwargs["is_template_graph"] = True build_kwargs["components"] = self._components # Override scope and device, and ignore surrounding control dependencies. with tfutil.absolute_variable_scope( self.scope, reuse=False), tfutil.absolute_name_scope( self.scope), tf.device( self.device), tf.control_dependencies(None): assert tf.get_variable_scope().name == self.scope assert tf.get_default_graph().get_name_scope() == self.scope # Create input templates. self._input_templates = [] for param in inspect.signature( self._build_func).parameters.values(): if param.kind == param.POSITIONAL_OR_KEYWORD and param.default is param.empty: self._input_templates.append( tf.placeholder(tf.float32, name=param.name)) # Call build func. out_expr = self._build_func(*self._input_templates, **build_kwargs) # Collect output templates and variables. assert tfutil.is_tf_expression(out_expr) or isinstance(out_expr, tuple) self._output_templates = [ out_expr ] if tfutil.is_tf_expression(out_expr) else list(out_expr) self._own_vars = OrderedDict( (var.name[len(self.scope) + 1:].split(":")[0], var) for var in tf.global_variables(self.scope + "/")) # Check for errors. if len(self._input_templates) == 0: raise ValueError("Network build func did not list any inputs.") if len(self._output_templates) == 0: raise ValueError("Network build func did not return any outputs.") if any(not tfutil.is_tf_expression(t) for t in self._output_templates): raise ValueError("Network outputs must be TensorFlow expressions.") if any(t.shape.ndims is None for t in self._input_templates): raise ValueError( "Network input shapes not defined. Please call x.set_shape() for each input." ) if any(t.shape.ndims is None for t in self._output_templates): raise ValueError( "Network output shapes not defined. Please call x.set_shape() where applicable." ) if any(not isinstance(comp, Network) for comp in self._components.values()): raise ValueError( "Components of a Network must be Networks themselves.") if len(self._components) != len( set(comp.name for comp in self._components.values())): raise ValueError("Components of a Network must have unique names.") # Initialize variables. if len(self._var_inits): tfutil.set_vars({ self._get_vars()[name]: value for name, value in self._var_inits.items() if name in self._get_vars() }) remaining_inits = [ var.initializer for name, var in self._own_vars.items() if name not in self._var_inits ] if self._all_inits_known: assert len(remaining_inits) == 0 else: tfutil.run(remaining_inits) self._var_inits = None
def default_attributes(cls) -> dict: init_signature = inspect.signature(cls) return {k: v.default for k, v in init_signature.parameters.items()}
def _format_args(func: Callable[..., Any]) -> str: return str(signature(func))
def model_setup(algorithm_name, env, policy, **kwargs): """ This function takes in the algorithm name specified by the user in the CLI, the environment to train on, the policy, and finally an optional dictionary of parameters to set for the agent. Parameter(s): ------------- algorithm_name: type(String) The name of the algorithm to be used. Must be supported in stable baselines. env: type(Gym) The environment to train the agent on. Must be of type openAI Gym. policy: type(String) The policy to train the agent with. Not all policies are compatible with all algorithms. kwargs: type(dict) Dictionary of algorithm variables that can be used instead of the default values. Returns: -------- model: type(Object) A machine learning model """ assert algorithm_name in list_of_algorithms, 'Algorithm must be supported by stable baselines' model = None if algorithm_name in [list_of_algorithms[0]]: # TD3 algorithm # Get the default values in case no user specifications signature = inspect.signature(TD3.__init__) g = kwargs.pop('g', signature.parameters['gamma'].default) bf = kwargs.pop('bf', signature.parameters['buffer_size'].default) nstd = kwargs.pop('nstd', signature.parameters['target_policy_noise'].default) lst = kwargs.pop('lst', signature.parameters['learning_starts'].default) bch = kwargs.pop('bch', signature.parameters['batch_size'].default) lr = kwargs.pop('lr', signature.parameters['learning_rate'].default) tf = kwargs.pop('tf', signature.parameters['train_freq'].default) grad = kwargs.pop('grad', signature.parameters['gradient_steps'].default) pkwargs = kwargs.pop('pkwargs', signature.parameters['policy_kwargs'].default) v = kwargs.pop('v', signature.parameters['verbose'].default) model = TD3(policy=policy, env=env, gamma=g, buffer_size=bf, target_policy_noise=nstd, learning_starts=lst, batch_size=bch, learning_rate=lr, train_freq=tf, gradient_steps=grad, verbose=v, policy_kwargs=pkwargs) elif algorithm_name in [list_of_algorithms[1]]: # DDPG algorithm pass elif algorithm_name in [list_of_algorithms[2]]: # PPO2 algorithm # Get the default values in case no user specifications signature = inspect.signature(PPO2.__init__) lr = kwargs.pop('lr', signature.parameters['learning_rate'].default) nsteps = kwargs.pop('nsteps', signature.parameters['n_steps'].default) nbtch = kwargs.pop('nbtch', signature.parameters['nminibatches'].default) lbd = kwargs.pop('lbd', signature.parameters['lam'].default) g = kwargs.pop('g', signature.parameters['gamma'].default) nep = kwargs.pop('nep', signature.parameters['noptepochs'].default) ent = kwargs.pop('ent', signature.parameters['ent_coef'].default) cl = kwargs.pop('cl', signature.parameters['cliprange'].default) v = kwargs.pop('v', signature.parameters['verbose'].default) model = PPO2(policy=policy, env=env, learning_rate=lr, verbose=v, ent_coef=ent, lam=lbd, gamma=g, n_steps=nsteps, nminibatches=nbtch, noptepochs=nep, cliprange=cl) elif algorithm_name in [list_of_algorithms[3]]: # PPO1 algorithm pass elif algorithm_name in [list_of_algorithms[4]]: # A2C algorithm pass elif algorithm_name in [list_of_algorithms[5]]: # ACER algorithm pass elif algorithm_name in [list_of_algorithms[6]]: # ACKTR algorithm pass elif algorithm_name in [list_of_algorithms[7]]: # DQN algorithm pass elif algorithm_name in [list_of_algorithms[8]]: # GAIL algorithm pass elif algorithm_name in [list_of_algorithms[9]]: # HER algorithm pass elif algorithm_name in [list_of_algorithms[10]]: # SAC algorithm # Get the default values in case no user specifications signature = inspect.signature(SAC.__init__) lr = kwargs.pop('lr', signature.parameters['learning_rate'].default) bf = kwargs.pop('bf', signature.parameters['buffer_size'].default) bch = kwargs.pop('bch', signature.parameters['batch_size'].default) ent = kwargs.pop('ent', signature.parameters['ent_coef'].default) tf = kwargs.pop('tf', signature.parameters['train_freq'].default) grad = kwargs.pop('grad', signature.parameters['gradient_steps'].default) lst = kwargs.pop('lst', signature.parameters['learning_starts'].default) v = kwargs.pop('v', signature.parameters['verbose'].default) model = SAC(policy=policy, env=env, learning_rate=lr, buffer_size=bf, batch_size=bch, ent_coef=ent, train_freq=tf, gradient_steps=grad, learning_starts=lst, verbose=v) elif algorithm_name in [list_of_algorithms[11]]: # TRPO algorithm pass return model