def transform(self, transformation, **kwargs):
"""Run transform operation as defined in the workspace.
Args:
transformation: Function or method used to perform a transformation upon
the transform-dataset.
**kwargs: Arbitrary keyword arguments; passed through to the transformation.
Returns:
arcetl.etl.ArcETL: Reference to the instance.
"""
# Unless otherwise stated, dataset path is self.transform_path.
kwargs.setdefault("dataset_path", self.transform_path)
# Add output_path to kwargs if needed.
if "output_path" in funcsigs.signature(transformation).parameters:
kwargs.setdefault(
"output_path",
unique_path(getattr(transformation, "__name__", "transform")),
)
transformation(**kwargs)
# If there"s a new output, replace old transform.
if "output_path" in funcsigs.signature(transformation).parameters:
if dataset.is_valid(self.transform_path):
dataset.delete(self.transform_path, log_level=None)
self.transform_path = kwargs["output_path"]
return self
def test_signature_on_callable_objects(self):
class Foo(object):
def __call__(self, a):
pass
self.assertEqual(self.signature(Foo()),
((('a', Ellipsis, Ellipsis, "positional_or_keyword"),),
Ellipsis))
class Spam(object):
pass
with self.assertRaisesRegex(TypeError, "is not a callable object"):
inspect.signature(Spam())
class Bar(Spam, Foo):
pass
self.assertEqual(self.signature(Bar()),
((('a', Ellipsis, Ellipsis, "positional_or_keyword"),),
Ellipsis))
class ToFail(object):
__call__ = type
with self.assertRaisesRegex(ValueError, "not supported by signature"):
inspect.signature(ToFail())
if sys.version_info[0] < 3:
return
class Wrapped(object):
pass
Wrapped.__wrapped__ = lambda a: None
self.assertEqual(self.signature(Wrapped),
((('a', Ellipsis, Ellipsis, "positional_or_keyword"),),
Ellipsis))
def test_signature_bound_arguments_apply_defaults_common(self):
def foo(a, b=1, *args, **kw): pass
sig = inspect.signature(foo)
ba = sig.bind(20)
ba.apply_defaults()
self.assertEqual(
list(ba.arguments.items()),
[('a', 20), ('b', 1), ('args', ()), ('kw', {})])
# Make sure that BoundArguments produced by bind_partial()
# are supported.
def foo(a, b): pass
sig = inspect.signature(foo)
ba = sig.bind_partial(20)
ba.apply_defaults()
self.assertEqual(
list(ba.arguments.items()),
[('a', 20)])
# Test no args
def foo(): pass
sig = inspect.signature(foo)
ba = sig.bind()
ba.apply_defaults()
self.assertEqual(list(ba.arguments.items()), [])
# Make sure a no-args binding still acquires proper defaults.
def foo(a='spam'): pass
sig = inspect.signature(foo)
ba = sig.bind()
ba.apply_defaults()
self.assertEqual(list(ba.arguments.items()), [('a', 'spam')])
def test_mixin_has_list_method(self):
mixin = RestFrameworkGenericViewSetAutoMetricsMixin()
function = getattr(mixin, 'list')
self.assertTrue(callable(function))
self.assertEqual(len(signature(function).parameters), 3)
self.assertIsNotNone(signature(function).parameters['request'])
self.assertIsNotNone(signature(function).parameters['args'])
self.assertIsNotNone(signature(function).parameters['kwargs'])
def fails_binding(func, args, kwargs):
"""Check if the given `args` and `kwargs` fails binding to `func`.
:returns: False if bound successfully, string message if failed.
"""
try:
signature(func).bind(*args, **kwargs)
return False
except TypeError as error:
return error.args[0] if error.args and error.args[0] else "Failed to bind"
def inspect_signature(cls, subject):
positional_names = []
keyword_names = []
accepts_args = False
accepts_kwargs = False
if not isinstance(subject, type):
subject = type(subject)
if platform.python_implementation() == 'PyPy':
# Under PyPy, funcsigs can't automatically find the correct
# initializer or constructor, so must be told what's desired.
# PyPy also seems to create function objects that don't exist
# under CPython, so we have to check for the existence of a
# filename to prove that those functions / methods are actually
# being pulled from code that exists on the filesystem.
if hasattr(subject.__init__.func_code, 'co_filename'):
signature = funcsigs.signature(subject.__init__)
elif hasattr(subject.__new__.func_code, 'co_filename'):
signature = funcsigs.signature(subject.__new__)
else:
return (
positional_names,
keyword_names,
accepts_args,
accepts_kwargs,
)
else:
try:
signature = funcsigs.signature(subject)
except ValueError:
return (
positional_names,
keyword_names,
accepts_args,
accepts_kwargs,
)
for name, parameter in signature.parameters.items():
if parameter.kind == funcsigs._POSITIONAL_OR_KEYWORD:
if parameter.default == parameter.empty:
positional_names.append(name)
else:
keyword_names.append(name)
elif parameter.kind == funcsigs._VAR_POSITIONAL:
accepts_args = True
elif parameter.kind == funcsigs._VAR_KEYWORD:
accepts_kwargs = True
if platform.python_implementation() == 'PyPy':
# Funcsigs under PyPy doesn't discard cls or self.
if positional_names:
positional_names.pop(0)
return (
positional_names,
keyword_names,
accepts_args,
accepts_kwargs,
)
def format_traceback(e):
import types
import traceback
import sys
from funcsigs import signature
def log_to_str(v):
if isinstance(v, bytes):
return ''.join(["'", v.replace('\n', '\\n'), "'"])
else:
try:
return str(v).replace('\n', '\\n')
except:
return '<ERROR: CANNOT PRINT>'
frame = sys.exc_info()[2]
formattedTb = traceback.format_tb(frame)
exception_string = "Globals:\n"
for k, v in list(frame.tb_frame.f_globals.items()):
exception_string += "{:>4}{:<20}:{}{:<.100}\n".format(
"", k, " ", log_to_str(v))
exception_string += "Traceback:\n"
while frame:
this_frame_tb = formattedTb.pop(0)
exception_string += this_frame_tb
call_re_search = re.search("^[\t ]+(.+)\(.*?\)$", this_frame_tb, re.M)
co_name = frame.tb_frame.f_code.co_name
if call_re_search and call_re_search.group(1) in list(
frame.tb_frame.f_locals.keys()) + list(
frame.tb_frame.f_globals.keys()):
call_name = call_re_search.group(1)
if call_name in frame.tb_frame.f_locals:
if call_name != frame.tb_frame.f_locals[call_name].__name__:
exception_string = exception_string[:-1]
exception_string += " => {}{}\n".format(
frame.tb_frame.f_locals[call_name].__name__,
str(signature(frame.tb_frame.f_locals[call_name])))
elif co_name in frame.tb_frame.f_globals:
if call_name != frame.tb_frame.f_globals[call_name].__name__:
exception_string = exception_string[:-1]
exception_string += " => {}{}\n".format(
frame.tb_frame.f_globals[call_name].__name__,
str(signature(frame.tb_frame.f_globals[call_name])))
exception_string += " Locals:\n"
for k, v in list(frame.tb_frame.f_locals.items()):
exception_string += "{:<8}{:<20}:{}{:<.100}\n".format(
"", k, " ", log_to_str(v))
frame = frame.tb_next
exception_string += 'Exception thrown, {}: {}\n'.format(type(e), str(e))
return exception_string
def _validate_arguments_against_signature(self, func):
"""Check if arguments match a function signature and can therefore be
passed to it.
:param func: The function object.
:param args: List of positional arguments (or None).
:param kwargs: Dict of keyword arguments (or None).
:raises InvalidParams: If the arguments cannot be passed to the
function.
"""
try:
signature(func).bind(*(self.args or []), **(self.kwargs or {}))
except TypeError as exc:
raise InvalidParams(str(exc))
def test_spec_inspect_signature(self):
def myfunc(x, y): pass
mock = create_autospec(myfunc)
mock(1, 2)
mock(x=1, y=2)
if six.PY2:
self.assertEqual(funcsigs.signature(mock), funcsigs.signature(myfunc))
else:
self.assertEqual(inspect.getfullargspec(mock), inspect.getfullargspec(myfunc))
self.assertEqual(mock.mock_calls, [call(1, 2), call(x=1, y=2)])
self.assertRaises(TypeError, mock, 1)
async def dispatch(self, transport, request: JSONRPCRequest):
if self.has_hevents and request.method == 'rpc.on':
return await self.method_subscribe(transport, request.params[0])
elif self.has_hevents and request.method == 'rpc.off':
return await self.method_unsubscribe(transport, request.params[0])
method = self.get_method(request.method)
try:
params = request.params
except AttributeError:
params = None
if inspect.isawaitable(method) or inspect.iscoroutinefunction(method):
# print("is awaitable", method)
if params is None:
return await method()
if isinstance(params, list):
signature(method).bind(*params)
return await method(*params)
elif isinstance(params, dict):
signature(method).bind(**params)
return await method(**params)
else:
# print("not awaitable", method)
if params is None:
return method()
if isinstance(params, list):
signature(method).bind(*params)
return method(*params)
elif isinstance(params, dict):
signature(method).bind(**params)
return method(**params)
def _validate_arguments_against_signature(self, func):
"""Check if arguments match a function signature and can therefore be
passed to it.
:param func: The function object.
:param args: List of positional arguments (or None).
:param kwargs: Dict of keyword arguments (or None).
:raises InvalidParams: If the arguments cannot be passed to the
function.
"""
try:
signature(func).bind(*(self.args or []), **(self.kwargs or {}))
except TypeError as exc:
raise InvalidParams(str(exc))
def add_queue_if_needed(c_queue, param_combination):
param_combination_copy = param_combination.copy()
args = signature(parameter_creator_function).parameters
if "queue" in args:
param_combination_copy.update({"queue": c_queue})
return param_combination_copy
def bind_function_to_object(f, obj):
if 'self' not in signature(f).parameters.keys():
raise ValueError('%s does not have a self argument' % f)
setattr(obj, f.__name__, MethodType(f, None, obj))
return obj
def get_signature_params(func):
"""Get signature parameters
Support Cython functions by grabbing relevant attributes from the Cython
function and attaching to a no-op function. This is somewhat brittle, since
funcsigs may change, but given that funcsigs is written to a PEP, we hope
it is relatively stable. Future versions of Python may allow overloading
the inspect 'isfunction' and 'ismethod' functions / create ABC for Python
functions. Until then, it appears that Cython won't do anything about
compatability with the inspect module.
Args:
func: The function whose signature should be checked.
Raises:
TypeError: A type error if the signature is not supported
"""
# The first condition for Cython functions, the latter for Cython instance
# methods
if is_cython(func):
attrs = ["__code__", "__annotations__",
"__defaults__", "__kwdefaults__"]
if all([hasattr(func, attr) for attr in attrs]):
original_func = func
def func(): return
for attr in attrs:
setattr(func, attr, getattr(original_func, attr))
else:
raise TypeError("{0!r} is not a Python function we can process"
.format(func))
return list(funcsigs.signature(func).parameters.items())
def clean(self, outstream, dryrun):
"""Execute task's clean
@ivar outstream: 'write' output into this stream
@ivar dryrun (bool): if True clean tasks are not executed
(just print out what would be executed)
"""
self.init_options()
# if clean is True remove all targets
if self._remove_targets is True:
clean_targets(self, dryrun)
else:
# clean contains a list of actions...
for action in self.clean_actions:
msg = "%s - executing '%s'\n"
outstream.write(msg % (self.name, action))
# add extra arguments used by clean actions
if isinstance(action, PythonAction):
#action_sig = inspect.signature(action.py_callable) # KGEN deletion
action_sig = signature(action.py_callable) # KGEN addition
if 'dryrun' in action_sig.parameters:
action.kwargs['dryrun'] = dryrun
if not dryrun:
result = action.execute(out=outstream)
if isinstance(result, CatchedException):
sys.stderr.write(str(result))
def test_signature_unhashable(self):
def foo(a):
pass
sig = inspect.signature(foo)
with self.assertRaisesRegex(TypeError, "unhashable type"):
hash(sig)
def _get_output_map(stage_name, cmd_fxn, tags, input_map, task_output_dir, execution_output_dir):
sig = funcsigs.signature(cmd_fxn)
for param_name, param in sig.parameters.iteritems():
if param_name not in tags and param.default is funcsigs._empty:
raise ValueError("Required output file parameter `%s` not specified for %s." % (param_name, stage_name))
value = tags.get(param_name, param.default)
if isinstance(value, Forward):
forward_instance = value
try:
input_value = input_map[forward_instance.input_parameter_name]
except KeyError:
raise KeyError(
"Cannot forward name `%s`,it is not a valid input parameter of "
"%s in stage %s" % (forward_instance.input_parameter_name, cmd_fxn, stage_name)
)
yield param_name, input_value
elif isinstance(value, OutputDir):
output_dir_instance = value
if task_output_dir is not None:
if output_dir_instance.prepend_execution_output_dir:
task_output_dir = os.path.join(execution_output_dir, task_output_dir)
output_file = os.path.join(task_output_dir, output_dir_instance.basename.format(**tags))
else:
output_file = output_dir_instance.basename.format(**tags)
# output_file = value.format(**tags)
yield param_name, output_file
elif param_name.startswith("out_"):
if isinstance(value, str):
yield param_name, value
def help_commands(self):
''' Returns a big string of Workbench commands and signatures '''
help_string = 'Workbench Commands:'
for name, meth in inspect.getmembers(self, predicate=inspect.ismethod):
if not name.startswith('_'):
help_string += '\n\t%s%s' % (name,funcsigs.signature(meth))
return help_string
def __init__(self, conffile=None):
if conffile is not None:
if isinstance(conffile, dict):
self.conf = copy.deepcopy(conffile)
else:
with io.open(conffile, encoding='utf-8') as f:
self.conf = yaml.safe_load(f)
assert isinstance(self.conf, dict), type(self.conf)
else:
self.conf = {'mode': 'sequential', 'process': []}
self.functions = OrderedDict()
if self.conf.get('mode', 'sequential') == 'sequential':
for idx, process in enumerate(self.conf['process']):
assert isinstance(process, dict), type(process)
opts = dict(process)
process_type = opts.pop('type')
class_obj = dynamic_import(process_type, import_alias)
# TODO(karita): assert issubclass(class_obj, TransformInterface)
check_kwargs(class_obj, opts)
try:
self.functions[idx] = class_obj(**opts)
except TypeError:
try:
signa = signature(class_obj)
except ValueError:
# Some function, e.g. built-in function, are failed
pass
else:
logging.error('Expected signature: {}({})'.format(
class_obj.__name__, signa))
raise
else:
raise NotImplementedError('Not supporting mode={}'.format(
self.conf['mode']))
def decorator(f):
sig = signature(f)
if file_arg_name in sig.parameters:
raise ValueError('{} already has a parameter named {}'
.format(f, file_arg_name))
@wraps(f)
def _(*args, **kwargs):
# remove file_arg_name from function list
a_file = kwargs.pop(file_arg_name, None)
# bind remaining arguments
pbargs = sig.bind_partial(*args, **kwargs)
# get data argument
a_data = pbargs.arguments.get(argname, None)
# if a Data object is already being passed in, use it
# instead of creating a new instance
if a_file is None and isinstance(a_data, Data):
d = a_data
else:
# create data replacement
d = Data(data=a_data, file=a_file)
# replace with data instance
pbargs.parameters[argname] = d
# call original function with instantiated data argument
return f(*pbargs.args, **pbargs.kwargs)
return _
def run(func, *args, **kwargs):
"""
Similar to bash_call, but actually just returns a string that is the source code of this function instead of importing it.
"""
import inspect
import pprint
source = inspect.getsource(func)
sig = funcsigs.signature(func)
kwargs = dict(zip(sig.parameters.keys(), args))
return r"""
python - <<EOF
{soource}
{func.__name__}(**
{param_str}
)
EOF""".format(func=func,
source=source,
param_str=pprint.pformat(kwargs, width=1, indent=1))
def nonmutating(fn, *argnames):
sig = funcsigs.signature(fn)
mutating = dict((name, None) for name in sig.parameters)
for name in argnames:
mutating[name] = False
fn._peval_mutating = mutating
return fn
def __init__(self, conffile=None):
if conffile is not None:
if isinstance(conffile, dict):
self.conf = copy.deepcopy(conffile)
else:
with io.open(conffile, encoding="utf-8") as f:
self.conf = yaml.safe_load(f)
assert isinstance(self.conf, dict), type(self.conf)
else:
self.conf = {"mode": "sequential", "process": []}
self.functions = OrderedDict()
if self.conf.get("mode", "sequential") == "sequential":
for idx, process in enumerate(self.conf["process"]):
assert isinstance(process, dict), type(process)
opts = dict(process)
process_type = opts.pop("type")
class_obj = dynamic_import(process_type, import_alias)
# TODO(karita): assert issubclass(class_obj, TransformInterface)
try:
self.functions[idx] = class_obj(**opts)
except TypeError:
try:
signa = signature(class_obj)
except ValueError:
# Some function, e.g. built-in function, are failed
pass
else:
logging.error("Expected signature: {}({})".format(
class_obj.__name__, signa))
raise
else:
raise NotImplementedError("Not supporting mode={}".format(
self.conf["mode"]))
def run(func, *args, **kwargs):
"""
Similar to bash_call, but actually just returns a string that is the source code of this function instead of importing it.
"""
import inspect
import pprint
source = inspect.getsource(func)
sig = funcsigs.signature(func)
kwargs = dict(zip(sig.parameters.keys(), args))
return r"""
python - <<EOF
{soource}
{func.__name__}(**
{param_str}
)
EOF""".format(func=func,
source=source,
param_str=pprint.pformat(kwargs, width=1, indent=1))
def get_allowed_args(fn_or_class):
# type: (Union[Callable, Type]) -> Tuple[List[str], Dict[str, Any]]
"""
Given a callable or a class, returns the arguments and default kwargs passed in.
:param Union[Callable, Type] fn_or_class: A function, method or class to inspect.
:return: A 2-tuple with a list of arguments and a dictionary of keywords mapped to
default values.
:rtype: Tuple[List[str], Dict[str, Any]]
"""
try:
signature = inspect.signature(fn_or_class)
except AttributeError:
import funcsigs
signature = funcsigs.signature(fn_or_class)
args = []
kwargs = {}
for arg, param in signature.parameters.items():
if (param.kind
in (param.POSITIONAL_OR_KEYWORD,
param.POSITIONAL_ONLY)) and param.default is param.empty:
args.append(arg)
else:
kwargs[
arg] = param.default if param.default is not param.empty else None
return args, kwargs
def __init__(self, clsname, bases, clsdict):
super(MatchSignaturesMeta, self).__init__(clsname, bases, clsdict)
sup = super(self, self) # type: ignore # See python/mypy #857
for name, value in clsdict.items():
if name.startswith('_') or not callable(value):
continue
# Get the previous definition (if any) and compare the signatures
prev_dfn = getattr(sup, name, None)
if prev_dfn:
prev_sig = inspect.signature(prev_dfn)
val_sig = inspect.signature(value)
if prev_sig != val_sig:
value_name = getattr(value, '__qualname__', value.__name__)
logging.warning('Signature mismatch in %s. %s != %s',
value_name, prev_sig, val_sig)
def endpoint_evaluator(endpoint):
"""
Function to evaluate endpoint functions and and return the url and methods if they comply
:param endpoint: The endpoint function to be allowed.
:return: url and methods specified in the function
"""
args = signature(endpoint).parameters
try:
url = args['url'].default
except KeyError:
raise EndpointException('Url for endpoint "{}" not supplied.'.format(
endpoint.__name__))
if not isinstance(url, str):
raise EndpointException('Url for endpoint "{}" is not a string'.format(
endpoint.__name__))
try:
methods = args['methods'].default
except KeyError:
# default method 'GET' is applied.
methods = ['GET']
allowed_methods = ['PUT', 'GET', 'POST', 'DELETE', 'PATCH']
for method in methods:
if method not in allowed_methods:
raise EndpointException(
'Supplied methods for "{}" endpoint is invalid.'
'Allowed methods are PUT, GET, POST, DELETE, PATCH'.format(
endpoint.__name__))
return url, methods
def bash_call(func, *args, **kwargs):
"""
Experimental way to not have to write boiler plate argparse code. Converts the function call to a bash script, when will be subsequently submitted
like a normal command.
Current Limitations:
* function must be importable from anywhere in the VE
* This means no partials!!! Parameters must all be passed as tags :(
"""
# decorator.decorator passes everything as *args, use function signature to turn it into kwargs which is more explicit
import pprint
import json
from collections import OrderedDict
sig = funcsigs.signature(func)
kwargs = dict(zip(sig.parameters.keys(), args))
return r"""
python - <<EOF
from {func.__module__} import {func.__name__}
{func.__name__}(**
{param_str}
)
EOF""".format(func=func,
param_str=pprint.pformat(kwargs, width=1, indent=1)) # todo assert values are basetypes
def _get_output_map(stage_name, cmd_fxn, tags, input_map, task_output_dir, execution_output_dir):
sig = funcsigs.signature(cmd_fxn)
for param_name, param in sig.parameters.iteritems():
if param_name.startswith('out_'):
value = tags.get(param_name, param.default)
if value == funcsigs._empty:
raise ValueError('Required output file parameter `%s` not specified for %s.' % (param_name, stage_name))
elif isinstance(value, Forward):
forward_instance = value
try:
input_value = input_map[forward_instance.input_parameter_name]
except KeyError:
raise KeyError('Cannot forward name `%s`,it is not a valid input parameter of '
'%s in stage %s' % (forward_instance.input_parameter_name, cmd_fxn, stage_name))
yield param_name, input_value
elif isinstance(value, OutputDir):
output_dir_instance = value
if task_output_dir is not None:
if output_dir_instance.prepend_execution_output_dir:
task_output_dir = os.path.join(execution_output_dir, task_output_dir)
output_file = os.path.join(task_output_dir, output_dir_instance.basename.format(**tags))
else:
output_file = output_dir_instance.basename.format(**tags)
# output_file = value.format(**tags)
yield param_name, output_file
elif isinstance(value, basestring):
# allows a user to remove an output_file by passing None for its value
yield param_name, value
def __inf_with_proposal_vars(self):
import funcsigs
return 'proposal_vars' in funcsigs.signature(getattr(ed.inferences, self.infMethod)).parameters
#import inspect # not working in 2.7
#return 'proposal_vars' in inspect.signature(getattr(ed.inferences, self.infMethod)).parameters
def run(self, **passedkwargs):
""" Runs the tests that are defined in the config object.
Returns a dictionary of the results as defined by the config
"""
results = OrderedDict()
for modu, tests in self.config.items():
try:
testpackage = import_module('ioos_qc.{}'.format(modu))
except ImportError:
raise ValueError('No ioos_qc test package "{}" was found, skipping.'.format(modu))
results[modu] = OrderedDict()
for testname, kwargs in tests.items():
if not hasattr(testpackage, testname):
L.warning('No test named "{}.{}" was found, skipping'.format(modu, testname))
elif kwargs is None:
L.debug('Test "{}.{}" had no config, skipping'.format(modu, testname))
else:
# Get our own copy of the kwargs object so we can change it
testkwargs = deepcopy(passedkwargs)
# Merges dicts
testkwargs = dict(kwargs, **testkwargs) # noqa
# Get the arguments that the test functions support
runfunc = getattr(testpackage, testname)
sig = signature(runfunc)
valid_keywords = [ p.name for p in sig.parameters.values() if p.kind == p.POSITIONAL_OR_KEYWORD ]
testkwargs = { k: v for k, v in testkwargs.items() if k in valid_keywords }
results[modu][testname] = runfunc(**testkwargs) # noqa
return results
def makespec_args():
names = ['datas'] # signature does not detect datas for some reason
for name, parameter in signature(makespec_main).parameters.items():
if name not in (excluded_args + ['args', 'kwargs']):
names.append(name)
return names
def call(cmd_fxn, task, input_map, output_map):
sig = funcsigs.signature(cmd_fxn)
def gen_params():
for param_name, param in sig.parameters.iteritems():
if param_name in input_map:
yield param_name, input_map[param_name]
elif param_name in output_map:
yield param_name, output_map[param_name]
elif param_name in task.tags:
yield param_name, task.tags[param_name]
elif param.default != funcsigs._empty:
yield param_name, param.default
else:
raise AttributeError('%s requires the parameter `%s`, are you missing a tag? Either provide a default in the cmd() '
'method signature, or pass a value for `%s` with a tag' % (cmd_fxn, param_name, param_name))
kwargs = dict(gen_params())
if 'out_ploidy' in kwargs:
import IPython;
IPython.embed()
out = cmd_fxn(**kwargs)
for param_name in ['cpu_req', 'mem_req', 'drm']:
if param_name in sig.parameters:
param_val = kwargs.get(param_name, sig.parameters[param_name].default)
setattr(task, param_name, param_val)
assert isinstance(out, str) or out is None, 'cmd_fxn %s did not return a str or None' % cmd_fxn
return out
def get_string(cls, device, strname):
"""Retrieve a string from the USB device, dealing with PyUSB API breaks
:param device: USB device instance
:type device: usb.core.Device
:param str strname: the string identifier
:return: the string read from the USB device
:rtype: str
"""
if cls.UsbApi is None:
try:
import inspect
args, varargs, varkw, defaults = \
inspect.signature(usb.core.Device.read).parameters
except AttributeError:
import funcsigs
args, varargs, varkw, defaults = funcsigs.signature(
usb.core.Device.read).parameters
if (len(args) >= 3) and args[1] == 'length':
cls.UsbApi = 1
else:
cls.UsbApi = 2
if cls.UsbApi == 2:
return usb.util.get_string(device, strname)
else:
return usb.util.get_string(device, 64, strname)
def run(parser):
args = parser.parse_args()
available_apps = list_apps()
if not args.name:
print json.dumps(available_apps)
elif (args.name in available_apps) and args.jsonrpc:
# Care must be taken that imported modules
# do not write to std out or it will break the jsonrps parsing
with Capturing() as output:
func_dict = dict(app_functions(args.name))
result = dispatch(func_dict, json.loads(args.jsonrpc))
out = '\n'.join(output)
print json.dumps({"result": result, "out": out})
elif args.name in available_apps:
funcs = app_functions(args.name)
doc_dict = {}
for name, func in funcs:
doc = inspect.getdoc(func)
sig = name + str(signature(func))
doc_dict[name] = sig
if doc:
doc_dict[name] += '\n\n' + doc
print json.dumps(doc_dict)
else:
print args
def __bind_commands(self):
if not self.parallel:
for attr in ['complete_kill', 'do_kill', 'do_status']:
delattr(FrameworkConsole, attr)
for name, func in get_commands():
longname = 'do_{}'.format(name)
# set the behavior of the console command (multi-processed or not)
# setattr(Console, longname, MethodType(FrameworkConsole.start_process_template(func) \
# if self.parallel and func.behavior.is_multiprocessed else func, self))
setattr(Console, longname, MethodType(func, self))
# retrieve parts of function's docstring to make console command's docstring
parts = func.__doc__.split(':param ')
description = parts[0].strip()
arguments = [" ".join([l.strip() for l in x.split(":")[-1].split('\n')]) for x in parts[1:]]
docstring = COMMAND_DOCSTRING["description"].format(description)
if len(arguments) > 0:
arg_descrs = [' - {}:\t{}'.format(n, d or "[no description]") \
for n, d in list(zip_longest(signature(func).parameters.keys(), arguments or []))]
docstring += COMMAND_DOCSTRING["arguments"].format('\n'.join(arg_descrs))
if hasattr(func, 'examples') and isinstance(func.examples, list):
args_examples = [' >>> {} {}'.format(name, e) for e in func.examples]
docstring += COMMAND_DOCSTRING["examples"].format('\n'.join(args_examples))
setattr(getattr(getattr(Console, longname), '__func__'), '__doc__', docstring)
# set the autocomplete list of values (can be lazy by using lambda) if relevant
if hasattr(func, 'autocomplete'):
setattr(Console, 'complete_{}'.format(name),
MethodType(FrameworkConsole.complete_template(func.autocomplete), self))
if hasattr(func, 'reexec_on_emptyline') and func.reexec_on_emptyline:
self.reexec.append(name)
def test_signature_unhashable(self):
def foo(a):
pass
sig = inspect.signature(foo)
with self.assertRaisesRegex(TypeError, 'unhashable type'):
hash(sig)
def client(self, client):
self._initialize_session()
service_name, client_name = client.rsplit('.', 1)
svc_module = importlib.import_module(service_name)
klass = getattr(svc_module, client_name)
klass_parameters = None
if sys.version_info[0] < 3:
import funcsigs
klass_parameters = funcsigs.signature(klass).parameters
else:
klass_parameters = inspect.signature(klass).parameters
client = None
if 'subscription_id' in klass_parameters:
client = klass(credentials=self.credentials,
subscription_id=self.subscription_id)
else:
client = klass(credentials=self.credentials)
# Override send() method to log request limits & custom retries
service_client = client._client
service_client.orig_send = service_client.send
service_client.send = types.MethodType(custodian_azure_send_override,
service_client)
# Don't respect retry_after_header to implement custom retries
service_client.config.retry_policy.policy.respect_retry_after_header = False
return client
def test_signature_bound_arguments_equality(self):
def foo(a): pass
ba = inspect.signature(foo).bind(1)
self.assertEqual(ba, ba)
ba2 = inspect.signature(foo).bind(1)
self.assertEqual(ba, ba2)
ba3 = inspect.signature(foo).bind(2)
self.assertNotEqual(ba, ba3)
ba3.arguments['a'] = 1
self.assertEqual(ba, ba3)
def bar(b): pass
ba4 = inspect.signature(bar).bind(1)
self.assertNotEqual(ba, ba4)
def _get_input_map(cmd_name, cmd_fxn, tags, parents):
# todo handle inputs without default
sig = funcsigs.signature(cmd_fxn)
# funcsigs._empty
for param_name, param in sig.parameters.iteritems():
if param_name.startswith('in_'):
value = tags.get(param_name, param.default)
if value == funcsigs._empty:
raise AssertionError, '%s Bad input `%s`, with default `%s`. Set its default to find(), or specify ' \
'its value via tags' % (cmd_name, param_name, param.default)
elif isinstance(value, FindFromParents):
# user used find()
find_instance = value
def get_available_files():
for p in parents:
if all(p.tags.get(k) == v for k, v in (find_instance.tags or dict()).items()):
yield p.output_files
available_files = it.chain(*get_available_files())
input_taskfiles = list(_find(available_files, find_instance.regex, error_if_missing=False))
_validate_input_mapping(cmd_name, param_name, find_instance, input_taskfiles, parents)
input_taskfile_or_input_taskfiles = unpack_if_cardinality_1(find_instance, input_taskfiles)
yield param_name, input_taskfile_or_input_taskfiles
elif isinstance(value, basestring):
# allows a user to remove an input_file by passing None for its value
yield param_name, value
def _register_info(self):
"""Register local methods in the Workbench Information system"""
# Stores information on Workbench commands and signatures
for name, meth in inspect.getmembers(self,
predicate=inspect.isroutine):
if not name.startswith('_') and name != 'run':
info = {
'command': name,
'sig': str(funcsigs.signature(meth)),
'docstring': meth.__doc__
}
self.workbench.store_info(info, name, 'command')
# Register help information
self.workbench.store_info({'help': self.help.help_cli()}, 'cli',
'help')
self.workbench.store_info({'help': self.help.help_cli_basic()},
'cli_basic', 'help')
self.workbench.store_info({'help': self.help.help_cli_search()},
'search', 'help')
self.workbench.store_info({'help': self.help.help_dataframe()},
'dataframe', 'help')
self.workbench.store_info({'help': self.help.help_dataframe_memory()},
'dataframe_memory', 'help')
self.workbench.store_info({'help': self.help.help_dataframe_pe()},
'dataframe_pe', 'help')
def _store_information(self):
""" Store infomation about Workbench and its commands """
print '<<< Generating Information Storage >>>'
# Stores information on Workbench commands and signatures
for name, meth in inspect.getmembers(self,
predicate=inspect.isroutine):
if not name.startswith('_'):
info = {
'command': name,
'sig': str(funcsigs.signature(meth)),
'docstring': meth.__doc__
}
self.store_info(info, name, type_tag='command')
# Stores help text into the workbench information system
self.store_info(
{'help': '<<< Workbench Server Version %s >>>' % self.version},
'version',
type_tag='help')
self.store_info({'help': self._help_workbench()},
'workbench',
type_tag='help')
self.store_info({'help': self._help_basic()}, 'basic', type_tag='help')
self.store_info({'help': self._help_commands()},
'commands',
type_tag='help')
self.store_info({'help': self._help_workers()},
'workers',
type_tag='help')
def _np_signature(f):
"""An enhanced funcsigs.signature that can handle numpy.ufunc."""
if not isinstance(f, np.ufunc):
return funcsigs.signature(f)
def names_from_num(prefix, n):
if n <= 0:
return []
elif n == 1:
return [prefix]
else:
return [prefix + str(i + 1) for i in range(n)]
input_names = names_from_num('x', f.nin)
output_names = names_from_num('out', f.nout)
keyword_only_params = [
('where', True),
('casting', 'same_kind'),
('order', 'K'),
('dtype', None),
('subok', True),
('signature', None),
('extobj', None)]
params = []
params += [Parameter(name, Parameter.POSITIONAL_ONLY) for name in input_names]
if f.nout > 1:
params += [Parameter(name, Parameter.POSITIONAL_ONLY, default=None)
for name in output_names]
params += [Parameter('out', Parameter.POSITIONAL_OR_KEYWORD,
default=None if f.nout == 1 else (None,) * f.nout)]
params += [Parameter(name, Parameter.KEYWORD_ONLY, default=default)
for name, default in keyword_only_params]
return funcsigs.Signature(params)
def precision_recall_plot(targs, preds, figsize=(6,6)):
"""
Plots the precision recall curve
----------
targs: array-like true class labels
preds: array-like predicted probabilities
figsize: size of figure
Returns:
-------
A precision and recall curve
"""
average_precision = average_precision_score(targs, preds)
precision, recall, _ = precision_recall_curve(targs, preds)
plt.figure(figsize=figsize)
step_kwargs = ({'step': 'post'}
if 'step' in signature(plt.fill_between).parameters
else {})
plt.step(recall, precision, color='b', alpha=0.2,
where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', **step_kwargs)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(
average_precision))
plt.show()
def get_policy_args(policy, obs, t, i_tr, step_data=None):
policy_args = {}
policy_signature = signature(
policy.act) # Gets arguments required by policy
for arg in policy_signature.parameters: # Fills out arguments according to their keyword
value = policy_signature.parameters[arg].default
if arg in obs:
value = obs[arg]
elif step_data is not None and arg in step_data:
value = step_data[arg]
# everthing that is not cached in post_process_obs is assigned here:
elif arg == 't':
value = t
elif arg == 'i_tr':
value = i_tr
elif arg == 'obs': # policy can ask for all arguments from environment
value = obs
elif arg == 'step_data':
value = step_data
elif arg == 'goal_pos':
value = step_data['goal_pos']
if value is Parameter.empty:
# required parameters MUST be set by agent
raise ValueError(
"Required Policy Param {} not set in agent".format(arg))
policy_args[arg] = value
# import pdb; pdb.set_trace()
return policy_args
def validate_arguments_against_signature(func, args, kwargs):
"""
Check if the request's arguments match a function's signature.
Raises InvalidParams exception if arguments cannot be passed to a
function.
:param func: The function to check.
:param args: Positional arguments.
:param kwargs: Keyword arguments.
:raises InvalidParams: If the arguments cannot be passed to the function.
"""
try:
signature(func).bind(*(args or []), **(kwargs or {}))
except TypeError as exc:
raise InvalidParams(str(exc))
def set_testcase(self, test_case_fn_str, test_case_args):
"""
Args:
test_case_fn_str (str): name of function in test_cases.py
"""
# Provide a fn (which returns list of agents) and the fn's args,
# to be called on each env.reset()
test_case_fn = getattr(tc, test_case_fn_str, None)
assert (callable(test_case_fn))
# Before running test_case_fn, make sure we didn't provide any args it doesn't accept
if sys.version[0] == '3':
signature = inspect.signature(test_case_fn)
elif sys.version[0] == '2':
import funcsigs
signature = funcsigs.signature(test_case_fn)
test_case_fn_args = signature.parameters
test_case_args_keys = list(test_case_args.keys())
for key in test_case_args_keys:
# print("checking if {} accepts {}".format(test_case_fn, key))
if key not in test_case_fn_args:
# print("{} doesn't accept {} -- removing".format(test_case_fn, key))
del test_case_args[key]
self.test_case_fn = test_case_fn
self.test_case_args = test_case_args
def default_docs(func):
"""
A decorator which automatically takes care of default parameter
documentation for common pipeline factory parameters
"""
docs = func.__doc__
new_docs = ""
signature = funcsigs.signature(func)
try:
buf = StringIO(docs)
line = True
while line:
line = buf.readline()
if "Parameters" in line:
artificial_param_docs = [line, buf.readline()]
# Include the `-----` line
for param in signature.parameters.keys():
doc = _PARAMETER_MAPPING.get(param, None)
if doc:
if not doc.endswith("\n"):
doc += "\n"
if doc.startswith("\n"):
doc = doc[1:]
artificial_param_docs.append(doc)
new_docs += "".join(artificial_param_docs)
continue
new_docs = "".join([new_docs, line])
func.__doc__ = new_docs
except Exception as ex:
func.__doc__ = str(ex)
return func
def test_signature_bound_arguments_unhashable(self):
def foo(a):
pass
ba = inspect.signature(foo).bind(1)
with self.assertRaisesRegex(TypeError, "unhashable type"):
hash(ba)
def test_bind_self(self):
class F:
def f(a, self):
return a, self
an_f = F()
partial_f = functools.partial(F.f, an_f)
ba = inspect.signature(partial_f).bind(self=10)
self.assertEqual((an_f, 10), partial_f(*ba.args, **ba.kwargs))
def _prepare_kwargs(task, func, args, kwargs):
"""
Prepare keyword arguments (targets, dependencies, changed,
cmd line options)
Inspect python callable and add missing arguments:
- that the callable expects
- have not been passed (as a regular arg or as keyword arg)
- are available internally through the task object
"""
# Return just what was passed in task generator
# dictionary if the task isn't available
if not task:
return kwargs
#func_sig = inspect.signature(func) # KGEN deletion
func_sig = signature(func) # KGEN addition
sig_params = func_sig.parameters.values()
func_has_kwargs = any(p.kind==p.VAR_KEYWORD for p in sig_params)
# use task meta information as extra_args
meta_args = {
'task': task,
'targets': task.targets,
'dependencies': task.file_dep,
'changed': task.dep_changed,
}
extra_args = dict(meta_args)
# tasks parameter options
extra_args.update(task.options)
if task.pos_arg is not None:
extra_args[task.pos_arg] = task.pos_arg_val
kwargs = kwargs.copy()
bound_args = func_sig.bind_partial(*args)
for key in extra_args.keys():
# check key is a positional parameter
if key in func_sig.parameters:
sig_param = func_sig.parameters[key]
# it is forbidden to use default values for this arguments
# because the user might be unware of this magic.
if (key in meta_args and sig_param.default!=sig_param.empty):
msg = ("Task %s, action %s(): The argument '%s' is not "
"allowed to have a default value (reserved by doit)"
% (task.name, func.__name__, key))
raise InvalidTask(msg)
# if value not taken from position parameter
if key not in bound_args.arguments:
kwargs[key] = extra_args[key]
# if function has **kwargs include extra_arg on it
elif func_has_kwargs and key not in kwargs:
kwargs[key] = extra_args[key]
return kwargs
def get_signature(func_obj):
if func_obj in KNOWN_SIGNATURES:
return KNOWN_SIGNATURES[func_obj]
try:
return funcsigs.signature(func_obj)
except:
pass
raise ValueError("Cannot get signature from", func_obj)
def signature(func):
sig = inspect.signature(func)
return (tuple((param.name,
(Ellipsis if param.default is param.empty else param.default),
(Ellipsis if param.annotation is param.empty
else param.annotation),
str(param.kind).lower())
for param in sig.parameters.values()),
(Ellipsis if sig.return_annotation is sig.empty
else sig.return_annotation))
def system_methodSignature(self,methodName):
try :
from funcsigs import signature
except:
return self.returnFault(-10,'You have to install funcsigs to get method signature')
if not self.isPublicMethod(methodName):
return self.returnFault(-5,'Not Existing Method: %s' % methodName)
h=getattr(self,methodName)
sig = signature(h).parameters
return sig
def mutating(fn, *argnames):
"""
Marks the function as mutating some of its arguments.
"""
sig = funcsigs.signature(fn)
mutating = dict((name, None) for name in sig.parameters)
for name in argnames:
mutating[name] = True
fn._peval_mutating = mutating
return fn
def pre_actions(*args, **kwargs):
if not pre_func:
raise NotImplementedError('You have to provide an implementation'
'for the pre-validation step!')
new_kwargs = {}
sig = signature(pre_func).parameters
for idx, (k, v) in enumerate(sig.iteritems()):
if k in kwargs.keys():
new_kwargs[k] = kwargs[k]
return pre_func(*args, **new_kwargs)
