def test_better_wraps():
def wraps_decorator(func):
@wraps(func)
def _wrap_wraps_decorator(*args, **kwargs):
return func(*args, **kwargs)
return _wrap_wraps_decorator
def better_decorator(func):
@better_wraps(func)
def _wrap_better_decorator(*args, **kwargs):
return func(*args, **kwargs)
return _wrap_better_decorator
@wraps_decorator
def function1(a, b, c):
return "function1"
@better_decorator
def function2(a, b, c):
return "function2"
eq_("function1", function1(1, 2, 3))
# getargspec shim now can handle @wraps'ed functions just fine
eq_(getargspec(function1)[0], ['a', 'b', 'c'])
eq_("function2", function2(1, 2, 3))
eq_(getargspec(function2)[0], ['a', 'b', 'c'])
def test_better_wraps():
from functools import wraps
from datalad.utils import getargspec
def wraps_decorator(func):
@wraps(func)
def _wrap_wraps_decorator(*args, **kwargs):
return func(*args, **kwargs)
return _wrap_wraps_decorator
def better_decorator(func):
@better_wraps(func)
def _wrap_better_decorator(*args, **kwargs):
return func(*args, **kwargs)
return _wrap_better_decorator
@wraps_decorator
def function1(a, b, c):
return "function1"
@better_decorator
def function2(a, b, c):
return "function2"
eq_("function1", function1(1, 2, 3))
eq_(getargspec(function1)[0], [])
eq_("function2", function2(1, 2, 3))
eq_(getargspec(function2)[0], ['a', 'b', 'c'])
def eq_argspec(f, expected, has_kwonlyargs=False):
"""A helper to centralize testing of getargspec on original and wrapped function
has_kwonlyargs is to instruct if function has kwonly args so we do not try to compare
to inspect.get*spec functions, which would barf ValueError if attempted to run on a
function with kwonlys. And also we pass it as include_kwonlyargs to our getargspec
"""
# so we know that our expected is correct
if not has_kwonlyargs:
# if False - we test function with kwonlys - inspect.getargspec would barf
eq_(inspect.getargspec(f), expected)
# and getfullargspec[:4] wouldn't provide a full picture
eq_(inspect.getfullargspec(f)[:4], expected)
else:
assert_raises(ValueError, inspect.getargspec, f)
inspect.getfullargspec(f) # doesn't barf
eq_(getargspec(f, include_kwonlyargs=has_kwonlyargs), expected)
# and lets try on a wrapped one -- only ours can do the right thing
def decorator(f):
@wraps(f)
def wrapper(*args, **kwargs): # pragma: no cover
return f(*args, **kwargs)
return wrapper
fw = decorator(f)
if has_kwonlyargs:
# We barf ValueError similarly to inspect.getargspec, unless explicitly requested
# to include kwonlyargs
assert_raises(ValueError, getargspec, fw)
eq_(getargspec(fw, include_kwonlyargs=has_kwonlyargs), expected)
def call_from_parser(cls, args):
# XXX needs safety check for name collisions
from datalad.utils import getargspec
argspec = getargspec(cls.__call__)
if argspec[2] is None:
# no **kwargs in the call receiver, pull argnames from signature
argnames = getargspec(cls.__call__)[0]
else:
# common options
# XXX define or better get from elsewhere
common_opts = ('change_path', 'common_debug', 'common_idebug', 'func',
'help', 'log_level', 'logger', 'pbs_runner',
'result_renderer', 'subparser')
argnames = [name for name in dir(args)
if not (name.startswith('_') or name in common_opts)]
kwargs = {k: getattr(args, k)
for k in argnames
# some arguments might be Python-only and do not appear in the
# parser Namespace
if hasattr(args, k) and is_api_arg(k)}
# we are coming from the entry point, this is the toplevel command,
# let it run like generator so we can act on partial results quicker
# TODO remove following condition test when transition is complete and
# run indented code unconditionally
if _has_eval_results_call(cls):
# set all common args explicitly to override class defaults
# that are tailored towards the the Python API
kwargs['return_type'] = 'generator'
kwargs['result_xfm'] = None
# allow commands to override the default, unless something other than
# default is requested
kwargs['result_renderer'] = \
args.common_output_format if args.common_output_format != 'tailored' \
else getattr(cls, 'result_renderer', 'default')
if '{' in args.common_output_format:
# stupid hack, could and should become more powerful
kwargs['result_renderer'] = DefaultOutputRenderer(args.common_output_format)
if args.common_on_failure:
kwargs['on_failure'] = args.common_on_failure
# compose filter function from to be invented cmdline options
res_filter = cls._get_result_filter(args)
if res_filter is not None:
# Don't add result_filter if it's None because then
# eval_results can't distinguish between --report-{status,type}
# not specified via the CLI and None passed via the Python API.
kwargs['result_filter'] = res_filter
try:
ret = cls.__call__(**kwargs)
if inspect.isgenerator(ret):
ret = list(ret)
return ret
except KeyboardInterrupt as exc:
ui.error("\nInterrupted by user while doing magic: %s" % exc_str(exc))
if cls._interrupted_exit_code is not None:
sys.exit(cls._interrupted_exit_code)
else:
raise
def test_eval_results_plus_build_doc():
# test docs
# docstring was build already:
with swallow_logs(new_level=logging.DEBUG) as cml:
TestUtils().__call__(1)
assert_not_in("Building doc for", cml.out)
# docstring accessible both ways:
doc1 = Dataset.fake_command.__doc__
doc2 = TestUtils().__call__.__doc__
# docstring was built from Test_Util's definition:
assert_equal(doc1, doc2)
assert_in("TestUtil's fake command", doc1)
assert_in("Parameters", doc1)
assert_in("It's a number", doc1)
# docstring shows correct override values of defaults in eval_params
assert_in("Default: 'tailored'", doc1)
assert_in("Default: 'item-or-list'", doc1)
# docstring also contains eval_result's parameters:
assert_in("result_filter", doc1)
assert_in("return_type", doc1)
assert_in("list", doc1)
assert_in("None", doc1)
assert_in("return value behavior", doc1)
assert_in("dictionary is passed", doc1)
# test eval_results is able to determine the call, a method of which it is
# decorating:
with swallow_logs(new_level=logging.DEBUG) as cml:
Dataset('/does/not/matter').fake_command(3)
assert_in(
"Determined class of decorated function: {}"
"".format(TestUtils().__class__), cml.out)
# test results:
result = TestUtils().__call__(2)
assert_equal(len(list(result)), 2)
result = Dataset('/does/not/matter').fake_command(3)
assert_equal(len(list(result)), 3)
# test absent side-effect of popping eval_defaults
kwargs = dict(return_type='list')
TestUtils().__call__(2, **kwargs)
assert_equal(list(kwargs), ['return_type'])
# test signature:
from datalad.utils import getargspec
assert_equal(
getargspec(Dataset.fake_command)[0],
['number', 'dataset', 'result_fn'])
assert_equal(
getargspec(TestUtils.__call__)[0], ['number', 'dataset', 'result_fn'])
def get_allargs_as_kwargs(call, args, kwargs):
"""Generate a kwargs dict from a call signature and *args, **kwargs
Basically resolving the argnames for all positional arguments, and
resolvin the defaults for all kwargs that are not given in a kwargs
dict
"""
from datalad.utils import getargspec
argspec = getargspec(call)
defaults = argspec.defaults
nargs = len(argspec.args)
assert (nargs >= len(defaults))
# map any args to their name
argmap = list(zip(argspec.args[:len(args)], args))
kwargs_ = OrderedDict(argmap)
# map defaults of kwargs to their names (update below)
for k, v in zip(argspec.args[-len(defaults):], defaults):
if k not in kwargs_:
kwargs_[k] = v
# update with provided kwarg args
kwargs_.update(kwargs)
# XXX we cannot assert the following, because our own highlevel
# API commands support more kwargs than what is discoverable
# from their signature...
#assert (nargs == len(kwargs_))
return kwargs_
def _test_consistent_order_of_args(intf, spec_posargs):
f = getattr(intf, '__call__')
args, varargs, varkw, defaults = getargspec(f)
# now verify that those spec_posargs are first among args
if not spec_posargs:
raise SkipTest("no positional args") # print intf, "skipped"
# else:
# print intf, spec_posargs
if intf.__name__ == 'Save':
# it makes sense there to have most common argument first
# -- the message. But we don't enforce it on cmdline so it is
# optional
spec_posargs.add('message')
elif intf.__name__ == 'ExtractMetadata':
# MIH I was never sure what this test enforces and it takes
# me ages each time to try to wrap my head around it
# I am confident that I do not want to change the API of this
# command now that it is a command and no longer a plugin
# hence this exception
eq_(spec_posargs, spec_posargs)
return
eq_(set(args[:len(spec_posargs)]), spec_posargs)
def setup_parser_for_interface(parser, cls, completing=False):
# XXX needs safety check for name collisions
# XXX allow for parser kwargs customization
# get the signature, order of arguments is taken from it
ndefaults = 0
args, varargs, varkw, defaults = getargspec(cls.__call__,
include_kwonlyargs=True)
if defaults is not None:
ndefaults = len(defaults)
default_offset = ndefaults - len(args)
prefix_chars = parser.prefix_chars
for i, arg in enumerate(args):
if not is_api_arg(arg):
continue
param = cls._params_[arg]
defaults_idx = default_offset + i
if param.cmd_args == tuple():
# explicitly provided an empty sequence of argument names
# this shall not appear in the parser
continue
# set up the parameter
setup_parserarg_for_interface(parser,
arg,
param,
defaults_idx,
prefix_chars,
defaults,
completing=completing)
def apply_func(wrapped, instance, args, kwargs):
# Wrapper function to assign arguments of the bound function to
# original function.
#
# Note
# ----
# This wrapper is NOT returned by the decorator, but only used to bind
# the function `f` to the Dataset class.
kwargs = kwargs.copy()
from datalad.utils import getargspec
orig_pos = getargspec(f).args
# If bound function is used with wrong signature (especially by
# explicitly passing a dataset, let's raise a proper exception instead
# of a 'list index out of range', that is not very telling to the user.
if len(args) >= len(orig_pos):
raise TypeError(
"{0}() takes at most {1} arguments ({2} given):"
" {3}".format(name, len(orig_pos), len(args), ['self'] +
[a for a in orig_pos if a != dataset_argname]))
if dataset_argname in kwargs:
raise TypeError("{}() got an unexpected keyword argument {}"
"".format(name, dataset_argname))
kwargs[dataset_argname] = instance
ds_index = orig_pos.index(dataset_argname)
for i in range(0, len(args)):
if i < ds_index:
kwargs[orig_pos[i]] = args[i]
elif i >= ds_index:
kwargs[orig_pos[i + 1]] = args[i]
return f(**kwargs)
def _test_consistent_order_of_args(intf, spec_posargs):
f = getattr(intf, '__call__')
args, varargs, varkw, defaults = getargspec(f)
# now verify that those spec_posargs are first among args
if not spec_posargs:
raise SkipTest("no positional args") # print intf, "skipped"
# else:
# print intf, spec_posargs
if intf.__name__ == 'Save':
# it makes sense there to have most common argument first
# -- the message. But we don't enforce it on cmdline so it is
# optional
spec_posargs.add('message')
elif intf.__name__ in ('AddReadme', 'Addurls', 'ExportArchive',
'ExportToFigshare', 'ExtractMetadata'):
if intf.__name__ != 'ExtractMetadata':
# ex-plugins had 'dataset' as the first positional argument
# and ExtractMetadata has 'types' as the first positional arg
eq_(args[0], 'dataset')
eq_(spec_posargs, spec_posargs)
return
eq_(set(args[:len(spec_posargs)]), spec_posargs)
def get_result_filter(fx):
"""Wrap a filter into a helper to be able to accept additional
arguments, if the filter doesn't support it already"""
_fx = fx
if fx and not getargspec(fx).keywords:
def _fx(res, **kwargs):
return fx(res)
return _fx
def update_docstring_with_parameters(func,
params,
prefix=None,
suffix=None,
add_args=None):
"""Generate a useful docstring from a parameter spec
Amends any existing docstring of a callable with a textual
description of its parameters. The Parameter spec needs to match
the number and names of the callables arguments.
"""
from datalad.utils import getargspec
# get the signature
args, varargs, varkw, defaults = getargspec(func, include_kwonlyargs=True)
defaults = defaults or tuple()
if add_args:
add_argnames = sorted(add_args.keys())
args.extend(add_argnames)
defaults = defaults + tuple(add_args[k] for k in add_argnames)
ndefaults = len(defaults)
# start documentation with what the callable brings with it
doc = prefix if prefix else u''
if len(args) > 1:
if len(doc):
if not doc.endswith('\n'):
doc += '\n'
doc += '\n'
doc += "Parameters\n----------\n"
for i, arg in enumerate(args):
if not is_api_arg(arg):
continue
# we need a parameter spec for each argument
if not arg in params:
raise ValueError(
"function has argument '%s' not described as a parameter" %
arg)
param = params[arg]
# validate the default -- to make sure that the parameter description is
# somewhat OK
defaults_idx = ndefaults - len(args) + i
if defaults_idx >= 0:
if param.constraints is not None:
param.constraints(defaults[defaults_idx])
orig_docs = param._doc
param._doc = alter_interface_docs_for_api(param._doc)
doc += param.get_autodoc(
arg,
default=defaults[defaults_idx] if defaults_idx >= 0 else None,
has_default=defaults_idx >= 0)
param._doc = orig_docs
doc += '\n'
doc += suffix if suffix else u""
# assign the amended docs
func.__doc__ = doc
return func
def get_function_nargs(f):
while hasattr(f, 'wrapped'):
f = f.wrapped
argspec = getargspec(f)
assert not argspec.keywords, \
"ATM we have none defined with keywords, so disabling having them"
if argspec.varargs:
# Variable number of arguments
return -1
else:
assert argspec.args, "ATM no static methods"
assert argspec.args[0] == "self"
return len(argspec.args) - 1
def setup_parser(cls, parser):
# XXX needs safety check for name collisions
# XXX allow for parser kwargs customization
parser_kwargs = {}
from datalad.utils import getargspec
# get the signature
ndefaults = 0
args, varargs, varkw, defaults = getargspec(cls.__call__)
if not defaults is None:
ndefaults = len(defaults)
for i, arg in enumerate(args):
if not is_api_arg(arg):
continue
param = cls._params_[arg]
defaults_idx = ndefaults - len(args) + i
cmd_args = param.cmd_args
if cmd_args == tuple():
# explicitly provided an empty sequence of argument names
# this shall not appear in the parser
continue
elif cmd_args is None:
cmd_args = []
if not len(cmd_args):
if defaults_idx >= 0:
# dealing with a kwarg
template = '--%s'
else:
# positional arg
template = '%s'
# use parameter name as default argument name
parser_args = (template % arg.replace('_', '-'), )
else:
parser_args = [c.replace('_', '-') for c in cmd_args]
parser_kwargs = param.cmd_kwargs
if defaults_idx >= 0:
parser_kwargs['default'] = defaults[defaults_idx]
help = alter_interface_docs_for_cmdline(param._doc)
if help and help.rstrip()[-1] != '.':
help = help.rstrip() + '.'
if param.constraints is not None:
parser_kwargs['type'] = param.constraints
# include value constraint description and default
# into the help string
cdoc = alter_interface_docs_for_cmdline(
param.constraints.long_description())
if cdoc[0] == '(' and cdoc[-1] == ')':
cdoc = cdoc[1:-1]
help += ' Constraints: %s' % cdoc
if 'metavar' not in parser_kwargs and \
isinstance(param.constraints, EnsureChoice):
parser_kwargs['metavar'] = \
'{%s}' % '|'.join(
# don't use short_description(), because
# it also needs to give valid output for
# Python syntax (quotes...), but here we
# can simplify to shell syntax where everything
# is a string
p for p in param.constraints._allowed
# in the cmdline None pretty much means
# don't give the options, so listing it
# doesn't make sense. Moreover, any non-string
# value cannot be given and very likely only
# serves a special purpose in the Python API
# or implementation details
if isinstance(p, str))
if defaults_idx >= 0:
# if it is a flag, in commandline it makes little sense to show
# showing the Default: (likely boolean).
# See https://github.com/datalad/datalad/issues/3203
if not parser_kwargs.get('action', '').startswith('store_'):
# [Default: None] also makes little sense for cmdline
if defaults[defaults_idx] is not None:
help += " [Default: %r]" % (defaults[defaults_idx], )
# create the parameter, using the constraint instance for type
# conversion
parser.add_argument(*parser_args, help=help, **parser_kwargs)
def setup_parser(cls, parser):
# XXX needs safety check for name collisions
# XXX allow for parser kwargs customization
parser_kwargs = {}
from datalad.utils import getargspec
# get the signature
ndefaults = 0
args, varargs, varkw, defaults = getargspec(cls.__call__)
if defaults is not None:
ndefaults = len(defaults)
default_offset = ndefaults - len(args)
prefix_chars = parser.prefix_chars
for i, arg in enumerate(args):
if not is_api_arg(arg):
continue
param = cls._params_[arg]
defaults_idx = default_offset + i
cmd_args = param.cmd_args
if cmd_args == tuple():
# explicitly provided an empty sequence of argument names
# this shall not appear in the parser
continue
parser_kwargs = param.cmd_kwargs
has_default = defaults_idx >= 0
if cmd_args:
if cmd_args[0][0] in prefix_chars:
# TODO: All the Paramteter(args=...) values in this code
# base use hyphens, so there is no point in the below
# conversion. If it looks like no extensions rely on this
# behavior either, this could be dropped.
parser_args = [c.replace('_', '-') for c in cmd_args]
else:
# Argparse will not convert dashes to underscores for
# arguments that don't start with a prefix character, so
# the above substitution must be avoided so that
# call_from_parser() can find the corresponding parameter.
parser_args = cmd_args
elif has_default:
# Construct the option from the Python parameter name.
parser_args = ("--{}".format(arg.replace("_", "-")),)
else:
# If args= wasn't given and its a positional argument in the
# function, add a positional argument to argparse. If `dest` is
# specified, we need to remove it from the keyword arguments
# because add_argument() expects it as the first argument. Note
# that `arg` shouldn't have a dash here, but `metavar` can be
# used if a dash is preferred for the command-line help.
parser_args = (parser_kwargs.pop("dest", arg),)
if has_default:
parser_kwargs['default'] = defaults[defaults_idx]
help = alter_interface_docs_for_cmdline(param._doc)
if help and help.rstrip()[-1] != '.':
help = help.rstrip() + '.'
if param.constraints is not None:
parser_kwargs['type'] = param.constraints
# include value constraint description and default
# into the help string
cdoc = alter_interface_docs_for_cmdline(
param.constraints.long_description())
if cdoc[0] == '(' and cdoc[-1] == ')':
cdoc = cdoc[1:-1]
help += ' Constraints: %s' % cdoc
if 'metavar' not in parser_kwargs and \
isinstance(param.constraints, EnsureChoice):
parser_kwargs['metavar'] = \
'{%s}' % '|'.join(
# don't use short_description(), because
# it also needs to give valid output for
# Python syntax (quotes...), but here we
# can simplify to shell syntax where everything
# is a string
p for p in param.constraints._allowed
# in the cmdline None pretty much means
# don't give the options, so listing it
# doesn't make sense. Moreover, any non-string
# value cannot be given and very likely only
# serves a special purpose in the Python API
# or implementation details
if isinstance(p, str))
if defaults_idx >= 0:
# if it is a flag, in commandline it makes little sense to show
# showing the Default: (likely boolean).
# See https://github.com/datalad/datalad/issues/3203
if not parser_kwargs.get('action', '').startswith('store_'):
# [Default: None] also makes little sense for cmdline
if defaults[defaults_idx] is not None:
help += " [Default: %r]" % (defaults[defaults_idx],)
# create the parameter, using the constraint instance for type
# conversion
parser.add_argument(*parser_args, help=help,
**parser_kwargs)
class Parameter(object):
"""This class shall serve as a representation of a parameter.
"""
# Known keyword arguments which we want to allow to pass over into
# argparser.add_argument . Mentioned explicitly, since otherwise
# are not verified while working in Python-only API
_KNOWN_ARGS = getargspec(argparse.Action.__init__)[0] + ['action']
def __init__(self, constraints=None, doc=None, args=None, **kwargs):
"""Add constraints (validator) specifications and a docstring for
a parameter.
Parameters
----------
constraints : callable
A functor that takes any input value, performs checks or type
conversions and finally returns a value that is appropriate for a
parameter or raises an exception. This will also be used to set up
the ``type`` functionality of argparse.add_argument.
doc : str
Documentation about the purpose of this parameter.
args : tuple or None
Any additional positional args for argparser.add_argument. This is
most useful for assigned multiple alternative argument names or
create positional arguments.
**kwargs :
Any additional keyword args for argparser.add_argument.
Examples
--------
Ensure a parameter is a float
>>> from datalad.support.param import Parameter
>>> from datalad.support.constraints import (EnsureFloat, EnsureRange,
... AltConstraints, Constraints)
>>> C = Parameter(constraints=EnsureFloat())
Ensure a parameter is of type float or None:
>>> C = Parameter(constraints=AltConstraints(EnsureFloat(), None))
Ensure a parameter is None or of type float and lies in the inclusive
range (7.0,44.0):
>>> C = Parameter(
... AltConstraints(
... Constraints(EnsureFloat(),
... EnsureRange(min=7.0, max=44.0)),
... None))
"""
self.constraints = expand_constraint_spec(constraints)
self._doc = doc
self.cmd_args = args
# Verify that no mistyped kwargs present
unknown_args = set(kwargs).difference(self._KNOWN_ARGS)
if unknown_args:
raise ValueError(
"Detected unknown argument(s) for the Parameter: %s. Known are: %s"
% (', '.join(unknown_args), ', '.join(self._KNOWN_ARGS))
)
self.cmd_kwargs = kwargs
def get_autodoc(self, name, indent=" ", width=70, default=None, has_default=False):
"""Docstring for the parameter to be used in lists of parameters
Returns
-------
string or list of strings (if indent is None)
"""
paramsdoc = '%s' % name
sdoc = None
if self.constraints is not None:
sdoc = self.constraints.short_description()
elif 'action' in self.cmd_kwargs \
and self.cmd_kwargs['action'] in ("store_true", "store_false"):
sdoc = 'bool'
if sdoc is not None:
if sdoc[0] == '(' and sdoc[-1] == ')':
sdoc = sdoc[1:-1]
nargs = self.cmd_kwargs.get('nargs', '')
if isinstance(nargs, int):
sdoc = '{}-item sequence of {}'.format(nargs, sdoc)
elif nargs == '+':
sdoc = 'non-empty sequence of {}'.format(sdoc)
elif nargs == '*':
sdoc = 'sequence of {}'.format(sdoc)
if self.cmd_kwargs.get('action', None) == 'append':
sdoc = 'list of {}'.format(sdoc)
paramsdoc += " : %s" % sdoc
if has_default:
paramsdoc += ", optional"
paramsdoc = [paramsdoc]
doc = self._doc
if doc is None:
doc = ''
doc = doc.strip()
if len(doc) and not doc.endswith('.'):
doc += '.'
if has_default:
doc += " [Default: %r]" % (default,)
# Explicitly deal with multiple spaces, for some reason
# replace_whitespace is non-effective
doc = _whitespace_re.sub(' ', doc)
paramsdoc += [indent + x
for x in textwrap.wrap(doc, width=width - len(indent),
replace_whitespace=True)]
return '\n'.join(paramsdoc)
