def test_docs_match_signature(name, func):
sig = inspect.signature(func)
docs = FunctionDoc(func)
sig_params = set(sig.parameters)
doc_params = {p.name for p in docs.get('Parameters')}
assert sig_params == doc_params, (
f"Signature parameters for hook specification '{name}' do "
"not match the parameters listed in the docstring:\n"
f"{sig_params} != {doc_params}"
)
# we know the parameters names match, now check that their types match...
# but only emit a warning if not
for doc_param in docs.get('Parameters'):
sig_param = sig.parameters.get(doc_param.name)
name = getattr(sig_param.annotation, '_name', None)
name = name or getattr(sig_param.annotation, '__name__', None)
if doc_param.type != name:
msg = (
f'The type ({name}) for parameter '
f'"{sig_param.name}" in hook specification "{name}" does not '
'match the type specified in the docstring '
f'({doc_param.type})'
)
warnings.warn(msg)
def test_docs_match_signature(name, func):
sig = inspect.signature(func)
docs = FunctionDoc(func)
sig_params = set(sig.parameters)
doc_params = {p.name for p in docs.get('Parameters')}
assert sig_params == doc_params, (
f"Signature parameters for hook specification '{name}' do "
"not match the parameters listed in the docstring:\n"
f"{sig_params} != {doc_params}")
def parse_methods(class_object, doc):
markdown = ""
# avoid private and builtin methods
methods = [m for m in dir(class_object) if should_parse_method(class_object, m)]
# sanity check
if len(methods) > 0:
markdown += f"{lb * 2}"
for method_key in methods:
try:
method = getattr(class_object, method_key)
# check if the func is wrapped (decorator)
if hasattr(method, "__closure__") and method.__closure__ is not None:
wrapped_method = extract_wrapped(method)
method = method if wrapped_method is None else wrapped_method
method_doc = FunctionDoc(method)
markdown += parse_signature(method)
markdown += parse_summary(method_doc)
markdown += parse_parameters(method_doc, method)
markdown += parse_returns(method_doc, method)
markdown += flb
except:
raise
warnings.warn(
f"Skip parsing method {class_object.__name__}.{method}.")
return markdown
def get_function_summary(func):
"""Get summary of doc string of function."""
doc = FunctionDoc(func)
summary = ''
for s in doc['Summary']:
summary += s
return summary.rstrip('.')
def check_types(f):
"""Decorator that adds runtime type checking.
Parmeters
---------
f : function
The function to wrap a type checker around.
Returns
-------
wrapper : function
The wrapped function.
"""
doc = FunctionDoc(f)
checkers = dict()
for param in doc['Parameters']:
checkers[param.name] = find_checker_function(param.name, param.type)
# Make sure that the checkers match the function parameters
if list(checkers.keys()) != inspect.getargspec(f).args:
raise ParseError('The arguments listed in the docstring for '
f'function `{f.__name__}` do not match the '
'arguments in the function signature.')
@wraps(f)
def wrapper(*args, **kwds):
# Check positional arguments
for arg, checker in zip(args, checkers.values()):
checker(arg)
# Check keyword arguments
for name, arg in kwds.items():
checkers[name](arg)
return f(*args, **kwds)
return wrapper
def guess_numpydoc_annotations(func: Callable) -> Dict[str, Dict[str, Any]]:
"""Return a dict with type hints and/or choices for each parameter in the docstring.
Parameters
----------
func : function
The function to parse
Returns
-------
param_dict : dict
dict where the keys are names of parameters in the signature of ``func``, and
the value is a dict with possible keys ``type``, and ``choices``. ``type``
provides a typing object from ``typing`` that can be used in function signatures
"""
param_dict: Dict[str, Dict[str, Any]] = OrderedDict()
sig = inspect.signature(func)
for name, type_, description in FunctionDoc(func).get("Parameters"):
if name not in sig.parameters:
continue
if name not in param_dict:
param_dict[name] = {}
if type_.startswith("{'"):
try:
param_dict[name]["choices"] = literal_eval(type_.split("},")[0] + "}")
except Exception:
pass
param_dict[name]["type"] = typestr_to_typeobj(type_)
return param_dict
def get_function_summary(func):
"""Get summary of doc string of function."""
doc = FunctionDoc(func)
summary = ''
summary += doc['Signature']
for s in doc['Summary']:
summary += '<br> ' + s
return summary
def test_escape_stars():
signature = str(doc3).split('\n')[0]
assert_equal(signature, r'my_signature(\*params, \*\*kwds)')
def my_func(a, b, **kwargs):
pass
fdoc = FunctionDoc(func=my_func)
assert_equal(fdoc['Signature'], r'my_func(a, b, \*\*kwargs)')
def _function_doc_markdown(func,
reference,
outstream=sys.stdout,
indent='',
method=False):
"""
Generate markdown documentation for the given function object.
Parameters
----------
func : function
The function object to be documented.
reference : str
The dotted path to the function in the API.
Returns
-------
str
The markdown representation of the function documentation.
"""
doc = FunctionDoc(func)
sig = _sub_unspecified_in_signature(doc['Signature'])
if not method:
print(f'{indent}!!! abstract "{reference}"\n', file=outstream)
else:
print(f'{indent}!!! abstract ""\n', file=outstream)
indent = indent + ' '
print(f"{indent}**{sig}**\n", file=outstream)
print('', file=outstream)
if doc['Summary']:
txt = textwrap.indent('\n'.join(doc['Summary']), indent) + '\n'
print(txt, file=outstream)
if doc['Extended Summary']:
txt = textwrap.indent('\n'.join(doc['Extended Summary']),
indent) + '\n'
print(txt, file=outstream)
print('', file=outstream)
print(f'{indent}**Arguments:**\n', file=outstream)
for p in doc['Parameters']:
print(f'{indent}**{p.name}**: {" ".join(p.desc)}', file=outstream)
print('', file=outstream)
if doc['Raises']:
print('{indent}**Raises:**\n', file=outstream)
for p in doc['Raises']:
print(f'{indent}**{p.name}**: {" ".join(p.desc)}', file=outstream)
print('', file=outstream)
def pytest_runtest_makereport(item, call):
pytest_html = item.config.pluginmanager.getplugin('html')
outcome = yield
report = outcome.get_result()
documentation = FunctionDoc(item.function)
# Add description, markers and tier to the report
report.description = '. '.join(documentation["Summary"])
report.tier = ', '.join(str(mark.kwargs['level']) for mark in item.iter_markers(name="tier"))
report.markers = ', '.join(mark.name for mark in item.iter_markers() if
mark.name != 'tier' and mark.name != 'parametrize')
if report.location[0] not in results:
results[report.location[0]] = {'passed': 0, 'failed': 0, 'skipped': 0, 'xfailed': 0, 'error': 0}
extra = getattr(report, 'extra', [])
if report.when == 'call':
# Apply hack to fix length filename problem
pytest_html.HTMLReport.TestResult.create_asset = create_asset
# Add extended information from docstring inside 'Result' section
extra.append(pytest_html.extras.html('<div><h2>Test function details</h2></div>'))
for section in ('Extended Summary', 'Parameters'):
extra.append(pytest_html.extras.html(f'<div><h3>{section}</h3></div>'))
for line in documentation[section]:
extra.append(pytest_html.extras.html(f'<div>{line}</div>'))
arguments = dict()
# Add arguments of each text as a json file
for key, value in item.funcargs.items():
if isinstance(value, set):
arguments[key] = list(value)
try:
json.dumps(value)
arguments[key] = value
except (TypeError, OverflowError):
arguments[key] = str(value)
extra.append(pytest_html.extras.json(arguments, name="Test arguments"))
# Extra files to be added in 'Links' section
for filepath in (LOG_FILE_PATH, WAZUH_CONF):
with open(filepath, mode='r', errors='replace') as f:
content = f.read()
extra.append(pytest_html.extras.text(content, name=os.path.split(filepath)[-1]))
if not report.passed and not report.skipped:
report.extra = extra
if report.longrepr is not None and report.longreprtext.split()[-1] == 'XFailed':
results[report.location[0]]['xfailed'] += 1
else:
results[report.location[0]][report.outcome] += 1
elif report.outcome == 'failed':
results[report.location[0]]['error'] += 1
def test_foo_docs():
assert 'Warns' in foo_with_docs_15.__doc__
docs = FunctionDoc(foo_with_docs_15)
assert 'Warns' in docs
assert 'FutureWarning' == docs['Warns'][0].type
assert "`bar` : `'world'` -> `'hello'`" in '\n'.join(docs['Warns'][0][2])
assert foo_with_docs_13.__doc__ == foo_with_docs.__doc__
with warns(FutureWarning, match='In release 0.15 of mylib'):
assert foo_with_docs_15() == 'world'
def numpydoc_type_desc(thing):
if inspect.isfunction(thing) or inspect.ismethod(thing):
docs = FunctionDoc(thing)
elif inspect.isclass(thing):
docs = ClassDoc(thing)
else:
raise RuntimeError("Don't know how to handle " + repr(thing))
npdoc_params = docs["Parameters"]
types = [p.type for p in npdoc_params]
descs = [" ".join(p.desc) for p in npdoc_params]
return types, descs
def test_docstring(layer):
name = layer.__name__
method_name = f'add_{camel_to_snake(name)}'
method = getattr(Viewer, method_name)
method_doc = FunctionDoc(method)
layer_doc = ClassDoc(layer)
# check summary section
method_summary = ' '.join(method_doc['Summary']) # join multi-line summary
summary_format = 'Add an? .+? layer to the layers list.'
assert re.match(
summary_format, method_summary
), f"improper 'Summary' section of '{method_name}'"
# check parameters section
method_params = method_doc['Parameters']
layer_params = layer_doc['Parameters']
try:
assert len(method_params) == len(layer_params)
for method_param, layer_param in zip(method_params, layer_params):
m_name, m_type, m_description = method_param
l_name, l_type, l_description = layer_param
# descriptions are treated as lists where each line is an element
m_description = ' '.join(m_description)
l_description = ' '.join(l_description)
assert m_name == l_name, 'different parameter names or order'
assert m_type == l_type, f"type mismatch of parameter '{m_name}'"
assert (
m_description == l_description
), f"description mismatch of parameter '{m_name}'"
except AssertionError as e:
raise AssertionError(f"docstrings don't match for class {name}") from e
# check returns section
method_returns, = method_doc[
'Returns'
] # only one thing should be returned
description = ' '.join(method_returns[-1]) # join multi-line description
method_returns = *method_returns[:-1], description
assert method_returns == (
'layer',
f':class:`napari.layers.{name}`',
f'The newly-created {name.lower()} layer.',
), f"improper 'Returns' section of '{method_name}'"
def signature_parameters(self):
if inspect.isclass(self.obj):
if hasattr(self.obj, "_accessors") and (
self.name.split(".")[-1] in self.obj._accessors
):
# accessor classes have a signature but don't want to show this
return ()
try:
sig = inspect.signature(self.obj)
except (TypeError, ValueError):
# Some objects, mainly in C extensions do not support introspection
# of the signature
try:
from numpydoc.docscrape import FunctionDoc
doc = FunctionDoc(self.obj)
if "Signature" in doc:
sig = doc["Signature"].split("(")[-1].split(")")[0]
sig = sig.split(",")
params = []
for param in sig:
if param.strip() in ("*", "/"):
continue
param = param.split("=")[0]
params.append(param)
return tuple([param.name for param in doc["Parameters"]])
except Exception as exc:
print("!! numpydoc failed on {0}!!".format(str(self.obj)))
print(exc)
return ()
params = list(sig.parameters.keys())
out_params = params[:]
kind = inspect.Parameter.VAR_POSITIONAL
varargs = [key for key in params if sig.parameters[key].kind == kind]
if varargs:
out_params = [
"*" + param if param == varargs[0] else param
for param in out_params
]
kind = inspect.Parameter.VAR_KEYWORD
varkw = [key for key in params if sig.parameters[key].kind == kind]
if varkw:
out_params = [
"**" + param if param == varkw[0] else param
for param in out_params
]
params = tuple(out_params)
if params and params[0] in ("self", "cls"):
return params[1:]
return params
def test_numpydoc_function():
r"""
Summary line.
Extended description of function.
Parameters
----------
arg1 : int, default: 5
Description of arg1
arg2 : str
Description of arg2
arg3 : str
The [JMESpath](https://jmespath.org) query.
Returns
-------
bool
Description of return value
Raises
------
AttributeError
The ``Raises`` section is a list of all exceptions
that are relevant to the interface.
ValueError
If `arg2` is equal to `arg1`.
Examples
--------
Examples should be written in doctest format, and should illustrate how
to use the function.
>>> a=1
>>> b=2
>>> func(a,b)
True
Notes
-----
blabla
"""
doc = FunctionDoc(func=test_numpydoc_function.__doc__)
assert doc["Parameters"] == [
Parameter(name="a", type="int, default: 5", desc=["Does something cool"]),
Parameter(name="b", type="str", desc=["Wow"]),
]
assert doc["See Also"] == [([("blabla", None)], [])]
assert doc["Notes"] == ["alias: blabla", "adesso tu"]
def test_see_also_print():
class Dummy(object):
"""
See Also
--------
func_a, func_b
func_c : some relationship
goes here
func_d
"""
pass
s = str(FunctionDoc(Dummy, role='func'))
assert (':func:`func_a`, :func:`func_b`' in s)
assert (' some relationship' in s)
assert (':func:`func_d`' in s)
def test_ex9():
"""Ex9: Document your function using numpydoc"""
from numpydoc.docscrape import FunctionDoc
import gizmo
assert hasattr(gizmo, 'generate_fibonacci_sequence')
assert hasattr(gizmo.generate_fibonacci_sequence, '__doc__')
doc = FunctionDoc(gizmo.generate_fibonacci_sequence)
assert doc['Summary'] != ''
assert doc['Extended Summary'] != ''
assert len(doc['Parameters']) == 1
assert doc['Parameters'][0].name != ''
assert doc['Parameters'][0].type != ''
assert doc['Parameters'][0].desc != ''
assert len(doc['Yields']) == 1
assert doc['Yields'][0].name != ''
assert doc['Yields'][0].type != ''
assert doc['Yields'][0].desc != ''
def _get_numpydoc_obj(py_obj:Any):
# special case where __doc__ is None e.g. with non overwritten dunder methods
# such as "__dict__"
if hasattr(py_obj, '__doc__') and getattr(py_obj, '__doc__') is None:
return NumpyDocString('')
# "normal" cases
if inspect.isclass(py_obj):
doc = ClassDoc(py_obj)
elif inspect.isfunction(py_obj) or inspect.ismethod(py_obj):
doc = FunctionDoc(py_obj)
elif hasattr(py_obj, '__doc__'):
doc = NumpyDocString(py_obj.__doc__)
else: # pragma: no cover
raise TypeError(f'The object {py_obj} is not a class, function, method '
'or any other Python object that has a __doc__ attribute')
return doc
def extract_args(self, val):
docstring = getattr(val, '__doc__', '')
if not docstring:
docstring = ''
docstring = docstring.strip()
npdoc = FunctionDoc(val)
params = npdoc['Parameters']
paragraphs = self._split_paras(docstring)
if (len(paragraphs) == 1 and not params) or len(paragraphs) > 1:
summary = paragraphs[0]
else:
summary = ''
if params: # Assuming the Parameters section is the last 'paragraph'
paragraphs = paragraphs[:-1]
detail = '\n'.join(x for x in paragraphs if x)
return summary, detail, params
def test_ex10():
"""Ex10: Document your function using numpydoc"""
from numpydoc.docscrape import FunctionDoc
import gizmo
assert hasattr(gizmo, 'multiplication_table')
assert hasattr(gizmo.multiplication_table, '__doc__')
doc = FunctionDoc(gizmo.multiplication_table)
assert doc['Summary'] != ''
assert doc['Extended Summary'] != ''
assert len(doc['Parameters']) == 1
assert doc['Parameters'][0].name != ''
assert doc['Parameters'][0].type != ''
assert doc['Parameters'][0].desc != ''
assert len(doc['Returns']) == 1
assert doc['Returns'][0].name != ''
assert doc['Returns'][0].type != ''
assert doc['Returns'][0].desc != ''
def docstring_func(pyobj):
"""Get the docstring dictionary of a function
Parameters
----------
pyobj : function name
Any object in Python for which you want the docstring
Returns
-------
FunctionDoc
If pyobj is a function or class method
A dictionary of the formatted numpy docstring can be
accessed by :code:`return_val._parsed_data`
Keys:
'Signature': '',
'Summary': [''],
'Extended Summary': [],
'Parameters': [],
'Returns': [],
'Raises': [],
'Warns': [],
'Other Parameters': [],
'Attributes': [],
'Methods': [],
'See Also': [],
'Notes': [],
'Warnings': [],
'References': '',
'Examples': '',
'index': {}
Taken from:
https://github.com/numpy/numpydoc/blob/master/numpydoc/docscrape.py#L94
"""
if inspect.isfunction(pyobj) or inspect.ismethod(pyobj):
return FunctionDoc(pyobj)
else:
raise ValueError("The pyobj input parameter is not a function."
"Your parameter returned {0} from "
"type(pyobj)".format(type(pyobj)))
def write_doc(mod,
functions,
name,
add_info=None,
param_append=None,
ignore_params=None):
if add_info is None:
add_info = []
if param_append is None:
param_append = {}
if ignore_params is None:
ignore_params = []
add_info.extend([[]] * (len(functions) - len(add_info)))
for i, function in enumerate(functions):
func_name = getattr(mod, function)
doc = FunctionDoc(func_name)
if verbose:
name += "." + function
else:
name = function
write_function(name, doc, add_info[i], param_append, ignore_params)
def docstring_func(pyobj):
"""Get the docstring dictionary of a function
Parameters
----------
pyobj : function name
Any object in Python for which you want the docstring
Returns
-------
FunctionDoc
If pyobj is a function or class method
A dictionary of the formatted numpy docstring can be
accessed by :code:`return_val[]`
Keys:
'Signature': '',
'Summary': [''],
'Extended Summary': [],
'Parameters': [],
'Returns': [],
'Raises': [],
'Warns': [],
'Other Parameters': [],
'Attributes': [],
'Methods': [],
'See Also': [],
'Notes': [],
'Warnings': [],
'References': '',
'Examples': '',
'index': {}
Taken from:
https://github.com/numpy/numpydoc/blob/master/numpydoc/docscrape.py#L94
"""
return FunctionDoc(pyobj)
def test_docstring(layer):
name = layer.__name__
method_name = f'add_{camel_to_snake(name)}'
method = getattr(Viewer, method_name)
method_doc = FunctionDoc(method)
layer_doc = ClassDoc(layer)
# check summary section
method_summary = ' '.join(method_doc['Summary']) # join multi-line summary
summary_format = 'Add an? .+? layer to the layer list.'
assert re.match(
summary_format, method_summary
), f"improper 'Summary' section of '{method_name}'"
# check parameters section
method_params = method_doc['Parameters']
layer_params = layer_doc['Parameters']
# Remove path parameter from viewer method if it exists
method_params = [m for m in method_params if m.name != 'path']
if name == 'Image':
# For Image just test arguments that are in layer are in method
named_method_params = [m.name for m in method_params]
for layer_param in layer_params:
l_name, l_type, l_description = layer_param
assert l_name in named_method_params
else:
try:
assert len(method_params) == len(layer_params)
for method_param, layer_param in zip(method_params, layer_params):
m_name, m_type, m_description = method_param
l_name, l_type, l_description = layer_param
# descriptions are treated as lists where each line is an
# element
m_description = ' '.join(m_description)
l_description = ' '.join(l_description)
assert m_name == l_name, 'different parameter names or order'
assert (
m_type == l_type
), f"type mismatch of parameter '{m_name}'"
assert (
m_description == l_description
), f"description mismatch of parameter '{m_name}'"
except AssertionError as e:
raise AssertionError(
f"docstrings don't match for class {name}"
) from e
# check returns section
(method_returns,) = method_doc[
'Returns'
] # only one thing should be returned
description = ' '.join(method_returns[-1]) # join multi-line description
method_returns = *method_returns[:-1], description
if name == 'Image':
assert method_returns == (
'layer',
f':class:`napari.layers.{name}` or list',
f'The newly-created {name.lower()} layer or list of {name.lower()} layers.', # noqa: E501
), f"improper 'Returns' section of '{method_name}'"
else:
assert method_returns == (
'layer',
f':class:`napari.layers.{name}`',
f'The newly-created {name.lower()} layer.',
), f"improper 'Returns' section of '{method_name}'"
def parse_args():
"""
Goes through all detected functions that are
bound and adds them to the argument parser,
along with their scopes. Then parses the
command line and returns a dictionary.
Parameters
----------
output_path : str, optional
Saves the args that are parsed to the given
file for running the command again, by default None
"""
p = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
p.add_argument('--args.save',
type=str,
required=False,
help="Path to save all arguments used to run script to.")
p.add_argument('--args.load',
type=str,
required=False,
help="Path to load arguments from, stored as a .yml file.")
p.add_argument('--args.debug',
type=int,
required=False,
default=0,
help="Print arguments as they are passed to each function.")
# Add kwargs from function to parser
for func_name in PARSE_FUNCS:
func, patterns, no_global = PARSE_FUNCS[func_name]
sig = inspect.signature(func)
prefix = func.__name__
docstring = FunctionDoc(func)
parameter_help = docstring['Parameters']
parameter_help = {x.name: ' '.join(x.desc) for x in parameter_help}
f = p.add_argument_group(
title=f"Generated arguments for function {prefix}", )
for key, val in sig.parameters.items():
arg_type = val.annotation
arg_val = val.default
if arg_val is not inspect.Parameter.empty:
arg_names = []
arg_help = {}
help_text = ''
if key in parameter_help:
help_text = textwrap.fill(parameter_help[key],
width=HELP_WIDTH)
if not no_global:
arg_names.append(f'--{prefix}.{key}')
arg_help[arg_names[-1]] = help_text
for pattern in patterns:
arg_names.append(f'--{pattern}/{prefix}.{key}')
arg_help[arg_names[-1]] = argparse.SUPPRESS
for arg_name in arg_names:
inner_types = [str, int, float, bool]
list_types = [List[x] for x in inner_types]
if arg_type is bool:
f.add_argument(arg_name,
action='store_true',
help=arg_help[arg_name])
elif arg_type in list_types:
_type = inner_types[list_types.index(arg_type)]
f.add_argument(arg_name,
type=str_to_list(_type),
default=arg_val,
help=arg_help[arg_name])
elif arg_type is Dict:
f.add_argument(arg_name,
type=str_to_dict(),
default=arg_val,
help=arg_help[arg_name])
else:
f.add_argument(arg_name,
type=arg_type,
default=arg_val,
help=arg_help[arg_name])
desc = docstring['Summary']
desc = ' '.join(desc)
if patterns:
desc += (
f" Additional scope patterns: {', '.join(list(patterns))}. "
"Use these by prefacing any of the args below with one "
"of these patterns. For example: "
f"--{patterns[0]}/{prefix}.{key} VALUE.")
desc = textwrap.fill(desc, width=HELP_WIDTH)
f.description = desc
used_args = [x.replace('--', '') for x in sys.argv if x.startswith('--')]
used_args.extend(['args.save', 'args.load'])
args = vars(p.parse_args())
load_args_path = args.pop('args.load')
save_args_path = args.pop('args.save')
debug_args = args.pop('args.debug')
pattern_keys = [key for key in args if '/' in key]
top_level_args = [key for key in args if '/' not in key]
for key in pattern_keys:
# If the top-level arguments were altered but the ones
# in patterns were not, change the scoped ones to
# match the top-level (inherit arguments from top-level).
pattern, arg_name = key.split('/')
if key not in used_args:
args[key] = args[arg_name]
if load_args_path:
loaded_args = load_args(load_args_path)
# Overwrite defaults with things in loaded arguments.
# except for things that came from the command line.
for key in loaded_args:
if key not in used_args:
args[key] = loaded_args[key]
for key in pattern_keys:
pattern, arg_name = key.split('/')
if key not in loaded_args and key not in used_args:
if arg_name in loaded_args:
args[key] = args[arg_name]
for key in top_level_args:
if key in used_args:
for pattern_key in pattern_keys:
pattern, arg_name = pattern_key.split('/')
if key == arg_name and pattern_key not in used_args:
args[pattern_key] = args[key]
if save_args_path:
dump_args(args, save_args_path)
# Put them back in case the script wants to use them
args['args.load'] = load_args_path
args['args.save'] = save_args_path
args['args.debug'] = debug_args
return args
def _get_script_args(self):
"""Generate all tlbx Args for this Glider"""
node_lookup = self.glider.get_node_lookup()
script_args = OrderedDict() # Map of arg names to Args
arg_dests = {} # Map of arg dests back to names
node_arg_names = defaultdict(set)
requires_data = not isinstance(self.glider.top_node, NoInputNode)
if requires_data and not self.blacklisted("", SCRIPT_DATA_ARG):
script_args[SCRIPT_DATA_ARG] = Arg(SCRIPT_DATA_ARG, nargs="+")
def add_script_arg(node, arg_name, **kwargs):
script_arg = self._get_script_arg(node, arg_name, **kwargs)
if not script_arg:
return
script_args[script_arg.name] = script_arg
arg_dests[script_arg.dest] = script_arg.name
node_arg_names[arg_name].add(script_arg.name)
for node in node_lookup.values():
node_help = {}
if FunctionDoc:
try:
# Only works if run() has docs in numpydoc format
docs = FunctionDoc(node.run)
node_help = {
v.name: "\n".join(v.desc)
for v in docs["Parameters"]
}
except Exception as e:
info("failed to parse node '%s' run() docs: %s" %
(node.name, str(e)))
for arg_name, _ in node.run_args.items():
add_script_arg(
node,
arg_name,
required=True,
arg_help=node_help.get(arg_name, None),
)
for kwarg_name, kwarg_default in node.run_kwargs.items():
add_script_arg(
node,
kwarg_name,
required=False,
default=kwarg_default,
arg_help=node_help.get(kwarg_name, None),
)
def assert_arg_present(custom_arg, arg_name):
raiseifnot(
arg_name in script_args,
("Custom arg %s with dest=%s maps to node arg=%s "
"which is not in the script arg list. Check for "
"conflicting args that cover the same node arg." %
(custom_arg.name, custom_arg.dest, arg_name)),
)
for custom_arg in self.custom_args:
raiseif(
self.blacklisted("", custom_arg.name),
"Blacklisted arg '%s' passed as a custom arg" %
custom_arg.name,
)
if custom_arg.dest in node_arg_names:
# Find and delete all node-based args this will cover
for arg_name in node_arg_names[custom_arg.dest]:
assert_arg_present(custom_arg, arg_name)
del script_args[arg_name]
if custom_arg.dest in arg_dests:
# Remove the original arg that this custom arg will satisfy
arg_name = arg_dests[custom_arg.dest]
assert_arg_present(custom_arg, arg_name)
del script_args[arg_name]
script_args[custom_arg.name] = custom_arg
arg_dests[custom_arg.dest] = custom_arg.name
return script_args.values()
def construct_parser(function, parser=None):
"""Automatically construct an argument parser from the function signature and docstring.
Parameters
----------
function : callable
The function that we want to turn into a script.
parser : argparse.ArgumentParser, optional
An existing argument parser.
Returns
-------
parser : argparse.ArgumentParser
The parser for this particular function.
Raises
------
ArgumentError
If parser is not None and already has a corresponding argument.
ValueError
If it cannot automatically extract the dtype from the function.
"""
# Try extracting the docstring
docstring = FunctionDoc(function)
docstring_params = {
param_name: (param_type, param_doc)
for param_name, param_type, param_doc in docstring['Parameters']
}
# Get the function description
if docstring['Summary']:
description = '\n'.join(docstring['Summary'])
else:
description = f'The {function.__name__} function.'
if docstring['Extended Summary']:
description += '\n\n' + '\n'.join(docstring['Extended Summary'])
# Construct parser if none is provided
if parser is None:
parser = argparse.ArgumentParser(
description=description,
formatter_class=argparse.RawTextHelpFormatter)
# Set default callback for the function.
parser.set_defaults(_function=function)
# Go through parameters of function signature
signature = inspect.signature(function)
for name, parameter in signature.parameters.items():
# Whether the parameter is a kwarg
is_kwarg = parameter.default is not parameter.empty
# Get the parameter dtype
param_type = None
if parameter.annotation is not parameter.empty:
# Annotation is provided (e : int)
param_type = parameter.annotation
elif is_kwarg and parameter.default is not None:
# Kwargs have default parameters / dtypes
param_type = type(parameter.default)
elif param_type is None and name in docstring_params:
# Otherwise, let's get it from the docstring if provided
str_type = docstring_params[name][0]
if str_type:
# Remove optional parameter if specified
str_type = str_type.split(',')[0].strip()
param_type = eval(str_type)
if param_type is None:
raise ValueError(
f"Unable to identify type of `{name}` parameter. Please "
f"specify the dtype as part the annotation, kwarg, "
f"or in the docstring.")
# Get the default value if it's a keyword argument
if parameter.default is parameter.empty:
param_default = dict(required=True)
else:
param_default = dict(default=parameter.default)
# Get the parameter description if it's available
if name in docstring_params and docstring_params[name][1]:
param_help = '\n'.join(docstring_params[name][1])
else:
param_help = None
# Nothing provided, use some sane defaults
if param_type is not None:
param_help = param_type.__name__
if is_kwarg:
param_help += f', default: {parameter.default}'
parser.add_argument(f'--{name}',
type=param_type,
help=param_help,
**param_default)
return parser
def docstring_add_deprecated(func, kwarg_mapping, deprecated_version):
"""Add deprecated kwarg(s) to the "Other Params" section of a docstring.
Parameters
---------
func : function
The function whose docstring we wish to update.
kwarg_mapping : dict
A dict containing {old_arg: new_arg} key/value pairs as used by
`deprecate_kwarg`.
deprecated_version : str
A major.minor version string specifying when old_arg was
deprecated.
Returns
-------
new_doc : str
The updated docstring. Returns the original docstring if numpydoc is
not available.
"""
if func.__doc__ is None:
return None
try:
from numpydoc.docscrape import FunctionDoc, Parameter
except ImportError:
# Return an unmodified docstring if numpydoc is not available.
return func.__doc__
Doc = FunctionDoc(func)
for old_arg, new_arg in kwarg_mapping.items():
desc = [
f'Deprecated in favor of `{new_arg}`.', f'',
f'.. deprecated:: {deprecated_version}'
]
Doc['Other Parameters'].append(
Parameter(name=old_arg, type='DEPRECATED', desc=desc))
new_docstring = str(Doc)
# new_docstring will have a header starting with:
#
# .. function:: func.__name__
#
# and some additional blank lines. We strip these off below.
split = new_docstring.split('\n')
no_header = split[1:]
while not no_header[0].strip():
no_header.pop(0)
# Store the initial description before any of the Parameters fields.
# Usually this is a single line, but the while loop covers any case
# where it is not.
descr = no_header.pop(0)
while no_header[0].strip():
descr += '\n ' + no_header.pop(0)
descr += '\n\n'
# '\n ' rather than '\n' here to restore the original indentation.
final_docstring = descr + '\n '.join(no_header)
# strip any extra spaces from ends of lines
final_docstring = '\n'.join(
[line.rstrip() for line in final_docstring.split('\n')])
return final_docstring
def numpy_fn_parser(fn):
data = FunctionDoc(fn)
# print(data)
return numpy_doc_parser(data)
def print_docstring(obj, file, depth):
"""Prints a classes's docstring to a file."""
doc = ClassDoc(obj) if inspect.isclass(obj) else FunctionDoc(obj)
printf = functools.partial(print, file=file)
printf(h1(obj.__name__))
printf(
linkifier.linkify_fences(paragraph(concat_lines(doc["Summary"])),
depth))
printf(
linkifier.linkify_fences(
paragraph(concat_lines(doc["Extended Summary"])), depth))
# We infer the type annotations from the signatures, and therefore rely on the signature
# instead of the docstring for documenting parameters
try:
signature = inspect.signature(obj)
except ValueError:
signature = (
inspect.Signature()
) # TODO: this is necessary for Cython classes, but it's not correct
params_desc = {
param.name: " ".join(param.desc)
for param in doc["Parameters"]
}
# Parameters
if signature.parameters:
printf(h2("Parameters"))
for param in signature.parameters.values():
# Name
printf(f"- **{param.name}**", end="")
# Type annotation
if param.annotation is not param.empty:
anno = inspect.formatannotation(param.annotation)
anno = linkifier.linkify_dotted(anno, depth)
printf(f" (*{anno}*)", end="")
# Default value
if param.default is not param.empty:
printf(f" – defaults to `{param.default}`", end="")
printf("\n", file=file)
# Description
desc = params_desc[param.name]
if desc:
printf(f" {desc}\n")
printf("")
# Attributes
if doc["Attributes"]:
printf(h2("Attributes"))
for attr in doc["Attributes"]:
# Name
printf(f"- **{attr.name}**", end="")
# Type annotation
if attr.type:
printf(f" (*{attr.type}*)", end="")
printf("\n", file=file)
# Description
desc = " ".join(attr.desc)
if desc:
printf(f" {desc}\n")
printf("")
# Examples
if doc["Examples"]:
printf(h2("Examples"))
in_code = False
after_space = False
for line in inspect.cleandoc("\n".join(doc["Examples"])).splitlines():
if (in_code and after_space and line and not line.startswith(">>>")
and not line.startswith("...")):
printf("```\n")
in_code = False
after_space = False
if not in_code and line.startswith(">>>"):
printf("```python")
in_code = True
after_space = False
if not line:
after_space = True
printf(line)
if in_code:
printf("```")
printf("")
# Methods
if inspect.isclass(obj) and doc["Methods"]:
printf(h2("Methods"))
printf_indent = lambda x, **kwargs: printf(f" {x}", **kwargs)
for meth in doc["Methods"]:
printf(paragraph(f'???- note "{meth.name}"'))
# Parse method docstring
docstring = inherit_docstring(c=obj, meth=meth.name)
if not docstring:
continue
meth_doc = FunctionDoc(func=None, doc=docstring)
printf_indent(paragraph(" ".join(meth_doc["Summary"])))
if meth_doc["Extended Summary"]:
printf_indent(paragraph(" ".join(
meth_doc["Extended Summary"])))
# We infer the type annotations from the signatures, and therefore rely on the signature
# instead of the docstring for documenting parameters
signature = inherit_signature(obj, meth.name)
params_desc = {
param.name: " ".join(param.desc)
for param in doc["Parameters"]
}
# Parameters
if len(signature.parameters
) > 1: # signature is never empty, but self doesn't count
printf_indent("**Parameters**\n")
for param in signature.parameters.values():
if param.name == "self":
continue
# Name
printf_indent(f"- **{param.name}**", end="")
# Type annotation
if param.annotation is not param.empty:
printf_indent(
f" (*{inspect.formatannotation(param.annotation)}*)",
end="")
# Default value
if param.default is not param.empty:
printf_indent(f" – defaults to `{param.default}`", end="")
printf_indent("", file=file)
# Description
desc = params_desc.get(param.name)
if desc:
printf_indent(f" {desc}")
printf_indent("")
# Returns
if meth_doc["Returns"]:
printf_indent("**Returns**\n")
return_val = meth_doc["Returns"][0]
if signature.return_annotation is not inspect._empty:
if inspect.isclass(signature.return_annotation):
printf_indent(
f"*{signature.return_annotation.__name__}*: ",
end="")
else:
printf_indent(f"*{signature.return_annotation}*: ",
end="")
printf_indent(return_val.type)
printf_indent("")
# Notes
if doc["Notes"]:
printf(h2("Notes"))
printf(paragraph("\n".join(doc["Notes"])))
# References
if doc["References"]:
printf(h2("References"))
printf(paragraph("\n".join(doc["References"])))
