def build(mcs, p_doc: str, c_doc: str) -> str:
if not p_doc or not c_doc:
return p_doc or c_doc
p_doc = NumpyDocString(p_doc)
c_doc = NumpyDocString(c_doc)
# reuse parents' doc except for `Extended Summary`
for section, content in c_doc.items():
if section != "Extended Summary" and (not content
and p_doc[section]):
c_doc[section] = p_doc[section]
# merge parameters
c_params = [param for param in c_doc['Parameters']]
c_param_names = set(param.name for param in c_params)
p_params = [
param for param in p_doc['Parameters']
if param.name not in c_param_names
]
c_params += p_params
c_doc['Parameters'] = c_params
return str(c_doc)
def test_no_index_in_str():
assert "index" not in str(NumpyDocString("""Test idx
"""))
assert "index" in str(NumpyDocString("""Test idx
.. index :: random
"""))
assert "index" in str(NumpyDocString("""Test idx
.. index ::
foo
"""))
def _docspec_comments(obj):
u"""
Inspect the docstring and get the comments for each parameter.
"""
# Sometimes our docstring is on the class, and sometimes it's on the initializer,
# so we've got to check both.
class_docstring = getattr(obj, u'__doc__', None)
init_docstring = getattr(obj.__init__, u'__doc__', None) if hasattr(
obj, u'__init__') else None
docstring = class_docstring or init_docstring or u''
doc = NumpyDocString(docstring)
params = doc[u"Parameters"]
comments = {}
for line in params:
# It looks like when there's not a space after the parameter name,
# numpydocstring parses it incorrectly.
name_bad = line[0]
name = name_bad.split(u":")[0]
# Sometimes the line has 3 fields, sometimes it has 4 fields.
comment = u"\n".join(line[-1])
comments[name] = comment
return comments
def extract_section(obj, section):
docstr = NumpyDocString(obj.__doc__)
section = section.lower().capitalize()
if section in ('Parameters', 'Returns', 'Yields', 'Other Parameters',
'Raises', 'Warns'):
return docstr[section]
return []
def enum_doc(name, enum, header_level, source_location):
"""
Generate markdown for an enum
Parameters
----------
name : str
Name of the thing being documented
enum : EnumMeta
Enum to document
header_level : int
Heading level
source_location : str
URL of repo containing source code
"""
lines = [f"{'#'*header_level} Enum **{name}**\n\n"]
lines.append(f"```python\n{name}\n```\n")
lines.append(get_source_link(enum, source_location))
try:
doc = NumpyDocString(inspect.getdoc(enum))._parsed_data
lines += summary(doc)
except:
pass
lines.append(f"{'#'*(header_level + 1)} Members\n\n")
lines += [f"- `{str(v).split('.').pop()}`: `{v.value}` \n\n" for v in enum]
return lines
def check_docstring_indention(doc: Docstring) -> list:
"""
Check indention of docstring since numpydoc reports weird results.
Parameters
----------
doc : numpydoc.validate.Docstring
Docstring handler.
Returns
-------
list
List of tuples with Modin error code and its description.
"""
from modin.utils import _get_indent
numpy_docstring = NumpyDocString(doc.clean_doc)
numpy_docstring._doc.reset()
numpy_docstring._parse_summary()
sections = list(numpy_docstring._read_sections())
errors = []
for section in sections:
description = "\n".join(section[1])
if _get_indent(description) != 0:
errors.append(
("MD03", MODIN_ERROR_CODES["MD03"].format(section=section[0]))
)
return errors
def _search_return_in_numpydocstr(docstr):
r"""
Search `docstr` (in numpydoc format) for type(-s) of `param_str`.
>>> from jedi.evaluate.docstrings import * # NOQA
>>> from jedi.evaluate.docstrings import _search_return_in_numpydocstr
>>> from jedi.evaluate.docstrings import _expand_typestr
>>> docstr = (
... 'Returns\n'
... '----------\n'
... 'int\n'
... ' can return an anoymous integer\n'
... 'out : ndarray\n'
... ' can return a named value\n'
... )
>>> _search_return_in_numpydocstr(docstr)
['int', 'ndarray']
"""
doc = NumpyDocString(docstr)
returns = doc._parsed_data['Returns']
returns += doc._parsed_data['Yields']
found = []
for p_name, p_type, p_descr in returns:
if not p_type:
p_type = p_name
p_name = ''
m = re.match(r'([^,]+(,[^,]+)*?)$', p_type)
if m:
p_type = m.group(1)
found.extend(_expand_typestr(p_type))
return found
def test_unknown_section():
doc_text = """
Test having an unknown section
Mope
----
This should be ignored and warned about
"""
class BadSection:
"""Class with bad section.
Nope
----
This class has a nope section.
"""
pass
with pytest.warns(UserWarning, match="Unknown section Mope") as record:
NumpyDocString(doc_text)
assert len(record) == 1
# SphinxClassDoc has _obj.__name__ == "BadSection". Test that this is
# included in the message
msg_match = "Unknown section Nope in the docstring of BadSection"
with pytest.warns(UserWarning, match=msg_match) as record:
SphinxClassDoc(BadSection)
assert len(record) == 1
def test_unknown_section():
doc_text = """
Test having an unknown section
Mope
----
This should be ignored and warned about
"""
class BadSection(object):
"""Class with bad section.
Nope
----
This class has a nope section.
"""
pass
with warnings.catch_warnings(record=True) as w:
NumpyDocString(doc_text)
assert len(w) == 1
assert "Unknown section Mope" == str(w[0].message)
with warnings.catch_warnings(record=True) as w:
SphinxClassDoc(BadSection)
assert len(w) == 1
assert_true('test_docscrape.test_unknown_section.<locals>.BadSection'
in str(w[0].message)
or 'test_docscrape.BadSection' in str(w[0].message))
def _search_param_in_numpydocstr(docstr, param_str):
r"""
Search `docstr` (in numpydoc format) for type(-s) of `param_str`.
>>> from jedi.evaluate.docstrings import * # NOQA
>>> from jedi.evaluate.docstrings import _search_param_in_numpydocstr
>>> docstr = (
... 'Parameters\n'
... '----------\n'
... 'x : ndarray\n'
... 'y : int or str or list\n'
... 'z : {"foo", "bar", 100500}, optional\n'
... )
>>> _search_param_in_numpydocstr(docstr, 'x')
['ndarray']
>>> sorted(_search_param_in_numpydocstr(docstr, 'y'))
['int', 'list', 'str']
>>> sorted(_search_param_in_numpydocstr(docstr, 'z'))
['int', 'str']
"""
params = NumpyDocString(docstr)._parsed_data['Parameters']
for p_name, p_type, p_descr in params:
if p_name == param_str:
m = re.match(r'([^,]+(,[^,]+)*?)(,[ ]*optional)?$', p_type)
if m:
p_type = m.group(1)
return _expand_typestr(p_type)
return []
def getDicoParams(instanceClassifier):
"""
This function allows to retrieve the name of the parameter,
the types for the accepted values in case of modification and that description.
:param instanceClassifier: the instance of the estimator issue by dictEstimator
:return: A dictionary containing the name of the estimator as a key and a tuple
as a value. The tuple contains (instance of type, default value of params, description)
"""
dico = {}
# regex to find type
types_re = re.compile(
r"(?P<type>(float|int(eger)?|str(ing)?|bool(ean)?|dict?|list?|array?))"
)
# Map string to object
type_map = {
'string': str,
'str': str,
'boolean': bool,
'bool': bool,
'int': int,
'integer': int,
'float': float,
'dict': dict,
'array': list,
'list': list,
}
classifierTemp = instanceClassifier().get_params()
doc = NumpyDocString(
" " +
instanceClassifier.__doc__) # hack, get the doc for the classifier
# For each params in get_params, take the name, first row contain type and the description
for name, type_, descriptions in doc['Parameters']:
# Find types in this row
types = types_re.match(type_)
# Check if the type was finded and if is not equal to "Infinity"
if types != None and classifierTemp[name] != "Infinity":
# Creates a complete description for this params
if type_map.get(types.group()) == dict:
completeDescription = str(
type_
) + "\n \n" + "Example : {'key':45} " + "\n \n" + " ".join(
str(e) for e in descriptions)
elif type_map.get(types.group()) == list:
completeDescription = str(
type_
) + "\n \n" + "Example : [value1,value2] " + "\n \n" + " ".join(
str(e) for e in descriptions)
else:
completeDescription = str(type_) + "\n \n" + " ".join(
str(e) for e in descriptions)
# add into the dict at the key (name of the params)
# a tuple (instance of type, default value of params, description)
dico[name] = (type_map.get(types.group()), classifierTemp[name],
completeDescription)
return dico
def check_function(self, dir_name, file_name, func_name, func, failures):
"""
Perform docstring checks on a function.
Parameters
----------
dir_name : str
The name of the directory in which the method is defined.
file_name : str
The name of the file in which the method is defined.
func_name : str
The name of the function being checked
func : function
The function being tested.
failures : dict
The failures encountered by the method. These are all stored
so that we can fail once at the end of the check_method method
with information about every failure. Form is
{ 'dir_name/file_name:class_name.method_name': [ messages ] }
"""
if PY3:
argspec = inspect.getfullargspec(func)
else:
argspec = inspect.getargspec(func)
doc = inspect.getdoc(func)
new_failures = []
# If the method is decorated with @contextmanager, skip it for now
if _is_context_manager(func):
return
# Check if docstring is missing
if doc is None:
new_failures.append('is missing docstring')
return
if not func.__doc__.startswith('\n'):
new_failures.append('docstring should start with a new line')
# Check if docstring references another function
if doc[:3] == 'See':
return
nds = NumpyDocString(doc)
new_failures.extend(self.check_summary(nds))
new_failures.extend(self.check_parameters(argspec, nds))
new_failures.extend(self.check_returns(func, nds))
if new_failures:
key = '{0}/{1}:{2}'.format(dir_name, file_name, func_name)
if key in failures:
failures[key] += new_failures
else:
failures[key] = new_failures
def _is_deprecated(real_name):
try:
obj, parent, modname = _import_by_name(real_name)
except ImportError:
return False
doc = NumpyDocString(obj.__doc__ or "")
summary = "".join(doc["Summary"] + doc["Extended Summary"])
return ".. deprecated::" in summary
def parse_methods(self, key, item):
# todo
s = []
for name, _, lst in item:
doc = NumpyDocString('\n'.join(lst))
doc._parsed_data['Signature'] = name
s.append(self.get_doctext(doc))
return '\n'.join(s)
def __init__(self, name):
self.name = name
obj = self._load_obj(name)
self.obj = obj
self.code_obj = self._to_original_callable(obj)
self.raw_doc = obj.__doc__ or ''
self.clean_doc = pydoc.getdoc(obj)
self.doc = NumpyDocString(self.clean_doc)
def _is_deprecated(real_name):
try:
obj, parent, modname = _import_by_name(real_name)
except ImportError:
return False
doc = NumpyDocString(obj.__doc__ or '')
summary = ''.join(doc['Summary'] + doc['Extended Summary'])
return '.. deprecated::' in summary
def _get_numpydoc_obj(py_obj:Any):
# special case where __doc__ is None e.g. with non overwritten dunder methods
# such as "__dict__"
if hasattr(py_obj, '__doc__') and getattr(py_obj, '__doc__') is None:
return NumpyDocString('')
# "normal" cases
if inspect.isclass(py_obj):
doc = ClassDoc(py_obj)
elif inspect.isfunction(py_obj) or inspect.ismethod(py_obj):
doc = FunctionDoc(py_obj)
elif hasattr(py_obj, '__doc__'):
doc = NumpyDocString(py_obj.__doc__)
else: # pragma: no cover
raise TypeError(f'The object {py_obj} is not a class, function, method '
'or any other Python object that has a __doc__ attribute')
return doc
def test_see_also_parse_error():
text = ("""
z(x,theta)
See Also
--------
:func:`~foo`
""")
with pytest.raises(ValueError, match="See Also entry ':func:`~foo`'"):
NumpyDocString(text)
def _search_param_in_numpydocstr(docstr, param_str):
"""Search `docstr` (in numpydoc format) for type(-s) of `param_str`."""
params = NumpyDocString(docstr)._parsed_data['Parameters']
for p_name, p_type, p_descr in params:
if p_name == param_str:
m = re.match('([^,]+(,[^,]+)*?)(,[ ]*optional)?$', p_type)
if m:
p_type = m.group(1)
return _expand_typestr(p_type)
return []
def test_section_twice():
doc_text = """
Test having a section Notes twice
Notes
-----
See the next note for more information
Notes
-----
That should break...
"""
with pytest.raises(ValueError, match="The section Notes appears twice"):
NumpyDocString(doc_text)
# if we have a numpydoc object, we know where the error came from
class Dummy:
"""
Dummy class.
Notes
-----
First note.
Notes
-----
Second note.
"""
def spam(self, a, b):
"""Spam\n\nSpam spam."""
pass
def ham(self, c, d):
"""Cheese\n\nNo cheese."""
pass
def dummy_func(arg):
"""
Dummy function.
Notes
-----
First note.
Notes
-----
Second note.
"""
with pytest.raises(ValueError, match="Dummy class"):
SphinxClassDoc(Dummy)
with pytest.raises(ValueError, match="dummy_func"):
SphinxFunctionDoc(dummy_func)
def splice(self, func):
"""
Main work of creating or updating the function docstring happens here.
This is a signature preserving decorator that only alters the `__doc__`
attribute of the object.
"""
# avoid circular import :|
from recipes.string import sub
# docstring of decorated function. The one to be adapted.
self.origin = docstring = func.__doc__
# make substitutions. verbatim substitutions happen
if docstring is None:
# if decorated function has no docstring carbon copy the
# docstring from input source
if callable(self.from_func):
docstring = self.from_func.__doc__
else:
# splicing from multiple sources. Start from a blank slate
docstring = ''
else:
# decorated function has a docstring. Look for directives and
# substitute them
if callable(self.from_func):
# new = self.sub(docstring)
self.subs.update(get_subs(docstring, self.from_func))
if self.subs:
new = sub(docstring, self.subs)
# TODO: do things in the order in which arguments were passed
if new == docstring:
wrn.warn('Docstring for function {func} identical '
'after substitution')
docstring = new
else:
'multi-source without explicit mapping. might be ambiguous'
# parse the update docstring
doc = NumpyDocString(docstring)
# parse directives and insert
doc = self.insert(func, doc)
# remove omitted sections / parameters
self.get_remove(self.from_func)
# write the new docstring
if (self.subs or self.insertion):
func.__doc__ = str(doc)
else:
wrn.warn(
f'{self.__class__.__name__} did not alter docstring for {func}.'
)
return func
def load_docstring(obj):
if obj not in load_docstring.cache:
docstring = normalize_docstring(getattr(obj, "__doc__", None) or "")
doc = dict(NumpyDocString(docstring))
enhance_multilines_doc(doc, "Summary")
enhance_multilines_doc(doc, "Extended Summary")
if doc["Extended Summary"] == [[]]:
doc["Extended Summary"] = []
doc["Summary"], *more_doc = doc["Summary"]
if more_doc:
doc["Extended Summary"] = more_doc + doc["Extended Summary"]
if isinstance(obj, type):
enhance_attributes(doc, "Attributes",
getattr(obj, "__annotations__", {}))
for base in obj.__mro__[1:-1]:
base_doc = load_docstring(base)
for name in base_doc.get("Attributes", {}).keys():
if name not in doc["Attributes"]:
doc["Attributes"][name] = base_doc["Attributes"][name]
else:
annotations = getattr(obj, '__annotations__', {})
enhance_attributes(doc, "Parameters", annotations)
klass = get_class_that_defined_method(obj)
if klass:
for base in klass.__mro__[:-1]:
try:
base_obj = getattr(base, obj.__name__)
except AttributeError:
continue
if base_obj is obj:
continue
try:
if base_obj.__func__ is obj.__func__:
continue
except AttributeError:
pass
base_doc = load_docstring(base_obj)
for name in base_doc["Parameters"].keys():
if name not in doc["Parameters"]:
doc["Parameters"][name] = base_doc["Parameters"][
name]
enhance_return(doc, "Returns", annotations)
enhance_raises(doc, "Raises")
if doc.get("Examples"):
doc["Examples"] = "\n".join(doc["Examples"])
load_docstring.cache[obj] = doc
return load_docstring.cache[obj]
def getNames():
# for reference: at the time of this writing, the list of names was:
# Methods, Parameters, Warnings, Warns, Other Parameters, Summary,
# Returns, References, Examples, Signature, Raises, Attributes,
# See Also, Notes, Extended Summary
global _sectnames
if _sectnames is None:
_sectnames = set(NumpyDocString('')._parsed_data.keys())
_sectnames -= set(['index'])
_sectnames |= set(['Parameters', 'Returns', 'Raises'])
return _sectnames
def test_duplicate_signature():
# Duplicate function signatures occur e.g. in ufuncs, when the
# automatic mechanism adds one, and a more detailed comes from the
# docstring itself.
doc = NumpyDocString("""
z(x1, x2)
z(a, theta)
""")
assert doc['Signature'].strip() == 'z(a, theta)'
def parse_docstring(doc: str, module: str, nested: bool) -> str:
"""Parse docstring and returns formatted markdown
Parameters
----------
doc : str
The docstring of a python object
Returns
-------
str
Formatted markdown
"""
lines = []
if not doc:
return ''
doc = replace_links(doc, module, nested)
doc = NumpyDocString(doc)
if doc.get('Summary'):
lines.append('{}'.format('\n'.join(doc['Summary'])))
lines.append('')
if doc.get('Extended Summary'):
lines.append('{}'.format('\n'.join(doc['Extended Summary'])))
lines.append('')
if doc.get('Examples'):
lines.append('__Examples__\n\n')
lines.extend(doc['Examples'])
lines.append('')
if doc.get('Attributes'):
lines.append('__Attributes__\n\n')
for name, dtype, description in doc['Attributes']:
lines.append('- `{}` (`{}`): {}'.format(name, dtype,
'\n\t'.join(description)))
lines.append('')
if doc.get('Parameters'):
lines.append('__Args__\n\n')
for name, dtype, description in doc['Parameters']:
lines.append('- `{}` (`{}`): {}'.format(name, dtype,
'\n\t'.join(description)))
lines.append('')
if doc.get('Returns'):
lines.append('__Returns__\n\n')
for name, dtype, description in doc['Returns']:
lines.append('- `{}`: {}'.format(name, '\n'.join(description)))
lines.append('')
return '\n'.join(lines)
def merge_docs(add_method, layer_string):
# create combined docstring with parameters from add_* and Viewer methods
add_method_doc = NumpyDocString(add_method.__doc__)
params = ("\n".join(add_method_doc._str_param_list('Parameters')) +
VIEW_PARAMS)
# this ugliness is because the indentation of the parsed numpydocstring
# is different for the first parameter :(
lines = params.splitlines()
lines = lines[:3] + textwrap.dedent("\n".join(lines[3:])).splitlines()
params = "\n".join(lines)
n = 'n' if layer_string.startswith(tuple('aeiou')) else ''
return DOC.format(n=n, name=layer_string, params=params)
def check_docstring(f):
# can't inspect builtins
if inspect.isbuiltin(f):
return False
with warnings.catch_warnings():
warnings.simplefilter('error')
try:
parsed = NumpyDocString(inspect.getdoc(f))
except:
print('ERROR PARSING DOCSTRING: %s' % fullname(f))
print('')
return False
if len(parsed['Parameters']) == 0:
return False
def iter_docargs():
for item in chain(parsed['Parameters'], parsed['Other Parameters']):
for rep in item[0].split(','):
yield rep.strip()
doc_args = set(iter_docargs())
try:
argspec = inspect.getargspec(f)
except TypeError as e:
return False
# ignore 'self' or 'cls' in the signature for instance methods or
# class methods.
args = set(argspec.args)
args.discard('cls')
args.discard('self')
# the docstring might legitimately mention parameters that aren't in
# the signature if the function takes *args, or **kwargs
if args != doc_args and len(doc_args) > len(args) and (
(argspec.varargs is not None) or (argspec.keywords is not None)):
return False
# if doc_params != args and len(parsed['Parameters']) > 0:
if args != doc_args:
print('%s ( %s )' % (fullname(f), inspect.getfile(f)))
undoc_args = args.difference(doc_args)
doc_nonargs = doc_args.difference(args)
if undoc_args:
print(' Undocumented arguments: ', undoc_args)
if doc_nonargs:
print(' Documented non-arguments: ', doc_nonargs)
print()
def check_dist_keyword_names():
# Look for collisions between names of distribution shape parameters and
# keywords of distribution methods. See gh-5982.
distnames = set(distdata[0] for distdata in distcont + distdiscrete)
mod_results = []
for distname in distnames:
dist = getattr(stats, distname)
method_members = inspect.getmembers(dist, predicate=inspect.ismethod)
method_names = [
method[0] for method in method_members
if not method[0].startswith('_')
]
for methodname in method_names:
method = getattr(dist, methodname)
try:
params = NumpyDocString(method.__doc__)['Parameters']
except TypeError:
result = (f'stats.{distname}.{methodname}', False,
"Method parameters are not documented properly.")
mod_results.append(result)
continue
if not dist.shapes: # can't have collision if there are no shapes
continue
shape_names = dist.shapes.split(', ')
param_names1 = set(param.name for param in params)
param_names2 = set(inspect.signature(method).parameters)
param_names = param_names1.union(param_names2)
# # Disabling this check in this PR;
# # these discrepancies are a separate issue.
# no_doc_params = {'args', 'kwds', 'kwargs'} # no need to document
# undoc_params = param_names2 - param_names1 - no_doc_params
# if un_doc_params:
# result = (f'stats.{distname}.{methodname}', False,
# f'Parameter(s) {undoc_params} are not documented.')
# mod_results.append(result)
# continue
intersection = param_names.intersection(shape_names)
if intersection:
message = ("Distribution/method keyword collision: "
f"{intersection} ")
result = (f'stats.{distname}.{methodname}', False, message)
else:
result = (f'stats.{distname}.{methodname}', True, '')
mod_results.append(result)
return mod_results
def test_see_also():
doc6 = NumpyDocString("""
z(x,theta)
See Also
--------
func_a, func_b, func_c
func_d : some equivalent func
foo.func_e : some other func over
multiple lines
func_f, func_g, :meth:`func_h`, func_j,
func_k
func_f1, func_g1, :meth:`func_h1`, func_j1
func_f2, func_g2, :meth:`func_h2`, func_j2 : description of multiple
:obj:`baz.obj_q`
:obj:`~baz.obj_r`
:class:`class_j`: fubar
foobar
""")
assert len(doc6['See Also']) == 10
for funcs, desc in doc6['See Also']:
for func, role in funcs:
if func in ('func_a', 'func_b', 'func_c', 'func_f', 'func_g',
'func_h', 'func_j', 'func_k', 'baz.obj_q', 'func_f1',
'func_g1', 'func_h1', 'func_j1', '~baz.obj_r'):
assert not desc, str([func, desc])
elif func in ('func_f2', 'func_g2', 'func_h2', 'func_j2'):
assert desc, str([func, desc])
else:
assert desc, str([func, desc])
if func == 'func_h':
assert role == 'meth'
elif func == 'baz.obj_q' or func == '~baz.obj_r':
assert role == 'obj'
elif func == 'class_j':
assert role == 'class'
elif func in ['func_h1', 'func_h2']:
assert role == 'meth'
else:
assert role is None, str([func, role])
if func == 'func_d':
assert desc == ['some equivalent func']
elif func == 'foo.func_e':
assert desc == ['some other func over', 'multiple lines']
elif func == 'class_j':
assert desc == ['fubar', 'foobar']
elif func in ['func_f2', 'func_g2', 'func_h2', 'func_j2']:
assert desc == ['description of multiple'
], str([desc, ['description of multiple']])
def test_see_also_parse_error():
text = ("""
z(x,theta)
See Also
--------
:func:`~foo`
""")
with assert_raises(ParseError) as err:
NumpyDocString(text)
assert_equal(
str(r":func:`~foo` is not a item name in '\n z(x,theta)\n\n See Also\n --------\n :func:`~foo`\n '"
), str(err.exception))
