def google_to_numpy_docstr(docstr):
"""
Convert a google-style docstring to a numpy-style docstring
Args:
docstr (str): contents of ``func.__doc__`` for some ``func``, assumed
to be in google-style.
Returns:
str: numpy style docstring
"""
from xdoctest.utils.util_str import indent as indent_fn
from xdoctest.docstr import docscrape_google
docblocks = docscrape_google.split_google_docblocks(docstr)
new_parts = []
for key, block in docblocks:
old_body, relpos = block
new_key = key
new_body = old_body
if key == '__DOC__':
new_key = None
new_text = new_body
elif key in {'Args'}:
new_key = 'Parameters'
arginfos = list(docscrape_google.parse_google_argblock(old_body))
parts = []
for info in arginfos:
info['desc'] = indent_fn(info['desc'])
p = '{name}: {type}\n{desc}'.format(**info)
parts.append(p)
parts.append('')
new_body = '\n'.join(parts)
if key in {'Returns', 'Yields'}:
retinfos = list(docscrape_google.parse_google_retblock(old_body))
parts = []
for info in retinfos:
info['desc'] = indent_fn(info['desc'])
info['name'] = info.get('name', '')
parts.append('{name}: {type}\n{desc}'.format(**info))
parts.append('')
new_body = '\n'.join(parts)
if new_key is not None:
new_text = '\n'.join([new_key, '-' * len(new_key), new_body])
if new_text.strip():
new_parts.append(new_text)
new_docstr = '\n'.join(new_parts)
new_docstr = new_docstr.strip('\n')
return new_docstr
def parse_google_docstr_examples(docstr,
callname=None,
modpath=None,
lineno=1,
fpath=None,
eager_parse=True):
"""
Parses Google-style doctests from a docstr and generates example objects
Args:
lineno (int): the line number (starting from 1) of the docstring.
(i.e. if you were to go to this line number in the source file
the starting quotes of the docstr would be on this line).
Raises:
.exceptions.MalformedDocstr: if an error occurs in finding google blocks
.exceptions.DoctestParseError: if an error occurs in parsing
"""
blocks = docscrape_google.split_google_docblocks(docstr)
example_blocks = []
for type, block in blocks:
if type.startswith('Example'):
example_blocks.append((type, block))
if type.startswith('Doctest'):
example_blocks.append((type, block))
if type.startswith('Script'):
example_blocks.append((type, block))
if type.startswith('Benchmark'):
example_blocks.append((type, block))
for num, (type, (docsrc, offset)) in enumerate(example_blocks):
# Add one because offset indicates the position of the block-label
# and the body of the block always starts on the next line.
label_lineno = lineno + offset
body_lineno = label_lineno + 1
example = doctest_example.DocTest(docsrc,
modpath,
callname,
num,
lineno=body_lineno,
fpath=fpath,
block_type=type)
if eager_parse:
# parse on the fly to be consistent with freeform?
example._parse()
yield example
def parse_google_docstr_examples(docstr,
callname=None,
modpath=None,
lineno=1,
fpath=None,
eager_parse=True):
"""
Parses Google-style doctests from a docstr and generates example objects
Args:
docstr (str): an extracted docstring
callname (str, default=None):
the name of the callable (e.g. function, class, or method)
that this docstring belongs to.
modpath (str | PathLike, default=None):
original module the docstring is from
lineno (int, default=1):
the line number (starting from 1) of the docstring. i.e. if you
were to go to this line number in the source file the starting
quotes of the docstr would be on this line.
fpath (str | PathLike, default=None):
the file that the docstring is from (if the file was not a module,
needed for backwards compatibility)
eager_parse (bool, default=True):
if True eagerly evaluate the parser inside the google example
blocks
Yields:
xdoctest.doctest_example.DocTest : doctest object
Raises:
xdoctest.exceptions.MalformedDocstr: if an error occurs in finding google blocks
xdoctest.exceptions.DoctestParseError: if an error occurs in parsing
"""
try:
blocks = docscrape_google.split_google_docblocks(docstr)
except exceptions.MalformedDocstr:
print('ERROR PARSING {} GOOGLE BLOCKS IN {} ON line {}'.format(
callname, modpath, lineno))
print('Did you forget to make a docstr with newlines raw?')
raise
example_blocks = []
example_tags = ('Example', 'Doctest', 'Script', 'Benchmark')
for type, block in blocks:
if type.startswith(example_tags):
example_blocks.append((type, block))
for num, (type, (docsrc, offset)) in enumerate(example_blocks):
# Add one because offset indicates the position of the block-label
# and the body of the block always starts on the next line.
label_lineno = lineno + offset
body_lineno = label_lineno + 1
example = doctest_example.DocTest(docsrc,
modpath,
callname,
num,
lineno=body_lineno,
fpath=fpath,
block_type=type)
if eager_parse:
# parse on the fly to be consistent with freeform?
example._parse()
yield example
def visit_func_def(self,
o: FuncDef,
is_abstract: bool = False,
is_overload: bool = False) -> None:
import ubelt as ub
if (self.is_private_name(o.name, o.fullname)
or self.is_not_in_all(o.name)
or (self.is_recorded_name(o.name) and not is_overload)):
self.clear_decorators()
return
if not self._indent and self._state not in (
EMPTY, FUNC) and not o.is_awaitable_coroutine:
self.add('\n')
if not self.is_top_level():
self_inits = find_self_initializers(o)
for init, value in self_inits:
if init in self.method_names:
# Can't have both an attribute and a method/property with the same name.
continue
init_code = self.get_init(init, value)
if init_code:
self.add(init_code)
# dump decorators, just before "def ..."
for s in self._decorators:
self.add(s)
self.clear_decorators()
self.add(
"%s%sdef %s(" %
(self._indent, 'async ' if o.is_coroutine else '', o.name))
self.record_name(o.name)
# import ubelt as ub
# if o.name == 'dzip':
# import xdev
# xdev.embed()
def _hack_for_info(info):
type_name = info['type']
if type_name is not None:
results = hacked_typing_info(type_name)
for typing_arg in results['typing_imports']:
self.add_typing_import(typing_arg)
for line in results['import_lines']:
self.add_import_line(line)
info['type'] = results['type_name']
name_to_parsed_docstr_info = {}
return_parsed_docstr_info = None
fullname = o.name
if getattr(self, '_IN_CLASS', None) is not None:
fullname = self._IN_CLASS + '.' + o.name
curr = ub.import_module_from_name(self.module)
# curr = sys.modules.get(self.module)
# print('o.name = {!r}'.format(o.name))
# print('fullname = {!r}'.format(fullname))
for part in fullname.split('.'):
# print('part = {!r}'.format(part))
# print('curr = {!r}'.format(curr))
curr = getattr(curr, part, None)
# print('curr = {!r}'.format(curr))
real_func = curr
# print('real_func = {!r}'.format(real_func))
# if o.name == 'dict_union':
# import xdev
# xdev.embed()
if real_func is not None and real_func.__doc__ is not None:
from mypy import fastparse
from xdoctest.docstr import docscrape_google
parsed_args = None
# parsed_ret = None
blocks = docscrape_google.split_google_docblocks(
real_func.__doc__)
# print('blocks = {}'.format(ub.repr2(blocks, nl=1)))
for key, block in blocks:
# print(f'key={key}')
lines = block[0]
if key == 'Returns':
# print(f'lines={lines}')
for retdict in docscrape_google.parse_google_retblock(
lines):
# print(f'retdict={retdict}')
_hack_for_info(retdict)
return_parsed_docstr_info = (key, retdict['type'])
if return_parsed_docstr_info is None:
print(
'Warning: return block for {} might be malformed'
.format(real_func))
if key == 'Yields':
for retdict in docscrape_google.parse_google_retblock(
lines):
_hack_for_info(retdict)
return_parsed_docstr_info = (key, retdict['type'])
if return_parsed_docstr_info is None:
print(
'Warning: return block for {} might be malformed'
.format(real_func))
if key == 'Args':
# hack for *args
lines = '\n'.join(
[line.lstrip('*') for line in lines.split('\n')])
# print('lines = {!r}'.format(lines))
parsed_args = list(
docscrape_google.parse_google_argblock(lines))
for info in parsed_args:
_hack_for_info(info)
name = info['name'].replace('*', '')
name_to_parsed_docstr_info[name] = info
parsed_rets = list(
docscrape_google.parse_google_returns(real_func.__doc__))
ret_infos = []
for info in parsed_rets:
try:
got = fastparse.parse_type_string(
info['type'], 'Any', 0, 0)
ret_infos.append(got)
except Exception:
pass
# print('o = {!r}'.format(o))
# print('o.arguments = {!r}'.format(o.arguments))
args: List[str] = []
for i, arg_ in enumerate(o.arguments):
var = arg_.variable
kind = arg_.kind
name = var.name
annotated_type = (o.unanalyzed_type.arg_types[i] if isinstance(
o.unanalyzed_type, CallableType) else None)
if annotated_type is None:
if name in name_to_parsed_docstr_info:
name = name.replace('*', '')
doc_type_str = name_to_parsed_docstr_info[name].get(
'type', None)
if doc_type_str is not None:
doc_type_str = doc_type_str.split(', default')[0]
# annotated_type = doc_type_str
# import mypy.types as mypy_types
from mypy import fastparse
# globals_ = {**mypy_types.__dict__}
try:
# # got = mypy_types.deserialize_type(doc_type_str)
# got = eval(doc_type_str, globals_)
# got = mypy_types.get_proper_type(got)
# got = mypy_types.Iterable
got = fastparse.parse_type_string(
doc_type_str, 'Any', 0, 0)
except Exception as ex:
print('ex = {!r}'.format(ex))
print('Failed to parse doc_type_str = {!r}'.
format(doc_type_str))
else:
annotated_type = got
# print('PARSED: annotated_type = {!r}'.format(annotated_type))
# print('annotated_type = {!r}'.format(annotated_type))
# I think the name check is incorrect: there are libraries which
# name their 0th argument other than self/cls
is_self_arg = i == 0 and name == 'self'
is_cls_arg = i == 0 and name == 'cls'
annotation = ""
if annotated_type and not is_self_arg and not is_cls_arg:
# Luckily, an argument explicitly annotated with "Any" has
# type "UnboundType" and will not match.
if not isinstance(get_proper_type(annotated_type),
AnyType):
annotation = ": {}".format(
self.print_annotation(annotated_type))
# xdev change, where we try to port the defaults over to the stubs
# as well (otherwise they dont show up in the function help text)
XDEV_KEEP_SOME_DEFAULTS = True
if arg_.initializer:
if kind.is_named() and not any(
arg.startswith('*') for arg in args):
args.append('*')
if not annotation:
typename = self.get_str_type_of_node(
arg_.initializer, True, False)
if typename == '':
if XDEV_KEEP_SOME_DEFAULTS:
# TODO
annotation = '=...'
else:
annotation = '=...'
else:
annotation = ': {} = ...'.format(typename)
else:
if XDEV_KEEP_SOME_DEFAULTS:
import mypy
# arg_.initializer.is_special_form
if isinstance(
arg_.initializer,
(mypy.nodes.IntExpr, mypy.nodes.FloatExpr)):
annotation += '={!r}'.format(
arg_.initializer.value)
elif isinstance(arg_.initializer,
mypy.nodes.StrExpr):
annotation += '={!r}'.format(
arg_.initializer.value)
elif isinstance(arg_.initializer,
mypy.nodes.NameExpr):
annotation += '={}'.format(
arg_.initializer.name)
else:
# fallback, unhandled default
print(
f'todo: Unhandled arg_.initializer={type(arg_.initializer)}'
)
annotation += '=...'
else:
annotation += ' = ...'
arg = name + annotation
elif kind == ARG_STAR:
arg = '*%s%s' % (name, annotation)
elif kind == ARG_STAR2:
arg = '**%s%s' % (name, annotation)
else:
arg = name + annotation
args.append(arg)
retname = None
if o.name != '__init__' and isinstance(o.unanalyzed_type,
CallableType):
if isinstance(get_proper_type(o.unanalyzed_type.ret_type),
AnyType):
# Luckily, a return type explicitly annotated with "Any" has
# type "UnboundType" and will enter the else branch.
retname = None # implicit Any
else:
retname = self.print_annotation(o.unanalyzed_type.ret_type)
elif isinstance(o, FuncDef) and (o.is_abstract or o.name
in METHODS_WITH_RETURN_VALUE):
# Always assume abstract methods return Any unless explicitly annotated. Also
# some dunder methods should not have a None return type.
retname = None # implicit Any
elif has_yield_expression(o):
self.add_abc_import('Generator')
yield_name = 'None'
send_name = 'None'
return_name = 'None'
for expr, in_assignment in all_yield_expressions(o):
if expr.expr is not None and not self.is_none_expr(
expr.expr):
self.add_typing_import('Any')
yield_name = 'Any'
if in_assignment:
self.add_typing_import('Any')
send_name = 'Any'
if has_return_statement(o):
self.add_typing_import('Any')
return_name = 'Any'
generator_name = self.typing_name('Generator')
if return_parsed_docstr_info is not None:
print(
f'return_parsed_docstr_info={return_parsed_docstr_info}'
)
yield_name = return_parsed_docstr_info[1]
retname = f'{generator_name}[{yield_name}, {send_name}, {return_name}]'
# print('o.name = {}'.format(ub.repr2(o.name, nl=1)))
# print('retname = {!r}'.format(retname))
# print('retfield = {!r}'.format(retfield))
elif not has_return_statement(o) and not is_abstract:
retname = 'None'
if retname is None:
if return_parsed_docstr_info is not None:
retname = return_parsed_docstr_info[1]
retfield = ''
if retname is not None:
retfield = ' -> ' + retname
self.add(', '.join(args))
self.add("){}: ...\n".format(retfield))
self._state = FUNC
def auto_argparse(func):
"""
Transform a function with a Google Style Docstring into an
`argparse.ArgumentParser`.
TODO:
- [ ] Handle booleans consistently, allow --flag=True and --flag=False
Args:
func (callable): function with kwargs
"""
from xdoctest.docstr import docscrape_google as scrape
import ast
import argparse
import ubelt as ub
import inspect
spec = inspect.getargspec(func)
# Parse default values from the function dynamically
try:
import xinspect
kwdefaults = xinspect.get_func_kwargs(func)
except Exception as ex:
raise
kwdefaults = dict(zip(spec.args[-len(spec.defaults):], spec.defaults))
# Parse help and description information from a google-style docstring
docstr = func.__doc__
description = scrape.split_google_docblocks(docstr)[0][1][0].strip()
# TODO: allow scraping from the kwargs block as well
google_args = {
argdict['name']: argdict
for argdict in scrape.parse_google_args(docstr)
}
argnames = ub.oset(spec.args) | ub.oset(kwdefaults)
argnames = (ub.oset(google_args) & argnames) | argnames
# DEBUG = 1
# if DEBUG:
# print(ub.repr2(google_args))
# Create the argument parser and register each argument
parser = argparse.ArgumentParser(
description=description,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
for arg in argnames:
argkw = {}
if arg in kwdefaults:
argkw['default'] = kwdefaults[arg]
if arg in google_args:
garg = google_args[arg]
argkw['help'] = garg['desc']
# print('-----')
# print('argkw = {}, {}'.format(arg, ub.repr2(argkw)))
try:
if garg['type'] == 'PathLike':
argkw['type'] = str
elif garg['type'] == 'bool':
def _parse_bool(s):
return bool(ast.literal_eval(s))
argkw['type'] = _parse_bool
else:
argkw['type'] = eval(garg['type'], {})
# literal_eval doesnt handle types
# argkw['type'] = ast.literal_eval(garg['type'])
except Exception as ex:
# print('{}, ex = {!r}'.format(arg, ex))
pass
# print('-----')
# print('argkw = {}, {!r}'.format(arg, argkw))
parser.add_argument('--' + arg, **argkw)
return parser
