def test_extract_default_with_parens(self) -> None:
""" Tests that `extract_default` works when wrapped in parentheses """
sample = "learning rate (default: 1)"
self.assertTupleEqual(
extract_default(sample, emit_default_doc=True),
(sample, 1),
)
sample = (
"tolerance_change (float): termination tolerance on function\n"
" value/parameter changes (default: 1e-9).")
self.assertTupleEqual(
extract_default(sample, emit_default_doc=True),
(sample, 1e-9),
)
def test_extract_default_with_float(self) -> None:
""" Tests that `extract_default` works when there is a `.` in the default referring to a decimal place """
sample = "learning rate. Defaults to 0.001."
self.assertTupleEqual(
extract_default(sample, emit_default_doc=False),
("learning rate.", 0.001),
)
def test_extract_default_with_int(self) -> None:
""" Tests that `extract_default` works for an integer default """
sample = "learning rate. Defaults to 0001."
self.assertTupleEqual(
extract_default(sample, emit_default_doc=False),
("learning rate.", 1),
)
def test_extract_default_with_dot(self) -> None:
""" Tests that `extract_default` works when there is a `.` in the default """
sample = "This. defaults to (np.empty(0), np.empty(0))"
self.assertTupleEqual(
extract_default(sample, emit_default_doc=False),
("This.", "(np.empty(0), np.empty(0))"),
)
def sqlalchemy_table(call_or_name):
"""
Parse out a `sqlalchemy.Table`, or a `name = sqlalchemy.Table`, into the IR
:param call_or_name: The call to `sqlalchemy.Table` or an assignment followed by the call
:type call_or_name: ```Union[AnnAssign, Assign, Call]```
:returns: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:rtype: ```dict```
"""
if isinstance(call_or_name, Assign):
name, call_or_name = call_or_name.targets[0].id, call_or_name.value
elif isinstance(call_or_name, AnnAssign):
name, call_or_name = call_or_name.target.id, call_or_name.value
else:
if not isinstance(call_or_name, Call):
call_or_name = get_value(call_or_name)
name = get_value(call_or_name.args[0])
# Binding should be same name as table… I guess?
assert_equal(get_value(call_or_name.args[0]), name)
comment = next(
map(
get_value,
map(get_value,
filter(lambda kw: kw.arg == "comment", call_or_name.keywords)),
),
None,
)
intermediate_repr = ({
"type": None,
"doc": "",
"params": OrderedDict()
} if comment is None else docstring(comment))
intermediate_repr["name"] = name
assert isinstance(call_or_name, Call)
assert_equal(call_or_name.func.id.rpartition(".")[2], "Table")
assert len(call_or_name.args) > 2
merge_ir = {
"params": OrderedDict(map(column_call_to_param,
call_or_name.args[2:])),
"returns": None,
}
ir_merge(target=intermediate_repr, other=merge_ir)
if intermediate_repr["returns"] and intermediate_repr["returns"].get(
"return_type", {}).get("doc"):
intermediate_repr["returns"]["return_type"]["doc"] = extract_default(
intermediate_repr["returns"]["return_type"]["doc"],
emit_default_doc=False)[0]
return intermediate_repr
def test_extract_default_middle(self) -> None:
""" Tests that `extract_default` produces the expected output """
sample = "Why would you. Have this defaults to something. In the middle?"
self.assertTupleEqual(extract_default(sample), (sample, "something"))
self.assertTupleEqual(
extract_default(sample,
rstrip_default=False,
emit_default_doc=False),
("Why would you. Have this. In the middle?", "something"),
)
self.assertTupleEqual(
extract_default(sample,
rstrip_default=True,
emit_default_doc=False),
("Why would you. Have thisIn the middle?", "something"),
)
def test_extract_default_with_ast_default(self) -> None:
""" Tests that `extract_default` works when default parses to an AST type """
sample = ("maximal number of function evaluations per optimization\n"
" step (default: max_iter * 1.25).")
self.assertTupleEqual(
extract_default(sample, emit_default_doc=True),
(sample, "max_iter * 1.25"),
)
def _param2docstring_param(
param,
docstring_format="rest",
emit_default_doc=True,
indent_level=1,
emit_types=False,
):
"""
Converts param dict from intermediate_repr to the right string representation
:param param: Name, dict with keys: 'typ', 'doc', 'default'
:type param: ```Tuple[str, dict]```
:param docstring_format: Format of docstring
:type docstring_format: ```Literal['rest', 'numpy', 'google']```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:param indent_level: indentation level whence: 0=no_tabs, 1=one tab; 2=two tabs
:type indent_level: ```int```
:param emit_types: whether to show `:type` lines
:type emit_types: ```bool```
"""
# if not isinstance(param, tuple):
assert isinstance(param, tuple), "Expected 'tuple' got `{!r}`".format(
type(param).__name__)
assert docstring_format == "rest", docstring_format
name, _param = param
del param
if "doc" in _param:
doc, default = extract_default(_param["doc"],
emit_default_doc=emit_default_doc)
if default is not None:
_param["default"] = default
_param["typ"] = ("**{name}".format(
name=name) if _param.get("typ") == "dict"
and name.endswith("kwargs") else _param.get("typ"))
return "".join(
filter(
None,
(
(lambda name_param: "{tab}:param {name}: {doc}".format(
tab=tab * indent_level,
name=name_param[0],
doc=name_param[1].get("doc"),
))(set_default_doc(
(name, _param), emit_default_doc=emit_default_doc)),
None if _param["typ"] is None or not emit_types else
"\n{tab}:type {name}: ```{_param[typ]}```".format(
tab=tab * indent_level, name=name, _param=_param),
),
))
def test_extract_default_with_bool(self) -> None:
""" Tests that `extract_default` works for an integer default """
sample = (
"Boolean. Whether to apply AMSGrad variant of this algorithm from"
'the paper "On the Convergence of Adam and beyond". Defaults to `True`.'
)
self.assertTupleEqual(
extract_default(sample, emit_default_doc=True),
(sample, True),
)
def test_extract_default_with_many_parens(self) -> None:
""" Tests that `extract_default` works when default parses to an AST type """
sample = (
"betas (Tuple[float, float], optional): coefficients used for computing\n"
" running averages of gradient and its square (default: (0.9, 0.999))"
)
default = "(0.9, 0.999)"
self.assertTupleEqual(
extract_default(sample, emit_default_doc=True),
(sample, default),
)
self.assertTupleEqual(
extract_default(sample, emit_default_doc=False),
(
"betas (Tuple[float, float], optional): coefficients used for computing\n"
" running averages of gradient and its square",
default,
),
)
def _parse_return(e, intermediate_repr, function_def, emit_default_doc):
"""
Parse return into a param dict
:param e: Return AST node
:type e: Return
:param intermediate_repr: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:type intermediate_repr: ```dict```
:param function_def: AST node for function definition
:type function_def: ```FunctionDef```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:returns: Name, dict with keys: 'typ', 'doc', 'default'
:rtype: ```Tuple[str, dict]```
"""
assert isinstance(e, Return)
return set_default_doc(
(
"return_type",
{
"doc":
extract_default(
next(
line.partition(",")[2].lstrip() for line in get_value(
function_def.body[0].value).split("\n")
if line.lstrip().startswith(":return")),
emit_default_doc=emit_default_doc,
)[0],
"default":
to_code(e.value.elts[1]).rstrip("\n"),
"typ":
to_code(
get_value(
ast.parse(
intermediate_repr["returns"]["return_type"]
["typ"]).body[0].value.slice).elts[1]).rstrip()
# 'Tuple[ArgumentParser, {typ}]'.format(typ=intermediate_repr['returns']['typ'])
},
),
emit_default_doc=emit_default_doc,
)
def interpolate_defaults(param,
default_search_announce=None,
require_default=False,
emit_default_doc=True):
"""
Correctly set the 'default' and 'doc' parameters
:param param: Name, dict with keys: 'typ', 'doc', 'default'
:type param: ```Tuple[str, dict]```
:param default_search_announce: Default text(s) to look for. If None, uses default specified in default_utils.
:type default_search_announce: ```Optional[Union[str, Iterable[str]]]```
:param require_default: Whether a default is required, if not found in doc, infer the proper default from type
:type require_default: ```bool```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:returns: Name, dict with keys: 'typ', 'doc', 'default'
:rtype: ```Tuple[str, dict]```
"""
name, _param = param
del param
if "doc" in _param:
doc, default = extract_default(
_param["doc"],
typ=_param.get("typ"),
default_search_announce=default_search_announce,
emit_default_doc=emit_default_doc,
)
_param["doc"] = doc
if default is not None:
_param["default"] = unquote(default)
if require_default and _param.get("default") is None:
# if (
# "typ" in _param
# and _param["typ"] not in frozenset(("Any", "object"))
# and not _param["typ"].startswith("Optional")
# ):
# _param["typ"] = "Optional[{}]".format(_param["typ"])
_param["default"] = (simple_types[_param["typ"]] if _param.get(
"typ", memoryview) in simple_types else NoneStr)
return name, _param
def param2argparse_param(param, emit_default_doc=True):
"""
Converts a param to an Expr `argparse.add_argument` call
:param param: Name, dict with keys: 'typ', 'doc', 'default'
:type param: ```Tuple[str, Dict[str, Any]]```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:return: `argparse.add_argument` call—with arguments—as an AST node
:rtype: ```Expr```
"""
name, _param = param
del param
typ, choices, required, action = (
"str",
None,
_param.get("default") is not None,
None,
)
_param.setdefault("typ", "Any")
action, choices, required, typ, (name, _param) = _resolve_arg(
action, choices, (name, _param), required, typ
)
# is_kwarg = param[0].endswith("kwargs")
_param.setdefault("doc", "")
doc, _default = extract_default(_param["doc"], emit_default_doc=emit_default_doc)
_action, default, _required, typ = infer_type_and_default(
_param.get("default", _default),
typ,
required=required
# _default, _param, action, required, typ#
)
if default is None and _param.get("default") == NoneStr:
default, required = NoneStr, False
if _action:
action = _action
if typ == "pickle.loads":
required = False
# elif _required is False and required is True:
# required = _required
# if _param.get("default") == NoneStr:
# required = False
# if typ in frozenset(("Any", "object")):
# required, typ = False, "str"
# if param[1].get("typ") and typ == "str":
# pass
# if is_kwarg and required:
# required = False
return Expr(
Call(
args=[set_value("--{name}".format(name=name))],
func=Attribute(
Name("argument_parser", Load()),
"add_argument",
Load(),
),
keywords=list(
filter(
None,
(
typ
if typ is None
else keyword(
arg="type",
value=FALLBACK_ARGPARSE_TYP
if typ == "globals().__getitem__"
else Name(typ, Load()),
identifier=None,
),
choices
if choices is None
else keyword(
arg="choices",
value=Tuple(
ctx=Load(),
elts=list(map(set_value, choices)),
expr=None,
),
identifier=None,
),
action
if action is None
else keyword(
arg="action",
value=set_value(action),
identifier=None,
),
keyword(
arg="help",
value=set_value(doc),
identifier=None,
)
if doc
else None,
keyword(
arg="required",
value=set_value(True),
identifier=None,
)
if required is True
else None,
default
if default is None
else keyword(
arg="default",
value=set_value(None if default == NoneStr else default),
identifier=None,
),
),
)
),
expr=None,
expr_func=None,
)
)
def test_extract_default(self) -> None:
""" Tests that `extract_default` produces the expected output """
sample = "This defaults to foo."
self.assertTupleEqual(extract_default(sample), (sample, "foo"))
self.assertTupleEqual(extract_default(sample, emit_default_doc=False),
("This", "foo"))
def _param2docstring_param(
param,
docstring_format="rest",
emit_default_doc=True,
indent_level=1,
emit_types=False,
):
"""
Converts param dict from intermediate_repr to the right string representation
:param param: Name, dict with keys: 'typ', 'doc', 'default'
:type param: ```Tuple[str, dict]```
:param docstring_format: Format of docstring
:type docstring_format: ```Literal['rest', 'numpydoc', 'google']```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:param indent_level: indentation level whence: 0=no_tabs, 1=one tab; 2=two tabs
:type indent_level: ```int```
:param emit_types: whether to show `:type` lines
:type emit_types: ```bool```
"""
# if not isinstance(param, tuple):
assert isinstance(param, tuple), "Expected 'tuple' got `{!r}`".format(
type(param).__name__)
name, _param = param
del param
if "doc" in _param:
doc, default = extract_default(_param["doc"],
emit_default_doc=emit_default_doc)
if default is not None:
_param["default"] = default
_sep = abs(indent_level) * tab
def _joiner(__param, param_type):
"""
Internal function to join new lines
:param __param: The `:param`
:type __param: ```Optional[str]```
:param param_type: The `:type`
:type param_type: ```Optional[str]```
:returns: Newline joined string
:rtype: ```str```
"""
if param_type is None and __param is not None:
return "{}\n{}".format(_fill(__param), _sep)
elif __param is None:
return __param
return "".join((
_fill(__param.replace("\n", "\n{sep}".format(sep=_sep))),
"\n",
_sep,
_fill(param_type.replace("\n", "\n{sep}".format(sep=_sep))),
"\n",
_sep,
))
return _joiner(
(emit_param_str(
(
name,
update_d(
_param,
doc=multiline(
indent_all_but_first(
set_default_doc(
(name, _param),
emit_default_doc=emit_default_doc,
)[1]["doc"],
indent_level=indent_level - 1,
),
quote_with=("", ""),
),
),
),
emit_type=False,
emit_default_doc=emit_default_doc,
style=docstring_format,
word_wrap=word_wrap,
)) if _param.get("doc") else None,
(None if _param.get("typ") is None or not emit_types else
emit_param_str(
(name, _param),
emit_doc=False,
emit_default_doc=emit_default_doc,
style=docstring_format,
word_wrap=word_wrap,
)),
)
def parse_out_param(expr, require_default=False, emit_default_doc=True):
"""
Turns the class_def repr of '--dataset_name', type=str, help='name of dataset.', required=True, default='mnist'
into
Tuple[Literal['dataset_name'], {"typ": Literal["str"], "doc": Literal["name of dataset."],
"default": Literal["mnist"]}]
:param expr: Expr
:type expr: ```Expr```
:param require_default: Whether a default is required, if not found in doc, infer the proper default from type
:type require_default: ```bool```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:returns: Name, dict with keys: 'typ', 'doc', 'default'
:rtype: ```Tuple[str, dict]```
"""
required = get_value(
get_value(
next(
(keyword for keyword in expr.value.keywords
if keyword.arg == "required"),
set_value(False),
)))
typ = next(
(_handle_value(get_value(key_word))
for key_word in expr.value.keywords if key_word.arg == "type"),
"str",
)
name = get_value(expr.value.args[0])[len("--"):]
default = next(
(get_value(key_word.value)
for key_word in expr.value.keywords if key_word.arg == "default"),
None,
)
doc = (lambda help_: help_ if help_ is None else
(help_ if default is None or emit_default_doc is False or
(hasattr(default, "__len__") and len(default) == 0
) or "defaults to" in help_ or "Defaults to" in help_ else
"{help} Defaults to {default}".format(
help=help_ if help_.endswith(".") else "{}.".format(help_),
default=default,
)))(next(
(get_value(key_word.value) for key_word in expr.value.keywords
if key_word.arg == "help" and key_word.value),
None,
))
if default is None:
doc, default = extract_default(doc, emit_default_doc=emit_default_doc)
if default is None:
if required:
# if name.endswith("kwargs"):
# default = NoneStr
# else:
default = simple_types[typ] if typ in simple_types else NoneStr
elif require_default or typ.startswith("Optional"):
default = NoneStr
# nargs = next(
# (
# get_value(key_word.value)
# for key_word in expr.value.keywords
# if key_word.arg == "nargs"
# ),
# None,
# )
action = next(
(get_value(key_word.value)
for key_word in expr.value.keywords if key_word.arg == "action"),
None,
)
typ = next(
(_handle_keyword(keyword, typ)
for keyword in expr.value.keywords if keyword.arg == "choices"),
typ,
)
if action == "append":
typ = "List[{typ}]".format(typ=typ)
if not required and "Optional" not in typ:
typ = "Optional[{typ}]".format(typ=typ)
# if "str" in typ or "Literal" in typ and (typ.count("'") > 1 or typ.count('"') > 1):
# default = quote(default)
return name, dict(doc=doc,
typ=typ,
**({} if default is None else {
"default": default
}))
