def test_extract_default_with_parens(self) -> None:
"""Tests that `extract_default` works when wrapped in parentheses"""
sample = "learning rate (default: 1)"
self.assertTupleEqual(
extract_default(sample, emit_default_doc=True),
(sample, 1),
)
sample = (
"tolerance_change (float): termination tolerance on function\n"
" value/parameter changes (default: 1e-9).")
self.assertTupleEqual(
extract_default(sample, emit_default_doc=True),
(sample, 1e-9),
)
def sqlalchemy_table(call_or_name):
"""
Parse out a `sqlalchemy.Table`, or a `name = sqlalchemy.Table`, into the IR
:param call_or_name: The call to `sqlalchemy.Table` or an assignment followed by the call
:type call_or_name: ```Union[AnnAssign, Assign, Call]```
:returns: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:rtype: ```dict```
"""
if isinstance(call_or_name, Assign):
name, call_or_name = call_or_name.targets[0].id, call_or_name.value
elif isinstance(call_or_name, AnnAssign):
name, call_or_name = call_or_name.target.id, call_or_name.value
else:
if not isinstance(call_or_name, Call):
call_or_name = get_value(call_or_name)
name = get_value(call_or_name.args[0])
# Binding should be same name as table… I guess?
assert_equal(get_value(call_or_name.args[0]), name)
comment = next(
map(
get_value,
map(
get_value, filter(lambda kw: kw.arg == "comment", call_or_name.keywords)
),
),
None,
)
intermediate_repr = (
{"type": None, "doc": "", "params": OrderedDict()}
if comment is None
else docstring(comment)
)
intermediate_repr["name"] = name
assert isinstance(call_or_name, Call)
assert_equal(call_or_name.func.id.rpartition(".")[2], "Table")
assert len(call_or_name.args) > 2
merge_ir = {
"params": OrderedDict(map(column_call_to_param, call_or_name.args[2:])),
"returns": None,
}
ir_merge(target=intermediate_repr, other=merge_ir)
if intermediate_repr["returns"] and intermediate_repr["returns"].get(
"return_type", {}
).get("doc"):
intermediate_repr["returns"]["return_type"]["doc"] = extract_default(
intermediate_repr["returns"]["return_type"]["doc"], emit_default_doc=False
)[0]
return intermediate_repr
def test_extract_default_with_int(self) -> None:
"""Tests that `extract_default` works for an integer default"""
sample = "learning rate. Defaults to 0001."
self.assertTupleEqual(
extract_default(sample, emit_default_doc=False),
("learning rate.", 1),
)
def test_extract_default_with_float(self) -> None:
"""Tests that `extract_default` works when there is a `.` in the default referring to a decimal place"""
sample = "learning rate. Defaults to 0.001."
self.assertTupleEqual(
extract_default(sample, emit_default_doc=False),
("learning rate.", 0.001),
)
def test_extract_default_with_dot(self) -> None:
"""Tests that `extract_default` works when there is a `.` in the default"""
sample = "This. defaults to (np.empty(0), np.empty(0))"
self.assertTupleEqual(
extract_default(sample, emit_default_doc=False),
("This.", "(np.empty(0), np.empty(0))"),
)
def test_extract_default_with_ast_default(self) -> None:
"""Tests that `extract_default` works when default parses to an AST type"""
sample = ("maximal number of function evaluations per optimization\n"
" step (default: max_iter * 1.25).")
self.assertTupleEqual(
extract_default(sample, emit_default_doc=True),
(sample, "max_iter * 1.25"),
)
def test_extract_default_middle(self) -> None:
"""Tests that `extract_default` produces the expected output"""
sample = "Why would you. Have this defaults to something. In the middle?"
default = "something"
self.assertTupleEqual(extract_default(sample), (sample, default))
self.assertTupleEqual(
extract_default(sample,
rstrip_default=False,
emit_default_doc=False),
("Why would you. Have this. In the middle?", default),
)
self.assertTupleEqual(
extract_default(sample,
rstrip_default=True,
emit_default_doc=False),
("Why would you. Have thisIn the middle?", default),
)
def test_extract_default_with_many_parens(self) -> None:
"""Tests that `extract_default` works when default parses to an AST type"""
sample = (
"betas (Tuple[float, float], optional): coefficients used for computing\n"
" running averages of gradient and its square (default: (0.9, 0.999))"
)
default = "(0.9, 0.999)"
self.assertTupleEqual(
extract_default(sample, emit_default_doc=True),
(sample, default),
)
self.assertTupleEqual(
extract_default(sample, emit_default_doc=False),
(
"betas (Tuple[float, float], optional): coefficients used for computing\n"
" running averages of gradient and its square",
default,
),
)
def test_extract_default_with_bool(self) -> None:
"""Tests that `extract_default` works for an integer default"""
sample = (
"Boolean. Whether to apply AMSGrad variant of this algorithm from"
'the paper "On the Convergence of Adam and beyond". Defaults to `True`.'
)
self.assertTupleEqual(
extract_default(sample, emit_default_doc=True),
(sample, True),
)
def _parse_return(e, intermediate_repr, function_def, emit_default_doc):
"""
Parse return into a param dict
:param e: Return AST node
:type e: Return
:param intermediate_repr: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:type intermediate_repr: ```dict```
:param function_def: AST node for function definition
:type function_def: ```FunctionDef```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:returns: Name, dict with keys: 'typ', 'doc', 'default'
:rtype: ```Tuple[str, dict]```
"""
assert isinstance(e, Return)
return set_default_doc(
(
"return_type",
{
"doc":
extract_default(
next(
line.partition(",")[2].lstrip() for line in get_value(
function_def.body[0].value).split("\n")
if line.lstrip().startswith(":return")),
emit_default_doc=emit_default_doc,
)[0],
"default":
to_code(e.value.elts[1]).rstrip("\n"),
"typ":
to_code(
get_value(
ast.parse(
intermediate_repr["returns"]["return_type"]
["typ"]).body[0].value.slice).elts[1]).rstrip()
# 'Tuple[ArgumentParser, {typ}]'.format(typ=intermediate_repr['returns']['typ'])
},
),
emit_default_doc=emit_default_doc,
)
def interpolate_defaults(param,
default_search_announce=None,
require_default=False,
emit_default_doc=True):
"""
Correctly set the 'default' and 'doc' parameters
:param param: Name, dict with keys: 'typ', 'doc', 'default'
:type param: ```Tuple[str, dict]```
:param default_search_announce: Default text(s) to look for. If None, uses default specified in default_utils.
:type default_search_announce: ```Optional[Union[str, Iterable[str]]]```
:param require_default: Whether a default is required, if not found in doc, infer the proper default from type
:type require_default: ```bool```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:returns: Name, dict with keys: 'typ', 'doc', 'default'
:rtype: ```Tuple[str, dict]```
"""
name, _param = param
del param
if "doc" in _param:
doc, default = extract_default(
_param["doc"],
typ=_param.get("typ"),
default_search_announce=default_search_announce,
emit_default_doc=emit_default_doc,
)
_param["doc"] = doc
if default is not None:
_param["default"] = unquote(default)
if require_default and _param.get("default") is None:
# if (
# "typ" in _param
# and _param["typ"] not in frozenset(("Any", "object"))
# and not _param["typ"].startswith("Optional")
# ):
# _param["typ"] = "Optional[{}]".format(_param["typ"])
_param["default"] = (simple_types[_param["typ"]] if _param.get(
"typ", memoryview) in simple_types else NoneStr)
return name, _param
def parse_out_param(expr, require_default=False, emit_default_doc=True):
"""
Turns the class_def repr of '--dataset_name', type=str, help='name of dataset.', required=True, default='mnist'
into
Tuple[Literal['dataset_name'], {"typ": Literal["str"], "doc": Literal["name of dataset."],
"default": Literal["mnist"]}]
:param expr: Expr
:type expr: ```Expr```
:param require_default: Whether a default is required, if not found in doc, infer the proper default from type
:type require_default: ```bool```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:returns: Name, dict with keys: 'typ', 'doc', 'default'
:rtype: ```Tuple[str, dict]```
"""
required = get_value(
get_value(
next(
(keyword for keyword in expr.value.keywords
if keyword.arg == "required"),
set_value(False),
)))
typ = next(
(_handle_value(get_value(key_word))
for key_word in expr.value.keywords if key_word.arg == "type"),
"str",
)
name = get_value(expr.value.args[0])[len("--"):]
default = next(
(get_value(key_word.value)
for key_word in expr.value.keywords if key_word.arg == "default"),
None,
)
doc = (lambda help_: help_ if help_ is None else
(help_ if default is None or emit_default_doc is False or
(hasattr(default, "__len__") and len(default) == 0
) or "defaults to" in help_ or "Defaults to" in help_ else
"{help} Defaults to {default}".format(
help=help_ if help_.endswith(".") else "{}.".format(help_),
default=default,
)))(next(
(get_value(key_word.value) for key_word in expr.value.keywords
if key_word.arg == "help" and key_word.value),
None,
))
if default is None:
doc, default = extract_default(doc, emit_default_doc=emit_default_doc)
if default is None:
if required:
# if name.endswith("kwargs"):
# default = NoneStr
# else:
default = simple_types[typ] if typ in simple_types else NoneStr
elif require_default or typ.startswith("Optional"):
default = NoneStr
action = next(
(get_value(key_word.value)
for key_word in expr.value.keywords if key_word.arg == "action"),
None,
)
typ = next(
(_handle_keyword(keyword, typ)
for keyword in expr.value.keywords if keyword.arg == "choices"),
typ,
)
if action == "append":
typ = "List[{typ}]".format(typ=typ)
if not required and "Optional" not in typ:
typ = "Optional[{typ}]".format(typ=typ)
return name, dict(doc=doc,
typ=typ,
**({} if default is None else {
"default": default
}))
def test_extract_default(self) -> None:
"""Tests that `extract_default` produces the expected output"""
sample = "This defaults to foo."
self.assertTupleEqual(extract_default(sample), (sample, "foo"))
self.assertTupleEqual(extract_default(sample, emit_default_doc=False),
("This", "foo"))
def param2argparse_param(param, word_wrap=True, emit_default_doc=True):
"""
Converts a param to an Expr `argparse.add_argument` call
:param param: Name, dict with keys: 'typ', 'doc', 'default'
:type param: ```Tuple[str, Dict[str, Any]]```
:param word_wrap: Whether to word-wrap. Set `DOCTRANS_LINE_LENGTH` to configure length.
:type word_wrap: ```bool```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:returns: `argparse.add_argument` call—with arguments—as an AST node
:rtype: ```Expr```
"""
name, _param = param
del param
typ, choices, required, action = (
"str",
None,
_param.get("default") is not None,
None,
)
_param.setdefault("typ", "Any")
action, choices, required, typ, (name, _param) = _resolve_arg(
action, choices, (name, _param), required, typ
)
# is_kwarg = param[0].endswith("kwargs")
_param.setdefault("doc", "")
doc, _default = extract_default(_param["doc"], emit_default_doc=emit_default_doc)
_action, default, _required, _typ = infer_type_and_default(
action,
_param.get("default", _default),
typ,
required=required
# _default, _param, action, required, typ#
)
if default is None and _param.get("default") == NoneStr:
required = False
if _action:
action = _action
if _typ is not None:
typ = _typ
if typ == "pickle.loads":
required = False
elif typ == "str" and action is None:
typ = None # Because `str` is default anyway
return Expr(
Call(
args=[set_value("--{name}".format(name=name))],
func=Attribute(
Name("argument_parser", Load()),
"add_argument",
Load(),
),
keywords=list(
filter(
None,
(
typ
if typ is None
else keyword(
arg="type",
value=FALLBACK_ARGPARSE_TYP
if typ == "globals().__getitem__"
else Name(typ, Load()),
identifier=None,
),
choices
if choices is None
else keyword(
arg="choices",
value=Tuple(
ctx=Load(),
elts=list(map(set_value, choices)),
expr=None,
),
identifier=None,
),
action
if action is None
else keyword(
arg="action",
value=set_value(action),
identifier=None,
),
keyword(
arg="help",
value=set_value((fill if word_wrap else identity)(doc)),
identifier=None,
)
if doc
else None,
keyword(
arg="required",
value=set_value(True),
identifier=None,
)
if required is True
else None,
default
if default is None
else keyword(
arg="default",
value=set_value(default),
identifier=None,
),
),
)
),
expr=None,
expr_func=None,
)
)