def test_get_function_type(self) -> None:
""" Test get_function_type returns the right type """
self.assertEqual(
get_function_type(
FunctionDef(
args=arguments(
args=[set_arg("something else")],
arg=None,
),
arguments_args=None,
identifier_name=None,
stmt=None,
)
),
"static",
)
self.assertEqual(
get_function_type(
FunctionDef(
args=arguments(args=[], arg=None),
arguments_args=None,
identifier_name=None,
stmt=None,
)
),
"static",
)
self.assertEqual(
get_function_type(
FunctionDef(
args=arguments(
args=[set_arg("self")],
arg=None,
),
arguments_args=None,
identifier_name=None,
stmt=None,
)
),
"self",
)
self.assertEqual(
get_function_type(
FunctionDef(
args=arguments(
args=[set_arg("cls")],
arg=None,
),
arguments_args=None,
identifier_name=None,
stmt=None,
)
),
"cls",
)
def _make_call_meth(body, return_type, param_names):
"""
Construct a `__call__` method from the provided `body`
:param body: The body, probably from a FunctionDef.body
:type body: ```List[AST]```
:param return_type: The return type of the parent symbol (probably class). Used to fill in `__call__` return.
:type return_type: ```Optional[str]```
:param param_names: Container of AST `id`s to match for rename
:type param_names: ```Optional[Iterator[str]]```
:return: Internal function for `__call__`
:rtype: ```FunctionDef```
"""
body_len = len(body)
return_ = (ast.fix_missing_locations(
RewriteName(param_names).visit(
Return(get_value(ast.parse(return_type[3:-3]).body[0]),
expr=None))) if return_type is not None
and len(return_type) > 6 and return_type.startswith("```")
and return_type.endswith("```") else None)
if body_len:
if isinstance(body[0], Expr):
doc_str = get_value(body[0].value)
if isinstance(doc_str, str) and body_len > 0:
body = (body[1:] if body_len > 1 else ([
set_value(doc_str.replace(":cvar", ":param"))
if return_ is None else return_
] if body_len == 1 else body))
# elif not isinstance(body[0], Return) and return_ is not None:
# body.append(return_)
# elif return_ is not None:
# body = [return_]
return FunctionDef(args=arguments(
args=[set_arg("self")],
defaults=[],
kw_defaults=[],
kwarg=None,
kwonlyargs=[],
posonlyargs=[],
vararg=None,
arg=None,
),
body=body,
decorator_list=[],
name="__call__",
returns=None,
arguments_args=None,
identifier_name=None,
stmt=None,
lineno=None,
**maybe_type_comment)
def test_find_in_ast_no_val(self) -> None:
"""Tests that `find_in_ast` correctly gives AST node from
`def class C(object): def function_name(self,dataset_name: str,…)`"""
run_ast_test(
self,
find_in_ast(
"C.function_name.dataset_name".split("."),
class_with_optional_arg_method_ast,
),
set_arg(
annotation=Name(
"str",
Load(),
),
arg="dataset_name",
),
)
def test_emit_arg(self) -> None:
""" Tests that `arg` is emitted from `emit_arg` """
self.assertIsInstance(
class_with_method_and_body_types_ast.body[1].args.args[1], arg)
self.assertIsInstance(
emit_arg(
class_with_method_and_body_types_ast.body[1].args.args[1]),
arg)
assign = Assign(targets=[Name("yup", Store())],
value=set_value("nup"),
expr=None,
**maybe_type_comment)
gen_ast = emit_arg(assign)
self.assertIsInstance(gen_ast, arg)
run_ast_test(
self,
gen_ast=gen_ast,
gold=set_arg("yup"),
)
def test_find_in_ast_with_val(self) -> None:
"""Tests that `find_in_ast` correctly gives AST node from
`def class C(object): def function_name(self,dataset_name: str='foo',…)`"""
gen_ast = find_in_ast(
"C.function_name.dataset_name".split("."),
class_with_method_and_body_types_ast,
)
self.assertIsInstance(gen_ast.default, Constant if PY_GTE_3_8 else Str)
self.assertEqual(get_value(gen_ast.default), "~/tensorflow_datasets")
run_ast_test(
self,
gen_ast,
set_arg(
annotation=Name(
"str",
Load(),
),
arg="dataset_name",
),
)
"RemoteMonitor",
"TensorBoard",
"TerminateOnNaN",
),
action="append",
help="Collection of callables that are run inside the training loop",
)
return argument_parser
'''.format(
_cli_doc_str=_cli_doc_str, header_doc_str=docstring_header_str
)
argparse_func_ast = fix_missing_locations(
FunctionDef(
args=arguments(
args=[set_arg("argument_parser")],
defaults=[],
kw_defaults=[],
kwarg=None,
kwonlyargs=[],
posonlyargs=[],
vararg=None,
arg=None,
),
body=[
_cli_doc_expr,
Assign(
targets=[
Attribute(
Name("argument_parser", Load()),
"description",
def argparse_function(
intermediate_repr,
emit_default_doc=False,
emit_default_doc_in_return=False,
function_name="set_cli_args",
function_type="static",
):
"""
Convert to an argparse FunctionDef
:param intermediate_repr: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:type intermediate_repr: ```dict```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:param emit_default_doc_in_return: Whether help/docstring in return should include 'With default' text
:type emit_default_doc_in_return: ```bool```
:param function_name: name of function_def
:type function_name: ```str```
:param function_type: Type of function, static is static or global method, others just become first arg
:type function_type: ```Literal['self', 'cls', 'static']```
:return: AST node for function definition which constructs argparse
:rtype: ```FunctionDef```
"""
function_name = function_name or intermediate_repr["name"]
function_type = function_type or intermediate_repr["type"]
internal_body = get_internal_body(
target_name=function_name,
target_type=function_type,
intermediate_repr=intermediate_repr,
)
return FunctionDef(
args=arguments(
args=[set_arg("argument_parser")],
# None if function_type in frozenset((None, "static"))
# else set_arg(function_type),
defaults=[],
kw_defaults=[],
kwarg=None,
kwonlyargs=[],
posonlyargs=[],
vararg=None,
arg=None,
),
body=list(
chain.from_iterable((
iter((
Expr(
set_value(
"\n Set CLI arguments\n\n "
":param argument_parser: argument parser\n "
":type argument_parser: ```ArgumentParser```\n\n "
"{return_params}".format(return_params=(
lambda returns:
":return: argument_parser{return_doc}\n "
"{rtype}".format(
return_doc=", {}".format(returns["doc"])
if "doc" in returns else "",
rtype=":rtype: ```ArgumentParser```\n "
if intermediate_repr["returns"][
"return_type"].get("typ") in
(None, "None", NoneStr) else
":rtype: ```Tuple[ArgumentParser, {returns[typ]}]```\n{tab}"
"".format(returns=returns, tab=tab),
)
)(returns=set_default_doc(
next(iter(intermediate_repr["returns"].items())
),
emit_default_doc=emit_default_doc_in_return,
)[1]) if "return_type" in (
intermediate_repr.get("returns") or iter(())
) else ":return: argument_parser\n "
":rtype: ```ArgumentParser```\n "
))),
Assign(targets=[
Attribute(
Name("argument_parser", Load()),
"description",
Store(),
)
],
value=set_value(intermediate_repr["doc"]),
lineno=None,
expr=None,
**maybe_type_comment),
)),
filter(
None,
(
*((map(
partial(
param2argparse_param,
emit_default_doc=emit_default_doc,
),
intermediate_repr["params"].items(),
)) if "params" in intermediate_repr else ()),
*(internal_body[
2 if len(internal_body) > 1 and isinstance(
internal_body[1], Assign) and internal_body[1].
targets[0].id == "argument_parser" else 1:] if
internal_body and isinstance(internal_body[0], Expr)
and isinstance(get_value(internal_body[0].value),
str) else internal_body),
None if internal_body
and isinstance(internal_body[-1], Return) else (Return(
value=Tuple(
ctx=Load(),
elts=[
Name("argument_parser", Load()),
set_value(intermediate_repr["returns"]
["return_type"]["default"]) if
intermediate_repr["returns"]["return_type"]
["default"].startswith("```")
and intermediate_repr["returns"]
["return_type"]["default"].endswith("```")
else ast.parse(intermediate_repr["returns"]
["return_type"]
["default"]).body[0].value,
],
expr=None,
),
expr=None,
) if "default" in (
intermediate_repr.get("returns") or {
"return_type": iter(())
})["return_type"] else Return(value=Name(
"argument_parser", Load()),
expr=None)),
),
),
))),
decorator_list=[],
name=function_name,
returns=None,
lineno=None,
arguments_args=None,
identifier_name=None,
stmt=None,
**maybe_type_comment)
def function(
intermediate_repr,
function_name,
function_type,
emit_default_doc=False,
docstring_format="rest",
indent_level=2,
emit_separating_tab=PY3_8,
inline_types=True,
emit_as_kwonlyargs=True,
):
"""
Construct a function from our IR
:param intermediate_repr: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:type intermediate_repr: ```dict```
:param function_name: name of function_def
:type function_name: ```Optional[str]```
:param function_type: Type of function, static is static or global method, others just become first arg
:type function_type: ```Optional[Literal['self', 'cls', 'static']]```
:param docstring_format: Format of docstring
:type docstring_format: ```Literal['rest', 'numpy', 'google']```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:param indent_level: docstring indentation level whence: 0=no_tabs, 1=one tab; 2=two tabs
:type indent_level: ```int```
:param emit_separating_tab: docstring decider for whether to put a tab between :param and return and desc
:type emit_separating_tab: ```bool```
:param inline_types: Whether the type should be inline or in docstring
:type inline_types: ```bool```
:param emit_as_kwonlyargs: Whether argument(s) emitted must be keyword only
:type emit_as_kwonlyargs: ```bool```
:return: AST node for function definition
:rtype: ```FunctionDef```
"""
params_no_kwargs = tuple(
filter(
lambda param: not param[0].endswith("kwargs"),
intermediate_repr["params"].items(),
))
function_name = function_name or intermediate_repr["name"]
function_type = function_type or intermediate_repr["type"]
args = ([] if function_type in frozenset(
(None, "static")) else [set_arg(function_type)])
from doctrans.emitter_utils import ast_parse_fix
args_from_params = list(
map(
lambda param: set_arg(
annotation=(Name(param[1]["typ"], Load()) if param[1]["typ"] in
simple_types else ast_parse_fix(param[1]["typ"]))
if inline_types and "typ" in param[1] else None,
arg=param[0],
),
params_no_kwargs,
), )
defaults_from_params = list(
map(
lambda param: set_value(None) if param[1].get("default") in
(None, "None", NoneStr) else set_value(param[1].get("default")),
params_no_kwargs,
))
if emit_as_kwonlyargs:
kwonlyargs, kw_defaults, defaults = args_from_params, defaults_from_params, []
else:
kwonlyargs, kw_defaults, defaults = [], [], defaults_from_params
args += args_from_params
internal_body = get_internal_body(
target_name=function_name,
target_type=function_type,
intermediate_repr=intermediate_repr,
)
return_val = (Return(
value=ast.parse(intermediate_repr["returns"]["return_type"]
["default"].strip("`")).body[0].value,
expr=None,
) if (intermediate_repr.get("returns") or {
"return_type": {}
})["return_type"].get("default") else None)
return FunctionDef(
args=arguments(
args=args,
defaults=defaults,
kw_defaults=kw_defaults,
kwarg=next(
map(
lambda param: set_arg(param[0]),
filter(
lambda param: param[0].endswith("kwargs"),
intermediate_repr["params"].items(),
),
),
None,
),
kwonlyargs=kwonlyargs,
posonlyargs=[],
vararg=None,
arg=None,
),
body=list(
filter(
None,
(
Expr(
set_value(
to_docstring(
intermediate_repr,
emit_default_doc=emit_default_doc,
docstring_format=docstring_format,
emit_types=not inline_types,
indent_level=indent_level,
emit_separating_tab=emit_separating_tab,
))),
*(internal_body[:-1] if internal_body
and isinstance(internal_body[-1], Return) and return_val
else internal_body),
return_val,
),
)),
decorator_list=[],
name=function_name,
returns=(ast.parse(
intermediate_repr["returns"]["return_type"]["typ"]).body[0].value
if inline_types and (intermediate_repr.get("returns") or {
"return_type": {}
})["return_type"].get("typ") else None),
lineno=None,
arguments_args=None,
identifier_name=None,
stmt=None,
**maybe_type_comment)
def _make_call_meth(body, return_type, param_names, docstring_format,
word_wrap):
"""
Construct a `__call__` method from the provided `body`
:param body: The body, probably from a FunctionDef.body
:type body: ```List[AST]```
:param return_type: The return type of the parent symbol (probably class). Used to fill in `__call__` return.
:type return_type: ```Optional[str]```
:param param_names: Container of AST `id`s to match for rename
:type param_names: ```Optional[Iterator[str]]```
:param docstring_format: Format of docstring
:type docstring_format: ```Literal['rest', 'numpydoc', 'google']```
:param word_wrap: Whether to word-wrap. Set `DOCTRANS_LINE_LENGTH` to configure length.
:type word_wrap: ```bool```
:returns: Internal function for `__call__`
:rtype: ```FunctionDef```
"""
body_len = len(body)
# return_ = (
# ast.fix_missing_locations(
# RewriteName(param_names).visit(
# Return(get_value(ast.parse(return_type.strip("`")).body[0]), expr=None)
# )
# )
# if return_type is not None and code_quoted(return_type)
# else None
# )
if body_len:
if isinstance(body, dict):
body = list(
filter(
None,
(
None if body.get("doc") in none_types else Expr(
set_value(
emit_param_str(
(
"return_type",
{
"doc":
multiline(
indent_all_but_first(
body["doc"]))
},
),
style=docstring_format,
word_wrap=word_wrap,
))),
RewriteName(param_names).visit(
Return(
get_value(
ast.parse(return_type.strip("`")).body[0]),
expr=None,
)) if code_quoted(body["default"]) else Return(
set_value(body["default"]), expr=None),
),
))
# elif isinstance(body[0], Expr):
# doc_str = get_value(body[0].value)
# if isinstance(doc_str, str) and body_len > 0:
# body = (
# body[1:]
# if body_len > 1
# else (
# [
# set_value(doc_str.replace(":cvar", ":param"))
# if return_ is None
# else return_
# ]
# if body_len == 1
# else body
# )
# )
# elif not isinstance(body[0], Return) and return_ is not None:
# body.append(return_)
# elif return_ is not None:
# body = [return_]
return (ast.fix_missing_locations(
FunctionDef(args=arguments(
args=[set_arg("self")],
defaults=[],
kw_defaults=[],
kwarg=None,
kwonlyargs=[],
posonlyargs=[],
vararg=None,
arg=None,
),
body=body,
decorator_list=[],
name="__call__",
returns=None,
arguments_args=None,
identifier_name=None,
stmt=None,
lineno=None,
**maybe_type_comment)) if body else None)
'''
class_with_method_and_body_types_ast = fix_missing_locations(
ClassDef(
name="C",
bases=[Name("object", Load())],
keywords=[],
body=[
Expr(set_value(" C class (mocked!) ")),
FunctionDef(
name="function_name",
args=arguments(
posonlyargs=[],
vararg=None,
args=[
set_arg("self"),
set_arg(arg="dataset_name",
annotation=Name("str", Load())),
set_arg(
arg="tfds_dir",
annotation=Subscript(
Name("Optional", Load()),
set_slice(Name("str", Load())),
Load(),
),
),
set_arg(
arg="K",
annotation=Subscript(
Name("Literal", Load()),
set_slice(
expr_annotation=None,
),
AnnAssign(
annotation=Name("str", Load()),
simple=1,
target=Name("return_type", Store()),
value=set_value(
"```K.mean(math_ops.square(math_ops.maximum(1.0 - y_true * y_pred, 0.0)), axis=-1)```"
),
expr=None,
expr_target=None,
expr_annotation=None,
),
FunctionDef(
args=arguments(
args=[set_arg("self")],
defaults=[],
kw_defaults=[],
kwarg=None,
kwonlyargs=[],
vararg=None,
arg=None,
posonlyargs=[],
),
body=[
Assign(
targets=[
Attribute(
Name("self", Load()),
"y_pred",
Load(),
def argparse_function(
intermediate_repr,
emit_default_doc=False,
function_name="set_cli_args",
function_type="static",
wrap_description=False,
word_wrap=True,
docstring_format="rest",
):
"""
Convert to an argparse FunctionDef
:param intermediate_repr: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:type intermediate_repr: ```dict```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:param function_name: name of function_def
:type function_name: ```str```
:param function_type: Type of function, static is static or global method, others just become first arg
:type function_type: ```Literal['self', 'cls', 'static']```
:param wrap_description: Whether to word-wrap the description. Set `DOCTRANS_LINE_LENGTH` to configure length.
:type wrap_description: ```bool```
:param word_wrap: Whether to word-wrap. Set `DOCTRANS_LINE_LENGTH` to configure length.
:type word_wrap: ```bool```
:param docstring_format: Format of docstring
:type docstring_format: ```Literal['rest', 'numpydoc', 'google']```
:returns: AST node for function definition which constructs argparse
:rtype: ```FunctionDef```
"""
function_name = function_name or intermediate_repr["name"]
function_type = function_type or intermediate_repr["type"]
internal_body = get_internal_body(
target_name=function_name,
target_type=function_type,
intermediate_repr=intermediate_repr,
)
return FunctionDef(
args=arguments(
args=[set_arg("argument_parser")],
# None if function_type in frozenset((None, "static"))
# else set_arg(function_type),
defaults=[],
kw_defaults=[],
kwarg=None,
kwonlyargs=[],
posonlyargs=[],
vararg=None,
arg=None,
),
body=list(
chain.from_iterable(
(
iter(
(
Expr(
set_value(
indent(
docstring(
{
"doc": "Set CLI arguments",
"params": OrderedDict(
(
(
"argument_parser",
{
"doc": "argument parser",
"typ": "ArgumentParser",
},
),
)
),
"returns": OrderedDict(
(
(
"return_type",
{
"doc": "argument_parser, {}".format(
intermediate_repr[
"returns"
]["return_type"][
"doc"
]
)
if intermediate_repr[
"returns"
]["return_type"].get(
"doc"
)
else "argument_parser",
"typ": "Tuple[ArgumentParser, {typ}]".format(
typ=intermediate_repr[
"returns"
][
"return_type"
][
"typ"
]
),
}
if "return_type"
in (
(
intermediate_repr
or {}
).get("returns")
or iter(())
)
and intermediate_repr[
"returns"
]["return_type"].get("typ")
not in none_types
else {
"doc": "argument_parser",
"typ": "ArgumentParser",
},
),
),
),
},
docstring_format=docstring_format,
word_wrap=word_wrap,
),
tab,
)
+ tab
)
),
Assign(
targets=[
Attribute(
Name("argument_parser", Load()),
"description",
Store(),
)
],
value=set_value(
(fill if wrap_description else identity)(
intermediate_repr["doc"]
)
),
lineno=None,
expr=None,
**maybe_type_comment
),
)
),
filter(
None,
(
*(
(
map(
partial(
param2argparse_param,
word_wrap=word_wrap,
emit_default_doc=emit_default_doc,
),
intermediate_repr["params"].items(),
)
)
if "params" in intermediate_repr
else ()
),
*(
internal_body[
2
if len(internal_body) > 1
and isinstance(internal_body[1], Assign)
and internal_body[1].targets[0].id
== "argument_parser"
else 1 :
]
if internal_body
and isinstance(internal_body[0], Expr)
and isinstance(get_value(internal_body[0].value), str)
else internal_body
),
None
if internal_body and isinstance(internal_body[-1], Return)
else (
Return(
value=Tuple(
ctx=Load(),
elts=[
Name("argument_parser", Load()),
set_value(
intermediate_repr["returns"][
"return_type"
]["default"]
)
if code_quoted(
intermediate_repr["returns"][
"return_type"
]["default"]
)
else ast.parse(
intermediate_repr["returns"][
"return_type"
]["default"]
)
.body[0]
.value,
],
expr=None,
),
expr=None,
)
if "default"
in (
intermediate_repr.get("returns")
or {"return_type": iter(())}
)["return_type"]
else Return(
value=Name("argument_parser", Load()), expr=None
)
),
),
),
)
)
),
decorator_list=[],
name=function_name,
returns=None,
lineno=None,
arguments_args=None,
identifier_name=None,
stmt=None,
**maybe_type_comment
)
