def test_from_class_with_body_in_method_to_method_with_body(self) -> None:
"""Tests if this can make the roundtrip from a full function to a full function"""
annotate_ancestry(class_with_method_and_body_types_ast)
function_def = reindent_docstring(
next(
filter(
rpartial(isinstance, FunctionDef),
class_with_method_and_body_types_ast.body,
)))
ir = parse.function(
find_in_ast(
"C.function_name".split("."),
class_with_method_and_body_types_ast,
), )
gen_ast = emit.function(
ir,
emit_default_doc=False,
function_name="function_name",
function_type="self",
indent_level=1,
emit_separating_tab=True,
emit_as_kwonlyargs=False,
)
# emit.file(gen_ast, os.path.join(os.path.dirname(__file__),
# "delme{extsep}py".format(extsep=extsep), mode="wt")
run_ast_test(
self,
gen_ast=gen_ast,
gold=function_def,
)
def ground_truth(args, truth_file):
"""
There is but one truth. Conform.
:param args: Namespace with the values of the CLI arguments
:type args: ```Namespace```
:param truth_file: contains the filename of the one true source
:type truth_file: ```str```
:returns: Filenames and whether they were changed
:rtype: ```OrderedDict```
"""
arg2parse_emit_type = {
"argparse_function":
(parse.argparse_ast, emit.argparse_function, FunctionDef),
"class": (parse.class_, emit.class_, ClassDef),
"function": (parse.function, emit.function, FunctionDef),
}
parse_func, emit_func, type_wanted = arg2parse_emit_type[args.truth]
search = _get_name_from_namespace(args, args.truth).split(".")
with open(truth_file, "rt") as f:
true_ast = ast_parse(f.read(), filename=truth_file)
original_node = find_in_ast(search, true_ast)
gold_ir = parse_func(
original_node,
**_default_options(node=original_node,
search=search,
type_wanted=type_wanted)())
effect = OrderedDict()
# filter(lambda arg: arg != args.truth, arg2parse_emit_type.keys()):
for fun_name, (parse_func, emit_func,
type_wanted) in arg2parse_emit_type.items():
search = list(
strip_split(_get_name_from_namespace(args, fun_name), "."))
filenames = getattr(args, pluralise(fun_name))
assert isinstance(
filenames,
(list,
tuple)), "Expected Union[list, tuple] got {type_name!r}".format(
type_name=type(filenames).__name__)
effect.update(
map(
lambda filename: _conform_filename(
filename=filename,
search=search,
emit_func=emit_func,
replacement_node_ir=gold_ir,
type_wanted=type_wanted,
),
filenames,
))
return effect
def test_emit_ann_assign(self) -> None:
"""Tests that AnnAssign is emitted from `emit_ann_assign`"""
self.assertIsInstance(class_ast.body[1], AnnAssign)
self.assertIsInstance(emit_ann_assign(class_ast.body[1]), AnnAssign)
self.assertIsInstance(emit_ann_assign(class_ast.body[1]), AnnAssign)
gen_ast = emit_ann_assign(
find_in_ast(
"C.function_name.dataset_name".split("."),
class_with_method_and_body_types_ast,
)
)
self.assertIsInstance(gen_ast, AnnAssign)
run_ast_test(
self,
gen_ast,
AnnAssign(
annotation=Name(
"str",
Load(),
),
simple=1,
target=Name("dataset_name", Store()),
value=set_value("~/tensorflow_datasets"),
expr=None,
expr_target=None,
expr_annotation=None,
),
)
def test_find_in_ast_self(self) -> None:
"""Tests that `find_in_ast` successfully finds itself in AST"""
run_ast_test(self, find_in_ast(["ConfigClass"], class_ast), class_ast)
module = Module(body=[], type_ignores=[], stmt=None)
run_ast_test(self, find_in_ast([], module), module)
module_with_fun = Module(
body=[
FunctionDef(
name="call_peril",
args=arguments(
args=[],
defaults=[],
kw_defaults=[],
kwarg=None,
kwonlyargs=[],
posonlyargs=[],
vararg=None,
arg=None,
),
body=[],
decorator_list=[],
lineno=None,
arguments_args=None,
identifier_name=None,
stmt=None,
)
],
stmt=None,
)
annotate_ancestry(module_with_fun)
run_ast_test(
self,
find_in_ast(["call_peril"], module_with_fun),
module_with_fun.body[0],
skip_black=True,
)
def test_find_in_ast_no_val(self) -> None:
"""Tests that `find_in_ast` correctly gives AST node from
`def class C(object): def function_name(self,dataset_name: str,…)`"""
run_ast_test(
self,
find_in_ast(
"C.function_name.dataset_name".split("."),
class_with_optional_arg_method_ast,
),
set_arg(
annotation=Name(
"str",
Load(),
),
arg="dataset_name",
),
)
def test_find_in_ast(self) -> None:
"""Tests that `find_in_ast` successfully finds nodes in AST"""
run_ast_test(
self,
find_in_ast("ConfigClass.dataset_name".split("."), class_ast),
AnnAssign(
annotation=Name(
"str",
Load(),
),
simple=1,
target=Name("dataset_name", Store()),
value=set_value("mnist"),
expr=None,
expr_target=None,
expr_annotation=None,
),
)
def test_find_in_ast_with_val(self) -> None:
"""Tests that `find_in_ast` correctly gives AST node from
`def class C(object): def function_name(self,dataset_name: str='foo',…)`"""
gen_ast = find_in_ast(
"C.function_name.dataset_name".split("."),
class_with_method_and_body_types_ast,
)
self.assertIsInstance(gen_ast.default, Constant if PY_GTE_3_8 else Str)
self.assertEqual(get_value(gen_ast.default), "~/tensorflow_datasets")
run_ast_test(
self,
gen_ast,
set_arg(
annotation=Name(
"str",
Load(),
),
arg="dataset_name",
),
)
def _get_ass_typ(self, node):
"""
Get the type of the assignment
:param node: Assignment
:type node: ```Union[Assign, AnnAssign]```
:returns: The type of the assignment, e.g., `int`
:rtype: ```Optional[str]```
"""
name, typ_dict = ((node.targets[0], {
"typ": node.type_comment
}) if isinstance(node, Assign) else
(node.target, {
"typ": node.annotation or node.type_comment
}))
if not hasattr(node, "_location"):
return typ_dict["typ"]
search = node._location[:-1]
search_str = ".".join(search)
self.memoized[search_str] = ir = (self.memoized.get(
search_str,
(lambda parent:
((lambda doc_str: None
if doc_str is None else parse.docstring(doc_str))
(get_docstring(parent))
if isinstance(parent,
(ClassDef, AsyncFunctionDef, FunctionDef)) else {
"params": OrderedDict()
}))(find_in_ast(search, self.whole_ast)),
) or {
"params": OrderedDict()
})
return ir["params"].get(name, typ_dict)[
"typ"] # if ir is not None and ir.get("params") else None
def _conform_filename(
filename,
search,
emit_func,
replacement_node_ir,
type_wanted,
):
"""
Conform the given file to the `intermediate_repr`
:param filename: Location of file
:type filename: ```str```
:param search: Search query, e.g., ['node_name', 'function_name', 'arg_name']
:type search: ```List[str]```
:param replacement_node_ir: Replace what is found with the contents of this param
:type replacement_node_ir: ```dict```
:param type_wanted: AST instance
:type type_wanted: ```AST```
:returns: filename, whether the file was modified
:rtype: ```Tuple[str, bool]```
"""
filename = path.realpath(path.expanduser(filename))
if not path.isfile(filename):
emit.file(
emit_func(
replacement_node_ir,
emit_default_doc=False, # emit_func.__name__ == "class_"
),
filename=filename,
mode="wt",
skip_black=False,
)
return filename, True
with open(filename, "rt") as f:
parsed_ast = ast_parse(f.read(), filename=filename)
assert isinstance(parsed_ast, Module)
original_node = find_in_ast(search, parsed_ast)
replacement_node = emit_func(
replacement_node_ir,
**_default_options(node=original_node,
search=search,
type_wanted=type_wanted)())
if original_node is None:
emit.file(replacement_node,
filename=filename,
mode="a",
skip_black=False)
return filename, True
assert len(search) > 0
assert (type(replacement_node) == type_wanted
), "Expected {type_wanted!r} got {type_replacement_node!r}".format(
type_wanted=type_wanted,
type_replacement_node=type(replacement_node).__name__)
replaced = False
if not cmp_ast(original_node, replacement_node):
rewrite_at_query = RewriteAtQuery(
search=search,
replacement_node=replacement_node,
)
rewrite_at_query.visit(parsed_ast)
print("modified" if rewrite_at_query.replaced else "unchanged",
filename,
sep="\t")
if rewrite_at_query.replaced:
emit.file(parsed_ast, filename, mode="wt", skip_black=False)
replaced = rewrite_at_query.replaced
return filename, replaced
def test_find_in_ast_None(self) -> None:
"""Tests that `find_in_ast` fails correctly in AST"""
self.assertIsNone(find_in_ast(["John Galt"], class_ast))
def sync_property(
input_eval,
input_param,
input_ast,
input_filename,
output_param,
output_param_wrap,
output_ast,
):
"""
Sync a single property
:param input_eval: Whether to evaluate the `param`, or just leave it
:type input_eval: ```bool```
:param input_param: Location within file of property.
Can be top level like `'a'` for `a=5` or with the `.` syntax as in `output_params`.
:type input_param: ```List[str]```
:param input_ast: AST of the input file
:type input_ast: ```AST```
:param input_filename: Filename of the input (used in `eval`)
:type input_filename: ```str```
:param output_param: Parameters to update. E.g., `'A.F'` for `class A: F = None`, `'f.g'` for `def f(g): pass`
:type output_param: ```str```
:param output_param_wrap: Wrap all input_str params with this. E.g., `Optional[Union[{output_param}, str]]`
:param output_param_wrap: ```Optional[str]```
:param output_ast: AST of the input file
:type output_ast: ```AST```
:returns: New AST derived from `output_ast`
:rtype: ```AST```
"""
search = list(strip_split(output_param, "."))
if input_eval:
if input_param.count(".") != 0:
raise NotImplementedError("Anything not on the top-level of the module")
local = {}
output = eval(compile(input_ast, filename=input_filename, mode="exec"), local)
assert output is None
replacement_node = ast.AnnAssign(
annotation=it2literal(local[input_param]),
simple=1,
target=ast.Name(
# input_param
search[-1],
ast.Store(),
),
value=None,
expr=None,
expr_annotation=None,
expr_target=None,
)
else:
assert isinstance(input_ast, ast.Module)
annotate_ancestry(input_ast)
replacement_node = find_in_ast(list(strip_split(input_param, ".")), input_ast)
assert replacement_node is not None
if output_param_wrap is not None:
if hasattr(replacement_node, "annotation"):
if replacement_node.annotation is not None:
replacement_node.annotation = (
ast.parse(
output_param_wrap.format(
output_param=to_code(replacement_node.annotation)
)
)
.body[0]
.value
)
else:
raise NotImplementedError(type(replacement_node).__name__)
rewrite_at_query = RewriteAtQuery(
search=search,
replacement_node=replacement_node,
)
gen_ast = rewrite_at_query.visit(output_ast)
assert (
rewrite_at_query.replaced is True
), "Failed to update with {replacement_node_str!r}".format(
replacement_node_str=to_code(replacement_node)
)
return gen_ast