def test_roundtrip(self):
only_localizable = False
for name, example in EXAMPLES.items():
if ' with eol comments' in name or name.startswith('multiline '):
continue
data = {}
with self.subTest(name=name, example=example, data=data):
tree = typed_ast.ast3.parse(example)
code = typed_astunparse.unparse(tree)
complete_tree = parse(example)
data['complete_tree'] = complete_tree
complete_code = unparse(complete_tree)
data['complete_code'] = complete_code
self.assertGreaterEqual(len(complete_code.replace(' ', '')),
len(code.replace(' ', '')),
(complete_code, code))
reparsed_tree = typed_ast.ast3.parse(code)
tree_nodes = ast_to_list(tree, only_localizable)
reparsed_tree_nodes = ast_to_list(reparsed_tree,
only_localizable)
self.assertEqual(len(reparsed_tree_nodes), len(tree_nodes))
self.assertEqual(typed_ast.ast3.dump(reparsed_tree),
typed_ast.ast3.dump(tree))
reparsed_complete_tree = parse(complete_code)
complete_tree_nodes = ast_to_list(complete_tree,
only_localizable)
reparsed_complete_tree_nodes = ast_to_list(
reparsed_complete_tree, only_localizable)
self.assertEqual(len(reparsed_complete_tree_nodes),
len(complete_tree_nodes), complete_code)
self.assertEqual(
typed_ast.ast3.dump(reparsed_complete_tree),
typed_ast.ast3.dump(complete_tree),
'"""\n{}\n""" vs. original """\n{}\n"""'.format(
complete_code, example))
def test_ast_to_list(self):
for name, example in EXAMPLES.items():
for only_localizable in (False, True):
with self.subTest(name=name, example=example, only_localizable=only_localizable):
tree = typed_ast.ast3.parse(example)
nodes = ast_to_list(tree, only_localizable)
self.assertIsInstance(nodes, list)
if not only_localizable:
self.assertGreater(len(nodes), 0)
def test_comment_tokens_approx(self):
for (name, example), only_localizable in itertools.product(
EXAMPLES.items(), (False, True)):
# for only_localizable in:
with self.subTest(name=name,
example=example,
only_localizable=only_localizable):
tree = insert_comment_tokens_approx(
typed_ast.ast3.parse(example), get_comment_tokens(example))
nodes = ast_to_list(tree, only_localizable)
self.assertIsInstance(tree, typed_ast.ast3.AST)
non_comment_nodes = ast_to_list(typed_ast.ast3.parse(example),
only_localizable)
comments = get_comment_tokens(example)
expected_count = max(1 if comments else 0, len(non_comment_nodes)) \
+ (0 if only_localizable else 1) * len(comments)
self.assertEqual(len(nodes), expected_count,
(nodes, non_comment_nodes, comments))
def test_comment_tokens(self):
only_localizable = False
for name, example in EXAMPLES.items():
if ' with eol comments' in name or name.startswith('multiline '):
continue
with self.subTest(name=name, example=example):
tree = insert_comment_tokens(example,
typed_ast.ast3.parse(example),
get_comment_tokens(example))
nodes = ast_to_list(tree, only_localizable)
self.assertIsInstance(tree, typed_ast.ast3.AST)
non_comment_nodes = ast_to_list(typed_ast.ast3.parse(example),
only_localizable)
comments = get_comment_tokens(example)
expected_count = max(1 if comments else 0,
len(non_comment_nodes)) + len(comments)
self.assertEqual(len(nodes), expected_count,
(nodes, non_comment_nodes, comments))
def test_get_ast_node_locations(self):
for name, example in EXAMPLES.items():
for only_localizable in (False, True):
with self.subTest(name=name, example=example, only_localizable=only_localizable):
tree = typed_ast.ast3.parse(example)
nodes = ast_to_list(tree, only_localizable)
locations = get_ast_node_locations(nodes)
self.assertIsInstance(locations, list)
for location in locations:
self.assertIsInstance(location, tuple)
self.assertEqual(len(location), 2)
def test_find_in_ast_real(self):
for name, example in EXAMPLES.items():
if ' with eol comments' in name or name.startswith('multiline '):
continue
comment_tokens = get_comment_tokens(example)
for token in comment_tokens:
scope = get_token_scope(token)
with self.subTest(name=name, example=example, scope=scope):
tree = typed_ast.ast3.parse(example)
nodes = ast_to_list(tree)
path, before = find_in_ast(example, tree, nodes, scope)
self.assertIsInstance(path, list)
self.assertIsInstance(before, bool)
def test_find_in_ast(self):
for name, example in EXAMPLES.items():
if ' with eol comments' in name or name.startswith('multiline '):
continue
for index in range(0, len(example) + 1, 64):
for end_index in range(index + 1, len(example) + 1, 64):
# TODO: decrease distance step without failing assertions
scope = Scope(convert_1d_str_index_to_2d(example, index),
convert_1d_str_index_to_2d(example, end_index))
with self.subTest(name=name, example=example, scope=scope):
tree = typed_ast.ast3.parse(example)
nodes = ast_to_list(tree)
path, before = find_in_ast(example, tree, nodes, scope)
self.assertIsInstance(path, list)
self.assertIsInstance(before, bool)
def test_get_ast_node_scopes(self):
for name, example in EXAMPLES.items():
with self.subTest(name=name, example=example):
tree = typed_ast.ast3.parse(example)
nodes = ast_to_list(tree)
scopes = get_ast_node_scopes(example, nodes)
self.assertIsInstance(scopes, list)
self.assertEqual(len(scopes), len(nodes))
for scope in scopes:
self.assertIsInstance(scope, Scope)
locations = get_ast_node_locations(nodes)
self.assertEqual(len(scopes), len(locations))
for node, scope, location in zip(nodes, scopes, locations):
if None in location:
continue
self.assertEqual(scope.start, location, '{} in: """\n{}\n"""'.format(
typed_ast.ast3.dump(node), example))