def verify_all_nodes(self, test_case):
"""
Generically test atok.get_text() on the ast tree: for each statement and expression in the
tree, we extract the text, parse it, and see if it produces an equivalent tree. Returns the
number of nodes that were tested this way.
"""
test_case.longMessage = True
tested_nodes = 0
for node in self.all_nodes:
if not (util.is_stmt(node) or util.is_expr(node)
or util.is_module(node)):
continue
if isinstance(node, astroid.nodes.Yield):
# Astroid stringifies Yield nodes differently depending on parent, so these are too
# annoying to verify.
continue
text = self.atok.get_text(node)
rebuilt_node = parse_snippet(text,
is_expr=util.is_expr(node),
is_module=util.is_module(node))
# Now we need to check if the two nodes are equivalent.
left = to_source(rebuilt_node)
right = to_source(node)
test_case.assertEqual(left, right)
tested_nodes += 1
return tested_nodes
def parse_snippet(self, text, node):
"""
Returns the parsed AST tree for the given text, handling issues with indentation and newlines
when text is really an extracted part of larger code.
"""
# If text is indented, it's a statement, and we need to put in a scope for indents to be valid
# (using textwrap.dedent is insufficient because some lines may not indented, e.g. comments or
# multiline strings). If text is an expression but has newlines, we parenthesize it to make it
# parsable.
# For expressions and statements, we add a dummy statement '_' before it because if it's just a
# string contained in an astroid.Const or astroid.Expr it will end up in the doc attribute and be
# a pain to extract for comparison
# For starred expressions, e.g. `*args`, we wrap it in a function call to make it parsable.
# For slices, e.g. `x:`, we wrap it in an indexing expression to make it parsable.
indented = re.match(r'^[ \t]+\S', text)
if indented:
return self.module.parse('def dummy():\n' + text).body[0].body[0]
if util.is_starred(node):
return self.module.parse('f(' + text + ')').body[0].value.args[0]
if util.is_slice(node):
return self.module.parse('a[' + text + ']').body[0].value.slice
if util.is_expr(node):
return self.module.parse('_\n(' + text + ')').body[1].value
if util.is_module(node):
return self.module.parse(text)
return self.module.parse('_\n' + text).body[1]
def verify_all_nodes(self, test_case):
"""
Generically test atok.get_text() on the ast tree: for each statement and expression in the
tree, we extract the text, parse it, and see if it produces an equivalent tree. Returns the
number of nodes that were tested this way.
"""
test_case.longMessage = True
tested_nodes = 0
for node in self.all_nodes:
if not (util.is_stmt(node) or util.is_expr(node)
or util.is_module(node)):
continue
# slices currently only get the correct tokens for ast, not astroid.
if util.is_slice(node) and test_case.is_astroid_test:
continue
text = self.atok.get_text(node)
# await is not allowed outside async functions below 3.7
# parsing again would give a syntax error
if 'await' in text and 'async def' not in text and sys.version_info < (
3, 7):
continue
# `elif:` is really just `else: if:` to the AST,
# so get_text can return text starting with elif when given an If node.
# This is generally harmless and there's probably no good alternative,
# but in isolation it's invalid syntax
text = re.sub(r'^(\s*)elif(\W)', r'\1if\2', text, re.MULTILINE)
rebuilt_node = test_case.parse_snippet(text, node)
try:
test_case.assert_nodes_equal(node, rebuilt_node)
except AssertionError:
if test_case.is_astroid_test:
# This can give a more helpful failure message with a diff
test_case.assertEqual(
repr_tree(node),
repr_tree(rebuilt_node),
)
raise
tested_nodes += 1
return tested_nodes