def verify_all_nodes(self, test_case): """ Generically test atok.get_text() on the ast tree: for each statement and expression in the tree, we extract the text, parse it, and see if it produces an equivalent tree. Returns the number of nodes that were tested this way. """ test_case.longMessage = True tested_nodes = 0 for node in self.all_nodes: if not (util.is_stmt(node) or util.is_expr(node) or util.is_module(node)): continue if isinstance(node, astroid.nodes.Yield): # Astroid stringifies Yield nodes differently depending on parent, so these are too # annoying to verify. continue text = self.atok.get_text(node) rebuilt_node = parse_snippet(text, is_expr=util.is_expr(node), is_module=util.is_module(node)) # Now we need to check if the two nodes are equivalent. left = to_source(rebuilt_node) right = to_source(node) test_case.assertEqual(left, right) tested_nodes += 1 return tested_nodes
def parse_snippet(self, text, node): """ Returns the parsed AST tree for the given text, handling issues with indentation and newlines when text is really an extracted part of larger code. """ # If text is indented, it's a statement, and we need to put in a scope for indents to be valid # (using textwrap.dedent is insufficient because some lines may not indented, e.g. comments or # multiline strings). If text is an expression but has newlines, we parenthesize it to make it # parsable. # For expressions and statements, we add a dummy statement '_' before it because if it's just a # string contained in an astroid.Const or astroid.Expr it will end up in the doc attribute and be # a pain to extract for comparison # For starred expressions, e.g. `*args`, we wrap it in a function call to make it parsable. # For slices, e.g. `x:`, we wrap it in an indexing expression to make it parsable. indented = re.match(r'^[ \t]+\S', text) if indented: return self.module.parse('def dummy():\n' + text).body[0].body[0] if util.is_starred(node): return self.module.parse('f(' + text + ')').body[0].value.args[0] if util.is_slice(node): return self.module.parse('a[' + text + ']').body[0].value.slice if util.is_expr(node): return self.module.parse('_\n(' + text + ')').body[1].value if util.is_module(node): return self.module.parse(text) return self.module.parse('_\n' + text).body[1]
def verify_all_nodes(self, test_case): """ Generically test atok.get_text() on the ast tree: for each statement and expression in the tree, we extract the text, parse it, and see if it produces an equivalent tree. Returns the number of nodes that were tested this way. """ test_case.longMessage = True tested_nodes = 0 for node in self.all_nodes: if not (util.is_stmt(node) or util.is_expr(node) or util.is_module(node)): continue # slices currently only get the correct tokens for ast, not astroid. if util.is_slice(node) and test_case.is_astroid_test: continue text = self.atok.get_text(node) # await is not allowed outside async functions below 3.7 # parsing again would give a syntax error if 'await' in text and 'async def' not in text and sys.version_info < ( 3, 7): continue # `elif:` is really just `else: if:` to the AST, # so get_text can return text starting with elif when given an If node. # This is generally harmless and there's probably no good alternative, # but in isolation it's invalid syntax text = re.sub(r'^(\s*)elif(\W)', r'\1if\2', text, re.MULTILINE) rebuilt_node = test_case.parse_snippet(text, node) try: test_case.assert_nodes_equal(node, rebuilt_node) except AssertionError: if test_case.is_astroid_test: # This can give a more helpful failure message with a diff test_case.assertEqual( repr_tree(node), repr_tree(rebuilt_node), ) raise tested_nodes += 1 return tested_nodes