def n_mkfunc(self, node): """If the function has a docstring (this is found in the code constants), pull that out and make it part of the syntax tree. When generating the source string that AST node rather than the code field is seen and used. """ code = find_code_node(node, -3).attr mkfunc_pattr = node[-1].pattr if isinstance(mkfunc_pattr, tuple): assert isinstance(mkfunc_pattr, tuple) assert len(mkfunc_pattr, 4) and isinstance(mkfunc_pattr, int) is_closure = node[-1].pattr[3] != 0 else: # FIXME: This is what we had before. It is hoaky and probably wrong. is_closure = mkfunc_pattr == "closure" if ((not is_closure) and len(code.co_consts) > 0 and isinstance(code.co_consts[0], str)): docstring_node = SyntaxTree( "docstring", [Token("LOAD_STR", has_arg=True, pattr=code.co_consts[0])], transformed_by="n_mkfunc", ) node = SyntaxTree( "mkfunc", node[:-1] + [docstring_node, node[-1]], transformed_by="n_mkfunc", ) return node
def n_mkfunc(self, node): """If the function has a docstring (this is found in the code constants), pull that out and make it part of the syntax tree. When generating the source string that AST node rather than the code field is seen and used. """ code = find_code_node(node, -3).attr if ( node[-1].pattr != "closure" and len(code.co_consts) > 0 and code.co_consts[0] is not None ): docstring_node = SyntaxTree( "docstring", [Token("LOAD_STR", has_arg=True, pattr=code.co_consts[0])], transformed_by="n_mkfunc", ) node = SyntaxTree( "mkfunc", node[:-1] + [docstring_node, node[-1]], transformed_by="n_mkfunc", ) return node
def n_import_from37(self, node): importlist37 = node[3] assert importlist37 == "importlist37" if len(importlist37) == 1: alias37 = importlist37[0] store = alias37[1] assert store == "store" alias_name = store[0].attr import_name_attr = node[2] assert import_name_attr == "IMPORT_NAME_ATTR" dotted_names = import_name_attr.attr.split(".") if len(dotted_names) > 1 and dotted_names[-1] == alias_name: # Simulate: # Instead of # import_from37 ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR importlist37 POP_TOP # import_as37 ::= LOAD_CONST LOAD_CONST importlist37 store POP_TOP # 'import_as37': ( '%|import %c as %c\n', 2, -2), node = SyntaxTree( "import_as37", [node[0], node[1], import_name_attr, store, node[-1]], transformed_by="n_import_from37", ) pass pass return node
def transform(self, ast): self.maybe_show_tree(ast) self.ast = copy(ast) self.ast = self.traverse(self.ast, is_lambda=False) try: for i in range(len(self.ast)): if is_docstring(self.ast[i]): docstring_ast = SyntaxTree( "docstring", [ Token( "LOAD_STR", has_arg=True, offset=0, pattr=self.ast[i][0][0][0][0].attr, ) ], transformed_by="transform", ) del self.ast[i] self.ast.insert(0, docstring_ast) break if self.ast[-1] == RETURN_NONE: self.ast.pop() # remove last node # todo: if empty, add 'pass' except: pass return self.ast
def n_stmts(self, node): if node.first_child() == "SETUP_ANNOTATIONS": prev = node[0] new_stmts = [node[0]] for i, sstmt in enumerate(node[1:]): ann_assign = sstmt[0] if ( sstmt[0] == "stmt" and ann_assign == "ann_assign" and prev == "assign" ): annotate_var = ann_assign[-2] if annotate_var.attr == prev[-1][0].attr: del new_stmts[-1] sstmt[0][0] = SyntaxTree( "ann_assign_init", [ann_assign[0], prev[0], annotate_var], transformed_by="n_stmts", ) pass pass new_stmts.append(sstmt) prev = ann_assign pass node.data = new_stmts return node
def transform(self, ast, code): self.maybe_show_tree(ast) self.ast = copy(ast) self.ast = self.traverse(self.ast, is_lambda=False) n = len(self.ast) try: # Disambiguate a string (expression) which appears as a "call_stmt" at # the beginning of a function versus a docstring. Seems pretty academic, # but this is Python. call_stmt = ast[0][0] if is_not_docstring(call_stmt): call_stmt.kind = "string_at_beginning" call_stmt.transformed_by = "transform" pass except: pass try: for i in range(n): if is_docstring(self.ast[i], self.version, code.co_consts): load_const = self.ast[i].first_child() docstring_ast = SyntaxTree( "docstring", [ Token( "LOAD_STR", has_arg=True, offset=0, attr=load_const.attr, pattr=load_const.pattr, ) ], transformed_by="transform", ) del self.ast[i] self.ast.insert(0, docstring_ast) break if self.ast[-1] == RETURN_NONE: self.ast.pop() # remove last node # todo: if empty, add 'pass' except: pass return self.ast
"dict": 0, # {expressions...} "dict_comp": 0, "generator_exp": 0, # (expressions...) "list": 0, # [expressions...] "list_comp": 0, "set_comp": 0, "set_comp_expr": 0, "unary_convert": 0, } LINE_LENGTH = 80 # Some parse trees created below are used for comparing code # fragments (like "return None" at the end of functions). NONE = SyntaxTree("expr", [NoneToken]) RETURN_NONE = SyntaxTree( "stmt", [SyntaxTree("return", [NONE, Token("RETURN_VALUE")])]) PASS = SyntaxTree( "stmts", [SyntaxTree("sstmt", [SyntaxTree("stmt", [SyntaxTree("pass", [])])])]) ASSIGN_DOC_STRING = lambda doc_string: \ SyntaxTree("stmt", [ SyntaxTree("assign", [ SyntaxTree("expr", [ Token("LOAD_STR", pattr=doc_string) ]), SyntaxTree("store", [ Token("STORE_NAME", pattr="__doc__")]) ])])
def n_ifelsestmt(self, node, preprocess=False): """ Transformation involving if..else statments. For example if ... else if .. into: if .. elif ... [else ...] where appropriate. """ else_suite = node[3] n = else_suite[0] old_stmts = None else_suite_index = 1 if len(n) and n[0] == "suite_stmts": n = n[0] len_n = len(n) if len_n == 1 == len(n[0]) and n[0] in ("stmt", "stmts"): n = n[0][0] elif len_n == 0: return node elif n[0].kind in ("lastc_stmt", ): n = n[0] if n[0].kind in ( "ifstmt", "iflaststmt", "iflaststmtc", "ifelsestmtc", "ifpoplaststmtc", ): n = n[0] if n.kind == "ifpoplaststmtc": old_stmts = n[2] else_suite_index = 2 pass pass else: if (len_n > 1 and isinstance(n[0], SyntaxTree) and 1 == len(n[0]) and n[0] == "stmt" and n[1].kind == "stmt"): else_suite_stmts = n[0] elif len_n == 1: else_suite_stmts = n else: return node if else_suite_stmts[0].kind in ( "ifstmt", "iflaststmt", "ifelsestmt", "ifelsestmtl", ): old_stmts = n n = else_suite_stmts[0] else: return node if n.kind == "last_stmt": n = n[0] if n.kind in ("ifstmt", "iflaststmt", "iflaststmtc", "ifpoplaststmtc"): node.kind = "ifelifstmt" n.kind = "elifstmt" elif n.kind in ("ifelsestmtr", ): node.kind = "ifelifstmt" n.kind = "elifelsestmtr" elif n.kind in ("ifelsestmt", "ifelsestmtc", "ifelsestmtc"): node.kind = "ifelifstmt" self.n_ifelsestmt(n, preprocess=True) if n == "ifelifstmt": n.kind = "elifelifstmt" elif n.kind in ("ifelsestmt", "ifelsestmtc"): n.kind = "elifelsestmt" if not preprocess: if old_stmts: if n.kind == "elifstmt": trailing_else = SyntaxTree("stmts", old_stmts[1:]) if len(trailing_else): # We use elifelsestmtr because it has 3 nodes elifelse_stmt = SyntaxTree( "elifelsestmtr", [n[0], n[else_suite_index], trailing_else]) node[3] = elifelse_stmt else: elif_stmt = SyntaxTree("elifstmt", [n[0], n[else_suite_index]]) node[3] = elif_stmt node.transformed_by = "n_ifelsestmt" pass else: # Other cases for n.kind may happen here pass pass return node
def n_ifstmt(self, node): """Here we check if we can turn an `ifstmt` or 'iflaststmtc` into some kind of `assert` statement""" testexpr = node[0] if testexpr not in ("testexpr", "testexprc"): return node if node.kind in ("ifstmt", "ifstmtc"): ifstmts_jump = node[1] if ifstmts_jump == "ifstmts_jumpc" and ifstmts_jump[ 0] == "ifstmts_jump": ifstmts_jump = ifstmts_jump[0] elif ifstmts_jump not in ("ifstmts_jump", "ifstmts_jumpc"): return node stmts = ifstmts_jump[0] else: # iflaststmt{c,} works this way stmts = node[1] if stmts in ("c_stmts", "stmts", "stmts_opt") and len(stmts) == 1: raise_stmt = stmts[0] if raise_stmt != "raise_stmt1" and len(raise_stmt) > 0: raise_stmt = raise_stmt[0] testtrue_or_false = testexpr[0] if testtrue_or_false == "testexpr": testtrue_or_false = testtrue_or_false[0] if (raise_stmt == "raise_stmt1" and 1 <= len(testtrue_or_false) <= 2 and raise_stmt.first_child().pattr == "AssertionError"): if testtrue_or_false in ("testtrue", "testtruec"): # Skip over the testtrue because because it would # produce a "not" and we don't want that here. assert_expr = testtrue_or_false[0] jump_cond = NoneToken else: assert testtrue_or_false in ( "testfalse", "testfalsec", ), testtrue_or_false assert_expr = testtrue_or_false[0] if assert_expr in ("and_not", "nand", "not_or", "and"): # FIXME: come back to stuff like this return node if testtrue_or_false[0] == "expr_pjif": jump_cond = testtrue_or_false[0][1] else: jump_cond = testtrue_or_false[1] assert_expr.kind = "assert_expr" pass expr = raise_stmt[0] RAISE_VARARGS_1 = raise_stmt[1] call = expr[0] if call == "call": # ifstmt # 0. testexpr # testtrue (2) # 0. expr # 1. _ifstmts_jump (2) # 0. c_stmts # stmt # raise_stmt1 (2) # 0. expr # call (3) # 1. RAISE_VARARGS_1 # becomes: # assert2 ::= assert_expr POP_JUMP_IF_TRUE LOAD_ASSERT expr RAISE_VARARGS_1 COME_FROM if jump_cond in ("POP_JUMP_IF_TRUE", NoneToken): kind = "assert2" else: if jump_cond == "POP_JUMP_IF_FALSE": # FIXME: We don't handle this kind of thing yet. return node kind = "assert2not" LOAD_ASSERT = call[0].first_child() if LOAD_ASSERT not in ("LOAD_ASSERT", "LOAD_GLOBAL"): return node if isinstance(call[1], SyntaxTree): expr = call[1][0] node = SyntaxTree( kind, [ assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1, ], ) pass pass else: # ifstmt # 0. testexpr (2) # testtrue # 0. expr # 1. _ifstmts_jump (2) # 0. c_stmts # stmts # raise_stmt1 (2) # 0. expr # LOAD_ASSERT # 1. RAISE_VARARGS_1 # becomes: # assert ::= assert_expr POP_JUMP_IF_TRUE LOAD_ASSERT RAISE_VARARGS_1 COME_FROM if jump_cond in ( "POP_JUMP_IF_TRUE", "POP_JUMP_IF_TRUE_BACK", NoneToken, ): kind = "assert" else: assert jump_cond.kind.startswith("POP_JUMP_IF_") kind = "assertnot" LOAD_ASSERT = expr[0] node = SyntaxTree( kind, [assert_expr, jump_cond, LOAD_ASSERT, RAISE_VARARGS_1], transformed_by="n_ifstmt", ) pass pass return node
'dict_comp': 0, 'generator_exp': 0, # (expressions...) 'list': 0, # [expressions...] 'list_comp': 0, 'set_comp': 0, 'set_comp_expr': 0, 'unary_convert': 0, } LINE_LENGTH = 80 # Some parse trees created below are used for comparing code # fragments (like 'return None' at the end of functions). RETURN_LOCALS = SyntaxTree('return', [ SyntaxTree('ret_expr', [SyntaxTree('expr', [Token('LOAD_LOCALS')])]), Token('RETURN_VALUE') ]) NONE = SyntaxTree('expr', [NoneToken]) RETURN_NONE = SyntaxTree( 'stmt', [SyntaxTree('return', [NONE, Token('RETURN_VALUE')])]) PASS = SyntaxTree( 'stmts', [SyntaxTree('sstmt', [SyntaxTree('stmt', [SyntaxTree('pass', [])])])]) ASSIGN_DOC_STRING = lambda doc_string: \ SyntaxTree('stmt', [ SyntaxTree('assign', [ SyntaxTree('expr', [ Token('LOAD_STR', pattr=doc_string) ]),
def n_ifstmt(self, node): """Here we check if we can turn an `ifstmt` or 'iflaststmtl` into some kind of `assert` statement""" testexpr = node[0] if testexpr != "testexpr": return node if node.kind in ("ifstmt", "ifstmtl"): ifstmts_jump = node[1] if ifstmts_jump == "_ifstmts_jumpl" and ifstmts_jump[0] == "_ifstmts_jump": ifstmts_jump = ifstmts_jump[0] elif ifstmts_jump not in ("_ifstmts_jump", "ifstmts_jumpl"): return node stmts = ifstmts_jump[0] else: # iflaststmtl works this way stmts = node[1] if stmts in ("c_stmts",) and len(stmts) == 1: stmt = stmts[0] raise_stmt = stmt[0] if ( raise_stmt == "raise_stmt1" and len(testexpr[0]) == 2 and raise_stmt.first_child().pattr == "AssertionError" ): assert_expr = testexpr[0][0] assert_expr.kind = "assert_expr" jump_cond = testexpr[0][1] expr = raise_stmt[0] RAISE_VARARGS_1 = raise_stmt[1] call = expr[0] if call == "call": # ifstmt # 0. testexpr # testtrue (2) # 0. expr # 1. _ifstmts_jump (2) # 0. c_stmts # stmt # raise_stmt1 (2) # 0. expr # call (3) # 1. RAISE_VARARGS_1 # becomes: # assert2 ::= assert_expr jmp_true LOAD_ASSERT expr RAISE_VARARGS_1 COME_FROM if jump_cond == "jmp_true": kind = "assert2" else: assert jump_cond == "jmp_false" kind = "assert2not" LOAD_ASSERT = call[0].first_child() if LOAD_ASSERT != "LOAD_ASSERT": return node expr = call[1][0] node = SyntaxTree( kind, [assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1], transformed_by="n_ifstmt", ) else: # ifstmt # 0. testexpr (2) # testtrue # 0. expr # 1. _ifstmts_jump (2) # 0. c_stmts # stmts # raise_stmt1 (2) # 0. expr # LOAD_ASSERT # 1. RAISE_VARARGS_1 # becomes: # assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM if jump_cond == "jmp_true": kind = "assert" else: assert jump_cond == "jmp_false" kind = "assertnot" LOAD_ASSERT = expr[0] node = SyntaxTree( kind, [assert_expr, jump_cond, LOAD_ASSERT, RAISE_VARARGS_1], transformed_by="n_ifstmt", ) pass pass return node
def n_ifstmt(self, node): """Here we check if we can turn an `ifstmt` or 'iflaststmtl` into some kind of `assert` statement""" testexpr = node[0] if testexpr != "testexpr": return node if node.kind == "ifstmt": ifstmts_jump = node[1] if node[1] != "_ifstmts_jump": return node stmts = ifstmts_jump[0] else: # iflaststmtl works this way stmts = node[1] if stmts in ("c_stmts", ) and len(stmts) == 1: stmt = stmts[0] raise_stmt = stmt[0] if raise_stmt == "raise_stmt1" and len(testexpr[0]) == 2: assert_expr = testexpr[0][0] assert_expr.kind = "assert_expr" jmp_true = testexpr[0][1] expr = raise_stmt[0] RAISE_VARARGS_1 = raise_stmt[1] if expr[0] == "call": # ifstmt # 0. testexpr # testtrue (2) # 0. expr # 1. _ifstmts_jump (2) # 0. c_stmts # stmt # raise_stmt1 (2) # 0. expr # call (3) # 1. RAISE_VARARGS_1 # becomes: # assert2 ::= assert_expr jmp_true LOAD_ASSERT expr RAISE_VARARGS_1 COME_FROM call = expr[0] LOAD_ASSERT = call[0] expr = call[1][0] node = SyntaxTree( "assert2", [ assert_expr, jmp_true, LOAD_ASSERT, expr, RAISE_VARARGS_1 ], transformed_by="n_ifstmt", ) else: # ifstmt # 0. testexpr (2) # testtrue # 0. expr # 1. _ifstmts_jump (2) # 0. c_stmts # stmts # raise_stmt1 (2) # 0. expr # LOAD_ASSERT # 1. RAISE_VARARGS_1 # becomes: # assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM LOAD_ASSERT = expr[0] node = SyntaxTree( "assert", [assert_expr, jmp_true, LOAD_ASSERT, RAISE_VARARGS_1], transformed_by="n_ifstmt", ) pass pass return node
def n_ifelsestmt(self, node, preprocess=False): """ Here we turn: if ... else if .. into: if .. elif ... [else ...] where appropriate """ else_suite = node[3] n = else_suite[0] old_stmts = None if len(n) == 1 == len(n[0]) and n[0] == "stmt": n = n[0][0] elif n[0].kind in ("lastc_stmt", "lastl_stmt"): n = n[0] if n[0].kind in ( "ifstmt", "iflaststmt", "iflaststmtl", "ifelsestmtl", "ifelsestmtc", ): # This seems needed for Python 2.5-2.7 n = n[0] pass pass elif len(n) > 1 and 1 == len( n[0]) and n[0] == "stmt" and n[1].kind == "stmt": else_suite_stmts = n[0] if else_suite_stmts[0].kind not in ("ifstmt", "iflaststmt", "ifelsestmtl"): return node old_stmts = n n = else_suite_stmts[0] else: return node if n.kind in ("ifstmt", "iflaststmt", "iflaststmtl"): node.kind = "ifelifstmt" n.kind = "elifstmt" elif n.kind in ("ifelsestmtr", ): node.kind = "ifelifstmt" n.kind = "elifelsestmtr" elif n.kind in ("ifelsestmt", "ifelsestmtc", "ifelsestmtl"): node.kind = "ifelifstmt" self.n_ifelsestmt(n, preprocess=True) if n == "ifelifstmt": n.kind = "elifelifstmt" elif n.kind in ("ifelsestmt", "ifelsestmtc", "ifelsestmtl"): n.kind = "elifelsestmt" if not preprocess: if old_stmts: if n.kind == "elifstmt": trailing_else = SyntaxTree("stmts", old_stmts[1:]) # We use elifelsestmtr because it has 3 nodes elifelse_stmt = SyntaxTree("elifelsestmtr", [n[0], n[1], trailing_else]) node[3] = elifelse_stmt pass else: # Other cases for n.kind may happen here pass pass node.transformed_by = "n_ifelsestmt" return node