def n_mkfunc(self, node):
"""If the function has a docstring (this is found in the code
constants), pull that out and make it part of the syntax
tree. When generating the source string that AST node rather
than the code field is seen and used.
"""
code = find_code_node(node, -3).attr
mkfunc_pattr = node[-1].pattr
if isinstance(mkfunc_pattr, tuple):
assert isinstance(mkfunc_pattr, tuple)
assert len(mkfunc_pattr, 4) and isinstance(mkfunc_pattr, int)
is_closure = node[-1].pattr[3] != 0
else:
# FIXME: This is what we had before. It is hoaky and probably wrong.
is_closure = mkfunc_pattr == "closure"
if ((not is_closure) and len(code.co_consts) > 0
and isinstance(code.co_consts[0], str)):
docstring_node = SyntaxTree(
"docstring",
[Token("LOAD_STR", has_arg=True, pattr=code.co_consts[0])],
transformed_by="n_mkfunc",
)
node = SyntaxTree(
"mkfunc",
node[:-1] + [docstring_node, node[-1]],
transformed_by="n_mkfunc",
)
return node
def n_mkfunc(self, node):
"""If the function has a docstring (this is found in the code
constants), pull that out and make it part of the syntax
tree. When generating the source string that AST node rather
than the code field is seen and used.
"""
code = find_code_node(node, -3).attr
if (
node[-1].pattr != "closure"
and len(code.co_consts) > 0
and code.co_consts[0] is not None
):
docstring_node = SyntaxTree(
"docstring",
[Token("LOAD_STR", has_arg=True, pattr=code.co_consts[0])],
transformed_by="n_mkfunc",
)
node = SyntaxTree(
"mkfunc",
node[:-1] + [docstring_node, node[-1]],
transformed_by="n_mkfunc",
)
return node
def n_import_from37(self, node):
importlist37 = node[3]
assert importlist37 == "importlist37"
if len(importlist37) == 1:
alias37 = importlist37[0]
store = alias37[1]
assert store == "store"
alias_name = store[0].attr
import_name_attr = node[2]
assert import_name_attr == "IMPORT_NAME_ATTR"
dotted_names = import_name_attr.attr.split(".")
if len(dotted_names) > 1 and dotted_names[-1] == alias_name:
# Simulate:
# Instead of
# import_from37 ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR importlist37 POP_TOP
# import_as37 ::= LOAD_CONST LOAD_CONST importlist37 store POP_TOP
# 'import_as37': ( '%|import %c as %c\n', 2, -2),
node = SyntaxTree(
"import_as37",
[node[0], node[1], import_name_attr, store, node[-1]],
transformed_by="n_import_from37",
)
pass
pass
return node
def transform(self, ast):
self.maybe_show_tree(ast)
self.ast = copy(ast)
self.ast = self.traverse(self.ast, is_lambda=False)
try:
for i in range(len(self.ast)):
if is_docstring(self.ast[i]):
docstring_ast = SyntaxTree(
"docstring",
[
Token(
"LOAD_STR",
has_arg=True,
offset=0,
pattr=self.ast[i][0][0][0][0].attr,
)
],
transformed_by="transform",
)
del self.ast[i]
self.ast.insert(0, docstring_ast)
break
if self.ast[-1] == RETURN_NONE:
self.ast.pop() # remove last node
# todo: if empty, add 'pass'
except:
pass
return self.ast
def n_stmts(self, node):
if node.first_child() == "SETUP_ANNOTATIONS":
prev = node[0]
new_stmts = [node[0]]
for i, sstmt in enumerate(node[1:]):
ann_assign = sstmt[0]
if (
sstmt[0] == "stmt"
and ann_assign == "ann_assign"
and prev == "assign"
):
annotate_var = ann_assign[-2]
if annotate_var.attr == prev[-1][0].attr:
del new_stmts[-1]
sstmt[0][0] = SyntaxTree(
"ann_assign_init",
[ann_assign[0], prev[0], annotate_var],
transformed_by="n_stmts",
)
pass
pass
new_stmts.append(sstmt)
prev = ann_assign
pass
node.data = new_stmts
return node
def transform(self, ast, code):
self.maybe_show_tree(ast)
self.ast = copy(ast)
self.ast = self.traverse(self.ast, is_lambda=False)
n = len(self.ast)
try:
# Disambiguate a string (expression) which appears as a "call_stmt" at
# the beginning of a function versus a docstring. Seems pretty academic,
# but this is Python.
call_stmt = ast[0][0]
if is_not_docstring(call_stmt):
call_stmt.kind = "string_at_beginning"
call_stmt.transformed_by = "transform"
pass
except:
pass
try:
for i in range(n):
if is_docstring(self.ast[i], self.version, code.co_consts):
load_const = self.ast[i].first_child()
docstring_ast = SyntaxTree(
"docstring",
[
Token(
"LOAD_STR",
has_arg=True,
offset=0,
attr=load_const.attr,
pattr=load_const.pattr,
)
],
transformed_by="transform",
)
del self.ast[i]
self.ast.insert(0, docstring_ast)
break
if self.ast[-1] == RETURN_NONE:
self.ast.pop() # remove last node
# todo: if empty, add 'pass'
except:
pass
return self.ast
"dict": 0, # {expressions...}
"dict_comp": 0,
"generator_exp": 0, # (expressions...)
"list": 0, # [expressions...]
"list_comp": 0,
"set_comp": 0,
"set_comp_expr": 0,
"unary_convert": 0,
}
LINE_LENGTH = 80
# Some parse trees created below are used for comparing code
# fragments (like "return None" at the end of functions).
NONE = SyntaxTree("expr", [NoneToken])
RETURN_NONE = SyntaxTree(
"stmt", [SyntaxTree("return", [NONE, Token("RETURN_VALUE")])])
PASS = SyntaxTree(
"stmts",
[SyntaxTree("sstmt", [SyntaxTree("stmt", [SyntaxTree("pass", [])])])])
ASSIGN_DOC_STRING = lambda doc_string: \
SyntaxTree("stmt",
[ SyntaxTree("assign",
[ SyntaxTree("expr", [ Token("LOAD_STR", pattr=doc_string) ]),
SyntaxTree("store", [ Token("STORE_NAME", pattr="__doc__")])
])])
def n_ifelsestmt(self, node, preprocess=False):
"""
Transformation involving if..else statments.
For example
if ...
else
if ..
into:
if ..
elif ...
[else ...]
where appropriate.
"""
else_suite = node[3]
n = else_suite[0]
old_stmts = None
else_suite_index = 1
if len(n) and n[0] == "suite_stmts":
n = n[0]
len_n = len(n)
if len_n == 1 == len(n[0]) and n[0] in ("stmt", "stmts"):
n = n[0][0]
elif len_n == 0:
return node
elif n[0].kind in ("lastc_stmt", ):
n = n[0]
if n[0].kind in (
"ifstmt",
"iflaststmt",
"iflaststmtc",
"ifelsestmtc",
"ifpoplaststmtc",
):
n = n[0]
if n.kind == "ifpoplaststmtc":
old_stmts = n[2]
else_suite_index = 2
pass
pass
else:
if (len_n > 1 and isinstance(n[0], SyntaxTree) and 1 == len(n[0])
and n[0] == "stmt" and n[1].kind == "stmt"):
else_suite_stmts = n[0]
elif len_n == 1:
else_suite_stmts = n
else:
return node
if else_suite_stmts[0].kind in (
"ifstmt",
"iflaststmt",
"ifelsestmt",
"ifelsestmtl",
):
old_stmts = n
n = else_suite_stmts[0]
else:
return node
if n.kind == "last_stmt":
n = n[0]
if n.kind in ("ifstmt", "iflaststmt", "iflaststmtc", "ifpoplaststmtc"):
node.kind = "ifelifstmt"
n.kind = "elifstmt"
elif n.kind in ("ifelsestmtr", ):
node.kind = "ifelifstmt"
n.kind = "elifelsestmtr"
elif n.kind in ("ifelsestmt", "ifelsestmtc", "ifelsestmtc"):
node.kind = "ifelifstmt"
self.n_ifelsestmt(n, preprocess=True)
if n == "ifelifstmt":
n.kind = "elifelifstmt"
elif n.kind in ("ifelsestmt", "ifelsestmtc"):
n.kind = "elifelsestmt"
if not preprocess:
if old_stmts:
if n.kind == "elifstmt":
trailing_else = SyntaxTree("stmts", old_stmts[1:])
if len(trailing_else):
# We use elifelsestmtr because it has 3 nodes
elifelse_stmt = SyntaxTree(
"elifelsestmtr",
[n[0], n[else_suite_index], trailing_else])
node[3] = elifelse_stmt
else:
elif_stmt = SyntaxTree("elifstmt",
[n[0], n[else_suite_index]])
node[3] = elif_stmt
node.transformed_by = "n_ifelsestmt"
pass
else:
# Other cases for n.kind may happen here
pass
pass
return node
def n_ifstmt(self, node):
"""Here we check if we can turn an `ifstmt` or 'iflaststmtc` into
some kind of `assert` statement"""
testexpr = node[0]
if testexpr not in ("testexpr", "testexprc"):
return node
if node.kind in ("ifstmt", "ifstmtc"):
ifstmts_jump = node[1]
if ifstmts_jump == "ifstmts_jumpc" and ifstmts_jump[
0] == "ifstmts_jump":
ifstmts_jump = ifstmts_jump[0]
elif ifstmts_jump not in ("ifstmts_jump", "ifstmts_jumpc"):
return node
stmts = ifstmts_jump[0]
else:
# iflaststmt{c,} works this way
stmts = node[1]
if stmts in ("c_stmts", "stmts", "stmts_opt") and len(stmts) == 1:
raise_stmt = stmts[0]
if raise_stmt != "raise_stmt1" and len(raise_stmt) > 0:
raise_stmt = raise_stmt[0]
testtrue_or_false = testexpr[0]
if testtrue_or_false == "testexpr":
testtrue_or_false = testtrue_or_false[0]
if (raise_stmt == "raise_stmt1"
and 1 <= len(testtrue_or_false) <= 2
and raise_stmt.first_child().pattr == "AssertionError"):
if testtrue_or_false in ("testtrue", "testtruec"):
# Skip over the testtrue because because it would
# produce a "not" and we don't want that here.
assert_expr = testtrue_or_false[0]
jump_cond = NoneToken
else:
assert testtrue_or_false in (
"testfalse",
"testfalsec",
), testtrue_or_false
assert_expr = testtrue_or_false[0]
if assert_expr in ("and_not", "nand", "not_or", "and"):
# FIXME: come back to stuff like this
return node
if testtrue_or_false[0] == "expr_pjif":
jump_cond = testtrue_or_false[0][1]
else:
jump_cond = testtrue_or_false[1]
assert_expr.kind = "assert_expr"
pass
expr = raise_stmt[0]
RAISE_VARARGS_1 = raise_stmt[1]
call = expr[0]
if call == "call":
# ifstmt
# 0. testexpr
# testtrue (2)
# 0. expr
# 1. _ifstmts_jump (2)
# 0. c_stmts
# stmt
# raise_stmt1 (2)
# 0. expr
# call (3)
# 1. RAISE_VARARGS_1
# becomes:
# assert2 ::= assert_expr POP_JUMP_IF_TRUE LOAD_ASSERT expr RAISE_VARARGS_1 COME_FROM
if jump_cond in ("POP_JUMP_IF_TRUE", NoneToken):
kind = "assert2"
else:
if jump_cond == "POP_JUMP_IF_FALSE":
# FIXME: We don't handle this kind of thing yet.
return node
kind = "assert2not"
LOAD_ASSERT = call[0].first_child()
if LOAD_ASSERT not in ("LOAD_ASSERT", "LOAD_GLOBAL"):
return node
if isinstance(call[1], SyntaxTree):
expr = call[1][0]
node = SyntaxTree(
kind,
[
assert_expr,
jump_cond,
LOAD_ASSERT,
expr,
RAISE_VARARGS_1,
],
)
pass
pass
else:
# ifstmt
# 0. testexpr (2)
# testtrue
# 0. expr
# 1. _ifstmts_jump (2)
# 0. c_stmts
# stmts
# raise_stmt1 (2)
# 0. expr
# LOAD_ASSERT
# 1. RAISE_VARARGS_1
# becomes:
# assert ::= assert_expr POP_JUMP_IF_TRUE LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
if jump_cond in (
"POP_JUMP_IF_TRUE",
"POP_JUMP_IF_TRUE_BACK",
NoneToken,
):
kind = "assert"
else:
assert jump_cond.kind.startswith("POP_JUMP_IF_")
kind = "assertnot"
LOAD_ASSERT = expr[0]
node = SyntaxTree(
kind,
[assert_expr, jump_cond, LOAD_ASSERT, RAISE_VARARGS_1],
transformed_by="n_ifstmt",
)
pass
pass
return node
'dict_comp': 0,
'generator_exp': 0, # (expressions...)
'list': 0, # [expressions...]
'list_comp': 0,
'set_comp': 0,
'set_comp_expr': 0,
'unary_convert': 0,
}
LINE_LENGTH = 80
# Some parse trees created below are used for comparing code
# fragments (like 'return None' at the end of functions).
RETURN_LOCALS = SyntaxTree('return', [
SyntaxTree('ret_expr', [SyntaxTree('expr', [Token('LOAD_LOCALS')])]),
Token('RETURN_VALUE')
])
NONE = SyntaxTree('expr', [NoneToken])
RETURN_NONE = SyntaxTree(
'stmt', [SyntaxTree('return', [NONE, Token('RETURN_VALUE')])])
PASS = SyntaxTree(
'stmts',
[SyntaxTree('sstmt', [SyntaxTree('stmt', [SyntaxTree('pass', [])])])])
ASSIGN_DOC_STRING = lambda doc_string: \
SyntaxTree('stmt',
[ SyntaxTree('assign',
[ SyntaxTree('expr', [ Token('LOAD_STR', pattr=doc_string) ]),
def n_ifstmt(self, node):
"""Here we check if we can turn an `ifstmt` or 'iflaststmtl` into
some kind of `assert` statement"""
testexpr = node[0]
if testexpr != "testexpr":
return node
if node.kind in ("ifstmt", "ifstmtl"):
ifstmts_jump = node[1]
if ifstmts_jump == "_ifstmts_jumpl" and ifstmts_jump[0] == "_ifstmts_jump":
ifstmts_jump = ifstmts_jump[0]
elif ifstmts_jump not in ("_ifstmts_jump", "ifstmts_jumpl"):
return node
stmts = ifstmts_jump[0]
else:
# iflaststmtl works this way
stmts = node[1]
if stmts in ("c_stmts",) and len(stmts) == 1:
stmt = stmts[0]
raise_stmt = stmt[0]
if (
raise_stmt == "raise_stmt1"
and len(testexpr[0]) == 2
and raise_stmt.first_child().pattr == "AssertionError"
):
assert_expr = testexpr[0][0]
assert_expr.kind = "assert_expr"
jump_cond = testexpr[0][1]
expr = raise_stmt[0]
RAISE_VARARGS_1 = raise_stmt[1]
call = expr[0]
if call == "call":
# ifstmt
# 0. testexpr
# testtrue (2)
# 0. expr
# 1. _ifstmts_jump (2)
# 0. c_stmts
# stmt
# raise_stmt1 (2)
# 0. expr
# call (3)
# 1. RAISE_VARARGS_1
# becomes:
# assert2 ::= assert_expr jmp_true LOAD_ASSERT expr RAISE_VARARGS_1 COME_FROM
if jump_cond == "jmp_true":
kind = "assert2"
else:
assert jump_cond == "jmp_false"
kind = "assert2not"
LOAD_ASSERT = call[0].first_child()
if LOAD_ASSERT != "LOAD_ASSERT":
return node
expr = call[1][0]
node = SyntaxTree(
kind,
[assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1],
transformed_by="n_ifstmt",
)
else:
# ifstmt
# 0. testexpr (2)
# testtrue
# 0. expr
# 1. _ifstmts_jump (2)
# 0. c_stmts
# stmts
# raise_stmt1 (2)
# 0. expr
# LOAD_ASSERT
# 1. RAISE_VARARGS_1
# becomes:
# assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
if jump_cond == "jmp_true":
kind = "assert"
else:
assert jump_cond == "jmp_false"
kind = "assertnot"
LOAD_ASSERT = expr[0]
node = SyntaxTree(
kind,
[assert_expr, jump_cond, LOAD_ASSERT, RAISE_VARARGS_1],
transformed_by="n_ifstmt",
)
pass
pass
return node
def n_ifstmt(self, node):
"""Here we check if we can turn an `ifstmt` or 'iflaststmtl` into
some kind of `assert` statement"""
testexpr = node[0]
if testexpr != "testexpr":
return node
if node.kind == "ifstmt":
ifstmts_jump = node[1]
if node[1] != "_ifstmts_jump":
return node
stmts = ifstmts_jump[0]
else:
# iflaststmtl works this way
stmts = node[1]
if stmts in ("c_stmts", ) and len(stmts) == 1:
stmt = stmts[0]
raise_stmt = stmt[0]
if raise_stmt == "raise_stmt1" and len(testexpr[0]) == 2:
assert_expr = testexpr[0][0]
assert_expr.kind = "assert_expr"
jmp_true = testexpr[0][1]
expr = raise_stmt[0]
RAISE_VARARGS_1 = raise_stmt[1]
if expr[0] == "call":
# ifstmt
# 0. testexpr
# testtrue (2)
# 0. expr
# 1. _ifstmts_jump (2)
# 0. c_stmts
# stmt
# raise_stmt1 (2)
# 0. expr
# call (3)
# 1. RAISE_VARARGS_1
# becomes:
# assert2 ::= assert_expr jmp_true LOAD_ASSERT expr RAISE_VARARGS_1 COME_FROM
call = expr[0]
LOAD_ASSERT = call[0]
expr = call[1][0]
node = SyntaxTree(
"assert2",
[
assert_expr, jmp_true, LOAD_ASSERT, expr,
RAISE_VARARGS_1
],
transformed_by="n_ifstmt",
)
else:
# ifstmt
# 0. testexpr (2)
# testtrue
# 0. expr
# 1. _ifstmts_jump (2)
# 0. c_stmts
# stmts
# raise_stmt1 (2)
# 0. expr
# LOAD_ASSERT
# 1. RAISE_VARARGS_1
# becomes:
# assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
LOAD_ASSERT = expr[0]
node = SyntaxTree(
"assert",
[assert_expr, jmp_true, LOAD_ASSERT, RAISE_VARARGS_1],
transformed_by="n_ifstmt",
)
pass
pass
return node
def n_ifelsestmt(self, node, preprocess=False):
"""
Here we turn:
if ...
else
if ..
into:
if ..
elif ...
[else ...]
where appropriate
"""
else_suite = node[3]
n = else_suite[0]
old_stmts = None
if len(n) == 1 == len(n[0]) and n[0] == "stmt":
n = n[0][0]
elif n[0].kind in ("lastc_stmt", "lastl_stmt"):
n = n[0]
if n[0].kind in (
"ifstmt",
"iflaststmt",
"iflaststmtl",
"ifelsestmtl",
"ifelsestmtc",
):
# This seems needed for Python 2.5-2.7
n = n[0]
pass
pass
elif len(n) > 1 and 1 == len(
n[0]) and n[0] == "stmt" and n[1].kind == "stmt":
else_suite_stmts = n[0]
if else_suite_stmts[0].kind not in ("ifstmt", "iflaststmt",
"ifelsestmtl"):
return node
old_stmts = n
n = else_suite_stmts[0]
else:
return node
if n.kind in ("ifstmt", "iflaststmt", "iflaststmtl"):
node.kind = "ifelifstmt"
n.kind = "elifstmt"
elif n.kind in ("ifelsestmtr", ):
node.kind = "ifelifstmt"
n.kind = "elifelsestmtr"
elif n.kind in ("ifelsestmt", "ifelsestmtc", "ifelsestmtl"):
node.kind = "ifelifstmt"
self.n_ifelsestmt(n, preprocess=True)
if n == "ifelifstmt":
n.kind = "elifelifstmt"
elif n.kind in ("ifelsestmt", "ifelsestmtc", "ifelsestmtl"):
n.kind = "elifelsestmt"
if not preprocess:
if old_stmts:
if n.kind == "elifstmt":
trailing_else = SyntaxTree("stmts", old_stmts[1:])
# We use elifelsestmtr because it has 3 nodes
elifelse_stmt = SyntaxTree("elifelsestmtr",
[n[0], n[1], trailing_else])
node[3] = elifelse_stmt
pass
else:
# Other cases for n.kind may happen here
pass
pass
node.transformed_by = "n_ifelsestmt"
return node