def n_mkfunc(self, node):
"""If the function has a docstring (this is found in the code
constants), pull that out and make it part of the syntax
tree. When generating the source string that AST node rather
than the code field is seen and used.
"""
code = find_code_node(node, -3).attr
if (
node[-1].pattr != "closure"
and len(code.co_consts) > 0
and code.co_consts[0] is not None
):
docstring_node = SyntaxTree(
"docstring",
[Token("LOAD_STR", has_arg=True, pattr=code.co_consts[0])],
transformed_by="n_mkfunc",
)
node = SyntaxTree(
"mkfunc",
node[:-1] + [docstring_node, node[-1]],
transformed_by="n_mkfunc",
)
return node
def n_mkfunc(self, node):
"""If the function has a docstring (this is found in the code
constants), pull that out and make it part of the syntax
tree. When generating the source string that AST node rather
than the code field is seen and used.
"""
code = find_code_node(node, -3).attr
mkfunc_pattr = node[-1].pattr
if isinstance(mkfunc_pattr, tuple):
assert isinstance(mkfunc_pattr, tuple)
assert len(mkfunc_pattr, 4) and isinstance(mkfunc_pattr, int)
is_closure = node[-1].pattr[3] != 0
else:
# FIXME: This is what we had before. It is hoaky and probably wrong.
is_closure = mkfunc_pattr == "closure"
if ((not is_closure) and len(code.co_consts) > 0
and isinstance(code.co_consts[0], str)):
docstring_node = SyntaxTree(
"docstring",
[Token("LOAD_STR", has_arg=True, pattr=code.co_consts[0])],
transformed_by="n_mkfunc",
)
node = SyntaxTree(
"mkfunc",
node[:-1] + [docstring_node, node[-1]],
transformed_by="n_mkfunc",
)
return node
def transform(self, ast):
self.maybe_show_tree(ast)
self.ast = copy(ast)
self.ast = self.traverse(self.ast, is_lambda=False)
try:
for i in range(len(self.ast)):
if is_docstring(self.ast[i]):
docstring_ast = SyntaxTree(
"docstring",
[
Token(
"LOAD_STR",
has_arg=True,
offset=0,
pattr=self.ast[i][0][0][0][0].attr,
)
],
transformed_by="transform",
)
del self.ast[i]
self.ast.insert(0, docstring_ast)
break
if self.ast[-1] == RETURN_NONE:
self.ast.pop() # remove last node
# todo: if empty, add 'pass'
except:
pass
return self.ast
def test_token():
# Test token formatting of: LOAD_CONST None
t = Token("LOAD_CONST", offset=0, attr=None, pattr=None, has_arg=True)
expect = " 0 LOAD_CONST None"
# print(t.format())
assert t
assert t.format() == expect
# Make sure equality testing of tokens ignores offset
t2 = Token("LOAD_CONST", offset=2, attr=None, pattr=None, has_arg=True)
assert t2 == t
# Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
# of co_consts.
t = Token("LOAD_CONST", offset=1, attr=False, pattr=False, has_arg=True)
expect = " 1 LOAD_CONST False"
assert t.format() == expect, t.format()
# Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
expect = "(005) 1 LOAD_CONST False"
assert t.format(token_num=5) == expect, t.format(token_num=5)
def transform(self, ast, code):
self.maybe_show_tree(ast)
self.ast = copy(ast)
self.ast = self.traverse(self.ast, is_lambda=False)
n = len(self.ast)
try:
# Disambiguate a string (expression) which appears as a "call_stmt" at
# the beginning of a function versus a docstring. Seems pretty academic,
# but this is Python.
call_stmt = ast[0][0]
if is_not_docstring(call_stmt):
call_stmt.kind = "string_at_beginning"
call_stmt.transformed_by = "transform"
pass
except:
pass
try:
for i in range(n):
if is_docstring(self.ast[i], self.version, code.co_consts):
load_const = self.ast[i].first_child()
docstring_ast = SyntaxTree(
"docstring",
[
Token(
"LOAD_STR",
has_arg=True,
offset=0,
attr=load_const.attr,
pattr=load_const.pattr,
)
],
transformed_by="transform",
)
del self.ast[i]
self.ast.insert(0, docstring_ast)
break
if self.ast[-1] == RETURN_NONE:
self.ast.pop() # remove last node
# todo: if empty, add 'pass'
except:
pass
return self.ast
def test_token():
# Test token formatting of: LOAD_CONST None
t = Token('LOAD_CONST', offset=0, attr=None, pattr=None, has_arg=True)
expect = ' 0 LOAD_CONST None'
# print(t.format())
assert t
assert t.format() == expect
# Make sure equality testing of tokens ignores offset
t2 = Token('LOAD_CONST', offset=2, attr=None, pattr=None, has_arg=True)
assert t2 == t
# Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
# of co_consts.
t = Token('LOAD_CONST', offset=1, attr=False, pattr=False, has_arg=True)
expect = ' 1 LOAD_CONST 0 False'
assert t.format() == expect
"list": 0, # [expressions...]
"list_comp": 0,
"set_comp": 0,
"set_comp_expr": 0,
"unary_convert": 0,
}
LINE_LENGTH = 80
# Some parse trees created below are used for comparing code
# fragments (like "return None" at the end of functions).
NONE = SyntaxTree("expr", [NoneToken])
RETURN_NONE = SyntaxTree(
"stmt", [SyntaxTree("return", [NONE, Token("RETURN_VALUE")])])
PASS = SyntaxTree(
"stmts",
[SyntaxTree("sstmt", [SyntaxTree("stmt", [SyntaxTree("pass", [])])])])
ASSIGN_DOC_STRING = lambda doc_string: \
SyntaxTree("stmt",
[ SyntaxTree("assign",
[ SyntaxTree("expr", [ Token("LOAD_STR", pattr=doc_string) ]),
SyntaxTree("store", [ Token("STORE_NAME", pattr="__doc__")])
])])
NAME_MODULE = SyntaxTree("assign", [
SyntaxTree("expr",
[Token("LOAD_NAME", pattr="__name__", offset=0, has_arg=True)]),
'dict_comp': 0,
'generator_exp': 0, # (expressions...)
'list': 0, # [expressions...]
'list_comp': 0,
'set_comp': 0,
'set_comp_expr': 0,
'unary_convert': 0,
}
LINE_LENGTH = 80
# Some parse trees created below are used for comparing code
# fragments (like 'return None' at the end of functions).
RETURN_LOCALS = SyntaxTree('return', [
SyntaxTree('ret_expr', [SyntaxTree('expr', [Token('LOAD_LOCALS')])]),
Token('RETURN_VALUE')
])
NONE = SyntaxTree('expr', [NoneToken])
RETURN_NONE = SyntaxTree(
'stmt', [SyntaxTree('return', [NONE, Token('RETURN_VALUE')])])
PASS = SyntaxTree(
'stmts',
[SyntaxTree('sstmt', [SyntaxTree('stmt', [SyntaxTree('pass', [])])])])
ASSIGN_DOC_STRING = lambda doc_string: \
SyntaxTree('stmt',
[ SyntaxTree('assign',