def print_list_tree(l, blank_nums=0):
blanks = ' ' * blank_nums
name = '' if token.ISNONTERMINAL(l[0]) else l[1]
print('{}--{} {}'.format(blanks, str(l[0]), name))
if token.ISNONTERMINAL(l[0]):
for c in l[1:]:
print_list_tree(c, blank_nums + 1)
def safe_node(self, node):
if token.ISNONTERMINAL(node[0]):
return reduce(operator.and_, map(self.safe_node, node[1:]))
elif node[0] == token.NAME:
return node[1] in self._safe_names
else:
return True
def get_suite_spots(self, tree, spots):
import symbol, token
for i in range(1, len(tree)):
if type(tree[i]) == type(()):
if tree[i][0] == symbol.suite:
# Found a suite, look back for the colon and keyword.
lineno_colon = lineno_word = None
for j in range(i - 1, 0, -1):
if tree[j][0] == token.COLON:
lineno_colon = tree[j][2]
elif tree[j][0] == token.NAME:
if tree[j][1] == 'elif':
# Find the line number of the first non-terminal
# after the keyword.
t = tree[j + 1]
while t and token.ISNONTERMINAL(t[0]):
t = t[1]
if t:
lineno_word = t[2]
else:
lineno_word = tree[j][2]
break
elif tree[j][0] == symbol.except_clause:
# "except" clauses look like:
# ('except_clause', ('NAME', 'except', lineno), ...)
if tree[j][1][0] == token.NAME:
lineno_word = tree[j][1][2]
break
if lineno_colon and lineno_word:
# Found colon and keyword, mark all the lines
# between the two with the two line numbers.
for l in range(lineno_word, lineno_colon + 1):
spots[l] = (lineno_word, lineno_colon)
self.get_suite_spots(tree[i], spots)
def walkCst(ctx, cst):
""" Scan the CST (tuple) for tokens, appending index lines to the buffer.
"""
indent = 0
lineno = 1
stack = [(cst, indent)]
try:
while stack:
cst, indent = stack.pop()
#print("%5d%s%s" % (lineno, " " * indent, nodeNames[cst[0]]))
if token.ISNONTERMINAL(cst[0]):
processNonTerminal(ctx, cst)
else:
lineno = processTerminal(ctx, cst)
indented = False
for i in range(len(cst) - 1, 0, -1):
if type(cst[i]) == tuple:
# Push it onto the processing stack
# Mirrors a recursive solution
if not indented:
indent += 2
indented = True
stack.append((cst[i], indent))
except Exception as e:
e.lineno = lineno
raise e
def walk(root):
for i in range(0, len(root)):
if token.ISNONTERMINAL(root[i]):
if root[i] == symbol.import_stmt:
modules.append(handle_import(root))
return
if type(root[i]) == tuple:
walk(root[i])
def find_statements(self, tree, dict):
import symbol, token
if token.ISNONTERMINAL(tree[0]):
for t in tree[1:]:
self.find_statements(t, dict)
if tree[0] == symbol.stmt:
self.find_statement(tree[1], dict)
elif (tree[0] == token.NAME
and tree[1] in ['elif', 'except', 'finally']):
dict[tree[2]] = 1
def convert_tree(node):
symbol_id = node[0]
symbol_value = node[1]
if token.ISNONTERMINAL(symbol_id):
# non terminal
result = [True, gen_node_id(), symbol.sym_name[symbol_id]]
for i in range(1, len(node)):
result.append(convert_tree(node[i]))
return result
else:
# terminal
return [False, gen_node_id(), token.tok_name[symbol_id], symbol_value]
def get_suite_spots(self, tree, spots):
""" Analyze a parse tree to find suite introducers which span a number
of lines.
"""
for i in range(1, len(tree)):
if type(tree[i]) == type(()):
if tree[i][0] == symbol.suite:
# Found a suite, look back for the colon and keyword.
lineno_colon = lineno_word = None
for j in range(i - 1, 0, -1):
if tree[j][0] == token.COLON:
# Colons are never executed themselves: we want the
# line number of the last token before the colon.
lineno_colon = self.last_line_of_tree(tree[j - 1])
elif tree[j][0] == token.NAME:
if tree[j][1] == 'elif':
# Find the line number of the first non-terminal
# after the keyword.
t = tree[j + 1]
while t and token.ISNONTERMINAL(t[0]):
t = t[1]
if t:
lineno_word = t[2]
else:
lineno_word = tree[j][2]
break
elif tree[j][0] == symbol.except_clause:
# "except" clauses look like:
# ('except_clause', ('NAME', 'except', lineno), ...)
if tree[j][1][0] == token.NAME:
lineno_word = tree[j][1][2]
break
if lineno_colon and lineno_word:
# Found colon and keyword, mark all the lines
# between the two with the two line numbers.
self.record_multiline(spots, lineno_word, lineno_colon)
# "pass" statements are tricky: different versions of Python
# treat them differently, especially in the common case of a
# function with a doc string and a single pass statement.
self.find_docstring_pass_pair(tree[i], spots)
elif tree[i][0] == symbol.simple_stmt:
first_line = self.first_line_of_tree(tree[i])
last_line = self.last_line_of_tree(tree[i])
if first_line != last_line:
self.record_multiline(spots, first_line, last_line)
self.get_suite_spots(tree[i], spots)
def construct_filter_list_tree(l, test_mode=False):
if len(l) == 0:
return l
if token.ISTERMINAL(l[0]):
return l
current_id = l[0]
current_test_mode = True if current_id in test_nodes or test_mode else False
l_c = []
for c in l[1:]:
l_c += [construct_filter_list_tree(c, current_test_mode)]
# remove current node(replace it by it child) if current node is (meet all conditions)
# (1) under a test node
# (2) has only one child
# (3) the only child is a non-terminal node
# (4) for special, the child is not the atom node(to keep the parent of the atom node)
if current_test_mode \
and len(l_c) == 1 \
and token.ISNONTERMINAL(l_c[0][0]) \
and l_c[0][0] != symbol.atom:
return l_c[0]
return [l[0]] + l_c
def find_statement(self, tree, dict):
import token
while token.ISNONTERMINAL(tree[0]):
tree = tree[1]
dict[tree[2]] = 1
def update_event(self, inp=-1):
self.set_output_val(0, token.ISNONTERMINAL(self.input(0)))
