def print_list_tree(l, blank_nums=0): blanks = ' ' * blank_nums name = '' if token.ISNONTERMINAL(l[0]) else l[1] print('{}--{} {}'.format(blanks, str(l[0]), name)) if token.ISNONTERMINAL(l[0]): for c in l[1:]: print_list_tree(c, blank_nums + 1)
def safe_node(self, node): if token.ISNONTERMINAL(node[0]): return reduce(operator.and_, map(self.safe_node, node[1:])) elif node[0] == token.NAME: return node[1] in self._safe_names else: return True
def get_suite_spots(self, tree, spots): import symbol, token for i in range(1, len(tree)): if type(tree[i]) == type(()): if tree[i][0] == symbol.suite: # Found a suite, look back for the colon and keyword. lineno_colon = lineno_word = None for j in range(i - 1, 0, -1): if tree[j][0] == token.COLON: lineno_colon = tree[j][2] elif tree[j][0] == token.NAME: if tree[j][1] == 'elif': # Find the line number of the first non-terminal # after the keyword. t = tree[j + 1] while t and token.ISNONTERMINAL(t[0]): t = t[1] if t: lineno_word = t[2] else: lineno_word = tree[j][2] break elif tree[j][0] == symbol.except_clause: # "except" clauses look like: # ('except_clause', ('NAME', 'except', lineno), ...) if tree[j][1][0] == token.NAME: lineno_word = tree[j][1][2] break if lineno_colon and lineno_word: # Found colon and keyword, mark all the lines # between the two with the two line numbers. for l in range(lineno_word, lineno_colon + 1): spots[l] = (lineno_word, lineno_colon) self.get_suite_spots(tree[i], spots)
def walkCst(ctx, cst): """ Scan the CST (tuple) for tokens, appending index lines to the buffer. """ indent = 0 lineno = 1 stack = [(cst, indent)] try: while stack: cst, indent = stack.pop() #print("%5d%s%s" % (lineno, " " * indent, nodeNames[cst[0]])) if token.ISNONTERMINAL(cst[0]): processNonTerminal(ctx, cst) else: lineno = processTerminal(ctx, cst) indented = False for i in range(len(cst) - 1, 0, -1): if type(cst[i]) == tuple: # Push it onto the processing stack # Mirrors a recursive solution if not indented: indent += 2 indented = True stack.append((cst[i], indent)) except Exception as e: e.lineno = lineno raise e
def walk(root): for i in range(0, len(root)): if token.ISNONTERMINAL(root[i]): if root[i] == symbol.import_stmt: modules.append(handle_import(root)) return if type(root[i]) == tuple: walk(root[i])
def find_statements(self, tree, dict): import symbol, token if token.ISNONTERMINAL(tree[0]): for t in tree[1:]: self.find_statements(t, dict) if tree[0] == symbol.stmt: self.find_statement(tree[1], dict) elif (tree[0] == token.NAME and tree[1] in ['elif', 'except', 'finally']): dict[tree[2]] = 1
def convert_tree(node): symbol_id = node[0] symbol_value = node[1] if token.ISNONTERMINAL(symbol_id): # non terminal result = [True, gen_node_id(), symbol.sym_name[symbol_id]] for i in range(1, len(node)): result.append(convert_tree(node[i])) return result else: # terminal return [False, gen_node_id(), token.tok_name[symbol_id], symbol_value]
def get_suite_spots(self, tree, spots): """ Analyze a parse tree to find suite introducers which span a number of lines. """ for i in range(1, len(tree)): if type(tree[i]) == type(()): if tree[i][0] == symbol.suite: # Found a suite, look back for the colon and keyword. lineno_colon = lineno_word = None for j in range(i - 1, 0, -1): if tree[j][0] == token.COLON: # Colons are never executed themselves: we want the # line number of the last token before the colon. lineno_colon = self.last_line_of_tree(tree[j - 1]) elif tree[j][0] == token.NAME: if tree[j][1] == 'elif': # Find the line number of the first non-terminal # after the keyword. t = tree[j + 1] while t and token.ISNONTERMINAL(t[0]): t = t[1] if t: lineno_word = t[2] else: lineno_word = tree[j][2] break elif tree[j][0] == symbol.except_clause: # "except" clauses look like: # ('except_clause', ('NAME', 'except', lineno), ...) if tree[j][1][0] == token.NAME: lineno_word = tree[j][1][2] break if lineno_colon and lineno_word: # Found colon and keyword, mark all the lines # between the two with the two line numbers. self.record_multiline(spots, lineno_word, lineno_colon) # "pass" statements are tricky: different versions of Python # treat them differently, especially in the common case of a # function with a doc string and a single pass statement. self.find_docstring_pass_pair(tree[i], spots) elif tree[i][0] == symbol.simple_stmt: first_line = self.first_line_of_tree(tree[i]) last_line = self.last_line_of_tree(tree[i]) if first_line != last_line: self.record_multiline(spots, first_line, last_line) self.get_suite_spots(tree[i], spots)
def construct_filter_list_tree(l, test_mode=False): if len(l) == 0: return l if token.ISTERMINAL(l[0]): return l current_id = l[0] current_test_mode = True if current_id in test_nodes or test_mode else False l_c = [] for c in l[1:]: l_c += [construct_filter_list_tree(c, current_test_mode)] # remove current node(replace it by it child) if current node is (meet all conditions) # (1) under a test node # (2) has only one child # (3) the only child is a non-terminal node # (4) for special, the child is not the atom node(to keep the parent of the atom node) if current_test_mode \ and len(l_c) == 1 \ and token.ISNONTERMINAL(l_c[0][0]) \ and l_c[0][0] != symbol.atom: return l_c[0] return [l[0]] + l_c
def find_statement(self, tree, dict): import token while token.ISNONTERMINAL(tree[0]): tree = tree[1] dict[tree[2]] = 1
def update_event(self, inp=-1): self.set_output_val(0, token.ISNONTERMINAL(self.input(0)))