def list_tree_to_sequence_preorder(l): seq = [l[0]] if token.ISTERMINAL(l[0]): return seq for c in l[1:]: seq += list_tree_to_sequence_preorder(c) return seq
def walk(self, elements, symtabs): if type(elements) != types.TupleType: return if token.ISTERMINAL(elements[0]): return production = elements[0] if production == symbol.funcdef: self.handle_funcdef(elements, symtabs) elif production == symbol.varargslist: self.handle_varargslist(elements, symtabs) elif production == symbol.fpdef: self.handle_fpdef(elements, symtabs) elif production == symbol.import_as_name: self.handle_import_as_name(elements, symtabs) elif production == symbol.dotted_as_name: self.handle_dotted_as_name(elements, symtabs) elif production == symbol.dotted_name: self.handle_dotted_name(elements, symtabs) elif production == symbol.global_stmt: self.handle_global_stmt(elements, symtabs) elif production == symbol.atom: self.handle_atom(elements, symtabs) elif production == symbol.trailer: self.handle_trailer(elements, symtabs) elif production == symbol.classdef: self.handle_classdef(elements, symtabs) elif production == symbol.argument: self.handle_argument(elements, symtabs) elif production == symbol.lambdef: self.handle_lambdef(elements, symtabs) elif production == symbol.decorator: self.handle_decorator(elements, symtabs) else: for node in elements: self.walk(node, symtabs)
def get_fpdef_names(elements): result = [] if type(elements) != types.TupleType: return result if token.ISTERMINAL(elements[0]): return result name = elements[1] assert name[0] == token.NAME id = name[1] line = name[2] result.append((line, id)) for node in elements: result.extend(CSTWalker.get_fpdef_names(node)) return result
def _render_tree(node): nonlocal prev, pos, source out = '' if type(node) == list: for n in node: out += _render_tree(n) if type(node) == int: if token.ISTERMINAL(node): name = token.tok_name[node] color = parsing.token_colors[name] else: name = symbol.sym_name[node] color = parsing.symbol_colors[name] if color: out += color if debug: print((color if color else colors.reset) + '[' + name + ']') prev = name if prev == 'NAME': if node == 'self': out += colors.syntax.self if node in ['super', 'elif', 'else', 'print']: out += colors.syntax.keyword if type(node) == str: if debug: print(node) out += node pos += len(node) whitespace_and_comments = r_whitespace_and_comments.search( source[pos:]) if whitespace_and_comments: if whitespace_and_comments.group(1): # syntax highlight comments out += colors.syntax.comment # add trailing whitespace to output (because parser removes whitespace and comma's) out += whitespace_and_comments.group(0) pos += len(whitespace_and_comments.group(0)) out += default_color return out
def printToken(tok, level): """ Print the symbolic name for a token or symbol from a python parse tree. @param tok: token to be printed. @type tok: integer, values defined in symbol or token modules @param level: recursion depth, specified as a string of spaces. @type level: string """ if (token.ISTERMINAL(tok)): print level + token.tok_name[tok] else: print level + symbol.sym_name[tok]
def parsetree_to_string(parsetree): """function that takes a parse tree and turns it back into Python source code""" if len(parsetree)==2 and token.ISTERMINAL(parsetree[0]): return parsetree[1] else: result = '' for t in parsetree[1:]: # Insert a space if need be. We need a space if the last thing # in result and the first thing in the new string are both alphnumeric. st = parsetree_to_string(t) if len(result)>0 and len(st)>0 and (result[-1]).isalnum() and (st[0]).isalnum(): result = result + ' ' + st else: result = result + st return result
def get_fpdef_names(elements): """Extract all argument names from fpdef""" result = [] # We are not interested in terminal tokens if type(elements) != types.TupleType: return result if token.ISTERMINAL(elements[0]): return result name = elements[1] assert name[0] == token.NAME id = name[1] line = name[2] result.append((line, id)) for node in elements: result.extend(CSTWalker.get_fpdef_names(node)) return result
def replace_list_preorder(l): nonlocal seq_index if token.ISTERMINAL(l[0]): ter_l = [seq_index] seq_index += 1 if keep_terminal_info: ter_l += l[1:] ter_l += [l[0]] return ter_l index_list = [] for i in l: if type(i) == int: index_list += [seq_index] seq_index += 1 elif type(i) == list: index_list += [replace_list_preorder(i)] return index_list
def list_tree_to_sequence(l, convert_to_str=False, walk_type='preorder'): """ :param l: a list-tree :param walk_type: four type: {'preorder', 'inorder', 'postorder', 'breadth'} :return: """ if walk_type == 'preorder': seq = list_tree_to_sequence_preorder(l) elif walk_type == 'inorder': raise NotImplementedError elif walk_type == 'postorder': raise NotImplementedError elif walk_type == 'breadth': raise NotImplementedError if convert_to_str: seq = [ token.tok_name[s] if token.ISTERMINAL(s) else symbol.sym_name[s] for s in seq ] return seq
def construct_filter_list_tree(l, test_mode=False): if len(l) == 0: return l if token.ISTERMINAL(l[0]): return l current_id = l[0] current_test_mode = True if current_id in test_nodes or test_mode else False l_c = [] for c in l[1:]: l_c += [construct_filter_list_tree(c, current_test_mode)] # remove current node(replace it by it child) if current node is (meet all conditions) # (1) under a test node # (2) has only one child # (3) the only child is a non-terminal node # (4) for special, the child is not the atom node(to keep the parent of the atom node) if current_test_mode \ and len(l_c) == 1 \ and token.ISNONTERMINAL(l_c[0][0]) \ and l_c[0][0] != symbol.atom: return l_c[0] return [l[0]] + l_c
class parse_text_x_python: def __init__(self, viewer, reload=0): self.__viewer = viewer self.__source = '' viewer.new_font((None, 0, 0, 1)) def feed(self, data): self.__source = self.__source + data self.__viewer.send_literal_data(data) IGNORED_TERMINALS = ( token.ENDMARKER, token.NEWLINE, token.INDENT, token.DEDENT) __wanted_terminals = {} for ntype in token.tok_name.keys(): if token.ISTERMINAL(ntype) and ntype not in IGNORED_TERMINALS: __wanted_terminals[ntype] = ntype def close(self): self.show("Colorizing Python source text - parsing...") import parser try: nodes = parser.ast2list(parser.suite(self.__source), 1) except parser.ParserError, err: self.__viewer.context.message( "Syntax error in Python source: %s" % err) return self.setup_tags() from types import IntType, ListType ISTERMINAL = token.ISTERMINAL wanted = self.__wanted_terminals.has_key tag_add = self.tag_add = self.__viewer.text.tag_add colorize = self.colorize prevline, prevcol = 0, 0 sourcetext = string.split(self.__source, "\n") sourcetext.insert(0, '') self.show("Colorizing Python source text - coloring...") steps = 0 while nodes: steps = steps + 1 if not (steps % 2000): self.show() node = nodes[0] del nodes[0] if type(node) is ListType: ntype = node[0] if wanted(ntype): [ntype, nstr, lineno] = node # The parser spits out the line number the token ENDS on, # not the line it starts on! if ntype == token.STRING and "\n" in nstr: strlines = string.split(nstr, "\n") endpos = lineno, len(strlines[-1]), sourcetext[lineno] lineno = lineno - len(strlines) + 1 else: endpos = () if prevline != lineno: tag_add('python:comment', "%d.%d" % (prevline, prevcol), "%d.0" % lineno) prevcol = 0 prevline = lineno sourceline = sourcetext[lineno] match = ws_width(sourceline, prevcol) if match: prevcol = match.end() colorize(ntype, nstr, lineno, prevcol) # point prevline/prevcol to 1st char after token: if endpos: prevline, prevcol, sourceline = endpos else: prevcol = prevcol + len(nstr) else: nodes = node[1:] + nodes
class parse_text_x_python: def __init__(self, viewer, reload=0): self.__viewer = viewer self.__source = '' viewer.new_font((None, 0, 0, 1)) def feed(self, data): self.__source = self.__source + data self.__viewer.send_literal_data(data) IGNORED_TERMINALS = (token.ENDMARKER, token.NEWLINE, token.INDENT, token.DEDENT) __wanted_terminals = {} for ntype in token.tok_name.keys(): if token.ISTERMINAL(ntype) and ntype not in IGNORED_TERMINALS: __wanted_terminals[ntype] = ntype __ws_width = re.compile("[%s]*" % string.whitespace).match def close(self): self.show("Colorizing Python source text - parsing...") import parser try: nodes = parser.ast2list(parser.suite(self.__source), 1) except parser.ParserError as err: self.__viewer.context.message("Syntax error in Python source: %s" % err) self.setup_tags() from types import IntType, ListType ISTERMINAL = token.ISTERMINAL wanted = self.__wanted_terminals.has_key ws_width = self.__ws_width tag_add = self.tag_add = self.__viewer.text.tag_add colorize = self.colorize prevline, prevcol = 0, 0 sourcetext = str.split(self.__source, "\n") sourcetext.insert(0, '') self.show("Colorizing Python source text - coloring...") steps = 0 while nodes: steps = steps + 1 if not (steps % 2000): self.show() node = nodes[0] del nodes[0] if type(node) is ListType: ntype = node[0] if wanted(ntype): [ntype, nstr, lineno] = node # The parser spits out the line number the token ENDS on, # not the line it starts on! if ntype == token.STRING and "\n" in nstr: strlines = str.split(nstr, "\n") endpos = lineno, len(strlines[-1]), sourcetext[lineno] lineno = lineno - len(strlines) + 1 else: endpos = () if prevline != lineno: tag_add('python:comment', "%d.%d" % (prevline, prevcol), "%d.0" % lineno) prevcol = 0 prevline = lineno sourceline = sourcetext[lineno] prevcol = prevcol + ws_width(sourceline, prevcol) colorize(ntype, nstr, lineno, prevcol) # point prevline/prevcol to 1st char after token: if endpos: prevline, prevcol, sourceline = endpos else: prevcol = prevcol + len(nstr) else: nodes = node[1:] + nodes # end of last token to EOF is a comment... start = "%d.%d" % (prevline or 1, prevcol) tag_add('python:comment', start, tkinter.END) self.__viewer.context.message_clear() self.tag_add = None def show(self, message=None): if message: self.__viewer.context.message(message) self.__viewer.context.browser.root.update_idletasks() # Each element in this table maps an identifier to a tuple of # the tag it should be marked with and the tag the next token # should be marked with (or None). # __keywords = { # real keywords 'and': ('python:operator', None), 'break': ('python:control', None), 'class': ('python:define', 'python:class'), 'continue': ('python:control', None), 'def': ('python:define', 'python:def'), 'del': ('python:statement', None), 'elif': ('python:control', None), 'else': ('python:control', None), 'except': ('python:control', None), 'finally': ('python:control', None), 'for': ('python:control', None), 'from': ('python:statement', None), 'global': ('python:statement', None), 'if': ('python:control', None), 'import': ('python:statement', None), 'in': ('python:operator', None), 'is': ('python:operator', None), 'lambda': ('python:operator', None), 'not': ('python:operator', None), 'or': ('python:operator', None), 'pass': ('python:statement', None), 'print': ('python:statement', None), 'raise': ('python:control', None), 'return': ('python:control', None), 'try': ('python:control', None), 'while': ('python:control', None), # others I'd like made special 'None': ('python:special', None), } import types for name in dir(types): if len(name) > 4 and name[-4:] == "Type": __keywords[name] = ('python:special', None) __next_tag = None def colorize(self, ntype, nstr, lineno, colno): """Colorize a single token. ntype Node type. This is guaranteed to be a terminal token type not listed in self.IGNORE_TERMINALS. nstr String containing the token, uninterpreted. lineno Line number (1-based) of the line on which the token starts. colno Index into the source line at which the token starts. <TAB>s are not counted specially. """ start = "%d.%d" % (lineno, colno) end = "%s + %d chars" % (start, len(nstr)) if self.__next_tag: self.tag_add(self.__next_tag, start, end) self.__next_tag = None elif self.__keywords.has_key(nstr): tag, self.__next_tag = self.__keywords[nstr] self.tag_add(tag, start, end) elif ntype == token.STRING: qw = 1 # number of leading/trailing quotation if nstr[0] == nstr[1]: # marks -- `quote width' qw = 3 start = "%d.%d" % (lineno, colno + qw) end = "%s + %d chars" % (start, len(nstr) - (2 * qw)) self.tag_add("python:string", start, end) # Set foreground colors from this tag==>color table: __foregrounds = { 'python:class': 'darkGreen', 'python:comment': 'mediumBlue', 'python:control': 'midnightBlue', 'python:def': 'saddleBrown', 'python:define': 'midnightBlue', 'python:operator': 'midnightBlue', 'python:special': 'darkGreen', 'python:statement': 'midnightBlue', 'python:string': 'steelblue4', } def setup_tags(self): """Configure the display tags associated with Python source coloring. This is called only if the source is correctly parsed. All mapping of logical tags to physical style is accomplished in this method. """ self.__viewer.configure_fonttag('_tt_b') self.__viewer.configure_fonttag('_tt_i') text = self.__viewer.text boldfont = text.tag_cget('_tt_b', '-font') italicfont = text.tag_cget('_tt_i', '-font') text.tag_config('python:string', font=italicfont) for tag in ('python:class', 'python:def', 'python:define'): text.tag_config(tag, font=boldfont) for tag, color in self.__foregrounds.items(): text.tag_config(tag, foreground=color)
def update_event(self, inp=-1): self.set_output_val(0, token.ISTERMINAL(self.input(0)))