def list_tree_to_sequence_preorder(l):
seq = [l[0]]
if token.ISTERMINAL(l[0]):
return seq
for c in l[1:]:
seq += list_tree_to_sequence_preorder(c)
return seq
def walk(self, elements, symtabs):
if type(elements) != types.TupleType:
return
if token.ISTERMINAL(elements[0]):
return
production = elements[0]
if production == symbol.funcdef:
self.handle_funcdef(elements, symtabs)
elif production == symbol.varargslist:
self.handle_varargslist(elements, symtabs)
elif production == symbol.fpdef:
self.handle_fpdef(elements, symtabs)
elif production == symbol.import_as_name:
self.handle_import_as_name(elements, symtabs)
elif production == symbol.dotted_as_name:
self.handle_dotted_as_name(elements, symtabs)
elif production == symbol.dotted_name:
self.handle_dotted_name(elements, symtabs)
elif production == symbol.global_stmt:
self.handle_global_stmt(elements, symtabs)
elif production == symbol.atom:
self.handle_atom(elements, symtabs)
elif production == symbol.trailer:
self.handle_trailer(elements, symtabs)
elif production == symbol.classdef:
self.handle_classdef(elements, symtabs)
elif production == symbol.argument:
self.handle_argument(elements, symtabs)
elif production == symbol.lambdef:
self.handle_lambdef(elements, symtabs)
elif production == symbol.decorator:
self.handle_decorator(elements, symtabs)
else:
for node in elements:
self.walk(node, symtabs)
def get_fpdef_names(elements):
result = []
if type(elements) != types.TupleType:
return result
if token.ISTERMINAL(elements[0]):
return result
name = elements[1]
assert name[0] == token.NAME
id = name[1]
line = name[2]
result.append((line, id))
for node in elements:
result.extend(CSTWalker.get_fpdef_names(node))
return result
def _render_tree(node):
nonlocal prev, pos, source
out = ''
if type(node) == list:
for n in node:
out += _render_tree(n)
if type(node) == int:
if token.ISTERMINAL(node):
name = token.tok_name[node]
color = parsing.token_colors[name]
else:
name = symbol.sym_name[node]
color = parsing.symbol_colors[name]
if color:
out += color
if debug:
print((color if color else colors.reset) + '[' + name + ']')
prev = name
if prev == 'NAME':
if node == 'self':
out += colors.syntax.self
if node in ['super', 'elif', 'else', 'print']:
out += colors.syntax.keyword
if type(node) == str:
if debug:
print(node)
out += node
pos += len(node)
whitespace_and_comments = r_whitespace_and_comments.search(
source[pos:])
if whitespace_and_comments:
if whitespace_and_comments.group(1):
# syntax highlight comments
out += colors.syntax.comment
# add trailing whitespace to output (because parser removes whitespace and comma's)
out += whitespace_and_comments.group(0)
pos += len(whitespace_and_comments.group(0))
out += default_color
return out
def printToken(tok, level):
"""
Print the symbolic name for a token or symbol from a python
parse tree.
@param tok: token to be printed.
@type tok: integer, values defined in symbol or token modules
@param level: recursion depth, specified as a string of spaces.
@type level: string
"""
if (token.ISTERMINAL(tok)):
print level + token.tok_name[tok]
else:
print level + symbol.sym_name[tok]
def parsetree_to_string(parsetree):
"""function that takes a parse tree and turns it back into Python source code"""
if len(parsetree)==2 and token.ISTERMINAL(parsetree[0]):
return parsetree[1]
else:
result = ''
for t in parsetree[1:]:
# Insert a space if need be. We need a space if the last thing
# in result and the first thing in the new string are both alphnumeric.
st = parsetree_to_string(t)
if len(result)>0 and len(st)>0 and (result[-1]).isalnum() and (st[0]).isalnum():
result = result + ' ' + st
else:
result = result + st
return result
def get_fpdef_names(elements):
"""Extract all argument names from fpdef"""
result = []
# We are not interested in terminal tokens
if type(elements) != types.TupleType:
return result
if token.ISTERMINAL(elements[0]):
return result
name = elements[1]
assert name[0] == token.NAME
id = name[1]
line = name[2]
result.append((line, id))
for node in elements:
result.extend(CSTWalker.get_fpdef_names(node))
return result
def replace_list_preorder(l):
nonlocal seq_index
if token.ISTERMINAL(l[0]):
ter_l = [seq_index]
seq_index += 1
if keep_terminal_info:
ter_l += l[1:]
ter_l += [l[0]]
return ter_l
index_list = []
for i in l:
if type(i) == int:
index_list += [seq_index]
seq_index += 1
elif type(i) == list:
index_list += [replace_list_preorder(i)]
return index_list
def list_tree_to_sequence(l, convert_to_str=False, walk_type='preorder'):
"""
:param l: a list-tree
:param walk_type: four type: {'preorder', 'inorder', 'postorder', 'breadth'}
:return:
"""
if walk_type == 'preorder':
seq = list_tree_to_sequence_preorder(l)
elif walk_type == 'inorder':
raise NotImplementedError
elif walk_type == 'postorder':
raise NotImplementedError
elif walk_type == 'breadth':
raise NotImplementedError
if convert_to_str:
seq = [
token.tok_name[s] if token.ISTERMINAL(s) else symbol.sym_name[s]
for s in seq
]
return seq
def construct_filter_list_tree(l, test_mode=False):
if len(l) == 0:
return l
if token.ISTERMINAL(l[0]):
return l
current_id = l[0]
current_test_mode = True if current_id in test_nodes or test_mode else False
l_c = []
for c in l[1:]:
l_c += [construct_filter_list_tree(c, current_test_mode)]
# remove current node(replace it by it child) if current node is (meet all conditions)
# (1) under a test node
# (2) has only one child
# (3) the only child is a non-terminal node
# (4) for special, the child is not the atom node(to keep the parent of the atom node)
if current_test_mode \
and len(l_c) == 1 \
and token.ISNONTERMINAL(l_c[0][0]) \
and l_c[0][0] != symbol.atom:
return l_c[0]
return [l[0]] + l_c
class parse_text_x_python:
def __init__(self, viewer, reload=0):
self.__viewer = viewer
self.__source = ''
viewer.new_font((None, 0, 0, 1))
def feed(self, data):
self.__source = self.__source + data
self.__viewer.send_literal_data(data)
IGNORED_TERMINALS = (
token.ENDMARKER, token.NEWLINE, token.INDENT, token.DEDENT)
__wanted_terminals = {}
for ntype in token.tok_name.keys():
if token.ISTERMINAL(ntype) and ntype not in IGNORED_TERMINALS:
__wanted_terminals[ntype] = ntype
def close(self):
self.show("Colorizing Python source text - parsing...")
import parser
try:
nodes = parser.ast2list(parser.suite(self.__source), 1)
except parser.ParserError, err:
self.__viewer.context.message(
"Syntax error in Python source: %s" % err)
return
self.setup_tags()
from types import IntType, ListType
ISTERMINAL = token.ISTERMINAL
wanted = self.__wanted_terminals.has_key
tag_add = self.tag_add = self.__viewer.text.tag_add
colorize = self.colorize
prevline, prevcol = 0, 0
sourcetext = string.split(self.__source, "\n")
sourcetext.insert(0, '')
self.show("Colorizing Python source text - coloring...")
steps = 0
while nodes:
steps = steps + 1
if not (steps % 2000): self.show()
node = nodes[0]
del nodes[0]
if type(node) is ListType:
ntype = node[0]
if wanted(ntype):
[ntype, nstr, lineno] = node
# The parser spits out the line number the token ENDS on,
# not the line it starts on!
if ntype == token.STRING and "\n" in nstr:
strlines = string.split(nstr, "\n")
endpos = lineno, len(strlines[-1]), sourcetext[lineno]
lineno = lineno - len(strlines) + 1
else:
endpos = ()
if prevline != lineno:
tag_add('python:comment',
"%d.%d" % (prevline, prevcol), "%d.0" % lineno)
prevcol = 0
prevline = lineno
sourceline = sourcetext[lineno]
match = ws_width(sourceline, prevcol)
if match:
prevcol = match.end()
colorize(ntype, nstr, lineno, prevcol)
# point prevline/prevcol to 1st char after token:
if endpos:
prevline, prevcol, sourceline = endpos
else:
prevcol = prevcol + len(nstr)
else:
nodes = node[1:] + nodes
class parse_text_x_python:
def __init__(self, viewer, reload=0):
self.__viewer = viewer
self.__source = ''
viewer.new_font((None, 0, 0, 1))
def feed(self, data):
self.__source = self.__source + data
self.__viewer.send_literal_data(data)
IGNORED_TERMINALS = (token.ENDMARKER, token.NEWLINE, token.INDENT,
token.DEDENT)
__wanted_terminals = {}
for ntype in token.tok_name.keys():
if token.ISTERMINAL(ntype) and ntype not in IGNORED_TERMINALS:
__wanted_terminals[ntype] = ntype
__ws_width = re.compile("[%s]*" % string.whitespace).match
def close(self):
self.show("Colorizing Python source text - parsing...")
import parser
try:
nodes = parser.ast2list(parser.suite(self.__source), 1)
except parser.ParserError as err:
self.__viewer.context.message("Syntax error in Python source: %s" %
err)
self.setup_tags()
from types import IntType, ListType
ISTERMINAL = token.ISTERMINAL
wanted = self.__wanted_terminals.has_key
ws_width = self.__ws_width
tag_add = self.tag_add = self.__viewer.text.tag_add
colorize = self.colorize
prevline, prevcol = 0, 0
sourcetext = str.split(self.__source, "\n")
sourcetext.insert(0, '')
self.show("Colorizing Python source text - coloring...")
steps = 0
while nodes:
steps = steps + 1
if not (steps % 2000): self.show()
node = nodes[0]
del nodes[0]
if type(node) is ListType:
ntype = node[0]
if wanted(ntype):
[ntype, nstr, lineno] = node
# The parser spits out the line number the token ENDS on,
# not the line it starts on!
if ntype == token.STRING and "\n" in nstr:
strlines = str.split(nstr, "\n")
endpos = lineno, len(strlines[-1]), sourcetext[lineno]
lineno = lineno - len(strlines) + 1
else:
endpos = ()
if prevline != lineno:
tag_add('python:comment', "%d.%d" % (prevline, prevcol),
"%d.0" % lineno)
prevcol = 0
prevline = lineno
sourceline = sourcetext[lineno]
prevcol = prevcol + ws_width(sourceline, prevcol)
colorize(ntype, nstr, lineno, prevcol)
# point prevline/prevcol to 1st char after token:
if endpos:
prevline, prevcol, sourceline = endpos
else:
prevcol = prevcol + len(nstr)
else:
nodes = node[1:] + nodes
# end of last token to EOF is a comment...
start = "%d.%d" % (prevline or 1, prevcol)
tag_add('python:comment', start, tkinter.END)
self.__viewer.context.message_clear()
self.tag_add = None
def show(self, message=None):
if message:
self.__viewer.context.message(message)
self.__viewer.context.browser.root.update_idletasks()
# Each element in this table maps an identifier to a tuple of
# the tag it should be marked with and the tag the next token
# should be marked with (or None).
#
__keywords = {
# real keywords
'and': ('python:operator', None),
'break': ('python:control', None),
'class': ('python:define', 'python:class'),
'continue': ('python:control', None),
'def': ('python:define', 'python:def'),
'del': ('python:statement', None),
'elif': ('python:control', None),
'else': ('python:control', None),
'except': ('python:control', None),
'finally': ('python:control', None),
'for': ('python:control', None),
'from': ('python:statement', None),
'global': ('python:statement', None),
'if': ('python:control', None),
'import': ('python:statement', None),
'in': ('python:operator', None),
'is': ('python:operator', None),
'lambda': ('python:operator', None),
'not': ('python:operator', None),
'or': ('python:operator', None),
'pass': ('python:statement', None),
'print': ('python:statement', None),
'raise': ('python:control', None),
'return': ('python:control', None),
'try': ('python:control', None),
'while': ('python:control', None),
# others I'd like made special
'None': ('python:special', None),
}
import types
for name in dir(types):
if len(name) > 4 and name[-4:] == "Type":
__keywords[name] = ('python:special', None)
__next_tag = None
def colorize(self, ntype, nstr, lineno, colno):
"""Colorize a single token.
ntype
Node type. This is guaranteed to be a terminal token type
not listed in self.IGNORE_TERMINALS.
nstr
String containing the token, uninterpreted.
lineno
Line number (1-based) of the line on which the token starts.
colno
Index into the source line at which the token starts. <TAB>s
are not counted specially.
"""
start = "%d.%d" % (lineno, colno)
end = "%s + %d chars" % (start, len(nstr))
if self.__next_tag:
self.tag_add(self.__next_tag, start, end)
self.__next_tag = None
elif self.__keywords.has_key(nstr):
tag, self.__next_tag = self.__keywords[nstr]
self.tag_add(tag, start, end)
elif ntype == token.STRING:
qw = 1 # number of leading/trailing quotation
if nstr[0] == nstr[1]: # marks -- `quote width'
qw = 3
start = "%d.%d" % (lineno, colno + qw)
end = "%s + %d chars" % (start, len(nstr) - (2 * qw))
self.tag_add("python:string", start, end)
# Set foreground colors from this tag==>color table:
__foregrounds = {
'python:class': 'darkGreen',
'python:comment': 'mediumBlue',
'python:control': 'midnightBlue',
'python:def': 'saddleBrown',
'python:define': 'midnightBlue',
'python:operator': 'midnightBlue',
'python:special': 'darkGreen',
'python:statement': 'midnightBlue',
'python:string': 'steelblue4',
}
def setup_tags(self):
"""Configure the display tags associated with Python source coloring.
This is called only if the source is correctly parsed. All mapping
of logical tags to physical style is accomplished in this method.
"""
self.__viewer.configure_fonttag('_tt_b')
self.__viewer.configure_fonttag('_tt_i')
text = self.__viewer.text
boldfont = text.tag_cget('_tt_b', '-font')
italicfont = text.tag_cget('_tt_i', '-font')
text.tag_config('python:string', font=italicfont)
for tag in ('python:class', 'python:def', 'python:define'):
text.tag_config(tag, font=boldfont)
for tag, color in self.__foregrounds.items():
text.tag_config(tag, foreground=color)
def update_event(self, inp=-1):
self.set_output_val(0, token.ISTERMINAL(self.input(0)))
