def match(self, kind):
## if match return token, else ParseException
tok = self.dequeue()
if (tok.kind != kind):
raise ParseException(u"cannot find token "+ \
Token.get_name(kind) + u" got " \
+ unicode(tok) + u" instead!")
return tok
def match(self,kind):
## if match return token, else ParseException
tok = self.dequeue()
if ( tok.kind != kind ):
raise ParseException(u"cannot find token "+ \
Token.get_name(kind) + u" got " \
+ unicode(tok) + u" instead!")
return tok
def _set_dimension(self, statement, dimension):
assignment = statement.children[0]
expression = assignment.children[1]
arithop = expression.children[0]
relation = arithop.children[0]
term = relation.children[0]
factor = term.children[0]
number = factor.children[0]
token = Token(dimension, "NUMBER")
number.set_token(token)
return statement
def _parse_type_mark(self):
type_mark = ParseTreeNode('type_mark')
if self._next_token_value_matches(Parser.DATA_TYPES):
type_mark.set_token(self._next())
else:
token = self._next()
report_expected_vs_encountered(str(Parser.DATA_TYPES), token.value,
token.line)
self._register_error()
type_mark.set_token(
Token(value='[filler]', type='WORD', line=token.line))
return type_mark
def __init__(self, scanner):
self.it = scanner.get_Token() #词法的迭代器
self.token = Token()
self.x = None
self.y = None
self.x_scale = None
self.y_scale = None
self.rot = None #旋转角度
self.start = None #T的起始
self.end = None #T的终点
self.step = None #步长
self.x_origin = None
self.y_origin = None
def new_tokens(lis):
'''
Добавляет тоекны в список
Принимает список с параметрами токенов
'''
global tokens
tokens = []
for line in lis:
tokens_line = []
for element in line:
token = Token(element[0], element[1], element[2], element[3])
tokens_line += [token]
tokens += [tokens_line]
def build_ast(self, tokens, customize, isLambda=0, noneInNames=False):
assert type(tokens) == ListType
#assert isinstance(tokens[0], Token)
if isLambda:
tokens.append(Token('LAMBDA_MARKER'))
try:
ast = parser.parse(tokens, customize)
except parser.ParserError, e:
raise ParserError(e, tokens)
if self.showast:
self.print_(repr(ast))
return ast
def n_return_if_stmt(self, node):
if self.__params['isLambda']:
self.preorder(node[0])
self.prune()
else:
self.write(self.indent, 'return')
if self.return_none or node != AST(
'return_stmt',
[AST('ret_expr', [NONE]),
Token('RETURN_END_IF')]):
self.write(' ')
self.preorder(node[0])
self.print_()
self.prune() # stop recursing
class Room:
'''
Класс для всех построенных комнат
'''
# Высота комнаты
height = Token(0, 0, "HEIGHT", HEIGHT)
# Ширина комнаты
width = Token(0, 0, "WIDTH", WIDTH)
# Левая дверь
left = None
# Правая дверь
right = None
# Верхняя дверь
up = None
# Нижняя дверь
down = None
# Название картинки
img = None
def __str__(self):
'''
Возвращает строку с описанием комнаты
'''
string = "height: " + str(self.height.value) + ", width: " + str(
self.width.value)
if self.left != None:
string += ", left: " + self.left.value
if self.right != None:
string += ", right: " + self.right.value
if self.up != None:
string += ", up: " + self.up.value
if self.down != None:
string += ", down: " + self.down.value
if self.img != None:
string += ", img: " + self.img.value
return string
def _index(self):
identifier = ParseTreeNode("identifier")
token = Token(value=self._index_identifier, type="WORD")
identifier.set_token(token)
name = ParseTreeNode("name")
name.add_child(identifier)
factor = ParseTreeNode("factor")
factor.add_child(name)
term = ParseTreeNode("term")
term.add_child(factor)
relation = ParseTreeNode("relation")
relation.add_child(term)
arithop = ParseTreeNode("arithop")
arithop.add_child(relation)
expression = ParseTreeNode("expression")
expression.add_child(arithop)
return expression
def n_yield(self, node):
self.write('yield')
if node != AST('yield', [NONE, Token('YIELD_VALUE')]):
self.write(' ')
self.preorder(node[0])
self.prune() # stop recursing
class Walker(GenericASTTraversal, object):
stacked_params = ('f', 'indent', 'isLambda', '_globals')
def __init__(self, out, scanner, showast=0):
GenericASTTraversal.__init__(self, ast=None)
self.scanner = scanner
params = {
'f': out,
'indent': '',
}
self.showast = showast
self.__params = params
self.__param_stack = []
self.ERROR = None
self.prec = 100
self.return_none = False
self.mod_globs = set()
self.currentclass = None
self.pending_newlines = 0
f = property(lambda s: s.__params['f'],
lambda s, x: s.__params.__setitem__('f', x),
lambda s: s.__params.__delitem__('f'), None)
indent = property(lambda s: s.__params['indent'],
lambda s, x: s.__params.__setitem__('indent', x),
lambda s: s.__params.__delitem__('indent'), None)
isLambda = property(lambda s: s.__params['isLambda'],
lambda s, x: s.__params.__setitem__('isLambda', x),
lambda s: s.__params.__delitem__('isLambda'), None)
_globals = property(lambda s: s.__params['_globals'],
lambda s, x: s.__params.__setitem__('_globals', x),
lambda s: s.__params.__delitem__('_globals'), None)
def indentMore(self, indent=TAB):
self.indent += indent
def indentLess(self, indent=TAB):
self.indent = self.indent[:-len(indent)]
def traverse(self, node, indent=None, isLambda=0):
self.__param_stack.append(self.__params)
if indent is None: indent = self.indent
p = self.pending_newlines
self.pending_newlines = 0
self.__params = {
'_globals': {},
'f': cStringIO.StringIO(),
'indent': indent,
'isLambda': isLambda,
}
self.preorder(node)
self.f.write('\n' * self.pending_newlines)
result = self.f.getvalue()
self.__params = self.__param_stack.pop()
self.pending_newlines = p
return result
def write(self, *data):
if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
return
# import pdb; pdb.set_trace()
out = ''.join((str(j) for j in data))
n = 0
for i in out:
if i == '\n':
n += 1
if n == len(out):
self.pending_newlines = max(self.pending_newlines, n)
return
elif n:
self.pending_newlines = max(self.pending_newlines, n)
out = out[n:]
break
else:
break
if self.pending_newlines > 0:
self.f.write('\n' * self.pending_newlines)
self.pending_newlines = 0
for i in out[::-1]:
if i == '\n':
self.pending_newlines += 1
else:
break
if self.pending_newlines:
out = out[:-self.pending_newlines]
self.f.write(out)
def print_(self, *data):
if data and not (len(data) == 1 and data[0] == ''):
self.write(*data)
self.pending_newlines = max(self.pending_newlines, 1)
def print_docstring(self, indent, docstring):
quote = '"""'
self.write(indent)
if type(docstring) == unicode:
self.write('u')
docstring = repr(docstring.expandtabs())[2:-1]
else:
docstring = repr(docstring.expandtabs())[1:-1]
for (orig, replace) in (('\\\\', '\t'), ('\\r\\n', '\n'), ('\\n',
'\n'),
('\\r', '\n'), ('\\"', '"'), ("\\'", "'")):
docstring = docstring.replace(orig, replace)
#Do a raw string if there are backslashes but no other escaped characters:
#also check some edge cases
if ('\t' in docstring and '\\' not in docstring and len(docstring) >= 2
and docstring[-1] != '\t'
and (docstring[-1] != '"' or docstring[-2] == '\t')):
self.write('r') #raw string
#restore backslashes unescaped since raw
docstring = docstring.replace('\t', '\\')
else:
#Escape '"' if it's the last character, so it doesn't ruin the ending triple quote
if len(docstring) and docstring[-1] == '"':
docstring = docstring[:-1] + '\\"'
#Escape triple quote anywhere
docstring = docstring.replace('"""', '\\"\\"\\"')
#Restore escaped backslashes
docstring = docstring.replace('\t', '\\\\')
lines = docstring.split('\n')
calculate_indent = sys.maxint
for line in lines[1:]:
stripped = line.lstrip()
if len(stripped) > 0:
calculate_indent = min(calculate_indent,
len(line) - len(stripped))
calculate_indent = min(calculate_indent,
len(lines[-1]) - len(lines[-1].lstrip()))
# Remove indentation (first line is special):
trimmed = [lines[0]]
if calculate_indent < sys.maxint:
trimmed += [line[calculate_indent:] for line in lines[1:]]
self.write(quote)
if len(trimmed) == 0:
self.print_(quote)
elif len(trimmed) == 1:
self.print_(trimmed[0], quote)
else:
self.print_(trimmed[0])
for line in trimmed[1:-1]:
self.print_(indent, line)
self.print_(indent, trimmed[-1], quote)
def n_return_stmt(self, node):
if self.__params['isLambda']:
self.preorder(node[0])
self.prune()
else:
self.write(self.indent, 'return')
if self.return_none or node != AST(
'return_stmt',
[AST('ret_expr', [NONE]),
Token('RETURN_VALUE')]):
self.write(' ')
self.preorder(node[0])
self.print_()
self.prune() # stop recursing
def n_return_if_stmt(self, node):
if self.__params['isLambda']:
self.preorder(node[0])
self.prune()
else:
self.write(self.indent, 'return')
if self.return_none or node != AST(
'return_stmt',
[AST('ret_expr', [NONE]),
Token('RETURN_END_IF')]):
self.write(' ')
self.preorder(node[0])
self.print_()
self.prune() # stop recursing
def n_yield(self, node):
self.write('yield')
if node != AST('yield', [NONE, Token('YIELD_VALUE')]):
self.write(' ')
self.preorder(node[0])
self.prune() # stop recursing
def n_buildslice3(self, node):
p = self.prec
self.prec = 100
if node[0] != NONE:
self.preorder(node[0])
self.write(':')
if node[1] != NONE:
self.preorder(node[1])
self.write(':')
if node[2] != NONE:
self.preorder(node[2])
self.prec = p
self.prune() # stop recursing
def n_buildslice2(self, node):
p = self.prec
self.prec = 100
if node[0] != NONE:
self.preorder(node[0])
self.write(':')
if node[1] != NONE:
self.preorder(node[1])
self.prec = p
self.prune() # stop recursing
# def n_l_stmts(self, node):
# if node[0] == '_stmts':
# if len(node[0]) >= 2 and node[0][1] == 'stmt':
# if node[0][-1][0] == 'continue_stmt':
# del node[0][-1]
# self.default(node)
def n_expr(self, node):
p = self.prec
if node[0].type.startswith('binary_expr'):
n = node[0][-1][0]
else:
n = node[0]
self.prec = PRECEDENCE.get(n, -2)
if n == 'LOAD_CONST' and repr(n.pattr)[0] == '-':
self.prec = 6
if p < self.prec:
self.write('(')
self.preorder(node[0])
self.write(')')
else:
self.preorder(node[0])
self.prec = p
self.prune()
def n_ret_expr(self, node):
if len(node) == 1 and node[0] == 'expr':
self.n_expr(node[0])
else:
self.n_expr(node)
n_ret_expr_or_cond = n_expr
def n_binary_expr(self, node):
self.preorder(node[0])
self.write(' ')
self.preorder(node[-1])
self.write(' ')
self.prec -= 1
self.preorder(node[1])
self.prec += 1
self.prune()
def n_LOAD_CONST(self, node):
data = node.pattr
datatype = type(data)
if datatype is IntType and data == minint:
# convert to hex, since decimal representation
# would result in 'LOAD_CONST; UNARY_NEGATIVE'
# change:hG/2002-02-07: this was done for all negative integers
# todo: check whether this is necessary in Python 2.1
self.write(hex(data))
elif datatype is EllipsisType:
self.write('...')
elif data is None:
# LOAD_CONST 'None' only occurs, when None is
# implicit eg. in 'return' w/o params
# pass
self.write('None')
else:
self.write(repr(data))
# LOAD_CONST is a terminal, so stop processing/recursing early
self.prune()
def n_delete_subscr(self, node):
if node[-2][0] == 'build_list' and node[-2][0][-1].type.startswith(
'BUILD_TUPLE'):
if node[-2][0][-1] != 'BUILD_TUPLE_0':
node[-2][0].type = 'build_tuple2'
self.default(node)
# maybe_tuple = node[-2][-1]
# if maybe_tuple.type.startswith('BUILD_TUPLE'):
# maybe_tuple.type = 'build_tuple2'
# self.default(node)
n_store_subscr = n_binary_subscr = n_delete_subscr
# 'tryfinallystmt': ( '%|try:\n%+%c%-%|finally:\n%+%c%-', 1, 5 ),
def n_tryfinallystmt(self, node):
if len(node[1][0]) == 1 and node[1][0][0] == 'stmt':
if node[1][0][0][0] == 'trystmt':
node[1][0][0][0].type = 'tf_trystmt'
if node[1][0][0][0] == 'tryelsestmt':
node[1][0][0][0].type = 'tf_tryelsestmt'
self.default(node)
def n_exec_stmt(self, node):
"""
exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
exec_stmt ::= expr exprlist EXEC_STMT
"""
self.write(self.indent, 'exec ')
self.preorder(node[0])
if node[1][0] != NONE:
sep = ' in '
for subnode in node[1]:
self.write(sep)
sep = ", "
self.preorder(subnode)
self.print_()
self.prune() # stop recursing
def n_ifelsestmt(self, node, preprocess=0):
n = node[3][0]
if len(n) == 1 == len(n[0]) and n[0] == '_stmts':
n = n[0][0][0]
elif n[0].type in ('lastc_stmt', 'lastl_stmt'):
n = n[0][0]
else:
if not preprocess:
self.default(node)
return
if n.type in ('ifstmt', 'iflaststmt', 'iflaststmtl'):
node.type = 'ifelifstmt'
n.type = 'elifstmt'
elif n.type in ('ifelsestmtr', ):
node.type = 'ifelifstmt'
n.type = 'elifelsestmtr'
elif n.type in ('ifelsestmt', 'ifelsestmtc', 'ifelsestmtl'):
node.type = 'ifelifstmt'
self.n_ifelsestmt(n, preprocess=1)
if n == 'ifelifstmt':
n.type = 'elifelifstmt'
elif n.type in ('ifelsestmt', 'ifelsestmtc', 'ifelsestmtl'):
n.type = 'elifelsestmt'
if not preprocess:
self.default(node)
n_ifelsestmtc = n_ifelsestmtl = n_ifelsestmt
def n_ifelsestmtr(self, node):
if len(node[2]) != 2:
self.default(node)
if not (node[2][0][0][0] == 'ifstmt' and node[2][0][0][0][1][0] == 'return_if_stmts') \
and not (node[2][0][-1][0] == 'ifstmt' and node[2][0][-1][0][1][0] == 'return_if_stmts'):
self.default(node)
return
self.write(self.indent, 'if ')
self.preorder(node[0])
self.print_(':')
self.indentMore()
self.preorder(node[1])
self.indentLess()
if_ret_at_end = False
if len(node[2][0]) >= 3:
if node[2][0][-1][0] == 'ifstmt' and node[2][0][-1][0][1][
0] == 'return_if_stmts':
if_ret_at_end = True
past_else = False
prev_stmt_is_if_ret = True
for n in node[2][0]:
if (n[0] == 'ifstmt' and n[0][1][0] == 'return_if_stmts'):
if prev_stmt_is_if_ret:
n[0].type = 'elifstmt'
prev_stmt_is_if_ret = True
else:
prev_stmt_is_if_ret = False
if not past_else and not if_ret_at_end:
self.print_(self.indent, 'else:')
self.indentMore()
past_else = True
self.preorder(n)
if not past_else or if_ret_at_end:
self.print_(self.indent, 'else:')
self.indentMore()
self.preorder(node[2][1])
self.indentLess()
self.prune()
def n_elifelsestmtr(self, node):
if len(node[2]) != 2:
self.default(node)
for n in node[2][0]:
if not (n[0] == 'ifstmt' and n[0][1][0] == 'return_if_stmts'):
self.default(node)
return
self.write(self.indent, 'elif ')
self.preorder(node[0])
self.print_(':')
self.indentMore()
self.preorder(node[1])
self.indentLess()
if_ret_at_end = False
if len(node[2][0]) >= 3:
if node[2][0][-1][0] == 'ifstmt' and node[2][0][-1][0][1][
0] == 'return_if_stmts':
if_ret_at_end = True
past_else = False
prev_stmt_is_if_ret = True
for n in node[2][0]:
n[0].type = 'elifstmt'
self.preorder(n)
self.print_(self.indent, 'else:')
self.indentMore()
self.preorder(node[2][1])
self.indentLess()
self.prune()
def n_import_as(self, node):
iname = node[0].pattr
assert node[-1][-1].type.startswith('STORE_')
sname = node[-1][-1].pattr # assume one of STORE_.... here
if iname == sname or iname.startswith(sname + '.'):
self.write(iname)
else:
self.write(iname, ' as ', sname)
self.prune() # stop recursing
n_import_as_cont = n_import_as
def n_importfrom(self, node):
if node[0].pattr > 0:
node[2].pattr = '.' * node[0].pattr + node[2].pattr
self.default(node)
n_importstar = n_importfrom
def n_mkfunc(self, node):
self.write(node[-2].attr.co_name) # = code.co_name
self.indentMore()
self.make_function(node, isLambda=0)
if len(self.__param_stack) > 1:
self.write('\n\n')
else:
self.write('\n\n\n')
self.indentLess()
self.prune() # stop recursing
def n_mklambda(self, node):
self.make_function(node, isLambda=1)
self.prune() # stop recursing
def n_list_compr(self, node):
p = self.prec
self.prec = 27
n = node[-1]
assert n == 'list_iter'
# find innerst node
while n == 'list_iter':
n = n[0] # recurse one step
if n == 'list_for': n = n[3]
elif n == 'list_if': n = n[2]
elif n == 'list_if_not': n = n[2]
assert n == 'lc_body'
self.write('[ ')
self.preorder(n[0]) # lc_body
self.preorder(node[-1]) # for/if parts
self.write(' ]')
self.prec = p
self.prune() # stop recursing
def comprehension_walk(self, node, iter_index):
p = self.prec
self.prec = 27
code = node[-5].attr
assert type(code) == CodeType
code = Code(code, self.scanner, self.currentclass)
#assert isinstance(code, Code)
ast = self.build_ast(code._tokens, code._customize)
self.customize(code._customize)
ast = ast[0][0][0]
n = ast[iter_index]
assert n == 'comp_iter'
# find innerst node
while n == 'comp_iter':
n = n[0] # recurse one step
if n == 'comp_for': n = n[3]
elif n == 'comp_if': n = n[2]
elif n == 'comp_ifnot': n = n[2]
assert n == 'comp_body', ast
self.preorder(n[0])
self.write(' for ')
self.preorder(ast[iter_index - 1])
self.write(' in ')
self.preorder(node[-3])
self.preorder(ast[iter_index])
self.prec = p
def n_genexpr(self, node):
self.write('(')
self.comprehension_walk(node, 3)
self.write(')')
self.prune()
def n_setcomp(self, node):
self.write('{')
self.comprehension_walk(node, 4)
self.write('}')
self.prune()
n_dictcomp = n_setcomp
def n_classdef(self, node):
# class definition ('class X(A,B,C):')
cclass = self.currentclass
self.currentclass = str(node[0].pattr)
self.write('\n\n')
self.write(self.indent, 'class ', self.currentclass)
self.print_super_classes(node)
self.print_(':')
# class body
self.indentMore()
self.build_class(node[2][-2].attr)
self.indentLess()
self.currentclass = cclass
if len(self.__param_stack) > 1:
self.write('\n\n')
else:
self.write('\n\n\n')
self.prune()
n_classdefdeco2 = n_classdef
def print_super_classes(self, node):
node = node[1][0]
if not (node == 'build_list'):
return
self.write('(')
line_separator = ', '
sep = ''
for elem in node[:-1]:
value = self.traverse(elem)
self.write(sep, value)
sep = line_separator
self.write(')')
def n_mapexpr(self, node):
"""
prettyprint a mapexpr
'mapexpr' is something like k = {'a': 1, 'b': 42 }"
"""
p = self.prec
self.prec = 100
assert node[-1] == 'kvlist'
node = node[-1] # goto kvlist
self.indentMore(INDENT_PER_LEVEL)
line_seperator = ',\n' + self.indent
sep = INDENT_PER_LEVEL[:-1]
self.write('{')
for kv in node:
assert kv in ('kv', 'kv2', 'kv3')
# kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR
# kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR
# kv3 ::= expr expr STORE_MAP
if kv == 'kv':
name = self.traverse(kv[-2], indent='')
value = self.traverse(kv[1],
indent=self.indent +
(len(name) + 2) * ' ')
elif kv == 'kv2':
name = self.traverse(kv[1], indent='')
value = self.traverse(kv[-3],
indent=self.indent +
(len(name) + 2) * ' ')
elif kv == 'kv3':
name = self.traverse(kv[-2], indent='')
value = self.traverse(kv[0],
indent=self.indent +
(len(name) + 2) * ' ')
self.write(sep, name, ': ', value)
sep = line_seperator
self.write('}')
self.indentLess(INDENT_PER_LEVEL)
self.prec = p
self.prune()
def n_build_list(self, node):
"""
prettyprint a list or tuple
"""
p = self.prec
self.prec = 100
lastnode = node.pop().type
if lastnode.startswith('BUILD_LIST'):
self.write('[')
endchar = ']'
elif lastnode.startswith('BUILD_TUPLE'):
self.write('(')
endchar = ')'
elif lastnode.startswith('BUILD_SET'):
self.write('{')
endchar = '}'
elif lastnode.startswith('ROT_TWO'):
self.write('(')
endchar = ')'
else:
raise 'Internal Error: n_build_list expects list or tuple'
self.indentMore(INDENT_PER_LEVEL)
if len(node) > 3:
line_separator = ',\n' + self.indent
else:
line_separator = ', '
sep = INDENT_PER_LEVEL[:-1]
for elem in node:
if (elem == 'ROT_THREE'):
continue
assert elem == 'expr'
value = self.traverse(elem)
self.write(sep, value)
sep = line_separator
if len(node) == 1 and lastnode.startswith('BUILD_TUPLE'):
self.write(',')
self.write(endchar)
self.indentLess(INDENT_PER_LEVEL)
self.prec = p
self.prune()
def n_unpack(self, node):
for n in node[1:]:
if n[0].type == 'unpack':
n[0].type = 'unpack_w_parens'
self.default(node)
n_unpack_w_parens = n_unpack
def n_assign2(self, node):
for n in node[-2:]:
if n[0] == 'unpack':
n[0].type = 'unpack_w_parens'
self.default(node)
def n_assign3(self, node):
for n in node[-3:]:
if n[0] == 'unpack':
n[0].type = 'unpack_w_parens'
self.default(node)
def n_except_cond2(self, node):
if node[5][0] == 'unpack':
node[5][0].type = 'unpack_w_parens'
self.default(node)
def engine(self, entry, startnode):
#self.print_("-----")
#self.print_(str(startnode.__dict__))
fmt = entry[0]
## no longer used, since BUILD_TUPLE_n is pretty printed:
##lastC = 0
arg = 1
i = 0
m = escape.search(fmt)
while m:
i = m.end()
self.write(m.group('prefix'))
typ = m.group('type') or '{'
node = startnode
try:
if m.group('child'):
node = node[int(m.group('child'))]
except:
print node.__dict__
raise
if typ == '%': self.write('%')
elif typ == '+': self.indentMore()
elif typ == '-': self.indentLess()
elif typ == '|':
self.write(self.indent)
## no longer used, since BUILD_TUPLE_n is pretty printed:
elif typ == ',':
if lastC == 1:
self.write(',')
elif typ == 'c':
self.preorder(node[entry[arg]])
arg += 1
elif typ == 'p':
p = self.prec
(index, self.prec) = entry[arg]
self.preorder(node[index])
self.prec = p
arg += 1
elif typ == 'C':
low, high, sep = entry[arg]
lastC = remaining = len(node[low:high])
## remaining = len(node[low:high])
for subnode in node[low:high]:
self.preorder(subnode)
remaining -= 1
if remaining > 0:
self.write(sep)
arg += 1
elif typ == 'P':
p = self.prec
low, high, sep, self.prec = entry[arg]
lastC = remaining = len(node[low:high])
## remaining = len(node[low:high])
for subnode in node[low:high]:
self.preorder(subnode)
remaining -= 1
if remaining > 0:
self.write(sep)
self.prec = p
arg += 1
elif typ == '{':
d = node.__dict__
expr = m.group('expr')
try:
self.write(eval(expr, d, d))
except:
print node
raise
m = escape.search(fmt, i)
self.write(fmt[i:])
def default(self, node):
mapping = MAP.get(node, MAP_DIRECT)
table = mapping[0]
key = node
for i in mapping[1:]:
key = key[i]
if table.has_key(key):
self.engine(table[key], node)
self.prune()
def customize(self, customize):
"""
Special handling for opcodes that take a variable number
of arguments -- we add a new entry for each in TABLE_R.
"""
for k, v in customize.items():
if TABLE_R.has_key(k):
continue
op = k[:k.rfind('_')]
if op == 'CALL_FUNCTION':
TABLE_R[k] = ('%c(%P)', 0, (1, -1, ', ', 100))
elif op in ('CALL_FUNCTION_VAR', 'CALL_FUNCTION_VAR_KW',
'CALL_FUNCTION_KW'):
if v == 0:
str = '%c(%C' # '%C' is a dummy here ...
p2 = (0, 0, None) # .. because of this
else:
str = '%c(%C, '
p2 = (1, -2, ', ')
if op == 'CALL_FUNCTION_VAR':
str += '*%c)'
entry = (str, 0, p2, -2)
elif op == 'CALL_FUNCTION_KW':
str += '**%c)'
entry = (str, 0, p2, -2)
else:
str += '*%c, **%c)'
if p2[2]: p2 = (1, -3, ', ')
entry = (str, 0, p2, -3, -2)
TABLE_R[k] = entry
## handled by n_mapexpr:
##if op == 'BUILD_SLICE': TABLE_R[k] = ('%C' , (0,-1,':'))
## handled by n_build_list:
##if op == 'BUILD_LIST': TABLE_R[k] = ('[%C]' , (0,-1,', '))
##elif op == 'BUILD_TUPLE': TABLE_R[k] = ('(%C%,)', (0,-1,', '))
def get_tuple_parameter(self, ast, name):
"""
If the name of the formal parameter starts with dot,
it's a tuple parameter, like this:
# def MyFunc(xx, (a,b,c), yy):
# print a, b*2, c*42
In byte-code, the whole tuple is assigned to parameter '.1' and
then the tuple gets unpacked to 'a', 'b' and 'c'.
Since identifiers starting with a dot are illegal in Python,
we can search for the byte-code equivalent to '(a,b,c) = .1'
"""
assert ast == 'stmts'
for i in range(len(ast)):
# search for an assign-statement
assert ast[i][0] == 'stmt'
node = ast[i][0][0]
if node == 'assign' \
and node[0] == ASSIGN_TUPLE_PARAM(name):
# okay, this assigns '.n' to something
del ast[i]
# walk lhs; this
# returns a tuple of identifiers as used
# within the function definition
assert node[1] == 'designator'
# if lhs is not a UNPACK_TUPLE (or equiv.),
# add parenteses to make this a tuple
#if node[1][0] not in ('unpack', 'unpack_list'):
return '(' + self.traverse(node[1]) + ')'
#return self.traverse(node[1])
raise Exception("Can't find tuple parameter " + name)
def make_function(self, node, isLambda, nested=1):
"""Dump function defintion, doc string, and function body."""
def build_param(ast, name, default):
"""build parameters:
- handle defaults
- handle format tuple parameters
"""
# if formal parameter is a tuple, the paramater name
# starts with a dot (eg. '.1', '.2')
if name.startswith('.'):
# replace the name with the tuple-string
name = self.get_tuple_parameter(ast, name)
if default:
if self.showast:
print '--', name
print default
print '--'
result = '%s = %s' % (name, self.traverse(default, indent=''))
if result[-2:] == '= ': # default was 'LOAD_CONST None'
result += 'None'
return result
else:
return name
defparams = node[:node[-1].attr] # node[-1] == MAKE_xxx_n
code = node[-2].attr
assert type(code) == CodeType
code = Code(code, self.scanner, self.currentclass)
#assert isinstance(code, Code)
# add defaults values to parameter names
argc = code.co_argcount
paramnames = list(code.co_varnames[:argc])
# defaults are for last n parameters, thus reverse
paramnames.reverse()
defparams.reverse()
try:
ast = self.build_ast(code._tokens,
code._customize,
isLambda=isLambda,
noneInNames=('None' in code.co_names))
except ParserError as p:
self.write(str(p))
self.ERROR = p
return
# build parameters
##This would be a nicer piece of code, but I can't get this to work
## now, have to find a usable lambda constuct hG/2000-09-05
##params = map(lambda name, default: build_param(ast, name, default),
## paramnames, defparams)
params = []
for name, default in map(lambda a, b: (a, b), paramnames, defparams):
params.append(build_param(ast, name, default))
params.reverse() # back to correct order
if 4 & code.co_flags: # flag 2 -> variable number of args
params.append('*%s' % code.co_varnames[argc])
argc += 1
if 8 & code.co_flags: # flag 3 -> keyword args
params.append('**%s' % code.co_varnames[argc])
argc += 1
# dump parameter list (with default values)
indent = self.indent
if isLambda:
self.write("lambda ", ", ".join(params), ": ")
else:
self.print_("(", ", ".join(params), "):")
#self.print_(indent, '#flags:\t', int(code.co_flags))
if len(code.co_consts
) > 0 and code.co_consts[0] != None and not isLambda: # ugly
# docstring exists, dump it
self.print_docstring(indent, code.co_consts[0])
code._tokens = None # save memory
assert ast == 'stmts'
#if isLambda:
# convert 'return' statement to expression
#assert len(ast[0]) == 1 wrong, see 'lambda (r,b): r,b,g'
#assert ast[-1] == 'stmt'
#assert len(ast[-1]) == 1
# assert ast[-1][0] == 'return_stmt'
# ast[-1][0].type = 'return_lambda'
#else:
# if ast[-1] == RETURN_NONE:
# Python adds a 'return None' to the
# end of any function; remove it
# ast.pop() # remove last node
all_globals = find_all_globals(ast, set())
for g in ((all_globals & self.mod_globs) | find_globals(ast, set())):
self.print_(self.indent, 'global ', g)
self.mod_globs -= all_globals
rn = ('None' in code.co_names) and not find_none(ast)
self.gen_source(ast, code._customize, isLambda=isLambda, returnNone=rn)
code._tokens = None
code._customize = None # save memory
def build_class(self, code):
"""Dump class definition, doc string and class body."""
assert type(code) == CodeType
code = Code(code, self.scanner, self.currentclass)
#assert isinstance(code, Code)
indent = self.indent
#self.print_(indent, '#flags:\t', int(code.co_flags))
ast = self.build_ast(code._tokens, code._customize)
code._tokens = None # save memory
assert ast == 'stmts'
if ast[0][0] == NAME_MODULE:
del ast[0]
# if docstring exists, dump it
if code.co_consts and code.co_consts[0] != None and ast[0][
0] == ASSIGN_DOC_STRING(code.co_consts[0]):
self.print_docstring(indent, code.co_consts[0])
self.print_()
del ast[0]
# the function defining a class normally returns locals(); we
# don't want this to show up in the source, thus remove the node
if ast[-1][0] == RETURN_LOCALS:
del ast[-1] # remove last node
#else:
# print ast[-1][-1]
for g in find_globals(ast, set()):
self.print_(indent, 'global ', g)
self.gen_source(ast, code._customize)
code._tokens = None
code._customize = None # save memory
def gen_source(self, ast, customize, isLambda=0, returnNone=False):
"""convert AST to source code"""
rn = self.return_none
self.return_none = returnNone
# if code would be empty, append 'pass'
if len(ast) == 0:
self.print_(self.indent, 'pass')
else:
self.customize(customize)
if isLambda:
self.write(self.traverse(ast, isLambda=isLambda))
else:
self.print_(self.traverse(ast, isLambda=isLambda))
self.return_none = rn
def build_ast(self, tokens, customize, isLambda=0, noneInNames=False):
assert type(tokens) == ListType
#assert isinstance(tokens[0], Token)
if isLambda:
tokens.append(Token('LAMBDA_MARKER'))
try:
ast = parser.parse(tokens, customize)
except parser.ParserError, e:
raise ParserError(e, tokens)
if self.showast:
self.print_(repr(ast))
return ast
if len(tokens) >= 2 and not noneInNames:
if tokens[-1] == Token('RETURN_VALUE'):
if tokens[-2] == Token('LOAD_CONST'):
del tokens[-2:]
else:
tokens.append(Token('RETURN_LAST'))
if len(tokens) == 0:
return PASS
# Build AST from disassembly.
try:
ast = parser.parse(tokens, customize)
except parser.ParserError, e:
raise ParserError(e, tokens)
def parse(self, tokens):
tokens.append(Token('eof', 'keyword'))
tokens.append(self._END_OF_FILE_CHARACTER)
last_idx = -1
idx = 0
self._parse_stack.append(self._END_OF_FILE_CHARACTER)
self._parse_stack.append(self._start_variable)
top = self._start_variable
loop_counter = 0
open_parentheses_count = 0
while top != self._END_OF_FILE_CHARACTER:
loop_counter += 1
last_token = tokens[-2]
if top == self._BOOLEAN_EXPRESSION_STRING and open_parentheses_count == 1 and tokens[idx - 1].value == '(':
self._parse_stack.pop()
top = self._get_parse_stack_top()
boolean_expression_tokens = []
while open_parentheses_count != 0:
boolean_expression_tokens.append(tokens[idx])
if tokens[idx].value == "(":
open_parentheses_count += 1
if tokens[idx].value == ")":
open_parentheses_count -= 1
idx += 1
idx -= 1
open_parentheses_count += 1
boolean_expression_tokens.pop()
try:
self._boolean_expression_parser.parse(boolean_expression_tokens)
except:
error_handler("Syntax Error:", " error in boolean expression")
continue
elif top == self._BOOLEAN_EXPRESSION_STRING and tokens[idx - 1].value == ';':
self._parse_stack.pop()
top = self._get_parse_stack_top()
boolean_expression_tokens = []
while tokens[idx].value != ";":
boolean_expression_tokens.append(tokens[idx])
if tokens[idx].value == "(":
open_parentheses_count += 1
if tokens[idx].value == ")":
open_parentheses_count -= 1
idx += 1
try:
self._boolean_expression_parser.parse(boolean_expression_tokens)
except:
error_handler("Syntax Error:", " error in boolean expression")
continue
top = self._get_parse_stack_top()
if top == self._NIL_STRING:
top = self._get_parse_stack_top()
continue
if self.is_semantic_rule(top):
semantic = top
self._parse_stack.pop()
top = self._get_parse_stack_top()
self._code_generator.generate_code(semantic, tokens[idx])
continue
if loop_counter > len(tokens) * 20:
error_handler("Syntax Error:", " (1) next token should not be " + str(tokens[idx]))
if top == self._IDENTIFIER_STRING:
if tokens[idx].type == TokenType.identifier:
idx = idx + 1
self._parse_stack.pop()
top = self._get_parse_stack_top()
continue
else:
error_handler("Syntax Error:", " (2) next token should not be " + str(tokens[idx]))
elif top == self._NUMBER_STRING:
if tokens[idx].type == TokenType.number:
idx = idx + 1
self._parse_stack.pop()
top = self._get_parse_stack_top()
continue
else:
error_handler("Syntax Error:", " (3) next token should not be " + str(tokens[idx]))
elif self.is_terminal(top):
if tokens[idx].value == "(":
open_parentheses_count += 1
if tokens[idx].value == ")":
open_parentheses_count -= 1
if top == "{":
self._symbol_table.one_scope_in()
if top == "}":
self._symbol_table.one_scope_out()
if tokens[idx].value == top:
idx = idx + 1
self._parse_stack.pop()
top = self._get_parse_stack_top()
continue
else:
error_handler("Syntax Error:", " (4) next token should not be " + str(tokens[idx]))
try:
try:
nxt = tokens[idx].value
if tokens[idx].type == TokenType.identifier:
nxt = self._IDENTIFIER_STRING
if tokens[idx].type == TokenType.number:
nxt = self._NUMBER_STRING
rule_idx = self._parse_table[top][nxt]
product = self._rules[rule_idx][1:]
self._parse_stack.pop()
if product != [self._NIL_STRING]:
self._parse_stack.extend(reversed(product))
except:
error_handler("Syntax Error:", " (5)")
except KeyError:
error_handler("Syntax Error:", "6: Unable to find derivation of '{0}' on '{1}'".format(top, nxt))
top = self._get_parse_stack_top()
return True
def eval_lexeme(lexeme: Lexeme) -> Token:
def eval_first_c(location: Location, c: str):
assert c is not None, "The evaluator received an empty lexeme."
if c.isdigit():
return partial(
eval_integer_literal,
sign=1,
value=ord(c) - ord('0'),
)
elif c == '+':
return eval_plus
elif c == '-':
return eval_minus
return partial(eval_symbol, content=[c])
def eval_integer_literal(location: Location, c: str, *, sign: int,
value: int):
if c is None:
return sign * value
if c.isdigit():
return partial(
eval_integer_literal,
sign=sign,
value=10 * value + (ord(c) - ord('0')),
)
if c == '_':
return partial(eval_integer_literal, sign=sign, value=value)
if c == '#':
return partial(
eval_integer_literal_with_base,
sign=sign,
value=0,
base=value,
)
if c == '.':
return partial(
eval_real_literal,
sign=sign,
value=Fraction(value),
base=10,
denominator=10,
)
raise LispError(
'Invalid character in base 10 integer literal',
location,
)
def eval_integer_literal_with_base(location: Location, c: str, *,
sign: int, value: int, base: int):
if c is None:
return sign * value
try:
digit = basedigit(c, base)
except ValueError:
pass
else:
return partial(
eval_integer_literal_with_base,
sign=sign,
value=base * value + digit,
base=base,
)
if c == '_':
return partial(
eval_integer_literal_with_base,
sign=sign,
value=value,
base=base,
)
if c == '.':
return partial(
eval_real_literal,
sign=sign,
value=Fraction(value),
base=base,
denominator=base,
)
raise LispError(
f'Invalid character in base {base} integer literal',
location,
)
def eval_real_literal(location: Location, c: str, *, sign, value, base,
denominator):
if c is None:
return sign * value
try:
digit = basedigit(c, base)
except ValueError:
pass
else:
return partial(
eval_real_literal,
sign=sign,
value=value + Fraction(digit, denominator),
base=base,
denominator=denominator * base,
)
if c == '_':
return partial(
eval_real_literal,
sign=sign,
value=value,
base=base,
denominator=denominator,
)
raise LispError(
f'Invalid character in base {base} real literal',
location,
)
def eval_plus(location: Location, c: str):
if c is None:
return Symbol('+')
if c.isdigit():
return partial(
eval_integer_literal,
sign=1,
value=ord(c) - ord('0'),
)
return partial(eval_symbol, content=[f'+{c}'])
def eval_minus(location: Location, c: str):
if c is None:
return Symbol('-')
if c.isdigit():
return partial(
eval_integer_literal,
sign=-1,
value=ord(c) - ord('0'),
)
return partial(eval_symbol, content=[f'-{c}'])
def eval_symbol(location: Location, c: str, *, content: t.List[str]):
if c is None:
return Symbol(''.join(content))
content.append(c)
return partial(eval_symbol, content=content)
state = eval_first_c
for location, c in enumerate_lexeme_with_locations(lexeme):
state = state(location, c)
return Token(lexeme.location, state(None, None))
def is_keyword(kind):
if Token.is_keyword(kind):
return True
if Token.get_name(kind) in [u"FOREACH",u"DOWHILE",u"IN",u"IMPORT"]:
return True
return False
def is_keyword(kind):
if Token.is_keyword(kind):
return True
if Token.get_name(kind) in ["FOREACH","DOWHILE"]:
return True
return False
def is_keyword(kind):
if Token.is_keyword(kind):
return True
if Token.get_name(kind) in ["FOREACH", "DOWHILE"]:
return True
return False
import sys, re, cStringIO
from types import ListType, TupleType, DictType, \
EllipsisType, IntType, CodeType
from spark import GenericASTTraversal
import parser
from parser import AST
from scanner import Token, Code
minint = -sys.maxint - 1
# Some ASTs used for comparing code fragments (like 'return None' at
# the end of functions).
RETURN_LOCALS = AST('return_stmt', [
AST('ret_expr', [AST('expr', [Token('LOAD_LOCALS')])]),
Token('RETURN_VALUE')
])
NONE = AST('expr', [Token('LOAD_CONST', pattr=None)])
RETURN_NONE = AST('stmt', [AST('return_stmt', [NONE, Token('RETURN_VALUE')])])
PASS = AST('stmts', [AST('sstmt', [AST('stmt', [AST('passstmt', [])])])])
ASSIGN_DOC_STRING = lambda doc_string: \
AST('stmt',
[ AST('assign',
[ AST('expr', [ Token('LOAD_CONST', pattr=doc_string) ]),
AST('designator', [ Token('STORE_NAME', pattr='__doc__')])
])])
def _create_filler_token(value='[filler value]',
type='[filler type]',
line=-1):
return Token(value=value, type=type, line=line)
def match(self, kind):
## if match return token, else ParseException
tok = self.dequeue()
if tok.kind != kind:
raise ParseException("cannot find token " + Token.get_name(kind) + " got " + str(tok) + " instead!")
return tok
