def match(self, kind): ## if match return token, else ParseException tok = self.dequeue() if (tok.kind != kind): raise ParseException(u"cannot find token "+ \ Token.get_name(kind) + u" got " \ + unicode(tok) + u" instead!") return tok
def match(self,kind): ## if match return token, else ParseException tok = self.dequeue() if ( tok.kind != kind ): raise ParseException(u"cannot find token "+ \ Token.get_name(kind) + u" got " \ + unicode(tok) + u" instead!") return tok
def _set_dimension(self, statement, dimension): assignment = statement.children[0] expression = assignment.children[1] arithop = expression.children[0] relation = arithop.children[0] term = relation.children[0] factor = term.children[0] number = factor.children[0] token = Token(dimension, "NUMBER") number.set_token(token) return statement
def _parse_type_mark(self): type_mark = ParseTreeNode('type_mark') if self._next_token_value_matches(Parser.DATA_TYPES): type_mark.set_token(self._next()) else: token = self._next() report_expected_vs_encountered(str(Parser.DATA_TYPES), token.value, token.line) self._register_error() type_mark.set_token( Token(value='[filler]', type='WORD', line=token.line)) return type_mark
def __init__(self, scanner): self.it = scanner.get_Token() #词法的迭代器 self.token = Token() self.x = None self.y = None self.x_scale = None self.y_scale = None self.rot = None #旋转角度 self.start = None #T的起始 self.end = None #T的终点 self.step = None #步长 self.x_origin = None self.y_origin = None
def new_tokens(lis): ''' Добавляет тоекны в список Принимает список с параметрами токенов ''' global tokens tokens = [] for line in lis: tokens_line = [] for element in line: token = Token(element[0], element[1], element[2], element[3]) tokens_line += [token] tokens += [tokens_line]
def build_ast(self, tokens, customize, isLambda=0, noneInNames=False): assert type(tokens) == ListType #assert isinstance(tokens[0], Token) if isLambda: tokens.append(Token('LAMBDA_MARKER')) try: ast = parser.parse(tokens, customize) except parser.ParserError, e: raise ParserError(e, tokens) if self.showast: self.print_(repr(ast)) return ast
def n_return_if_stmt(self, node): if self.__params['isLambda']: self.preorder(node[0]) self.prune() else: self.write(self.indent, 'return') if self.return_none or node != AST( 'return_stmt', [AST('ret_expr', [NONE]), Token('RETURN_END_IF')]): self.write(' ') self.preorder(node[0]) self.print_() self.prune() # stop recursing
class Room: ''' Класс для всех построенных комнат ''' # Высота комнаты height = Token(0, 0, "HEIGHT", HEIGHT) # Ширина комнаты width = Token(0, 0, "WIDTH", WIDTH) # Левая дверь left = None # Правая дверь right = None # Верхняя дверь up = None # Нижняя дверь down = None # Название картинки img = None def __str__(self): ''' Возвращает строку с описанием комнаты ''' string = "height: " + str(self.height.value) + ", width: " + str( self.width.value) if self.left != None: string += ", left: " + self.left.value if self.right != None: string += ", right: " + self.right.value if self.up != None: string += ", up: " + self.up.value if self.down != None: string += ", down: " + self.down.value if self.img != None: string += ", img: " + self.img.value return string
def _index(self): identifier = ParseTreeNode("identifier") token = Token(value=self._index_identifier, type="WORD") identifier.set_token(token) name = ParseTreeNode("name") name.add_child(identifier) factor = ParseTreeNode("factor") factor.add_child(name) term = ParseTreeNode("term") term.add_child(factor) relation = ParseTreeNode("relation") relation.add_child(term) arithop = ParseTreeNode("arithop") arithop.add_child(relation) expression = ParseTreeNode("expression") expression.add_child(arithop) return expression
def n_yield(self, node): self.write('yield') if node != AST('yield', [NONE, Token('YIELD_VALUE')]): self.write(' ') self.preorder(node[0]) self.prune() # stop recursing
class Walker(GenericASTTraversal, object): stacked_params = ('f', 'indent', 'isLambda', '_globals') def __init__(self, out, scanner, showast=0): GenericASTTraversal.__init__(self, ast=None) self.scanner = scanner params = { 'f': out, 'indent': '', } self.showast = showast self.__params = params self.__param_stack = [] self.ERROR = None self.prec = 100 self.return_none = False self.mod_globs = set() self.currentclass = None self.pending_newlines = 0 f = property(lambda s: s.__params['f'], lambda s, x: s.__params.__setitem__('f', x), lambda s: s.__params.__delitem__('f'), None) indent = property(lambda s: s.__params['indent'], lambda s, x: s.__params.__setitem__('indent', x), lambda s: s.__params.__delitem__('indent'), None) isLambda = property(lambda s: s.__params['isLambda'], lambda s, x: s.__params.__setitem__('isLambda', x), lambda s: s.__params.__delitem__('isLambda'), None) _globals = property(lambda s: s.__params['_globals'], lambda s, x: s.__params.__setitem__('_globals', x), lambda s: s.__params.__delitem__('_globals'), None) def indentMore(self, indent=TAB): self.indent += indent def indentLess(self, indent=TAB): self.indent = self.indent[:-len(indent)] def traverse(self, node, indent=None, isLambda=0): self.__param_stack.append(self.__params) if indent is None: indent = self.indent p = self.pending_newlines self.pending_newlines = 0 self.__params = { '_globals': {}, 'f': cStringIO.StringIO(), 'indent': indent, 'isLambda': isLambda, } self.preorder(node) self.f.write('\n' * self.pending_newlines) result = self.f.getvalue() self.__params = self.__param_stack.pop() self.pending_newlines = p return result def write(self, *data): if (len(data) == 0) or (len(data) == 1 and data[0] == ''): return # import pdb; pdb.set_trace() out = ''.join((str(j) for j in data)) n = 0 for i in out: if i == '\n': n += 1 if n == len(out): self.pending_newlines = max(self.pending_newlines, n) return elif n: self.pending_newlines = max(self.pending_newlines, n) out = out[n:] break else: break if self.pending_newlines > 0: self.f.write('\n' * self.pending_newlines) self.pending_newlines = 0 for i in out[::-1]: if i == '\n': self.pending_newlines += 1 else: break if self.pending_newlines: out = out[:-self.pending_newlines] self.f.write(out) def print_(self, *data): if data and not (len(data) == 1 and data[0] == ''): self.write(*data) self.pending_newlines = max(self.pending_newlines, 1) def print_docstring(self, indent, docstring): quote = '"""' self.write(indent) if type(docstring) == unicode: self.write('u') docstring = repr(docstring.expandtabs())[2:-1] else: docstring = repr(docstring.expandtabs())[1:-1] for (orig, replace) in (('\\\\', '\t'), ('\\r\\n', '\n'), ('\\n', '\n'), ('\\r', '\n'), ('\\"', '"'), ("\\'", "'")): docstring = docstring.replace(orig, replace) #Do a raw string if there are backslashes but no other escaped characters: #also check some edge cases if ('\t' in docstring and '\\' not in docstring and len(docstring) >= 2 and docstring[-1] != '\t' and (docstring[-1] != '"' or docstring[-2] == '\t')): self.write('r') #raw string #restore backslashes unescaped since raw docstring = docstring.replace('\t', '\\') else: #Escape '"' if it's the last character, so it doesn't ruin the ending triple quote if len(docstring) and docstring[-1] == '"': docstring = docstring[:-1] + '\\"' #Escape triple quote anywhere docstring = docstring.replace('"""', '\\"\\"\\"') #Restore escaped backslashes docstring = docstring.replace('\t', '\\\\') lines = docstring.split('\n') calculate_indent = sys.maxint for line in lines[1:]: stripped = line.lstrip() if len(stripped) > 0: calculate_indent = min(calculate_indent, len(line) - len(stripped)) calculate_indent = min(calculate_indent, len(lines[-1]) - len(lines[-1].lstrip())) # Remove indentation (first line is special): trimmed = [lines[0]] if calculate_indent < sys.maxint: trimmed += [line[calculate_indent:] for line in lines[1:]] self.write(quote) if len(trimmed) == 0: self.print_(quote) elif len(trimmed) == 1: self.print_(trimmed[0], quote) else: self.print_(trimmed[0]) for line in trimmed[1:-1]: self.print_(indent, line) self.print_(indent, trimmed[-1], quote) def n_return_stmt(self, node): if self.__params['isLambda']: self.preorder(node[0]) self.prune() else: self.write(self.indent, 'return') if self.return_none or node != AST( 'return_stmt', [AST('ret_expr', [NONE]), Token('RETURN_VALUE')]): self.write(' ') self.preorder(node[0]) self.print_() self.prune() # stop recursing def n_return_if_stmt(self, node): if self.__params['isLambda']: self.preorder(node[0]) self.prune() else: self.write(self.indent, 'return') if self.return_none or node != AST( 'return_stmt', [AST('ret_expr', [NONE]), Token('RETURN_END_IF')]): self.write(' ') self.preorder(node[0]) self.print_() self.prune() # stop recursing def n_yield(self, node): self.write('yield') if node != AST('yield', [NONE, Token('YIELD_VALUE')]): self.write(' ') self.preorder(node[0]) self.prune() # stop recursing def n_buildslice3(self, node): p = self.prec self.prec = 100 if node[0] != NONE: self.preorder(node[0]) self.write(':') if node[1] != NONE: self.preorder(node[1]) self.write(':') if node[2] != NONE: self.preorder(node[2]) self.prec = p self.prune() # stop recursing def n_buildslice2(self, node): p = self.prec self.prec = 100 if node[0] != NONE: self.preorder(node[0]) self.write(':') if node[1] != NONE: self.preorder(node[1]) self.prec = p self.prune() # stop recursing # def n_l_stmts(self, node): # if node[0] == '_stmts': # if len(node[0]) >= 2 and node[0][1] == 'stmt': # if node[0][-1][0] == 'continue_stmt': # del node[0][-1] # self.default(node) def n_expr(self, node): p = self.prec if node[0].type.startswith('binary_expr'): n = node[0][-1][0] else: n = node[0] self.prec = PRECEDENCE.get(n, -2) if n == 'LOAD_CONST' and repr(n.pattr)[0] == '-': self.prec = 6 if p < self.prec: self.write('(') self.preorder(node[0]) self.write(')') else: self.preorder(node[0]) self.prec = p self.prune() def n_ret_expr(self, node): if len(node) == 1 and node[0] == 'expr': self.n_expr(node[0]) else: self.n_expr(node) n_ret_expr_or_cond = n_expr def n_binary_expr(self, node): self.preorder(node[0]) self.write(' ') self.preorder(node[-1]) self.write(' ') self.prec -= 1 self.preorder(node[1]) self.prec += 1 self.prune() def n_LOAD_CONST(self, node): data = node.pattr datatype = type(data) if datatype is IntType and data == minint: # convert to hex, since decimal representation # would result in 'LOAD_CONST; UNARY_NEGATIVE' # change:hG/2002-02-07: this was done for all negative integers # todo: check whether this is necessary in Python 2.1 self.write(hex(data)) elif datatype is EllipsisType: self.write('...') elif data is None: # LOAD_CONST 'None' only occurs, when None is # implicit eg. in 'return' w/o params # pass self.write('None') else: self.write(repr(data)) # LOAD_CONST is a terminal, so stop processing/recursing early self.prune() def n_delete_subscr(self, node): if node[-2][0] == 'build_list' and node[-2][0][-1].type.startswith( 'BUILD_TUPLE'): if node[-2][0][-1] != 'BUILD_TUPLE_0': node[-2][0].type = 'build_tuple2' self.default(node) # maybe_tuple = node[-2][-1] # if maybe_tuple.type.startswith('BUILD_TUPLE'): # maybe_tuple.type = 'build_tuple2' # self.default(node) n_store_subscr = n_binary_subscr = n_delete_subscr # 'tryfinallystmt': ( '%|try:\n%+%c%-%|finally:\n%+%c%-', 1, 5 ), def n_tryfinallystmt(self, node): if len(node[1][0]) == 1 and node[1][0][0] == 'stmt': if node[1][0][0][0] == 'trystmt': node[1][0][0][0].type = 'tf_trystmt' if node[1][0][0][0] == 'tryelsestmt': node[1][0][0][0].type = 'tf_tryelsestmt' self.default(node) def n_exec_stmt(self, node): """ exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT exec_stmt ::= expr exprlist EXEC_STMT """ self.write(self.indent, 'exec ') self.preorder(node[0]) if node[1][0] != NONE: sep = ' in ' for subnode in node[1]: self.write(sep) sep = ", " self.preorder(subnode) self.print_() self.prune() # stop recursing def n_ifelsestmt(self, node, preprocess=0): n = node[3][0] if len(n) == 1 == len(n[0]) and n[0] == '_stmts': n = n[0][0][0] elif n[0].type in ('lastc_stmt', 'lastl_stmt'): n = n[0][0] else: if not preprocess: self.default(node) return if n.type in ('ifstmt', 'iflaststmt', 'iflaststmtl'): node.type = 'ifelifstmt' n.type = 'elifstmt' elif n.type in ('ifelsestmtr', ): node.type = 'ifelifstmt' n.type = 'elifelsestmtr' elif n.type in ('ifelsestmt', 'ifelsestmtc', 'ifelsestmtl'): node.type = 'ifelifstmt' self.n_ifelsestmt(n, preprocess=1) if n == 'ifelifstmt': n.type = 'elifelifstmt' elif n.type in ('ifelsestmt', 'ifelsestmtc', 'ifelsestmtl'): n.type = 'elifelsestmt' if not preprocess: self.default(node) n_ifelsestmtc = n_ifelsestmtl = n_ifelsestmt def n_ifelsestmtr(self, node): if len(node[2]) != 2: self.default(node) if not (node[2][0][0][0] == 'ifstmt' and node[2][0][0][0][1][0] == 'return_if_stmts') \ and not (node[2][0][-1][0] == 'ifstmt' and node[2][0][-1][0][1][0] == 'return_if_stmts'): self.default(node) return self.write(self.indent, 'if ') self.preorder(node[0]) self.print_(':') self.indentMore() self.preorder(node[1]) self.indentLess() if_ret_at_end = False if len(node[2][0]) >= 3: if node[2][0][-1][0] == 'ifstmt' and node[2][0][-1][0][1][ 0] == 'return_if_stmts': if_ret_at_end = True past_else = False prev_stmt_is_if_ret = True for n in node[2][0]: if (n[0] == 'ifstmt' and n[0][1][0] == 'return_if_stmts'): if prev_stmt_is_if_ret: n[0].type = 'elifstmt' prev_stmt_is_if_ret = True else: prev_stmt_is_if_ret = False if not past_else and not if_ret_at_end: self.print_(self.indent, 'else:') self.indentMore() past_else = True self.preorder(n) if not past_else or if_ret_at_end: self.print_(self.indent, 'else:') self.indentMore() self.preorder(node[2][1]) self.indentLess() self.prune() def n_elifelsestmtr(self, node): if len(node[2]) != 2: self.default(node) for n in node[2][0]: if not (n[0] == 'ifstmt' and n[0][1][0] == 'return_if_stmts'): self.default(node) return self.write(self.indent, 'elif ') self.preorder(node[0]) self.print_(':') self.indentMore() self.preorder(node[1]) self.indentLess() if_ret_at_end = False if len(node[2][0]) >= 3: if node[2][0][-1][0] == 'ifstmt' and node[2][0][-1][0][1][ 0] == 'return_if_stmts': if_ret_at_end = True past_else = False prev_stmt_is_if_ret = True for n in node[2][0]: n[0].type = 'elifstmt' self.preorder(n) self.print_(self.indent, 'else:') self.indentMore() self.preorder(node[2][1]) self.indentLess() self.prune() def n_import_as(self, node): iname = node[0].pattr assert node[-1][-1].type.startswith('STORE_') sname = node[-1][-1].pattr # assume one of STORE_.... here if iname == sname or iname.startswith(sname + '.'): self.write(iname) else: self.write(iname, ' as ', sname) self.prune() # stop recursing n_import_as_cont = n_import_as def n_importfrom(self, node): if node[0].pattr > 0: node[2].pattr = '.' * node[0].pattr + node[2].pattr self.default(node) n_importstar = n_importfrom def n_mkfunc(self, node): self.write(node[-2].attr.co_name) # = code.co_name self.indentMore() self.make_function(node, isLambda=0) if len(self.__param_stack) > 1: self.write('\n\n') else: self.write('\n\n\n') self.indentLess() self.prune() # stop recursing def n_mklambda(self, node): self.make_function(node, isLambda=1) self.prune() # stop recursing def n_list_compr(self, node): p = self.prec self.prec = 27 n = node[-1] assert n == 'list_iter' # find innerst node while n == 'list_iter': n = n[0] # recurse one step if n == 'list_for': n = n[3] elif n == 'list_if': n = n[2] elif n == 'list_if_not': n = n[2] assert n == 'lc_body' self.write('[ ') self.preorder(n[0]) # lc_body self.preorder(node[-1]) # for/if parts self.write(' ]') self.prec = p self.prune() # stop recursing def comprehension_walk(self, node, iter_index): p = self.prec self.prec = 27 code = node[-5].attr assert type(code) == CodeType code = Code(code, self.scanner, self.currentclass) #assert isinstance(code, Code) ast = self.build_ast(code._tokens, code._customize) self.customize(code._customize) ast = ast[0][0][0] n = ast[iter_index] assert n == 'comp_iter' # find innerst node while n == 'comp_iter': n = n[0] # recurse one step if n == 'comp_for': n = n[3] elif n == 'comp_if': n = n[2] elif n == 'comp_ifnot': n = n[2] assert n == 'comp_body', ast self.preorder(n[0]) self.write(' for ') self.preorder(ast[iter_index - 1]) self.write(' in ') self.preorder(node[-3]) self.preorder(ast[iter_index]) self.prec = p def n_genexpr(self, node): self.write('(') self.comprehension_walk(node, 3) self.write(')') self.prune() def n_setcomp(self, node): self.write('{') self.comprehension_walk(node, 4) self.write('}') self.prune() n_dictcomp = n_setcomp def n_classdef(self, node): # class definition ('class X(A,B,C):') cclass = self.currentclass self.currentclass = str(node[0].pattr) self.write('\n\n') self.write(self.indent, 'class ', self.currentclass) self.print_super_classes(node) self.print_(':') # class body self.indentMore() self.build_class(node[2][-2].attr) self.indentLess() self.currentclass = cclass if len(self.__param_stack) > 1: self.write('\n\n') else: self.write('\n\n\n') self.prune() n_classdefdeco2 = n_classdef def print_super_classes(self, node): node = node[1][0] if not (node == 'build_list'): return self.write('(') line_separator = ', ' sep = '' for elem in node[:-1]: value = self.traverse(elem) self.write(sep, value) sep = line_separator self.write(')') def n_mapexpr(self, node): """ prettyprint a mapexpr 'mapexpr' is something like k = {'a': 1, 'b': 42 }" """ p = self.prec self.prec = 100 assert node[-1] == 'kvlist' node = node[-1] # goto kvlist self.indentMore(INDENT_PER_LEVEL) line_seperator = ',\n' + self.indent sep = INDENT_PER_LEVEL[:-1] self.write('{') for kv in node: assert kv in ('kv', 'kv2', 'kv3') # kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR # kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR # kv3 ::= expr expr STORE_MAP if kv == 'kv': name = self.traverse(kv[-2], indent='') value = self.traverse(kv[1], indent=self.indent + (len(name) + 2) * ' ') elif kv == 'kv2': name = self.traverse(kv[1], indent='') value = self.traverse(kv[-3], indent=self.indent + (len(name) + 2) * ' ') elif kv == 'kv3': name = self.traverse(kv[-2], indent='') value = self.traverse(kv[0], indent=self.indent + (len(name) + 2) * ' ') self.write(sep, name, ': ', value) sep = line_seperator self.write('}') self.indentLess(INDENT_PER_LEVEL) self.prec = p self.prune() def n_build_list(self, node): """ prettyprint a list or tuple """ p = self.prec self.prec = 100 lastnode = node.pop().type if lastnode.startswith('BUILD_LIST'): self.write('[') endchar = ']' elif lastnode.startswith('BUILD_TUPLE'): self.write('(') endchar = ')' elif lastnode.startswith('BUILD_SET'): self.write('{') endchar = '}' elif lastnode.startswith('ROT_TWO'): self.write('(') endchar = ')' else: raise 'Internal Error: n_build_list expects list or tuple' self.indentMore(INDENT_PER_LEVEL) if len(node) > 3: line_separator = ',\n' + self.indent else: line_separator = ', ' sep = INDENT_PER_LEVEL[:-1] for elem in node: if (elem == 'ROT_THREE'): continue assert elem == 'expr' value = self.traverse(elem) self.write(sep, value) sep = line_separator if len(node) == 1 and lastnode.startswith('BUILD_TUPLE'): self.write(',') self.write(endchar) self.indentLess(INDENT_PER_LEVEL) self.prec = p self.prune() def n_unpack(self, node): for n in node[1:]: if n[0].type == 'unpack': n[0].type = 'unpack_w_parens' self.default(node) n_unpack_w_parens = n_unpack def n_assign2(self, node): for n in node[-2:]: if n[0] == 'unpack': n[0].type = 'unpack_w_parens' self.default(node) def n_assign3(self, node): for n in node[-3:]: if n[0] == 'unpack': n[0].type = 'unpack_w_parens' self.default(node) def n_except_cond2(self, node): if node[5][0] == 'unpack': node[5][0].type = 'unpack_w_parens' self.default(node) def engine(self, entry, startnode): #self.print_("-----") #self.print_(str(startnode.__dict__)) fmt = entry[0] ## no longer used, since BUILD_TUPLE_n is pretty printed: ##lastC = 0 arg = 1 i = 0 m = escape.search(fmt) while m: i = m.end() self.write(m.group('prefix')) typ = m.group('type') or '{' node = startnode try: if m.group('child'): node = node[int(m.group('child'))] except: print node.__dict__ raise if typ == '%': self.write('%') elif typ == '+': self.indentMore() elif typ == '-': self.indentLess() elif typ == '|': self.write(self.indent) ## no longer used, since BUILD_TUPLE_n is pretty printed: elif typ == ',': if lastC == 1: self.write(',') elif typ == 'c': self.preorder(node[entry[arg]]) arg += 1 elif typ == 'p': p = self.prec (index, self.prec) = entry[arg] self.preorder(node[index]) self.prec = p arg += 1 elif typ == 'C': low, high, sep = entry[arg] lastC = remaining = len(node[low:high]) ## remaining = len(node[low:high]) for subnode in node[low:high]: self.preorder(subnode) remaining -= 1 if remaining > 0: self.write(sep) arg += 1 elif typ == 'P': p = self.prec low, high, sep, self.prec = entry[arg] lastC = remaining = len(node[low:high]) ## remaining = len(node[low:high]) for subnode in node[low:high]: self.preorder(subnode) remaining -= 1 if remaining > 0: self.write(sep) self.prec = p arg += 1 elif typ == '{': d = node.__dict__ expr = m.group('expr') try: self.write(eval(expr, d, d)) except: print node raise m = escape.search(fmt, i) self.write(fmt[i:]) def default(self, node): mapping = MAP.get(node, MAP_DIRECT) table = mapping[0] key = node for i in mapping[1:]: key = key[i] if table.has_key(key): self.engine(table[key], node) self.prune() def customize(self, customize): """ Special handling for opcodes that take a variable number of arguments -- we add a new entry for each in TABLE_R. """ for k, v in customize.items(): if TABLE_R.has_key(k): continue op = k[:k.rfind('_')] if op == 'CALL_FUNCTION': TABLE_R[k] = ('%c(%P)', 0, (1, -1, ', ', 100)) elif op in ('CALL_FUNCTION_VAR', 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'): if v == 0: str = '%c(%C' # '%C' is a dummy here ... p2 = (0, 0, None) # .. because of this else: str = '%c(%C, ' p2 = (1, -2, ', ') if op == 'CALL_FUNCTION_VAR': str += '*%c)' entry = (str, 0, p2, -2) elif op == 'CALL_FUNCTION_KW': str += '**%c)' entry = (str, 0, p2, -2) else: str += '*%c, **%c)' if p2[2]: p2 = (1, -3, ', ') entry = (str, 0, p2, -3, -2) TABLE_R[k] = entry ## handled by n_mapexpr: ##if op == 'BUILD_SLICE': TABLE_R[k] = ('%C' , (0,-1,':')) ## handled by n_build_list: ##if op == 'BUILD_LIST': TABLE_R[k] = ('[%C]' , (0,-1,', ')) ##elif op == 'BUILD_TUPLE': TABLE_R[k] = ('(%C%,)', (0,-1,', ')) def get_tuple_parameter(self, ast, name): """ If the name of the formal parameter starts with dot, it's a tuple parameter, like this: # def MyFunc(xx, (a,b,c), yy): # print a, b*2, c*42 In byte-code, the whole tuple is assigned to parameter '.1' and then the tuple gets unpacked to 'a', 'b' and 'c'. Since identifiers starting with a dot are illegal in Python, we can search for the byte-code equivalent to '(a,b,c) = .1' """ assert ast == 'stmts' for i in range(len(ast)): # search for an assign-statement assert ast[i][0] == 'stmt' node = ast[i][0][0] if node == 'assign' \ and node[0] == ASSIGN_TUPLE_PARAM(name): # okay, this assigns '.n' to something del ast[i] # walk lhs; this # returns a tuple of identifiers as used # within the function definition assert node[1] == 'designator' # if lhs is not a UNPACK_TUPLE (or equiv.), # add parenteses to make this a tuple #if node[1][0] not in ('unpack', 'unpack_list'): return '(' + self.traverse(node[1]) + ')' #return self.traverse(node[1]) raise Exception("Can't find tuple parameter " + name) def make_function(self, node, isLambda, nested=1): """Dump function defintion, doc string, and function body.""" def build_param(ast, name, default): """build parameters: - handle defaults - handle format tuple parameters """ # if formal parameter is a tuple, the paramater name # starts with a dot (eg. '.1', '.2') if name.startswith('.'): # replace the name with the tuple-string name = self.get_tuple_parameter(ast, name) if default: if self.showast: print '--', name print default print '--' result = '%s = %s' % (name, self.traverse(default, indent='')) if result[-2:] == '= ': # default was 'LOAD_CONST None' result += 'None' return result else: return name defparams = node[:node[-1].attr] # node[-1] == MAKE_xxx_n code = node[-2].attr assert type(code) == CodeType code = Code(code, self.scanner, self.currentclass) #assert isinstance(code, Code) # add defaults values to parameter names argc = code.co_argcount paramnames = list(code.co_varnames[:argc]) # defaults are for last n parameters, thus reverse paramnames.reverse() defparams.reverse() try: ast = self.build_ast(code._tokens, code._customize, isLambda=isLambda, noneInNames=('None' in code.co_names)) except ParserError as p: self.write(str(p)) self.ERROR = p return # build parameters ##This would be a nicer piece of code, but I can't get this to work ## now, have to find a usable lambda constuct hG/2000-09-05 ##params = map(lambda name, default: build_param(ast, name, default), ## paramnames, defparams) params = [] for name, default in map(lambda a, b: (a, b), paramnames, defparams): params.append(build_param(ast, name, default)) params.reverse() # back to correct order if 4 & code.co_flags: # flag 2 -> variable number of args params.append('*%s' % code.co_varnames[argc]) argc += 1 if 8 & code.co_flags: # flag 3 -> keyword args params.append('**%s' % code.co_varnames[argc]) argc += 1 # dump parameter list (with default values) indent = self.indent if isLambda: self.write("lambda ", ", ".join(params), ": ") else: self.print_("(", ", ".join(params), "):") #self.print_(indent, '#flags:\t', int(code.co_flags)) if len(code.co_consts ) > 0 and code.co_consts[0] != None and not isLambda: # ugly # docstring exists, dump it self.print_docstring(indent, code.co_consts[0]) code._tokens = None # save memory assert ast == 'stmts' #if isLambda: # convert 'return' statement to expression #assert len(ast[0]) == 1 wrong, see 'lambda (r,b): r,b,g' #assert ast[-1] == 'stmt' #assert len(ast[-1]) == 1 # assert ast[-1][0] == 'return_stmt' # ast[-1][0].type = 'return_lambda' #else: # if ast[-1] == RETURN_NONE: # Python adds a 'return None' to the # end of any function; remove it # ast.pop() # remove last node all_globals = find_all_globals(ast, set()) for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): self.print_(self.indent, 'global ', g) self.mod_globs -= all_globals rn = ('None' in code.co_names) and not find_none(ast) self.gen_source(ast, code._customize, isLambda=isLambda, returnNone=rn) code._tokens = None code._customize = None # save memory def build_class(self, code): """Dump class definition, doc string and class body.""" assert type(code) == CodeType code = Code(code, self.scanner, self.currentclass) #assert isinstance(code, Code) indent = self.indent #self.print_(indent, '#flags:\t', int(code.co_flags)) ast = self.build_ast(code._tokens, code._customize) code._tokens = None # save memory assert ast == 'stmts' if ast[0][0] == NAME_MODULE: del ast[0] # if docstring exists, dump it if code.co_consts and code.co_consts[0] != None and ast[0][ 0] == ASSIGN_DOC_STRING(code.co_consts[0]): self.print_docstring(indent, code.co_consts[0]) self.print_() del ast[0] # the function defining a class normally returns locals(); we # don't want this to show up in the source, thus remove the node if ast[-1][0] == RETURN_LOCALS: del ast[-1] # remove last node #else: # print ast[-1][-1] for g in find_globals(ast, set()): self.print_(indent, 'global ', g) self.gen_source(ast, code._customize) code._tokens = None code._customize = None # save memory def gen_source(self, ast, customize, isLambda=0, returnNone=False): """convert AST to source code""" rn = self.return_none self.return_none = returnNone # if code would be empty, append 'pass' if len(ast) == 0: self.print_(self.indent, 'pass') else: self.customize(customize) if isLambda: self.write(self.traverse(ast, isLambda=isLambda)) else: self.print_(self.traverse(ast, isLambda=isLambda)) self.return_none = rn def build_ast(self, tokens, customize, isLambda=0, noneInNames=False): assert type(tokens) == ListType #assert isinstance(tokens[0], Token) if isLambda: tokens.append(Token('LAMBDA_MARKER')) try: ast = parser.parse(tokens, customize) except parser.ParserError, e: raise ParserError(e, tokens) if self.showast: self.print_(repr(ast)) return ast if len(tokens) >= 2 and not noneInNames: if tokens[-1] == Token('RETURN_VALUE'): if tokens[-2] == Token('LOAD_CONST'): del tokens[-2:] else: tokens.append(Token('RETURN_LAST')) if len(tokens) == 0: return PASS # Build AST from disassembly. try: ast = parser.parse(tokens, customize) except parser.ParserError, e: raise ParserError(e, tokens)
def parse(self, tokens): tokens.append(Token('eof', 'keyword')) tokens.append(self._END_OF_FILE_CHARACTER) last_idx = -1 idx = 0 self._parse_stack.append(self._END_OF_FILE_CHARACTER) self._parse_stack.append(self._start_variable) top = self._start_variable loop_counter = 0 open_parentheses_count = 0 while top != self._END_OF_FILE_CHARACTER: loop_counter += 1 last_token = tokens[-2] if top == self._BOOLEAN_EXPRESSION_STRING and open_parentheses_count == 1 and tokens[idx - 1].value == '(': self._parse_stack.pop() top = self._get_parse_stack_top() boolean_expression_tokens = [] while open_parentheses_count != 0: boolean_expression_tokens.append(tokens[idx]) if tokens[idx].value == "(": open_parentheses_count += 1 if tokens[idx].value == ")": open_parentheses_count -= 1 idx += 1 idx -= 1 open_parentheses_count += 1 boolean_expression_tokens.pop() try: self._boolean_expression_parser.parse(boolean_expression_tokens) except: error_handler("Syntax Error:", " error in boolean expression") continue elif top == self._BOOLEAN_EXPRESSION_STRING and tokens[idx - 1].value == ';': self._parse_stack.pop() top = self._get_parse_stack_top() boolean_expression_tokens = [] while tokens[idx].value != ";": boolean_expression_tokens.append(tokens[idx]) if tokens[idx].value == "(": open_parentheses_count += 1 if tokens[idx].value == ")": open_parentheses_count -= 1 idx += 1 try: self._boolean_expression_parser.parse(boolean_expression_tokens) except: error_handler("Syntax Error:", " error in boolean expression") continue top = self._get_parse_stack_top() if top == self._NIL_STRING: top = self._get_parse_stack_top() continue if self.is_semantic_rule(top): semantic = top self._parse_stack.pop() top = self._get_parse_stack_top() self._code_generator.generate_code(semantic, tokens[idx]) continue if loop_counter > len(tokens) * 20: error_handler("Syntax Error:", " (1) next token should not be " + str(tokens[idx])) if top == self._IDENTIFIER_STRING: if tokens[idx].type == TokenType.identifier: idx = idx + 1 self._parse_stack.pop() top = self._get_parse_stack_top() continue else: error_handler("Syntax Error:", " (2) next token should not be " + str(tokens[idx])) elif top == self._NUMBER_STRING: if tokens[idx].type == TokenType.number: idx = idx + 1 self._parse_stack.pop() top = self._get_parse_stack_top() continue else: error_handler("Syntax Error:", " (3) next token should not be " + str(tokens[idx])) elif self.is_terminal(top): if tokens[idx].value == "(": open_parentheses_count += 1 if tokens[idx].value == ")": open_parentheses_count -= 1 if top == "{": self._symbol_table.one_scope_in() if top == "}": self._symbol_table.one_scope_out() if tokens[idx].value == top: idx = idx + 1 self._parse_stack.pop() top = self._get_parse_stack_top() continue else: error_handler("Syntax Error:", " (4) next token should not be " + str(tokens[idx])) try: try: nxt = tokens[idx].value if tokens[idx].type == TokenType.identifier: nxt = self._IDENTIFIER_STRING if tokens[idx].type == TokenType.number: nxt = self._NUMBER_STRING rule_idx = self._parse_table[top][nxt] product = self._rules[rule_idx][1:] self._parse_stack.pop() if product != [self._NIL_STRING]: self._parse_stack.extend(reversed(product)) except: error_handler("Syntax Error:", " (5)") except KeyError: error_handler("Syntax Error:", "6: Unable to find derivation of '{0}' on '{1}'".format(top, nxt)) top = self._get_parse_stack_top() return True
def eval_lexeme(lexeme: Lexeme) -> Token: def eval_first_c(location: Location, c: str): assert c is not None, "The evaluator received an empty lexeme." if c.isdigit(): return partial( eval_integer_literal, sign=1, value=ord(c) - ord('0'), ) elif c == '+': return eval_plus elif c == '-': return eval_minus return partial(eval_symbol, content=[c]) def eval_integer_literal(location: Location, c: str, *, sign: int, value: int): if c is None: return sign * value if c.isdigit(): return partial( eval_integer_literal, sign=sign, value=10 * value + (ord(c) - ord('0')), ) if c == '_': return partial(eval_integer_literal, sign=sign, value=value) if c == '#': return partial( eval_integer_literal_with_base, sign=sign, value=0, base=value, ) if c == '.': return partial( eval_real_literal, sign=sign, value=Fraction(value), base=10, denominator=10, ) raise LispError( 'Invalid character in base 10 integer literal', location, ) def eval_integer_literal_with_base(location: Location, c: str, *, sign: int, value: int, base: int): if c is None: return sign * value try: digit = basedigit(c, base) except ValueError: pass else: return partial( eval_integer_literal_with_base, sign=sign, value=base * value + digit, base=base, ) if c == '_': return partial( eval_integer_literal_with_base, sign=sign, value=value, base=base, ) if c == '.': return partial( eval_real_literal, sign=sign, value=Fraction(value), base=base, denominator=base, ) raise LispError( f'Invalid character in base {base} integer literal', location, ) def eval_real_literal(location: Location, c: str, *, sign, value, base, denominator): if c is None: return sign * value try: digit = basedigit(c, base) except ValueError: pass else: return partial( eval_real_literal, sign=sign, value=value + Fraction(digit, denominator), base=base, denominator=denominator * base, ) if c == '_': return partial( eval_real_literal, sign=sign, value=value, base=base, denominator=denominator, ) raise LispError( f'Invalid character in base {base} real literal', location, ) def eval_plus(location: Location, c: str): if c is None: return Symbol('+') if c.isdigit(): return partial( eval_integer_literal, sign=1, value=ord(c) - ord('0'), ) return partial(eval_symbol, content=[f'+{c}']) def eval_minus(location: Location, c: str): if c is None: return Symbol('-') if c.isdigit(): return partial( eval_integer_literal, sign=-1, value=ord(c) - ord('0'), ) return partial(eval_symbol, content=[f'-{c}']) def eval_symbol(location: Location, c: str, *, content: t.List[str]): if c is None: return Symbol(''.join(content)) content.append(c) return partial(eval_symbol, content=content) state = eval_first_c for location, c in enumerate_lexeme_with_locations(lexeme): state = state(location, c) return Token(lexeme.location, state(None, None))
def is_keyword(kind): if Token.is_keyword(kind): return True if Token.get_name(kind) in [u"FOREACH",u"DOWHILE",u"IN",u"IMPORT"]: return True return False
def is_keyword(kind): if Token.is_keyword(kind): return True if Token.get_name(kind) in ["FOREACH","DOWHILE"]: return True return False
def is_keyword(kind): if Token.is_keyword(kind): return True if Token.get_name(kind) in ["FOREACH", "DOWHILE"]: return True return False
import sys, re, cStringIO from types import ListType, TupleType, DictType, \ EllipsisType, IntType, CodeType from spark import GenericASTTraversal import parser from parser import AST from scanner import Token, Code minint = -sys.maxint - 1 # Some ASTs used for comparing code fragments (like 'return None' at # the end of functions). RETURN_LOCALS = AST('return_stmt', [ AST('ret_expr', [AST('expr', [Token('LOAD_LOCALS')])]), Token('RETURN_VALUE') ]) NONE = AST('expr', [Token('LOAD_CONST', pattr=None)]) RETURN_NONE = AST('stmt', [AST('return_stmt', [NONE, Token('RETURN_VALUE')])]) PASS = AST('stmts', [AST('sstmt', [AST('stmt', [AST('passstmt', [])])])]) ASSIGN_DOC_STRING = lambda doc_string: \ AST('stmt', [ AST('assign', [ AST('expr', [ Token('LOAD_CONST', pattr=doc_string) ]), AST('designator', [ Token('STORE_NAME', pattr='__doc__')]) ])])
def _create_filler_token(value='[filler value]', type='[filler type]', line=-1): return Token(value=value, type=type, line=line)
def match(self, kind): ## if match return token, else ParseException tok = self.dequeue() if tok.kind != kind: raise ParseException("cannot find token " + Token.get_name(kind) + " got " + str(tok) + " instead!") return tok