def parse_func_args(self): look_token = self.lex.look_ahead() exp_list = [] if look_token.kind == lexer.TokenKind.SEP_LPAREN: self.lex.next_token() if self.lex.look_ahead().kind != lexer.TokenKind.SEP_RPAREN: exp_list = self.parse_exp_list() self.lex.next_token_of_kind(lexer.TokenKind.SEP_RPAREN) elif look_token.kind == lexer.TokenKind.SEP_LCURLY: exp_list = [self.parse_table_constructor_exp()] else: exp_list = [ ast.String(self.lex.next_token_of_kind( lexer.TokenKind.STRING)).data ] return exp_list
def preparse(l: Lexer, delimiter: str) -> List[Node]: token: Token = l.next_token() elements: List[Node] = [] while token.ty != TokenType.TokenEOF: if token.ty == TokenType.TokenIdentifier: elements.append(ast.Name(token.val)) elif token.ty == TokenType.TokenIntegerLiteral: elements.append(ast.Int(token.val)) elif token.ty == TokenType.TokenFloatLiteral: elements.append(ast.Float(token.val)) elif token.ty == TokenType.TokenStringLiteral: elements.append(ast.String(token.val)) elif token.ty == TokenType.TokenOpenParen: tuple = ast.Tuple(preparse(l, '(')) elements.append(tuple) elif token.ty == TokenType.TokenCloseParen: if delimiter != '(': raise PreparseError('read: unexpected \')\'') return elements elif token.ty == TokenType.TokenQuote: quote: List[Node] = [ast.Name(constants.QUOTE)] quote += preparse(l, '\'') elements.append(ast.Tuple(quote)) elif token.ty == TokenType.TokenQuasiquote: quasiquote: List[Node] = [ast.Name(constants.QUASIQUOTE)] quasiquote += preparse(l, '`') elements.append(ast.Tuple(quasiquote)) elif token.ty == TokenType.TokenUnquote: unquote: List[Node] = [ast.Name(constants.UNQUOTE)] unquote += preparse(l, ',') elements.append(ast.Tuple(unquote)) elif token.ty == TokenType.TokenUnquoteSplicing: unquote_splicing: List[Node] = [ ast.Name(constants.UNQUOTE_SPLICING) ] unquote_splicing += preparse(l, ',@') elements.append(ast.Tuple(unquote_splicing)) elif token.ty == TokenType.TokenError: raise PreparseError(f'token error: {token.val}') else: raise PreparseError(f'unexpected token type: {token.ty}') if delimiter in ('\'', '`', ',', ',@'): return elements token = l.next_token() if delimiter != ' ': raise PreparseError(f'unclosed delimiter, expected: \'{delimiter}\'') return elements
def parse(s): code = pp.Forward() opcode = pp.Or([ pp.Literal('+'), pp.Literal('-'), pp.Literal('*'), pp.Literal('/'), pp.Literal('_'), pp.Literal('='), pp.Literal('>'), pp.Literal('&'), pp.Literal('|'), pp.Literal('~'), pp.Literal('$'), pp.Literal('%'), pp.Literal('\\'), pp.Literal('@'), pp.Literal('ø'), pp.Literal('p'), pp.Literal(':'), pp.Literal(';'), pp.Literal('!'), pp.Literal('?'), pp.Literal('#'), ]).setParseAction(lambda toks: ast.Opcode(toks[0])) number = (pp.Word('1234567890').setParseAction( lambda toks: ast.Number(int(toks[0])))) str_def = ((pp.Literal('"') + pp.SkipTo(pp.Literal('"'), include=True) ).setParseAction(lambda toks: ast.String(toks[1]))) varname = (pp.Word( 'qwertyuiopasdfghjklzxcvbnm', exact=1).setParseAction(lambda toks: ast.Varname(toks[0]))) fn_def = pp.Suppress(pp.Literal('[')) + code + pp.Suppress(pp.Literal(']')) expr = pp.Or([opcode, number, varname, str_def, fn_def]) atom = pp.Or([expr]) code << pp.ZeroOrMore(atom) code.setParseAction(lambda toks: ast.Function(toks)) return code.parseString(s)[0]
def parse_string(self): return ast.String(self.cur_tok)
def p_string_literal(self, p): """string_literal : STRING""" p[0] = ast.String(p[1])
def p_expression_string_constant(self, parse): ''' expression : STRING ''' parse[0] = AST.String(content=parse[1])
def translate(node, st=None, strings=None, funcName=False): if isinstance(node, oast.Add): left = translate(node.left, st, strings, funcName) right = translate(node.right, st, strings, funcName) return ast.Add(left, right) elif isinstance(node, oast.And): left = translate(node.nodes[0], st, strings, funcName) right = translate(node.nodes[1], st, strings, funcName) return ast.And(left, right) elif isinstance(node, oast.Assign): # Translate the right hand side first so it can use the older version # of the left hand side. exp = translate(node.expr, st, strings, funcName) var = node.nodes.pop() if isinstance(var, oast.AssAttr): string = strings.setdefault(var.attrname, ast.String(var.attrname)) var = translate(var.expr, st, strings, funcName) return ast.SetAttr(var, string, exp) else: var = translate(var, st, strings, funcName) return ast.Assign(var, exp) elif isinstance(node, oast.AssName): return st.getSymbol(node.name, True) elif isinstance(node, oast.CallFunc): name = translate(node.node, st, strings, True) args = [translate(a, st, strings) for a in node.args] return ast.FunctionCall(name, *args) elif isinstance(node, oast.Class): bases = [translate(base, st, strings, funcName) for base in node.bases] body = translate(node.code, st, strings, funcName) body = ast.BasicBlock(body) sym = st.getSymbol(node.name, True) name = st.getName(node.name, True) # This is here temporarily. It will be moved to the typify pass # later. sym['type'] = 'class' klass = ast.Class(name, bases, body) return ast.Assign(sym, klass) elif isinstance(node, oast.Compare): left = translate(node.expr, st, strings, funcName) op, right = node.ops[0] right = translate(right, st, strings, funcName) if op == '==': return ast.Eq(left, right) elif op == '!=': return ast.Ne(left, right) elif op == 'is': return ast.Is(left, right) elif isinstance(node, oast.Const): return ast.Integer(node.value) elif isinstance(node, oast.Dict): pairs = {} for pair in node.items: key, value = pair key = translate(key, st, strings, funcName) value = translate(value, st, strings, funcName) pairs[key] = value return ast.Dictionary(pairs) elif isinstance(node, oast.Discard): return translate(node.expr, st, strings, funcName) elif isinstance(node, oast.Div): left = translate(node.left, st, strings, funcName) right = translate(node.right, st, strings, funcName) return ast.Div(left, right) elif isinstance(node, oast.Function): sym = st.getSymbol(node.name, True) name = st.getName(node.name, True) sym['type'] = 'function' newST = SymbolTable(st) argSymbols = [ newST.getSymbol(argName, True) for argName in node.argnames ] body = translate(node.code, newST, strings, funcName) body = ast.BasicBlock(body) fun = ast.Function(name, argSymbols, body, newST) fun['simplified'] = False st.update(newST) return ast.Assign(sym, fun) elif isinstance(node, oast.Getattr): exp = translate(node.expr, st, strings, funcName) name = strings.setdefault(node.attrname, ast.String(node.attrname)) return ast.GetAttr(exp, name) elif isinstance(node, oast.If): tests = node.tests cond, then = tests.pop(0) # Translate the conditional expression. cond = translate(cond, st, strings) # Snapshot the SymbolTable st.snapshot() # Translate the 'then' clause. then = translate(then, st, strings, funcName) then = ast.BasicBlock(then) # Roll-back the SymbolTable for the 'else' clause. st.rollback() # Translate the 'else' clause. if len(tests) > 0: els = [translate(oast.If(tests, node.else_), st, funcName)] else: els = translate(node.else_, st, strings, funcName) els = ast.BasicBlock(els) return ast.If(cond, then, els, st) elif isinstance(node, oast.IfExp): cond = translate(node.test, st, strings, funcName) then = translate(node.then, st, strings, funcName) els = translate(node.else_, st, strings, funcName) return ast.IfExp(cond, then, els) elif isinstance(node, oast.Lambda): name = st.getName('lambda', True) newST = SymbolTable(st) argSymbols = map(lambda name: newST.getSymbol(name, True), node.argnames) code = ast.Return(translate(node.code, newST, strings, funcName)) block = ast.BasicBlock([code]) fun = ast.Function(name, argSymbols, block, newST) fun['simplified'] = False st.update(newST) return fun elif isinstance(node, oast.List): elements = [] for n in node.nodes: elements.append(translate(n, st, strings, funcName)) return ast.List(elements) elif isinstance(node, oast.Module): # Create a new SymbolTable for this module. st = SymbolTable() strings = {} children = translate(node.node, st, strings, funcName) block = ast.BasicBlock(children) fun = ast.Function(st.getBIF('main'), [], block, st) # Mark the main function as migrated so that it doesn't get moved # later. fun['simplified'] = True return ast.Module([fun], strings) elif isinstance(node, oast.Mul): left = translate(node.left, st, strings, funcName) right = translate(node.right, st, strings, funcName) return ast.Mul(left, right) elif isinstance(node, oast.Name): ret = 'input_int' if node.name == 'input' else node.name if ret == 'input_int': ret = st.getBIF(ret) else: if ret == 'True': ret = ast.Tru() elif ret == 'False': ret = ast.Fals() else: ret = st.getSymbol(ret) return ret elif isinstance(node, oast.Not): operand = translate(node.expr, st, strings, funcName) return ast.Not(operand) elif isinstance(node, oast.Or): left = translate(node.nodes[0], st, strings, funcName) right = translate(node.nodes[1], st, strings, funcName) return ast.Or(left, right) elif isinstance(node, oast.Printnl): children = [ translate(e, st, strings, funcName) for e in node.getChildNodes() ] children = util.flatten(children) return ast.FunctionCall(st.getBIF('print_any'), *children) elif isinstance(node, oast.Return): return ast.Return(translate(node.value, st, strings, funcName)) elif isinstance(node, oast.Stmt): stmts = [ translate(s, st, strings, funcName) for s in node.getChildNodes() ] return util.flatten(stmts) elif isinstance(node, oast.Sub): left = translate(node.left, st, strings, funcName) right = translate(node.right, st, strings, funcName) return ast.Sub(left, right) elif isinstance(node, oast.Subscript): sym = translate(node.expr, st, strings, funcName) sub = translate(node.subs[0], st, strings, funcName) return ast.Subscript(sym, sub) elif isinstance(node, oast.While): cond = translate(node.test, st, strings, funcName) body = translate(node.body, st, strings, funcName) body = ast.BasicBlock(body) return ast.While(cond, body, st) elif isinstance(node, oast.UnarySub): operand = translate(node.expr, st, strings, funcName) return ast.Negate(operand) else: raise Exception("Unsupported AST node encountered: {}".format( node.__class__.__name__))
def p_string(p): '''string : STRING''' p[0] = ast.String(p[1], lineno=p.lineno(1))
def string_indexing(p): index = p[2].value string = ast.String(p[0].getstr().strip("\"")) return ast.IndexOperation(string, index)
def string_exp_to_string(p): return ast.String(p[0].getstr().strip("\""))
def val_string(s): return ast.String(s[0].getstr())
# will still be parsed as program calls. This should be changed # later when program call generation is fixed. if self.peek(0).type == token.DOT and not self.peek(0).prec_by_space: return True if (self.peek(0).type == token.LBRACE and self.peek(1).type == token.IDENTIFIER and self.peek(2).type == token.COLON): return True def primary(self): if tok := self.match(token.INT_VALUE): return ast.Integer([tok], int(tok.lexeme)) if tok := self.match(token.FLOAT_VALUE): return ast.Float([tok], float(tok.lexeme)) if tok := self.match(token.STRING): return ast.String([tok], str(tok.lexeme)) if tok := self.match(token.BOOL_VALUE): return ast.Bool([tok], True if self.peek(-1).lexeme=="true" else False) toks = TokenList() # parentheses ( ... ) if toks.add(self.match(token.LPAREN)): exp = self.expression() if not toks.add(self.match(token.RPAREN)): raise ParseException("Missing closing parenthesis ).") exp.tokens.extend(toks) return exp # array values [ ... ] if toks.add(self.match(token.LBRACKET)): if toks.add(self.match(token.RBRACKET)): return ast.Array(toks, ast.ExpressionList([], [])) elements = self.parse_commata_expressions()
def parse(self, parser, tokens): token = tokens.consume_expected('STRING') return ast.String(token.value)