def parse_func_args(self):
look_token = self.lex.look_ahead()
exp_list = []
if look_token.kind == lexer.TokenKind.SEP_LPAREN:
self.lex.next_token()
if self.lex.look_ahead().kind != lexer.TokenKind.SEP_RPAREN:
exp_list = self.parse_exp_list()
self.lex.next_token_of_kind(lexer.TokenKind.SEP_RPAREN)
elif look_token.kind == lexer.TokenKind.SEP_LCURLY:
exp_list = [self.parse_table_constructor_exp()]
else:
exp_list = [
ast.String(self.lex.next_token_of_kind(
lexer.TokenKind.STRING)).data
]
return exp_list
def preparse(l: Lexer, delimiter: str) -> List[Node]:
token: Token = l.next_token()
elements: List[Node] = []
while token.ty != TokenType.TokenEOF:
if token.ty == TokenType.TokenIdentifier:
elements.append(ast.Name(token.val))
elif token.ty == TokenType.TokenIntegerLiteral:
elements.append(ast.Int(token.val))
elif token.ty == TokenType.TokenFloatLiteral:
elements.append(ast.Float(token.val))
elif token.ty == TokenType.TokenStringLiteral:
elements.append(ast.String(token.val))
elif token.ty == TokenType.TokenOpenParen:
tuple = ast.Tuple(preparse(l, '('))
elements.append(tuple)
elif token.ty == TokenType.TokenCloseParen:
if delimiter != '(':
raise PreparseError('read: unexpected \')\'')
return elements
elif token.ty == TokenType.TokenQuote:
quote: List[Node] = [ast.Name(constants.QUOTE)]
quote += preparse(l, '\'')
elements.append(ast.Tuple(quote))
elif token.ty == TokenType.TokenQuasiquote:
quasiquote: List[Node] = [ast.Name(constants.QUASIQUOTE)]
quasiquote += preparse(l, '`')
elements.append(ast.Tuple(quasiquote))
elif token.ty == TokenType.TokenUnquote:
unquote: List[Node] = [ast.Name(constants.UNQUOTE)]
unquote += preparse(l, ',')
elements.append(ast.Tuple(unquote))
elif token.ty == TokenType.TokenUnquoteSplicing:
unquote_splicing: List[Node] = [
ast.Name(constants.UNQUOTE_SPLICING)
]
unquote_splicing += preparse(l, ',@')
elements.append(ast.Tuple(unquote_splicing))
elif token.ty == TokenType.TokenError:
raise PreparseError(f'token error: {token.val}')
else:
raise PreparseError(f'unexpected token type: {token.ty}')
if delimiter in ('\'', '`', ',', ',@'):
return elements
token = l.next_token()
if delimiter != ' ':
raise PreparseError(f'unclosed delimiter, expected: \'{delimiter}\'')
return elements
def parse(s):
code = pp.Forward()
opcode = pp.Or([
pp.Literal('+'),
pp.Literal('-'),
pp.Literal('*'),
pp.Literal('/'),
pp.Literal('_'),
pp.Literal('='),
pp.Literal('>'),
pp.Literal('&'),
pp.Literal('|'),
pp.Literal('~'),
pp.Literal('$'),
pp.Literal('%'),
pp.Literal('\\'),
pp.Literal('@'),
pp.Literal('ø'),
pp.Literal('p'),
pp.Literal(':'),
pp.Literal(';'),
pp.Literal('!'),
pp.Literal('?'),
pp.Literal('#'),
]).setParseAction(lambda toks: ast.Opcode(toks[0]))
number = (pp.Word('1234567890').setParseAction(
lambda toks: ast.Number(int(toks[0]))))
str_def = ((pp.Literal('"') + pp.SkipTo(pp.Literal('"'), include=True)
).setParseAction(lambda toks: ast.String(toks[1])))
varname = (pp.Word(
'qwertyuiopasdfghjklzxcvbnm',
exact=1).setParseAction(lambda toks: ast.Varname(toks[0])))
fn_def = pp.Suppress(pp.Literal('[')) + code + pp.Suppress(pp.Literal(']'))
expr = pp.Or([opcode, number, varname, str_def, fn_def])
atom = pp.Or([expr])
code << pp.ZeroOrMore(atom)
code.setParseAction(lambda toks: ast.Function(toks))
return code.parseString(s)[0]
def parse_string(self):
return ast.String(self.cur_tok)
def p_string_literal(self, p):
"""string_literal : STRING"""
p[0] = ast.String(p[1])
def p_expression_string_constant(self, parse):
'''
expression : STRING
'''
parse[0] = AST.String(content=parse[1])
def translate(node, st=None, strings=None, funcName=False):
if isinstance(node, oast.Add):
left = translate(node.left, st, strings, funcName)
right = translate(node.right, st, strings, funcName)
return ast.Add(left, right)
elif isinstance(node, oast.And):
left = translate(node.nodes[0], st, strings, funcName)
right = translate(node.nodes[1], st, strings, funcName)
return ast.And(left, right)
elif isinstance(node, oast.Assign):
# Translate the right hand side first so it can use the older version
# of the left hand side.
exp = translate(node.expr, st, strings, funcName)
var = node.nodes.pop()
if isinstance(var, oast.AssAttr):
string = strings.setdefault(var.attrname, ast.String(var.attrname))
var = translate(var.expr, st, strings, funcName)
return ast.SetAttr(var, string, exp)
else:
var = translate(var, st, strings, funcName)
return ast.Assign(var, exp)
elif isinstance(node, oast.AssName):
return st.getSymbol(node.name, True)
elif isinstance(node, oast.CallFunc):
name = translate(node.node, st, strings, True)
args = [translate(a, st, strings) for a in node.args]
return ast.FunctionCall(name, *args)
elif isinstance(node, oast.Class):
bases = [translate(base, st, strings, funcName) for base in node.bases]
body = translate(node.code, st, strings, funcName)
body = ast.BasicBlock(body)
sym = st.getSymbol(node.name, True)
name = st.getName(node.name, True)
# This is here temporarily. It will be moved to the typify pass
# later.
sym['type'] = 'class'
klass = ast.Class(name, bases, body)
return ast.Assign(sym, klass)
elif isinstance(node, oast.Compare):
left = translate(node.expr, st, strings, funcName)
op, right = node.ops[0]
right = translate(right, st, strings, funcName)
if op == '==':
return ast.Eq(left, right)
elif op == '!=':
return ast.Ne(left, right)
elif op == 'is':
return ast.Is(left, right)
elif isinstance(node, oast.Const):
return ast.Integer(node.value)
elif isinstance(node, oast.Dict):
pairs = {}
for pair in node.items:
key, value = pair
key = translate(key, st, strings, funcName)
value = translate(value, st, strings, funcName)
pairs[key] = value
return ast.Dictionary(pairs)
elif isinstance(node, oast.Discard):
return translate(node.expr, st, strings, funcName)
elif isinstance(node, oast.Div):
left = translate(node.left, st, strings, funcName)
right = translate(node.right, st, strings, funcName)
return ast.Div(left, right)
elif isinstance(node, oast.Function):
sym = st.getSymbol(node.name, True)
name = st.getName(node.name, True)
sym['type'] = 'function'
newST = SymbolTable(st)
argSymbols = [
newST.getSymbol(argName, True) for argName in node.argnames
]
body = translate(node.code, newST, strings, funcName)
body = ast.BasicBlock(body)
fun = ast.Function(name, argSymbols, body, newST)
fun['simplified'] = False
st.update(newST)
return ast.Assign(sym, fun)
elif isinstance(node, oast.Getattr):
exp = translate(node.expr, st, strings, funcName)
name = strings.setdefault(node.attrname, ast.String(node.attrname))
return ast.GetAttr(exp, name)
elif isinstance(node, oast.If):
tests = node.tests
cond, then = tests.pop(0)
# Translate the conditional expression.
cond = translate(cond, st, strings)
# Snapshot the SymbolTable
st.snapshot()
# Translate the 'then' clause.
then = translate(then, st, strings, funcName)
then = ast.BasicBlock(then)
# Roll-back the SymbolTable for the 'else' clause.
st.rollback()
# Translate the 'else' clause.
if len(tests) > 0:
els = [translate(oast.If(tests, node.else_), st, funcName)]
else:
els = translate(node.else_, st, strings, funcName)
els = ast.BasicBlock(els)
return ast.If(cond, then, els, st)
elif isinstance(node, oast.IfExp):
cond = translate(node.test, st, strings, funcName)
then = translate(node.then, st, strings, funcName)
els = translate(node.else_, st, strings, funcName)
return ast.IfExp(cond, then, els)
elif isinstance(node, oast.Lambda):
name = st.getName('lambda', True)
newST = SymbolTable(st)
argSymbols = map(lambda name: newST.getSymbol(name, True),
node.argnames)
code = ast.Return(translate(node.code, newST, strings, funcName))
block = ast.BasicBlock([code])
fun = ast.Function(name, argSymbols, block, newST)
fun['simplified'] = False
st.update(newST)
return fun
elif isinstance(node, oast.List):
elements = []
for n in node.nodes:
elements.append(translate(n, st, strings, funcName))
return ast.List(elements)
elif isinstance(node, oast.Module):
# Create a new SymbolTable for this module.
st = SymbolTable()
strings = {}
children = translate(node.node, st, strings, funcName)
block = ast.BasicBlock(children)
fun = ast.Function(st.getBIF('main'), [], block, st)
# Mark the main function as migrated so that it doesn't get moved
# later.
fun['simplified'] = True
return ast.Module([fun], strings)
elif isinstance(node, oast.Mul):
left = translate(node.left, st, strings, funcName)
right = translate(node.right, st, strings, funcName)
return ast.Mul(left, right)
elif isinstance(node, oast.Name):
ret = 'input_int' if node.name == 'input' else node.name
if ret == 'input_int':
ret = st.getBIF(ret)
else:
if ret == 'True':
ret = ast.Tru()
elif ret == 'False':
ret = ast.Fals()
else:
ret = st.getSymbol(ret)
return ret
elif isinstance(node, oast.Not):
operand = translate(node.expr, st, strings, funcName)
return ast.Not(operand)
elif isinstance(node, oast.Or):
left = translate(node.nodes[0], st, strings, funcName)
right = translate(node.nodes[1], st, strings, funcName)
return ast.Or(left, right)
elif isinstance(node, oast.Printnl):
children = [
translate(e, st, strings, funcName) for e in node.getChildNodes()
]
children = util.flatten(children)
return ast.FunctionCall(st.getBIF('print_any'), *children)
elif isinstance(node, oast.Return):
return ast.Return(translate(node.value, st, strings, funcName))
elif isinstance(node, oast.Stmt):
stmts = [
translate(s, st, strings, funcName) for s in node.getChildNodes()
]
return util.flatten(stmts)
elif isinstance(node, oast.Sub):
left = translate(node.left, st, strings, funcName)
right = translate(node.right, st, strings, funcName)
return ast.Sub(left, right)
elif isinstance(node, oast.Subscript):
sym = translate(node.expr, st, strings, funcName)
sub = translate(node.subs[0], st, strings, funcName)
return ast.Subscript(sym, sub)
elif isinstance(node, oast.While):
cond = translate(node.test, st, strings, funcName)
body = translate(node.body, st, strings, funcName)
body = ast.BasicBlock(body)
return ast.While(cond, body, st)
elif isinstance(node, oast.UnarySub):
operand = translate(node.expr, st, strings, funcName)
return ast.Negate(operand)
else:
raise Exception("Unsupported AST node encountered: {}".format(
node.__class__.__name__))
def p_string(p):
'''string : STRING'''
p[0] = ast.String(p[1], lineno=p.lineno(1))
def string_indexing(p):
index = p[2].value
string = ast.String(p[0].getstr().strip("\""))
return ast.IndexOperation(string, index)
def string_exp_to_string(p):
return ast.String(p[0].getstr().strip("\""))
def val_string(s):
return ast.String(s[0].getstr())
# will still be parsed as program calls. This should be changed
# later when program call generation is fixed.
if self.peek(0).type == token.DOT and not self.peek(0).prec_by_space:
return True
if (self.peek(0).type == token.LBRACE
and self.peek(1).type == token.IDENTIFIER
and self.peek(2).type == token.COLON):
return True
def primary(self):
if tok := self.match(token.INT_VALUE):
return ast.Integer([tok], int(tok.lexeme))
if tok := self.match(token.FLOAT_VALUE):
return ast.Float([tok], float(tok.lexeme))
if tok := self.match(token.STRING):
return ast.String([tok], str(tok.lexeme))
if tok := self.match(token.BOOL_VALUE):
return ast.Bool([tok], True if self.peek(-1).lexeme=="true" else False)
toks = TokenList()
# parentheses ( ... )
if toks.add(self.match(token.LPAREN)):
exp = self.expression()
if not toks.add(self.match(token.RPAREN)):
raise ParseException("Missing closing parenthesis ).")
exp.tokens.extend(toks)
return exp
# array values [ ... ]
if toks.add(self.match(token.LBRACKET)):
if toks.add(self.match(token.RBRACKET)):
return ast.Array(toks, ast.ExpressionList([], []))
elements = self.parse_commata_expressions()
def parse(self, parser, tokens):
token = tokens.consume_expected('STRING')
return ast.String(token.value)
