def test_read_number(self):
token_stream = TokenStream(InputStream('123='))
result = token_stream._read_number()
self.assertEqual(Token('num', 123.0), result)
token_stream = TokenStream(InputStream('123.3.='))
result = token_stream._read_number()
self.assertEqual(Token('num', 123.3), result)
def test_next(self):
self.assertEqual(InputStream('').next(), '')
for length in range(10):
input_ = ''.join([choice(printable) for _ in range(length)])
input_stream = InputStream(input_)
for i in range(length):
self.assertEqual(input_[i], input_stream.next())
for i in range(length, 2 * length + 1):
self.assertEqual('', input_stream.next())
def test_read_string(self):
token_stream = TokenStream(InputStream('"ab"'))
result = token_stream._read_string()
self.assertEqual(Token('str', 'ab'), result)
token_stream = TokenStream(InputStream('"ab\\c"'))
result = token_stream._read_string()
self.assertEqual(Token('str', 'abc'), result)
token_stream = TokenStream(InputStream('"abc'))
with self.assertRaises(Exception):
token_stream._read_string()
def test_read_ident(self):
token_stream = TokenStream(InputStream('a=1'))
result = token_stream._read_identifier()
self.assertEqual(Token('var', 'a=1'), result)
token_stream = TokenStream(InputStream('a = 1'))
result = token_stream._read_identifier()
self.assertEqual(Token('var', 'a'), result)
token_stream = TokenStream(InputStream('let(a = 1'))
result = token_stream._read_identifier()
self.assertEqual(Token('kw', 'let'), result)
token_stream = TokenStream(InputStream('js "aaa"'))
result = token_stream._read_identifier()
self.assertEqual(Token('kw', 'js'), result)
def main():
code = "sum = lambda(x, y) x + y; print(sum(2, 3));"
code = """
fib = λ(n) if n < 2 then n else fib(n - 1) + fib(n - 2);
time( λ() println(fib(25)) );
"""
code = """
sum = lambda(n, ret)
if n == 0 then ret
else sum(n - 1, ret + n);
time(lambda() println(sum(50000, 0)));
"""
code = """
println("foo");
halt();
println("bar");
"""
global_env = Environment()
for name, func in primitive.items():
global_env.define(name, func)
with open(sys.argv[1]) as file:
code = file.read()
parser = Parser(TokenStream(InputStream(code)))
execute(evaluate, (parser(), global_env,
lambda result: print(f"*** Result: {result}")))
def main():
with open(sys.argv[1]) as file:
code = file.read()
# code = 'let foo(x = 1, y = 1) foo(x + y)'
# code = 'lambda foo(x) x'
parser = Parser(TokenStream(InputStream(code)))
js_code = to_js(parser())
print(js_code)
def main():
with open(sys.argv[1]) as file:
code = file.read()
parser = Parser(TokenStream(InputStream(code)))
cps_code = to_cps(parser(), lambda ast: CallAst(
VarAst('β_TOPLEVEL'),
[ast],
))
print(cps_code)
def test_read_next(self):
token_stream = TokenStream(
InputStream(' # comment\n123 abc "nba" let a=2 >= js;'))
self.assertEqual(token_stream._read_next(), Token('num', 123.0))
self.assertEqual(token_stream._read_next(), Token('var', 'abc'))
self.assertEqual(token_stream._read_next(), Token('str', 'nba'))
self.assertEqual(token_stream._read_next(), Token('kw', 'let'))
self.assertEqual(token_stream._read_next(), Token('var', 'a=2'))
self.assertEqual(token_stream._read_next(), Token('op', '>='))
self.assertEqual(token_stream._read_next(), Token('kw', 'js'))
self.assertEqual(token_stream._read_next(), Token('punc', ';'))
self.assertEqual(token_stream._read_next(), Token('null', 'null'))
token_stream = TokenStream(InputStream('\x08'))
with self.assertRaises(Exception):
token_stream._read_next()
token_stream = TokenStream(InputStream('λ (n) 1'))
self.assertEqual(token_stream._read_next(), Token("kw", 'λ'))
def main():
global_env = Environment()
for name, func in primitive.items():
global_env.define(name, func)
lambda_file_path = sys.argv[1]
with open(lambda_file_path) as file:
code = file.read()
parser = Parser(TokenStream(InputStream(code)))
evaluate(parser(), global_env)
def main():
with open(sys.argv[1]) as file:
code = file.read()
parser = Parser(TokenStream(InputStream(code)))
ast = parser()
ast = to_cps(ast, lambda ast: CallAst(VarAst('β_TOPLEVEL'), [ast]))
# print(ast)
ast = Optimizer().optimize(ast)
# print(ast)
js_code = to_js(ast)
print(js_code)
def test_peek_and_next(self):
token_stream = TokenStream(
InputStream(' # comment\n123 abc let a=2 >=;'))
self.assertEqual(token_stream.peek(), Token('num', 123.0))
self.assertEqual(token_stream.peek(), Token('num', 123.0))
self.assertEqual(token_stream.next(), Token('num', 123.0))
token_stream = TokenStream(
InputStream(' # comment\n123 abc let a=2 js >=;'))
self.assertEqual(token_stream.next(), Token('num', 123.0))
self.assertEqual(token_stream.next(), Token('var', 'abc'))
self.assertEqual(token_stream.next(), Token('kw', 'let'))
self.assertEqual(token_stream.next(), Token('var', 'a=2'))
self.assertEqual(token_stream.next(), Token('kw', 'js'))
token_stream = TokenStream(InputStream('λ (n) 1'))
self.assertEqual(token_stream.next(), Token("kw", "λ"))
self.assertEqual(token_stream.next(), Token("punc", "("))
self.assertEqual(token_stream.next(), Token("var", "n"))
self.assertEqual(token_stream.next(), Token("punc", ")"))
self.assertEqual(token_stream.next(), Token("num", 1.0))
def test_eof(self):
for length in range(10):
input_ = ''.join([choice(printable) for _ in range(length)])
input_stream = InputStream(input_)
for _ in range(length):
self.assertFalse(input_stream.eof())
input_stream.next()
self.assertTrue(input_stream.eof())
def test_to_cps(self):
js_raw_ast = JsAst("aa")
cps_ast = _cps_js_raw(js_raw_ast, lambda x: x)
self.assertEqual(cps_ast, js_raw_ast)
atom_ast = LiteralAst(1.0)
cps_ast = to_cps(atom_ast, lambda x: x)
self.assertEqual(atom_ast, cps_ast)
let_ast = LetAst([], LiteralAst(False))
cps_ast = to_cps(let_ast, lambda x: x)
self.assertEqual(cps_ast, LiteralAst(False))
prog_ast = ProgAst([])
cps_ast = to_cps(prog_ast, lambda x: x)
self.assertEqual(cps_ast, LiteralAst(False))
prog_ast = ProgAst([LiteralAst(1)])
cps_ast = to_cps(prog_ast, lambda x: x)
self.assertEqual(cps_ast, LiteralAst(1))
prog_ast = ProgAst([LiteralAst(1), LiteralAst(2)])
cps_ast = to_cps(prog_ast, lambda x: x)
self.assertEqual(cps_ast, ProgAst([LiteralAst(1), LiteralAst(2)]))
if_ast = IfAst(LiteralAst(1), LiteralAst(2), LiteralAst(3))
cps_ast: CallAst = to_cps(if_ast, lambda x: x)
expected_ast = CallAst(
LambdaAst(
'', cps_ast.func.params,
IfAst(LiteralAst(1),
CallAst(VarAst(cps_ast.func.params[0]), [LiteralAst(2)]),
CallAst(VarAst(cps_ast.func.params[0]),
[LiteralAst(3)]))), [
LambdaAst('', cps_ast.args[0].params,
VarAst(cps_ast.args[0].params[0]))
])
self.assertEqual(cps_ast, expected_ast)
lambda_ast = LambdaAst('', ['x', 'y'], LiteralAst(1))
cps_ast = to_cps(lambda_ast, lambda x: x)
expected_ast = LambdaAst(
'', [cps_ast.params[0]] + ['x', 'y'],
CallAst(VarAst(cps_ast.params[0]), [LiteralAst(1)]))
self.assertEqual(cps_ast, expected_ast)
binary_ast = BinaryAst('+', LiteralAst(1), LiteralAst(2))
cps_ast = to_cps(binary_ast, lambda x: x)
self.assertEqual(cps_ast, binary_ast)
parse = Parser(TokenStream(InputStream("a = foo(10);")))
cps_ast = to_cps(parse(), lambda x: x)
expected_ast = CallAst(VarAst('foo'), [
LambdaAst(
'', [cps_ast.args[0].params[0]],
AssignAst(VarAst('a'), VarAst(cps_ast.args[0].params[0]))),
LiteralAst(10)
])
self.assertEqual(cps_ast, expected_ast)
def main():
# code = "sum = lambda(x, y) x + y; print(sum(2, 3));"
code = """
fib = λ(n) if n < 2 then n else fib(n - 1) + fib(n - 2);
time( λ() println(fib(12)) );
"""
# code = "print(1 + 2 * 3)"
# code = """
# fib = λ(n) if n < 2 then n else fib(n - 1) + fib(n - 2);
# println(fib(8));
# """
parser = Parser(TokenStream(InputStream(code)))
global_env = Environment()
for name, func in primitive.items():
global_env.define(name, func)
evaluate(parser(), global_env, lambda result: result)
def test_eof(self):
token_stream = TokenStream(InputStream(' # comment\n'))
self.assertTrue(token_stream.eof())
def __init__(self, text):
self.input_stream = InputStream(text + '\n')
self.current = None
self.has_peeked = False
class TokenStream:
def __init__(self, text):
self.input_stream = InputStream(text + '\n')
self.current = None
self.has_peeked = False
def _is_whitespace(self, char):
return char in ' \t'
def _is_digit(self, char):
return char.isdigit()
def _is_operator(self, char):
return char in operators
def _read_while(self, predicate_func):
_str = ""
while not self.input_stream.is_eof() and predicate_func(
self.input_stream.peek()):
_str += self.input_stream.next()
return _str
def _read_number(self):
number = self._read_while(self._is_digit)
return {'type': 'num', 'value': int(number)}
def _read_operator(self):
return {'type': 'op', 'value': self.input_stream.next()}
def _read_next(self):
_ = self._read_while(self._is_whitespace)
if self.input_stream.is_eof():
return None
char = self.input_stream.peek()
if self._is_digit(char):
return self._read_number()
if self._is_operator(char):
return self._read_operator()
self.input_stream.croak("Can't handle character: " + char)
self.input_stream.next()
return None
def next(self):
if self.has_peeked:
self.has_peeked = False
return self.current
self.current = self._read_next()
return self.current
def peek(self):
if self.has_peeked:
return self.current
self.current = self._read_next()
self.has_peeked = True
return self.current
def is_eof(self):
return self.peek() is None
def croak(self, msg):
self.input_stream.croak(msg)
def test_evaluate(self):
ast = LiteralAst(1.0)
environment = Environment()
evaluate(ast, environment, lambda value: self.assertEqual(value, 1.0))
ast = LiteralAst(True)
environment = Environment()
evaluate(ast, environment, self.assertTrue)
ast = LiteralAst(False)
environment = Environment()
evaluate(ast, environment, self.assertFalse)
ast = LiteralAst("aaa")
evaluate(ast, Environment(),
lambda value: self.assertEqual(value, "aaa"))
ast = BinaryAst('+', LiteralAst(1), LiteralAst(2))
evaluate(ast, Environment(),
lambda value: self.assertEqual(value, 3.0))
ast = ProgAst([])
evaluate(ast, Environment(), self.assertFalse)
ast = ProgAst([LiteralAst(1)])
evaluate(ast, Environment(),
lambda value: self.assertEqual(value, 1.0))
ast = ProgAst([LiteralAst(1), LiteralAst(2)])
evaluate(ast, Environment(),
lambda value: self.assertEqual(value, 2.0))
ast = AssignAst(LiteralAst(1), LiteralAst("a"))
with self.assertRaises(Exception):
evaluate(ast, Environment(), lambda value: value)
ast = ProgAst([AssignAst(VarAst('a'), LiteralAst("foo")), VarAst('a')])
evaluate(ast, Environment(),
lambda value: self.assertEqual(value, "foo"))
ast = AssignAst(VarAst("a"), LiteralAst("foo"))
with self.assertRaises(Exception):
evaluate(ast, Environment(Environment()), lambda value: value)
ast = CallAst(
LambdaAst("", ["a"], VarAst("a")),
[LiteralAst(1)],
)
evaluate(ast, Environment(),
lambda value: self.assertEqual(value, 1.0))
ast = CallAst(LambdaAst("", ["a"], VarAst("a")), [LiteralAst("abc")])
evaluate(ast, Environment(),
lambda value: self.assertEqual(value, "abc"))
# # (λ loop (n) if n > 0 then n + loop(n - 1) else 0) (10)
ast = CallAst(
LambdaAst(
"loop", ["n"],
IfAst(
BinaryAst(">", VarAst("n"), LiteralAst(0)),
BinaryAst(
"+", VarAst("n"),
CallAst(VarAst("loop"),
[BinaryAst('-', VarAst('n'), LiteralAst(1))])),
LiteralAst(0))), [LiteralAst(10)])
evaluate(ast, Environment(),
lambda value: self.assertEqual(value, 55.0))
# # let (x) x;
ast = LetAst([VarDefAst("x", None)], VarAst("x"))
evaluate(ast, Environment(), self.assertFalse)
# # let (x = 2, y = x + 1, z = x + y) x + y + z
ast = LetAst([
VarDefAst("x", LiteralAst(2)),
VarDefAst("y", BinaryAst("+", VarAst("x"), LiteralAst(1))),
VarDefAst("z", BinaryAst("+", VarAst("x"), VarAst("y")))
], BinaryAst("+", BinaryAst("+", VarAst("x"), VarAst("y")),
VarAst("z")))
evaluate(ast, Environment(),
lambda value: self.assertEqual(value, 10.0))
# # the second expression will result an errors,
# since x, y, z are bound to the let body
# # let (x = 2, y = x + 1, z = x + y) x + y + z; x + y + z
ast = ProgAst([
LetAst([
VarDefAst('x', LiteralAst(2)),
VarDefAst('y', BinaryAst('+', VarAst('x'), LiteralAst(1))),
VarDefAst('z', BinaryAst('+', VarAst('x'), VarAst('y')))
],
BinaryAst('+', BinaryAst('+', VarAst('x'), VarAst('y')),
VarAst('z'))),
BinaryAst('+', BinaryAst('+', VarAst('x'), VarAst('y')),
VarAst('z'))
])
with self.assertRaises(Exception):
evaluate(ast, Environment(), lambda value: value)
ast = IfAst(LiteralAst(""), LiteralAst(1), None)
evaluate(ast, Environment(),
lambda value: self.assertEqual(value, 1.0))
ast = IfAst(LiteralAst(False), LiteralAst(1), LiteralAst(2))
evaluate(ast, Environment(),
lambda value: self.assertEqual(value, 2.0))
ast = IfAst(LiteralAst(False), LiteralAst(1), LiteralAst(False))
evaluate(ast, Environment(), self.assertFalse)
ast = {"type": "foo", "value": 'foo'}
with self.assertRaises(Exception):
evaluate(ast, Environment(), lambda value: value)
# fib = λ(n) if n < 2 then n else fib(n - 1) + fib(n - 2);
# fib(6);
#
ast = ProgAst([
AssignAst(
VarAst('fib'),
LambdaAst(
'n', ['n'],
IfAst(
BinaryAst('<', VarAst('n'), LiteralAst(2)),
VarAst('n'),
BinaryAst(
'+',
CallAst(
VarAst('fib'),
[BinaryAst('-', VarAst('n'), LiteralAst(1))]),
CallAst(
VarAst('fib'),
[BinaryAst('-', VarAst('n'), LiteralAst(2))
]))))),
CallAst(VarAst('fib'), [LiteralAst(6)])
])
evaluate(ast, Environment(),
lambda value: self.assertEqual(value, 8.0))
ast = IfAst(LiteralAst(False), LiteralAst(1), LiteralAst(False))
evaluate(ast, Environment(), self.assertFalse)
ast = CallAst(LiteralAst(1), [])
with self.assertRaises(Exception):
evaluate(ast, Environment(), self.assertFalse)
code = """
2 + twice(3, 4)
"""
global_env = Environment()
for name, func in primitive.items():
global_env.define(name, func)
parser = Parser(TokenStream(InputStream(code)))
evaluate(parser(), global_env, lambda result: result)
if token.type != 'op':
return left
his_prec = self.PRECEDENCE[token.value]
if his_prec > my_prec:
self._token_stream.next()
right = self._maybe_binary(self._parse_atom(), his_prec)
if token.value == '=':
binary = AssignAst(left, right)
else:
binary = BinaryAst(token.value, left, right)
return self._maybe_binary(binary, my_prec)
return left
def __call__(self) -> ProgAst:
return self._parse_toplevel()
def unexpected(self):
"""
raise exception with error msg and error location
whenever encountered error.
"""
self._token_stream.croak(
f'Unexpected token: {self._token_stream.peek()}')
if __name__ == '__main__':
with open(sys.argv[1]) as f:
code = f.read()
ast = Parser(TokenStream(InputStream(code)))()
print(ast)
def test_croak(self):
token_stream = TokenStream(InputStream(' # comment\n'))
with self.assertRaises(Exception):
token_stream.croak('foo')
def test_skip_comment(self):
token_stream = TokenStream(InputStream('# abc\ndef'))
token_stream._skip_comment()
self.assertEqual(token_stream._input_stream.peek(), 'd')
def test_read_while(self):
token_stream = TokenStream(InputStream('ab123='))
result = token_stream._read_while(lambda ch: ch.isalnum())
self.assertEqual(result, 'ab123')