def test_function_literal_parsing(self):
line = "fn(x, y) { x + y; }"
lex = lexer_.Lexer(input=line)
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
assert len(program.statements) == 1
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
function = stmt.expression
assert type(function) is ast_.FunctionLiteral
assert len(function.parameters) == 2
assert self.check_literal_expression(function.parameters[0], "x")
assert self.check_literal_expression(function.parameters[1], "y")
assert len(function.body.statements) == 1
body_stmt = function.body.statements[0]
assert type(body_stmt) is ast_.ExpressionStatement
assert self.check_infix_expression(body_stmt.expression, "x", "+", "y")
for v in program.statements:
print(v.string())
print(type(v))
print(type(v.expression))
print(type(v.expression.token))
print(type(v.expression.string()))
print(v.expression.string())
def test_Eval(self, input):
lex = lexer_.Lexer(input)
p = parser_.Parser(lex)
program = p.parse_program()
env = env_.NewEnvironment()
return evaluator_.Eval(program, env)
def test_infix(self):
line = """
5 + 5.;
5 - 5.;
5 * 5.;
5 / 5.;
5 > 5.;
5 < 5.;
5 == 5.;
5 != 5.;
"""
lex = lexer_.Lexer(input=line)
obj = parser_.Parser(lex)
program = obj.parse_program()
print(program)
print(len(program.statements))
assert self.check_parser_errors(obj)
for v in program.statements:
print(v.string())
print(v.expression.operator)
print(v.expression.left.value)
print(v.expression.right.value)
print("-" * 30)
def test_next_token1(self):
line = """
let five = 5.2.36;
let ten = 10;
let add = fn(x, y) {
x + y;
};
let result = add(five, ten);
!-/*5;
5 < 10.236 > 5;
if (5 < 10) {
return true;
} else {
return false;
}
10 == 10;
10 != .9;
"foobar"
"foo bar"
[1, 2];
"""
lex = lexer_.Lexer(input=line)
while True:
tok = lex.next_token()
print(tok.token_type)
print(tok.literal)
print("-----")
if tok.token_type == token_.TokenType.EOF:
break
def test_parsing_infix_expressions(self):
tests = [
("5 + 5;", 5, "+", 5),
("5 - 5;", 5, "-", 5),
("5 * 5;", 5, "*", 5),
("5 / 5;", 5, "/", 5),
("5 > 5;", 5, ">", 5),
("5 < 5;", 5, "<", 5),
("5 == 5;", 5, "==", 5),
("5 != 5;", 5, "!=", 5),
("foobar + barfoo;", "foobar", "+", "barfoo"),
("foobar - barfoo;", "foobar", "-", "barfoo"),
("foobar * barfoo;", "foobar", "*", "barfoo"),
("foobar / barfoo;", "foobar", "/", "barfoo"),
("foobar > barfoo;", "foobar", ">", "barfoo"),
("foobar < barfoo;", "foobar", "<", "barfoo"),
("foobar == barfoo;", "foobar", "==", "barfoo"),
("foobar != barfoo;", "foobar", "!=", "barfoo"),
("true == true", True, "==", True),
("true != false", True, "!=", False),
("false == false", False, "==", False),
]
for v in tests:
lex = lexer_.Lexer(input=v[0])
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
assert len(program.statements) == 1
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
exp = stmt.expression
assert type(exp) is ast_.InfixExpression
assert self.check_infix_expression(exp, v[1], v[2], v[3])
def test_parse_identifier(self):
line = """foobar;"""
lex = lexer_.Lexer(input=line)
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
for v in program.statements:
print(v.string())
def test_TestStringLiteralExpression(self):
input = '"hello world";'
lex = lexer_.Lexer(input)
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
literal = stmt.expression
assert type(literal) is ast_.StringLiteral
assert literal.value == "hello world"
def test_TestParsingEmptyArrayLiterals(self):
input = "[]"
lex = lexer_.Lexer(input)
obj = parser_.Parser(lex=lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
assert len(program.statements) == 1
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
exp = stmt.expression
assert type(exp) is ast_.ArrayLiteral
assert len(exp.elements) == 0
def test_integer_literal_expression(self):
input = "5;"
lex = lexer_.Lexer(input=input)
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
assert len(program.statements) == 1
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
literal = stmt.expression
assert type(literal) is ast_.IntegerLiteral
assert literal.value == 5
assert literal.token_literal() == "5"
def test_identifier_expression(self):
input = "foobar;"
lex = lexer_.Lexer(input=input)
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
assert len(program.statements) == 1
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
ident = stmt.expression
assert type(ident) is ast_.Identifier
assert ident.value == "foobar"
assert ident.token_literal() == "foobar"
def test_if(self):
line = """
if (x < y) { x }
if (5 < 10) { (1 + 2) * 3 }
"""
lex = lexer_.Lexer(input=line)
obj = parser_.Parser(lex)
program = obj.parse_program()
print(program)
assert self.check_parser_errors(obj)
for v in program.statements:
print(v.string())
def test_parse_return_statement(self):
line = """
return 5;
return 10;
return 838383;
"""
lex = lexer_.Lexer(input=line)
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
for v in program.statements:
print(v.token_literal())
def test_parse_let_statement(self):
line = """
let x = 5;
let ssss = 10;
let foobar = 838383;
"""
lex = lexer_.Lexer(input=line)
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
for v in program.statements:
print(v.token_literal())
def test_parse_int_float(self):
line = """
5;
3.14;
"""
lex = lexer_.Lexer(input=line)
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
for v in program.statements:
print(v.string())
print(type(v))
print(type(v.expression))
def test_TestParsingArrayLiterals(self):
input = "[1, 2 * 2, 3 + 3]"
lex = lexer_.Lexer(input)
obj = parser_.Parser(lex=lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
assert len(program.statements) == 1
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
exp = stmt.expression
assert type(exp) is ast_.ArrayLiteral
assert len(exp.elements) == 3
assert self.check_literal_expression(exp.elements[0], 1)
assert self.check_infix_expression(exp.elements[1], 2, "*", 2)
assert self.check_infix_expression(exp.elements[2], 3, "+", 3)
def test_prefix(self):
line = """
!525;
-3.1415;
"""
lex = lexer_.Lexer(input=line)
obj = parser_.Parser(lex)
program = obj.parse_program()
print(program)
assert self.check_parser_errors(obj)
for v in program.statements:
print(v.string())
print(v.expression.operator)
print(v.expression.right.value)
def test_call_expression_parsing(self):
input = "add(1, 2 * 3, 4 + 5);"
lex = lexer_.Lexer(input)
obj = parser_.Parser(lex=lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
assert len(program.statements) == 1
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
exp = stmt.expression
assert type(exp) is ast_.CallExpression
assert self.check_identifier(exp.function, "add")
assert len(exp.arguments) == 3
assert self.check_literal_expression(exp.arguments[0], 1)
assert self.check_infix_expression(exp.arguments[1], 2, "*", 3)
assert self.check_infix_expression(exp.arguments[2], 4, "+", 5)
def test_boolean_expression(self):
tests = [
("true;", True),
("false;", False),
]
for v in tests:
lex = lexer_.Lexer(input=v[0])
obj = parser_.Parser(lex=lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
assert len(program.statements) == 1
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
boolean = stmt.expression
assert type(boolean) is ast_.Boolean
assert boolean.value == v[1]
def test_if_expression(self):
input = "if (x < y) { x }"
lex = lexer_.Lexer(input)
obj = parser_.Parser(lex=lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
assert len(program.statements) == 1
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
exp = stmt.expression
assert type(exp) is ast_.IfExpression
assert self.check_infix_expression(exp.condition, "x", "<", "y")
assert len(exp.consequence.statements) == 1
consequence = exp.consequence.statements[0]
assert type(consequence) is ast_.ExpressionStatement
assert self.check_identifier(consequence.expression, "x")
assert exp.alternative is None
def test_parse_prefix_expression(self):
line = """
!5;
-3.14;
"""
lex = lexer_.Lexer(input=line)
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
for v in program.statements:
print(v.string())
print(type(v))
print(type(v.expression))
print(type(v.expression.operator))
print(type(v.expression.right))
def test_function_parameter_parsing(self):
tests = [
("fn() {};", []),
("fn(x) {};", ["x"]),
("fn(x, y, z) {};", ["x", "y", "z"]),
]
for v in tests:
lex = lexer_.Lexer(input=v[0])
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
function = stmt.expression
assert type(function) is ast_.FunctionLiteral
assert len(function.parameters) == len(v[1])
for v2, v3 in zip(function.parameters, v[1]):
assert self.check_literal_expression(v2, v3)
def start():
env = env_.NewEnvironment()
try:
while True:
print(PROMPT, end="")
line = input()
lex = lexer_.Lexer(line)
p = parser_.Parser(lex)
program = p.parse_program()
if len(p.Errors()) != 0:
print(print_parser_errors(p.Errors()))
continue
evaluated = evaluator_.Eval(program, env)
if evaluated is not None:
print(evaluated.Inspect())
except KeyboardInterrupt:
sys.exit()
def test_return_statements(self):
tests = [
("return 5;", 5),
("return true;", True),
("return foobar;", "foobar"),
]
for v in tests:
lex = lexer_.Lexer(input=v[0])
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
assert len(program.statements) == 1,\
f"program.Statements does not contain 1 statements. got={len(program.statements)}"
stmt = program.statements[0]
assert type(stmt) is ast_.ReturnStatement
assert stmt.token_literal() == "return"
assert self.check_literal_expression(stmt.return_value, v[1])
def test_call_expression_parameter_parsing(self):
tests = [
("add();", "add", []),
("add(1);", "add", ["1"]),
("add(1, 2 * 3, 4 + 5);", "add", ["1", "(2 * 3)", "(4 + 5)"]),
]
for v in tests:
lex = lexer_.Lexer(input=v[0])
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
exp = stmt.expression
assert type(exp) is ast_.CallExpression
assert self.check_identifier(exp.function, v[1])
assert len(exp.arguments) == len(v[2])
for i, arg in enumerate(v[2]):
assert exp.arguments[i].string() == arg
def test_let_statements(self):
tests = [
("let x = 5;", "x", 5),
("let y = true;", "y", True),
("let foobar = y;", "foobar", "y"),
]
for v in tests:
lex = lexer_.Lexer(input=v[0])
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
assert len(program.statements) == 1,\
f"program.Statements does not contain 1 statements. got={len(program.statements)}"
stmt = program.statements[0]
assert self.check_let_statement(stmt, v[1])
val = stmt.value
assert self.check_literal_expression(val, v[2])
def test_parsing_prefix_expressions(self):
tests = [
("!5;", "!", 5),
("-15;", "-", 15),
("!foobar;", "!", "foobar"),
("-foobar;", "-", "foobar"),
("!true;", "!", True),
("!false;", "!", False),
]
for v in tests:
lex = lexer_.Lexer(input=v[0])
obj = parser_.Parser(lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
assert len(program.statements) == 1
stmt = program.statements[0]
assert type(stmt) is ast_.ExpressionStatement
exp = stmt.expression
assert type(exp) is ast_.PrefixExpression
assert exp.operator == v[1]
assert self.check_literal_expression(exp.right, v[2])
def test_next_token2(self):
input = """let five = 5;
let ten = 10;
let add = fn(x, y) {
x + y;
};
let result = add(five, ten);
!-/*5;
5 < 10 > 5;
if (5 < 10) {
return true;
} else {
return false;
}
10 == 10;
10 != 9;
"foobar"
"foo bar"
"日本語"
[1, 2];
"""
tests = [
(token_.TokenType.LET, "let"),
(token_.TokenType.IDENT, "five"),
(token_.TokenType.ASSIGN, "="),
(token_.TokenType.INT, "5"),
(token_.TokenType.SEMICOLON, ";"),
(token_.TokenType.LET, "let"),
(token_.TokenType.IDENT, "ten"),
(token_.TokenType.ASSIGN, "="),
(token_.TokenType.INT, "10"),
(token_.TokenType.SEMICOLON, ";"),
(token_.TokenType.LET, "let"),
(token_.TokenType.IDENT, "add"),
(token_.TokenType.ASSIGN, "="),
(token_.TokenType.FUNCTION, "fn"),
(token_.TokenType.LPAREN, "("),
(token_.TokenType.IDENT, "x"),
(token_.TokenType.COMMA, ","),
(token_.TokenType.IDENT, "y"),
(token_.TokenType.RPAREN, ")"),
(token_.TokenType.LBRACE, "{"),
(token_.TokenType.IDENT, "x"),
(token_.TokenType.PLUS, "+"),
(token_.TokenType.IDENT, "y"),
(token_.TokenType.SEMICOLON, ";"),
(token_.TokenType.RBRACE, "}"),
(token_.TokenType.SEMICOLON, ";"),
(token_.TokenType.LET, "let"),
(token_.TokenType.IDENT, "result"),
(token_.TokenType.ASSIGN, "="),
(token_.TokenType.IDENT, "add"),
(token_.TokenType.LPAREN, "("),
(token_.TokenType.IDENT, "five"),
(token_.TokenType.COMMA, ","),
(token_.TokenType.IDENT, "ten"),
(token_.TokenType.RPAREN, ")"),
(token_.TokenType.SEMICOLON, ";"),
(token_.TokenType.BANG, "!"),
(token_.TokenType.MINUS, "-"),
(token_.TokenType.SLASH, "/"),
(token_.TokenType.ASTERISK, "*"),
(token_.TokenType.INT, "5"),
(token_.TokenType.SEMICOLON, ";"),
(token_.TokenType.INT, "5"),
(token_.TokenType.LT, "<"),
(token_.TokenType.INT, "10"),
(token_.TokenType.GT, ">"),
(token_.TokenType.INT, "5"),
(token_.TokenType.SEMICOLON, ";"),
(token_.TokenType.IF, "if"),
(token_.TokenType.LPAREN, "("),
(token_.TokenType.INT, "5"),
(token_.TokenType.LT, "<"),
(token_.TokenType.INT, "10"),
(token_.TokenType.RPAREN, ")"),
(token_.TokenType.LBRACE, "{"),
(token_.TokenType.RETURN, "return"),
(token_.TokenType.TRUE, "true"),
(token_.TokenType.SEMICOLON, ";"),
(token_.TokenType.RBRACE, "}"),
(token_.TokenType.ELSE, "else"),
(token_.TokenType.LBRACE, "{"),
(token_.TokenType.RETURN, "return"),
(token_.TokenType.FALSE, "false"),
(token_.TokenType.SEMICOLON, ";"),
(token_.TokenType.RBRACE, "}"),
(token_.TokenType.INT, "10"),
(token_.TokenType.EQ, "=="),
(token_.TokenType.INT, "10"),
(token_.TokenType.SEMICOLON, ";"),
(token_.TokenType.INT, "10"),
(token_.TokenType.NOT_EQ, "!="),
(token_.TokenType.INT, "9"),
(token_.TokenType.SEMICOLON, ";"),
(token_.TokenType.STRING, "foobar"),
(token_.TokenType.STRING, "foo bar"),
(token_.TokenType.STRING, "日本語"),
(token_.TokenType.LBRACKET, "["),
(token_.TokenType.INT, "1"),
(token_.TokenType.COMMA, ","),
(token_.TokenType.INT, "2"),
(token_.TokenType.RBRACKET, "]"),
(token_.TokenType.SEMICOLON, ";"),
(token_.TokenType.EOF, ""),
]
lex = lexer_.Lexer(input)
for v in tests:
tok = lex.next_token()
assert tok.token_type == v[0],\
f"tokentype wrong. expected={tok.token_type}, got={v[0]}"
assert tok.literal == v[1],\
f"tokentype wrong. expected={tok.literal}, got={v[1]}"
def test_OperatorPrecedenceParsing(self):
tests = [
(
"1 + 2 + 3",
"((1 + 2) + 3)",
),
(
"-a * b",
"((-a) * b)",
),
(
"!-a",
"(!(-a))",
),
(
"a + b + c",
"((a + b) + c)",
),
(
"a + b - c",
"((a + b) - c)",
),
(
"a * b * c",
"((a * b) * c)",
),
(
"a * b / c",
"((a * b) / c)",
),
(
"a + b / c",
"(a + (b / c))",
),
(
"a + b * c + d / e - f",
"(((a + (b * c)) + (d / e)) - f)",
),
(
"3 + 4; -5 * 5",
"(3 + 4)((-5) * 5)",
),
(
"5 > 4 == 3 < 4",
"((5 > 4) == (3 < 4))",
),
(
"5 < 4 != 3 > 4",
"((5 < 4) != (3 > 4))",
),
(
"3 + 4 * 5 == 3 * 1 + 4 * 5",
"((3 + (4 * 5)) == ((3 * 1) + (4 * 5)))",
),
(
"true",
"true",
),
(
"false",
"false",
),
(
"3 > 5 == false",
"((3 > 5) == false)",
),
(
"3 < 5 == true",
"((3 < 5) == true)",
),
(
"1 + (2 + 3) + 4",
"((1 + (2 + 3)) + 4)",
),
(
"(5 + 5) * 2",
"((5 + 5) * 2)",
),
(
"2 / (5 + 5)",
"(2 / (5 + 5))",
),
(
"(5 + 5) * 2 * (5 + 5)",
"(((5 + 5) * 2) * (5 + 5))",
),
(
"-(5 + 5)",
"(-(5 + 5))",
),
(
"!(true == true)",
"(!(true == true))",
),
(
"a + add(b * c) + d",
"((a + add((b * c))) + d)",
),
(
"add(a, b, 1, 2 * 3, 4 + 5, add(6, 7 * 8))",
"add(a, b, 1, (2 * 3), (4 + 5), add(6, (7 * 8)))",
),
(
"add(a + b + c * d / f + g)",
"add((((a + b) + ((c * d) / f)) + g))",
),
]
for v in tests:
lex = lexer_.Lexer(input=v[0])
obj = parser_.Parser(lex=lex)
program = obj.parse_program()
assert self.check_parser_errors(obj)
actual = program.string()
assert actual == v[1]
