def test_function_literal_parsing(self): line = "fn(x, y) { x + y; }" lex = lexer_.Lexer(input=line) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) assert len(program.statements) == 1 stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement function = stmt.expression assert type(function) is ast_.FunctionLiteral assert len(function.parameters) == 2 assert self.check_literal_expression(function.parameters[0], "x") assert self.check_literal_expression(function.parameters[1], "y") assert len(function.body.statements) == 1 body_stmt = function.body.statements[0] assert type(body_stmt) is ast_.ExpressionStatement assert self.check_infix_expression(body_stmt.expression, "x", "+", "y") for v in program.statements: print(v.string()) print(type(v)) print(type(v.expression)) print(type(v.expression.token)) print(type(v.expression.string())) print(v.expression.string())
def test_Eval(self, input): lex = lexer_.Lexer(input) p = parser_.Parser(lex) program = p.parse_program() env = env_.NewEnvironment() return evaluator_.Eval(program, env)
def test_infix(self): line = """ 5 + 5.; 5 - 5.; 5 * 5.; 5 / 5.; 5 > 5.; 5 < 5.; 5 == 5.; 5 != 5.; """ lex = lexer_.Lexer(input=line) obj = parser_.Parser(lex) program = obj.parse_program() print(program) print(len(program.statements)) assert self.check_parser_errors(obj) for v in program.statements: print(v.string()) print(v.expression.operator) print(v.expression.left.value) print(v.expression.right.value) print("-" * 30)
def test_next_token1(self): line = """ let five = 5.2.36; let ten = 10; let add = fn(x, y) { x + y; }; let result = add(five, ten); !-/*5; 5 < 10.236 > 5; if (5 < 10) { return true; } else { return false; } 10 == 10; 10 != .9; "foobar" "foo bar" [1, 2]; """ lex = lexer_.Lexer(input=line) while True: tok = lex.next_token() print(tok.token_type) print(tok.literal) print("-----") if tok.token_type == token_.TokenType.EOF: break
def test_parsing_infix_expressions(self): tests = [ ("5 + 5;", 5, "+", 5), ("5 - 5;", 5, "-", 5), ("5 * 5;", 5, "*", 5), ("5 / 5;", 5, "/", 5), ("5 > 5;", 5, ">", 5), ("5 < 5;", 5, "<", 5), ("5 == 5;", 5, "==", 5), ("5 != 5;", 5, "!=", 5), ("foobar + barfoo;", "foobar", "+", "barfoo"), ("foobar - barfoo;", "foobar", "-", "barfoo"), ("foobar * barfoo;", "foobar", "*", "barfoo"), ("foobar / barfoo;", "foobar", "/", "barfoo"), ("foobar > barfoo;", "foobar", ">", "barfoo"), ("foobar < barfoo;", "foobar", "<", "barfoo"), ("foobar == barfoo;", "foobar", "==", "barfoo"), ("foobar != barfoo;", "foobar", "!=", "barfoo"), ("true == true", True, "==", True), ("true != false", True, "!=", False), ("false == false", False, "==", False), ] for v in tests: lex = lexer_.Lexer(input=v[0]) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) assert len(program.statements) == 1 stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement exp = stmt.expression assert type(exp) is ast_.InfixExpression assert self.check_infix_expression(exp, v[1], v[2], v[3])
def test_parse_identifier(self): line = """foobar;""" lex = lexer_.Lexer(input=line) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) for v in program.statements: print(v.string())
def test_TestStringLiteralExpression(self): input = '"hello world";' lex = lexer_.Lexer(input) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement literal = stmt.expression assert type(literal) is ast_.StringLiteral assert literal.value == "hello world"
def test_TestParsingEmptyArrayLiterals(self): input = "[]" lex = lexer_.Lexer(input) obj = parser_.Parser(lex=lex) program = obj.parse_program() assert self.check_parser_errors(obj) assert len(program.statements) == 1 stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement exp = stmt.expression assert type(exp) is ast_.ArrayLiteral assert len(exp.elements) == 0
def test_integer_literal_expression(self): input = "5;" lex = lexer_.Lexer(input=input) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) assert len(program.statements) == 1 stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement literal = stmt.expression assert type(literal) is ast_.IntegerLiteral assert literal.value == 5 assert literal.token_literal() == "5"
def test_identifier_expression(self): input = "foobar;" lex = lexer_.Lexer(input=input) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) assert len(program.statements) == 1 stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement ident = stmt.expression assert type(ident) is ast_.Identifier assert ident.value == "foobar" assert ident.token_literal() == "foobar"
def test_if(self): line = """ if (x < y) { x } if (5 < 10) { (1 + 2) * 3 } """ lex = lexer_.Lexer(input=line) obj = parser_.Parser(lex) program = obj.parse_program() print(program) assert self.check_parser_errors(obj) for v in program.statements: print(v.string())
def test_parse_return_statement(self): line = """ return 5; return 10; return 838383; """ lex = lexer_.Lexer(input=line) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) for v in program.statements: print(v.token_literal())
def test_parse_let_statement(self): line = """ let x = 5; let ssss = 10; let foobar = 838383; """ lex = lexer_.Lexer(input=line) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) for v in program.statements: print(v.token_literal())
def test_parse_int_float(self): line = """ 5; 3.14; """ lex = lexer_.Lexer(input=line) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) for v in program.statements: print(v.string()) print(type(v)) print(type(v.expression))
def test_TestParsingArrayLiterals(self): input = "[1, 2 * 2, 3 + 3]" lex = lexer_.Lexer(input) obj = parser_.Parser(lex=lex) program = obj.parse_program() assert self.check_parser_errors(obj) assert len(program.statements) == 1 stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement exp = stmt.expression assert type(exp) is ast_.ArrayLiteral assert len(exp.elements) == 3 assert self.check_literal_expression(exp.elements[0], 1) assert self.check_infix_expression(exp.elements[1], 2, "*", 2) assert self.check_infix_expression(exp.elements[2], 3, "+", 3)
def test_prefix(self): line = """ !525; -3.1415; """ lex = lexer_.Lexer(input=line) obj = parser_.Parser(lex) program = obj.parse_program() print(program) assert self.check_parser_errors(obj) for v in program.statements: print(v.string()) print(v.expression.operator) print(v.expression.right.value)
def test_call_expression_parsing(self): input = "add(1, 2 * 3, 4 + 5);" lex = lexer_.Lexer(input) obj = parser_.Parser(lex=lex) program = obj.parse_program() assert self.check_parser_errors(obj) assert len(program.statements) == 1 stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement exp = stmt.expression assert type(exp) is ast_.CallExpression assert self.check_identifier(exp.function, "add") assert len(exp.arguments) == 3 assert self.check_literal_expression(exp.arguments[0], 1) assert self.check_infix_expression(exp.arguments[1], 2, "*", 3) assert self.check_infix_expression(exp.arguments[2], 4, "+", 5)
def test_boolean_expression(self): tests = [ ("true;", True), ("false;", False), ] for v in tests: lex = lexer_.Lexer(input=v[0]) obj = parser_.Parser(lex=lex) program = obj.parse_program() assert self.check_parser_errors(obj) assert len(program.statements) == 1 stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement boolean = stmt.expression assert type(boolean) is ast_.Boolean assert boolean.value == v[1]
def test_if_expression(self): input = "if (x < y) { x }" lex = lexer_.Lexer(input) obj = parser_.Parser(lex=lex) program = obj.parse_program() assert self.check_parser_errors(obj) assert len(program.statements) == 1 stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement exp = stmt.expression assert type(exp) is ast_.IfExpression assert self.check_infix_expression(exp.condition, "x", "<", "y") assert len(exp.consequence.statements) == 1 consequence = exp.consequence.statements[0] assert type(consequence) is ast_.ExpressionStatement assert self.check_identifier(consequence.expression, "x") assert exp.alternative is None
def test_parse_prefix_expression(self): line = """ !5; -3.14; """ lex = lexer_.Lexer(input=line) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) for v in program.statements: print(v.string()) print(type(v)) print(type(v.expression)) print(type(v.expression.operator)) print(type(v.expression.right))
def test_function_parameter_parsing(self): tests = [ ("fn() {};", []), ("fn(x) {};", ["x"]), ("fn(x, y, z) {};", ["x", "y", "z"]), ] for v in tests: lex = lexer_.Lexer(input=v[0]) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement function = stmt.expression assert type(function) is ast_.FunctionLiteral assert len(function.parameters) == len(v[1]) for v2, v3 in zip(function.parameters, v[1]): assert self.check_literal_expression(v2, v3)
def start(): env = env_.NewEnvironment() try: while True: print(PROMPT, end="") line = input() lex = lexer_.Lexer(line) p = parser_.Parser(lex) program = p.parse_program() if len(p.Errors()) != 0: print(print_parser_errors(p.Errors())) continue evaluated = evaluator_.Eval(program, env) if evaluated is not None: print(evaluated.Inspect()) except KeyboardInterrupt: sys.exit()
def test_return_statements(self): tests = [ ("return 5;", 5), ("return true;", True), ("return foobar;", "foobar"), ] for v in tests: lex = lexer_.Lexer(input=v[0]) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) assert len(program.statements) == 1,\ f"program.Statements does not contain 1 statements. got={len(program.statements)}" stmt = program.statements[0] assert type(stmt) is ast_.ReturnStatement assert stmt.token_literal() == "return" assert self.check_literal_expression(stmt.return_value, v[1])
def test_call_expression_parameter_parsing(self): tests = [ ("add();", "add", []), ("add(1);", "add", ["1"]), ("add(1, 2 * 3, 4 + 5);", "add", ["1", "(2 * 3)", "(4 + 5)"]), ] for v in tests: lex = lexer_.Lexer(input=v[0]) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement exp = stmt.expression assert type(exp) is ast_.CallExpression assert self.check_identifier(exp.function, v[1]) assert len(exp.arguments) == len(v[2]) for i, arg in enumerate(v[2]): assert exp.arguments[i].string() == arg
def test_let_statements(self): tests = [ ("let x = 5;", "x", 5), ("let y = true;", "y", True), ("let foobar = y;", "foobar", "y"), ] for v in tests: lex = lexer_.Lexer(input=v[0]) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) assert len(program.statements) == 1,\ f"program.Statements does not contain 1 statements. got={len(program.statements)}" stmt = program.statements[0] assert self.check_let_statement(stmt, v[1]) val = stmt.value assert self.check_literal_expression(val, v[2])
def test_parsing_prefix_expressions(self): tests = [ ("!5;", "!", 5), ("-15;", "-", 15), ("!foobar;", "!", "foobar"), ("-foobar;", "-", "foobar"), ("!true;", "!", True), ("!false;", "!", False), ] for v in tests: lex = lexer_.Lexer(input=v[0]) obj = parser_.Parser(lex) program = obj.parse_program() assert self.check_parser_errors(obj) assert len(program.statements) == 1 stmt = program.statements[0] assert type(stmt) is ast_.ExpressionStatement exp = stmt.expression assert type(exp) is ast_.PrefixExpression assert exp.operator == v[1] assert self.check_literal_expression(exp.right, v[2])
def test_next_token2(self): input = """let five = 5; let ten = 10; let add = fn(x, y) { x + y; }; let result = add(five, ten); !-/*5; 5 < 10 > 5; if (5 < 10) { return true; } else { return false; } 10 == 10; 10 != 9; "foobar" "foo bar" "日本語" [1, 2]; """ tests = [ (token_.TokenType.LET, "let"), (token_.TokenType.IDENT, "five"), (token_.TokenType.ASSIGN, "="), (token_.TokenType.INT, "5"), (token_.TokenType.SEMICOLON, ";"), (token_.TokenType.LET, "let"), (token_.TokenType.IDENT, "ten"), (token_.TokenType.ASSIGN, "="), (token_.TokenType.INT, "10"), (token_.TokenType.SEMICOLON, ";"), (token_.TokenType.LET, "let"), (token_.TokenType.IDENT, "add"), (token_.TokenType.ASSIGN, "="), (token_.TokenType.FUNCTION, "fn"), (token_.TokenType.LPAREN, "("), (token_.TokenType.IDENT, "x"), (token_.TokenType.COMMA, ","), (token_.TokenType.IDENT, "y"), (token_.TokenType.RPAREN, ")"), (token_.TokenType.LBRACE, "{"), (token_.TokenType.IDENT, "x"), (token_.TokenType.PLUS, "+"), (token_.TokenType.IDENT, "y"), (token_.TokenType.SEMICOLON, ";"), (token_.TokenType.RBRACE, "}"), (token_.TokenType.SEMICOLON, ";"), (token_.TokenType.LET, "let"), (token_.TokenType.IDENT, "result"), (token_.TokenType.ASSIGN, "="), (token_.TokenType.IDENT, "add"), (token_.TokenType.LPAREN, "("), (token_.TokenType.IDENT, "five"), (token_.TokenType.COMMA, ","), (token_.TokenType.IDENT, "ten"), (token_.TokenType.RPAREN, ")"), (token_.TokenType.SEMICOLON, ";"), (token_.TokenType.BANG, "!"), (token_.TokenType.MINUS, "-"), (token_.TokenType.SLASH, "/"), (token_.TokenType.ASTERISK, "*"), (token_.TokenType.INT, "5"), (token_.TokenType.SEMICOLON, ";"), (token_.TokenType.INT, "5"), (token_.TokenType.LT, "<"), (token_.TokenType.INT, "10"), (token_.TokenType.GT, ">"), (token_.TokenType.INT, "5"), (token_.TokenType.SEMICOLON, ";"), (token_.TokenType.IF, "if"), (token_.TokenType.LPAREN, "("), (token_.TokenType.INT, "5"), (token_.TokenType.LT, "<"), (token_.TokenType.INT, "10"), (token_.TokenType.RPAREN, ")"), (token_.TokenType.LBRACE, "{"), (token_.TokenType.RETURN, "return"), (token_.TokenType.TRUE, "true"), (token_.TokenType.SEMICOLON, ";"), (token_.TokenType.RBRACE, "}"), (token_.TokenType.ELSE, "else"), (token_.TokenType.LBRACE, "{"), (token_.TokenType.RETURN, "return"), (token_.TokenType.FALSE, "false"), (token_.TokenType.SEMICOLON, ";"), (token_.TokenType.RBRACE, "}"), (token_.TokenType.INT, "10"), (token_.TokenType.EQ, "=="), (token_.TokenType.INT, "10"), (token_.TokenType.SEMICOLON, ";"), (token_.TokenType.INT, "10"), (token_.TokenType.NOT_EQ, "!="), (token_.TokenType.INT, "9"), (token_.TokenType.SEMICOLON, ";"), (token_.TokenType.STRING, "foobar"), (token_.TokenType.STRING, "foo bar"), (token_.TokenType.STRING, "日本語"), (token_.TokenType.LBRACKET, "["), (token_.TokenType.INT, "1"), (token_.TokenType.COMMA, ","), (token_.TokenType.INT, "2"), (token_.TokenType.RBRACKET, "]"), (token_.TokenType.SEMICOLON, ";"), (token_.TokenType.EOF, ""), ] lex = lexer_.Lexer(input) for v in tests: tok = lex.next_token() assert tok.token_type == v[0],\ f"tokentype wrong. expected={tok.token_type}, got={v[0]}" assert tok.literal == v[1],\ f"tokentype wrong. expected={tok.literal}, got={v[1]}"
def test_OperatorPrecedenceParsing(self): tests = [ ( "1 + 2 + 3", "((1 + 2) + 3)", ), ( "-a * b", "((-a) * b)", ), ( "!-a", "(!(-a))", ), ( "a + b + c", "((a + b) + c)", ), ( "a + b - c", "((a + b) - c)", ), ( "a * b * c", "((a * b) * c)", ), ( "a * b / c", "((a * b) / c)", ), ( "a + b / c", "(a + (b / c))", ), ( "a + b * c + d / e - f", "(((a + (b * c)) + (d / e)) - f)", ), ( "3 + 4; -5 * 5", "(3 + 4)((-5) * 5)", ), ( "5 > 4 == 3 < 4", "((5 > 4) == (3 < 4))", ), ( "5 < 4 != 3 > 4", "((5 < 4) != (3 > 4))", ), ( "3 + 4 * 5 == 3 * 1 + 4 * 5", "((3 + (4 * 5)) == ((3 * 1) + (4 * 5)))", ), ( "true", "true", ), ( "false", "false", ), ( "3 > 5 == false", "((3 > 5) == false)", ), ( "3 < 5 == true", "((3 < 5) == true)", ), ( "1 + (2 + 3) + 4", "((1 + (2 + 3)) + 4)", ), ( "(5 + 5) * 2", "((5 + 5) * 2)", ), ( "2 / (5 + 5)", "(2 / (5 + 5))", ), ( "(5 + 5) * 2 * (5 + 5)", "(((5 + 5) * 2) * (5 + 5))", ), ( "-(5 + 5)", "(-(5 + 5))", ), ( "!(true == true)", "(!(true == true))", ), ( "a + add(b * c) + d", "((a + add((b * c))) + d)", ), ( "add(a, b, 1, 2 * 3, 4 + 5, add(6, 7 * 8))", "add(a, b, 1, (2 * 3), (4 + 5), add(6, (7 * 8)))", ), ( "add(a + b + c * d / f + g)", "add((((a + b) + ((c * d) / f)) + g))", ), ] for v in tests: lex = lexer_.Lexer(input=v[0]) obj = parser_.Parser(lex=lex) program = obj.parse_program() assert self.check_parser_errors(obj) actual = program.string() assert actual == v[1]