from lexer.lexer import Lexer from util.manage_file import ManageFile import sys file = ManageFile(sys.argv[1]) lexer = Lexer() run(lexer) def run(lexer) token = lexer.next_token() token_list = [] while(token != Grammar.ERR): token_list.append(token) token = lexer.next_token()
def test_tokens(self): lexer = Lexer( """ let five = 5; let ten = 10; let add = fn(x, y) { x + y; }; let result = add(five, ten); !-/*5; 5 < 10 > 5; if (5 < 10) { return true; } else { return false; } 10 == 10; 10 != 9; "foobar" "foo bar" [1, 2]; {"foo": "bar"}; while (true) {}; """ ) expected_tokens = [ Token(Token.LET, "let"), Token(Token.IDENT, "five"), Token(Token.ASSIGN, "="), Token(Token.INT, "5"), Token(Token.SEMICOLON, ";"), Token(Token.LET, "let"), Token(Token.IDENT, "ten"), Token(Token.ASSIGN, "="), Token(Token.INT, "10"), Token(Token.SEMICOLON, ";"), Token(Token.LET, "let"), Token(Token.IDENT, "add"), Token(Token.ASSIGN, "="), Token(Token.FUNCTION, "fn"), Token(Token.LPAREN, "("), Token(Token.IDENT, "x"), Token(Token.COMMA, ","), Token(Token.IDENT, "y"), Token(Token.RPAREN, ")"), Token(Token.LBRACE, "{"), Token(Token.IDENT, "x"), Token(Token.PLUS, "+"), Token(Token.IDENT, "y"), Token(Token.SEMICOLON, ";"), Token(Token.RBRACE, "}"), Token(Token.SEMICOLON, ";"), Token(Token.LET, "let"), Token(Token.IDENT, "result"), Token(Token.ASSIGN, "="), Token(Token.IDENT, "add"), Token(Token.LPAREN, "("), Token(Token.IDENT, "five"), Token(Token.COMMA, ","), Token(Token.IDENT, "ten"), Token(Token.RPAREN, ")"), Token(Token.SEMICOLON, ";"), Token(Token.BANG, "!"), Token(Token.MINUS, "-"), Token(Token.SLASH, "/"), Token(Token.ASTERISK, "*"), Token(Token.INT, "5"), Token(Token.SEMICOLON, ";"), Token(Token.INT, "5"), Token(Token.LT, "<"), Token(Token.INT, "10"), Token(Token.GT, ">"), Token(Token.INT, "5"), Token(Token.SEMICOLON, ";"), Token(Token.IF, "if"), Token(Token.LPAREN, "("), Token(Token.INT, "5"), Token(Token.LT, "<"), Token(Token.INT, "10"), Token(Token.RPAREN, ")"), Token(Token.LBRACE, "{"), Token(Token.RETURN, "return"), Token(Token.TRUE, "true"), Token(Token.SEMICOLON, ";"), Token(Token.RBRACE, "}"), Token(Token.ELSE, "else"), Token(Token.LBRACE, "{"), Token(Token.RETURN, "return"), Token(Token.FALSE, "false"), Token(Token.SEMICOLON, ";"), Token(Token.RBRACE, "}"), Token(Token.INT, "10"), Token(Token.EQ, "=="), Token(Token.INT, "10"), Token(Token.SEMICOLON, ";"), Token(Token.INT, "10"), Token(Token.NOT_EQ, "!="), Token(Token.INT, "9"), Token(Token.SEMICOLON, ";"), Token(Token.STRING, "foobar"), Token(Token.STRING, "foo bar"), Token(Token.LBRACKET, "["), Token(Token.INT, "1"), Token(Token.COMMA, ","), Token(Token.INT, "2"), Token(Token.RBRACKET, "]"), Token(Token.SEMICOLON, ";"), Token(Token.LBRACE, "{"), Token(Token.STRING, "foo"), Token(Token.COLON, ":"), Token(Token.STRING, "bar"), Token(Token.RBRACE, "}"), Token(Token.SEMICOLON, ";"), Token(Token.WHILE, "while"), Token(Token.LPAREN, "("), Token(Token.TRUE, "true"), Token(Token.RPAREN, ")"), Token(Token.LBRACE, "{"), Token(Token.RBRACE, "}"), Token(Token.SEMICOLON, ";"), Token(Token.EOF, ""), ] for expected_token in expected_tokens: actual = lexer.next_token() self.assertEqual(expected_token.token_type, actual.token_type) self.assertEqual(expected_token.literal, actual.literal)