def test_id_with_whitespaces(): lexer = Lexer(code=" IDdTest ") tokens = lexer.generate_tokens() assert (Token(token_type=TOKENS.NAME, value="IDdTest", line_number=0, column=3) == tokens[0]) lexer = Lexer(code="\tIDdTest\t") tokens = lexer.generate_tokens() assert (Token(token_type=TOKENS.NAME, value="IDdTest", line_number=0, column=1) == tokens[0])
def test_integer(): lexer = Lexer(code="12345") tokens = lexer.generate_tokens() assert (Token(token_type=TOKENS.INTEGER, value=12345, line_number=0, column=0) == tokens[0])
def test_bin_digit(): lexer = Lexer(code="'B1010'") tokens = lexer.generate_tokens() assert (Token(token_type=TOKENS.INTEGER, value=int("1010", 2), line_number=0, column=0) == tokens[0])
def test_hex_digit(): lexer = Lexer(code="'H1F") tokens = lexer.generate_tokens() assert (Token(token_type=TOKENS.INTEGER, value=int("1f", 16), line_number=0, column=0) == tokens[0])
def test_id(): lexer = Lexer(code="IDdTest") tokens = lexer.generate_tokens() assert (Token(token_type=TOKENS.NAME, value="IDdTest", line_number=0, column=0) == tokens[0])
def test_real(): lexer = Lexer(code="1.1") tokens = lexer.generate_tokens() assert (Token(token_type=TOKENS.REAL, value=1.1, line_number=0, column=0) == tokens[0]) lexer = Lexer(code="1E3") tokens = lexer.generate_tokens() assert (Token(token_type=TOKENS.REAL, value=1E3, line_number=0, column=0) == tokens[0]) lexer = Lexer(code="1.3E3") tokens = lexer.generate_tokens() assert (Token(token_type=TOKENS.REAL, value=1.3E3, line_number=0, column=0) == tokens[0]) lexer = Lexer(code="1.3E-3") tokens = lexer.generate_tokens() assert (Token(token_type=TOKENS.REAL, value=1.3E-3, line_number=0, column=0) == tokens[0])