def test_id_with_whitespaces():
lexer = Lexer(code=" IDdTest ")
tokens = lexer.generate_tokens()
assert (Token(token_type=TOKENS.NAME,
value="IDdTest",
line_number=0,
column=3) == tokens[0])
lexer = Lexer(code="\tIDdTest\t")
tokens = lexer.generate_tokens()
assert (Token(token_type=TOKENS.NAME,
value="IDdTest",
line_number=0,
column=1) == tokens[0])
def test_integer():
lexer = Lexer(code="12345")
tokens = lexer.generate_tokens()
assert (Token(token_type=TOKENS.INTEGER,
value=12345,
line_number=0,
column=0) == tokens[0])
def test_bin_digit():
lexer = Lexer(code="'B1010'")
tokens = lexer.generate_tokens()
assert (Token(token_type=TOKENS.INTEGER,
value=int("1010", 2),
line_number=0,
column=0) == tokens[0])
def test_hex_digit():
lexer = Lexer(code="'H1F")
tokens = lexer.generate_tokens()
assert (Token(token_type=TOKENS.INTEGER,
value=int("1f", 16),
line_number=0,
column=0) == tokens[0])
def test_id():
lexer = Lexer(code="IDdTest")
tokens = lexer.generate_tokens()
assert (Token(token_type=TOKENS.NAME,
value="IDdTest",
line_number=0,
column=0) == tokens[0])
def test_real():
lexer = Lexer(code="1.1")
tokens = lexer.generate_tokens()
assert (Token(token_type=TOKENS.REAL, value=1.1, line_number=0,
column=0) == tokens[0])
lexer = Lexer(code="1E3")
tokens = lexer.generate_tokens()
assert (Token(token_type=TOKENS.REAL, value=1E3, line_number=0,
column=0) == tokens[0])
lexer = Lexer(code="1.3E3")
tokens = lexer.generate_tokens()
assert (Token(token_type=TOKENS.REAL, value=1.3E3, line_number=0,
column=0) == tokens[0])
lexer = Lexer(code="1.3E-3")
tokens = lexer.generate_tokens()
assert (Token(token_type=TOKENS.REAL,
value=1.3E-3,
line_number=0,
column=0) == tokens[0])
