def test_lexer(self):
e1 = "1"
self.assertEqual(list(simplify(tokenize(e1))),
[('NUMBER', 1), 'EOF'])
input_str = "(1 + 11 * 22)"
result = list(simplify(tokenize(input_str)))
expected = ['LPAREN', ('NUMBER', 1), 'PLUS', ('NUMBER', 11),
'TIMES', ('NUMBER', 22), 'RPAREN', 'EOF']
self.assertEqual(expected, result)
def test_lexer_comment(self):
input_str = "// this is a comment\n+"
result = list(simplify(tokenize(input_str)))
expected = ['PLUS', 'EOF']
self.assertEqual(expected, result)
def test_lexer_continue(self):
input_str = "continue;"
result = list(simplify(tokenize(input_str)))
expected = ['CONTINUE', 'SEMI', 'EOF']
self.assertEqual(expected, result)
def test_lexer_break(self):
input_str = "break;"
result = list(simplify(tokenize(input_str)))
expected = ['BREAK', 'SEMI', 'EOF']
self.assertEqual(expected, result)
def test_lexer_print(self):
input_str = "print(x);"
result = list(simplify(tokenize(input_str)))
expected = ['PRINT', 'LPAREN', ('ID', 'x'), 'RPAREN', 'SEMI', 'EOF']
self.assertEqual(expected, result)
def test_lexer_binops(self):
input_str = "x && y || z"
result = list(simplify(tokenize(input_str)))
expected = [('ID', 'x'), 'AND', ('ID', 'y'), 'OR', ('ID', 'z'), 'EOF']
self.assertEqual(expected, result)
def test_lexer_address(self):
inputs = [
("d = &n", [('ID', 'd'), 'EQUAL', 'AMPERSAND', ('ID', 'n'),
'EOF']),
]
for (input, expected) in inputs:
with self.subTest(input=input):
self.assertEqual(list(simplify(tokenize(input))), expected)
def test_lexer_return(self):
input_str = "return x;"
result = list(simplify(tokenize(input_str)))
expected = ['RETURN', ('ID', 'x'), 'SEMI', 'EOF']
self.assertEqual(expected, result)
def test_lexer_with_let(self):
input_str = "let x := 1 in x + x end"
result = list(simplify(tokenize(input_str)))
expected = ['LET', ('ID', "x"), "ASSIGN", ('NUMBER', 1), "IN",
('ID', "x"), 'PLUS', ('ID', "x"), 'END', 'EOF']
self.assertEqual(expected, result)
def test_lexer_decl_pointers(self):
inputs = [
("int* d", [('TYPE', 'int'), 'TIMES', ('ID', 'd'), 'EOF']),
("int *d", [('TYPE', 'int'), 'TIMES', ('ID', 'd'), 'EOF']),
("long* d", [('TYPE', 'long'), 'TIMES', ('ID', 'd'), 'EOF']),
]
for (input, expected) in inputs:
with self.subTest(input=input):
self.assertEqual(list(simplify(tokenize(input))), expected)
def test_lexer_ident(self):
inputs = [
("x", [('ID', 'x'), 'EOF']),
("x123", [('ID', 'x123'), 'EOF']),
("x_123", [('ID', 'x_123'), 'EOF']),
]
for (input, expected) in inputs:
with self.subTest(input=input):
self.assertEqual(list(simplify(tokenize(input))), expected)
def test_lexer_while(self):
input_str = "while (x > 0) { x = 10; } "
result = list(simplify(tokenize(input_str)))
expected = ['WHILE', 'LPAREN', ('ID', 'x'),
'GREATER', ('NUMBER', 0), 'RPAREN',
'LBRACE', ('ID', 'x'), 'EQUAL', ('NUMBER', 10),
'SEMI', 'RBRACE', 'EOF']
self.assertEqual(expected, result)
def test_lexer_fun_decl(self):
input_str = "void foo(long a, int b, char c) { }"
result = list(simplify(tokenize(input_str)))
expected = [('TYPE', 'void'), ('ID', 'foo'),
'LPAREN', ('TYPE', 'long'), ('ID', 'a'), 'COMMA',
('TYPE', 'int'), ('ID', 'b'), 'COMMA', ('TYPE', 'char'),
('ID', 'c'), 'RPAREN', 'LBRACE', 'RBRACE', 'EOF']
self.assertEqual(expected, result)
def test_lexer_compound_assign(self):
inputs = [
("x += 1", [('ID', 'x'), 'PLUS_EQ', ('NUMBER', 1), 'EOF']),
("x -= 1", [('ID', 'x'), 'MINUS_EQ', ('NUMBER', 1), 'EOF']),
("x *= 1", [('ID', 'x'), 'TIMES_EQ', ('NUMBER', 1), 'EOF']),
("x /= 1", [('ID', 'x'), 'DIVIDE_EQ', ('NUMBER', 1), 'EOF']),
("x %= 1", [('ID', 'x'), 'MOD_EQ', ('NUMBER', 1), 'EOF']),
]
for (input, expected) in inputs:
with self.subTest(input=input):
self.assertEqual(list(simplify(tokenize(input))), expected)
def test_lexer_ops(self):
inputs = [
("x == 0", [('ID', 'x'), 'DBL_EQ', ('NUMBER', 0), 'EOF']),
("x != 0", [('ID', 'x'), 'NOT_EQ', ('NUMBER', 0), 'EOF']),
("x >= 0", [('ID', 'x'), 'GREATER_EQ', ('NUMBER', 0), 'EOF']),
("x > 0", [('ID', 'x'), 'GREATER', ('NUMBER', 0), 'EOF']),
("x <= 0", [('ID', 'x'), 'LESS_EQ', ('NUMBER', 0), 'EOF']),
("x < 0", [('ID', 'x'), 'LESS', ('NUMBER', 0), 'EOF']),
("!x", ['BANG', ('ID', 'x'), 'EOF']),
("! (x == 0)", [
'BANG', 'LPAREN', ('ID', 'x'), 'DBL_EQ', ('NUMBER', 0),
'RPAREN', 'EOF'
]),
]
for (input, expected) in inputs:
with self.subTest(input=input):
self.assertEqual(list(simplify(tokenize(input))), expected)
def test_lexer_decl(self):
input = "long x"
result = [('TYPE', 'long'), ('ID', 'x'), 'EOF']
self.assertEqual(list(simplify(tokenize(input))), result)
def test_lexer_incorrect_char(self):
input_str = "\b"
with self.assertRaises(Exception):
list(simplify(tokenize(input_str)))
