def test_lexer(self): e1 = "1" self.assertEqual(list(simplify(tokenize(e1))), [('NUMBER', 1), 'EOF']) input_str = "(1 + 11 * 22)" result = list(simplify(tokenize(input_str))) expected = ['LPAREN', ('NUMBER', 1), 'PLUS', ('NUMBER', 11), 'TIMES', ('NUMBER', 22), 'RPAREN', 'EOF'] self.assertEqual(expected, result)
def test_lexer_comment(self): input_str = "// this is a comment\n+" result = list(simplify(tokenize(input_str))) expected = ['PLUS', 'EOF'] self.assertEqual(expected, result)
def test_lexer_continue(self): input_str = "continue;" result = list(simplify(tokenize(input_str))) expected = ['CONTINUE', 'SEMI', 'EOF'] self.assertEqual(expected, result)
def test_lexer_break(self): input_str = "break;" result = list(simplify(tokenize(input_str))) expected = ['BREAK', 'SEMI', 'EOF'] self.assertEqual(expected, result)
def test_lexer_print(self): input_str = "print(x);" result = list(simplify(tokenize(input_str))) expected = ['PRINT', 'LPAREN', ('ID', 'x'), 'RPAREN', 'SEMI', 'EOF'] self.assertEqual(expected, result)
def test_lexer_binops(self): input_str = "x && y || z" result = list(simplify(tokenize(input_str))) expected = [('ID', 'x'), 'AND', ('ID', 'y'), 'OR', ('ID', 'z'), 'EOF'] self.assertEqual(expected, result)
def test_lexer_address(self): inputs = [ ("d = &n", [('ID', 'd'), 'EQUAL', 'AMPERSAND', ('ID', 'n'), 'EOF']), ] for (input, expected) in inputs: with self.subTest(input=input): self.assertEqual(list(simplify(tokenize(input))), expected)
def test_lexer_return(self): input_str = "return x;" result = list(simplify(tokenize(input_str))) expected = ['RETURN', ('ID', 'x'), 'SEMI', 'EOF'] self.assertEqual(expected, result)
def test_lexer_with_let(self): input_str = "let x := 1 in x + x end" result = list(simplify(tokenize(input_str))) expected = ['LET', ('ID', "x"), "ASSIGN", ('NUMBER', 1), "IN", ('ID', "x"), 'PLUS', ('ID', "x"), 'END', 'EOF'] self.assertEqual(expected, result)
def test_lexer_decl_pointers(self): inputs = [ ("int* d", [('TYPE', 'int'), 'TIMES', ('ID', 'd'), 'EOF']), ("int *d", [('TYPE', 'int'), 'TIMES', ('ID', 'd'), 'EOF']), ("long* d", [('TYPE', 'long'), 'TIMES', ('ID', 'd'), 'EOF']), ] for (input, expected) in inputs: with self.subTest(input=input): self.assertEqual(list(simplify(tokenize(input))), expected)
def test_lexer_ident(self): inputs = [ ("x", [('ID', 'x'), 'EOF']), ("x123", [('ID', 'x123'), 'EOF']), ("x_123", [('ID', 'x_123'), 'EOF']), ] for (input, expected) in inputs: with self.subTest(input=input): self.assertEqual(list(simplify(tokenize(input))), expected)
def test_lexer_while(self): input_str = "while (x > 0) { x = 10; } " result = list(simplify(tokenize(input_str))) expected = ['WHILE', 'LPAREN', ('ID', 'x'), 'GREATER', ('NUMBER', 0), 'RPAREN', 'LBRACE', ('ID', 'x'), 'EQUAL', ('NUMBER', 10), 'SEMI', 'RBRACE', 'EOF'] self.assertEqual(expected, result)
def test_lexer_fun_decl(self): input_str = "void foo(long a, int b, char c) { }" result = list(simplify(tokenize(input_str))) expected = [('TYPE', 'void'), ('ID', 'foo'), 'LPAREN', ('TYPE', 'long'), ('ID', 'a'), 'COMMA', ('TYPE', 'int'), ('ID', 'b'), 'COMMA', ('TYPE', 'char'), ('ID', 'c'), 'RPAREN', 'LBRACE', 'RBRACE', 'EOF'] self.assertEqual(expected, result)
def test_lexer_compound_assign(self): inputs = [ ("x += 1", [('ID', 'x'), 'PLUS_EQ', ('NUMBER', 1), 'EOF']), ("x -= 1", [('ID', 'x'), 'MINUS_EQ', ('NUMBER', 1), 'EOF']), ("x *= 1", [('ID', 'x'), 'TIMES_EQ', ('NUMBER', 1), 'EOF']), ("x /= 1", [('ID', 'x'), 'DIVIDE_EQ', ('NUMBER', 1), 'EOF']), ("x %= 1", [('ID', 'x'), 'MOD_EQ', ('NUMBER', 1), 'EOF']), ] for (input, expected) in inputs: with self.subTest(input=input): self.assertEqual(list(simplify(tokenize(input))), expected)
def test_lexer_ops(self): inputs = [ ("x == 0", [('ID', 'x'), 'DBL_EQ', ('NUMBER', 0), 'EOF']), ("x != 0", [('ID', 'x'), 'NOT_EQ', ('NUMBER', 0), 'EOF']), ("x >= 0", [('ID', 'x'), 'GREATER_EQ', ('NUMBER', 0), 'EOF']), ("x > 0", [('ID', 'x'), 'GREATER', ('NUMBER', 0), 'EOF']), ("x <= 0", [('ID', 'x'), 'LESS_EQ', ('NUMBER', 0), 'EOF']), ("x < 0", [('ID', 'x'), 'LESS', ('NUMBER', 0), 'EOF']), ("!x", ['BANG', ('ID', 'x'), 'EOF']), ("! (x == 0)", [ 'BANG', 'LPAREN', ('ID', 'x'), 'DBL_EQ', ('NUMBER', 0), 'RPAREN', 'EOF' ]), ] for (input, expected) in inputs: with self.subTest(input=input): self.assertEqual(list(simplify(tokenize(input))), expected)
def test_lexer_decl(self): input = "long x" result = [('TYPE', 'long'), ('ID', 'x'), 'EOF'] self.assertEqual(list(simplify(tokenize(input))), result)
def test_lexer_incorrect_char(self): input_str = "\b" with self.assertRaises(Exception): list(simplify(tokenize(input_str)))