def test_precedence(self):
buf = """
fn prec() -> int {
2 + 3 * 2 + 2 / 8;
}
"""
exprs = [
ast.Function(
"prec",
[],
[
ast.ExprStatement(
ast.BinaryOp(
ast.BinaryOp(
ast.Number(2),
ast.BinaryOp(ast.Number(3), ast.Number(2),
Token(TokenType.MULTIPLY, "*")),
Token(TokenType.ADD, "+"),
),
ast.BinaryOp(ast.Number(2), ast.Number(8),
Token(TokenType.DIVIDE, "/")),
Token(TokenType.ADD, "+"),
))
],
ast.Type(ast.TypeKind.INT),
)
]
self._test_parse_impl(buf, exprs)
def _const_decl():
self._expect(Token('IDENTIFIER', None))
record.name = self.current_token.value
self._forward()
self._expect(Token(None, '='))
self._forward()
self._expect(Token('NUMBER', None))
record.value = int(self.current_token.value)
self.table.enter(record)
self._forward()
def _procedure():
record = Record('procedure', None, None, self.current_level)
while self.current_token.value == 'procedure':
self._forward()
self._expect(Token('IDENTIFIER', None))
record.name = self.current_token.value
self.table.enter(record)
self._forward()
self._expect(Token(None, ';'))
self._forward()
self._block(3)
self._expect(Token(None, ';'))
self._forward()
def test_while_loop(self):
buf = """
fn loopTest(int x) -> int {
while (x < 5) {
print("blah");
}
return 0;
}
"""
exprs = [
ast.Function(
"loopTest",
[("x", ast.Type(ast.TypeKind.INT))],
[
ast.WhileLoop(
ast.BinaryOp(ast.VariableRef("x"), ast.Number(5),
Token(TokenType.LESS_THAN, "<")),
[
ast.ExprStatement(
ast.FunctionCall("print",
[ast.String("blah")]))
],
),
ast.ReturnStatement(ast.Number(0)),
],
ast.Type(ast.TypeKind.INT),
)
]
self._test_parse_impl(buf, exprs)
def test_if_statements(self):
buf = """
fn foo() -> string {
if (3 == 3) {
return "foo";
}
return "bar";
}
"""
exprs = [
ast.Function(
"foo",
[],
[
ast.IfStatement(
ast.BinaryOp(ast.Number(3), ast.Number(3),
Token(TokenType.EQUALS, "==")),
[ast.ReturnStatement(ast.String("foo"))],
list(),
),
ast.ReturnStatement(ast.String("bar")),
],
ast.Type(ast.TypeKind.STRING),
)
]
self._test_parse_impl(buf, exprs)
def _var_decl():
nonlocal dx
self._expect(Token('IDENTIFIER', None))
record.name = self.current_token.value
record.address = dx
dx += 1
self.table.enter(record)
self._forward()
def _program(self):
""" The following is rec-descent parser.
Each handler will move forward 1 token before return, therefore self.current_token is assigned at
the beginning of each function.
"""
self._forward()
self.table.enter(Record())
self._block(3)
self._expect(Token(None, '.'))
def _factor(self):
if self.current_token.type == 'IDENTIFIER':
record = self.table.get(self.current_token.value)
if record.type == 'const':
self.pcode.gen(OpCode.LIT, 0, record.value)
elif record.type == 'var':
self.pcode.gen(OpCode.LOD, self.current_level - record.level,
record.address)
elif record.type == 'procedure':
raise ParserError('Wrong variable type')
self._forward()
elif self.current_token.type == 'NUMBER':
self.pcode.gen(OpCode.LIT, 0, int(self.current_token.value))
self._forward()
else:
self._expect(Token(None, '('))
self._forward()
self._expression()
self._expect(Token(None, ')'))
self._forward()
def _var():
def _var_decl():
nonlocal dx
self._expect(Token('IDENTIFIER', None))
record.name = self.current_token.value
record.address = dx
dx += 1
self.table.enter(record)
self._forward()
record = Record('var', None, None, self.current_level)
self._expect(Token(None, 'var'))
self._forward()
while True:
_var_decl()
if self.current_token.value == ',':
self._forward()
else:
break
self._expect(Token(None, ';'))
self._forward()
return dx
def test_dot_operator(self):
buf = """
print(foo.bar);
"""
tokens = [
Token(TokenType.IDENTIFIER, "print"),
Token(TokenType.L_BRACKET, "("),
Token(TokenType.IDENTIFIER, "foo"),
Token(TokenType.DOT, "."),
Token(TokenType.IDENTIFIER, "bar"),
Token(TokenType.R_BRACKET, ")"),
Token(TokenType.SEMICOLON, ";"),
Token(TokenType.EOF, str()),
]
self._test_lex_impl(buf, tokens)
def _const():
def _const_decl():
self._expect(Token('IDENTIFIER', None))
record.name = self.current_token.value
self._forward()
self._expect(Token(None, '='))
self._forward()
self._expect(Token('NUMBER', None))
record.value = int(self.current_token.value)
self.table.enter(record)
self._forward()
record = Record('const', None, None, 0)
self._expect(Token(None, 'const'))
self._forward()
while True:
_const_decl()
if self.current_token.value == ',':
self._forward()
else:
break
self._expect(Token(None, ';'))
self._forward()
def _condition(self):
if self.current_token.value == 'odd':
self._forward()
self._expression()
self.pcode.gen(OpCode.OPR, 0, 6)
else:
self._expression()
self._expect(Token('RELATIONAL_OPERATOR', None))
op = self.current_token.value
self._forward()
self._expression()
if op == '=':
self.pcode.gen(OpCode.OPR, 0, 7)
elif op == '<>':
self.pcode.gen(OpCode.OPR, 0, 8)
elif op == '<':
self.pcode.gen(OpCode.OPR, 0, 9)
elif op == '>=':
self.pcode.gen(OpCode.OPR, 0, 10)
elif op == '>':
self.pcode.gen(OpCode.OPR, 0, 11)
elif op == '<=':
self.pcode.gen(OpCode.OPR, 0, 12)
def _statement(self):
if self.current_token.type == 'IDENTIFIER':
record = self.table.get(self.current_token.value, 'var')
self._forward()
self._expect(Token(None, ':='))
self._forward()
self._expression()
self.pcode.gen(OpCode.STO, self.current_level - record.level,
record.address)
elif self.current_token.value == 'if':
self._forward()
self._condition()
self._expect(Token(None, 'then'))
self._forward()
code1 = len(self.pcode)
self.pcode.gen(OpCode.JPC, 0, 0)
self._statement() # then statement
code2 = len(self.pcode)
self.pcode.gen(OpCode.JMP, 0, 0)
if self.current_token.value == 'else':
self._forward()
self.pcode[code1].a = len(self.pcode)
self._statement() # else statement
else:
self.pcode[code1].a = len(self.pcode)
self.pcode[code2].a = len(self.pcode)
elif self.current_token.value == 'while':
code1 = len(self.pcode)
self._forward()
self._condition()
code2 = len(self.pcode)
self.pcode.gen(OpCode.JPC, 0, 0)
self._expect(Token(None, 'do'))
self._forward()
self._statement()
self.pcode.gen(OpCode.JMP, 0, code1)
self.pcode[code2].a = len(self.pcode)
elif self.current_token.value == 'call':
self._forward()
self._expect(Token('IDENTIFIER', None))
record = self.table.get(self.current_token.value, 'procedure')
self.pcode.gen(OpCode.CAL, self.current_level - record.level,
record.address)
self._forward()
elif self.current_token.value == 'begin':
self._forward()
self._statement()
while self.current_token.value == ';':
self._forward()
self._statement()
self._expect(Token(None, 'end'))
self._forward()
elif self.current_token.value == 'repeat':
self._forward()
code1 = len(self.pcode)
self._statement()
while self.current_token.value == ';':
self._forward()
self._statement()
self._expect(Token(None, 'until'))
self._forward()
self._condition()
self.pcode.gen(OpCode.JPC, 0, code1)
elif self.current_token.value == 'read':
self._forward()
self._expect(Token(None, '('))
self._forward()
while True:
self._expect(Token('IDENTIFIER', None))
record = self.table.get(self.current_token.value, 'var')
self.pcode.gen(OpCode.RED, self.current_level - record.level,
record.address)
self._forward()
if self.current_token.value != ',':
break
else:
self._forward()
self._expect(Token(None, ')'))
self._forward()
elif self.current_token.value == 'write':
self._forward()
self._expect(Token(None, '('))
self._forward()
while True:
self._expression()
self.pcode.gen(OpCode.WRT, 0, 0)
if self.current_token.value != ',':
break
else:
self._forward()
self._expect(Token(None, ')'))
self._forward()
def test_lexer_on_inputfile(self):
path = self.__get_path("example.nl")
with open(path, "r") as f:
lexer = Lexer(inFile=f)
expected_tokens = [
Token('ID', 'x', 1),
Token('ASSIGN', '=', 1),
Token('NUMBER', 4, 1),
Token('SEMICOLON', ';', 1),
Token('FUNC', 'func', 2),
Token('ID', 'random', 2),
Token('LPAREN', '(', 2),
Token('RPAREN', ')', 2),
Token('INT', 'int', 2),
Token('LBRACE', '{', 2),
Token('RETURN', 'return', 3),
Token('NUMBER', '5', 3),
Token('SEMICOLON', ';', 3),
Token('RBRACE', '}', 4)
]
for i in range(0, len(lexer.tokens)):
self.assertEqual(expected_tokens[i], lexer.tokens[i])
def test_control_flow(self):
buf = """
if (x == 3) {
let y = 3;
}
"""
tokens = [
Token(TokenType.IF, "if"),
Token(TokenType.L_BRACKET, "("),
Token(TokenType.IDENTIFIER, "x"),
Token(TokenType.EQUALS, "=="),
Token(TokenType.NUMBER_LITERAL, "3"),
Token(TokenType.R_BRACKET, ")"),
Token(TokenType.L_BRACE, "{"),
Token(TokenType.LET, "let"),
Token(TokenType.IDENTIFIER, "y"),
Token(TokenType.ASSIGN, "="),
Token(TokenType.NUMBER_LITERAL, "3"),
Token(TokenType.SEMICOLON, ";"),
Token(TokenType.R_BRACE, "}"),
Token(TokenType.EOF, str()),
]
self._test_lex_impl(buf, tokens)
def test_function_w_args(self):
buf = """
fn foo(int one, string two) -> int {
print("blah");
};
"""
tokens = [
Token(TokenType.FUNCTION, "fn"),
Token(TokenType.IDENTIFIER, "foo"),
Token(TokenType.L_BRACKET, "("),
Token(TokenType.INT, "int"),
Token(TokenType.IDENTIFIER, "one"),
Token(TokenType.COMMA, ","),
Token(TokenType.STRING, "string"),
Token(TokenType.IDENTIFIER, "two"),
Token(TokenType.R_BRACKET, ")"),
Token(TokenType.ARROW, "->"),
Token(TokenType.INT, "int"),
Token(TokenType.L_BRACE, "{"),
Token(TokenType.IDENTIFIER, "print"),
Token(TokenType.L_BRACKET, "("),
Token(TokenType.STRING_LITERAL, "blah"),
Token(TokenType.R_BRACKET, ")"),
Token(TokenType.SEMICOLON, ";"),
Token(TokenType.R_BRACE, "}"),
Token(TokenType.SEMICOLON, ";"),
Token(TokenType.EOF, str()),
]
self._test_lex_impl(buf, tokens)
def test_struct(self):
buf = """
struct Foo {
int id = 3;
string name;
fn foo() -> int {
return 3;
}
};
"""
tokens = [
Token(TokenType.STRUCTURE, "struct"),
Token(TokenType.IDENTIFIER, "Foo"),
Token(TokenType.L_BRACE, "{"),
Token(TokenType.INT, "int"),
Token(TokenType.IDENTIFIER, "id"),
Token(TokenType.ASSIGN, "="),
Token(TokenType.NUMBER_LITERAL, "3"),
Token(TokenType.SEMICOLON, ";"),
Token(TokenType.STRING, "string"),
Token(TokenType.IDENTIFIER, "name"),
Token(TokenType.SEMICOLON, ";"),
Token(TokenType.FUNCTION, "fn"),
Token(TokenType.IDENTIFIER, "foo"),
Token(TokenType.L_BRACKET, "("),
Token(TokenType.R_BRACKET, ")"),
Token(TokenType.ARROW, "->"),
Token(TokenType.INT, "int"),
Token(TokenType.L_BRACE, "{"),
Token(TokenType.RETURN, "return"),
Token(TokenType.NUMBER_LITERAL, "3"),
Token(TokenType.SEMICOLON, ";"),
Token(TokenType.R_BRACE, "}"),
Token(TokenType.R_BRACE, "}"),
Token(TokenType.SEMICOLON, ";"),
Token(TokenType.EOF, str()),
]
self._test_lex_impl(buf, tokens)