def test_parse_and_ast_assignment(): token_tree, remainder = parse(g, simple_assign) # Just check it does not crash for now. my_ast = to_ast(token_tree) token_tree, remainder = parse(g, assign_string) # Just check it does not crash for now. my_ast = to_ast(token_tree)
def test_integer(self): self.assertEqual(parse('1'), [Token('1', TokenType.INTEGER)]) self.assertEqual(parse('234'), [Token('234', TokenType.INTEGER)]) self.assertEqual(parse('1 23 456'), [ Token('1', TokenType.INTEGER), Token('23', TokenType.INTEGER), Token('456', TokenType.INTEGER), ])
def test_real(self): self.assertEqual(parse('1.1'), [Token('1.1', TokenType.REAL)]) self.assertEqual(parse('03.1415'), [Token('03.1415', TokenType.REAL)]) self.assertEqual(parse('1.1 2.2 3.3'), [ Token('1.1', TokenType.REAL), Token('2.2', TokenType.REAL), Token('3.3', TokenType.REAL), ])
def test_operator(self): self.assertEqual(parse('+'), [Token('+', TokenType.OPERATOR)]) self.assertEqual(parse('-'), [Token('-', TokenType.OPERATOR)]) self.assertEqual(parse('+ - + -'), [ Token('+', TokenType.OPERATOR), Token('-', TokenType.OPERATOR), Token('+', TokenType.OPERATOR), Token('-', TokenType.OPERATOR), ])
def test_parser_assignment(): token_tree, remainder = parse(g, simple_assign) assert remainder == '' assert token_tree == ['Wrap', ['Block', ['Statement', ['Declaration', ['Type', 'int'], ['Identifier', 'valid_identifier'], '=', ['Expr', ['Expr2', ['SimpleExpr', ['Integer', '42']]]]], ';']]] # Note we dont necessarily raise an exception when we cant parse. Just when we cant parse *anything*. token_tree, remainder = parse(g, invalid_identifier) assert remainder != '' token_tree, remainder = parse(g, invalid_identifier_big) assert remainder != '' token_tree, remainder = parse(g, assign_string) assert remainder == ''
def test_ast_binop_precedence(): binop = "(1 + 2) * 3" token_tree, remainder = parse(g, binop) assert remainder == '', 'Parsing failed, check parser tests.' ast = to_ast(token_tree) nodes = list(ast.walk()) expected_order = [BinOp.MULTIPLY, BinOp.ADD] assert [node.operation for node in nodes if isinstance(node, BinOp)] == expected_order binop_2 = "1 + 2 * 3" ast = get_ast(binop_2) nodes = list(ast.walk()) expected_order = [BinOp.ADD, BinOp.MULTIPLY] assert [node.operation for node in nodes if isinstance(node, BinOp)] == expected_order binop_3 = "1 * 2 + 3" ast = get_ast(binop_3) nodes = list(ast.walk()) expected_order = [BinOp.ADD, BinOp.MULTIPLY] assert [node.operation for node in nodes if isinstance(node, BinOp)] == expected_order binop_3 = "4 / 3 * 2" ast = get_ast(binop_3) nodes = list(ast.walk()) expected_order = [BinOp.MULTIPLY, BinOp.DIVIDE] assert [node.operation for node in nodes if isinstance(node, BinOp)] == expected_order
def test_numeric_expression(self): self.assertEqual(parse('1+1'), [ Token('1', TokenType.INTEGER), Token('+', TokenType.OPERATOR), Token('1', TokenType.INTEGER), ]) self.assertEqual(parse('+-3.14 + 1'), [ Token('+', TokenType.OPERATOR), Token('-', TokenType.OPERATOR), Token('3.14', TokenType.REAL), Token('+', TokenType.OPERATOR), Token('1', TokenType.INTEGER), ]) self.assertEqual(parse('1+++--2.2---+1'), [ Token('1', TokenType.INTEGER), Token('+', TokenType.OPERATOR), Token('+', TokenType.OPERATOR), Token('+', TokenType.OPERATOR), Token('-', TokenType.OPERATOR), Token('-', TokenType.OPERATOR), Token('2.2', TokenType.REAL), Token('-', TokenType.OPERATOR), Token('-', TokenType.OPERATOR), Token('-', TokenType.OPERATOR), Token('+', TokenType.OPERATOR), Token('1', TokenType.INTEGER), ]) self.assertEqual(parse('10**2'), [ Token('10', TokenType.INTEGER), Token('**', TokenType.OPERATOR), Token('2', TokenType.INTEGER) ]) self.assertEqual(parse('2*10**2*2'), [ Token('2', TokenType.INTEGER), Token('*', TokenType.OPERATOR), Token('10', TokenType.INTEGER), Token('**', TokenType.OPERATOR), Token('2', TokenType.INTEGER), Token('*', TokenType.OPERATOR), Token('2', TokenType.INTEGER) ])
def test_simple_module_function_to_llvm(): token_tree, remainder = parse(g, simple_function) my_ast = to_ast(token_tree) ir_code = to_llvm(my_ast) expected_ir_code = """define i64 @"fibo"() { entry: ret i64 42 }""" assert expected_ir_code.strip() in str(ir_code)
def __init__(self, filename): assert isinstance(filename, str) ui.debug("parsing " + filename) with ui.ident: self.filename = filename self.name = self.__get_module_name(filename) self.tokens = lexer.parse(filename) self.__parse() variables.add(self.name, "$__path", os.path.dirname(filename)) variables.add_empty(self.name, "$__null")
def test_paren(self): self.assertEqual(parse('2*((1+3)/2+1)'), [ Token('2', TokenType.INTEGER), Token('*', TokenType.OPERATOR), Token('(', TokenType.PAREN), Token('(', TokenType.PAREN), Token('1', TokenType.INTEGER), Token('+', TokenType.OPERATOR), Token('3', TokenType.INTEGER), Token(')', TokenType.PAREN), Token('/', TokenType.OPERATOR), Token('2', TokenType.INTEGER), Token('+', TokenType.OPERATOR), Token('1', TokenType.INTEGER), Token(')', TokenType.PAREN), ])
def test_function_node_to_llvm(): token_tree, remainder = parse(g, simple_function) my_ast = to_ast(token_tree) # Let's get the function node func_node = None for node in my_ast.walk(): if isinstance(node, Function): func_node = node break module = ir.Module('test') ir_code = function_to_llvm(func_node, module) expected_ir_code = """define i64 @"fibo"() { entry: ret i64 42 }""" assert expected_ir_code.strip() in str(ir_code)
def compile(source_file): token_list, remainder = parse(g, source_file.read()) assert remainder.strip() == '', 'Failed to parse!' ast = to_ast(token_list) print(to_llvm(ast))
#!/usr/bin/env python # -*- coding: utf-8 -*- import os import lexer import parser import compiler ROOT_PATH = os.path.dirname(__file__) + "/source.txt" lexer = lexer.Lexer(ROOT_PATH) lexer.parse() parser = parser.Parser(lexer) interpreter = compiler.Interpreter(parser.parse()) interpreter.interpretate()
def test_parser_binop(): binop = "40 + 2" token_tree, remainder = parse(g, binop) assert remainder == ''
def test_parse_return(): t_tree, rem = parse(g, 'return 42;') assert rem == ''
def get_ast(source): token_tree, remainder = parse(g, source) assert remainder == '', 'Parsing failed, check parser tests.' return to_ast(token_tree)
def weak_parse_test(source): token_tree, remainder = parse(g, source) assert remainder == ''