def test_simple(self): bos = self.ast.parent.children[0] new = TextNode(Terminal("1+2")) bos.insert_after(new) self.lexer.relex(new) assert self.parser.inc_parse([]) == True assert self.ast.parent.symbol == Nonterminal("Root") assert isinstance(self.ast.parent.children[0], BOS) assert isinstance(self.ast.parent.children[-1], EOS) bos = self.ast.parent.children[0] root = TextNode(Nonterminal("Root")) bos = BOS(Terminal("")) eos = EOS(FinishSymbol()) Start = TextNode(Nonterminal("Startrule")) root.set_children([bos, Start, eos]) E1 = TextNode(Nonterminal("E")) Start.set_children([TextNode(N("WS")), E1]) E1.set_children(self.make_nodes([N("E"), T("+"), N("WS"), N("T")])) E2 = E1.children[0] E2.set_children(self.make_nodes([N("T")])) T1 = E2.children[0] T1.set_children(self.make_nodes([N("P")])) P1 = T1.children[0] P1.set_children(self.make_nodes([T("1"), N("WS")])) T2 = E1.children[3] T2.set_children(self.make_nodes([N("P")])) P2 = T2.children[0] P2.set_children(self.make_nodes([T("2"), N("WS")])) self.compare_trees(self.ast.parent, root)
def test_closure_0(): s1 = StateSet() s = State(Production(Nonterminal("Z"), [Nonterminal("S")]), 0) # first state Z ::= .S s1.add(s) closure = helper1.closure_0(s1) assert len(closure.elements) == 4 assert State(Production(Z, [S]), 0) in closure assert State(Production(S, [S, b]), 0) in closure assert State(Production(S, [b, A, a]), 0) in closure assert State(Production(S, [a]), 0) in closure s2 = StateSet() s = State(Production(F, [C, D, f]), 0) s2.add(s) closure = helper1.closure_0(s2) assert len(closure.elements) == 4 assert State(Production(F, [C, D, f]), 0) in closure assert State(Production(C, [D, A]), 0) in closure assert State(Production(D, [d]), 0) in closure assert State(Production(D, [Epsilon()]), 1) in closure s3 = StateSet() s = State(Production(C, [D, A]), 1) s3.add(s) closure = helper1.closure_0(s3) assert len(closure.elements) == 4 assert State(Production(C, [D, A]), 1) in closure assert State(Production(A, [a, S, c]), 0) in closure assert State(Production(A, [a, S, b]), 0) in closure assert State(Production(A, [a]), 0) in closure
def test_annotations(self): grammar = """ X ::= "x" {Node(child=#0)}; Y ::= "y" {[#0,#3]}; Z ::= "z" {#1+#2}; A ::= "a" {Node2(child=#0, child2=[#1,#3], child3=#3+[#4])}; %% x:"x" """ bp = BootstrapParser() bp.parse(grammar) assert bp.rules[Nonterminal("X")].annotations == [ AstNode("Node", {"child": LookupExpr(0)}) ] assert bp.rules[Nonterminal("Y")].annotations == [ ListExpr([LookupExpr(0), LookupExpr(3)]) ] assert bp.rules[Nonterminal("Z")].annotations == [ AddExpr(LookupExpr(1), LookupExpr(2)) ] assert bp.rules[Nonterminal("A")].annotations == [ AstNode( "Node2", { 'child': LookupExpr(0), 'child2': ListExpr([LookupExpr(1), LookupExpr(3)]), 'child3': AddExpr(LookupExpr(3), ListExpr([LookupExpr(4)])) }) ]
def incparse_from_dict(self, rules): if not rules: print("Warning: incparser has not access to comment tokens") elif rules.has_key(Nonterminal("comment")): rule = rules[Nonterminal("comment")] for a in rule.alternatives: if len(a) > 0: self.comment_tokens.append(a[0].name)
def test_terminal(): t1 = Nonterminal("E") t2 = Nonterminal("E") assert t1 == t2 t1 = Terminal("a") t2 = Terminal("a") assert t1 == t2
def test_multiple_rules(): p = Parser(""" E ::= A A ::= \"a\" """) p.parse() assert p.rules[Nonterminal("E")].alternatives == [[Nonterminal("A")]] assert p.rules[Nonterminal("A")].alternatives == [[Terminal("a")]]
def test_loop_rule(): p = Parser(""" A ::= "a" { "b" } "g" """) p.parse() print(p.rules) assert p.rules[Nonterminal("A")].alternatives == [[ Terminal("a"), Nonterminal("A_loop") ]] assert p.rules[Nonterminal("A_loop")].alternatives == [[ Terminal("b"), Nonterminal("A_loop") ], [Terminal("g")]]
def test_more_complex_grammar(): p = Parser(""" name ::= "ID" | "&" "ID" | splice | insert """) p.parse() assert p.rules[Nonterminal("name")].alternatives == [ [Terminal("ID")], [Terminal("&"), Terminal("ID")], [Nonterminal("splice")], [Nonterminal("insert")] ]
def test_option_rule(): p = Parser(""" A ::= "a" [ "b" ] "g" """) p.parse() print(p.rules) assert p.rules[Nonterminal("A")].alternatives == [[ Terminal("a"), Nonterminal("A_option") ]] assert p.rules[Nonterminal("A_option")].alternatives == [[ Terminal("b"), Terminal("g") ], [Terminal("g")]]
def test_empty_alternative(self): grammar = """ A ::= "a" | ; %% a:"a" """ bp = BootstrapParser() bp.parse(grammar) assert bp.start_symbol == Nonterminal("A") assert bp.rules[Nonterminal("A")].alternatives == [[Terminal("a")], []]
def test_nodes(self): root = TextNode(Nonterminal("Root")) bos = BOS(Terminal("")) eos = EOS(FinishSymbol()) a = TextNode(Terminal("a")) b = TextNode(Terminal("b")) nB = TextNode(Nonterminal("B")) nB.set_children([b]) root.set_children([bos, a, nB, eos]) a.next_term = b b.next_term = eos l = Lexer([("name", "[a-z]+")]) assert l.treelex(a) == [("ab", "name", 0)]
def test_empty_alternative(): p = Parser(""" E ::= "a" | """) p.parse() assert p.rules[Nonterminal("E")].alternatives == [[Terminal("a")], []]
def test_relex3(self): ast = AST() ast.init() bos = ast.parent.children[0] new1 = TextNode(Terminal("1+2")) new2 = TextNode(Terminal("345")) new3 = TextNode(Terminal("6+")) new4 = TextNode(Terminal("789")) # this should never be touched new4.lookup = "INT" new5 = TextNode(Terminal("+")) # this should never be touched new5.lookup = "plus" bos.insert_after(new1) new1.insert_after(new2) new2.insert_after(new3) new3.insert_after(new4) new4.insert_after(new5) self.relex(new1) assert ast.parent.symbol == Nonterminal("Root") assert isinstance(ast.parent.children[0], BOS) assert isinstance(ast.parent.children[-1], EOS) node = bos.next_term assert node.symbol == Terminal("1") node = node.next_term assert node.symbol == Terminal("+") node = node.next_term assert node.symbol == Terminal("23456") node = node.next_term assert node.symbol == Terminal("+") # check that 789 hasn't been relexed assert node.next_term is new4 assert node.next_term.symbol is new4.symbol
def test_relex(self): ast = AST() ast.init() bos = ast.parent.children[0] new = TextNode(Terminal("1 + 2 * 3")) bos.insert_after(new) self.relex(new) assert ast.parent.symbol == Nonterminal("Root") assert isinstance(ast.parent.children[0], BOS) assert isinstance(ast.parent.children[-1], EOS) node = bos.next_term assert node.symbol == Terminal("1") assert node.lookahead == 1 node = node.next_term assert node.symbol == Terminal(" ") assert node.lookahead == 1 node = node.next_term assert node.symbol == Terminal("+") assert node.lookahead == 0 node = node.next_term assert node.symbol == Terminal(" ") node = node.next_term assert node.symbol == Terminal("2") node = node.next_term assert node.symbol == Terminal(" ") node = node.next_term assert node.symbol == Terminal("*") node = node.next_term assert node.symbol == Terminal(" ") node = node.next_term assert node.symbol == Terminal("3") node = node.next_term assert isinstance(node, EOS)
def init(self): bos = BOS(Terminal(""), 0, []) eos = EOS(FinishSymbol(), 0, []) bos.next_term = eos eos.prev_term = bos root = TextNode(Nonterminal("Root"), 0, [bos, eos]) self.parent = root
def test_relex_stop(self): ast = AST() ast.init() bos = ast.parent.children[0] new = TextNode(Terminal("1+2")) old1 = TextNode(Terminal("*")) old2 = TextNode(Terminal("3")) old2.lookup = "INT" bos.insert_after(new) new.insert_after(old1) old1.insert_after(old2) self.relex(new) assert ast.parent.symbol == Nonterminal("Root") assert isinstance(ast.parent.children[0], BOS) assert isinstance(ast.parent.children[-1], EOS) node = bos.next_term assert node.symbol == Terminal("1") node = node.next_term assert node.symbol == Terminal("+") node = node.next_term assert node.symbol == Terminal("2") node = node.next_term assert node.symbol == Terminal("*") node = node.next_term assert node.symbol == Terminal("3") node = node.next_term assert isinstance(node, EOS)
def parse_rule(self, node): name = node.children[0].symbol.name self.current_rulename = name alternatives = self.parse_alternatives(node.children[4]) symbol = Nonterminal(name) if self.start_symbol is None: self.start_symbol = symbol if self.change_startrule and symbol.name == self.change_startrule: self.start_symbol = symbol r = Rule(symbol) for a in alternatives: r.add_alternative(a[0], a[1], a[2]) # add additional alternatives to the grammar (grammar extension feature, e.g. languageboxes) if self.extra_alternatives.has_key(symbol.name): for n in self.extra_alternatives[symbol.name]: r.add_alternative([MagicTerminal(n), Nonterminal("WS")], None) self.rules[symbol] = r
def init_ast(self, magic_parent=None): bos = BOS(Terminal(""), 0, []) eos = EOS(FinishSymbol(), 0, []) bos.magic_parent = magic_parent eos.magic_parent = magic_parent bos.next_term = eos eos.prev_term = bos root = TextNode(Nonterminal("Root"), 0, [bos, eos]) self.previous_version = AST(root)
def get_next_symbols_no_ws(self): symbols = set() for state in self.elements: if state.p.left == Nonterminal("WS"): continue symbol = state.next_symbol_no_ws() if symbol: symbols.add(symbol) return symbols
def next_symbol_no_ws(self): try: s = self.p.right[self.d] if s == Nonterminal("WS"): #XXX if state of s is nullable, return next symbol as well s = self.p.right[self.d + 1] return s except IndexError: return None
def parse_symbols(self, node): if node.children[0].symbol.name == "symbols": symbols = self.parse_symbols(node.children[0]) symbol = self.parse_symbol(node.children[1]) symbols.append(symbol) if ( isinstance(symbol, Terminal) or isinstance(symbol, MagicTerminal) ) and self.implicit_ws( ) and self.current_rulename not in self.options["nowhitespace"]: symbols.append(Nonterminal("WS")) return symbols elif node.children[0].symbol.name == "symbol": l = [] symbol = self.parse_symbol(node.children[0]) l.append(symbol) if isinstance(symbol, Terminal) and self.implicit_ws( ) and self.current_rulename not in self.options["nowhitespace"]: l.append(Nonterminal("WS")) return l
def parse_symbol(self, node): node = node.children[0] if node.lookup == "nonterminal": return Nonterminal(node.symbol.name) elif node.lookup == "terminal": if node.symbol.name != "\"<eos>\"": self.terminals.add(node.symbol.name[1:-1]) return Terminal(node.symbol.name[1:-1]) elif node.lookup == "languagebox": return MagicTerminal(node.symbol.name) elif node.symbol.name == "function": return self.parse_function(node)
def test_lookup_reference(self): grammar = """ X ::= "x" {X(name=#0.x)} ; %% x:"x" """ bp = BootstrapParser() bp.parse(grammar) assert bp.rules[Nonterminal("X")].annotations == [ AstNode("X", {"name": LookupExpr(0, "x")}) ]
def test_alternatives(self): grammar = """ X ::= "x" {Node(child=#0)} | "y" "z" {#1}; %% x:"x" """ bp = BootstrapParser() bp.parse(grammar) assert bp.rules[Nonterminal("X")].annotations == [ AstNode("Node", {"child": LookupExpr(0)}), LookupExpr(1) ]
def test_annotations_extension(self): grammar = """ X ::= "x" {foreach(#2) X(name=item.x)} ; %% x:"x" """ bp = BootstrapParser() bp.parse(grammar) assert bp.rules[Nonterminal("X")].annotations == [ Foreach("foreach", LookupExpr(2), AstNode("X", {"name": ReferenceExpr("item", "x")})) ]
def test_group_rule(): p = Parser(""" A ::= "a" ( "b" | "c" ) "g" """) p.parse() print(p.rules) assert p.rules[Nonterminal("A")].alternatives == [[ Terminal("a"), Nonterminal("A_group1") ]] assert p.rules[Nonterminal("A_group1")].alternatives == [[ Terminal("b"), Nonterminal("A_group2") ], [Terminal("c"), Nonterminal("A_group2")]] assert p.rules[Nonterminal("A_group2")].alternatives == [[Terminal("g")]]
def test_simple(self): grammar = """ X ::= A "b" | C; A ::= "a"; C ::= "c"; %% a:"a" b:"b" c:"c" """ bp = BootstrapParser() bp.parse(grammar) assert bp.start_symbol == Nonterminal("X") assert bp.rules[Nonterminal("X")].symbol == Nonterminal("X") assert bp.rules[Nonterminal("X")].alternatives == [[ Nonterminal("A"), Terminal("b") ], [Nonterminal("C")]] assert bp.rules[Nonterminal("A")].alternatives == [[Terminal("a")]] assert bp.rules[Nonterminal("C")].alternatives == [[Terminal("c")]]
def test_relex_newline(self): ast = AST() ast.init() bos = ast.parent.children[0] new1 = TextNode(Terminal("1+2\r3+4")) bos.insert_after(new1) self.relex(new1) assert ast.parent.symbol == Nonterminal("Root") assert isinstance(ast.parent.children[0], BOS) assert isinstance(ast.parent.children[-1], EOS) node = bos.next_term assert node.symbol == Terminal("1") node = node.next_term assert node.symbol == Terminal("+") node = node.next_term assert node.symbol == Terminal("2") node = node.next_term assert node.symbol == Terminal("\r") node = node.next_term assert node.symbol == Terminal("3") node = node.next_term assert node.symbol == Terminal("+") node = node.next_term assert node.symbol == Terminal("4")
def test_relex4(self): ast = AST() ast.init() bos = ast.parent.children[0] new1 = TextNode(Terminal("1")) new2 = TextNode(Terminal("2")) new3 = TextNode(Terminal("+")) new4 = TextNode(Terminal("3+4")) new5 = TextNode(Terminal("+4")) new6 = TextNode(Terminal("5")) bos.insert_after(new1) new1.insert_after(new2) new2.insert_after(new3) new3.insert_after(new4) new4.insert_after(new5) new5.insert_after(new6) self.relex(new1) assert ast.parent.symbol == Nonterminal("Root") assert isinstance(ast.parent.children[0], BOS) assert isinstance(ast.parent.children[-1], EOS) node = bos.next_term assert node.symbol == Terminal("12") node = node.next_term assert node.symbol == Terminal("+") node = node.next_term assert node.symbol == Terminal("3") node = node.next_term assert node.symbol == Terminal("+") node = node.next_term assert node.symbol == Terminal("4") node = node.next_term assert node.symbol == Terminal("+") node = node.next_term assert node.symbol == Terminal("45") node = node.next_term assert isinstance(node, EOS)
def get_ast(self): bos = Node(Terminal("bos"), 0, []) eos = Node(FinishSymbol(), 0, []) root = Node(Nonterminal("Root"), 0, [bos, self.ast_stack[0], eos]) return AST(root)