def test_annotations(self): grammar = """ X ::= "x" {Node(child=#0)}; Y ::= "y" {[#0,#3]}; Z ::= "z" {#1+#2}; A ::= "a" {Node2(child=#0, child2=[#1,#3], child3=#3+[#4])}; %% x:"x" """ bp = BootstrapParser() bp.parse(grammar) assert bp.rules[Nonterminal("X")].annotations == [ AstNode("Node", {"child": LookupExpr(0)}) ] assert bp.rules[Nonterminal("Y")].annotations == [ ListExpr([LookupExpr(0), LookupExpr(3)]) ] assert bp.rules[Nonterminal("Z")].annotations == [ AddExpr(LookupExpr(1), LookupExpr(2)) ] assert bp.rules[Nonterminal("A")].annotations == [ AstNode( "Node2", { 'child': LookupExpr(0), 'child2': ListExpr([LookupExpr(1), LookupExpr(3)]), 'child3': AddExpr(LookupExpr(3), ListExpr([LookupExpr(4)])) }) ]
def test_lookup_reference(self): grammar = """ X ::= item {#0.x} ; item ::= "a" {Var(x=#0)} | "b" {Var(x=#0)} ; %% a:"a" b:"b" comma:"," """ bootstrap = BootstrapParser(lr_type=1, whitespaces=False) bootstrap.parse(grammar) self.treemanager = TreeManager() self.treemanager.add_parser(bootstrap.incparser, bootstrap.inclexer, "") self.treemanager.set_font_test(7, 17) self.treemanager.key_normal("a") assert bootstrap.incparser.last_status == True # test ast generation root = bootstrap.incparser.previous_version.parent assert root.symbol.name == "Root" assert root.children[1].symbol.name == "X" E = root.children[1] assert E.alternate.symbol.name == "a"
def test_empty_alternative(self): grammar = """ A ::= "a" | ; %% a:"a" """ bp = BootstrapParser() bp.parse(grammar) assert bp.start_symbol == Nonterminal("A") assert bp.rules[Nonterminal("A")].alternatives == [[Terminal("a")], []]
def test_bootstrapping1(self): bootstrap = BootstrapParser(lr_type=1, whitespaces=False) test_grammar = 'S ::= "abc"; %% abc:\"abc\"' bootstrap.parse(test_grammar) self.treemanager = TreeManager() self.treemanager.add_parser(bootstrap.incparser, bootstrap.inclexer, "") self.treemanager.set_font_test(7, 17) self.treemanager.key_normal("a") assert bootstrap.incparser.last_status == False self.treemanager.key_normal("b") assert bootstrap.incparser.last_status == False self.treemanager.key_normal("c") assert bootstrap.incparser.last_status == True
def test_lookup_reference(self): grammar = """ X ::= "x" {X(name=#0.x)} ; %% x:"x" """ bp = BootstrapParser() bp.parse(grammar) assert bp.rules[Nonterminal("X")].annotations == [ AstNode("X", {"name": LookupExpr(0, "x")}) ]
def test_calculator(self): calc = """ E ::= T {#0} | E "plus" T {Plus(arg1=#0,arg2=#2)} ; T ::= P {#0} | T "mul" P {Mul(arg1=#0,arg2=#2)} ; P ::= "INT" {#0} ; %% INT:"[0-9]+" plus:"\+" mul:"\*" <ws>:"[ \\t]+" <return>:"[\\n\\r]" """ bootstrap = BootstrapParser(lr_type=1, whitespaces=False) bootstrap.parse(calc) self.treemanager = TreeManager() self.treemanager.add_parser(bootstrap.incparser, bootstrap.inclexer, "") self.treemanager.set_font_test(7, 17) self.treemanager.key_normal("1") assert bootstrap.incparser.last_status == True self.treemanager.key_normal("+") assert bootstrap.incparser.last_status == False self.treemanager.key_normal("2") assert bootstrap.incparser.last_status == True self.treemanager.key_normal("*") assert bootstrap.incparser.last_status == False self.treemanager.key_normal("3") assert bootstrap.incparser.last_status == True # test ast generation root = bootstrap.incparser.previous_version.parent assert root.symbol.name == "Root" assert root.children[1].symbol.name == "E" E = root.children[1] assert isinstance(E.alternate, AstNode) assert E.alternate.name == "Plus" plus = E.alternate assert plus.children['arg1'].symbol.name == "1" mul = plus.children['arg2'] assert isinstance(mul, AstNode) assert mul.name == "Mul" assert mul.children['arg1'].symbol.name == "2" assert mul.children['arg2'].symbol.name == "3"
def test_simple(self): grammar = """ X ::= "x"; %% numbers:"[0-9]+" text:"[a-z][a-z]*" escaped:"abc\\\"escaped" """ bp = BootstrapParser() bp.parse(grammar) assert bp.lrules[0] == ("numbers", "[0-9]+") assert bp.lrules[1] == ("text", "[a-z][a-z]*") assert bp.lrules[2] == ("escaped", "abc\\\"escaped")
def test_simple(self): grammar = """ X ::= "x"; %% a:"a" b:"b" c:"c" """ bp = BootstrapParser() bp.parse(grammar) assert bp.lrules[0] == ("a", "a") assert bp.lrules[1] == ("b", "b") assert bp.lrules[2] == ("c", "c")
def test_annotations_extension(self): grammar = """ X ::= "x" {foreach(#2) X(name=item.x)} ; %% x:"x" """ bp = BootstrapParser() bp.parse(grammar) assert bp.rules[Nonterminal("X")].annotations == [ Foreach("foreach", LookupExpr(2), AstNode("X", {"name": ReferenceExpr("item", "x")})) ]
def test_alternatives(self): grammar = """ X ::= "x" {Node(child=#0)} | "y" "z" {#1}; %% x:"x" """ bp = BootstrapParser() bp.parse(grammar) assert bp.rules[Nonterminal("X")].annotations == [ AstNode("Node", {"child": LookupExpr(0)}), LookupExpr(1) ]
def test_foreach(self): grammar = """ X ::= items {foreach(#0) Field(b=item.x)} ; items ::= items "comma" item {#0 + [#2]} | item {[#0]} ; item ::= "a" {Var(x=#0)} | "b" {Var(x=#0)} ; %% a:"a" b:"b" comma:"," """ bootstrap = BootstrapParser(lr_type=1, whitespaces=False) bootstrap.parse(grammar) self.treemanager = TreeManager() self.treemanager.add_parser(bootstrap.incparser, bootstrap.inclexer, "") self.treemanager.set_font_test(7, 17) self.treemanager.key_normal("a") assert bootstrap.incparser.last_status == True self.treemanager.key_normal(",") assert bootstrap.incparser.last_status == False self.treemanager.key_normal("b") assert bootstrap.incparser.last_status == True # test ast generation root = bootstrap.incparser.previous_version.parent assert root.symbol.name == "Root" assert root.children[1].symbol.name == "X" E = root.children[1] assert isinstance(E.alternate, ListNode) assert isinstance(E.alternate.children[0], AstNode) assert isinstance(E.alternate.children[1], AstNode) assert E.alternate.children[0].name == "Field" assert E.alternate.children[1].name == "Field" assert E.alternate.children[0].children['b'].symbol.name == "a" assert E.alternate.children[1].children['b'].symbol.name == "b"
def test_simple(self): grammar = """ X ::= A "b" | C; A ::= "a"; C ::= "c"; %% a:"a" b:"b" c:"c" """ bp = BootstrapParser() bp.parse(grammar) assert bp.start_symbol == Nonterminal("X") assert bp.rules[Nonterminal("X")].symbol == Nonterminal("X") assert bp.rules[Nonterminal("X")].alternatives == [[ Nonterminal("A"), Terminal("b") ], [Nonterminal("C")]] assert bp.rules[Nonterminal("A")].alternatives == [[Terminal("a")]] assert bp.rules[Nonterminal("C")].alternatives == [[Terminal("c")]]
def test_simple(self): bootstrap = BootstrapParser() test_grammar = """S ::= A {If(child1=#1, child2=[#3, #4])}; A ::= \"a\"; %% a:\"a\"""" bootstrap.parse(test_grammar)
def test_error(self): bootstrap = BootstrapParser() test_grammar = """S ::= A {If(child1=#1, child2=[#3, #4]}; %% a:\"a\"""" py.test.raises(Exception, "bootstrap.parse(test_grammar)")
def load(self, buildlexer=True): from grammar_parser.bootstrap import BootstrapParser from jsonmanager import JsonManager from incparser.incparser import IncParser if _cache.has_key(self.name + "::parser"): syntaxtable, whitespaces = _cache[self.name + "::parser"] incparser = IncParser() incparser.syntaxtable = syntaxtable incparser.whitespaces = whitespaces incparser.init_ast() inclexer = _cache[self.name + "::lexer"] incparser.lexer = inclexer # give parser a reference to its lexer (needed for multiline comments) return (incparser, inclexer) else: manager = JsonManager(unescape=True) root, language, whitespaces = manager.load(self.filename)[0] pickle_id = hash(self) bootstrap = BootstrapParser(lr_type=1, whitespaces=whitespaces) bootstrap.ast = root bootstrap.extra_alternatives = self.alts bootstrap.change_startrule = self.extract bootstrap.read_options() bootstrap.parse_both() bootstrap.create_parser(pickle_id) bootstrap.create_lexer(buildlexer) whitespace = bootstrap.implicit_ws() _cache[self.name + "::lexer"] = bootstrap.inclexer _cache[self.name + "::json"] = (root, language, whitespaces) _cache[self.name + "::parser"] = (bootstrap.incparser.syntaxtable, whitespace) bootstrap.incparser.lexer = bootstrap.inclexer return (bootstrap.incparser, bootstrap.inclexer)
def load(self): from grammar_parser.bootstrap import BootstrapParser from jsonmanager import JsonManager if _cache.has_key(self.name + "::parser"): root, language, whitespaces = _cache[self.name + "::json"] # parse rules as they are needed by the incremental parser to # detect comments manager = JsonManager(unescape=True) root, language, whitespaces = manager.load(self.filename)[0] pickle_id = hash(self) bootstrap = BootstrapParser(lr_type=1, whitespaces=whitespaces) bootstrap.ast = root bootstrap.parse_rules(root.children[1].children[1].children[0]) pickle_id, whitespace = _cache[self.name + "::parser"] from incparser.incparser import IncParser incparser = IncParser() incparser.from_dict(bootstrap.rules, None, None, whitespace, pickle_id, None) incparser.init_ast() inclexer = _cache[self.name + "::lexer"] incparser.lexer = inclexer # give parser a reference to its lexer (needed for multiline comments) return (incparser, inclexer) else: manager = JsonManager(unescape=True) root, language, whitespaces = manager.load(self.filename)[0] pickle_id = hash(self) bootstrap = BootstrapParser(lr_type=1, whitespaces=whitespaces) bootstrap.ast = root bootstrap.extra_alternatives = self.alts bootstrap.change_startrule = self.extract bootstrap.read_options() bootstrap.parse_both() bootstrap.create_parser(pickle_id) bootstrap.create_lexer() whitespace = bootstrap.implicit_ws() _cache[self.name + "::lexer"] = bootstrap.inclexer _cache[self.name + "::json"] = (root, language, whitespaces) _cache[self.name + "::parser"] = (pickle_id, whitespace) bootstrap.incparser.lexer = bootstrap.inclexer return (bootstrap.incparser, bootstrap.inclexer)