def test_simple(self):
bos = self.ast.parent.children[0]
new = TextNode(Terminal("1+2"))
bos.insert_after(new)
self.lexer.relex(new)
assert self.parser.inc_parse([]) == True
assert self.ast.parent.symbol == Nonterminal("Root")
assert isinstance(self.ast.parent.children[0], BOS)
assert isinstance(self.ast.parent.children[-1], EOS)
bos = self.ast.parent.children[0]
root = TextNode(Nonterminal("Root"))
bos = BOS(Terminal(""))
eos = EOS(FinishSymbol())
Start = TextNode(Nonterminal("Startrule"))
root.set_children([bos, Start, eos])
E1 = TextNode(Nonterminal("E"))
Start.set_children([TextNode(N("WS")), E1])
E1.set_children(self.make_nodes([N("E"), T("+"), N("WS"), N("T")]))
E2 = E1.children[0]
E2.set_children(self.make_nodes([N("T")]))
T1 = E2.children[0]
T1.set_children(self.make_nodes([N("P")]))
P1 = T1.children[0]
P1.set_children(self.make_nodes([T("1"), N("WS")]))
T2 = E1.children[3]
T2.set_children(self.make_nodes([N("P")]))
P2 = T2.children[0]
P2.set_children(self.make_nodes([T("2"), N("WS")]))
self.compare_trees(self.ast.parent, root)
def test_closure_0():
s1 = StateSet()
s = State(Production(Nonterminal("Z"), [Nonterminal("S")]), 0) # first state Z ::= .S
s1.add(s)
closure = helper1.closure_0(s1)
assert len(closure.elements) == 4
assert State(Production(Z, [S]), 0) in closure
assert State(Production(S, [S, b]), 0) in closure
assert State(Production(S, [b, A, a]), 0) in closure
assert State(Production(S, [a]), 0) in closure
s2 = StateSet()
s = State(Production(F, [C, D, f]), 0)
s2.add(s)
closure = helper1.closure_0(s2)
assert len(closure.elements) == 4
assert State(Production(F, [C, D, f]), 0) in closure
assert State(Production(C, [D, A]), 0) in closure
assert State(Production(D, [d]), 0) in closure
assert State(Production(D, [Epsilon()]), 1) in closure
s3 = StateSet()
s = State(Production(C, [D, A]), 1)
s3.add(s)
closure = helper1.closure_0(s3)
assert len(closure.elements) == 4
assert State(Production(C, [D, A]), 1) in closure
assert State(Production(A, [a, S, c]), 0) in closure
assert State(Production(A, [a, S, b]), 0) in closure
assert State(Production(A, [a]), 0) in closure
def test_annotations(self):
grammar = """
X ::= "x" {Node(child=#0)};
Y ::= "y" {[#0,#3]};
Z ::= "z" {#1+#2};
A ::= "a" {Node2(child=#0, child2=[#1,#3], child3=#3+[#4])};
%%
x:"x"
"""
bp = BootstrapParser()
bp.parse(grammar)
assert bp.rules[Nonterminal("X")].annotations == [
AstNode("Node", {"child": LookupExpr(0)})
]
assert bp.rules[Nonterminal("Y")].annotations == [
ListExpr([LookupExpr(0), LookupExpr(3)])
]
assert bp.rules[Nonterminal("Z")].annotations == [
AddExpr(LookupExpr(1), LookupExpr(2))
]
assert bp.rules[Nonterminal("A")].annotations == [
AstNode(
"Node2", {
'child': LookupExpr(0),
'child2': ListExpr([LookupExpr(1),
LookupExpr(3)]),
'child3': AddExpr(LookupExpr(3), ListExpr([LookupExpr(4)]))
})
]
def incparse_from_dict(self, rules):
if not rules:
print("Warning: incparser has not access to comment tokens")
elif rules.has_key(Nonterminal("comment")):
rule = rules[Nonterminal("comment")]
for a in rule.alternatives:
if len(a) > 0:
self.comment_tokens.append(a[0].name)
def test_terminal():
t1 = Nonterminal("E")
t2 = Nonterminal("E")
assert t1 == t2
t1 = Terminal("a")
t2 = Terminal("a")
assert t1 == t2
def test_multiple_rules():
p = Parser("""
E ::= A
A ::= \"a\"
""")
p.parse()
assert p.rules[Nonterminal("E")].alternatives == [[Nonterminal("A")]]
assert p.rules[Nonterminal("A")].alternatives == [[Terminal("a")]]
def test_loop_rule():
p = Parser("""
A ::= "a" { "b" } "g"
""")
p.parse()
print(p.rules)
assert p.rules[Nonterminal("A")].alternatives == [[
Terminal("a"), Nonterminal("A_loop")
]]
assert p.rules[Nonterminal("A_loop")].alternatives == [[
Terminal("b"), Nonterminal("A_loop")
], [Terminal("g")]]
def test_more_complex_grammar():
p = Parser("""
name ::= "ID"
| "&" "ID"
| splice
| insert
""")
p.parse()
assert p.rules[Nonterminal("name")].alternatives == [
[Terminal("ID")], [Terminal("&"), Terminal("ID")],
[Nonterminal("splice")], [Nonterminal("insert")]
]
def test_option_rule():
p = Parser("""
A ::= "a" [ "b" ] "g"
""")
p.parse()
print(p.rules)
assert p.rules[Nonterminal("A")].alternatives == [[
Terminal("a"), Nonterminal("A_option")
]]
assert p.rules[Nonterminal("A_option")].alternatives == [[
Terminal("b"), Terminal("g")
], [Terminal("g")]]
def test_empty_alternative(self):
grammar = """
A ::= "a"
| ;
%%
a:"a"
"""
bp = BootstrapParser()
bp.parse(grammar)
assert bp.start_symbol == Nonterminal("A")
assert bp.rules[Nonterminal("A")].alternatives == [[Terminal("a")], []]
def test_nodes(self):
root = TextNode(Nonterminal("Root"))
bos = BOS(Terminal(""))
eos = EOS(FinishSymbol())
a = TextNode(Terminal("a"))
b = TextNode(Terminal("b"))
nB = TextNode(Nonterminal("B"))
nB.set_children([b])
root.set_children([bos, a, nB, eos])
a.next_term = b
b.next_term = eos
l = Lexer([("name", "[a-z]+")])
assert l.treelex(a) == [("ab", "name", 0)]
def test_empty_alternative():
p = Parser("""
E ::= "a"
|
""")
p.parse()
assert p.rules[Nonterminal("E")].alternatives == [[Terminal("a")], []]
def test_relex3(self):
ast = AST()
ast.init()
bos = ast.parent.children[0]
new1 = TextNode(Terminal("1+2"))
new2 = TextNode(Terminal("345"))
new3 = TextNode(Terminal("6+"))
new4 = TextNode(Terminal("789")) # this should never be touched
new4.lookup = "INT"
new5 = TextNode(Terminal("+")) # this should never be touched
new5.lookup = "plus"
bos.insert_after(new1)
new1.insert_after(new2)
new2.insert_after(new3)
new3.insert_after(new4)
new4.insert_after(new5)
self.relex(new1)
assert ast.parent.symbol == Nonterminal("Root")
assert isinstance(ast.parent.children[0], BOS)
assert isinstance(ast.parent.children[-1], EOS)
node = bos.next_term
assert node.symbol == Terminal("1")
node = node.next_term
assert node.symbol == Terminal("+")
node = node.next_term
assert node.symbol == Terminal("23456")
node = node.next_term
assert node.symbol == Terminal("+")
# check that 789 hasn't been relexed
assert node.next_term is new4
assert node.next_term.symbol is new4.symbol
def test_relex(self):
ast = AST()
ast.init()
bos = ast.parent.children[0]
new = TextNode(Terminal("1 + 2 * 3"))
bos.insert_after(new)
self.relex(new)
assert ast.parent.symbol == Nonterminal("Root")
assert isinstance(ast.parent.children[0], BOS)
assert isinstance(ast.parent.children[-1], EOS)
node = bos.next_term
assert node.symbol == Terminal("1")
assert node.lookahead == 1
node = node.next_term
assert node.symbol == Terminal(" ")
assert node.lookahead == 1
node = node.next_term
assert node.symbol == Terminal("+")
assert node.lookahead == 0
node = node.next_term
assert node.symbol == Terminal(" ")
node = node.next_term
assert node.symbol == Terminal("2")
node = node.next_term
assert node.symbol == Terminal(" ")
node = node.next_term
assert node.symbol == Terminal("*")
node = node.next_term
assert node.symbol == Terminal(" ")
node = node.next_term
assert node.symbol == Terminal("3")
node = node.next_term
assert isinstance(node, EOS)
def init(self):
bos = BOS(Terminal(""), 0, [])
eos = EOS(FinishSymbol(), 0, [])
bos.next_term = eos
eos.prev_term = bos
root = TextNode(Nonterminal("Root"), 0, [bos, eos])
self.parent = root
def test_relex_stop(self):
ast = AST()
ast.init()
bos = ast.parent.children[0]
new = TextNode(Terminal("1+2"))
old1 = TextNode(Terminal("*"))
old2 = TextNode(Terminal("3"))
old2.lookup = "INT"
bos.insert_after(new)
new.insert_after(old1)
old1.insert_after(old2)
self.relex(new)
assert ast.parent.symbol == Nonterminal("Root")
assert isinstance(ast.parent.children[0], BOS)
assert isinstance(ast.parent.children[-1], EOS)
node = bos.next_term
assert node.symbol == Terminal("1")
node = node.next_term
assert node.symbol == Terminal("+")
node = node.next_term
assert node.symbol == Terminal("2")
node = node.next_term
assert node.symbol == Terminal("*")
node = node.next_term
assert node.symbol == Terminal("3")
node = node.next_term
assert isinstance(node, EOS)
def parse_rule(self, node):
name = node.children[0].symbol.name
self.current_rulename = name
alternatives = self.parse_alternatives(node.children[4])
symbol = Nonterminal(name)
if self.start_symbol is None:
self.start_symbol = symbol
if self.change_startrule and symbol.name == self.change_startrule:
self.start_symbol = symbol
r = Rule(symbol)
for a in alternatives:
r.add_alternative(a[0], a[1], a[2])
# add additional alternatives to the grammar (grammar extension feature, e.g. languageboxes)
if self.extra_alternatives.has_key(symbol.name):
for n in self.extra_alternatives[symbol.name]:
r.add_alternative([MagicTerminal(n), Nonterminal("WS")], None)
self.rules[symbol] = r
def init_ast(self, magic_parent=None):
bos = BOS(Terminal(""), 0, [])
eos = EOS(FinishSymbol(), 0, [])
bos.magic_parent = magic_parent
eos.magic_parent = magic_parent
bos.next_term = eos
eos.prev_term = bos
root = TextNode(Nonterminal("Root"), 0, [bos, eos])
self.previous_version = AST(root)
def get_next_symbols_no_ws(self):
symbols = set()
for state in self.elements:
if state.p.left == Nonterminal("WS"):
continue
symbol = state.next_symbol_no_ws()
if symbol:
symbols.add(symbol)
return symbols
def next_symbol_no_ws(self):
try:
s = self.p.right[self.d]
if s == Nonterminal("WS"):
#XXX if state of s is nullable, return next symbol as well
s = self.p.right[self.d + 1]
return s
except IndexError:
return None
def parse_symbols(self, node):
if node.children[0].symbol.name == "symbols":
symbols = self.parse_symbols(node.children[0])
symbol = self.parse_symbol(node.children[1])
symbols.append(symbol)
if (
isinstance(symbol, Terminal)
or isinstance(symbol, MagicTerminal)
) and self.implicit_ws(
) and self.current_rulename not in self.options["nowhitespace"]:
symbols.append(Nonterminal("WS"))
return symbols
elif node.children[0].symbol.name == "symbol":
l = []
symbol = self.parse_symbol(node.children[0])
l.append(symbol)
if isinstance(symbol, Terminal) and self.implicit_ws(
) and self.current_rulename not in self.options["nowhitespace"]:
l.append(Nonterminal("WS"))
return l
def parse_symbol(self, node):
node = node.children[0]
if node.lookup == "nonterminal":
return Nonterminal(node.symbol.name)
elif node.lookup == "terminal":
if node.symbol.name != "\"<eos>\"":
self.terminals.add(node.symbol.name[1:-1])
return Terminal(node.symbol.name[1:-1])
elif node.lookup == "languagebox":
return MagicTerminal(node.symbol.name)
elif node.symbol.name == "function":
return self.parse_function(node)
def test_lookup_reference(self):
grammar = """
X ::= "x" {X(name=#0.x)}
;
%%
x:"x"
"""
bp = BootstrapParser()
bp.parse(grammar)
assert bp.rules[Nonterminal("X")].annotations == [
AstNode("X", {"name": LookupExpr(0, "x")})
]
def test_alternatives(self):
grammar = """
X ::= "x" {Node(child=#0)}
| "y" "z" {#1};
%%
x:"x"
"""
bp = BootstrapParser()
bp.parse(grammar)
assert bp.rules[Nonterminal("X")].annotations == [
AstNode("Node", {"child": LookupExpr(0)}),
LookupExpr(1)
]
def test_annotations_extension(self):
grammar = """
X ::= "x" {foreach(#2) X(name=item.x)}
;
%%
x:"x"
"""
bp = BootstrapParser()
bp.parse(grammar)
assert bp.rules[Nonterminal("X")].annotations == [
Foreach("foreach", LookupExpr(2),
AstNode("X", {"name": ReferenceExpr("item", "x")}))
]
def test_group_rule():
p = Parser("""
A ::= "a" ( "b" | "c" ) "g"
""")
p.parse()
print(p.rules)
assert p.rules[Nonterminal("A")].alternatives == [[
Terminal("a"), Nonterminal("A_group1")
]]
assert p.rules[Nonterminal("A_group1")].alternatives == [[
Terminal("b"), Nonterminal("A_group2")
], [Terminal("c"), Nonterminal("A_group2")]]
assert p.rules[Nonterminal("A_group2")].alternatives == [[Terminal("g")]]
def test_simple(self):
grammar = """
X ::= A "b"
| C;
A ::= "a";
C ::= "c";
%%
a:"a"
b:"b"
c:"c"
"""
bp = BootstrapParser()
bp.parse(grammar)
assert bp.start_symbol == Nonterminal("X")
assert bp.rules[Nonterminal("X")].symbol == Nonterminal("X")
assert bp.rules[Nonterminal("X")].alternatives == [[
Nonterminal("A"), Terminal("b")
], [Nonterminal("C")]]
assert bp.rules[Nonterminal("A")].alternatives == [[Terminal("a")]]
assert bp.rules[Nonterminal("C")].alternatives == [[Terminal("c")]]
def test_relex_newline(self):
ast = AST()
ast.init()
bos = ast.parent.children[0]
new1 = TextNode(Terminal("1+2\r3+4"))
bos.insert_after(new1)
self.relex(new1)
assert ast.parent.symbol == Nonterminal("Root")
assert isinstance(ast.parent.children[0], BOS)
assert isinstance(ast.parent.children[-1], EOS)
node = bos.next_term
assert node.symbol == Terminal("1")
node = node.next_term
assert node.symbol == Terminal("+")
node = node.next_term
assert node.symbol == Terminal("2")
node = node.next_term
assert node.symbol == Terminal("\r")
node = node.next_term
assert node.symbol == Terminal("3")
node = node.next_term
assert node.symbol == Terminal("+")
node = node.next_term
assert node.symbol == Terminal("4")
def test_relex4(self):
ast = AST()
ast.init()
bos = ast.parent.children[0]
new1 = TextNode(Terminal("1"))
new2 = TextNode(Terminal("2"))
new3 = TextNode(Terminal("+"))
new4 = TextNode(Terminal("3+4"))
new5 = TextNode(Terminal("+4"))
new6 = TextNode(Terminal("5"))
bos.insert_after(new1)
new1.insert_after(new2)
new2.insert_after(new3)
new3.insert_after(new4)
new4.insert_after(new5)
new5.insert_after(new6)
self.relex(new1)
assert ast.parent.symbol == Nonterminal("Root")
assert isinstance(ast.parent.children[0], BOS)
assert isinstance(ast.parent.children[-1], EOS)
node = bos.next_term
assert node.symbol == Terminal("12")
node = node.next_term
assert node.symbol == Terminal("+")
node = node.next_term
assert node.symbol == Terminal("3")
node = node.next_term
assert node.symbol == Terminal("+")
node = node.next_term
assert node.symbol == Terminal("4")
node = node.next_term
assert node.symbol == Terminal("+")
node = node.next_term
assert node.symbol == Terminal("45")
node = node.next_term
assert isinstance(node, EOS)
def get_ast(self):
bos = Node(Terminal("bos"), 0, [])
eos = Node(FinishSymbol(), 0, [])
root = Node(Nonterminal("Root"), 0, [bos, self.ast_stack[0], eos])
return AST(root)
