def test_nodes(self):
root = TextNode(Nonterminal("Root"))
bos = BOS(Terminal(""))
eos = EOS(FinishSymbol())
a = TextNode(Terminal("a"))
b = TextNode(Terminal("b"))
nB = TextNode(Nonterminal("B"))
nB.set_children([b])
root.set_children([bos, a, nB, eos])
a.next_term = b
b.next_term = eos
l = Lexer([("name", "[a-z]+")])
assert l.treelex(a) == [("ab", "name", 0)]
def parse_after(self, la, split=None, maxtoks=1, maxdist=0):
"""Checks if la can be parsed in the current state. If la is whitespace,
continue until we can parse the next non-whitespace token."""
parsed_tokens = 0
parsed_distance = 0
if split:
token = self.lexer.lex(la.prev_term.symbol.name[split:])
tmpla = la
la = TextNode(Terminal(token[0][1]))
la.next_term = tmpla
while True:
lookup = get_lookup(la)
element = self.syntaxtable.lookup(self.state[-1], lookup)
# If we see the errornode here and the parse table action is
# either Shift or Accept, then the inserted language box has fixed
# the error without wrapping it inside the box
if la is self.errornode and type(element) in [Shift, Accept]:
self.seen_error = True
if type(element) is Reduce:
for i in range(element.amount()):
self.state.pop()
goto = self.syntaxtable.lookup(self.state[-1],
element.action.left)
assert goto is not None
self.state.append(goto.action)
continue
if type(element) is Shift:
# if whitespace continue
if la.lookup in ws_tokens:
self.state.append(element.action)
self.abs_parse_distance += len(la.symbol.name)
parsed_distance += len(la.symbol.name)
la = la.next_term
continue
self.state.append(element.action)
self.abs_parse_distance += len(la.symbol.name)
parsed_tokens += 1
parsed_distance += len(la.symbol.name)
if parsed_tokens >= maxtoks and parsed_distance >= maxdist:
return True
la = la.next_term
continue
if type(element) is Accept:
return True
if parsed_tokens > 0:
return True
return False
def parse_after(self, la, split=None, distance=1):
"""Checks if la can be parsed in the current state. If la is whitespace,
continue until we can parse the next non-whitespace token."""
parsed_tokens = 0
if split:
token = self.lexer.lex(la.prev_term.symbol.name[split:])
tmpla = la
la = TextNode(Terminal(token[0][1]))
la.next_term = tmpla
while True:
lookup = get_lookup(la)
element = self.syntaxtable.lookup(self.state[-1], lookup)
if type(element) is Reduce:
for i in range(element.amount()):
self.state.pop()
goto = self.syntaxtable.lookup(self.state[-1],
element.action.left)
assert goto is not None
self.state.append(goto.action)
continue
if type(element) is Shift:
# if whitespace continue
if la.lookup in ws_tokens:
self.state.append(element.action)
self.abs_parse_distance += len(la.symbol.name)
la = la.next_term
continue
self.state.append(element.action)
self.abs_parse_distance += len(la.symbol.name)
parsed_tokens += 1
if parsed_tokens == distance:
return True
la = la.next_term
continue
if type(element) is Accept:
return True
if parsed_tokens > 0:
return True
return False
def parse_after_lbox_h1(self,
lbox,
end,
cut,
errornode=None,
split=None,
distance=1):
self.abs_parse_distance = 0
parsed_tokens = 0
# copy stack
stack = []
for i in range(cut + 1):
stack.append(self.op.stack[i].state)
self.abs_parse_distance += self.op.stack[i].textlength()
after_end = end.next_term
# do all reductions until there's a shift or accept (whitespace doesn't
# count)
lboxnode = TextNode(lbox)
la = lboxnode
if split:
token = self.ol.lex(end.symbol.name[split:])
splitla = TextNode(Terminal(token[0][1]))
splitla.next_term = after_end
la.next_term = splitla
else:
la.next_term = after_end
while True:
if la.deleted:
la = la.next_term
continue
element = self.op.syntaxtable.lookup(stack[-1],
self.op.get_lookup(la))
if type(element) is Reduce:
for i in range(element.amount()):
stack.pop()
goto = self.op.syntaxtable.lookup(stack[-1],
element.action.left)
assert goto is not None
stack.append(goto.action)
continue
if type(element) is Shift:
if errornode and la is errornode:
return True
# if whitespace continue
if la.lookup in ws_tokens or la is lboxnode:
stack.append(element.action)
if la is not lboxnode:
self.abs_parse_distance += len(la.symbol.name)
la = la.next_term
continue
if not errornode:
stack.append(element.action)
self.abs_parse_distance += len(la.symbol.name)
parsed_tokens += 1
if parsed_tokens == distance:
return True
la = la.next_term
continue
if type(element) is Accept:
return True
if parsed_tokens > 0:
return True
return False