def is_finished(self):
result = self.syntaxtable.lookup(self.state[-1], FinishSymbol())
states = list(self.state)
while isinstance(result, Reduce):
i = 0
for i in range(result.amount()):
states.pop()
goto = self.syntaxtable.lookup(states[-1], result.action.left)
states.append(goto.action)
result = self.syntaxtable.lookup(states[-1], FinishSymbol())
if isinstance(result, Accept):
return True
return False
def next_token(self):
try:
t = self.tokeniter()
self.last_read = t[3][-1]
self.last_token_value = t[0]
self.last_split = t[4]
return Terminal(t[1])
except StopIteration:
self.reached_eos = True
return FinishSymbol() # No more tokens to read
except LexingError:
return FinishSymbol(
) # Couldn't continue lexing with given language
def is_finished(self):
states = list(self.state)
if self.temp_parse(states, Terminal("NEWLINE")):
element = self.syntaxtable.lookup(states[-1], FinishSymbol())
if element:
return True
elif self.temp_parse(states, Terminal("DEDENT")):
return True
return False
def next_token(self):
if self.todo:
return self.todo.pop(0)
tok1 = self.get_token_iter()
if tok1 is None:
self.todo.append(Terminal("NEWLINE"))
while self.indents[-1] != 0:
self.todo.append(Terminal("DEDENT"))
self.indents.pop()
self.todo.append(FinishSymbol())
return self.todo.pop(0)
if type(tok1[3][-1].symbol) is not MagicTerminal\
and tok1[3][-1].symbol.name.endswith(tok1[0]):
# only use fully parsed nodes as possible ends
self.last_read = tok1[3][-1]
self.last_token_value = tok1[0]
self.last_split = tok1[4]
if tok1[1] == "<return>":
if self.logical_line: # last line was logical
self.todo.append(Terminal("NEWLINE"))
self.logical_line = False
self.last_ws = 0
return Terminal(tok1[1]) # parse <return> token first
if tok1[1] == "<ws>":
self.last_ws = len(tok1[0])
return Terminal(tok1[1])
if self.is_logical(tok1[1]):
if self.logical_line is False: # first logical token in this line
self.logical_line = True
if self.last_ws > self.indents[-1]:
self.todo.append(Terminal("INDENT"))
self.indents.append(self.last_ws)
elif self.last_ws == self.indents[-1]:
pass
else:
while self.last_ws < self.indents[-1]:
self.todo.append(Terminal("DEDENT"))
self.indents.pop()
self.todo.append(Terminal(tok1[1]))
return self.todo.pop(0)
return Terminal(tok1[1])