def relex_import(self, startnode, version = 0):
"""Optimised relex for freshly imported files."""
success = self.lex(startnode.symbol.name)
bos = startnode.prev_term # bos
parent = bos.parent
eos = parent.children.pop()
last_node = bos
for match in success:
if match is success[0]:
# reuse old node for fist node to mimic the behaviour of a
# normal relex
node = startnode
node.symbol.name = match[0]
else:
node = TextNode(Terminal(match[0]))
node.lookup = match[1]
parent.children.append(node)
last_node.next_term = node
last_node.right = node
node.left = last_node
node.prev_term = last_node
node.parent = parent
last_node = node
node.mark_changed()
parent.children.append(eos)
last_node.right = eos # link to eos
last_node.next_term = eos
eos.left = last_node
eos.prev_term = last_node
bos.mark_changed()
eos.mark_changed()
parent.mark_changed()
def merge_back(self, read_nodes, generated_tokens):
any_changes = False
# insert new nodes into tree
it = iter(read_nodes)
for t in generated_tokens:
try:
node = it.next()
except StopIteration:
node = TextNode(Terminal(""))
last_node.insert_after(node)
any_changes = True
last_node = node
node.symbol.name = t.source
if node.lookup != t.name or t.source.find("*/") > 0:
any_changes = True
node.mark_changed()
else:
node.mark_version()
node.lookup = t.name
node.lookahead = t.lookahead
# delete left over nodes
while True:
try:
node = it.next()
node.parent.remove_child(node)
any_changes = True
except StopIteration:
break
return any_changes
def merge_back(self, read_nodes, generated_tokens):
any_changes = False
# insert new nodes into tree
it = iter(read_nodes)
for t in generated_tokens:
try:
node = it.next()
except StopIteration:
node = TextNode(Terminal(""))
last_node.insert_after(node)
any_changes = True
last_node = node
node.symbol.name = t.source
node.indent = None
if node.lookup != t.name:
node.mark_changed()
any_changes = True
else:
node.mark_version()
# we need to invalidate the newline if we changed whitespace or
# logical nodes that come after it
if node.lookup == "<ws>" or node.lookup != t.name:
prev = node.prev_term
while isinstance(prev.symbol, IndentationTerminal):
prev = prev.prev_term
if prev.lookup == "<return>":
prev.mark_changed()
any_changes = True
elif isinstance(prev, BOS):
# if there is no return, re-indentation won't be triggered
# in the incremental parser so we have to mark the next
# terminal. possibly only use case: bos <ws> pass DEDENT eos
node.next_term.mark_changed()
# XXX this should become neccessary with incparse optimisations turned on
if node.lookup == "\\" and node.next_term.lookup == "<return>":
node.next_term.mark_changed()
any_changes = True
node.lookup = t.name
node.lookahead = t.lookahead
# delete left over nodes
while True:
try:
node = it.next()
node.parent.remove_child(node)
any_changes = True
except StopIteration:
break
return any_changes
def merge_back(self, read_nodes, generated_tokens):
"""
Replace the symbols in the nodes with the newly generated tokens.
We loop over read_nodes and generated tokens at the same pace and replace the read_node.symbol.name with
the corresponding generated_token.source. We also update the node's lookup (the type of token). I
If it is changed, the node is marked changed (A node whom lookup did not change is identical to the parser)
If the arrays are of unequal length:
- If the length of generated_tokens is insufficient, we add extra nodes
- Excess nodes are removed
:param read_nodes: Nodes that have been read by the relexer
:param generated_tokens: Tokens that have been found during relexing
:return:
"""
any_changes = False
# insert new nodes into tree
it = iter(read_nodes)
for t in generated_tokens:
try:
node = it.next()
except StopIteration:
node = TextNode(Terminal(""))
last_node.insert_after(node)
any_changes = True
last_node = node
node.symbol.name = t.source
node.indent = None
if node.lookup != t.name:
node.mark_changed()
any_changes = True
else:
node.mark_version()
# we need to invalidate the newline if we changed whitespace or
# logical nodes that come after it
if node.lookup == "<ws>" or node.lookup != t.name:
prev = node.prev_term
while isinstance(prev.symbol, IndentationTerminal):
prev = prev.prev_term
if prev.lookup == "<return>":
prev.mark_changed()
any_changes = True
elif isinstance(prev, BOS):
# if there is no return, re-indentation won't be triggered
# in the incremental parser so we have to mark the next
# terminal. possibly only use case: bos <ws> pass DEDENT eos
node.next_term.mark_changed()
# XXX this should become neccessary with incparse optimisations turned on
if node.lookup == "\\" and node.next_term.lookup == "<return>":
node.next_term.mark_changed()
any_changes = True
node.lookup = t.name
node.lookahead = t.lookahead
# delete left over nodes
while True:
try:
node = it.next()
node.parent.remove_child(node)
any_changes = True
except StopIteration:
break
return any_changes
