def relex_import(self, startnode, version = 0): """Optimised relex for freshly imported files.""" success = self.lex(startnode.symbol.name) bos = startnode.prev_term # bos parent = bos.parent eos = parent.children.pop() last_node = bos for match in success: if match is success[0]: # reuse old node for fist node to mimic the behaviour of a # normal relex node = startnode node.symbol.name = match[0] else: node = TextNode(Terminal(match[0])) node.lookup = match[1] parent.children.append(node) last_node.next_term = node last_node.right = node node.left = last_node node.prev_term = last_node node.parent = parent last_node = node node.mark_changed() parent.children.append(eos) last_node.right = eos # link to eos last_node.next_term = eos eos.left = last_node eos.prev_term = last_node bos.mark_changed() eos.mark_changed() parent.mark_changed()
def merge_back(self, read_nodes, generated_tokens): any_changes = False # insert new nodes into tree it = iter(read_nodes) for t in generated_tokens: try: node = it.next() except StopIteration: node = TextNode(Terminal("")) last_node.insert_after(node) any_changes = True last_node = node node.symbol.name = t.source if node.lookup != t.name or t.source.find("*/") > 0: any_changes = True node.mark_changed() else: node.mark_version() node.lookup = t.name node.lookahead = t.lookahead # delete left over nodes while True: try: node = it.next() node.parent.remove_child(node) any_changes = True except StopIteration: break return any_changes
def merge_back(self, read_nodes, generated_tokens): any_changes = False # insert new nodes into tree it = iter(read_nodes) for t in generated_tokens: try: node = it.next() except StopIteration: node = TextNode(Terminal("")) last_node.insert_after(node) any_changes = True last_node = node node.symbol.name = t.source node.indent = None if node.lookup != t.name: node.mark_changed() any_changes = True else: node.mark_version() # we need to invalidate the newline if we changed whitespace or # logical nodes that come after it if node.lookup == "<ws>" or node.lookup != t.name: prev = node.prev_term while isinstance(prev.symbol, IndentationTerminal): prev = prev.prev_term if prev.lookup == "<return>": prev.mark_changed() any_changes = True elif isinstance(prev, BOS): # if there is no return, re-indentation won't be triggered # in the incremental parser so we have to mark the next # terminal. possibly only use case: bos <ws> pass DEDENT eos node.next_term.mark_changed() # XXX this should become neccessary with incparse optimisations turned on if node.lookup == "\\" and node.next_term.lookup == "<return>": node.next_term.mark_changed() any_changes = True node.lookup = t.name node.lookahead = t.lookahead # delete left over nodes while True: try: node = it.next() node.parent.remove_child(node) any_changes = True except StopIteration: break return any_changes
def merge_back(self, read_nodes, generated_tokens): """ Replace the symbols in the nodes with the newly generated tokens. We loop over read_nodes and generated tokens at the same pace and replace the read_node.symbol.name with the corresponding generated_token.source. We also update the node's lookup (the type of token). I If it is changed, the node is marked changed (A node whom lookup did not change is identical to the parser) If the arrays are of unequal length: - If the length of generated_tokens is insufficient, we add extra nodes - Excess nodes are removed :param read_nodes: Nodes that have been read by the relexer :param generated_tokens: Tokens that have been found during relexing :return: """ any_changes = False # insert new nodes into tree it = iter(read_nodes) for t in generated_tokens: try: node = it.next() except StopIteration: node = TextNode(Terminal("")) last_node.insert_after(node) any_changes = True last_node = node node.symbol.name = t.source node.indent = None if node.lookup != t.name: node.mark_changed() any_changes = True else: node.mark_version() # we need to invalidate the newline if we changed whitespace or # logical nodes that come after it if node.lookup == "<ws>" or node.lookup != t.name: prev = node.prev_term while isinstance(prev.symbol, IndentationTerminal): prev = prev.prev_term if prev.lookup == "<return>": prev.mark_changed() any_changes = True elif isinstance(prev, BOS): # if there is no return, re-indentation won't be triggered # in the incremental parser so we have to mark the next # terminal. possibly only use case: bos <ws> pass DEDENT eos node.next_term.mark_changed() # XXX this should become neccessary with incparse optimisations turned on if node.lookup == "\\" and node.next_term.lookup == "<return>": node.next_term.mark_changed() any_changes = True node.lookup = t.name node.lookahead = t.lookahead # delete left over nodes while True: try: node = it.next() node.parent.remove_child(node) any_changes = True except StopIteration: break return any_changes