def merge_escaped(stream): """Merge tokens whose escaped attribute is True together. Must be run after detect_escaped. Parameters ---------- stream: iterator """ stream = Peeker(stream, EOF) queue = [] t = six.advance_iterator(stream) while t: if t.escaped: queue.append(t) else: if t.type == "WORD": if queue: queue.append(t) n = stream.peek() if not n.escaped: t.value = "".join([c.value for c in queue]) yield t queue = [] else: n = stream.peek() if n.escaped: queue.append(t) else: yield t else: if queue: queue[-1].value = "".join([c.value for c in queue]) queue[-1].type = "WORD" yield queue[-1] queue = [] yield t try: t = six.advance_iterator(stream) except StopIteration: if queue: t.value = "".join([c.value for c in queue]) t.type = "WORD" yield t return
def merge_escaped(stream): stream = Peeker(stream, EOF) queue = [] t = six.advance_iterator(stream) while t: if t.escaped: queue.append(t) else: if t.type == "WORD": if queue: queue.append(t) n = stream.peek() if not n.escaped: t.value = "".join([c.value for c in queue]) yield t queue = [] else: n = stream.peek() if n.escaped: queue.append(t) else: yield t else: if queue: queue[-1].value = "".join([c.value for c in queue]) queue[-1].type = "WORD" yield queue[-1] queue = [] yield t try: t = six.advance_iterator(stream) except StopIteration: if queue: t.value = "".join([c.value for c in queue]) t.type = "WORD" yield t return