def merge_escaped(stream):
"""Merge tokens whose escaped attribute is True together.
Must be run after detect_escaped.
Parameters
----------
stream: iterator
"""
stream = Peeker(stream, EOF)
queue = []
t = six.advance_iterator(stream)
while t:
if t.escaped:
queue.append(t)
else:
if t.type == "WORD":
if queue:
queue.append(t)
n = stream.peek()
if not n.escaped:
t.value = "".join([c.value for c in queue])
yield t
queue = []
else:
n = stream.peek()
if n.escaped:
queue.append(t)
else:
yield t
else:
if queue:
queue[-1].value = "".join([c.value for c in queue])
queue[-1].type = "WORD"
yield queue[-1]
queue = []
yield t
try:
t = six.advance_iterator(stream)
except StopIteration:
if queue:
t.value = "".join([c.value for c in queue])
t.type = "WORD"
yield t
return
def merge_escaped(stream):
"""Merge tokens whose escaped attribute is True together.
Must be run after detect_escaped.
Parameters
----------
stream: iterator
"""
stream = Peeker(stream, EOF)
queue = []
t = six.advance_iterator(stream)
while t:
if t.escaped:
queue.append(t)
else:
if t.type == "WORD":
if queue:
queue.append(t)
n = stream.peek()
if not n.escaped:
t.value = "".join([c.value for c in queue])
yield t
queue = []
else:
n = stream.peek()
if n.escaped:
queue.append(t)
else:
yield t
else:
if queue:
queue[-1].value = "".join([c.value for c in queue])
queue[-1].type = "WORD"
yield queue[-1]
queue = []
yield t
try:
t = six.advance_iterator(stream)
except StopIteration:
if queue:
t.value = "".join([c.value for c in queue])
t.type = "WORD"
yield t
return
def merge_escaped(stream):
stream = Peeker(stream, EOF)
queue = []
t = six.advance_iterator(stream)
while t:
if t.escaped:
queue.append(t)
else:
if t.type == "WORD":
if queue:
queue.append(t)
n = stream.peek()
if not n.escaped:
t.value = "".join([c.value for c in queue])
yield t
queue = []
else:
n = stream.peek()
if n.escaped:
queue.append(t)
else:
yield t
else:
if queue:
queue[-1].value = "".join([c.value for c in queue])
queue[-1].type = "WORD"
yield queue[-1]
queue = []
yield t
try:
t = six.advance_iterator(stream)
except StopIteration:
if queue:
t.value = "".join([c.value for c in queue])
t.type = "WORD"
yield t
return
def merge_escaped(stream):
stream = Peeker(stream, EOF)
queue = []
t = six.advance_iterator(stream)
while t:
if t.escaped:
queue.append(t)
else:
if t.type == "WORD":
if queue:
queue.append(t)
n = stream.peek()
if not n.escaped:
t.value = "".join([c.value for c in queue])
yield t
queue = []
else:
n = stream.peek()
if n.escaped:
queue.append(t)
else:
yield t
else:
if queue:
queue[-1].value = "".join([c.value for c in queue])
queue[-1].type = "WORD"
yield queue[-1]
queue = []
yield t
try:
t = six.advance_iterator(stream)
except StopIteration:
if queue:
t.value = "".join([c.value for c in queue])
t.type = "WORD"
yield t
return
