def make_chunk_iter(stream, separator, limit=None, buffer_size=1024*10):
"""Works like :func:`make_line_iter` but accepts a separator
which divides chunks. If you want newline based processing
you should use :func:`make_line_iter` instead as it
supports arbitrary newline markers.
added support for iterators as input stream.
:param stream: the stream or iterate to iterate over.
:param separator: the separator that divides chunks.
:param limit: the limit in bytes for the stream. (Usually
content length. Not necessary if the `stream`
is otherwise already limited).
:param buffer_size: The optional buffer size.
"""
_iter = stream_iter(stream, limit, buffer_size)
first_item = next(_iter, '')
if not first_item:
return
_iter = chain((first_item,), _iter)
separator = safe_bytes(separator)
_split = re.compile(b'(' + re.escape(separator) + b')').split
_join = b''.join
buffer = []
while True:
new_data = next(_iter, '')
if not new_data:
break
chunks = _split(new_data)
new_buf = []
for i in chain(buffer, chunks):
if i == separator:
yield _join(new_buf)
new_buf = []
else:
new_buf.append(i)
buffer = new_buf
if buffer:
yield _join(buffer)
def __bytes__(self):
return safe_bytes(self.__str__())
