def run_zero_buffer_bench():
d = defaultdict(int)
cur_buffer = Buffer.alloc(8192)
last_pos = 0
collator = BufferCollator()
with open("/usr/share/dict/words") as f:
done = False
while not done:
try:
read = cur_buffer.read_from(f.fileno())
except BufferFull:
cur_buffer = Buffer.alloc(8192)
last_pos = 0
continue
except EOFError:
read = 0
done = True
view = cur_buffer.view(last_pos, last_pos + read)
last_pos += read
collator.append(view)
if b"\n" in view:
data = collator.collapse()
last_newline_pos = data.rfind(b"\n")
for line in data[:last_newline_pos].split(b"\n"):
d[chr(line[0])] += 1
collator.append(data[last_newline_pos + 1:])
def run_zero_buffer_bench():
d = defaultdict(int)
cur_buffer = Buffer.alloc(8192)
last_pos = 0
collator = BufferCollator()
with open("/usr/share/dict/words") as f:
done = False
while not done:
try:
read = cur_buffer.read_from(f.fileno())
except BufferFull:
cur_buffer = Buffer.alloc(8192)
last_pos = 0
continue
except EOFError:
read = 0
done = True
view = cur_buffer.view(last_pos, last_pos + read)
last_pos += read
collator.append(view)
if b"\n" in view:
data = collator.collapse()
last_newline_pos = data.rfind(b"\n")
for line in data[:last_newline_pos].split(b"\n"):
d[chr(line[0])] += 1
collator.append(data[last_newline_pos + 1:])
def test_splitlines(self):
b = Buffer.allocate(32)
b.add_bytes(b"abc\ndef\n\rghi")
assert list(b.view().splitlines()) == [b"abc", b"def", b"", b"ghi"]
b = Buffer.allocate(32)
b.add_bytes(b"abc\ndef\r\nghi")
assert list(b.view().splitlines()) == [b"abc", b"def", b"ghi"]
b = Buffer.allocate(32)
b.add_bytes(b"\nabc\ndef\r\nghi\n\r")
assert list(b.view().splitlines(True)) == [
b"\n", b"abc\n", b"def\r\n", b"ghi\n", b"\r"
]
def collapse(self):
if len(self._views) == 1:
result = self._views[0]
else:
data = _ffi.new("uint8_t[]", self._total_length)
pos = 0
for view in self._views:
_lib.memcpy(data + pos, view._data, len(view))
pos += len(view)
result = Buffer(data, self._total_length).view()
del self._views[:]
self._total_length = 0
return result
""" zero_buffer is a high-performance, zero-copy, implementation of a byte-buffer for Python.
https://warehouse.python.org/project/zero_buffer/
Documentation is available on ReadTheDocs.
"""
from zero_buffer import Buffer
# Create a buffer which has space for 8192 bytes.
b = Buffer.allocate(8192)
with open(path, "rb") as f:
# Read up to 8192 bytes from the file into the buffer
b.read_from(f.fileno())
# Create a read-only view of the buffer, this performs no copying.
view = b.view()
# Split the view on colons, this returns a generator which yields sub-views
# of the view.
for part in view.split(b":"):
print(part)
def buf():
return Buffer.allocate(16)