def run_zero_buffer_bench():
    d = defaultdict(int)
    cur_buffer = Buffer.alloc(8192)
    last_pos = 0
    collator = BufferCollator()
    with open("/usr/share/dict/words") as f:
        done = False
        while not done:
            try:
                read = cur_buffer.read_from(f.fileno())
            except BufferFull:
                cur_buffer = Buffer.alloc(8192)
                last_pos = 0
                continue
            except EOFError:
                read = 0
                done = True
            view = cur_buffer.view(last_pos, last_pos + read)
            last_pos += read
            collator.append(view)
            if b"\n" in view:
                data = collator.collapse()
                last_newline_pos = data.rfind(b"\n")
                for line in data[:last_newline_pos].split(b"\n"):
                    d[chr(line[0])] += 1
                collator.append(data[last_newline_pos + 1:])
def run_zero_buffer_bench():
    d = defaultdict(int)
    cur_buffer = Buffer.alloc(8192)
    last_pos = 0
    collator = BufferCollator()
    with open("/usr/share/dict/words") as f:
        done = False
        while not done:
            try:
                read = cur_buffer.read_from(f.fileno())
            except BufferFull:
                cur_buffer = Buffer.alloc(8192)
                last_pos = 0
                continue
            except EOFError:
                read = 0
                done = True
            view = cur_buffer.view(last_pos, last_pos + read)
            last_pos += read
            collator.append(view)
            if b"\n" in view:
                data = collator.collapse()
                last_newline_pos = data.rfind(b"\n")
                for line in data[:last_newline_pos].split(b"\n"):
                    d[chr(line[0])] += 1
                collator.append(data[last_newline_pos + 1:])
    def test_splitlines(self):
        b = Buffer.allocate(32)
        b.add_bytes(b"abc\ndef\n\rghi")
        assert list(b.view().splitlines()) == [b"abc", b"def", b"", b"ghi"]

        b = Buffer.allocate(32)
        b.add_bytes(b"abc\ndef\r\nghi")
        assert list(b.view().splitlines()) == [b"abc", b"def", b"ghi"]

        b = Buffer.allocate(32)
        b.add_bytes(b"\nabc\ndef\r\nghi\n\r")
        assert list(b.view().splitlines(True)) == [
            b"\n", b"abc\n", b"def\r\n", b"ghi\n", b"\r"
        ]
Exemple #4
0
 def collapse(self):
     if len(self._views) == 1:
         result = self._views[0]
     else:
         data = _ffi.new("uint8_t[]", self._total_length)
         pos = 0
         for view in self._views:
             _lib.memcpy(data + pos, view._data, len(view))
             pos += len(view)
         result = Buffer(data, self._total_length).view()
     del self._views[:]
     self._total_length = 0
     return result
Exemple #5
0
""" zero_buffer is a high-performance, zero-copy, implementation of a byte-buffer for Python.
 https://warehouse.python.org/project/zero_buffer/
Documentation is available on ReadTheDocs.
"""
from zero_buffer import Buffer

# Create a buffer which has space for 8192 bytes.
b = Buffer.allocate(8192)
with open(path, "rb") as f:
    # Read up to 8192 bytes from the file into the buffer
    b.read_from(f.fileno())
# Create a read-only view of the buffer, this performs no copying.
view = b.view()
# Split the view on colons, this returns a generator which yields sub-views
# of the view.
for part in view.split(b":"):
    print(part)

def buf():
    return Buffer.allocate(16)