def copy_struct(self, src, offset, data_size, ptrs_size, bufsize=None): if bufsize is None: bufsize = len(src) + 8 dst = SegmentBuilder(bufsize) dst_pos = dst.allocate(8) # allocate the space to store the pointer p p = ptr.new_struct(0, data_size, ptrs_size) copy_pointer(src, p, offset - 8, dst, dst_pos) return dst.as_string()
def test_null_pointers(self): NULL = b'\x00\x00\x00\x00\x00\x00\x00\x00' # NULL pointer buf = SegmentBuilder() pos = buf.allocate(24) buf.copy_from_struct(0, Struct, None) buf.alloc_text(8, None) buf.copy_from_list(16, PrimitiveItemType(Types.int64), None) s = buf.as_string() assert s == NULL*3
def test_resize_big_allocation(self): buf = SegmentBuilder(32) assert buf.length == 32 assert buf.end == 0 # make a big allocation, which is bigger than the normal exponential # growth of the buffer buf.allocate(4096) assert buf.length == 4096 assert buf.end == 4096
def test_resize(self): buf = SegmentBuilder() buf.allocate(64) print() buf.write_int64(0, 42) for i in range(63): buf.allocate(64) s = buf.as_string() assert len(s) == 64*64 assert s[:8] == struct.pack('q', 42) assert s[8:] == b'\x00' * (64*64-8)
def test_alloc_struct(self): buf = SegmentBuilder(64) buf.allocate(16) a = buf.alloc_struct(0, data_size=3, ptrs_size=0) b = buf.alloc_struct(8, data_size=1, ptrs_size=0) buf.write_int64(a, 1) buf.write_int64(a+8, 2) buf.write_int64(a+16, 3) buf.write_int64(b, 4) s = buf.as_string() assert s == ('\x04\x00\x00\x00\x03\x00\x00\x00' # ptr to a (3, 0) '\x0c\x00\x00\x00\x01\x00\x00\x00' # ptr to b (1, 0) '\x01\x00\x00\x00\x00\x00\x00\x00' # a: 1 '\x02\x00\x00\x00\x00\x00\x00\x00' # 2 '\x03\x00\x00\x00\x00\x00\x00\x00' # 3 '\x04\x00\x00\x00\x00\x00\x00\x00') # b: 4
def test_write_slice(self): src = Segment(b'1234foobar1234') buf = SegmentBuilder() buf.allocate(8) pos = buf.allocate(8) buf.write_slice(pos, src, start=4, n=6) s = buf.as_string() assert s == b('\x00\x00\x00\x00\x00\x00\x00\x00' 'foobar\x00\x00')
def test_copy_from_list_int8(self): buf = SegmentBuilder() buf.allocate(8) # allocate some garbage at the beginning pos = buf.allocate(8) item_type = PrimitiveItemType(Types.int8) buf.copy_from_list(pos, item_type, [1, 2, 3, 4]) s = buf.as_string() assert s == b( '\x00\x00\x00\x00\x00\x00\x00\x00' # garbage '\x01\x00\x00\x00\x22\x00\x00\x00' # ptrlist '\x01\x02\x03\x04\x00\x00\x00\x00') # 1,2,3,4 + padding
def copy_struct_segment(self, src_seg, offset, data_size, ptrs_size, bufsize): dst = SegmentBuilder(bufsize) dst_pos = dst.allocate(8) # allocate the space to store the pointer p p = ptr.new_struct(0, data_size, ptrs_size) dst.copy_from_pointer(dst_pos, src_seg, p, offset - 8) return dst.as_string()
def test_copy_from_list_of_structs(self): class Point(Struct): __static_data_size__ = 2 __static_ptrs_size__ = 0 buf1 = b('\x0a\x00\x00\x00\x00\x00\x00\x00' # 10 '\x64\x00\x00\x00\x00\x00\x00\x00') # 100 buf2 = b('\x14\x00\x00\x00\x00\x00\x00\x00' # 20 '\xc8\x00\x00\x00\x00\x00\x00\x00') # 200 buf3 = b('\x1e\x00\x00\x00\x00\x00\x00\x00' # 30 '\x2c\x01\x00\x00\x00\x00\x00\x00') # 300 buf4 = b('\x28\x00\x00\x00\x00\x00\x00\x00' # 40 '\x90\x01\x00\x00\x00\x00\x00\x00') # 400 p1 = Point.from_buffer(buf1, 0, 2, 0) p2 = Point.from_buffer(buf2, 0, 2, 0) p3 = Point.from_buffer(buf3, 0, 2, 0) p4 = Point.from_buffer(buf4, 0, 2, 0) # buf = SegmentBuilder() pos = buf.allocate(8) item_type = StructItemType(Point) buf.copy_from_list(pos, item_type, [p1, p2, p3, p4]) s = buf.as_string() expected_buf = b('\x01\x00\x00\x00\x47\x00\x00\x00' # ptrlist '\x10\x00\x00\x00\x02\x00\x00\x00' # list tag '\x0a\x00\x00\x00\x00\x00\x00\x00' # 10 '\x64\x00\x00\x00\x00\x00\x00\x00' # 100 '\x14\x00\x00\x00\x00\x00\x00\x00' # 20 '\xc8\x00\x00\x00\x00\x00\x00\x00' # 200 '\x1e\x00\x00\x00\x00\x00\x00\x00' # 30 '\x2c\x01\x00\x00\x00\x00\x00\x00' # 300 '\x28\x00\x00\x00\x00\x00\x00\x00' # 40 '\x90\x01\x00\x00\x00\x00\x00\x00') # 400 assert s == expected_buf
def test_copy_from_list_of_structs_with_pointers(self): class Person(Struct): __static_data_size__ = 1 __static_ptrs_size__ = 1 john = b('\x20\x00\x00\x00\x00\x00\x00\x00' # age=32 '\x01\x00\x00\x00\x2a\x00\x00\x00' # name=ptr 'J' 'o' 'h' 'n' '\x00\x00\x00\x00') # John # emily is a "split struct", with garbage between the body and the extra emily = b('garbage0' '\x18\x00\x00\x00\x00\x00\x00\x00' # age=24 '\x09\x00\x00\x00\x32\x00\x00\x00' # name=ptr 'garbage1' 'garbage2' '\x45\x6d\x69\x6c\x79\x00\x00\x00' # Emily 'garbage3') john = Person.from_buffer(john, 0, 1, 1) emily = Person.from_buffer(emily, 8, 1, 1) # buf = SegmentBuilder() pos = buf.allocate(8) item_type = StructItemType(Person) buf.copy_from_list(pos, item_type, [john, emily]) s = buf.as_string() expected_buf = b('\x01\x00\x00\x00\x27\x00\x00\x00' # ptrlist '\x08\x00\x00\x00\x01\x00\x01\x00' # list tag '\x20\x00\x00\x00\x00\x00\x00\x00' # age=32 '\x09\x00\x00\x00\x2a\x00\x00\x00' # name=ptr '\x18\x00\x00\x00\x00\x00\x00\x00' # age=24 '\x05\x00\x00\x00\x32\x00\x00\x00' # name=ptr 'J' 'o' 'h' 'n' '\x00\x00\x00\x00' # John 'E' 'm' 'i' 'l' 'y' '\x00\x00\x00') # Emily assert s == expected_buf
def test_copy_from_list_float64(self): buf = SegmentBuilder() buf.allocate(8) # allocate some garbage at the beginning pos = buf.allocate(8) item_type = PrimitiveItemType(Types.float64) buf.copy_from_list(pos, item_type, [1.234, 2.345, 3.456, 4.567]) s = buf.as_string() assert s == b( '\x00\x00\x00\x00\x00\x00\x00\x00' # garbage '\x01\x00\x00\x00\x25\x00\x00\x00' # ptrlist '\x58\x39\xb4\xc8\x76\xbe\xf3\x3f' # 1.234 '\xc3\xf5\x28\x5c\x8f\xc2\x02\x40' # 2.345 '\xd9\xce\xf7\x53\xe3\xa5\x0b\x40' # 3.456 '\xf8\x53\xe3\xa5\x9b\x44\x12\x40') # 4.567
def compact(self): """ Return a compact version of the struct. In case the struct was inside a bigger message, it "extracts" it into a new message which contains only the needed pieces. """ builder = SegmentBuilder() pos = builder.allocate(8) builder.copy_from_struct(pos, Struct, self) buf = builder.as_string() t = type(self) res = t.__new__(t) res._init_from_buffer(buf, 8, self._data_size, self._ptrs_size) return res
def test_copy_from_list_of_text(self): buf = SegmentBuilder() pos = buf.allocate(8) item_type = TextItemType(Types.text) buf.copy_from_list(pos, item_type, [b'A', b'BC', b'DEF', b'GHIJ']) s = buf.as_string() expected_buf = b('\x01\x00\x00\x00\x26\x00\x00\x00' # ptrlist '\x0d\x00\x00\x00\x12\x00\x00\x00' # ptr item 1 '\x0d\x00\x00\x00\x1a\x00\x00\x00' # ptr item 2 '\x0d\x00\x00\x00\x22\x00\x00\x00' # ptr item 3 '\x0d\x00\x00\x00\x2a\x00\x00\x00' # ptr item 4 'A' '\x00\x00\x00\x00\x00\x00\x00' # A 'B' 'C' '\x00\x00\x00\x00\x00\x00' # BC 'D' 'E' 'F' '\x00\x00\x00\x00\x00' # DEF 'G' 'H' 'I' 'J' '\x00\x00\x00\x00') # GHIJ assert s == expected_buf
def test_write(self): expected = struct.pack('<bBhHiIqQfd', 10, 20, 30, 40, 50, 60, 70, 80, 90, 100) n = len(expected) buf = SegmentBuilder(n) buf.allocate(n) buf.write_int8(0, 10) buf.write_uint8(1, 20) buf.write_int16(2, 30) buf.write_uint16(4, 40) buf.write_int32(6, 50) buf.write_uint32(10, 60) buf.write_int64(14, 70) buf.write_uint64(22, 80) buf.write_float32(30, 90) buf.write_float64(34, 100) s = buf.as_string() assert s == expected
def test_write_int64(self): buf = SegmentBuilder(8) buf.allocate(8) buf.write_int64(0, 0x1234ABCD) s = buf.as_string() assert s == '\xCD\xAB\x34\x12\x00\x00\x00\x00'
def dumps(obj, fastpath=True): """ Dump a struct into a message, returned as a string of bytes. The message is encoded using the recommended capnp format for serializing messages over a stream. It always uses a single segment. By default, it tries to follow a fast path: it checks if the object is "compact" (as defined by capnpy/visit.py) and, is so, uses a fast memcpy to dump the whole message. If the fast path can be taken, it is approximately 5x faster on CPython and 10x faster on PyPy. However, if the object is **not** compact, the fast path check makes it ~2x slower. If you are sure that the object is not compact, you can disable the check by passing ``fastpath=False``. """ if fastpath: # try the fast path: if the object is compact, we can dump the # object with a fast memcpy end = obj._get_end() else: end = -1 # if end != -1: # fast path. On CPython, the real speedup comes from the fact that we # do not create a temporary SegmentBuilder. The memcpy vs copy_pointer # difference seems negligible, for small objects at least. start = obj._data_offset p = ptr.new_struct(0, obj._data_size, obj._ptrs_size) return obj._seg.dump_message(p, start, end) else: builder = SegmentBuilder() builder.allocate( 16) # reserve space for segment header+the root pointer builder.copy_from_struct(8, Struct, obj) segment_count = 1 segment_size = (builder.get_length() - 8) / 8 # subtract the segment header # and convert to words builder.write_uint32(0, segment_count - 1) builder.write_uint32(4, segment_size) return builder.as_string()
def dumps(obj): """ Dump a struct into a message, returned as a string of bytes. The message is encoded using the recommended capnp format for serializing messages over a stream. It always uses a single segment. """ if obj._is_compact(): # fast path. On CPython, the real speedup comes from the fact that we # do not create a temporary SegmentBuilder. The memcpy vs copy_pointer # difference seems negligible, for small objects at least. start = obj._data_offset end = obj._get_end() end = (end + (8 - 1)) & -8 # Round up to 8-byte boundary p = ptr.new_struct(0, obj._data_size, obj._ptrs_size) return obj._seg.dump_message(p, start, end) else: builder = SegmentBuilder() builder.allocate( 16) # reserve space for segment header+the root pointer builder.copy_from_struct(8, Struct, obj) segment_count = 1 segment_size = (builder.get_length() - 8) / 8 # subtract the segment header # and convert to words builder.write_uint32(0, segment_count - 1) builder.write_uint32(4, segment_size) return builder.as_string()
def test_write_generic(self): expected = struct.pack('<bBhHiIqQfd', 10, 20, 30, 40, 50, 60, 70, 80, 90, 100) n = len(expected) buf = SegmentBuilder(n) buf.allocate(n) buf.write_generic(ord('b'), 0, 10) buf.write_generic(ord('B'), 1, 20) buf.write_generic(ord('h'), 2, 30) buf.write_generic(ord('H'), 4, 40) buf.write_generic(ord('i'), 6, 50) buf.write_generic(ord('I'), 10, 60) buf.write_generic(ord('q'), 14, 70) buf.write_generic(ord('Q'), 22, 80) buf.write_generic(ord('f'), 30, 90) buf.write_generic(ord('d'), 34, 100) s = buf.as_string() assert s == expected
def test_alloc_text_and_data(self): buf = SegmentBuilder() buf.allocate(32) buf.alloc_text(0, b'foo') buf.alloc_text(8, None) buf.alloc_text(16, b'bar') buf.alloc_data(24, b'bar') s = buf.as_string() print() print_buffer(s) assert s == b('\x0D\x00\x00\x00\x22\x00\x00\x00' # ptr to 'foo' '\x00\x00\x00\x00\x00\x00\x00\x00' # NULL '\x09\x00\x00\x00\x22\x00\x00\x00' # ptr to text 'bar' (4 items) '\x09\x00\x00\x00\x1A\x00\x00\x00' # ptr to data 'bar' (3 items) 'foo\x00\x00\x00\x00\x00' 'bar\x00\x00\x00\x00\x00' 'bar\x00\x00\x00\x00\x00')
def test_alloc_list(self): buf = SegmentBuilder(64) buf.allocate(16) a = buf.alloc_list(0, size_tag=ptr.LIST_SIZE_8, item_count=4, body_length=4) bb = buf.alloc_list(8, size_tag=ptr.LIST_SIZE_16, item_count=2, body_length=4) buf.write_int8(a, ord('f')) buf.write_int8(a+1, ord('o')) buf.write_int8(a+2, ord('o')) buf.write_int8(a+3, ord('\x00')) # buf.write_int16(bb, 0x1234) buf.write_int16(bb+2, 0x5678) # s = buf.as_string() assert s == b('\x05\x00\x00\x00\x22\x00\x00\x00' # ptrlist int8, item_count=4 '\x05\x00\x00\x00\x13\x00\x00\x00' # ptrlist int16, item_count=2 'foo\x00\x00\x00\x00\x00' '\x34\x12\x78\x56\x00\x00\x00\x00') # 0x1234 0x5678
def test_allocate(self): buf = SegmentBuilder(16) assert buf.allocate(8) == 0 s = buf.as_string() assert s == b'\x00' * 8