def test_new_struct_overflow(): def unpack(p): assert ptr.kind(p) == ptr.STRUCT return ptr.offset(p), ptr.struct_data_size(p), ptr.struct_ptrs_size(p) # p = ptr.new_struct(-1, 0, 0) assert unpack(p) == (-1, 0, 0) # p = ptr.new_struct(0, -1, 0) assert unpack(p) == (0, 2**16 - 1, 0) # p = ptr.new_struct(0, 0, -1) assert unpack(p) == (0, 0, 2**16 - 1)
def copy_struct_segment(self, src_seg, offset, data_size, ptrs_size, bufsize): dst = SegmentBuilder(bufsize) dst_pos = dst.allocate(8) # allocate the space to store the pointer p p = ptr.new_struct(0, data_size, ptrs_size) dst.copy_from_pointer(dst_pos, src_seg, p, offset - 8) return dst.as_string()
def copy_from_list(builder, pos, item_type, lst): if lst is None: builder.write_int64(pos, 0) return # item_length = item_type.item_length size_tag = item_type.size_tag item_count = len(lst) body_length = item_length * item_count if size_tag == ptr.LIST_SIZE_COMPOSITE: # alloc the list and write the tag struct_item_type = item_type data_size = struct_item_type.static_data_size ptrs_size = struct_item_type.static_ptrs_size total_words = (data_size + ptrs_size) * item_count pos = builder.alloc_list(pos, size_tag, total_words, body_length + 8) # +8 is for the tag tag = ptr.new_struct(item_count, data_size, ptrs_size) builder.write_int64(pos, tag) pos += 8 else: # alloc the list, no tag pos = builder.alloc_list(pos, size_tag, item_count, body_length) # for item in lst: item_type.write_item(builder, pos, item) pos += item_length
def _as_pointer(self, offset): """ Return a pointer p which points to this structure, assuming that p will be read at ``offset`` """ p_offset = (self._data_offset - offset - 8) / 8 return ptr.new_struct(p_offset, self._data_size, self._ptrs_size)
def dumps(obj): """ Dump a struct into a message, returned as a string of bytes. The message is encoded using the recommended capnp format for serializing messages over a stream. It always uses a single segment. """ if obj._is_compact(): # fast path. On CPython, the real speedup comes from the fact that we # do not create a temporary SegmentBuilder. The memcpy vs copy_pointer # difference seems negligible, for small objects at least. start = obj._data_offset end = obj._get_end() end = (end + (8 - 1)) & -8 # Round up to 8-byte boundary p = ptr.new_struct(0, obj._data_size, obj._ptrs_size) return obj._seg.dump_message(p, start, end) else: builder = SegmentBuilder() builder.allocate( 16) # reserve space for segment header+the root pointer builder.copy_from_struct(8, Struct, obj) segment_count = 1 segment_size = (builder.get_length() - 8) / 8 # subtract the segment header # and convert to words builder.write_uint32(0, segment_count - 1) builder.write_uint32(4, segment_size) return builder.as_string()
def test_new_struct(): p = ptr.new_struct(100, 2, 4) assert ptr.kind(p) == ptr.STRUCT assert ptr.offset(p) == 100 assert ptr.struct_data_size(p) == 2 assert ptr.struct_ptrs_size(p) == 4 assert p == 0x0004000200000190
def copy_struct(self, src, offset, data_size, ptrs_size, bufsize=None): if bufsize is None: bufsize = len(src) + 8 dst = SegmentBuilder(bufsize) dst_pos = dst.allocate(8) # allocate the space to store the pointer p p = ptr.new_struct(0, data_size, ptrs_size) copy_pointer(src, p, offset - 8, dst, dst_pos) return dst.as_string()
def is_compact(self, buf, offset, kind, **kwds): buf = Segment(buf) if kind == ptr.STRUCT: p = ptr.new_struct(0, **kwds) elif kind == ptr.LIST: p = ptr.new_list(0, **kwds) else: assert False return is_compact(buf, p, offset - 8)
def alloc_struct(self, pos, data_size, ptrs_size): """ Allocate a new struct of the given size, and write the resulting pointer at position i. Return the newly allocated position. """ length = (data_size + ptrs_size) * 8 result = self.allocate(length) offet = result - (pos + 8) p = ptr.new_struct(offet // 8, data_size, ptrs_size) self.write_int64(pos, p) return result
def _copy_struct(src, p, src_pos, dst, dst_pos): src_pos = ptr.deref(p, src_pos) data_size = ptr.struct_data_size(p) ptrs_size = ptr.struct_ptrs_size(p) if data_size + ptrs_size == 0: # "empty" struct, no need to allocate dst.write_int64(dst_pos, ptr.new_struct(-1, 0, 0)) return ds = data_size * 8 dst_pos = dst.alloc_struct(dst_pos, data_size, ptrs_size) check_bounds(src, ds, src_pos) dst.write_slice(dst_pos, src, src_pos, ds) # copy data section _copy_many_ptrs(ptrs_size, src, src_pos + ds, dst, dst_pos + ds)
def _new_ptrlist(self, size_tag, ptr_offset, item_type, item_count): if size_tag != ptr.LIST_SIZE_COMPOSITE: # a plain ptr return ptr.new_list(ptr_offset, size_tag, item_count) # # if size is composite, ptr contains the total size in words, and # we also need to emit a "list tag" data_size = item_type.structcls.__static_data_size__ # in words ptrs_size = item_type.structcls.__static_ptrs_size__ # in words total_words = (data_size + ptrs_size) * item_count # # emit the tag tag = ptr.new_struct(item_count, data_size, ptrs_size) self._alloc(struct.pack('<q', tag)) return ptr.new_list(ptr_offset, ptr.LIST_SIZE_COMPOSITE, total_words)
def alloc_struct(self, offset, struct_type, value): if value is None: return 0 # NULL if not isinstance(value, struct_type): raise TypeError("Expected %s instance, got %s" % (struct_type.__class__.__name__, value)) # data_size = struct_type.__static_data_size__ # in words ptrs_size = struct_type.__static_ptrs_size__ # in words ptr_offset = self._calc_relative_offset(offset) # in words # # we need to take the compact repr of the struct, else we might get # garbage and wrong offsets. See # test_alloc_list_of_structs_with_pointers p = ptr.new_struct(ptr_offset, data_size, ptrs_size) self._alloc(value.compact()._buf.s) self._record_allocation(offset, p) return p
def dumps(obj, fastpath=True): """ Dump a struct into a message, returned as a string of bytes. The message is encoded using the recommended capnp format for serializing messages over a stream. It always uses a single segment. By default, it tries to follow a fast path: it checks if the object is "compact" (as defined by capnpy/visit.py) and, is so, uses a fast memcpy to dump the whole message. If the fast path can be taken, it is approximately 5x faster on CPython and 10x faster on PyPy. However, if the object is **not** compact, the fast path check makes it ~2x slower. If you are sure that the object is not compact, you can disable the check by passing ``fastpath=False``. """ if fastpath: # try the fast path: if the object is compact, we can dump the # object with a fast memcpy end = obj._get_end() else: end = -1 # if end != -1: # fast path. On CPython, the real speedup comes from the fact that we # do not create a temporary SegmentBuilder. The memcpy vs copy_pointer # difference seems negligible, for small objects at least. start = obj._data_offset p = ptr.new_struct(0, obj._data_size, obj._ptrs_size) return obj._seg.dump_message(p, start, end) else: builder = SegmentBuilder() builder.allocate( 16) # reserve space for segment header+the root pointer builder.copy_from_struct(8, Struct, obj) segment_count = 1 segment_size = (builder.get_length() - 8) / 8 # subtract the segment header # and convert to words builder.write_uint32(0, segment_count - 1) builder.write_uint32(4, segment_size) return builder.as_string()
def dumps(obj): """ Dump a struct into a message, returned as a string of bytes. The message is encoded using the recommended capnp format for serializing messages over a stream. It always uses a single segment. """ if not obj._is_compact(): obj = obj.compact() a = obj._get_body_start() b = obj._get_end() buf = obj._buf.s[a:b] p = ptr.new_struct(0, obj._data_size, obj._ptrs_size) # segment_count = 1 if len(buf) % 8 != 0: padding = 8 - (len(buf) % 8) buf += '\x00' * padding segment_size = len(buf) / 8 + 1 # +1 is for the ptr header = pack_message_header(segment_count, segment_size, p) return header + buf
def end_of(self, buf, offset, data_size, ptrs_size): buf = CapnpBuffer(buf) p = ptr.new_struct(0, data_size, ptrs_size) return end_of(buf, p, offset - 8)
def test_hash_str_exception(): buf = '' p = ptr.new_struct(0, 1, 1) # this is the wrong type of pointer b = Segment(buf) py.test.raises(AssertionError, "b.hash_str(p, 0, 0, 0)")
def end_of(self, buf, offset, data_size, ptrs_size): buf = Segment(buf) p = ptr.new_struct(0, data_size, ptrs_size) return end_of(buf, p, offset - 8)
def _is_compact(self): p = ptr.new_struct(0, self._data_size, self._ptrs_size) return is_compact(self._seg, p, self._data_offset - 8)
def _get_end(self): p = ptr.new_struct(0, self._data_size, self._ptrs_size) return end_of(self._seg, p, self._data_offset - 8)
def endof(self, seg, offset, data_size, ptrs_size): if isinstance(seg, bytes): seg = Segment(seg) p = ptr.new_struct(0, data_size, ptrs_size) return endof(seg, p, offset - 8)
def test_hash_str_exception(): buf = b'' p = ptr.new_struct(0, 1, 1) # this is the wrong type of pointer bb = Segment(buf) with pytest.raises(AssertionError): bb.hash_str(p, 0, 0, 0)