def test_new_struct_overflow():
def unpack(p):
assert ptr.kind(p) == ptr.STRUCT
return ptr.offset(p), ptr.struct_data_size(p), ptr.struct_ptrs_size(p)
#
p = ptr.new_struct(-1, 0, 0)
assert unpack(p) == (-1, 0, 0)
#
p = ptr.new_struct(0, -1, 0)
assert unpack(p) == (0, 2**16 - 1, 0)
#
p = ptr.new_struct(0, 0, -1)
assert unpack(p) == (0, 0, 2**16 - 1)
def copy_struct_segment(self, src_seg, offset, data_size, ptrs_size,
bufsize):
dst = SegmentBuilder(bufsize)
dst_pos = dst.allocate(8) # allocate the space to store the pointer p
p = ptr.new_struct(0, data_size, ptrs_size)
dst.copy_from_pointer(dst_pos, src_seg, p, offset - 8)
return dst.as_string()
def copy_from_list(builder, pos, item_type, lst):
if lst is None:
builder.write_int64(pos, 0)
return
#
item_length = item_type.item_length
size_tag = item_type.size_tag
item_count = len(lst)
body_length = item_length * item_count
if size_tag == ptr.LIST_SIZE_COMPOSITE:
# alloc the list and write the tag
struct_item_type = item_type
data_size = struct_item_type.static_data_size
ptrs_size = struct_item_type.static_ptrs_size
total_words = (data_size + ptrs_size) * item_count
pos = builder.alloc_list(pos, size_tag, total_words,
body_length + 8) # +8 is for the tag
tag = ptr.new_struct(item_count, data_size, ptrs_size)
builder.write_int64(pos, tag)
pos += 8
else:
# alloc the list, no tag
pos = builder.alloc_list(pos, size_tag, item_count, body_length)
#
for item in lst:
item_type.write_item(builder, pos, item)
pos += item_length
def _as_pointer(self, offset):
"""
Return a pointer p which points to this structure, assuming that p will be
read at ``offset``
"""
p_offset = (self._data_offset - offset - 8) / 8
return ptr.new_struct(p_offset, self._data_size, self._ptrs_size)
def dumps(obj):
"""
Dump a struct into a message, returned as a string of bytes.
The message is encoded using the recommended capnp format for serializing
messages over a stream. It always uses a single segment.
"""
if obj._is_compact():
# fast path. On CPython, the real speedup comes from the fact that we
# do not create a temporary SegmentBuilder. The memcpy vs copy_pointer
# difference seems negligible, for small objects at least.
start = obj._data_offset
end = obj._get_end()
end = (end + (8 - 1)) & -8 # Round up to 8-byte boundary
p = ptr.new_struct(0, obj._data_size, obj._ptrs_size)
return obj._seg.dump_message(p, start, end)
else:
builder = SegmentBuilder()
builder.allocate(
16) # reserve space for segment header+the root pointer
builder.copy_from_struct(8, Struct, obj)
segment_count = 1
segment_size = (builder.get_length() -
8) / 8 # subtract the segment header
# and convert to words
builder.write_uint32(0, segment_count - 1)
builder.write_uint32(4, segment_size)
return builder.as_string()
def test_new_struct():
p = ptr.new_struct(100, 2, 4)
assert ptr.kind(p) == ptr.STRUCT
assert ptr.offset(p) == 100
assert ptr.struct_data_size(p) == 2
assert ptr.struct_ptrs_size(p) == 4
assert p == 0x0004000200000190
def copy_struct(self, src, offset, data_size, ptrs_size, bufsize=None):
if bufsize is None:
bufsize = len(src) + 8
dst = SegmentBuilder(bufsize)
dst_pos = dst.allocate(8) # allocate the space to store the pointer p
p = ptr.new_struct(0, data_size, ptrs_size)
copy_pointer(src, p, offset - 8, dst, dst_pos)
return dst.as_string()
def is_compact(self, buf, offset, kind, **kwds):
buf = Segment(buf)
if kind == ptr.STRUCT:
p = ptr.new_struct(0, **kwds)
elif kind == ptr.LIST:
p = ptr.new_list(0, **kwds)
else:
assert False
return is_compact(buf, p, offset - 8)
def alloc_struct(self, pos, data_size, ptrs_size):
"""
Allocate a new struct of the given size, and write the resulting pointer
at position i. Return the newly allocated position.
"""
length = (data_size + ptrs_size) * 8
result = self.allocate(length)
offet = result - (pos + 8)
p = ptr.new_struct(offet // 8, data_size, ptrs_size)
self.write_int64(pos, p)
return result
def _copy_struct(src, p, src_pos, dst, dst_pos):
src_pos = ptr.deref(p, src_pos)
data_size = ptr.struct_data_size(p)
ptrs_size = ptr.struct_ptrs_size(p)
if data_size + ptrs_size == 0:
# "empty" struct, no need to allocate
dst.write_int64(dst_pos, ptr.new_struct(-1, 0, 0))
return
ds = data_size * 8
dst_pos = dst.alloc_struct(dst_pos, data_size, ptrs_size)
check_bounds(src, ds, src_pos)
dst.write_slice(dst_pos, src, src_pos, ds) # copy data section
_copy_many_ptrs(ptrs_size, src, src_pos + ds, dst, dst_pos + ds)
def _new_ptrlist(self, size_tag, ptr_offset, item_type, item_count):
if size_tag != ptr.LIST_SIZE_COMPOSITE:
# a plain ptr
return ptr.new_list(ptr_offset, size_tag, item_count)
#
# if size is composite, ptr contains the total size in words, and
# we also need to emit a "list tag"
data_size = item_type.structcls.__static_data_size__ # in words
ptrs_size = item_type.structcls.__static_ptrs_size__ # in words
total_words = (data_size + ptrs_size) * item_count
#
# emit the tag
tag = ptr.new_struct(item_count, data_size, ptrs_size)
self._alloc(struct.pack('<q', tag))
return ptr.new_list(ptr_offset, ptr.LIST_SIZE_COMPOSITE, total_words)
def alloc_struct(self, offset, struct_type, value):
if value is None:
return 0 # NULL
if not isinstance(value, struct_type):
raise TypeError("Expected %s instance, got %s" %
(struct_type.__class__.__name__, value))
#
data_size = struct_type.__static_data_size__ # in words
ptrs_size = struct_type.__static_ptrs_size__ # in words
ptr_offset = self._calc_relative_offset(offset) # in words
#
# we need to take the compact repr of the struct, else we might get
# garbage and wrong offsets. See
# test_alloc_list_of_structs_with_pointers
p = ptr.new_struct(ptr_offset, data_size, ptrs_size)
self._alloc(value.compact()._buf.s)
self._record_allocation(offset, p)
return p
def dumps(obj, fastpath=True):
"""
Dump a struct into a message, returned as a string of bytes.
The message is encoded using the recommended capnp format for serializing
messages over a stream. It always uses a single segment.
By default, it tries to follow a fast path: it checks if the object is
"compact" (as defined by capnpy/visit.py) and, is so, uses a fast memcpy
to dump the whole message.
If the fast path can be taken, it is approximately 5x faster on CPython
and 10x faster on PyPy. However, if the object is **not** compact, the
fast path check makes it ~2x slower. If you are sure that the object is
not compact, you can disable the check by passing ``fastpath=False``.
"""
if fastpath:
# try the fast path: if the object is compact, we can dump the
# object with a fast memcpy
end = obj._get_end()
else:
end = -1
#
if end != -1:
# fast path. On CPython, the real speedup comes from the fact that we
# do not create a temporary SegmentBuilder. The memcpy vs copy_pointer
# difference seems negligible, for small objects at least.
start = obj._data_offset
p = ptr.new_struct(0, obj._data_size, obj._ptrs_size)
return obj._seg.dump_message(p, start, end)
else:
builder = SegmentBuilder()
builder.allocate(
16) # reserve space for segment header+the root pointer
builder.copy_from_struct(8, Struct, obj)
segment_count = 1
segment_size = (builder.get_length() -
8) / 8 # subtract the segment header
# and convert to words
builder.write_uint32(0, segment_count - 1)
builder.write_uint32(4, segment_size)
return builder.as_string()
def dumps(obj):
"""
Dump a struct into a message, returned as a string of bytes.
The message is encoded using the recommended capnp format for serializing
messages over a stream. It always uses a single segment.
"""
if not obj._is_compact():
obj = obj.compact()
a = obj._get_body_start()
b = obj._get_end()
buf = obj._buf.s[a:b]
p = ptr.new_struct(0, obj._data_size, obj._ptrs_size)
#
segment_count = 1
if len(buf) % 8 != 0:
padding = 8 - (len(buf) % 8)
buf += '\x00' * padding
segment_size = len(buf) / 8 + 1 # +1 is for the ptr
header = pack_message_header(segment_count, segment_size, p)
return header + buf
def end_of(self, buf, offset, data_size, ptrs_size):
buf = CapnpBuffer(buf)
p = ptr.new_struct(0, data_size, ptrs_size)
return end_of(buf, p, offset - 8)
def test_hash_str_exception():
buf = ''
p = ptr.new_struct(0, 1, 1) # this is the wrong type of pointer
b = Segment(buf)
py.test.raises(AssertionError, "b.hash_str(p, 0, 0, 0)")
def end_of(self, buf, offset, data_size, ptrs_size):
buf = Segment(buf)
p = ptr.new_struct(0, data_size, ptrs_size)
return end_of(buf, p, offset - 8)
def _is_compact(self):
p = ptr.new_struct(0, self._data_size, self._ptrs_size)
return is_compact(self._seg, p, self._data_offset - 8)
def _get_end(self):
p = ptr.new_struct(0, self._data_size, self._ptrs_size)
return end_of(self._seg, p, self._data_offset - 8)
def endof(self, seg, offset, data_size, ptrs_size):
if isinstance(seg, bytes):
seg = Segment(seg)
p = ptr.new_struct(0, data_size, ptrs_size)
return endof(seg, p, offset - 8)
def test_hash_str_exception():
buf = b''
p = ptr.new_struct(0, 1, 1) # this is the wrong type of pointer
bb = Segment(buf)
with pytest.raises(AssertionError):
bb.hash_str(p, 0, 0, 0)
