def _copy_list_composite(src, p, src_pos, dst, dst_pos):
src_pos = ptr.deref(p, src_pos)
total_words = ptr.list_item_count(p) # n of words NOT including the tag
body_length = (total_words + 1) * 8 # total length INCLUDING the tag
#
# check that there is enough data for both the tag AND the whole body;
# this way we do the bound checking only once
check_bounds(src, body_length, src_pos)
tag = read_int64_fast(src, src_pos)
count = ptr.offset(tag)
data_size = ptr.struct_data_size(tag)
ptrs_size = ptr.struct_ptrs_size(tag)
#
# allocate the list and copy the whole body at once
dst_pos = dst.alloc_list(dst_pos, ptr.LIST_SIZE_COMPOSITE, total_words,
body_length)
dst.write_slice(dst_pos, src, src_pos, body_length)
#
# iterate over the elements, fix the pointers and copy the content
i = 0
item_length = (data_size + ptrs_size) * 8
ptrs_section_offset = 0
for i in range(count):
ptrs_section_offset = 8 + item_length * i + data_size * 8
_copy_many_ptrs(ptrs_size, src, src_pos + ptrs_section_offset, dst,
dst_pos + ptrs_section_offset)
def test_new_list():
p = ptr.new_list(64, 7, 200)
assert ptr.kind(p) == ptr.LIST
assert ptr.offset(p) == 64
assert ptr.list_size_tag(p) == 7
assert ptr.list_item_count(p) == 200
assert p == 0x0000064700000101
def visit(self, buf, p, offset):
kind = ptr.kind(p)
offset = ptr.deref(p, offset)
if kind == ptr.STRUCT:
data_size = ptr.struct_data_size(p)
ptrs_size = ptr.struct_ptrs_size(p)
return self.visit_struct(buf, p, offset, data_size, ptrs_size)
elif kind == ptr.LIST:
item_size = ptr.list_size_tag(p)
count = ptr.list_item_count(p)
if item_size == ptr.LIST_SIZE_COMPOSITE:
tag = buf.read_ptr(offset)
count = ptr.offset(tag)
data_size = ptr.struct_data_size(tag)
ptrs_size = ptr.struct_ptrs_size(tag)
return self.visit_list_composite(buf, p, offset,
count, data_size, ptrs_size)
elif item_size == ptr.LIST_SIZE_PTR:
return self.visit_list_ptr(buf, p, offset, count)
elif item_size == ptr.LIST_SIZE_BIT:
return self.visit_list_bit(buf, p, offset, count)
else:
return self.visit_list_primitive(buf, p, offset, item_size, count)
elif kind == ptr.FAR:
raise NotImplementedError('Far pointer not supported')
else:
assert False, 'unknown ptr kind'
def _copy_list_ptr(src, p, src_pos, dst, dst_pos):
src_pos = ptr.deref(p, src_pos)
count = ptr.list_item_count(p)
body_length = count * 8
dst_pos = dst.alloc_list(dst_pos, ptr.LIST_SIZE_PTR, count, body_length)
check_bounds(src, body_length, src_pos)
_copy_many_ptrs(count, src, src_pos, dst, dst_pos)
def hash_str(self, p, offset, default_, additional_size):
if p == 0:
return default_
assert ptr.kind(p) == ptr.LIST
assert ptr.list_size_tag(p) == ptr.LIST_SIZE_8
start = ptr.deref(p, offset)
size = ptr.list_item_count(p) + additional_size
return _hash.strhash(self.buf, start, size)
def test_ListPtr():
# 0000064 item_count<<1
# 7 item_size
# 00000100 offset<<2
# 1 kind
p = 0x0000064700000101
assert ptr.kind(p) == ptr.LIST
assert ptr.offset(p) == 64
assert ptr.list_size_tag(p) == 7
assert ptr.list_item_count(p) == 200
def read_item(self, lst, i):
offset = lst._offset + (i * 8)
p = lst._seg.read_ptr(offset)
if ptr.kind(p) == ptr.FAR:
offset, p = lst._seg.read_far_ptr(offset)
obj = List.__new__(List)
obj._init_from_buffer(lst._seg, ptr.deref(p, offset),
ptr.list_size_tag(p), ptr.list_item_count(p),
self.inner_item_type)
return obj
def read_item(self, lst, i):
offset = lst._offset + (i * 8)
p = lst._seg.read_ptr(offset)
if ptr.kind(p) == ptr.FAR:
raise NotImplementedError('FAR pointers not supported here')
obj = List.__new__(List)
obj._init_from_buffer(lst._seg, ptr.deref(p, offset),
ptr.list_size_tag(p), ptr.list_item_count(p),
self.inner_item_type)
return obj
def _copy_list_primitive(src, p, src_pos, dst, dst_pos):
src_pos = ptr.deref(p, src_pos)
count = ptr.list_item_count(p)
size_tag = ptr.list_size_tag(p)
body_length = 0
if size_tag == ptr.LIST_SIZE_BIT:
body_length = (count + 8 - 1) / 8 # divide by 8 and round up
else:
body_length = count * ptr.list_item_length(size_tag)
#
dst_pos = dst.alloc_list(dst_pos, size_tag, count, body_length)
check_bounds(src, body_length, src_pos)
dst.write_slice(dst_pos, src, src_pos, body_length)
def read_str(self, p, offset, default_, additional_size):
"""
Read Text or Data from the pointer ``p``, which was read from the given
offset.
If you want to read a Text, pass additional_size=-1 to remove the
trailing '\0'. If you want to read a Data, pass additional_size=0.
"""
if p == 0:
return default_
assert ptr.kind(p) == ptr.LIST
assert ptr.list_size_tag(p) == ptr.LIST_SIZE_8
start = ptr.deref(p, offset)
end = start + ptr.list_item_count(p) + additional_size
return self.buf[start:end]
def _read_list(self, offset, item_type, default_=None):
p = self._read_fast_ptr(offset)
if ptr.kind(p) == ptr.FAR:
offset, p = self._read_far_ptr(offset)
else:
offset += self._ptrs_offset
if p == 0:
return default_
assert ptr.kind(p) == ptr.LIST
list_offset = ptr.deref(p, offset)
# in theory we could simply use List.from_buffer; however, Cython is
# not able to compile classmethods, so we create it manually
obj = List.__new__(List)
obj._init_from_buffer(self._seg, list_offset, ptr.list_size_tag(p),
ptr.list_item_count(p), item_type)
return obj
def _read_list_or_struct(self, ptr_offset, default_=None):
ptr_offset, p = self._read_ptr_generic(ptr_offset)
if p == 0:
return default_
blob_offet = ptr.deref(p, ptr_offset)
if ptr.kind(p) == ptr.STRUCT:
Struct = capnpy.struct_.Struct
return Struct.from_buffer(self._buf, blob_offet,
ptr.struct_data_size(p),
ptr.struct_ptrs_size(p))
elif ptr.kind(p) == ptr.LIST:
List = capnpy.list.List
return List.from_buffer(self._buf,
blob_offet, ptr.list_size_tag(p),
ptr.list_item_count(p),
capnpy.list.StructItemType(Blob))
else:
assert False, 'Unkwown pointer kind: %s' % ptr.kind(p)
def ptr(self, offset, s):
p = struct.unpack('q', s)[0]
if ptr.kind(p) not in (ptr.STRUCT, ptr.LIST, ptr.FAR):
return ' ' * 25
#
# try to display only "reasonable" ptrs; if the fields are too big, it
# probably means that the current word is not a pointer
def if_in_range(x, min, max):
if min <= x < max:
return str(x)
else:
return '?'
#
if p == 0:
return 'NULL'.ljust(25)
if ptr.kind(p) == ptr.STRUCT:
descr = 'struct {:>4} {:>3}'.format(
if_in_range(ptr.struct_data_size(p), 0, 100),
if_in_range(ptr.struct_ptrs_size(p), 0, 100))
elif ptr.kind(p) == ptr.LIST:
tag = '<%s>' % self._list_tag(ptr.list_size_tag(p))
descr = 'list{:<5} {:>5}'.format(
tag, if_in_range(ptr.list_item_count(p), 0, 65536))
elif ptr.kind(p) == ptr.FAR:
descr = 'far {:>7} {:>3}'.format(
ptr.far_landing_pad(p), if_in_range(ptr.far_target(p), 0, 100))
else:
descr = 'unknown ptr '
#
if -1000 < ptr.offset(p) < 1000:
dest = ptr.deref(p, offset)
dest = self.addr(dest)
dest = dest.ljust(16)
else:
dest = '? '
line = '{0} to {1}'.format(descr, dest)
if '?' in line:
return Color.set(Color.lightgray, line)
else:
return line
def endof(seg, p, offset):
"""
Check whether the given object is compact, and in that case compute its
end boundary. If it's not compact, return -1.
An object is compact if:
1. there is no gap between its data section and its ptrs section
2. there is no gap between children
3. its children are compact
4. there are no FAR pointers
"""
kind = ptr.kind(p)
offset = ptr.deref(p, offset)
if kind == ptr.STRUCT:
data_size = ptr.struct_data_size(p)
ptrs_size = ptr.struct_ptrs_size(p)
return _endof_struct(seg, p, offset, data_size, ptrs_size)
elif kind == ptr.LIST:
item_size = ptr.list_size_tag(p)
count = ptr.list_item_count(p)
if item_size == ptr.LIST_SIZE_COMPOSITE:
tag = seg.read_ptr(offset)
count = ptr.offset(tag)
data_size = ptr.struct_data_size(tag)
ptrs_size = ptr.struct_ptrs_size(tag)
return _endof_list_composite(seg, p, offset, count, data_size,
ptrs_size)
elif item_size == ptr.LIST_SIZE_PTR:
return _endof_list_ptr(seg, p, offset, count)
elif item_size == ptr.LIST_SIZE_BIT:
return _endof_list_bit(seg, p, offset, count)
else:
return _endof_list_primitive(seg, p, offset, item_size, count)
elif kind == ptr.FAR:
return -1
else:
assert False, 'unknown ptr kind'
def unpack(p):
assert ptr.kind(p) == ptr.LIST
return ptr.offset(p), ptr.list_size_tag(p), ptr.list_item_count(p)
