def _split(self, extra_offset):
"""
Split the body and the extra part. The extra part must be placed at the
specified offset, in words. The ptrs in the body will be adjusted
accordingly.
"""
body_start = self._get_body_start()
body_end = self._get_body_end()
if self._ptrs_size == 0:
# easy case, just copy the body
return self._buf.s[body_start:body_end], ''
#
# hard case. The layout of self._buf is like this:
# +----------+------+------+----------+-------------+
# | garbage0 | data | ptrs | garbage1 | extra |
# +----------+------+------+----------+-------------+
# | | ^ ^
# +-----------------+ |
# | |
# +-------------------+
#
# We recompute the pointers assumining len(garbage1) == extra_offset
#
# 1) the data section is copied verbatim
# 2) the offset of pointers in ptrs are adjusted
# 3) extra is copied verbatim
#
extra_start = self._get_extra_start()
extra_end = self._get_end()
#
# 1) data section
data_size = self._data_size
data_buf = self._buf.s[body_start:body_start+data_size*8]
#
# 2) ptrs section
# for each ptr:
# ptr.offset += (extra_offset - old_extra_offset)/8
#
# NOTE: ptr.offset is in words, extra_start and body_end in bytes
old_extra_offset = (extra_start - body_end)/8
additional_offset = extra_offset - old_extra_offset
#
# iterate over and fix the pointers
parts = [data_buf]
for j in range(self._ptrs_size):
# read pointer, update its offset, and pack it
p = self._read_raw_ptr(j*8)
if p != 0:
assert ptr.kind(p) != ptr.FAR
p = ptr.new_generic(ptr.kind(p),
ptr.offset(p)+additional_offset,
ptr.extra(p))
s = struct.pack('q', p)
parts.append(s)
#
body_buf = ''.join(parts)
# 3) extra part
extra_buf = self._buf.s[extra_start:extra_end]
#
return body_buf, extra_buf
def test_kind_offset():
p = 0x0004000200000190
assert ptr.kind(p) == ptr.STRUCT
assert ptr.offset(p) == 100
#
p2 = 0x0000064700000101
assert ptr.kind(p2) == ptr.LIST
def _read_struct(self, offset, structcls):
"""
Read and dereference a struct pointer at the given offset. It returns an
instance of ``structcls`` pointing to the dereferenced struct.
"""
p = self._read_fast_ptr(offset)
if ptr.kind(p) == ptr.FAR:
offset, p = self._read_far_ptr(offset)
else:
offset += self._ptrs_offset
if p == 0:
return None
assert ptr.kind(p) == ptr.STRUCT
obj = structcls.__new__(structcls)
obj._init_from_pointer(self._seg, offset, p)
return obj
def test_new_far():
p = ptr.new_far(0, 725, 1)
assert ptr.kind(p) == ptr.FAR
assert ptr.far_landing_pad(p) == 0
assert ptr.far_offset(p) == 725
assert ptr.far_target(p) == 1
assert p == 0x00000001000016aa
def _read_list(self, offset, item_type, default_=None):
p = self._read_fast_ptr(offset)
if ptr.kind(p) == ptr.FAR:
offset, p = self._read_far_ptr(offset)
else:
offset += self._ptrs_offset
if p == 0:
return default_
assert ptr.kind(p) == ptr.LIST
list_offset = ptr.deref(p, offset)
# in theory we could simply use List.from_buffer; however, Cython is
# not able to compile classmethods, so we create it manually
obj = List.__new__(List)
obj._init_from_buffer(self._seg, list_offset, ptr.list_size_tag(p),
ptr.list_item_count(p), item_type)
return obj
def test_new_struct():
p = ptr.new_struct(100, 2, 4)
assert ptr.kind(p) == ptr.STRUCT
assert ptr.offset(p) == 100
assert ptr.struct_data_size(p) == 2
assert ptr.struct_ptrs_size(p) == 4
assert p == 0x0004000200000190
def test_new_list():
p = ptr.new_list(64, 7, 200)
assert ptr.kind(p) == ptr.LIST
assert ptr.offset(p) == 64
assert ptr.list_size_tag(p) == 7
assert ptr.list_item_count(p) == 200
assert p == 0x0000064700000101
def visit(self, buf, p, offset):
kind = ptr.kind(p)
offset = ptr.deref(p, offset)
if kind == ptr.STRUCT:
data_size = ptr.struct_data_size(p)
ptrs_size = ptr.struct_ptrs_size(p)
return self.visit_struct(buf, p, offset, data_size, ptrs_size)
elif kind == ptr.LIST:
item_size = ptr.list_size_tag(p)
count = ptr.list_item_count(p)
if item_size == ptr.LIST_SIZE_COMPOSITE:
tag = buf.read_ptr(offset)
count = ptr.offset(tag)
data_size = ptr.struct_data_size(tag)
ptrs_size = ptr.struct_ptrs_size(tag)
return self.visit_list_composite(buf, p, offset,
count, data_size, ptrs_size)
elif item_size == ptr.LIST_SIZE_PTR:
return self.visit_list_ptr(buf, p, offset, count)
elif item_size == ptr.LIST_SIZE_BIT:
return self.visit_list_bit(buf, p, offset, count)
else:
return self.visit_list_primitive(buf, p, offset, item_size, count)
elif kind == ptr.FAR:
raise NotImplementedError('Far pointer not supported')
else:
assert False, 'unknown ptr kind'
def _hash_str_data(self, offset, default_=hash(None), additional_size=0):
p = self._read_fast_ptr(offset)
if ptr.kind(p) == ptr.FAR:
offset, p = self._read_far_ptr(offset)
else:
offset += self._ptrs_offset
return self._seg.hash_str(p, offset, default_, additional_size)
def hash_str(self, p, offset, default_, additional_size):
if p == 0:
return default_
assert ptr.kind(p) == ptr.LIST
assert ptr.list_size_tag(p) == ptr.LIST_SIZE_8
start = ptr.deref(p, offset)
size = ptr.list_item_count(p) + additional_size
return _hash.strhash(self.buf, start, size)
def _read_list_or_struct(self, ptr_offset, default_=None):
ptr_offset, p = self._read_ptr_generic(ptr_offset)
if p == 0:
return default_
blob_offet = ptr.deref(p, ptr_offset)
if ptr.kind(p) == ptr.STRUCT:
Struct = capnpy.struct_.Struct
return Struct.from_buffer(self._buf, blob_offet,
ptr.struct_data_size(p),
ptr.struct_ptrs_size(p))
elif ptr.kind(p) == ptr.LIST:
List = capnpy.list.List
return List.from_buffer(self._buf,
blob_offet, ptr.list_size_tag(p),
ptr.list_item_count(p),
capnpy.list.StructItemType(Blob))
else:
assert False, 'Unkwown pointer kind: %s' % ptr.kind(p)
def test_struct_ptr():
# 0004 ptrs size
# 0002 data size
# 00000190 offset<<2
# 0 kind
p = 0x0004000200000190
assert ptr.kind(p) == ptr.STRUCT
assert ptr.offset(p) == 100
assert ptr.struct_data_size(p) == 2
assert ptr.struct_ptrs_size(p) == 4
def test_ListPtr():
# 0000064 item_count<<1
# 7 item_size
# 00000100 offset<<2
# 1 kind
p = 0x0000064700000101
assert ptr.kind(p) == ptr.LIST
assert ptr.offset(p) == 64
assert ptr.list_size_tag(p) == 7
assert ptr.list_item_count(p) == 200
def test_FarPtr():
# 00000001 target
# 000016aa offset
# a landing
# a kind
p = 0x00000001000016aa
assert ptr.kind(p) == ptr.FAR
assert ptr.far_landing_pad(p) == 0
assert ptr.far_offset(p) == 725
assert ptr.far_target(p) == 1
def read_item(self, lst, i):
offset = lst._offset + (i * 8)
p = lst._seg.read_ptr(offset)
if ptr.kind(p) == ptr.FAR:
offset, p = lst._seg.read_far_ptr(offset)
obj = List.__new__(List)
obj._init_from_buffer(lst._seg, ptr.deref(p, offset),
ptr.list_size_tag(p), ptr.list_item_count(p),
self.inner_item_type)
return obj
def test__as_pointer():
buf = b('garbage0'
'\x01\x00\x00\x00\x00\x00\x00\x00' # 1
'\x02\x00\x00\x00\x00\x00\x00\x00') # 2
b1 = Struct.from_buffer(buf, 8, data_size=2, ptrs_size=0)
p = b1._as_pointer(24) # arbitrary offset
assert ptr.kind(p) == ptr.STRUCT
assert ptr.deref(p, 24) == 8
assert ptr.struct_data_size(p) == 2
assert ptr.struct_ptrs_size(p) == 0
def ptr(self, offset, s):
p = struct.unpack('q', s)[0]
if ptr.kind(p) not in (ptr.STRUCT, ptr.LIST, ptr.FAR):
return ' ' * 25
#
# try to display only "reasonable" ptrs; if the fields are too big, it
# probably means that the current word is not a pointer
def if_in_range(x, min, max):
if min <= x < max:
return str(x)
else:
return '?'
#
if p == 0:
return 'NULL'.ljust(25)
if ptr.kind(p) == ptr.STRUCT:
descr = 'struct {:>4} {:>3}'.format(
if_in_range(ptr.struct_data_size(p), 0, 100),
if_in_range(ptr.struct_ptrs_size(p), 0, 100))
elif ptr.kind(p) == ptr.LIST:
tag = '<%s>' % self._list_tag(ptr.list_size_tag(p))
descr = 'list{:<5} {:>5}'.format(
tag, if_in_range(ptr.list_item_count(p), 0, 65536))
elif ptr.kind(p) == ptr.FAR:
descr = 'far {:>7} {:>3}'.format(
ptr.far_landing_pad(p), if_in_range(ptr.far_target(p), 0, 100))
else:
descr = 'unknown ptr '
#
if -1000 < ptr.offset(p) < 1000:
dest = ptr.deref(p, offset)
dest = self.addr(dest)
dest = dest.ljust(16)
else:
dest = '? '
line = '{0} to {1}'.format(descr, dest)
if '?' in line:
return Color.set(Color.lightgray, line)
else:
return line
def read_item(self, lst, i):
offset = lst._offset + (i * 8)
p = lst._seg.read_ptr(offset)
if ptr.kind(p) == ptr.FAR:
raise NotImplementedError('FAR pointers not supported here')
obj = List.__new__(List)
obj._init_from_buffer(lst._seg, ptr.deref(p, offset),
ptr.list_size_tag(p), ptr.list_item_count(p),
self.inner_item_type)
return obj
def _get_extra_start(self):
if self._ptrs_size == 0:
return self._get_body_end()
for i in range(self._ptrs_size):
p = self._read_raw_ptr(i*8)
assert ptr.kind(p) != ptr.FAR
if p != 0:
return self._ptrs_offset + ptr.deref(p, i*8)
#
# if we are here, it means that all ptrs are null
return self._get_body_end()
def _get_extra_start(self):
if self._ptrs_size == 0:
return self._get_body_end()
i = 0
while i < self._ptrs_size:
p = self._read_fast_ptr(i * 8)
assert ptr.kind(p) != ptr.FAR
if p != 0:
return self._ptrs_offset + ptr.deref(p, i * 8)
i += 1
#
# if we are here, it means that all ptrs are null
return self._get_body_end()
def read_str(self, p, offset, default_, additional_size):
"""
Read Text or Data from the pointer ``p``, which was read from the given
offset.
If you want to read a Text, pass additional_size=-1 to remove the
trailing '\0'. If you want to read a Data, pass additional_size=0.
"""
if p == 0:
return default_
assert ptr.kind(p) == ptr.LIST
assert ptr.list_size_tag(p) == ptr.LIST_SIZE_8
start = ptr.deref(p, offset)
end = start + ptr.list_item_count(p) + additional_size
return self.buf[start:end]
def read_ptr(self, offset):
"""
Return the pointer at the specifield offet.
WARNING: you MUST check if the return value is E_IS_FAR_POINTER, and
in that case call read_far_ptr. We need this messy interface for
speed; the proper alternative would be to simply return a tuple
(offset, p) and handle the far ptr here: this is fine on PyPy but slow
on CPython, because this way we cannot give a static return type.
We could raise an exception instead of returning an error value:
however, this is ~20% slower.
"""
p = self.read_raw_ptr(offset)
if ptr.kind(p) == ptr.FAR:
return ptr.E_IS_FAR_POINTER
return p
def copy_pointer(src, p, src_pos, dst, dst_pos):
"""
Copy from: BaseSegment src, pointer p living at the src_pos offset
to: SegmentBuilder dst at position dst_pos
"""
kind = ptr.kind(p)
if kind == ptr.STRUCT:
return _copy_struct(src, p, src_pos, dst, dst_pos)
elif kind == ptr.LIST:
item_size = ptr.list_size_tag(p)
if item_size == ptr.LIST_SIZE_COMPOSITE:
return _copy_list_composite(src, p, src_pos, dst, dst_pos)
elif item_size == ptr.LIST_SIZE_PTR:
return _copy_list_ptr(src, p, src_pos, dst, dst_pos)
else:
return _copy_list_primitive(src, p, src_pos, dst, dst_pos)
elif kind == ptr.FAR:
src_pos, p = src.read_far_ptr(src_pos)
return copy_pointer(src, p, src_pos, dst, dst_pos)
def copy_pointer(src, p, src_pos, dst, dst_pos):
"""
Copy from: BaseSegment src, pointer p living at the src_pos offset
to: SegmentBuilder dst at position dst_pos
"""
kind = ptr.kind(p)
if kind == ptr.STRUCT:
return _copy_struct(src, p, src_pos, dst, dst_pos)
elif kind == ptr.LIST:
item_size = ptr.list_size_tag(p)
if item_size == ptr.LIST_SIZE_COMPOSITE:
return _copy_list_composite(src, p, src_pos, dst, dst_pos)
elif item_size == ptr.LIST_SIZE_PTR:
return _copy_list_ptr(src, p, src_pos, dst, dst_pos)
else:
return _copy_list_primitive(src, p, src_pos, dst, dst_pos)
## elif kind == ptr.FAR:
## raise NotImplementedError('Far pointer not supported')
assert False, 'unknown ptr kind: %s' % kind
def endof(seg, p, offset):
"""
Check whether the given object is compact, and in that case compute its
end boundary. If it's not compact, return -1.
An object is compact if:
1. there is no gap between its data section and its ptrs section
2. there is no gap between children
3. its children are compact
4. there are no FAR pointers
"""
kind = ptr.kind(p)
offset = ptr.deref(p, offset)
if kind == ptr.STRUCT:
data_size = ptr.struct_data_size(p)
ptrs_size = ptr.struct_ptrs_size(p)
return _endof_struct(seg, p, offset, data_size, ptrs_size)
elif kind == ptr.LIST:
item_size = ptr.list_size_tag(p)
count = ptr.list_item_count(p)
if item_size == ptr.LIST_SIZE_COMPOSITE:
tag = seg.read_ptr(offset)
count = ptr.offset(tag)
data_size = ptr.struct_data_size(tag)
ptrs_size = ptr.struct_ptrs_size(tag)
return _endof_list_composite(seg, p, offset, count, data_size,
ptrs_size)
elif item_size == ptr.LIST_SIZE_PTR:
return _endof_list_ptr(seg, p, offset, count)
elif item_size == ptr.LIST_SIZE_BIT:
return _endof_list_bit(seg, p, offset, count)
else:
return _endof_list_primitive(seg, p, offset, item_size, count)
elif kind == ptr.FAR:
return -1
else:
assert False, 'unknown ptr kind'
def read_far_ptr(self, offset):
"""
Read and return the ptr referenced by this far pointer
"""
p = self.read_ptr(offset)
segment_start = self.segment_offsets[ptr.far_target(p)] # in bytes
offset = segment_start + ptr.far_offset(p) * 8
if ptr.far_landing_pad(p) == 0:
# simple case: the landing pad is a normal pointer, just read it
p = self.read_ptr(offset)
return offset, p
else:
# complex case. From capnproto specs:
# If B == 1, then the "landing pad" is itself another far
# pointer that is interpreted differently: This far pointer
# (which always has B = 0) points to the start of the object's
# content, located in some other segment. The landing pad is
# itself immediately followed by a tag word. The tag word
# looks exactly like an intra-segment pointer to the target
# object would look, except that the offset is always zero.
#
# read the 2nd far pointer and the tag word
p = self.read_ptr(offset)
ptag = self.read_ptr(offset + 8)
assert ptr.kind(p) == ptr.FAR
assert ptr.far_landing_pad(p) == 0
assert ptr.offset(ptag) == 0
# compute the absolute offset which the object is located at
segment_start = self.segment_offsets[ptr.far_target(p)] # in bytes
offset = segment_start + ptr.far_offset(p) * 8
#
# ptag is a pointer which perfectly describes the object we want
# to read. Remember that normally when ptr_offset==0, capnproto
# expects the object to start at offset+8. So here we return
# offset-8, so that the object will be read at the expected place
return offset - 8, ptag
def unpack(p):
assert ptr.kind(p) == ptr.STRUCT
return ptr.offset(p), ptr.struct_data_size(p), ptr.struct_ptrs_size(p)
def _init_from_pointer(self, buf, offset, p):
assert ptr.kind(p) == ptr.STRUCT
struct_offset = ptr.deref(p, offset)
data_size = ptr.struct_data_size(p)
ptrs_size = ptr.struct_ptrs_size(p)
self._init_from_buffer(buf, struct_offset, data_size, ptrs_size)
def unpack(p):
assert ptr.kind(p) == ptr.FAR
return ptr.far_landing_pad(p), ptr.far_offset(p), ptr.far_target(p)
def unpack(p):
assert ptr.kind(p) == ptr.LIST
return ptr.offset(p), ptr.list_size_tag(p), ptr.list_item_count(p)