def visit(self, buf, p, offset): kind = ptr.kind(p) offset = ptr.deref(p, offset) if kind == ptr.STRUCT: data_size = ptr.struct_data_size(p) ptrs_size = ptr.struct_ptrs_size(p) return self.visit_struct(buf, p, offset, data_size, ptrs_size) elif kind == ptr.LIST: item_size = ptr.list_size_tag(p) count = ptr.list_item_count(p) if item_size == ptr.LIST_SIZE_COMPOSITE: tag = buf.read_ptr(offset) count = ptr.offset(tag) data_size = ptr.struct_data_size(tag) ptrs_size = ptr.struct_ptrs_size(tag) return self.visit_list_composite(buf, p, offset, count, data_size, ptrs_size) elif item_size == ptr.LIST_SIZE_PTR: return self.visit_list_ptr(buf, p, offset, count) elif item_size == ptr.LIST_SIZE_BIT: return self.visit_list_bit(buf, p, offset, count) else: return self.visit_list_primitive(buf, p, offset, item_size, count) elif kind == ptr.FAR: raise NotImplementedError('Far pointer not supported') else: assert False, 'unknown ptr kind'
def test_new_struct(): p = ptr.new_struct(100, 2, 4) assert ptr.kind(p) == ptr.STRUCT assert ptr.offset(p) == 100 assert ptr.struct_data_size(p) == 2 assert ptr.struct_ptrs_size(p) == 4 assert p == 0x0004000200000190
def _copy_list_composite(src, p, src_pos, dst, dst_pos): src_pos = ptr.deref(p, src_pos) total_words = ptr.list_item_count(p) # n of words NOT including the tag body_length = (total_words + 1) * 8 # total length INCLUDING the tag # # check that there is enough data for both the tag AND the whole body; # this way we do the bound checking only once check_bounds(src, body_length, src_pos) tag = read_int64_fast(src, src_pos) count = ptr.offset(tag) data_size = ptr.struct_data_size(tag) ptrs_size = ptr.struct_ptrs_size(tag) # # allocate the list and copy the whole body at once dst_pos = dst.alloc_list(dst_pos, ptr.LIST_SIZE_COMPOSITE, total_words, body_length) dst.write_slice(dst_pos, src, src_pos, body_length) # # iterate over the elements, fix the pointers and copy the content i = 0 item_length = (data_size + ptrs_size) * 8 ptrs_section_offset = 0 for i in range(count): ptrs_section_offset = 8 + item_length * i + data_size * 8 _copy_many_ptrs(ptrs_size, src, src_pos + ptrs_section_offset, dst, dst_pos + ptrs_section_offset)
def _copy_struct(src, p, src_pos, dst, dst_pos): src_pos = ptr.deref(p, src_pos) data_size = ptr.struct_data_size(p) ptrs_size = ptr.struct_ptrs_size(p) ds = data_size * 8 dst_pos = dst.alloc_struct(dst_pos, data_size, ptrs_size) check_bounds(src, ds, src_pos) dst.write_slice(dst_pos, src, src_pos, ds) # copy data section _copy_many_ptrs(ptrs_size, src, src_pos + ds, dst, dst_pos + ds)
def test_struct_ptr(): # 0004 ptrs size # 0002 data size # 00000190 offset<<2 # 0 kind p = 0x0004000200000190 assert ptr.kind(p) == ptr.STRUCT assert ptr.offset(p) == 100 assert ptr.struct_data_size(p) == 2 assert ptr.struct_ptrs_size(p) == 4
def test__as_pointer(): buf = b('garbage0' '\x01\x00\x00\x00\x00\x00\x00\x00' # 1 '\x02\x00\x00\x00\x00\x00\x00\x00') # 2 b1 = Struct.from_buffer(buf, 8, data_size=2, ptrs_size=0) p = b1._as_pointer(24) # arbitrary offset assert ptr.kind(p) == ptr.STRUCT assert ptr.deref(p, 24) == 8 assert ptr.struct_data_size(p) == 2 assert ptr.struct_ptrs_size(p) == 0
def endof(seg, p, offset): """ Check whether the given object is compact, and in that case compute its end boundary. If it's not compact, return -1. An object is compact if: 1. there is no gap between its data section and its ptrs section 2. there is no gap between children 3. its children are compact 4. there are no FAR pointers """ kind = ptr.kind(p) offset = ptr.deref(p, offset) if kind == ptr.STRUCT: data_size = ptr.struct_data_size(p) ptrs_size = ptr.struct_ptrs_size(p) return _endof_struct(seg, p, offset, data_size, ptrs_size) elif kind == ptr.LIST: item_size = ptr.list_size_tag(p) count = ptr.list_item_count(p) if item_size == ptr.LIST_SIZE_COMPOSITE: tag = seg.read_ptr(offset) count = ptr.offset(tag) data_size = ptr.struct_data_size(tag) ptrs_size = ptr.struct_ptrs_size(tag) return _endof_list_composite(seg, p, offset, count, data_size, ptrs_size) elif item_size == ptr.LIST_SIZE_PTR: return _endof_list_ptr(seg, p, offset, count) elif item_size == ptr.LIST_SIZE_BIT: return _endof_list_bit(seg, p, offset, count) else: return _endof_list_primitive(seg, p, offset, item_size, count) elif kind == ptr.FAR: return -1 else: assert False, 'unknown ptr kind'
def _copy_struct(src, p, src_pos, dst, dst_pos): src_pos = ptr.deref(p, src_pos) data_size = ptr.struct_data_size(p) ptrs_size = ptr.struct_ptrs_size(p) if data_size + ptrs_size == 0: # "empty" struct, no need to allocate dst.write_int64(dst_pos, ptr.new_struct(-1, 0, 0)) return ds = data_size * 8 dst_pos = dst.alloc_struct(dst_pos, data_size, ptrs_size) check_bounds(src, ds, src_pos) dst.write_slice(dst_pos, src, src_pos, ds) # copy data section _copy_many_ptrs(ptrs_size, src, src_pos + ds, dst, dst_pos + ds)
def _set_list_tag(self, size_tag, item_count): self._size_tag = size_tag if size_tag == ptr.LIST_SIZE_COMPOSITE: tag = self._seg.read_ptr(self._offset) self._tag = tag self._item_count = ptr.offset(tag) self._item_length = (ptr.struct_data_size(tag)+ptr.struct_ptrs_size(tag))*8 self._item_offset = 8 else: self._tag = -1 self._item_count = item_count self._item_length = ptr.list_item_length(size_tag) self._item_offset = 0
def _read_list_or_struct(self, ptr_offset, default_=None): ptr_offset, p = self._read_ptr_generic(ptr_offset) if p == 0: return default_ blob_offet = ptr.deref(p, ptr_offset) if ptr.kind(p) == ptr.STRUCT: Struct = capnpy.struct_.Struct return Struct.from_buffer(self._buf, blob_offet, ptr.struct_data_size(p), ptr.struct_ptrs_size(p)) elif ptr.kind(p) == ptr.LIST: List = capnpy.list.List return List.from_buffer(self._buf, blob_offet, ptr.list_size_tag(p), ptr.list_item_count(p), capnpy.list.StructItemType(Blob)) else: assert False, 'Unkwown pointer kind: %s' % ptr.kind(p)
def ptr(self, offset, s): p = struct.unpack('q', s)[0] if ptr.kind(p) not in (ptr.STRUCT, ptr.LIST, ptr.FAR): return ' ' * 25 # # try to display only "reasonable" ptrs; if the fields are too big, it # probably means that the current word is not a pointer def if_in_range(x, min, max): if min <= x < max: return str(x) else: return '?' # if p == 0: return 'NULL'.ljust(25) if ptr.kind(p) == ptr.STRUCT: descr = 'struct {:>4} {:>3}'.format( if_in_range(ptr.struct_data_size(p), 0, 100), if_in_range(ptr.struct_ptrs_size(p), 0, 100)) elif ptr.kind(p) == ptr.LIST: tag = '<%s>' % self._list_tag(ptr.list_size_tag(p)) descr = 'list{:<5} {:>5}'.format( tag, if_in_range(ptr.list_item_count(p), 0, 65536)) elif ptr.kind(p) == ptr.FAR: descr = 'far {:>7} {:>3}'.format( ptr.far_landing_pad(p), if_in_range(ptr.far_target(p), 0, 100)) else: descr = 'unknown ptr ' # if -1000 < ptr.offset(p) < 1000: dest = ptr.deref(p, offset) dest = self.addr(dest) dest = dest.ljust(16) else: dest = '? ' line = '{0} to {1}'.format(descr, dest) if '?' in line: return Color.set(Color.lightgray, line) else: return line
def _copy_struct_inline(src, p, src_pos, dst, dst_pos): # this does the same as _copy_struct, but instead of allocating space for # it, it fills an already-allocated space (useful e.g. for writing structs # into lists). # # I have tried several ways to reduce code duplication (such as adding a # do_allocation param to determine whether to allocate or not), but it # always caused a ~30% slowdown. Apparently, it seems to be related to the # number of call-sites to _copy_struct: at the moment of writing is it # called only from copy_pointer, but it seems enough to add another call # site (even if it's never actually called!) to cause the slowdown. Maybe # it's because this causes GCC not to inline it? Anyway, the only solution # I found, was to duplicate some of the code :( # src_pos = ptr.deref(p, src_pos) data_size = ptr.struct_data_size(p) ptrs_size = ptr.struct_ptrs_size(p) ds = data_size * 8 check_bounds(src, ds, src_pos) dst.write_slice(dst_pos, src, src_pos, ds) # copy data section _copy_many_ptrs(ptrs_size, src, src_pos + ds, dst, dst_pos + ds)
def unpack(p): assert ptr.kind(p) == ptr.STRUCT return ptr.offset(p), ptr.struct_data_size(p), ptr.struct_ptrs_size(p)
def _init_from_pointer(self, buf, offset, p): assert ptr.kind(p) == ptr.STRUCT struct_offset = ptr.deref(p, offset) data_size = ptr.struct_data_size(p) ptrs_size = ptr.struct_ptrs_size(p) self._init_from_buffer(buf, struct_offset, data_size, ptrs_size)
def read_item(self, lst, i): offset = self.offset_for_item(lst, i) return self.structcls.from_buffer(lst._seg, offset, ptr.struct_data_size(lst._tag), ptr.struct_ptrs_size(lst._tag))