def buffer_callback(buffer: PickleBuffer) -> None: value = buffer.raw() offset = self.__offset length = len(value) if offset + length > self.block.size: raise ValueTooLarge( f"value exceeds available space in block, {length} bytes needed but {self.block.size - offset} bytes free" ) self.block.buf[offset:offset + length] = value self.__offset += length buffers.append((offset, length))
def __reduce_ex__( self, protocol: int ) -> Tuple[Any, Tuple[Sequence[Any], Optional[datetime]]]: if protocol >= 5: return ( type(self), ([PickleBuffer(row) for row in self.rows], self.origin_timestamp), ) else: return type(self), (self.rows, self.origin_timestamp)
def test_ndarray_2d(self): # C-contiguous ndarray = support.import_module("_testbuffer").ndarray arr = ndarray(list(range(12)), shape=(4, 3), format='<i') self.assertTrue(arr.c_contiguous) self.assertFalse(arr.f_contiguous) pb = PickleBuffer(arr) self.check_memoryview(pb, arr) # Non-contiguous arr = arr[::2] self.assertFalse(arr.c_contiguous) self.assertFalse(arr.f_contiguous) pb = PickleBuffer(arr) self.check_memoryview(pb, arr) # F-contiguous arr = ndarray(list(range(12)), shape=(3, 4), strides=(4, 12), format='<i') self.assertTrue(arr.f_contiguous) self.assertFalse(arr.c_contiguous) pb = PickleBuffer(arr) self.check_memoryview(pb, arr)
def test_pickle_protocol(self): data = b"iamsomedata" pickled_data = PickleBuffer(data) with self.assertRaises(PickleError) as context: result = pickle.dumps(pickled_data) assert "PickleBuffer can only pickled with protocol >= 5" == str( context.exception) result = pickle.dumps(pickled_data, protocol=5) assert result
def test_release(self): pb = PickleBuffer(b"foo") pb.release() with self.assertRaises(ValueError) as raises: memoryview(pb) self.assertIn("operation forbidden on released PickleBuffer object", str(raises.exception)) # Idempotency pb.release()
def __reduce_ex__(self, protocol): if protocol < 5: raise RuntimeError('Needs at least pickle protocol 5') builder = self._reader.as_builder() protoname = builder.schema.node.displayName.replace('/', '.')\ .replace('.capnp:', '_capnp:') meta = { 'protoname': protoname, 'streamdir': str(self._streamdir), 'setdir': str(self._setdir), 'storedir': str(self._storedir), } segments = builder.to_segments() return (self.from_segments, (meta, *[PickleBuffer(x) for x in segments]))
def test_raw_released(self): pb = PickleBuffer(b"foo") pb.release() with self.assertRaises(ValueError) as raises: pb.raw()
def check_raw_non_contiguous(self, obj): pb = PickleBuffer(obj) with self.assertRaisesRegex(BufferError, "non-contiguous"): pb.raw()
def check_raw(self, obj, equiv): pb = PickleBuffer(obj) with pb.raw() as m: self.assertIsInstance(m, memoryview) self.check_memoryview(m, equiv)
def __reduce_ex__(self, protocol: int): if protocol >= 5: return (type(self), ([PickleBuffer(row) for row in self.rows], )) else: return type(self), (self.rows, )
def __reduce_ex__(self, protocol): if protocol >= 5: return MemoryviewHolder, (PickleBuffer(self.mv), ) else: return MemoryviewHolder, (self.mv.tobytes(), )