def test_serialize_list_compress(): x = np.ones(1000000) L = serialize_bytelist(x) assert sum(map(len, L)) < x.nbytes / 2 b = b''.join(L) y = deserialize_bytes(b) assert (x == y).all()
def test_serialize_list_compress(): pytest.importorskip("lz4") x = np.ones(1000000) L = serialize_bytelist(x) assert sum(map(nbytes, L)) < x.nbytes / 2 b = b"".join(L) y = deserialize_bytes(b) assert (x == y).all()
def test_serialize_list_compress(): pytest.importorskip('lz4') x = np.ones(1000000) L = serialize_bytelist(x) assert sum(map(nbytes, L)) < x.nbytes / 2 b = b''.join(L) y = deserialize_bytes(b) assert (x == y).all()
def test_serialize_cupy_collection(collection, length, value): # Avoid running test for length 0 (no collection) multiple times if length == 0 and collection is not list: return if isinstance(value, dict): cudf = pytest.importorskip("cudf") dd = pytest.importorskip("dask.dataframe") x = cudf.DataFrame(value) assert_func = dd.assert_eq else: x = cupy.arange(10) assert_func = assert_eq if length == 0: obj = device_to_host(x) elif collection is dict: obj = device_to_host(dict(zip(range(length), (x, ) * length))) else: obj = device_to_host(collection((x, ) * length)) if length > 0: assert all( [h["serializer"] == "dask" for h in obj.header["sub-headers"]]) else: assert obj.header["serializer"] == "dask" btslst = serialize_bytelist(obj) bts = deserialize_bytes(b"".join(btslst)) res = host_to_device(bts) if length == 0: assert_func(res, x) else: assert isinstance(res, collection) values = res.values() if collection is dict else res [assert_func(v, x) for v in values] header, frames = serialize(obj, serializers=["pickle"], on_error="raise") if HIGHEST_PROTOCOL >= 5: assert len(frames) == (1 + len(obj.frames)) else: assert len(frames) == 1 obj2 = deserialize(header, frames) res = host_to_device(obj2) if length == 0: assert_func(res, x) else: assert isinstance(res, collection) values = res.values() if collection is dict else res [assert_func(v, x) for v in values]
def psize(*objs) -> tuple[int, int]: return ( sum(sizeof(o) for o in objs), sum(len(frame) for obj in objs for frame in serialize_bytelist(obj)), )