def test_serialize_list_compress():
x = np.ones(1000000)
L = serialize_bytelist(x)
assert sum(map(len, L)) < x.nbytes / 2
b = b''.join(L)
y = deserialize_bytes(b)
assert (x == y).all()
def test_serialize_list_compress():
pytest.importorskip("lz4")
x = np.ones(1000000)
L = serialize_bytelist(x)
assert sum(map(nbytes, L)) < x.nbytes / 2
b = b"".join(L)
y = deserialize_bytes(b)
assert (x == y).all()
def test_serialize_list_compress():
pytest.importorskip('lz4')
x = np.ones(1000000)
L = serialize_bytelist(x)
assert sum(map(nbytes, L)) < x.nbytes / 2
b = b''.join(L)
y = deserialize_bytes(b)
assert (x == y).all()
def test_serialize_cupy_collection(collection, length, value):
# Avoid running test for length 0 (no collection) multiple times
if length == 0 and collection is not list:
return
if isinstance(value, dict):
cudf = pytest.importorskip("cudf")
dd = pytest.importorskip("dask.dataframe")
x = cudf.DataFrame(value)
assert_func = dd.assert_eq
else:
x = cupy.arange(10)
assert_func = assert_eq
if length == 0:
obj = device_to_host(x)
elif collection is dict:
obj = device_to_host(dict(zip(range(length), (x, ) * length)))
else:
obj = device_to_host(collection((x, ) * length))
if length > 0:
assert all(
[h["serializer"] == "dask" for h in obj.header["sub-headers"]])
else:
assert obj.header["serializer"] == "dask"
btslst = serialize_bytelist(obj)
bts = deserialize_bytes(b"".join(btslst))
res = host_to_device(bts)
if length == 0:
assert_func(res, x)
else:
assert isinstance(res, collection)
values = res.values() if collection is dict else res
[assert_func(v, x) for v in values]
header, frames = serialize(obj, serializers=["pickle"], on_error="raise")
if HIGHEST_PROTOCOL >= 5:
assert len(frames) == (1 + len(obj.frames))
else:
assert len(frames) == 1
obj2 = deserialize(header, frames)
res = host_to_device(obj2)
if length == 0:
assert_func(res, x)
else:
assert isinstance(res, collection)
values = res.values() if collection is dict else res
[assert_func(v, x) for v in values]
def psize(*objs) -> tuple[int, int]:
return (
sum(sizeof(o) for o in objs),
sum(len(frame) for obj in objs for frame in serialize_bytelist(obj)),
)
