def test_ensure_bytes():
data = [b'1', '1', memoryview(b'1'), bytearray(b'1')]
if PY2:
data.append(buffer(b'1')) # flake8: noqa
for d in data:
result = ensure_bytes(d)
assert isinstance(result, bytes)
assert result == b'1'
def test_ensure_bytes():
data = [b"1", "1", memoryview(b"1"), bytearray(b"1")]
if PY2:
data.append(buffer(b"1")) # noqa: F821
for d in data:
result = ensure_bytes(d)
assert isinstance(result, bytes)
assert result == b"1"
def test_ensure_bytes():
data = [b'1', '1', memoryview(b'1'), bytearray(b'1')]
if PY2:
data.append(buffer(b'1')) # noqa: F821
for d in data:
result = ensure_bytes(d)
assert isinstance(result, bytes)
assert result == b'1'
def maybe_compress(
payload,
min_size=1e4,
sample_size=1e4,
nsamples=5,
compression=dask.config.get("distributed.comm.compression"),
):
"""
Maybe compress payload
1. We don't compress small messages
2. We sample the payload in a few spots, compress that, and if it doesn't
do any good we return the original
3. We then compress the full original, it it doesn't compress well then we
return the original
4. We return the compressed result
"""
if compression == "auto":
compression = default_compression
if not compression:
return None, payload
if len(payload) < min_size:
return None, payload
if len(payload) > 2**31: # Too large, compression libraries often fail
return None, payload
min_size = int(min_size)
sample_size = int(sample_size)
compress = compressions[compression]["compress"]
# Compress a sample, return original if not very compressed
sample = byte_sample(payload, sample_size, nsamples)
if len(compress(sample)) > 0.9 * len(sample): # sample not very compressible
return None, payload
if type(payload) is memoryview:
nbytes = payload.itemsize * len(payload)
else:
nbytes = len(payload)
if default_compression and blosc and type(payload) is memoryview:
# Blosc does itemsize-aware shuffling, resulting in better compression
compressed = blosc.compress(
payload, typesize=payload.itemsize, cname="lz4", clevel=5
)
compression = "blosc"
else:
compressed = compress(ensure_bytes(payload))
if len(compressed) > 0.9 * nbytes: # full data not very compressible
return None, payload
else:
return compression, compressed
def test_ensure_bytes():
data = [
b"1", "1",
memoryview(b"1"),
bytearray(b"1"),
array.array("b", [49])
]
for d in data:
result = ensure_bytes(d)
assert isinstance(result, bytes)
assert result == b"1"
async def write(self, msg, serializers=None, on_error=None):
frames = await to_frames(
msg,
allow_offload=self.allow_offload,
serializers=serializers,
on_error=on_error,
context={
"sender": self.local_info,
"recipient": self.remote_info,
**self.handshake_options,
},
frame_split_size=BIG_BYTES_SHARD_SIZE,
)
n = struct.pack("Q", len(frames))
try:
await self.sock.write_message(n, binary=True)
for frame in frames:
await self.sock.write_message(ensure_bytes(frame), binary=True)
except WebSocketClosedError as e:
raise CommClosedError(e)
return sum(map(nbytes, frames))
def test_ensure_bytes_pyarrow_buffer():
pa = pytest.importorskip("pyarrow")
buf = pa.py_buffer(b"123")
result = ensure_bytes(buf)
assert isinstance(result, bytes)
def test_ensure_bytes_ndarray():
result = ensure_bytes(np.arange(12))
assert isinstance(result, bytes)
def test_ensure_bytes_ndarray():
np = pytest.importorskip("numpy")
result = ensure_bytes(np.arange(12))
assert isinstance(result, bytes)
def serialization_error_loads(header, frames):
msg = "\n".join([ensure_bytes(frame).decode("utf8") for frame in frames])
raise TypeError(msg)
