def eq_frames(a, b):
if b'headers' in a:
return (msgpack.loads(a,
use_list=False) == msgpack.loads(b,
use_list=False))
else:
return a == b
def eq_frames(a, b):
if b"headers" in a:
return msgpack.loads(a, use_list=False,
strict_map_key=False) == msgpack.loads(
b, use_list=False, strict_map_key=False)
else:
return a == b
def test_compression_2():
pytest.importorskip("lz4")
np = pytest.importorskip("numpy")
x = np.random.random(10000)
msg = dumps(to_serialize(x.tobytes()))
compression = msgpack.loads(msg[1]).get("compression")
assert all(c is None for c in compression)
def test_compression_2():
pytest.importorskip("lz4")
np = pytest.importorskip("numpy")
x = np.random.random(10000)
header, payload = dumps(x.tobytes())
assert not header or not msgpack.loads(header,
encoding="utf8").get("compression")
def test_compression():
pytest.importorskip('lz4')
np = pytest.importorskip('numpy')
x = np.random.random(10000)
header, payload = dumps(x.tobytes())
assert (not header
or not msgpack.loads(header, encoding='utf8').get('compression'))
def test_large_messages():
psutil = pytest.importorskip('psutil')
pytest.importorskip('lz4')
if psutil.virtual_memory().total < 8e9:
return
def f(n):
"""
Want to avoid compiling b'0' * 2**31 as a constant during
setup.py install, so we turn this into a function and call it in
the next line
Otherwise this takes up 2 GB of memory during install
"""
return b'0' * (2**n + 10)
big_bytes = f(31)
msg = {'x': [big_bytes, b'small_bytes'],
'y': {'a': big_bytes, 'b': b'small_bytes'}}
b = dumps(msg)
msg2 = loads(b)
assert msg == msg2
assert len(b) >= 2
big_header = msgpack.loads(b[2], encoding='utf8')
assert len(big_header['shards']) == 2
assert len(big_header['keys']) + 2 + 1 == len(b)
def test_compression_2():
pytest.importorskip('lz4')
np = pytest.importorskip('numpy')
x = np.random.random(10000)
header, payload = dumps(x.tobytes())
assert (not header or
not msgpack.loads(header, encoding='utf8').get('compression'))
def test_large_messages():
psutil = pytest.importorskip('psutil')
pytest.importorskip('lz4')
if psutil.virtual_memory().total < 8e9:
return
def f(n):
"""
Want to avoid compiling b'0' * 2**31 as a constant during
setup.py install, so we turn this into a function and call it in
the next line
Otherwise this takes up 2 GB of memory during install
"""
return b'0' * (2**n + 10)
big_bytes = f(31)
msg = {
'x': [big_bytes, b'small_bytes'],
'y': {
'a': big_bytes,
'b': b'small_bytes'
}
}
b = dumps(msg)
msg2 = loads(b)
assert msg == msg2
assert len(b) >= 2
big_header = msgpack.loads(b[2], encoding='utf8')
assert len(big_header['shards']) == 2
assert len(big_header['keys']) + 2 + 1 == len(b)
def test_compress_numpy():
pytest.importorskip("lz4")
x = np.ones(10000000, dtype="i4")
frames = dumps({"x": to_serialize(x)})
assert sum(map(nbytes, frames)) < x.nbytes
header = msgpack.loads(frames[2], raw=False, use_list=False)
try:
import blosc # noqa: F401
except ImportError:
pass
else:
assert all(c == "blosc" for c in header["headers"][("x",)]["compression"])
def test_compress_numpy():
pytest.importorskip('lz4')
x = np.ones(10000000, dtype='i4')
frames = dumps({'x': to_serialize(x)})
assert sum(map(len, frames)) < x.nbytes
header = msgpack.loads(frames[2], encoding='utf8', use_list=False)
try:
import blosc
except ImportError:
pass
else:
assert all(c == 'blosc'
for c in header['headers'][('x', )]['compression'])
def test_compress_numpy():
pytest.importorskip('lz4')
x = np.ones(10000000, dtype='i4')
frames = dumps({'x': to_serialize(x)})
assert sum(map(nbytes, frames)) < x.nbytes
header = msgpack.loads(frames[2], raw=False, use_list=False)
try:
import blosc # noqa: F401
except ImportError:
pass
else:
assert all(c == 'blosc' for c in
header['headers'][('x',)]['compression'])
def eq_frames(a, b):
if b'headers' in a:
return (msgpack.loads(a, use_list=False)
== msgpack.loads(b, use_list=False))
else:
return a == b
