def eq_frames(a, b): if b'headers' in a: return (msgpack.loads(a, use_list=False) == msgpack.loads(b, use_list=False)) else: return a == b
def eq_frames(a, b): if b"headers" in a: return msgpack.loads(a, use_list=False, strict_map_key=False) == msgpack.loads( b, use_list=False, strict_map_key=False) else: return a == b
def test_compression_2(): pytest.importorskip("lz4") np = pytest.importorskip("numpy") x = np.random.random(10000) msg = dumps(to_serialize(x.tobytes())) compression = msgpack.loads(msg[1]).get("compression") assert all(c is None for c in compression)
def test_compression_2(): pytest.importorskip("lz4") np = pytest.importorskip("numpy") x = np.random.random(10000) header, payload = dumps(x.tobytes()) assert not header or not msgpack.loads(header, encoding="utf8").get("compression")
def test_compression(): pytest.importorskip('lz4') np = pytest.importorskip('numpy') x = np.random.random(10000) header, payload = dumps(x.tobytes()) assert (not header or not msgpack.loads(header, encoding='utf8').get('compression'))
def test_large_messages(): psutil = pytest.importorskip('psutil') pytest.importorskip('lz4') if psutil.virtual_memory().total < 8e9: return def f(n): """ Want to avoid compiling b'0' * 2**31 as a constant during setup.py install, so we turn this into a function and call it in the next line Otherwise this takes up 2 GB of memory during install """ return b'0' * (2**n + 10) big_bytes = f(31) msg = {'x': [big_bytes, b'small_bytes'], 'y': {'a': big_bytes, 'b': b'small_bytes'}} b = dumps(msg) msg2 = loads(b) assert msg == msg2 assert len(b) >= 2 big_header = msgpack.loads(b[2], encoding='utf8') assert len(big_header['shards']) == 2 assert len(big_header['keys']) + 2 + 1 == len(b)
def test_compression_2(): pytest.importorskip('lz4') np = pytest.importorskip('numpy') x = np.random.random(10000) header, payload = dumps(x.tobytes()) assert (not header or not msgpack.loads(header, encoding='utf8').get('compression'))
def test_large_messages(): psutil = pytest.importorskip('psutil') pytest.importorskip('lz4') if psutil.virtual_memory().total < 8e9: return def f(n): """ Want to avoid compiling b'0' * 2**31 as a constant during setup.py install, so we turn this into a function and call it in the next line Otherwise this takes up 2 GB of memory during install """ return b'0' * (2**n + 10) big_bytes = f(31) msg = { 'x': [big_bytes, b'small_bytes'], 'y': { 'a': big_bytes, 'b': b'small_bytes' } } b = dumps(msg) msg2 = loads(b) assert msg == msg2 assert len(b) >= 2 big_header = msgpack.loads(b[2], encoding='utf8') assert len(big_header['shards']) == 2 assert len(big_header['keys']) + 2 + 1 == len(b)
def test_compress_numpy(): pytest.importorskip("lz4") x = np.ones(10000000, dtype="i4") frames = dumps({"x": to_serialize(x)}) assert sum(map(nbytes, frames)) < x.nbytes header = msgpack.loads(frames[2], raw=False, use_list=False) try: import blosc # noqa: F401 except ImportError: pass else: assert all(c == "blosc" for c in header["headers"][("x",)]["compression"])
def test_compress_numpy(): pytest.importorskip('lz4') x = np.ones(10000000, dtype='i4') frames = dumps({'x': to_serialize(x)}) assert sum(map(len, frames)) < x.nbytes header = msgpack.loads(frames[2], encoding='utf8', use_list=False) try: import blosc except ImportError: pass else: assert all(c == 'blosc' for c in header['headers'][('x', )]['compression'])
def test_compress_numpy(): pytest.importorskip('lz4') x = np.ones(10000000, dtype='i4') frames = dumps({'x': to_serialize(x)}) assert sum(map(nbytes, frames)) < x.nbytes header = msgpack.loads(frames[2], raw=False, use_list=False) try: import blosc # noqa: F401 except ImportError: pass else: assert all(c == 'blosc' for c in header['headers'][('x',)]['compression'])