def test_maybe_compress(): pass try_converters = [bytes, memoryview] try_compressions = ['zlib', 'lz4'] payload = b'123' with dask.config.set({'distributed.comm.compression': None}): for f in try_converters: assert maybe_compress(f(payload)) == (None, payload) for compression in try_compressions: try: __import__(compression) except ImportError: continue with dask.config.set({'distributed.comm.compression': compression}): for f in try_converters: payload = b'123' assert maybe_compress(f(payload)) == (None, payload) payload = b'0' * 10000 rc, rd = maybe_compress(f(payload)) # For some reason compressing memoryviews can force blosc... assert rc in (compression, 'blosc') assert compressions[rc]['decompress'](rd) == payload
def test_maybe_compress(): pass try_converters = [bytes, memoryview] try_compressions = ['zlib', 'lz4'] payload = b'123' with dask.set_options(compression=None): for f in try_converters: assert maybe_compress(f(payload)) == (None, payload) for compression in try_compressions: try: __import__(compression) except ImportError: continue with dask.set_options(compression=compression): for f in try_converters: payload = b'123' assert maybe_compress(f(payload)) == (None, payload) payload = b'0' * 10000 rc, rd = maybe_compress(f(payload)) # For some reason compressing memoryviews can force blosc... assert rc in (compression, 'blosc') assert compressions[rc]['decompress'](rd) == payload
def test_maybe_compress(): pass try_converters = [bytes, memoryview] try_compressions = ["zlib", "lz4"] payload = b"123" with dask.config.set({"distributed.comm.compression": None}): for f in try_converters: assert maybe_compress(f(payload)) == (None, payload) for compression in try_compressions: try: __import__(compression) except ImportError: continue with dask.config.set({"distributed.comm.compression": compression}): for f in try_converters: payload = b"123" assert maybe_compress(f(payload)) == (None, payload) payload = b"0" * 10000 rc, rd = maybe_compress(f(payload)) # For some reason compressing memoryviews can force blosc... assert rc in (compression, "blosc") assert compressions[rc]["decompress"](rd) == payload
def test_maybe_compress(): import zlib payload = b'123' assert maybe_compress(payload, None) == (None, payload) assert maybe_compress(payload, 'zlib') == (None, payload) assert maybe_compress(b'111', 'zlib') == (None, b'111') payload = b'0' * 10000 assert maybe_compress(payload, 'zlib') == ('zlib', zlib.compress(payload))
def test_maybe_compress(): import zlib payload = b'123' with dask.set_options(compression=None): assert maybe_compress(payload) == (None, payload) with dask.set_options(compression='zlib'): assert maybe_compress(payload) == (None, payload) assert maybe_compress(b'111') == (None, b'111') payload = b'0' * 10000 assert maybe_compress(payload) == ('zlib', zlib.compress(payload))
def test_maybe_compress(lib, compression): if lib: pytest.importorskip(lib) try_converters = [bytes, memoryview] for f in try_converters: payload = b"123" assert maybe_compress(f(payload), compression=compression) == (None, payload) payload = b"0" * 10000 rc, rd = maybe_compress(f(payload), compression=compression) # For some reason compressing memoryviews can force blosc... assert rc in (compression, "blosc") assert compressions[rc]["decompress"](rd) == payload
def test_maybe_compress_sample(): np = pytest.importorskip("numpy") lz4 = pytest.importorskip("lz4") payload = np.random.randint(0, 255, size=10000).astype("u1").tobytes() fmt, compressed = maybe_compress(payload) assert fmt is None assert compressed == payload
def test_maybe_compress(lib, compression): if lib: pytest.importorskip(lib) try_converters = [bytes, memoryview] with dask.config.set({"distributed.comm.compression": compression}): for f in try_converters: payload = b"123" assert maybe_compress(f(payload)) == (None, payload) payload = b"0" * 10000 rc, rd = maybe_compress(f(payload)) # For some reason compressing memoryviews can force blosc... assert rc in (compression, "blosc") assert compressions[rc]["decompress"](rd) == payload
def test_maybe_compress_sample(): np = pytest.importorskip('numpy') lz4 = pytest.importorskip('lz4') payload = np.random.randint(0, 255, dtype='u1', size=10000).tobytes() fmt, compressed = maybe_compress(payload, 'lz4') assert fmt == None assert compressed == payload
def test_maybe_compress_memoryviews(): np = pytest.importorskip("numpy") pytest.importorskip("lz4") x = np.arange(1000000, dtype="int64") compression, payload = maybe_compress(x.data) try: import blosc # noqa: F401 except ImportError: assert compression == "lz4" assert len(payload) < x.nbytes * 0.75 else: assert compression == "blosc" assert len(payload) < x.nbytes / 10
def test_maybe_compress_memoryviews(): np = pytest.importorskip('numpy') pytest.importorskip('lz4') x = np.arange(1000000, dtype='int64') compression, payload = maybe_compress(x.data) try: import blosc # noqa: F401 except ImportError: assert compression == 'lz4' assert len(payload) < x.nbytes * 0.75 else: assert compression == 'blosc' assert len(payload) < x.nbytes / 10