def test_compress_input_types(self):
import numpy as np
# assume the expected answer was compressed from bytes
expected = blosc2.compress(b"0123456789", typesize=1)
# now for all the things that support the buffer interface
self.assertEqual(
expected, blosc2.compress(memoryview(b"0123456789"), typesize=1))
self.assertEqual(expected,
blosc2.compress(bytearray(b"0123456789"), typesize=1))
self.assertEqual(
expected, blosc2.compress(np.array([b"0123456789"]), typesize=1))
def test_bitshuffle_not_multiple(self):
# Check the fix for #133
x = numpy.ones(27266, dtype="uint8")
xx = x.tobytes()
zxx = blosc2.compress(xx, typesize=8, shuffle=blosc2.Filter.BITSHUFFLE)
last_xx = blosc2.decompress(zxx)[-3:]
self.assertEqual(last_xx, b"\x01\x01\x01")
def test_all_filters(self):
s = b"0123456789" * 100
filters = list(blosc2.Filter)
for filter_ in filters:
c = blosc2.compress(s, typesize=1, shuffle=filter_)
d = blosc2.decompress(c)
self.assertEqual(s, d)
def test_raise_error(object, cname):
c = blosc2.compress(object, cname=cname)
dest = bytearray(object)
with pytest.raises(ValueError):
blosc2.decompress(c, dst=dest)
dest3 = blosc2.decompress(c)
if type(object) is bytes:
assert dest3 == object
else:
assert dest3 == object.tobytes()
dest4 = blosc2.decompress(c, as_bytearray=True)
if type(object) is bytes:
assert dest4 == object
else:
assert dest4 == object.tobytes()
dest5 = bytearray(object)
with pytest.raises(ValueError):
blosc2.decompress(numpy.array([c]), dst=dest5)
with pytest.raises(ValueError):
blosc2.decompress(b"")
def test_bitshuffle_leftovers(self):
# Test for https://github.com/blosc2/c-blosc22/pull/100
buffer = b" " * 641091 # a buffer that is not divisible by 8
cbuffer = blosc2.compress(buffer,
typesize=8,
shuffle=blosc2.Filter.BITSHUFFLE,
clevel=1)
dbuffer = blosc2.decompress(cbuffer)
self.assertTrue(buffer == dbuffer)
def test_compressors(typesize, clevel, filt, cname):
src = b"Something to be compressed" * 100
dest = blosc2.compress(src, typesize, clevel, filt, cname)
src2 = blosc2.decompress(dest)
assert src == src2
if cname == "lz4hc":
assert blosc2.get_clib(dest).lower() == b"lz4"
else:
assert blosc2.get_clib(dest).lower() == cname.encode("utf-8").lower()
blosc2.free_resources()
def test_decompress_releasegil(self):
import numpy as np
# assume the expected answer was compressed from bytes
blosc2.set_releasegil(True)
expected = b"0123456789"
compressed = blosc2.compress(expected, typesize=1)
# now for all the things that support the buffer interface
self.assertEqual(expected, blosc2.decompress(compressed))
self.assertEqual(expected, blosc2.decompress(memoryview(compressed)))
self.assertEqual(expected, blosc2.decompress(bytearray(compressed)))
self.assertEqual(expected, blosc2.decompress(np.array([compressed])))
blosc2.set_releasegil(False)
def test_decompress(object, cname):
c = blosc2.compress(object, cname=cname)
dest = bytearray(object)
blosc2.decompress(c, dst=dest)
assert dest == object
dest3 = blosc2.decompress(c)
assert dest3 == object
dest4 = blosc2.decompress(c, as_bytearray=True)
assert dest4 == object
dest5 = bytearray(object)
blosc2.decompress(numpy.array([c]), dst=dest5)
assert dest5 == object
def test_decompress_input_types_as_bytearray(self):
import numpy as np
# assume the expected answer was compressed from bytes
expected = bytearray(b"0123456789")
compressed = blosc2.compress(expected, typesize=1)
# now for all the things that support the buffer interface
self.assertEqual(expected,
blosc2.decompress(compressed, as_bytearray=True))
self.assertEqual(
expected,
blosc2.decompress(memoryview(compressed), as_bytearray=True))
self.assertEqual(
expected,
blosc2.decompress(bytearray(compressed), as_bytearray=True))
self.assertEqual(
expected,
blosc2.decompress(np.array([compressed]), as_bytearray=True))
def test_decompress_numpy(object, cname):
c = blosc2.compress(object, cname=cname)
dest = bytearray(object)
blosc2.decompress(c, dst=dest)
assert dest == object.tobytes()
dest2 = numpy.empty(object.shape, object.dtype)
blosc2.decompress(c, dst=dest2)
assert numpy.array_equal(dest2, object)
dest3 = blosc2.decompress(c)
assert dest3 == object.tobytes()
dest4 = blosc2.decompress(c, as_bytearray=True)
assert dest4 == object.tobytes()
dest5 = numpy.empty(object.shape, object.dtype)
blosc2.decompress(c, dst=memoryview(dest5))
assert numpy.array_equal(dest5, object)
dtoc = time.time()
assert np.array_equal(in_, out)
tc = (ctoc - ctic) / NREP
td = (dtoc - dtic) / NREP
print(
" Time for pack_array/unpack_array: %.3f/%.3f s (%.2f/%.2f GB/s)) "
% (tc, td, ((N * 8 / tc) / 2**30), ((N * 8 / td) / 2**30)),
end="",
)
print("\tcr: %5.1fx" % (in_.size * in_.dtype.itemsize * 1.0 / len(c)))
ctic = time.time()
for i in range(NREP):
c = blosc2.compress(in_,
clevel=clevel,
shuffle=blosc2.Filter.SHUFFLE,
cname=cname)
ctoc = time.time()
out = np.full_like(in_, fill_value=0)
dtic = time.time()
for i in range(NREP):
blosc2.decompress(c, dst=out)
dtoc = time.time()
assert np.array_equal(in_, out)
tc = (ctoc - ctic) / NREP
td = (dtoc - dtic) / NREP
print(
" Time for compress/decompress: %.3f/%.3f s (%.2f/%.2f GB/s)) "
% (tc, td, ((N * 8 / tc) / 2**30), ((N * 8 / td) / 2**30)),
end="",
print(" *** np.copyto() *** Time for memcpy():\t%.3f s\t(%.2f GB/s)" %
(tcpy, (N * 8 / tcpy) / 2**30))
print("\nTimes for compressing/decompressing:")
for (in_, label) in arrays:
print("\n*** %s ***" % label)
for cname in blosc2.compressor_list():
for filter in [
blosc2.Filter.NOFILTER, blosc2.Filter.SHUFFLE,
blosc2.Filter.BITSHUFFLE
]:
clevel = 6
t0 = time.time()
c = blosc2.compress(in_,
in_.itemsize,
clevel=clevel,
shuffle=filter,
cname=cname)
tc = time.time() - t0
# Cause a page fault here
out = np.full_like(in_, fill_value=0)
t0 = time.time()
for i in range(NREP):
blosc2.decompress(c, dst=out)
td = (time.time() - t0) / NREP
assert np.array_equal(in_, out)
print(
" *** %-7s, %-10s *** %6.3f s (%.2f GB/s) / %5.3f s (%.2f GB/s)"
% (cname, filter, tc, ((N * 8 / tc) / 2**30), td,
((N * 8 / td) / 2**30)),
end="",
def test_get_blocksize(self):
s = b"0123456789" * 1000
blosc2.set_blocksize(2**14)
blosc2.compress(s, typesize=1)
d = blosc2.get_blocksize()
self.assertEqual(d, 2**14)
def decompress():
cx = blosc2.compress(array, typesize, clevel=1)
blosc2.decompress(cx)
def compress():
blosc2.compress(array, typesize, clevel=1)
def test_basic_codec(self):
s = b"0123456789"
c = blosc2.compress(s, typesize=1)
d = blosc2.decompress(c)
self.assertEqual(s, d)
def test_all_compressors(self):
s = b"0123456789" * 100
for cname in blosc2.compressor_list():
c = blosc2.compress(s, typesize=1, cname=cname)
d = blosc2.decompress(c)
self.assertEqual(s, d)
def test_bytes_array(arr, gil):
blosc2.set_releasegil(gil)
dest = blosc2.compress(arr, 1)
assert arr == blosc2.decompress(dest)
######################################################################## # # Created: April 30, 2021 # Author: The Blosc development team - [email protected] # ######################################################################## import array # Compress and decompress different arrays import blosc2 a = array.array("i", range(1000 * 1000)) a_bytesobj = a.tobytes() c_bytesobj = blosc2.compress(a_bytesobj, typesize=4) assert len(c_bytesobj) < len(a_bytesobj) a_bytesobj2 = blosc2.decompress(c_bytesobj) assert a_bytesobj == a_bytesobj2 dest = blosc2.compress(b"", 1) assert b"" == blosc2.decompress(dest) assert type(blosc2.decompress(blosc2.compress(b"1" * 7, 8), as_bytearray=True)) is bytearray
