def test_update(contiguous, urlpath, nchunks, nupdates, copy, create_chunk):
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": {"nthreads": 2},
"dparams": {"nthreads": 2},
}
blosc2.remove_urlpath(urlpath)
nbytes = 23401
schunk = blosc2.SChunk(chunksize=nbytes * 2, **storage)
for i in range(nchunks):
bytes_obj = b"i " * nbytes
nchunks_ = schunk.append_data(bytes_obj)
assert nchunks_ == (i + 1)
for i in range(nupdates):
pos = random.randint(0, nchunks - 1)
bytes_obj = b"i " * nbytes
if create_chunk:
chunk = blosc2.compress2(bytes_obj)
schunk.update_chunk(pos, chunk)
else:
schunk.update_data(pos, bytes_obj, copy)
res = schunk.decompress_chunk(pos)
assert res == bytes_obj
for i in range(nchunks):
schunk.decompress_chunk(i)
blosc2.remove_urlpath(urlpath)
def test_schunk_update_numpy(contiguous, urlpath, nchunks, nupdates, copy, create_chunk):
storage = {
"contiguous": contiguous,
"urlpath": urlpath,
"cparams": {"nthreads": 2},
"dparams": {"nthreads": 2},
}
blosc2.remove_urlpath(urlpath)
schunk = blosc2.SChunk(chunksize=200 * 1000 * 4, **storage)
for i in range(nchunks):
buffer = i * numpy.arange(200 * 1000, dtype="int32")
nchunks_ = schunk.append_data(buffer)
assert nchunks_ == (i + 1)
for i in range(nupdates):
pos = random.randint(0, nchunks - 1)
buffer = pos * numpy.arange(200 * 1000, dtype="int32")
if create_chunk:
chunk = blosc2.compress2(buffer)
schunk.update_chunk(pos, chunk)
else:
schunk.update_data(pos, buffer, copy)
chunk_ = schunk.decompress_chunk(pos)
bytes_obj = buffer.tobytes()
assert chunk_ == bytes_obj
dest = numpy.empty(buffer.shape, buffer.dtype)
schunk.decompress_chunk(pos, dest)
assert numpy.array_equal(buffer, dest)
for i in range(nchunks):
schunk.decompress_chunk(i)
blosc2.remove_urlpath(urlpath)
def test_compress2(nbytes, cparams, dparams):
bytes_obj = b" " * nbytes
c = blosc2.compress2(bytes_obj, **cparams)
dest = bytearray(bytes_obj)
blosc2.decompress2(c, dst=dest, **dparams)
assert dest == bytes_obj
dest2 = blosc2.decompress2(c, **dparams)
assert dest2 == bytes_obj
dest3 = bytearray(bytes_obj)
blosc2.decompress2(numpy.array([c]), dst=dest3, **dparams)
assert dest3 == bytes_obj
def test_compress2_numpy(obj, cparams, dparams):
bytes_obj = obj.tobytes()
c = blosc2.compress2(obj, **cparams)
dest = bytearray(obj)
blosc2.decompress2(c, dst=dest, **dparams)
assert dest == bytes_obj
dest2 = numpy.empty(obj.shape, obj.dtype)
blosc2.decompress2(c, dst=dest2, **dparams)
assert numpy.array_equal(dest2, obj)
dest3 = blosc2.decompress2(c, **dparams)
assert dest3 == bytes_obj
dest4 = numpy.empty(obj.shape, obj.dtype)
blosc2.decompress2(c, dst=memoryview(dest4), **dparams)
assert numpy.array_equal(dest4, obj)
def test_raise_error(object, cparams, dparams):
c = blosc2.compress2(object, **cparams, **dparams)
dest = bytearray(object)
with pytest.raises(ValueError):
blosc2.decompress2(c, dst=dest)
dest3 = blosc2.decompress2(c)
if type(object) is bytes:
assert dest3 == object
else:
assert dest3 == object.tobytes()
dest5 = bytearray(object)
with pytest.raises(ValueError):
blosc2.decompress2(numpy.array([c]), dst=dest5)
with pytest.raises(ValueError):
blosc2.decompress2(b"")
######################################################################## # # Created: July 30, 2021 # Author: The Blosc development team - [email protected] # ######################################################################## import blosc2 import numpy as np a = np.linspace(0, 1, 1_000_000, dtype=np.float64) typesize = a.dtype.itemsize c_bytesobj = blosc2.compress2( a, typesize=typesize, compcode=blosc2.Codec.ZSTD, filters=[blosc2.Filter.TRUNC_PREC, blosc2.Filter.SHUFFLE], filters_meta=[20, 0], ) assert len(c_bytesobj) < (len(a) * typesize) cratio = (len(a) * typesize) / len(c_bytesobj) print("cratio: %.3f" % cratio) a_bytesobj2 = blosc2.decompress2(c_bytesobj) # The next check does not work when using truncation (obviously) #assert a_bytesobj == a_bytesobj2
res = schunk.decompress_chunk(1) assert res == bytes_obj dest = numpy.empty(buffer.shape, buffer.dtype) schunk.decompress_chunk(1, dest) assert numpy.array_equal(buffer, dest) schunk.decompress_chunk(1, memoryview(dest)) assert numpy.array_equal(buffer, dest) dest = bytearray(buffer) schunk.decompress_chunk(1, dest) assert dest == bytes_obj # Insert a chunk in the 5th position buffer = 10 * numpy.arange(200 * 1000, dtype="int32") schunk.insert_data(5, buffer, False) # Update a chunk compressing the data first buffer = 11 * numpy.arange(200 * 1000, dtype="int32") chunk = blosc2.compress2(buffer, **cparams) schunk.update_chunk(7, chunk) # Delete the 4th chunk schunk.delete_chunk(4) # Get the compressed chunk schunk.get_chunk(1) blosc2.remove_urlpath(urlpath)