def prepare_chunk(self):
assert self.buff is None
header = self.fs.read_block(self.key_path, 0, 16)
nbytes, self.cbytes, blocksize = cbuffer_sizes(header)
typesize, _shuffle, _memcpyd = cbuffer_metainfo(header)
self.buff = mmap.mmap(-1, self.cbytes)
self.buff[0:16] = header
self.nblocks = nbytes / blocksize
self.nblocks = (
int(self.nblocks)
if self.nblocks == int(self.nblocks)
else int(self.nblocks + 1)
)
if self.nblocks == 1:
self.buff = self.read_full()
return
start_points_buffer = self.fs.read_block(
self.key_path, 16, int(self.nblocks * 4)
)
self.start_points = np.frombuffer(
start_points_buffer, count=self.nblocks, dtype=np.int32
)
self.start_points_max = self.start_points.max()
self.buff[16: (16 + (self.nblocks * 4))] = start_points_buffer
self.n_per_block = blocksize / typesize
def test_compress_metainfo(dtype, use_threads):
arr = np.arange(1000, dtype=dtype)
for shuffle in Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE:
blosc.use_threads = use_threads
for cname in blosc.list_compressors():
enc = blosc.compress(arr, cname.encode(), 1, shuffle)
typesize, did_shuffle, _ = blosc.cbuffer_metainfo(enc)
assert typesize == arr.dtype.itemsize
assert did_shuffle == shuffle
def test_compress_metainfo():
for dtype in 'i1', 'i2', 'i4', 'i8':
arr = np.arange(1000, dtype=dtype)
for shuffle in Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE:
for use_threads in True, False, None:
blosc.use_threads = use_threads
for cname in blosc.list_compressors():
enc = blosc.compress(arr, cname.encode(), 1, shuffle)
typesize, did_shuffle, _ = blosc.cbuffer_metainfo(enc)
assert typesize == arr.dtype.itemsize
assert did_shuffle == shuffle
def test_compress_autoshuffle(use_threads):
arr = np.arange(8000)
for dtype in 'i1', 'i2', 'i4', 'i8', 'f2', 'f4', 'f8', 'bool', 'S10':
varr = arr.view(dtype)
blosc.use_threads = use_threads
for cname in blosc.list_compressors():
enc = blosc.compress(varr, cname.encode(), 1, Blosc.AUTOSHUFFLE)
typesize, did_shuffle, _ = blosc.cbuffer_metainfo(enc)
assert typesize == varr.dtype.itemsize
if typesize == 1:
assert did_shuffle == Blosc.BITSHUFFLE
else:
assert did_shuffle == Blosc.SHUFFLE