def prepare_chunk(self): assert self.buff is None header = self.fs.read_block(self.key_path, 0, 16) nbytes, self.cbytes, blocksize = cbuffer_sizes(header) typesize, _shuffle, _memcpyd = cbuffer_metainfo(header) self.buff = mmap.mmap(-1, self.cbytes) self.buff[0:16] = header self.nblocks = nbytes / blocksize self.nblocks = ( int(self.nblocks) if self.nblocks == int(self.nblocks) else int(self.nblocks + 1) ) if self.nblocks == 1: self.buff = self.read_full() return start_points_buffer = self.fs.read_block( self.key_path, 16, int(self.nblocks * 4) ) self.start_points = np.frombuffer( start_points_buffer, count=self.nblocks, dtype=np.int32 ) self.start_points_max = self.start_points.max() self.buff[16: (16 + (self.nblocks * 4))] = start_points_buffer self.n_per_block = blocksize / typesize
def test_compress_metainfo(dtype, use_threads): arr = np.arange(1000, dtype=dtype) for shuffle in Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE: blosc.use_threads = use_threads for cname in blosc.list_compressors(): enc = blosc.compress(arr, cname.encode(), 1, shuffle) typesize, did_shuffle, _ = blosc.cbuffer_metainfo(enc) assert typesize == arr.dtype.itemsize assert did_shuffle == shuffle
def test_compress_metainfo(): for dtype in 'i1', 'i2', 'i4', 'i8': arr = np.arange(1000, dtype=dtype) for shuffle in Blosc.NOSHUFFLE, Blosc.SHUFFLE, Blosc.BITSHUFFLE: for use_threads in True, False, None: blosc.use_threads = use_threads for cname in blosc.list_compressors(): enc = blosc.compress(arr, cname.encode(), 1, shuffle) typesize, did_shuffle, _ = blosc.cbuffer_metainfo(enc) assert typesize == arr.dtype.itemsize assert did_shuffle == shuffle
def test_compress_autoshuffle(use_threads): arr = np.arange(8000) for dtype in 'i1', 'i2', 'i4', 'i8', 'f2', 'f4', 'f8', 'bool', 'S10': varr = arr.view(dtype) blosc.use_threads = use_threads for cname in blosc.list_compressors(): enc = blosc.compress(varr, cname.encode(), 1, Blosc.AUTOSHUFFLE) typesize, did_shuffle, _ = blosc.cbuffer_metainfo(enc) assert typesize == varr.dtype.itemsize if typesize == 1: assert did_shuffle == Blosc.BITSHUFFLE else: assert did_shuffle == Blosc.SHUFFLE