def write_blosc(stream, data, compression='lz4', level=5, shuffle=True): if isinstance(compression, six.string_types) and compression.startswith('blosc:'): compression = compression[6:] data = np.asanyarray(data) if data.dtype == np.dtype('O'): raise ValueError('unable to serialize: invalid dtype') if not data.flags.contiguous: raise ValueError('expected contiguous array') payload = blosc.compress_ptr( data.__array_interface__['data'][0], data.size, data.dtype.itemsize, cname=compression, clevel=level, shuffle=shuffle ) meta = { 'size': data.size * data.dtype.itemsize, 'length': len(payload), 'comp': (compression, level, int(shuffle)), 'shape': data.shape, 'dtype': flatten_dtype(data.dtype) } meta_length = write_json(stream, meta) stream.write(payload) return len(payload) + meta_length
def pack(a): # assume a.shape = (n,) raw = blosc.compress_ptr(a.__array_interface__['data'][0], a.size, a.dtype.itemsize, cname='lz4') return {'size': a.size, 'dtype': str(a.dtype), 'data': Binary(raw)}
def __reduce__(self): _clevel, _cname, _shuffle = BloscItem.config a = np.ascontiguousarray(self.array) shape, size, dtype = a.shape, a.size, a.dtype compressed = blosc.compress_ptr(a.__array_interface__['data'][0], size, typesize=dtype.itemsize, clevel=_clevel, cname=_cname, shuffle=_shuffle) return BloscItem.unpickle, (shape, dtype, compressed,)
def compress_ndarray(vectors, compress_type=COMPRESS_FASTEST) -> bytes: """ compress_ndarray(vectors[, compress_type=COMPRESS_FASTEST, nthreads=blosc.ncores]) High speed compress numpy.ndarray with multi-threading. Implement from blosc.compress Raises ------ ValueError: if size of array larger than 2147483631 bytes. Example: array with float32 have itemsize=4 and size=614400000 ((1200000, 512) at 2D array) -> total size of array: 4*614400000 == 2457600000 bytes You must split array to small pieces. """ if not isinstance(vectors, numpy.ndarray): raise TypeError("Only support numpy.ndarray type.") if vectors.dtype.itemsize == 1: warnings.warn( f"The compressor isn't effective with `{vectors.dtype}` type.") # prepare header data header = b"" # convert numpy dtype dtype_bytes = cvt_str2hex(str(vectors.dtype)) header += cvt_dec2hex(len(dtype_bytes)) + dtype_bytes # convert numpy shape for size_of_dim in vectors.shape: size_of_dim_bytes = cvt_dec2hex(size_of_dim) len_size_of_dim_bytes = 2 - len(size_of_dim_bytes) if len_size_of_dim_bytes > 0: size_of_dim_bytes = b"\x00" * len_size_of_dim_bytes + size_of_dim_bytes header += size_of_dim_bytes # add info size of header header_size = len(header) if header_size > (1 << 8): raise ValueError("Vectors too large.") header = cvt_dec2hex(header_size) + header # compress vectors compressor = "lz4" if compress_type == COMPRESS_FASTEST else "zstd" level = 1 if compress_type == COMPRESS_FASTEST else 5 buffer = blosc.compress_ptr(vectors.__array_interface__['data'][0], vectors.size, typesize=max(1, min(255, vectors.dtype.itemsize)), clevel=level, cname=compressor, shuffle=blosc.BITSHUFFLE) return header + buffer
def arraydata_to_compressed_bytes(arraydata): cbytes = blosc.compress_ptr(arraydata.__array_interface__['data'][0], arraydata.size, arraydata.dtype.itemsize, cname='zlib', shuffle=blosc.SHUFFLE) return cbytes
def test_codec(chunk, codec, filter_name, clevel): """ Compress the chunk and return tested data. Parameters ---------- chunk: bytes-like object (supporting the buffer interface) The data to be compressed. codec : string The name of the compressor used internally in Blosc. It can be any of the supported by Blosc ('blosclz', 'lz4', 'lz4hc', 'snappy', 'zlib', 'zstd' and maybe others too). filter_name : int The shuffle filter to be activated. Allowed values are blosc.NOSHUFFLE, blosc.SHUFFLE and blosc.BITSHUFFLE. clevel : int The compression level from 0 (no compression) to 9 (maximum compression). Returns ------- out: tuple The associated compression rate, compression speed and decompression speed (in GB/s). Raises ------ TypeError If bytesobj doesn't support the buffer interface. ValueError If bytesobj is too long. If typesize is not within the allowed range. If clevel is not within the allowed range. If cname is not a valid codec. """ t0 = time() c = blosc.compress_ptr(chunk.__array_interface__['data'][0], chunk.size, chunk.dtype.itemsize, clevel=clevel, shuffle=filter_name, cname=codec) tc = time() - t0 out = np.empty(chunk.size, dtype=chunk.dtype) times = [] for i in range(3): t0 = time() blosc.decompress_ptr(c, out.__array_interface__['data'][0]) times.append(time() - t0) chunk_byte_size = chunk.size * chunk.dtype.itemsize rate = chunk_byte_size / len(c) c_speed = chunk_byte_size / tc / SPEED_UNIT d_speed = chunk_byte_size / min(times) / SPEED_UNIT # print(" *** %-8s, %-10s, CL%d *** %6.4f s / %5.4f s " % # ( codec, blosc.filters[filter], clevel, tc, td), end='') # print("\tCompr. ratio: %5.1fx" % rate) return rate, c_speed, d_speed
def compression(self, a): """ data compression method Parameters ------------- a : numpy array data to be compresses """ self.data = blosc.compress_ptr(a.__array_interface__['data'][0], a.size, a.dtype.itemsize, 9, True) self.size = a.size self.dtype = a.dtype
def ndarray_to_flatnpy(data, dataname, prefix='x', compression=False, clevel=9, shuffle=1, codec='blosclz'): shuffle = (blosc.NOSHUFFLE, blosc.SHUFFLE, blosc.BITSHUFFLE)[shuffle] if compression: for i, d in enumerate(data): fname = '{}/{}_{}.blosc'.format(dataname, prefix, i) d = blosc.compress_ptr(d.__array_interface__['data'][0], d.size, d.dtype.itemsize, clevel=clevel, shuffle=shuffle, cname=codec) with open(fname, 'wb') as f: f.write(d) else: for i, d in enumerate(data): fname = '{}/{}_{}.npy'.format(dataname, prefix, i) numpy.save(fname, d)
def compress(arr, Path): #c = blosc.compress_ptr(arr.__array_interface__['data'][0], arr.size, arr.dtype.itemsize, clevel=3,cname='lz4',shuffle=blosc.SHUFFLE) c = blosc.compress_ptr(arr.__array_interface__['data'][0], arr.size, arr.dtype.itemsize, clevel=3, cname='zstd', shuffle=blosc.SHUFFLE) f = open(Path, "wb") pickle.dump((arr.shape, arr.dtype), f) f.write(c) f.close() return c, arr.shape, arr.dtype
def _server_pack_data(name, compressor='blosc', downsample=None, **compressor_args): """Pack the data in the named ISM_Buffer into bytes for transfer over the network (or other serialization). Downsample parameter: int / None. If not None, only return every nth pixel. Valid compressor values are: - None: pack raw image bytes - 'blosc': use the fast, modern BLOSC compression library - 'zlib': use older, more widely supported zlib compression compressor_args are passed to zlib.compress() or blosc.compress() directly.""" array = release_array( name ) # get the array and release it from the list of to-be-transfered arrays if downsample: array = array[::downsample, ::downsample] dtype_str = numpy.lib.format.dtype_to_descr(array.dtype) if array.flags.f_contiguous: order = 'F' elif array.flags.c_contiguous: order = 'C' else: array = numpy.asfortranarray(array) order = 'F' descr = json.dumps((dtype_str, array.shape, order)).encode('ascii') output = bytearray(struct.pack( '<H', len(descr))) # put the len of the descr in a 2-byte uint16 output += descr if compressor is None: output += memoryview(array.flatten(order=order)) elif compressor == 'zlib': has_level_arg = 'level' in compressor_args if len(compressor_args) - has_level_arg > 0: raise RuntimeError( '"level" is the only valid valid zlib compression option.') zlib_compressor_args = [compressor_args['level'] ] if has_level_arg else [] output += zlib.compress(array.flatten(order=order), *zlib_compressor_args) elif compressor == 'blosc': import blosc # because blosc.compress can't handle a memoryview, we need to use blosc.compress_ptr output += blosc.compress_ptr(array.ctypes.data, array.size, typesize=array.dtype.itemsize, **compressor_args) else: raise RuntimeError('un-recognized compressor') return output
def compression(self, a): """ data compression method Parameters ------------- a : numpy array data to be compresses """ self.data = compress_ptr(a.__array_interface__['data'][0], a.size, typesize=a.dtype.itemsize, clevel=self._compression_level, shuffle=True) self.size = a.size self.dtype = a.dtype
def dump(self, array, file): """ Store compressed `NumPy` array to an opened file handler. Makes the data C-contigious, if it is not. Also stores shape and dtype of an array. """ if not array.data.c_contiguous: array = np.ascontiguousarray(array) compressed = blosc.compress_ptr(array.__array_interface__['data'][0], array.size, array.dtype.itemsize, clevel=self.clevel, cname=self.cname, shuffle=self.shuffle) dill.dump((array.shape, array.dtype), file) file.write(compressed)
def test_codec( chunk, codec, filter, clevel ): """ Compresses the array chunk with the given codec, filter and clevel and return the compression time and rate. Parameters ---------- chunk : bytes-like object (supporting the buffer interface) The data to be compressed. codec : string The name of the compressor used internally in Blosc. It can be any of the supported by Blosc ('blosclz', 'lz4', 'lz4hc', 'snappy', 'zlib', 'zstd' and maybe others too). clevel : int The compression level from 0 (no compression) to 9 (maximum compression). shuffle : int The shuffle filter to be activated. Allowed values are blosc.NOSHUFFLE, blosc.SHUFFLE and blosc.BITSHUFFLE. Returns ------- out : tuple The associated compression time, rate and decompression time. Raises ------ TypeError If bytesobj doesn't support the buffer interface. ValueError If bytesobj is too long. If typesize is not within the allowed range. If clevel is not within the allowed range. If cname is not a valid codec. """ t0 = time() c = blosc.compress_ptr(chunk.__array_interface__['data'][0], chunk.size, chunk.dtype.itemsize, clevel = clevel, shuffle = filter, cname = codec) tc = time() - t0 out = np.empty(chunk.size, dtype = chunk.dtype) t0 = time() blosc.decompress_ptr(c, out.__array_interface__['data'][0]) td = time() - t0 rate = (chunk.size * chunk.dtype.itemsize / len(c)) assert ((chunk == out).all()) # print(" *** %-8s, %-10s, CL%d *** %6.4f s / %5.4f s " % # ( codec, blosc.filters[filter], clevel, tc, td), end='') # print("\tCompr. ratio: %5.1fx" % rate) return (rate, tc, td)
def test_decompress_ptr_exceptions(self): # make sure we do have a valid address typesize, items = 8, 8 data = [float(i) for i in range(items)] Array = ctypes.c_double * items in_array = Array(*data) c = blosc.compress_ptr(ctypes.addressof(in_array), items, typesize) out_array = ctypes.create_string_buffer(items*typesize) self.assertRaises(TypeError, blosc.decompress_ptr, 1.0, ctypes.addressof(out_array)) self.assertRaises(TypeError, blosc.decompress_ptr, ['abc'], ctypes.addressof(out_array)) self.assertRaises(TypeError, blosc.decompress_ptr, c, 1.0) self.assertRaises(TypeError, blosc.decompress_ptr, c, ['abc'])
def compress_ndarray(vectors: numpy.ndarray, compress_type=COMPRESS_FASTEST, nthreads=blosc.ncores) -> bytes: assert type(vectors) is numpy.ndarray blosc.set_nthreads(nthreads) compressor = "lz4" if compress_type == COMPRESS_FASTEST else "zstd" level = 1 if compress_type == COMPRESS_FASTEST else 5 buffer = blosc.compress_ptr(vectors.__array_interface__['data'][0], vectors.size, typesize=max(1, min(255, vectors.dtype.itemsize)), clevel=level, cname=compressor, shuffle=blosc.BITSHUFFLE) return pickle.dumps([buffer, vectors.dtype, vectors.shape])
def test_decompress_ptr_exceptions(self): # make sure we do have a valid address typesize, items = 8, 8 data = [float(i) for i in range(items)] Array = ctypes.c_double * items in_array = Array(*data) c = blosc.compress_ptr(ctypes.addressof(in_array), items, typesize) out_array = ctypes.create_string_buffer(items * typesize) self.assertRaises(TypeError, blosc.decompress_ptr, 1.0, ctypes.addressof(out_array)) self.assertRaises(TypeError, blosc.decompress_ptr, ['abc'], ctypes.addressof(out_array)) self.assertRaises(TypeError, blosc.decompress_ptr, c, 1.0) self.assertRaises(TypeError, blosc.decompress_ptr, c, ['abc'])
def _squeezed_copy(obj, clevel, cname, shuffle): """Compress arrays within dicts, tuples and lists, do not dig other objects for now """ if isinstance(obj, np.ndarray): array = np.ascontiguousarray(obj) shape, size, dtype = array.shape, array.size, array.dtype comp = blosc.compress_ptr(array.__array_interface__['data'][0], size, typesize=dtype.itemsize, clevel=clevel, cname=cname, shuffle=shuffle) return _SqueezedArray(shape, dtype, comp) tpe = type(obj) if tpe is tuple or tpe is list: return tpe(_squeezed_copy(el, clevel, cname, shuffle) for el in obj) if tpe is dict: return tpe((k, _squeezed_copy(v, clevel, cname, shuffle)) for k, v in obj.items()) return obj
def compress_ndarray(vectors, compress_type=COMPRESS_FASTEST, nthreads=blosc.ncores) -> bytes: """ compress_ndarray(vectors[, compress_type=COMPRESS_FASTEST, nthreads=blosc.ncores]) High speed compress numpy.ndarray with multi-threading. Implement from blosc.compress Raise ValueError if size of array larger than 2147483631 bytes. Example: array with float32 have itemsize=4 and size=614400000 ((1200000, 512) at 2D array) -> total size of array: 4*614400000 == 2457600000 bytes You must split array to small pieces. """ assert type(vectors) is numpy.ndarray blosc.set_nthreads(nthreads) compressor = "lz4" if compress_type == COMPRESS_FASTEST else "zstd" level = 1 if compress_type == COMPRESS_FASTEST else 5 buffer = blosc.compress_ptr(vectors.__array_interface__['data'][0], vectors.size, typesize=max(1, min(255, vectors.dtype.itemsize)), clevel=level, cname=compressor, shuffle=blosc.BITSHUFFLE) return pickle.dumps([buffer, vectors.dtype, vectors.shape])
def processArray(self, arr, attr): self.log.debug('packing array with shape: %s', str(arr.shape)) # Pack a description of the array dimensions and datatype into a packet msg_array_desc = msgpack.packb({ 'dtype': str(arr.dtype), 'shape': arr.shape }) # Compress the data from the array without making a copy (i.e. by passing a read-only # pointer to the blosc library. arr_ptr = arr.__array_interface__['data'][0] msg_array = blosc.compress_ptr(arr_ptr, arr.size, arr.dtype.itemsize, self['level'], True) # Calculate the resulting compression ratio self['ratio'] = float(len(msg_array)) / float(arr.nbytes) self.log.debug("Compressed ratio: %d/%d %.3f", arr.nbytes, len(msg_array), self['ratio']) # Pack the NDAttribute dictionary into a packet self.log.debug('packing attributes') msg_attr = msgpack.packb(attr) self.log.debug(' result dict length: %d', len(msg_attr)) # Send the packets in a multipart zmq message. self.log.debug('sending multipart message') tracker = self.zmq_socket.send_multipart( [msg_array_desc, msg_array, msg_attr], copy=False, track=True) # Wait for ZMQ to report that it has completed. We can possibly ignore this step - but then we would be # relying on the ZMQ buffering rather than our areaDetector buffers - and that is much less configurable. #self.log.debug('waiting for send to complete') #try: # tracker.wait(1.0) #except zmq.NotDone: # self.log.exception('Timeout when waiting to complete transfer: %s', str(zmq.NotDone)) # All done, ready for new frame! self.log.debug('Processing done!') return None
def _server_pack_data(name, compressor='blosc', downsample=None, **compressor_args): """Pack the data in the named ISM_Buffer into bytes for transfer over the network (or other serialization). Downsample parameter: int / None. If not None, only return every nth pixel. Valid compressor values are: - None: pack raw image bytes - 'blosc': use the fast, modern BLOSC compression library - 'zlib': use older, more widely supported zlib compression compressor_args are passed to zlib.compress() or blosc.compress() directly.""" array = release_array(name) # get the array and release it from the list of to-be-transfered arrays if downsample: array = array[::downsample, ::downsample] dtype_str = numpy.lib.format.dtype_to_descr(array.dtype) if array.flags.f_contiguous: order = 'F' elif array.flags.c_contiguous: order = 'C' else: array = numpy.asfortranarray(array) order = 'F' descr = json.dumps((dtype_str, array.shape, order)).encode('ascii') output = bytearray(struct.pack('<H', len(descr))) # put the len of the descr in a 2-byte uint16 output += descr if compressor is None: output += memoryview(array.flatten(order=order)) elif compressor == 'zlib': has_level_arg = 'level' in compressor_args if len(compressor_args) - has_level_arg > 0: raise RuntimeError('"level" is the only valid valid zlib compression option.') zlib_compressor_args = [compressor_args['level']] if has_level_arg else [] output += zlib.compress(array.flatten(order=order), *zlib_compressor_args) elif compressor == 'blosc': import blosc # because blosc.compress can't handle a memoryview, we need to use blosc.compress_ptr output += blosc.compress_ptr(array.ctypes.data, array.size, typesize=array.dtype.itemsize, **compressor_args) else: raise RuntimeError('un-recognized compressor') return output
def _compress_chunk_ptr(chunk, blosc_args): ptr, size = chunk return blosc.compress_ptr(ptr, size, **blosc_args)
print("Using *** %s *** compressor::" % cname) ctic = time.time() c = blosc.pack_array(in_, clevel=clevel, shuffle=True, cname=cname) ctoc = time.time() dtic = time.time() out = blosc.unpack_array(c) dtoc = time.time() assert ((in_ == out).all()) print(" Time for pack_array/unpack_array: %.3f/%.3f s." % \ (ctoc-ctic, dtoc-dtic), end='') print("\tCompr ratio: %.2f" % (in_.size * in_.dtype.itemsize * 1. / len(c))) ctic = time.time() c = blosc.compress_ptr(in_.__array_interface__['data'][0], in_.size, in_.dtype.itemsize, clevel=clevel, shuffle=True, cname=cname) ctoc = time.time() out = np.empty(in_.size, dtype=in_.dtype) dtic = time.time() blosc.decompress_ptr(c, out.__array_interface__['data'][0]) dtoc = time.time() assert ((in_ == out).all()) print(" Time for compress_ptr/decompress_ptr: %.3f/%.3f s." % \ (ctoc-ctic, dtoc-dtic), end='') print("\tCompr ratio: %.2f" % (in_.size * in_.dtype.itemsize * 1. / len(c)))
def decompress_ptr(): cx = blosc.compress_ptr(address, num_elements, typesize, clevel=0) blosc.decompress_ptr(cx, address)
def compress_ptr(): blosc.compress_ptr(address, num_elements, typesize, clevel=0)
def __MRCExport(input_image, header, MRCfilename, slices, endchar='<', offset=0, idxnewfile=True): ''' MRCExport private interface with a dictionary rather than a mess of function arguments. ''' if idxnewfile: # If forcing a new file we truncate it even if it already exists: fmode = 'wb' else: # Otherwise we'll just update its header and append images as required: fmode = 'rb+' with open(MRCfilename, fmode, buffering=BUFFERSIZE) as f: extendedBytes = writeMRCHeader(f, header, slices, endchar=endchar) f.seek(DEFAULT_HEADER_LEN + extendedBytes + offset) dtype = header['dtype'] if ('compressor' in header) \ and (header['compressor'] in REVERSE_COMPRESSOR_ENUM) \ and (REVERSE_COMPRESSOR_ENUM[header['compressor']]) > 0: # compressed MRCZ logger.debug('Compressing %s with compressor %s%d' % (MRCfilename, header['compressor'], header['clevel'])) applyCast = False if slices > 0: chunkSize = input_image[0].size typeSize = input_image[0].dtype.itemsize if dtype != 'uint4' and input_image[0].dtype != dtype: applyCast = True else: chunkSize = input_image[0,:,:].size typeSize = input_image.dtype.itemsize if dtype != 'uint4' and input_image.dtype != dtype: applyCast = True blosc.set_nthreads(header['n_threads']) # for small image dimensions we need to scale blocksize appropriately # so we use the available cores block_size = np.minimum(BLOSC_BLOCK, chunkSize//header['n_threads']) blosc.set_blocksize(block_size) header['packedBytes'] = 0 clevel = header['clevel'] cname = header['compressor'] # For 3D frames in lists, we need to further sub-divide each frame # into slices so that each channel is compressed seperately by # blosc. if slices > 1: deep_image = input_image # grab a reference input_image = [] for frame in deep_image: for I in range(slices): input_image.append(frame[I,:,:]) for J, frame in enumerate(input_image): if applyCast: frame = frame.astype(dtype) if frame.flags['C_CONTIGUOUS'] and frame.flags['ALIGNED']: # Use pointer compressedData = blosc.compress_ptr(frame.__array_interface__['data'][0], frame.size, typeSize, clevel=header['clevel'], shuffle=blosc.BITSHUFFLE, cname=header['compressor']) else: # Use tobytes, which is slower in benchmarking compressedData = blosc.compress(frame.tobytes(), typeSize, clevel=clevel, shuffle=blosc.BITSHUFFLE, cname=cname) f.write(compressedData) header['packedBytes'] += len(compressedData) # Rewind and write out the total compressed size f.seek(144) np.int64(header['packedBytes']).astype(endchar + 'i8').tofile(f) else: # vanilla MRC if slices > 0: if dtype != 'uint4' and dtype != input_image[0].dtype: for z_slice in input_image: z_slice.astype(dtype).tofile(f) else: for z_slice in input_image: z_slice.tofile(f) else: if dtype != 'uint4' and dtype != input_image.dtype: input_image = input_image.astype(dtype) input_image.tofile(f) return
def compressSlice(args): """ args = (numpy array address, array_size, item_size, bytesList, bytesIndex) """ args[3][args[4]] = blosc.compress_ptr( args[0], args[1], args[2], \ clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
# Let's try instead just pool threads... #poolThreads = np.arange( 1, maxThreads+1 ) #bloscThreads = np.ones_like( poolThreads ) solo_times = np.zeros_like(poolThreads, dtype='float64') solo_unlocked_times = np.zeros_like(poolThreads, dtype='float64') locked_times = np.zeros_like(poolThreads, dtype='float64') unlocked_times = np.zeros_like(poolThreads, dtype='float64') for J in np.arange(nRuns): print("Run %d of %d" % (J + 1, nRuns)) blosc.set_releasegil(False) for I in np.arange(len(poolThreads)): t1 = time.time() blosc.set_nthreads(bloscThreads[I]) blosc.compress_ptr( stack.__array_interface__['data'][0], stack.size, stack.dtype.itemsize, \ clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR ) solo_times[I] += time.time() - t1 blosc.set_releasegil(True) for I in np.arange(len(poolThreads)): t2 = time.time() blosc.set_nthreads(bloscThreads[I]) blosc.compress_ptr( stack.__array_interface__['data'][0], stack.size, stack.dtype.itemsize, \ clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR ) solo_unlocked_times[I] += time.time() - t2 blosc.set_releasegil(True) for I in np.arange(len(poolThreads)): t3 = time.time() compressStack(stack, blosc_threads=bloscThreads[I],
def compressSlice( args ): """ args = (numpy array address, array_size, item_size, bytesList, bytesIndex) """ args[3][args[4]] = blosc.compress_ptr( args[0], args[1], args[2], \ clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
arrays = [None]*3 labels = [None]*3 arrays[0] = np.arange(N, dtype=np.int64) labels[0] = "the arange linear distribution" arrays[1] = np.linspace(0, 1000, N) labels[1] = "the linspace linear distribution" arrays[2] = np.random.random_integers(0, 1000, N) labels[2] = "the random distribution" tic = time.time() out_ = np.copy(arrays[0]) toc = time.time() print(" *** np.copy() **** Time for memcpy(): %.3f s" % (toc-tic,)) for (in_, label) in zip(arrays, labels): print("\n*** %s ***" % label) for cname in blosc.compressor_list(): ctic = time.time() c = blosc.compress_ptr(in_.__array_interface__['data'][0], in_.size, in_.dtype.itemsize, clevel=clevel, shuffle=True, cname=cname) ctoc = time.time() out = np.empty(in_.size, dtype=in_.dtype) dtic = time.time() blosc.decompress_ptr(c, out.__array_interface__['data'][0]) dtoc = time.time() assert((in_ == out).all()) print(" *** %-8s *** Time for comp/decomp: %.3f/%.3f s." % \ (cname, ctoc-ctic, dtoc-dtic), end='') print("\tCompr ratio: %6.2f" % (in_.size*in_.dtype.itemsize*1. / len(c)))
def _tensor_dump(info): return blosc.compress_ptr(info['data_ptr'], info['numel'], info['element_size'])
# Let's try instead just pool threads... #poolThreads = np.arange( 1, maxThreads+1 ) #bloscThreads = np.ones_like( poolThreads ) solo_times = np.zeros_like( poolThreads, dtype='float64' ) solo_unlocked_times = np.zeros_like( poolThreads, dtype='float64' ) locked_times = np.zeros_like( poolThreads, dtype='float64' ) unlocked_times = np.zeros_like( poolThreads, dtype='float64' ) for J in np.arange(nRuns): print( "Run %d of %d" % (J+1, nRuns) ) blosc.set_releasegil(False) for I in np.arange( len(poolThreads) ): t1 = time.time() blosc.set_nthreads( bloscThreads[I] ) blosc.compress_ptr( stack.__array_interface__['data'][0], stack.size, stack.dtype.itemsize, \ clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR ) solo_times[I] += time.time() - t1 blosc.set_releasegil(True) for I in np.arange( len(poolThreads) ): t2 = time.time() blosc.set_nthreads( bloscThreads[I] ) blosc.compress_ptr( stack.__array_interface__['data'][0], stack.size, stack.dtype.itemsize, \ clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR ) solo_unlocked_times[I] += time.time() - t2 blosc.set_releasegil(True) for I in np.arange( len(poolThreads) ): t3 = time.time() compressStack( stack, blosc_threads=bloscThreads[I], pool_threads=poolThreads[I] ) unlocked_times[I] += time.time() - t3
import asyncio import time import cbor import blosc import asyncio_redis from asyncio_redis.encoders import BaseEncoder class CborEncoder(BaseEncoder): native_type = object def encode_from_native(self, data): return cbor.dumps(data) def decode_to_native(self, data): return cbor.loads(data) compress = lambda in_: (in_.size, in_.dtype, blosc.compress_ptr(in_.__array_interface__['data'][0], in_.size, in_.dtype.itemsize, clevel = 1, shuffle = blosc.BITSHUFFLE, cname = 'lz4')) def decompress(size, dtype, data): out = np.empty(size, dtype) blosc.decompress_ptr(data, out.__array_interface__['data'][0]) return out def packed_bytes_to_iq(samples, out = None): if out is None: iq = np.empty(len(samples)//2, 'complex64') else: iq = out bytes_np = np.ctypeslib.as_array(samples) iq.real, iq.imag = bytes_np[::2], bytes_np[1::2] iq /= (255/2) iq -= (1 + 1j)