def write_blosc(stream, data, compression='lz4', level=5, shuffle=True):
if isinstance(compression, six.string_types) and compression.startswith('blosc:'):
compression = compression[6:]
data = np.asanyarray(data)
if data.dtype == np.dtype('O'):
raise ValueError('unable to serialize: invalid dtype')
if not data.flags.contiguous:
raise ValueError('expected contiguous array')
payload = blosc.compress_ptr(
data.__array_interface__['data'][0],
data.size,
data.dtype.itemsize,
cname=compression,
clevel=level,
shuffle=shuffle
)
meta = {
'size': data.size * data.dtype.itemsize,
'length': len(payload),
'comp': (compression, level, int(shuffle)),
'shape': data.shape,
'dtype': flatten_dtype(data.dtype)
}
meta_length = write_json(stream, meta)
stream.write(payload)
return len(payload) + meta_length
def pack(a):
# assume a.shape = (n,)
raw = blosc.compress_ptr(a.__array_interface__['data'][0],
a.size,
a.dtype.itemsize,
cname='lz4')
return {'size': a.size, 'dtype': str(a.dtype), 'data': Binary(raw)}
def __reduce__(self):
_clevel, _cname, _shuffle = BloscItem.config
a = np.ascontiguousarray(self.array)
shape, size, dtype = a.shape, a.size, a.dtype
compressed = blosc.compress_ptr(a.__array_interface__['data'][0], size,
typesize=dtype.itemsize, clevel=_clevel,
cname=_cname, shuffle=_shuffle)
return BloscItem.unpickle, (shape, dtype, compressed,)
def compress_ndarray(vectors, compress_type=COMPRESS_FASTEST) -> bytes:
"""
compress_ndarray(vectors[, compress_type=COMPRESS_FASTEST, nthreads=blosc.ncores])
High speed compress numpy.ndarray with multi-threading. Implement from blosc.compress
Raises
------
ValueError:
if size of array larger than 2147483631 bytes.
Example: array with float32 have itemsize=4 and size=614400000 ((1200000, 512) at 2D array)
-> total size of array: 4*614400000 == 2457600000 bytes
You must split array to small pieces.
"""
if not isinstance(vectors, numpy.ndarray):
raise TypeError("Only support numpy.ndarray type.")
if vectors.dtype.itemsize == 1:
warnings.warn(
f"The compressor isn't effective with `{vectors.dtype}` type.")
# prepare header data
header = b""
# convert numpy dtype
dtype_bytes = cvt_str2hex(str(vectors.dtype))
header += cvt_dec2hex(len(dtype_bytes)) + dtype_bytes
# convert numpy shape
for size_of_dim in vectors.shape:
size_of_dim_bytes = cvt_dec2hex(size_of_dim)
len_size_of_dim_bytes = 2 - len(size_of_dim_bytes)
if len_size_of_dim_bytes > 0:
size_of_dim_bytes = b"\x00" * len_size_of_dim_bytes + size_of_dim_bytes
header += size_of_dim_bytes
# add info size of header
header_size = len(header)
if header_size > (1 << 8):
raise ValueError("Vectors too large.")
header = cvt_dec2hex(header_size) + header
# compress vectors
compressor = "lz4" if compress_type == COMPRESS_FASTEST else "zstd"
level = 1 if compress_type == COMPRESS_FASTEST else 5
buffer = blosc.compress_ptr(vectors.__array_interface__['data'][0],
vectors.size,
typesize=max(1, min(255,
vectors.dtype.itemsize)),
clevel=level,
cname=compressor,
shuffle=blosc.BITSHUFFLE)
return header + buffer
def arraydata_to_compressed_bytes(arraydata):
cbytes = blosc.compress_ptr(arraydata.__array_interface__['data'][0],
arraydata.size,
arraydata.dtype.itemsize,
cname='zlib',
shuffle=blosc.SHUFFLE)
return cbytes
def test_codec(chunk, codec, filter_name, clevel):
"""
Compress the chunk and return tested data.
Parameters
----------
chunk: bytes-like object (supporting the buffer interface)
The data to be compressed.
codec : string
The name of the compressor used internally in Blosc. It can be
any of the supported by Blosc ('blosclz', 'lz4', 'lz4hc',
'snappy', 'zlib', 'zstd' and maybe others too).
filter_name : int
The shuffle filter to be activated. Allowed values are
blosc.NOSHUFFLE, blosc.SHUFFLE and blosc.BITSHUFFLE.
clevel : int
The compression level from 0 (no compression) to 9
(maximum compression).
Returns
-------
out: tuple
The associated compression rate, compression speed and
decompression speed (in GB/s).
Raises
------
TypeError
If bytesobj doesn't support the buffer interface.
ValueError
If bytesobj is too long.
If typesize is not within the allowed range.
If clevel is not within the allowed range.
If cname is not a valid codec.
"""
t0 = time()
c = blosc.compress_ptr(chunk.__array_interface__['data'][0],
chunk.size,
chunk.dtype.itemsize,
clevel=clevel,
shuffle=filter_name,
cname=codec)
tc = time() - t0
out = np.empty(chunk.size, dtype=chunk.dtype)
times = []
for i in range(3):
t0 = time()
blosc.decompress_ptr(c, out.__array_interface__['data'][0])
times.append(time() - t0)
chunk_byte_size = chunk.size * chunk.dtype.itemsize
rate = chunk_byte_size / len(c)
c_speed = chunk_byte_size / tc / SPEED_UNIT
d_speed = chunk_byte_size / min(times) / SPEED_UNIT
# print(" *** %-8s, %-10s, CL%d *** %6.4f s / %5.4f s " %
# ( codec, blosc.filters[filter], clevel, tc, td), end='')
# print("\tCompr. ratio: %5.1fx" % rate)
return rate, c_speed, d_speed
def compression(self, a):
""" data compression method
Parameters
-------------
a : numpy array
data to be compresses
"""
self.data = blosc.compress_ptr(a.__array_interface__['data'][0], a.size, a.dtype.itemsize, 9, True)
self.size = a.size
self.dtype = a.dtype
def ndarray_to_flatnpy(data, dataname, prefix='x',
compression=False, clevel=9, shuffle=1, codec='blosclz'):
shuffle = (blosc.NOSHUFFLE, blosc.SHUFFLE, blosc.BITSHUFFLE)[shuffle]
if compression:
for i, d in enumerate(data):
fname = '{}/{}_{}.blosc'.format(dataname, prefix, i)
d = blosc.compress_ptr(d.__array_interface__['data'][0], d.size, d.dtype.itemsize, clevel=clevel, shuffle=shuffle, cname=codec)
with open(fname, 'wb') as f:
f.write(d)
else:
for i, d in enumerate(data):
fname = '{}/{}_{}.npy'.format(dataname, prefix, i)
numpy.save(fname, d)
def compress(arr, Path):
#c = blosc.compress_ptr(arr.__array_interface__['data'][0], arr.size, arr.dtype.itemsize, clevel=3,cname='lz4',shuffle=blosc.SHUFFLE)
c = blosc.compress_ptr(arr.__array_interface__['data'][0],
arr.size,
arr.dtype.itemsize,
clevel=3,
cname='zstd',
shuffle=blosc.SHUFFLE)
f = open(Path, "wb")
pickle.dump((arr.shape, arr.dtype), f)
f.write(c)
f.close()
return c, arr.shape, arr.dtype
def _server_pack_data(name,
compressor='blosc',
downsample=None,
**compressor_args):
"""Pack the data in the named ISM_Buffer into bytes for transfer over
the network (or other serialization).
Downsample parameter: int / None. If not None, only return every nth pixel.
Valid compressor values are:
- None: pack raw image bytes
- 'blosc': use the fast, modern BLOSC compression library
- 'zlib': use older, more widely supported zlib compression
compressor_args are passed to zlib.compress() or blosc.compress() directly."""
array = release_array(
name
) # get the array and release it from the list of to-be-transfered arrays
if downsample:
array = array[::downsample, ::downsample]
dtype_str = numpy.lib.format.dtype_to_descr(array.dtype)
if array.flags.f_contiguous:
order = 'F'
elif array.flags.c_contiguous:
order = 'C'
else:
array = numpy.asfortranarray(array)
order = 'F'
descr = json.dumps((dtype_str, array.shape, order)).encode('ascii')
output = bytearray(struct.pack(
'<H', len(descr))) # put the len of the descr in a 2-byte uint16
output += descr
if compressor is None:
output += memoryview(array.flatten(order=order))
elif compressor == 'zlib':
has_level_arg = 'level' in compressor_args
if len(compressor_args) - has_level_arg > 0:
raise RuntimeError(
'"level" is the only valid valid zlib compression option.')
zlib_compressor_args = [compressor_args['level']
] if has_level_arg else []
output += zlib.compress(array.flatten(order=order),
*zlib_compressor_args)
elif compressor == 'blosc':
import blosc
# because blosc.compress can't handle a memoryview, we need to use blosc.compress_ptr
output += blosc.compress_ptr(array.ctypes.data,
array.size,
typesize=array.dtype.itemsize,
**compressor_args)
else:
raise RuntimeError('un-recognized compressor')
return output
def compression(self, a):
""" data compression method
Parameters
-------------
a : numpy array
data to be compresses
"""
self.data = compress_ptr(a.__array_interface__['data'][0],
a.size, typesize=a.dtype.itemsize,
clevel=self._compression_level,
shuffle=True)
self.size = a.size
self.dtype = a.dtype
def dump(self, array, file):
""" Store compressed `NumPy` array to an opened file handler. Makes the data C-contigious, if it is not.
Also stores shape and dtype of an array.
"""
if not array.data.c_contiguous:
array = np.ascontiguousarray(array)
compressed = blosc.compress_ptr(array.__array_interface__['data'][0],
array.size, array.dtype.itemsize,
clevel=self.clevel, cname=self.cname,
shuffle=self.shuffle)
dill.dump((array.shape, array.dtype), file)
file.write(compressed)
def test_codec( chunk, codec, filter, clevel ):
"""
Compresses the array chunk with the given codec, filter and clevel
and return the compression time and rate.
Parameters
----------
chunk : bytes-like object (supporting the buffer interface)
The data to be compressed.
codec : string
The name of the compressor used internally in Blosc. It can be
any of the supported by Blosc ('blosclz', 'lz4', 'lz4hc',
'snappy', 'zlib', 'zstd' and maybe others too).
clevel : int
The compression level from 0 (no compression) to 9
(maximum compression).
shuffle : int
The shuffle filter to be activated. Allowed values are
blosc.NOSHUFFLE, blosc.SHUFFLE and blosc.BITSHUFFLE.
Returns
-------
out : tuple
The associated compression time, rate and decompression time.
Raises
------
TypeError
If bytesobj doesn't support the buffer interface.
ValueError
If bytesobj is too long.
If typesize is not within the allowed range.
If clevel is not within the allowed range.
If cname is not a valid codec.
"""
t0 = time()
c = blosc.compress_ptr(chunk.__array_interface__['data'][0],
chunk.size, chunk.dtype.itemsize,
clevel = clevel, shuffle = filter, cname = codec)
tc = time() - t0
out = np.empty(chunk.size, dtype = chunk.dtype)
t0 = time()
blosc.decompress_ptr(c, out.__array_interface__['data'][0])
td = time() - t0
rate = (chunk.size * chunk.dtype.itemsize / len(c))
assert ((chunk == out).all())
# print(" *** %-8s, %-10s, CL%d *** %6.4f s / %5.4f s " %
# ( codec, blosc.filters[filter], clevel, tc, td), end='')
# print("\tCompr. ratio: %5.1fx" % rate)
return (rate, tc, td)
def compression(self, a):
""" data compression method
Parameters
-------------
a : numpy array
data to be compresses
"""
self.data = compress_ptr(a.__array_interface__['data'][0],
a.size,
typesize=a.dtype.itemsize,
clevel=self._compression_level,
shuffle=True)
self.size = a.size
self.dtype = a.dtype
def test_decompress_ptr_exceptions(self):
# make sure we do have a valid address
typesize, items = 8, 8
data = [float(i) for i in range(items)]
Array = ctypes.c_double * items
in_array = Array(*data)
c = blosc.compress_ptr(ctypes.addressof(in_array), items, typesize)
out_array = ctypes.create_string_buffer(items*typesize)
self.assertRaises(TypeError, blosc.decompress_ptr, 1.0,
ctypes.addressof(out_array))
self.assertRaises(TypeError, blosc.decompress_ptr, ['abc'],
ctypes.addressof(out_array))
self.assertRaises(TypeError, blosc.decompress_ptr, c, 1.0)
self.assertRaises(TypeError, blosc.decompress_ptr, c, ['abc'])
def compress_ndarray(vectors: numpy.ndarray,
compress_type=COMPRESS_FASTEST,
nthreads=blosc.ncores) -> bytes:
assert type(vectors) is numpy.ndarray
blosc.set_nthreads(nthreads)
compressor = "lz4" if compress_type == COMPRESS_FASTEST else "zstd"
level = 1 if compress_type == COMPRESS_FASTEST else 5
buffer = blosc.compress_ptr(vectors.__array_interface__['data'][0],
vectors.size,
typesize=max(1, min(255,
vectors.dtype.itemsize)),
clevel=level,
cname=compressor,
shuffle=blosc.BITSHUFFLE)
return pickle.dumps([buffer, vectors.dtype, vectors.shape])
def test_decompress_ptr_exceptions(self):
# make sure we do have a valid address
typesize, items = 8, 8
data = [float(i) for i in range(items)]
Array = ctypes.c_double * items
in_array = Array(*data)
c = blosc.compress_ptr(ctypes.addressof(in_array), items, typesize)
out_array = ctypes.create_string_buffer(items * typesize)
self.assertRaises(TypeError, blosc.decompress_ptr, 1.0,
ctypes.addressof(out_array))
self.assertRaises(TypeError, blosc.decompress_ptr, ['abc'],
ctypes.addressof(out_array))
self.assertRaises(TypeError, blosc.decompress_ptr, c, 1.0)
self.assertRaises(TypeError, blosc.decompress_ptr, c, ['abc'])
def _squeezed_copy(obj, clevel, cname, shuffle):
"""Compress arrays within dicts, tuples and lists, do not dig other objects for now
"""
if isinstance(obj, np.ndarray):
array = np.ascontiguousarray(obj)
shape, size, dtype = array.shape, array.size, array.dtype
comp = blosc.compress_ptr(array.__array_interface__['data'][0],
size,
typesize=dtype.itemsize,
clevel=clevel,
cname=cname,
shuffle=shuffle)
return _SqueezedArray(shape, dtype, comp)
tpe = type(obj)
if tpe is tuple or tpe is list:
return tpe(_squeezed_copy(el, clevel, cname, shuffle) for el in obj)
if tpe is dict:
return tpe((k, _squeezed_copy(v, clevel, cname, shuffle))
for k, v in obj.items())
return obj
def compress_ndarray(vectors, compress_type=COMPRESS_FASTEST, nthreads=blosc.ncores) -> bytes:
"""
compress_ndarray(vectors[, compress_type=COMPRESS_FASTEST, nthreads=blosc.ncores])
High speed compress numpy.ndarray with multi-threading. Implement from blosc.compress
Raise ValueError if size of array larger than 2147483631 bytes.
Example: array with float32 have itemsize=4 and size=614400000 ((1200000, 512) at 2D array)
-> total size of array: 4*614400000 == 2457600000 bytes
You must split array to small pieces.
"""
assert type(vectors) is numpy.ndarray
blosc.set_nthreads(nthreads)
compressor = "lz4" if compress_type == COMPRESS_FASTEST else "zstd"
level = 1 if compress_type == COMPRESS_FASTEST else 5
buffer = blosc.compress_ptr(vectors.__array_interface__['data'][0], vectors.size,
typesize=max(1, min(255, vectors.dtype.itemsize)),
clevel=level, cname=compressor, shuffle=blosc.BITSHUFFLE)
return pickle.dumps([buffer, vectors.dtype, vectors.shape])
def processArray(self, arr, attr):
self.log.debug('packing array with shape: %s', str(arr.shape))
# Pack a description of the array dimensions and datatype into a packet
msg_array_desc = msgpack.packb({
'dtype': str(arr.dtype),
'shape': arr.shape
})
# Compress the data from the array without making a copy (i.e. by passing a read-only
# pointer to the blosc library.
arr_ptr = arr.__array_interface__['data'][0]
msg_array = blosc.compress_ptr(arr_ptr, arr.size, arr.dtype.itemsize,
self['level'], True)
# Calculate the resulting compression ratio
self['ratio'] = float(len(msg_array)) / float(arr.nbytes)
self.log.debug("Compressed ratio: %d/%d %.3f", arr.nbytes,
len(msg_array), self['ratio'])
# Pack the NDAttribute dictionary into a packet
self.log.debug('packing attributes')
msg_attr = msgpack.packb(attr)
self.log.debug(' result dict length: %d', len(msg_attr))
# Send the packets in a multipart zmq message.
self.log.debug('sending multipart message')
tracker = self.zmq_socket.send_multipart(
[msg_array_desc, msg_array, msg_attr], copy=False, track=True)
# Wait for ZMQ to report that it has completed. We can possibly ignore this step - but then we would be
# relying on the ZMQ buffering rather than our areaDetector buffers - and that is much less configurable.
#self.log.debug('waiting for send to complete')
#try:
# tracker.wait(1.0)
#except zmq.NotDone:
# self.log.exception('Timeout when waiting to complete transfer: %s', str(zmq.NotDone))
# All done, ready for new frame!
self.log.debug('Processing done!')
return None
def _server_pack_data(name, compressor='blosc', downsample=None, **compressor_args):
"""Pack the data in the named ISM_Buffer into bytes for transfer over
the network (or other serialization).
Downsample parameter: int / None. If not None, only return every nth pixel.
Valid compressor values are:
- None: pack raw image bytes
- 'blosc': use the fast, modern BLOSC compression library
- 'zlib': use older, more widely supported zlib compression
compressor_args are passed to zlib.compress() or blosc.compress() directly."""
array = release_array(name) # get the array and release it from the list of to-be-transfered arrays
if downsample:
array = array[::downsample, ::downsample]
dtype_str = numpy.lib.format.dtype_to_descr(array.dtype)
if array.flags.f_contiguous:
order = 'F'
elif array.flags.c_contiguous:
order = 'C'
else:
array = numpy.asfortranarray(array)
order = 'F'
descr = json.dumps((dtype_str, array.shape, order)).encode('ascii')
output = bytearray(struct.pack('<H', len(descr))) # put the len of the descr in a 2-byte uint16
output += descr
if compressor is None:
output += memoryview(array.flatten(order=order))
elif compressor == 'zlib':
has_level_arg = 'level' in compressor_args
if len(compressor_args) - has_level_arg > 0:
raise RuntimeError('"level" is the only valid valid zlib compression option.')
zlib_compressor_args = [compressor_args['level']] if has_level_arg else []
output += zlib.compress(array.flatten(order=order), *zlib_compressor_args)
elif compressor == 'blosc':
import blosc
# because blosc.compress can't handle a memoryview, we need to use blosc.compress_ptr
output += blosc.compress_ptr(array.ctypes.data, array.size, typesize=array.dtype.itemsize, **compressor_args)
else:
raise RuntimeError('un-recognized compressor')
return output
def _compress_chunk_ptr(chunk, blosc_args):
ptr, size = chunk
return blosc.compress_ptr(ptr, size, **blosc_args)
print("Using *** %s *** compressor::" % cname)
ctic = time.time()
c = blosc.pack_array(in_, clevel=clevel, shuffle=True, cname=cname)
ctoc = time.time()
dtic = time.time()
out = blosc.unpack_array(c)
dtoc = time.time()
assert ((in_ == out).all())
print(" Time for pack_array/unpack_array: %.3f/%.3f s." % \
(ctoc-ctic, dtoc-dtic), end='')
print("\tCompr ratio: %.2f" %
(in_.size * in_.dtype.itemsize * 1. / len(c)))
ctic = time.time()
c = blosc.compress_ptr(in_.__array_interface__['data'][0],
in_.size,
in_.dtype.itemsize,
clevel=clevel,
shuffle=True,
cname=cname)
ctoc = time.time()
out = np.empty(in_.size, dtype=in_.dtype)
dtic = time.time()
blosc.decompress_ptr(c, out.__array_interface__['data'][0])
dtoc = time.time()
assert ((in_ == out).all())
print(" Time for compress_ptr/decompress_ptr: %.3f/%.3f s." % \
(ctoc-ctic, dtoc-dtic), end='')
print("\tCompr ratio: %.2f" %
(in_.size * in_.dtype.itemsize * 1. / len(c)))
def decompress_ptr():
cx = blosc.compress_ptr(address, num_elements, typesize, clevel=0)
blosc.decompress_ptr(cx, address)
def compress_ptr():
blosc.compress_ptr(address, num_elements, typesize, clevel=0)
def compress_ptr():
blosc.compress_ptr(address, num_elements, typesize, clevel=0)
def _compress_chunk_ptr(chunk, blosc_args):
ptr, size = chunk
return blosc.compress_ptr(ptr, size, **blosc_args)
def __MRCExport(input_image, header, MRCfilename, slices,
endchar='<', offset=0, idxnewfile=True):
'''
MRCExport private interface with a dictionary rather than a mess of function
arguments.
'''
if idxnewfile: # If forcing a new file we truncate it even if it already exists:
fmode = 'wb'
else: # Otherwise we'll just update its header and append images as required:
fmode = 'rb+'
with open(MRCfilename, fmode, buffering=BUFFERSIZE) as f:
extendedBytes = writeMRCHeader(f, header, slices, endchar=endchar)
f.seek(DEFAULT_HEADER_LEN + extendedBytes + offset)
dtype = header['dtype']
if ('compressor' in header) \
and (header['compressor'] in REVERSE_COMPRESSOR_ENUM) \
and (REVERSE_COMPRESSOR_ENUM[header['compressor']]) > 0:
# compressed MRCZ
logger.debug('Compressing %s with compressor %s%d' %
(MRCfilename, header['compressor'], header['clevel']))
applyCast = False
if slices > 0:
chunkSize = input_image[0].size
typeSize = input_image[0].dtype.itemsize
if dtype != 'uint4' and input_image[0].dtype != dtype:
applyCast = True
else:
chunkSize = input_image[0,:,:].size
typeSize = input_image.dtype.itemsize
if dtype != 'uint4' and input_image.dtype != dtype:
applyCast = True
blosc.set_nthreads(header['n_threads'])
# for small image dimensions we need to scale blocksize appropriately
# so we use the available cores
block_size = np.minimum(BLOSC_BLOCK, chunkSize//header['n_threads'])
blosc.set_blocksize(block_size)
header['packedBytes'] = 0
clevel = header['clevel']
cname = header['compressor']
# For 3D frames in lists, we need to further sub-divide each frame
# into slices so that each channel is compressed seperately by
# blosc.
if slices > 1:
deep_image = input_image # grab a reference
input_image = []
for frame in deep_image:
for I in range(slices):
input_image.append(frame[I,:,:])
for J, frame in enumerate(input_image):
if applyCast:
frame = frame.astype(dtype)
if frame.flags['C_CONTIGUOUS'] and frame.flags['ALIGNED']:
# Use pointer
compressedData = blosc.compress_ptr(frame.__array_interface__['data'][0],
frame.size,
typeSize,
clevel=header['clevel'],
shuffle=blosc.BITSHUFFLE,
cname=header['compressor'])
else:
# Use tobytes, which is slower in benchmarking
compressedData = blosc.compress(frame.tobytes(),
typeSize,
clevel=clevel,
shuffle=blosc.BITSHUFFLE,
cname=cname)
f.write(compressedData)
header['packedBytes'] += len(compressedData)
# Rewind and write out the total compressed size
f.seek(144)
np.int64(header['packedBytes']).astype(endchar + 'i8').tofile(f)
else: # vanilla MRC
if slices > 0:
if dtype != 'uint4' and dtype != input_image[0].dtype:
for z_slice in input_image:
z_slice.astype(dtype).tofile(f)
else:
for z_slice in input_image:
z_slice.tofile(f)
else:
if dtype != 'uint4' and dtype != input_image.dtype:
input_image = input_image.astype(dtype)
input_image.tofile(f)
return
def compressSlice(args):
"""
args = (numpy array address, array_size, item_size, bytesList, bytesIndex)
"""
args[3][args[4]] = blosc.compress_ptr( args[0], args[1], args[2], \
clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
# Let's try instead just pool threads...
#poolThreads = np.arange( 1, maxThreads+1 )
#bloscThreads = np.ones_like( poolThreads )
solo_times = np.zeros_like(poolThreads, dtype='float64')
solo_unlocked_times = np.zeros_like(poolThreads, dtype='float64')
locked_times = np.zeros_like(poolThreads, dtype='float64')
unlocked_times = np.zeros_like(poolThreads, dtype='float64')
for J in np.arange(nRuns):
print("Run %d of %d" % (J + 1, nRuns))
blosc.set_releasegil(False)
for I in np.arange(len(poolThreads)):
t1 = time.time()
blosc.set_nthreads(bloscThreads[I])
blosc.compress_ptr( stack.__array_interface__['data'][0], stack.size, stack.dtype.itemsize, \
clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
solo_times[I] += time.time() - t1
blosc.set_releasegil(True)
for I in np.arange(len(poolThreads)):
t2 = time.time()
blosc.set_nthreads(bloscThreads[I])
blosc.compress_ptr( stack.__array_interface__['data'][0], stack.size, stack.dtype.itemsize, \
clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
solo_unlocked_times[I] += time.time() - t2
blosc.set_releasegil(True)
for I in np.arange(len(poolThreads)):
t3 = time.time()
compressStack(stack,
blosc_threads=bloscThreads[I],
def compressSlice( args ):
"""
args = (numpy array address, array_size, item_size, bytesList, bytesIndex)
"""
args[3][args[4]] = blosc.compress_ptr( args[0], args[1], args[2], \
clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
arrays = [None]*3
labels = [None]*3
arrays[0] = np.arange(N, dtype=np.int64)
labels[0] = "the arange linear distribution"
arrays[1] = np.linspace(0, 1000, N)
labels[1] = "the linspace linear distribution"
arrays[2] = np.random.random_integers(0, 1000, N)
labels[2] = "the random distribution"
tic = time.time()
out_ = np.copy(arrays[0])
toc = time.time()
print(" *** np.copy() **** Time for memcpy(): %.3f s" % (toc-tic,))
for (in_, label) in zip(arrays, labels):
print("\n*** %s ***" % label)
for cname in blosc.compressor_list():
ctic = time.time()
c = blosc.compress_ptr(in_.__array_interface__['data'][0],
in_.size, in_.dtype.itemsize,
clevel=clevel, shuffle=True, cname=cname)
ctoc = time.time()
out = np.empty(in_.size, dtype=in_.dtype)
dtic = time.time()
blosc.decompress_ptr(c, out.__array_interface__['data'][0])
dtoc = time.time()
assert((in_ == out).all())
print(" *** %-8s *** Time for comp/decomp: %.3f/%.3f s." % \
(cname, ctoc-ctic, dtoc-dtic), end='')
print("\tCompr ratio: %6.2f" % (in_.size*in_.dtype.itemsize*1. / len(c)))
def _tensor_dump(info):
return blosc.compress_ptr(info['data_ptr'], info['numel'],
info['element_size'])
# Let's try instead just pool threads...
#poolThreads = np.arange( 1, maxThreads+1 )
#bloscThreads = np.ones_like( poolThreads )
solo_times = np.zeros_like( poolThreads, dtype='float64' )
solo_unlocked_times = np.zeros_like( poolThreads, dtype='float64' )
locked_times = np.zeros_like( poolThreads, dtype='float64' )
unlocked_times = np.zeros_like( poolThreads, dtype='float64' )
for J in np.arange(nRuns):
print( "Run %d of %d" % (J+1, nRuns) )
blosc.set_releasegil(False)
for I in np.arange( len(poolThreads) ):
t1 = time.time()
blosc.set_nthreads( bloscThreads[I] )
blosc.compress_ptr( stack.__array_interface__['data'][0], stack.size, stack.dtype.itemsize, \
clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
solo_times[I] += time.time() - t1
blosc.set_releasegil(True)
for I in np.arange( len(poolThreads) ):
t2 = time.time()
blosc.set_nthreads( bloscThreads[I] )
blosc.compress_ptr( stack.__array_interface__['data'][0], stack.size, stack.dtype.itemsize, \
clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
solo_unlocked_times[I] += time.time() - t2
blosc.set_releasegil(True)
for I in np.arange( len(poolThreads) ):
t3 = time.time()
compressStack( stack, blosc_threads=bloscThreads[I], pool_threads=poolThreads[I] )
unlocked_times[I] += time.time() - t3
def decompress_ptr():
cx = blosc.compress_ptr(address, num_elements, typesize, clevel=0)
blosc.decompress_ptr(cx, address)
import asyncio
import time
import cbor
import blosc
import asyncio_redis
from asyncio_redis.encoders import BaseEncoder
class CborEncoder(BaseEncoder):
native_type = object
def encode_from_native(self, data):
return cbor.dumps(data)
def decode_to_native(self, data):
return cbor.loads(data)
compress = lambda in_: (in_.size, in_.dtype, blosc.compress_ptr(in_.__array_interface__['data'][0], in_.size, in_.dtype.itemsize, clevel = 1, shuffle = blosc.BITSHUFFLE, cname = 'lz4'))
def decompress(size, dtype, data):
out = np.empty(size, dtype)
blosc.decompress_ptr(data, out.__array_interface__['data'][0])
return out
def packed_bytes_to_iq(samples, out = None):
if out is None:
iq = np.empty(len(samples)//2, 'complex64')
else:
iq = out
bytes_np = np.ctypeslib.as_array(samples)
iq.real, iq.imag = bytes_np[::2], bytes_np[1::2]
iq /= (255/2)
iq -= (1 + 1j)
