def get_raw_data(self):
nx = self._header["x_pixels_in_detector"]
ny = self._header["y_pixels_in_detector"]
depth = self._header["bit_depth_image"]
assert self._header["pixel_mask_applied"] is True
if depth == 16:
dtype = numpy.uint16
elif depth == 32:
dtype = numpy.uint32
else:
dtype = 1 / 0
dt = numpy.dtype(dtype)
data = msgpack.unpackb(self.open_file(self._image_file).read(),
raw=False)[2]
blob = numpy.fromstring(data[12:], dtype=numpy.uint8)
if dtype == numpy.uint32:
block = numpy.ndarray(shape=(), dtype=">u4", buffer=data[8:12]) / 4
image = bitshuffle.decompress_lz4(blob, (ny, nx), dt, block)
else:
image = bitshuffle.decompress_lz4(blob, (ny, nx), dt)
image = flex.int(image.astype("int32"))
# only need to overwrite values if read and used in 16-bit mode
if dtype == numpy.uint16:
bad = 2**16 - 1
sel = image.as_1d() >= bad
image.as_1d().set_selected(sel, -1)
return image
def chunk_read_image(d_id, image_number):
offset = (image_number, 0, 0)
filter_mask, chunk = d_id.read_direct_chunk(offset)
blob = numpy.fromstring(chunk[12:], dtype=numpy.uint8)
image = bitshuffle.decompress_lz4(blob, (ny, nx), dt)
flex_image = flex.int(image.astype("int32"))
return flex_image
def test1():
data = np.ones(1000, dtype=np.uint32)
compressed = bitshuffle.compress_lz4(data)
print(data.nbytes / compressed.nbytes)
decompressed = bitshuffle.decompress_lz4(compressed, data.shape,
data.dtype)
assert (np.array_equal(data, decompressed))
def readBS16LZ4(self, data, shape, dtype, size):
"""
Unpack bitshuffle-lz4 compressed 16 bit frame and return np array image data
"""
assert bitshuffle is not None, "No bitshuffle module"
blob = np.fromstring(data[12:], dtype=np.uint8)
return bitshuffle.decompress_lz4(blob, shape[::-1], np.dtype(dtype))
def chunk_to_image(self, chunk):
"""Decompress chunk to image."""
blob = numpy.fromstring(chunk[12:], dtype=numpy.uint8)
image = bitshuffle.decompress_lz4(blob, self._dim, self._dtype)
image = flex.int(image.astype("int32"))
return image
def _read_header(bytes, size):
hash = numpy.frombuffer(bytes.read(8), dtype='>i8')
compression = numpy.frombuffer(bytes.read(1), dtype='>i1')
raw_data = bytes.read(int(size - 2 - 8 - 1))
# print(size)
# print(id)
# print(hash)
# print(compression)
if compression == 0:
data = raw_data.decode()
elif compression == 1:
length = struct.unpack(">q", raw_data[:8])[0]
b_size = struct.unpack(">i", raw_data[8:12])[0]
# print(length)
# print(b_size)
byte_array = bitshuffle.decompress_lz4(numpy.frombuffer(
raw_data[12:], dtype=numpy.uint8),
shape=(length, ),
dtype=numpy.dtype('uint8'),
block_size=b_size)
data = byte_array.tobytes().decode()
else:
raise RuntimeError('Compression not supported')
return json.loads(data)
def read_stream_data(frames, bss_job_mode=4):
import lz4
import bitshuffle
if len(frames) != 5:
return None, None
header = json.loads(frames[0].bytes)
for i in (1, 3, 4):
header.update(json.loads(frames[i].bytes))
if header.get("bss_job_mode", 4) != bss_job_mode:
return None, None
dtype = header["type"]
shape = header["shape"][::-1]
if dtype in ("int32", "uint32"): byte = 4
elif dtype in ("int16", "uint16"): byte = 2
else: raise RuntimeError("Unknown dtype (%s)" % dtype)
size = byte * shape[0] * shape[1]
if header["encoding"] == "lz4<":
data = lz4.loads(struct.pack('<I', size) + frames[2].bytes)
data = numpy.fromstring(data, dtype=dtype).reshape(shape)
assert data.size * data.dtype.itemsize == size
elif header["encoding"] == "bs32-lz4<":
data = frames[2].bytes
blob = numpy.fromstring(data[12:], dtype=numpy.uint8)
# blocksize is big endian uint32 starting at byte 8, divided by element size
blocksize = numpy.ndarray(shape=(), dtype=">u4", buffer=data[8:12]) / 4
data = bitshuffle.decompress_lz4(blob, shape, numpy.dtype(dtype),
blocksize)
data = data.reshape(shape)
elif header["encoding"] == "bs16-lz4<":
data = frames[2].bytes
blob = numpy.fromstring(data[12:], dtype=numpy.uint8)
data = bitshuffle.decompress_lz4(blob, shape, numpy.dtype(dtype))
data = data.reshape(shape)
else:
RuntimeError("Unknown encoding (%s)" % header["encoding"])
bad_sel = data == 2**(byte * 8) - 1
data = data.astype(numpy.int32)
data[bad_sel] = -1
return header, data
def readBS16LZ4(self, data, shape, dtype, size):
"""
Unpack bitshuffle-lz4 compressed 16 bit frame and return np array image data
"""
import bitshuffle
import numpy
blob = numpy.fromstring(data[12:], dtype=numpy.uint8)
return bitshuffle.decompress_lz4(blob, shape[::-1], numpy.dtype(dtype))
def unpack_data(raw_bytes, dtype, shape=None):
"""
Convert raw bytes into the specified numpy type.
:param raw_bytes: Raw bytes to convert.
:param dtype: dtype to use for the result.
:param shape: Shape of the result.
:return: Numpy array of dtype and shape.
"""
# Interpret the bytes as a numpy array.
raw_data = numpy.frombuffer(raw_bytes, dtype=numpy.uint8)
# If the numpy array is empty, return it as such.
if raw_data.size == 0:
return None
# Uncompressed block size, big endian, int64 (long long)
unpacked_length = struct.unpack(">q", raw_data[0:8].tobytes())[0]
# Empty array was transmitted.
if unpacked_length == 0:
return None
# Type of the output array.
dtype = numpy.dtype(dtype)
n_bytes_per_element = dtype.itemsize
# Either the unpacked length or the dtype is wrong.
if unpacked_length % n_bytes_per_element != 0:
raise ValueError("Invalid unpacked length or dtype for raw bytes.")
# How many bytes per element we use.
n_elements = int(unpacked_length / n_bytes_per_element)
# TODO: This is so ugly.. discuss if strings really need a shape [1].
# shape == [1] and n_elements > 1 is used for strings.
if shape is None or (shape == [1] and n_elements > 1):
shape = (n_elements,)
# Compression block size, big endian, int32 (int). Divide by number of bytes per element.
header_compression_block_size = struct.unpack(">i", raw_data[8:12].tobytes())[0]
n_bytes_per_element = numpy.dtype(dtype).itemsize
compression_block_size = header_compression_block_size / n_bytes_per_element
# If shape is not provided use the original length.
if shape is None:
shape = (unpacked_length,)
# Numpy is slowest dimension first, but bsread is fastest dimension first.
shape = shape[::-1]
# Actual data.
byte_array = bitshuffle.decompress_lz4(raw_data[12:], block_size=compression_block_size,
shape=shape, dtype=dtype)
return byte_array
def read_stream_data(frames, bss_job_mode=4):
import lz4
import bitshuffle
if len(frames) != 5:
return None, None
header = json.loads(frames[0].bytes)
for i in (1,3,4): header.update(json.loads(frames[i].bytes))
if header.get("bss_job_mode", 4) != bss_job_mode:
return None, None
dtype = header["type"]
shape = header["shape"][::-1]
if dtype in ("int32","uint32"): byte = 4
elif dtype in ("int16","uint16"): byte = 2
else: raise RuntimeError("Unknown dtype (%s)"%dtype)
size = byte*shape[0]*shape[1]
if header["encoding"] == "lz4<":
data = lz4.loads(struct.pack('<I', size) + frames[2].bytes)
data = numpy.fromstring(data, dtype=dtype).reshape(shape)
assert data.size * data.dtype.itemsize == size
elif header["encoding"] == "bs32-lz4<":
data = frames[2].bytes
blob = numpy.fromstring(data[12:],dtype=numpy.uint8)
# blocksize is big endian uint32 starting at byte 8, divided by element size
blocksize = numpy.ndarray(shape=(),dtype=">u4", buffer=data[8:12])/4
data = bitshuffle.decompress_lz4(blob, shape, numpy.dtype(dtype), blocksize)
data = data.reshape(shape)
elif header["encoding"] == "bs16-lz4<":
data = frames[2].bytes
blob = numpy.fromstring(data[12:],dtype=numpy.uint8)
data = bitshuffle.decompress_lz4(blob, shape, numpy.dtype(dtype))
data = data.reshape(shape)
else:
RuntimeError("Unknown encoding (%s)"%header["encoding"])
data = data.astype(numpy.int32)
data[data==2**(byte*8)-1] = -1
return header, data
def __getitem__(self, index):
assert index is Ellipsis
if self.dtype is None:
return None
with file(self.datafilename, 'r') as ff:
compressed = numpy.fromfile(ff, dtype='uint8')
data = bitshuffle.decompress_lz4(compressed,
self.shape, self.dtype)
#data = data.reshape(self.shape)
return data
def extractor_do_uncompress(ts, pulse, buf, data, name, data_type, shape):
c_length = struct.unpack(">q", buf[0:8])[0]
b_size = struct.unpack(">i", buf[8:12])[0]
nbuf = numpy.frombuffer(buf[12:], dtype=numpy.uint8)
value = bitshuffle.decompress_lz4(nbuf,
shape=shape,
dtype=data_type,
block_size=int(b_size /
data_type.itemsize))
data.append({"timestamp": ts, "pulse_id": pulse, name: value})
def readBSLZ4(self, data, shape, dtype, size):
"""
Unpack bitshuffle-lz4 compressed frame and return np array image data
"""
assert bitshuffle is not None, "No bitshuffle module"
blob = np.fromstring(data[12:], dtype=np.uint8)
# blocksize is big endian uint32 starting at byte 8, divided by element size
blocksize = np.ndarray(shape=(), dtype=">u4", buffer=data[8:12]) / 4
imgData = bitshuffle.decompress_lz4(blob, shape[::-1], np.dtype(dtype),
blocksize)
return imgData
def extractor_writer_compressed_string_scalar(ts, pulse, buf, data, name,
shape):
clen = int(struct.unpack(">q", buf[0:8])[0])
bsize = int(struct.unpack(">i", buf[8:12])[0])
u8buf = numpy.frombuffer(buf[12:], dtype=numpy.uint8)
bval = bitshuffle.decompress_lz4(u8buf,
shape=(clen, ),
dtype=numpy.dtype(numpy.int8),
block_size=bsize)
value = bval.tobytes().decode()
data.append({"timestamp": ts, "pulse_id": pulse, name: value})
def readBSLZ4(self, data, shape, dtype, size):
"""
Unpack bitshuffle-lz4 compressed frame and return np array image data
"""
import numpy as np
import lz4, bitshuffle
blob = np.fromstring(data[12:], dtype=np.uint8)
# blocksize is big endian uint32 starting at byte 8, divided by element size
blocksize = np.ndarray(shape=(), dtype=">u4", buffer=data[8:12])/4
imgData = bitshuffle.decompress_lz4(blob, shape[::-1], np.dtype(dtype), blocksize)
return imgData
def readBSLZ4(self, frame, shape, dtype):
"""
unpack bitshuffle-lz4 compressed frame and return np array image data
frame: zmq data blob frame
shape: image shape
dtype: image data type
"""
data = frame.bytes
blob = np.fromstring(data[12:], dtype=np.uint8)
dtype = np.dtype(dtype)
# blocksize is big endian uint32 starting at byte 8, divided by element size
blocksize = np.ndarray(shape=(), dtype=">u4", buffer=data[8:12])/dtype.itemsize
imgData = bitshuffle.decompress_lz4(blob, shape[::-1], dtype, blocksize)
if self._verbose:
print("[OK] unpacked {0} bytes of bs-lz4 data".format(len(imgData)))
return imgData
def readBSLZ4(self, frame, shape, dtype):
"""
unpack bitshuffle-lz4 compressed frame and return np array image data
frame: zmq data blob frame
shape: image shape
dtype: image data type
"""
data = frame.bytes
blob = np.fromstring(data[12:], dtype=np.uint8)
dtype = np.dtype(dtype)
# blocksize is big endian uint32 starting at byte 8, divided by element size
blocksize = np.ndarray(shape=(), dtype=">u4",
buffer=data[8:12]) / dtype.itemsize
imgData = bitshuffle.decompress_lz4(blob, shape[::-1], dtype,
blocksize)
if self._verbose:
print "[OK] unpacked {0} bytes of bs-lz4 data".format(len(imgData))
return imgData
def readBSLZ4(datafile, headerfile):
"""
unpack bitshuffle-lz4 compressed frame and return np array image data
frame: zmq data blob frame
shape: image shape
dtype: image data type
"""
with open(datafile, "r") as f:
data = f.read()
with open(headerfile, "r") as f:
header = json.loads(f.read())
shape = header["shape"]
dtype = np.dtype(header["type"])
blob = np.fromstring(data[12:], dtype=np.uint8)
# blocksize is big endian uint32 starting at byte 8, divided by element size
blocksize = np.ndarray(shape=(), dtype=">u4",
buffer=data[8:12]) / dtype.itemsize
print blocksize, dtype.itemsize
imgData = bitshuffle.decompress_lz4(blob, shape[::-1], dtype, blocksize)
print "[OK] unpacked {0} bytes of bs-lz4 data".format(len(imgData))
return imgData
def zmq_worker(ai: AzimuthalIntegrator, host: str, port: int):
context = zmq.Context()
pull_sock = context.socket(zmq.PULL)
pull_sock.connect('tcp://%s:%d' % (host, port))
push_sock = context.socket(zmq.PUSH)
push_sock.connect('tcp://localhost:5550')
while True:
parts = pull_sock.recv_multipart(copy=False)
header = json.loads(parts[0].bytes)
#print(header)
if header['htype'] == 'image':
img = decompress_lz4(parts[1].buffer, header['shape'],
np.dtype(header['type']))
res = ai.integrate(img)
header['type'] = 'float32'
header['shape'] = res.shape
header['compression'] = 'none'
push_sock.send_json(header, flags=zmq.SNDMORE)
push_sock.send(res)
else:
push_sock.send_json(header)
ai.close()
def bitshuffle_lz4_decode(data, shape, dtype, blocksize=0, out=None):
"""Decompress LZ4 with Bitshuffle."""
return bitshuffle.decompress_lz4(data, shape, dtype, blocksize)
from dials.algorithms.spot_finding.threshold import DispersionThresholdStrategy
from dials.model.data import PixelList
from dials.model.data import PixelListLabeller
thresholder = DispersionThresholdStrategy(gain=1)
mask = image.as_1d() >= 0 # flex.bool(image.size(), True)
mask.reshape(flex.grid(*image.focus()))
threshold_mask = thresholder(image, mask=mask)
plist = PixelList(0, image, threshold_mask)
pixel_labeller = PixelListLabeller()
pixel_labeller.add(plist)
creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True, 2, 100,
False)
shoeboxes = creator.result()
return shoeboxes
for o in client.list_objects(bucket):
if o.object_name is "header":
continue
image_number = int(o.object_name)
chunk = client.get_object(o.bucket_name, o.object_name).read()
blob = numpy.fromstring(chunk[12:], dtype=numpy.uint8)
image = bitshuffle.decompress_lz4(blob, (ny, nx), dtype)
image = flex.int(image.astype("int32"))
shoeboxes = find_spots(image)
print(image_number, flex.max(image), len(shoeboxes))
def decode(bytes, collector_function=None):
"""
Decode idread decoded data
:param bytes: bytes to decode
:param collector_function: function to collect decoded values. The signature of the function is as follows:
def add_data(self, channel_name, backend, value, pulse_id, global_time, ioc_time, status, severity):
:return:
"""
channels = None
while True:
# read size
b = bytes.read(8)
if b == b'':
logger.debug('End of stream')
break
# size = numpy.frombuffer(b, dtype='>i8')
# size = int.from_bytes(b, byteorder='big')
size = struct.unpack(">q", b)[0]
# id = numpy.frombuffer(bytes.read(2), dtype='>i2')
# id = int.from_bytes(bytes.read(2), byteorder='big')
id = struct.unpack(">h", bytes.read(2))[0]
if id == 1: # Read Header
header = _read_header(bytes, size)
logging.debug(header)
channels = []
for channel in header['channels']:
encoding = '>' if 'encoding' in channel and channel["encoding"] == "big" else ''
n_channel = {}
if "type" not in channel or channel["type"] == "float64" or channel["type"] == "float": # default
n_channel = {'size': 8, 'dtype': encoding+'f8', 'stype': encoding+'d'}
elif channel["type"] == "uint8":
n_channel = {'size': 1, 'dtype': encoding+'u1', 'stype': encoding+'B'}
elif channel["type"] == "int8":
n_channel = {'size': 1, 'dtype': encoding+'i1', 'stype': encoding+'b'}
elif channel["type"] == "uint16":
n_channel = {'size': 2, 'dtype': encoding+'u2', 'stype': encoding+'H'}
elif channel["type"] == "int16":
n_channel = {'size': 2, 'dtype': encoding+'i2', 'stype': encoding+'h'}
elif channel["type"] == "uint32":
n_channel = {'size': 4, 'dtype': encoding+'u4', 'stype': encoding+'I'}
elif channel["type"] == "int32":
n_channel = {'size': 4, 'dtype': encoding+'i4', 'stype': encoding+'i'}
elif channel["type"] == "uint64":
n_channel = {'size': 8, 'dtype': encoding+'u8', 'stype': encoding+'Q'}
elif channel["type"] == "int64" or channel["type"] == "int":
n_channel = {'size': 8, 'dtype': encoding+'i8', 'stype': encoding+'q'}
elif channel["type"] == "float32":
n_channel = {'size': 4, 'dtype': encoding+'f4', 'stype': encoding+'f'}
else:
# Raise exception for others (including strings)
raise RuntimeError('Unsupported data type')
# need to fix dtype with encoding
n_channel['encoding'] = encoding
# n_channel['encoding'] = 'big' if 'encoding' in channel and channel["encoding"] == "big" else 'little'
# n_channel['dtype'] = n_channel['encoding']+n_channel['dtype']
n_channel['compression'] = channel['compression'] if 'compression' in channel else None
# Numpy is slowest dimension first, but bsread is fastest dimension first.
n_channel['shape'] = channel['shape'][::-1] if 'shape' in channel else [1]
n_channel['name'] = channel['name']
n_channel['backend'] = channel['backend']
# used for struct readout
n_channel['longtype'] = encoding + 'q'
n_channel['chartype'] = encoding + 'b'
n_channel['inttype'] = encoding + 'i'
channels.append(n_channel)
logger.debug(channels)
elif id == 0: # Read Values
if channels is None or channel == []: # Header was not yet received
bytes.read(int(size - 2))
logging.warning('No channels specified, cannot deserialize - drop remaining bytes')
else:
size_counter = 0
for channel in channels:
# event_size = numpy.frombuffer(bytes.read(4), dtype=channel['encoding']+'i4')
# ioc_time = numpy.frombuffer(bytes.read(8), dtype=channel['encoding']+'i8')
# pulse_id = numpy.frombuffer(bytes.read(8), dtype=channel['encoding']+'i8')
# global_time = numpy.frombuffer(bytes.read(8), dtype=channel['encoding']+'i8')
# status = numpy.frombuffer(bytes.read(1), dtype=channel['encoding']+'i1')
# severity = numpy.frombuffer(bytes.read(1), dtype=channel['encoding']+'i1')
event_size = struct.unpack(channel['inttype'], bytes.read(4))[0]
if event_size == 0:
ioc_time = None
pulse_id = None
global_time = None
status = None
severity = None
data = None
else:
ioc_time = struct.unpack(channel['longtype'], bytes.read(8))[0]
pulse_id = struct.unpack(channel['longtype'], bytes.read(8))[0]
global_time = struct.unpack(channel['longtype'], bytes.read(8))[0]
status = struct.unpack(channel['chartype'], bytes.read(1))[0]
severity = struct.unpack(channel['chartype'], bytes.read(1))[0]
# number of bytes to subtract from event_size = 8 - 8 - 8 - 1 - 1 = 26
n_bytes_to_read = int(event_size-26)
raw_bytes = bytes.read(n_bytes_to_read)
if channel['compression'] is not None:
# TODO need to check for compression type -
# Ideally this is done while header parsing, and here I would get the decode function
length = struct.unpack(">q", raw_bytes[:8])[0]
b_size = struct.unpack(">i", raw_bytes[8:12])[0]
data = bitshuffle.decompress_lz4(numpy.frombuffer(raw_bytes[12:],
dtype=numpy.uint8),
shape=(channel['shape']),
dtype=numpy.dtype(channel["dtype"]),
block_size=b_size / channel['size'])
else:
if channel['shape'] is None or channel['shape'] == [1]:
data = struct.unpack(channel['stype'], raw_bytes)[0]
elif len(channel['shape']) == 1:
data = struct.unpack(channel['stype'], raw_bytes)
else:
data = numpy.frombuffer(raw_bytes, dtype=channel["dtype"])
data = data.reshape(channel['shape'])
size_counter += (2 + 4 + event_size) # 2 for id, 4 for event_size
if collector_function is not None:
collector_function(channel['name'], channel["backend"], data, pulse_id, global_time, ioc_time, status, severity)
remaining_bytes = size-size_counter
if remaining_bytes > 0:
logger.warning("Remaining bytes - %d - drop remaining bytes" % remaining_bytes)
bytes.read(remaining_bytes)
else:
logging.warning("id %i not supported - drop remaining bytes" % id)
bytes.read(int(size-2))
def decode(bytes, serializer=None):
channels = None
while True:
# read size
b = bytes.read(8)
if b == b'':
logger.debug('end of file')
break
size = numpy.frombuffer(b, dtype='>i8')
# read id
id = numpy.frombuffer(bytes.read(2), dtype='>i2')
if id == 1: # Read Header
header = _read_header(bytes, size)
logging.debug(header)
channels = []
for channel in header['channels']:
n_channel = {}
if "type" not in channel or channel[
"type"] == "float64" or channel[
"type"] == "float": # default
n_channel = {'size': 8, 'dtype': 'f8'}
elif channel["type"] == "uint8":
n_channel = {'size': 1, 'dtype': 'u1'}
elif channel["type"] == "int8":
n_channel = {'size': 1, 'dtype': 'i1'}
elif channel["type"] == "uint16":
n_channel = {'size': 2, 'dtype': 'u2'}
elif channel["type"] == "int16":
n_channel = {'size': 2, 'dtype': 'i2'}
elif channel["type"] == "uint32":
n_channel = {'size': 4, 'dtype': 'u4'}
elif channel["type"] == "int32":
n_channel = {'size': 4, 'dtype': 'i4'}
elif channel["type"] == "uint64":
n_channel = {'size': 8, 'dtype': 'u8'}
elif channel["type"] == "int64" or channel["type"] == "int":
n_channel = {'size': 8, 'dtype': 'i8'}
elif channel["type"] == "float32":
n_channel = {'size': 4, 'dtype': 'f4'}
else:
# Raise exception for others (including strings)
raise RuntimeError('Unsupported data type')
# need to fix dtype with encoding
n_channel[
'encoding'] = '>' if 'encoding' in channel and channel[
"encoding"] == "big" else ''
# n_channel['dtype'] = n_channel['encoding']+n_channel['dtype']
n_channel['compression'] = channel[
'compression'] if 'compression' in channel else None
# Numpy is slowest dimension first, but bsread is fastest dimension first.
n_channel['shape'] = channel[
'shape'][::-1] if 'shape' in channel else [1]
n_channel['name'] = channel['name']
channels.append(n_channel)
logger.debug(channels)
elif id == 0: # Read Values
if channels is None or channel == []: # Header was not yet received
bytes.read(int(size - 2))
logging.warning(
'No channels specified, cannot deserialize - drop remaining bytes'
)
else:
size_counter = 0
for channel in channels:
# eventSize - int32
# iocTime - int64
# pulseId - int64
# globalTime - int64
# status - int8
# severity - int8
# value - dtype
event_size = numpy.frombuffer(bytes.read(4),
dtype=channel['encoding'] +
'i4')
ioc_time = numpy.frombuffer(bytes.read(8),
dtype=channel['encoding'] +
'i8')
pulse_id = numpy.frombuffer(bytes.read(8),
dtype=channel['encoding'] +
'i8')
global_time = numpy.frombuffer(bytes.read(8),
dtype=channel['encoding'] +
'i8')
status = numpy.frombuffer(bytes.read(1),
dtype=channel['encoding'] + 'i1')
severity = numpy.frombuffer(bytes.read(1),
dtype=channel['encoding'] +
'i1')
# number of bytes to subtract from event_size = 8 - 8 - 8 - 1 - 1 = 26
raw_bytes = bytes.read(int(event_size - 26))
if channel['compression'] is not None:
# TODO need to check for compression type -
# Ideally this is done while header parsing, and here I would get the decode function
length = struct.unpack(">q", raw_bytes[:8])[0]
b_size = struct.unpack(">i", raw_bytes[8:12])[0]
data = bitshuffle.decompress_lz4(
numpy.frombuffer(raw_bytes[12:],
dtype=numpy.uint8),
shape=(channel['shape']),
dtype=numpy.dtype(n_channel['encoding'] +
channel["dtype"]),
block_size=b_size / channel['size'])
else:
data = numpy.frombuffer(raw_bytes,
dtype=n_channel['encoding'] +
channel["dtype"])
# reshape the array
if channel['shape'] is not None and channel['shape'] != [
1
]:
data = data.reshape(channel['shape'])
size_counter += (2 + 4 + event_size
) # 2 for id, 4 for event_size
if serializer is not None:
serializer.append_dataset('/' + channel['name'] +
'/data',
data,
dtype=channel['dtype'],
shape=channel['shape'],
compress=True)
serializer.append_dataset('/' + channel['name'] +
'/pulse_id',
pulse_id,
dtype='i8')
serializer.append_dataset('/' + channel['name'] +
'/timestamp',
global_time,
dtype='i8')
serializer.append_dataset('/' + channel['name'] +
'/ioc_timestamp',
ioc_time,
dtype='i8')
serializer.append_dataset('/' + channel['name'] +
'/status',
status,
dtype='i1')
serializer.append_dataset('/' + channel['name'] +
'/severity',
severity,
dtype='i1')
remaining_bytes = size - size_counter
if remaining_bytes > 0:
logger.warning(
"Remaining bytes - %d - drop remaining bytes" %
remaining_bytes)
bytes.read(remaining_bytes)
else:
logging.warning("id %i not supported - drop remaining bytes" % id)
bytes.read(int(size - 2))
