def get_raw_data(self): nx = self._header["x_pixels_in_detector"] ny = self._header["y_pixels_in_detector"] depth = self._header["bit_depth_image"] assert self._header["pixel_mask_applied"] is True if depth == 16: dtype = numpy.uint16 elif depth == 32: dtype = numpy.uint32 else: dtype = 1 / 0 dt = numpy.dtype(dtype) data = msgpack.unpackb(self.open_file(self._image_file).read(), raw=False)[2] blob = numpy.fromstring(data[12:], dtype=numpy.uint8) if dtype == numpy.uint32: block = numpy.ndarray(shape=(), dtype=">u4", buffer=data[8:12]) / 4 image = bitshuffle.decompress_lz4(blob, (ny, nx), dt, block) else: image = bitshuffle.decompress_lz4(blob, (ny, nx), dt) image = flex.int(image.astype("int32")) # only need to overwrite values if read and used in 16-bit mode if dtype == numpy.uint16: bad = 2**16 - 1 sel = image.as_1d() >= bad image.as_1d().set_selected(sel, -1) return image
def chunk_read_image(d_id, image_number): offset = (image_number, 0, 0) filter_mask, chunk = d_id.read_direct_chunk(offset) blob = numpy.fromstring(chunk[12:], dtype=numpy.uint8) image = bitshuffle.decompress_lz4(blob, (ny, nx), dt) flex_image = flex.int(image.astype("int32")) return flex_image
def test1(): data = np.ones(1000, dtype=np.uint32) compressed = bitshuffle.compress_lz4(data) print(data.nbytes / compressed.nbytes) decompressed = bitshuffle.decompress_lz4(compressed, data.shape, data.dtype) assert (np.array_equal(data, decompressed))
def readBS16LZ4(self, data, shape, dtype, size): """ Unpack bitshuffle-lz4 compressed 16 bit frame and return np array image data """ assert bitshuffle is not None, "No bitshuffle module" blob = np.fromstring(data[12:], dtype=np.uint8) return bitshuffle.decompress_lz4(blob, shape[::-1], np.dtype(dtype))
def chunk_to_image(self, chunk): """Decompress chunk to image.""" blob = numpy.fromstring(chunk[12:], dtype=numpy.uint8) image = bitshuffle.decompress_lz4(blob, self._dim, self._dtype) image = flex.int(image.astype("int32")) return image
def _read_header(bytes, size): hash = numpy.frombuffer(bytes.read(8), dtype='>i8') compression = numpy.frombuffer(bytes.read(1), dtype='>i1') raw_data = bytes.read(int(size - 2 - 8 - 1)) # print(size) # print(id) # print(hash) # print(compression) if compression == 0: data = raw_data.decode() elif compression == 1: length = struct.unpack(">q", raw_data[:8])[0] b_size = struct.unpack(">i", raw_data[8:12])[0] # print(length) # print(b_size) byte_array = bitshuffle.decompress_lz4(numpy.frombuffer( raw_data[12:], dtype=numpy.uint8), shape=(length, ), dtype=numpy.dtype('uint8'), block_size=b_size) data = byte_array.tobytes().decode() else: raise RuntimeError('Compression not supported') return json.loads(data)
def read_stream_data(frames, bss_job_mode=4): import lz4 import bitshuffle if len(frames) != 5: return None, None header = json.loads(frames[0].bytes) for i in (1, 3, 4): header.update(json.loads(frames[i].bytes)) if header.get("bss_job_mode", 4) != bss_job_mode: return None, None dtype = header["type"] shape = header["shape"][::-1] if dtype in ("int32", "uint32"): byte = 4 elif dtype in ("int16", "uint16"): byte = 2 else: raise RuntimeError("Unknown dtype (%s)" % dtype) size = byte * shape[0] * shape[1] if header["encoding"] == "lz4<": data = lz4.loads(struct.pack('<I', size) + frames[2].bytes) data = numpy.fromstring(data, dtype=dtype).reshape(shape) assert data.size * data.dtype.itemsize == size elif header["encoding"] == "bs32-lz4<": data = frames[2].bytes blob = numpy.fromstring(data[12:], dtype=numpy.uint8) # blocksize is big endian uint32 starting at byte 8, divided by element size blocksize = numpy.ndarray(shape=(), dtype=">u4", buffer=data[8:12]) / 4 data = bitshuffle.decompress_lz4(blob, shape, numpy.dtype(dtype), blocksize) data = data.reshape(shape) elif header["encoding"] == "bs16-lz4<": data = frames[2].bytes blob = numpy.fromstring(data[12:], dtype=numpy.uint8) data = bitshuffle.decompress_lz4(blob, shape, numpy.dtype(dtype)) data = data.reshape(shape) else: RuntimeError("Unknown encoding (%s)" % header["encoding"]) bad_sel = data == 2**(byte * 8) - 1 data = data.astype(numpy.int32) data[bad_sel] = -1 return header, data
def readBS16LZ4(self, data, shape, dtype, size): """ Unpack bitshuffle-lz4 compressed 16 bit frame and return np array image data """ import bitshuffle import numpy blob = numpy.fromstring(data[12:], dtype=numpy.uint8) return bitshuffle.decompress_lz4(blob, shape[::-1], numpy.dtype(dtype))
def unpack_data(raw_bytes, dtype, shape=None): """ Convert raw bytes into the specified numpy type. :param raw_bytes: Raw bytes to convert. :param dtype: dtype to use for the result. :param shape: Shape of the result. :return: Numpy array of dtype and shape. """ # Interpret the bytes as a numpy array. raw_data = numpy.frombuffer(raw_bytes, dtype=numpy.uint8) # If the numpy array is empty, return it as such. if raw_data.size == 0: return None # Uncompressed block size, big endian, int64 (long long) unpacked_length = struct.unpack(">q", raw_data[0:8].tobytes())[0] # Empty array was transmitted. if unpacked_length == 0: return None # Type of the output array. dtype = numpy.dtype(dtype) n_bytes_per_element = dtype.itemsize # Either the unpacked length or the dtype is wrong. if unpacked_length % n_bytes_per_element != 0: raise ValueError("Invalid unpacked length or dtype for raw bytes.") # How many bytes per element we use. n_elements = int(unpacked_length / n_bytes_per_element) # TODO: This is so ugly.. discuss if strings really need a shape [1]. # shape == [1] and n_elements > 1 is used for strings. if shape is None or (shape == [1] and n_elements > 1): shape = (n_elements,) # Compression block size, big endian, int32 (int). Divide by number of bytes per element. header_compression_block_size = struct.unpack(">i", raw_data[8:12].tobytes())[0] n_bytes_per_element = numpy.dtype(dtype).itemsize compression_block_size = header_compression_block_size / n_bytes_per_element # If shape is not provided use the original length. if shape is None: shape = (unpacked_length,) # Numpy is slowest dimension first, but bsread is fastest dimension first. shape = shape[::-1] # Actual data. byte_array = bitshuffle.decompress_lz4(raw_data[12:], block_size=compression_block_size, shape=shape, dtype=dtype) return byte_array
def read_stream_data(frames, bss_job_mode=4): import lz4 import bitshuffle if len(frames) != 5: return None, None header = json.loads(frames[0].bytes) for i in (1,3,4): header.update(json.loads(frames[i].bytes)) if header.get("bss_job_mode", 4) != bss_job_mode: return None, None dtype = header["type"] shape = header["shape"][::-1] if dtype in ("int32","uint32"): byte = 4 elif dtype in ("int16","uint16"): byte = 2 else: raise RuntimeError("Unknown dtype (%s)"%dtype) size = byte*shape[0]*shape[1] if header["encoding"] == "lz4<": data = lz4.loads(struct.pack('<I', size) + frames[2].bytes) data = numpy.fromstring(data, dtype=dtype).reshape(shape) assert data.size * data.dtype.itemsize == size elif header["encoding"] == "bs32-lz4<": data = frames[2].bytes blob = numpy.fromstring(data[12:],dtype=numpy.uint8) # blocksize is big endian uint32 starting at byte 8, divided by element size blocksize = numpy.ndarray(shape=(),dtype=">u4", buffer=data[8:12])/4 data = bitshuffle.decompress_lz4(blob, shape, numpy.dtype(dtype), blocksize) data = data.reshape(shape) elif header["encoding"] == "bs16-lz4<": data = frames[2].bytes blob = numpy.fromstring(data[12:],dtype=numpy.uint8) data = bitshuffle.decompress_lz4(blob, shape, numpy.dtype(dtype)) data = data.reshape(shape) else: RuntimeError("Unknown encoding (%s)"%header["encoding"]) data = data.astype(numpy.int32) data[data==2**(byte*8)-1] = -1 return header, data
def __getitem__(self, index): assert index is Ellipsis if self.dtype is None: return None with file(self.datafilename, 'r') as ff: compressed = numpy.fromfile(ff, dtype='uint8') data = bitshuffle.decompress_lz4(compressed, self.shape, self.dtype) #data = data.reshape(self.shape) return data
def extractor_do_uncompress(ts, pulse, buf, data, name, data_type, shape): c_length = struct.unpack(">q", buf[0:8])[0] b_size = struct.unpack(">i", buf[8:12])[0] nbuf = numpy.frombuffer(buf[12:], dtype=numpy.uint8) value = bitshuffle.decompress_lz4(nbuf, shape=shape, dtype=data_type, block_size=int(b_size / data_type.itemsize)) data.append({"timestamp": ts, "pulse_id": pulse, name: value})
def readBSLZ4(self, data, shape, dtype, size): """ Unpack bitshuffle-lz4 compressed frame and return np array image data """ assert bitshuffle is not None, "No bitshuffle module" blob = np.fromstring(data[12:], dtype=np.uint8) # blocksize is big endian uint32 starting at byte 8, divided by element size blocksize = np.ndarray(shape=(), dtype=">u4", buffer=data[8:12]) / 4 imgData = bitshuffle.decompress_lz4(blob, shape[::-1], np.dtype(dtype), blocksize) return imgData
def extractor_writer_compressed_string_scalar(ts, pulse, buf, data, name, shape): clen = int(struct.unpack(">q", buf[0:8])[0]) bsize = int(struct.unpack(">i", buf[8:12])[0]) u8buf = numpy.frombuffer(buf[12:], dtype=numpy.uint8) bval = bitshuffle.decompress_lz4(u8buf, shape=(clen, ), dtype=numpy.dtype(numpy.int8), block_size=bsize) value = bval.tobytes().decode() data.append({"timestamp": ts, "pulse_id": pulse, name: value})
def readBSLZ4(self, data, shape, dtype, size): """ Unpack bitshuffle-lz4 compressed frame and return np array image data """ import numpy as np import lz4, bitshuffle blob = np.fromstring(data[12:], dtype=np.uint8) # blocksize is big endian uint32 starting at byte 8, divided by element size blocksize = np.ndarray(shape=(), dtype=">u4", buffer=data[8:12])/4 imgData = bitshuffle.decompress_lz4(blob, shape[::-1], np.dtype(dtype), blocksize) return imgData
def readBSLZ4(self, frame, shape, dtype): """ unpack bitshuffle-lz4 compressed frame and return np array image data frame: zmq data blob frame shape: image shape dtype: image data type """ data = frame.bytes blob = np.fromstring(data[12:], dtype=np.uint8) dtype = np.dtype(dtype) # blocksize is big endian uint32 starting at byte 8, divided by element size blocksize = np.ndarray(shape=(), dtype=">u4", buffer=data[8:12])/dtype.itemsize imgData = bitshuffle.decompress_lz4(blob, shape[::-1], dtype, blocksize) if self._verbose: print("[OK] unpacked {0} bytes of bs-lz4 data".format(len(imgData))) return imgData
def readBSLZ4(self, frame, shape, dtype): """ unpack bitshuffle-lz4 compressed frame and return np array image data frame: zmq data blob frame shape: image shape dtype: image data type """ data = frame.bytes blob = np.fromstring(data[12:], dtype=np.uint8) dtype = np.dtype(dtype) # blocksize is big endian uint32 starting at byte 8, divided by element size blocksize = np.ndarray(shape=(), dtype=">u4", buffer=data[8:12]) / dtype.itemsize imgData = bitshuffle.decompress_lz4(blob, shape[::-1], dtype, blocksize) if self._verbose: print "[OK] unpacked {0} bytes of bs-lz4 data".format(len(imgData)) return imgData
def readBSLZ4(datafile, headerfile): """ unpack bitshuffle-lz4 compressed frame and return np array image data frame: zmq data blob frame shape: image shape dtype: image data type """ with open(datafile, "r") as f: data = f.read() with open(headerfile, "r") as f: header = json.loads(f.read()) shape = header["shape"] dtype = np.dtype(header["type"]) blob = np.fromstring(data[12:], dtype=np.uint8) # blocksize is big endian uint32 starting at byte 8, divided by element size blocksize = np.ndarray(shape=(), dtype=">u4", buffer=data[8:12]) / dtype.itemsize print blocksize, dtype.itemsize imgData = bitshuffle.decompress_lz4(blob, shape[::-1], dtype, blocksize) print "[OK] unpacked {0} bytes of bs-lz4 data".format(len(imgData)) return imgData
def zmq_worker(ai: AzimuthalIntegrator, host: str, port: int): context = zmq.Context() pull_sock = context.socket(zmq.PULL) pull_sock.connect('tcp://%s:%d' % (host, port)) push_sock = context.socket(zmq.PUSH) push_sock.connect('tcp://localhost:5550') while True: parts = pull_sock.recv_multipart(copy=False) header = json.loads(parts[0].bytes) #print(header) if header['htype'] == 'image': img = decompress_lz4(parts[1].buffer, header['shape'], np.dtype(header['type'])) res = ai.integrate(img) header['type'] = 'float32' header['shape'] = res.shape header['compression'] = 'none' push_sock.send_json(header, flags=zmq.SNDMORE) push_sock.send(res) else: push_sock.send_json(header) ai.close()
def bitshuffle_lz4_decode(data, shape, dtype, blocksize=0, out=None): """Decompress LZ4 with Bitshuffle.""" return bitshuffle.decompress_lz4(data, shape, dtype, blocksize)
from dials.algorithms.spot_finding.threshold import DispersionThresholdStrategy from dials.model.data import PixelList from dials.model.data import PixelListLabeller thresholder = DispersionThresholdStrategy(gain=1) mask = image.as_1d() >= 0 # flex.bool(image.size(), True) mask.reshape(flex.grid(*image.focus())) threshold_mask = thresholder(image, mask=mask) plist = PixelList(0, image, threshold_mask) pixel_labeller = PixelListLabeller() pixel_labeller.add(plist) creator = flex.PixelListShoeboxCreator(pixel_labeller, 0, 0, True, 2, 100, False) shoeboxes = creator.result() return shoeboxes for o in client.list_objects(bucket): if o.object_name is "header": continue image_number = int(o.object_name) chunk = client.get_object(o.bucket_name, o.object_name).read() blob = numpy.fromstring(chunk[12:], dtype=numpy.uint8) image = bitshuffle.decompress_lz4(blob, (ny, nx), dtype) image = flex.int(image.astype("int32")) shoeboxes = find_spots(image) print(image_number, flex.max(image), len(shoeboxes))
def decode(bytes, collector_function=None): """ Decode idread decoded data :param bytes: bytes to decode :param collector_function: function to collect decoded values. The signature of the function is as follows: def add_data(self, channel_name, backend, value, pulse_id, global_time, ioc_time, status, severity): :return: """ channels = None while True: # read size b = bytes.read(8) if b == b'': logger.debug('End of stream') break # size = numpy.frombuffer(b, dtype='>i8') # size = int.from_bytes(b, byteorder='big') size = struct.unpack(">q", b)[0] # id = numpy.frombuffer(bytes.read(2), dtype='>i2') # id = int.from_bytes(bytes.read(2), byteorder='big') id = struct.unpack(">h", bytes.read(2))[0] if id == 1: # Read Header header = _read_header(bytes, size) logging.debug(header) channels = [] for channel in header['channels']: encoding = '>' if 'encoding' in channel and channel["encoding"] == "big" else '' n_channel = {} if "type" not in channel or channel["type"] == "float64" or channel["type"] == "float": # default n_channel = {'size': 8, 'dtype': encoding+'f8', 'stype': encoding+'d'} elif channel["type"] == "uint8": n_channel = {'size': 1, 'dtype': encoding+'u1', 'stype': encoding+'B'} elif channel["type"] == "int8": n_channel = {'size': 1, 'dtype': encoding+'i1', 'stype': encoding+'b'} elif channel["type"] == "uint16": n_channel = {'size': 2, 'dtype': encoding+'u2', 'stype': encoding+'H'} elif channel["type"] == "int16": n_channel = {'size': 2, 'dtype': encoding+'i2', 'stype': encoding+'h'} elif channel["type"] == "uint32": n_channel = {'size': 4, 'dtype': encoding+'u4', 'stype': encoding+'I'} elif channel["type"] == "int32": n_channel = {'size': 4, 'dtype': encoding+'i4', 'stype': encoding+'i'} elif channel["type"] == "uint64": n_channel = {'size': 8, 'dtype': encoding+'u8', 'stype': encoding+'Q'} elif channel["type"] == "int64" or channel["type"] == "int": n_channel = {'size': 8, 'dtype': encoding+'i8', 'stype': encoding+'q'} elif channel["type"] == "float32": n_channel = {'size': 4, 'dtype': encoding+'f4', 'stype': encoding+'f'} else: # Raise exception for others (including strings) raise RuntimeError('Unsupported data type') # need to fix dtype with encoding n_channel['encoding'] = encoding # n_channel['encoding'] = 'big' if 'encoding' in channel and channel["encoding"] == "big" else 'little' # n_channel['dtype'] = n_channel['encoding']+n_channel['dtype'] n_channel['compression'] = channel['compression'] if 'compression' in channel else None # Numpy is slowest dimension first, but bsread is fastest dimension first. n_channel['shape'] = channel['shape'][::-1] if 'shape' in channel else [1] n_channel['name'] = channel['name'] n_channel['backend'] = channel['backend'] # used for struct readout n_channel['longtype'] = encoding + 'q' n_channel['chartype'] = encoding + 'b' n_channel['inttype'] = encoding + 'i' channels.append(n_channel) logger.debug(channels) elif id == 0: # Read Values if channels is None or channel == []: # Header was not yet received bytes.read(int(size - 2)) logging.warning('No channels specified, cannot deserialize - drop remaining bytes') else: size_counter = 0 for channel in channels: # event_size = numpy.frombuffer(bytes.read(4), dtype=channel['encoding']+'i4') # ioc_time = numpy.frombuffer(bytes.read(8), dtype=channel['encoding']+'i8') # pulse_id = numpy.frombuffer(bytes.read(8), dtype=channel['encoding']+'i8') # global_time = numpy.frombuffer(bytes.read(8), dtype=channel['encoding']+'i8') # status = numpy.frombuffer(bytes.read(1), dtype=channel['encoding']+'i1') # severity = numpy.frombuffer(bytes.read(1), dtype=channel['encoding']+'i1') event_size = struct.unpack(channel['inttype'], bytes.read(4))[0] if event_size == 0: ioc_time = None pulse_id = None global_time = None status = None severity = None data = None else: ioc_time = struct.unpack(channel['longtype'], bytes.read(8))[0] pulse_id = struct.unpack(channel['longtype'], bytes.read(8))[0] global_time = struct.unpack(channel['longtype'], bytes.read(8))[0] status = struct.unpack(channel['chartype'], bytes.read(1))[0] severity = struct.unpack(channel['chartype'], bytes.read(1))[0] # number of bytes to subtract from event_size = 8 - 8 - 8 - 1 - 1 = 26 n_bytes_to_read = int(event_size-26) raw_bytes = bytes.read(n_bytes_to_read) if channel['compression'] is not None: # TODO need to check for compression type - # Ideally this is done while header parsing, and here I would get the decode function length = struct.unpack(">q", raw_bytes[:8])[0] b_size = struct.unpack(">i", raw_bytes[8:12])[0] data = bitshuffle.decompress_lz4(numpy.frombuffer(raw_bytes[12:], dtype=numpy.uint8), shape=(channel['shape']), dtype=numpy.dtype(channel["dtype"]), block_size=b_size / channel['size']) else: if channel['shape'] is None or channel['shape'] == [1]: data = struct.unpack(channel['stype'], raw_bytes)[0] elif len(channel['shape']) == 1: data = struct.unpack(channel['stype'], raw_bytes) else: data = numpy.frombuffer(raw_bytes, dtype=channel["dtype"]) data = data.reshape(channel['shape']) size_counter += (2 + 4 + event_size) # 2 for id, 4 for event_size if collector_function is not None: collector_function(channel['name'], channel["backend"], data, pulse_id, global_time, ioc_time, status, severity) remaining_bytes = size-size_counter if remaining_bytes > 0: logger.warning("Remaining bytes - %d - drop remaining bytes" % remaining_bytes) bytes.read(remaining_bytes) else: logging.warning("id %i not supported - drop remaining bytes" % id) bytes.read(int(size-2))
def decode(bytes, serializer=None): channels = None while True: # read size b = bytes.read(8) if b == b'': logger.debug('end of file') break size = numpy.frombuffer(b, dtype='>i8') # read id id = numpy.frombuffer(bytes.read(2), dtype='>i2') if id == 1: # Read Header header = _read_header(bytes, size) logging.debug(header) channels = [] for channel in header['channels']: n_channel = {} if "type" not in channel or channel[ "type"] == "float64" or channel[ "type"] == "float": # default n_channel = {'size': 8, 'dtype': 'f8'} elif channel["type"] == "uint8": n_channel = {'size': 1, 'dtype': 'u1'} elif channel["type"] == "int8": n_channel = {'size': 1, 'dtype': 'i1'} elif channel["type"] == "uint16": n_channel = {'size': 2, 'dtype': 'u2'} elif channel["type"] == "int16": n_channel = {'size': 2, 'dtype': 'i2'} elif channel["type"] == "uint32": n_channel = {'size': 4, 'dtype': 'u4'} elif channel["type"] == "int32": n_channel = {'size': 4, 'dtype': 'i4'} elif channel["type"] == "uint64": n_channel = {'size': 8, 'dtype': 'u8'} elif channel["type"] == "int64" or channel["type"] == "int": n_channel = {'size': 8, 'dtype': 'i8'} elif channel["type"] == "float32": n_channel = {'size': 4, 'dtype': 'f4'} else: # Raise exception for others (including strings) raise RuntimeError('Unsupported data type') # need to fix dtype with encoding n_channel[ 'encoding'] = '>' if 'encoding' in channel and channel[ "encoding"] == "big" else '' # n_channel['dtype'] = n_channel['encoding']+n_channel['dtype'] n_channel['compression'] = channel[ 'compression'] if 'compression' in channel else None # Numpy is slowest dimension first, but bsread is fastest dimension first. n_channel['shape'] = channel[ 'shape'][::-1] if 'shape' in channel else [1] n_channel['name'] = channel['name'] channels.append(n_channel) logger.debug(channels) elif id == 0: # Read Values if channels is None or channel == []: # Header was not yet received bytes.read(int(size - 2)) logging.warning( 'No channels specified, cannot deserialize - drop remaining bytes' ) else: size_counter = 0 for channel in channels: # eventSize - int32 # iocTime - int64 # pulseId - int64 # globalTime - int64 # status - int8 # severity - int8 # value - dtype event_size = numpy.frombuffer(bytes.read(4), dtype=channel['encoding'] + 'i4') ioc_time = numpy.frombuffer(bytes.read(8), dtype=channel['encoding'] + 'i8') pulse_id = numpy.frombuffer(bytes.read(8), dtype=channel['encoding'] + 'i8') global_time = numpy.frombuffer(bytes.read(8), dtype=channel['encoding'] + 'i8') status = numpy.frombuffer(bytes.read(1), dtype=channel['encoding'] + 'i1') severity = numpy.frombuffer(bytes.read(1), dtype=channel['encoding'] + 'i1') # number of bytes to subtract from event_size = 8 - 8 - 8 - 1 - 1 = 26 raw_bytes = bytes.read(int(event_size - 26)) if channel['compression'] is not None: # TODO need to check for compression type - # Ideally this is done while header parsing, and here I would get the decode function length = struct.unpack(">q", raw_bytes[:8])[0] b_size = struct.unpack(">i", raw_bytes[8:12])[0] data = bitshuffle.decompress_lz4( numpy.frombuffer(raw_bytes[12:], dtype=numpy.uint8), shape=(channel['shape']), dtype=numpy.dtype(n_channel['encoding'] + channel["dtype"]), block_size=b_size / channel['size']) else: data = numpy.frombuffer(raw_bytes, dtype=n_channel['encoding'] + channel["dtype"]) # reshape the array if channel['shape'] is not None and channel['shape'] != [ 1 ]: data = data.reshape(channel['shape']) size_counter += (2 + 4 + event_size ) # 2 for id, 4 for event_size if serializer is not None: serializer.append_dataset('/' + channel['name'] + '/data', data, dtype=channel['dtype'], shape=channel['shape'], compress=True) serializer.append_dataset('/' + channel['name'] + '/pulse_id', pulse_id, dtype='i8') serializer.append_dataset('/' + channel['name'] + '/timestamp', global_time, dtype='i8') serializer.append_dataset('/' + channel['name'] + '/ioc_timestamp', ioc_time, dtype='i8') serializer.append_dataset('/' + channel['name'] + '/status', status, dtype='i1') serializer.append_dataset('/' + channel['name'] + '/severity', severity, dtype='i1') remaining_bytes = size - size_counter if remaining_bytes > 0: logger.warning( "Remaining bytes - %d - drop remaining bytes" % remaining_bytes) bytes.read(remaining_bytes) else: logging.warning("id %i not supported - drop remaining bytes" % id) bytes.read(int(size - 2))