def test_to_file_v2_0_timestamp_packet(tmpfile, timestamp_packet):
to_file(tmpfile, [timestamp_packet], version='2.0')
f = h5py.File(tmpfile, 'r')
assert len(f['packets']) == 1
row = f['packets'][0]
props = dtype_property_index_lookup['2.0']['packets']
assert row[3] == 4 # packet_type
new_packet = TimestampPacket()
new_packet.timestamp = row[props['timestamp']]
assert new_packet == timestamp_packet
def test_message_format_test_packets(chip):
expected_packets = chip.get_configuration_packets(Packet.CONFIG_READ_PACKET)
expected_messages = [b'\x01\x00'+b'D'+b'\x02'+b'\x00'*4 + packet.bytes() + b'\x00' for packet in expected_packets]
ts_packet = TimestampPacket(timestamp=123456789)
expected_packets += [ts_packet]
expected_messages += [b'\x01\x00'+b'T'+b'\x00'*5 + ts_packet.bytes() + b'\x00']
print(expected_packets[-1])
print(expected_messages[-1])
print(dataserver_message_decode(expected_messages)[-1])
print(expected_messages[-1])
print(dataserver_message_encode(expected_packets)[-1])
assert expected_messages == dataserver_message_encode(expected_packets)
assert expected_packets == dataserver_message_decode(expected_messages)
def _parse_packets_v1_0(row, message_dset, *args, **kwargs):
if row['type'] == 4:
return TimestampPacket(row['timestamp'])
if row['type'] == 5:
index = row['counter']
message_row = message_dset[index]
message = message_row['message'].decode()
timestamp = row['timestamp']
return MessagePacket(message, timestamp)
if row['type'] < 4:
p = Packet_v1()
p.chip_key = row['chip_key']
p.packet_type = row['type']
p.chipid = row['chipid']
p.parity_bit_value = row['parity']
if p.packet_type == Packet_v1.DATA_PACKET:
p.channel = row['channel']
p.timestamp = row['timestamp']
p.dataword = row['adc_counts']
p.fifo_half_flag = row['fifo_half']
p.fifo_full_flag = row['fifo_full']
elif p.packet_type == Packet_v1.TEST_PACKET:
p.counter = row['counter']
elif (p.packet_type == Packet_v1.CONFIG_WRITE_PACKET
or p.packet_type == Packet_v1.CONFIG_READ_PACKET):
p.register_address = row['register']
p.register_data = row['value']
p.direction = row['direction']
return p
return None
def add_timestamps(packets, positions, timestamps=0):
'''
Insert timestamp packets into a list of packets in the given
positions.
Convenience method for modifying lists of packets to add to the
FakeIO queue. Modifies the list in-place.
The positions are with respect to the indexes of the original
list, so that the inserted element is just before the element
that originally had that index. e.g.
>>> add_timestamps([a, b, c, d], [1, 3])
[a, TimestampPacket(...), b, c, TimestampPacket(...), d]
If timestamps is a list, those timestamps will be used for the
TimestampPackets. If it is an int, it will be taken as the
starting time, and each subsequent packet will be incremented by
1. A default starting time of 0 is assumed.
'''
npositions = len(positions)
if isinstance(timestamps, int):
timestamps = list(range(timestamps, timestamps + npositions))
for position, timestamp in reversed(list(zip(positions, timestamps))):
packets.insert(position, TimestampPacket(timestamp))
def dataserver_message_decode(msgs,
key_generator=None,
version=(1, 0),
**kwargs):
r'''
Convert a list of larpix data server messages into packets. A key generator
should be provided if packets are to be used with an ``larpix.io.IO``
object. The data server messages provide a ``chip_id`` and ``io_chain`` for
keys. Additional keyword arguments can be passed along to the key generator. E.g.::
from larpix.larpix import Key
def ex_key_gen(chip_id, io_chain, io_group):
return Key(Key.key_format.format(
chip_id=chip_id,
io_channel=io_chain,
io_group=io_group
))
msg = b'\x01\x00D\x01\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00'
packets = dataserver_message_decode([msg], key_generator=ex_key_gen, io_group=1)
packets[0] # Packet(b'\x04\x00\x00\x00\x00\x00\x00'), key of '1-1-1'
:param msgs: list of bytestream messages each starting with a single 8-byte header word, followed by N 8-byte data words
:param key_generator: optional, a method that takes ``chip_id`` and ``io_chain`` as arguments and returns a ``larpix.Key`` object
:param version: optional, message version to validate against, ``tuple`` of major, minor version numbers
:returns: list of ``larpix.Packet`` and ``larpix.TimestampPacket`` objects
'''
packets = []
for msg in msgs:
major, minor = struct.unpack('BB', msg[:2])
if (major, minor) != version:
warnings.warn(
'Message version mismatch! Expected {}, received {}'.format(
version, (major, minor)))
msg_type = struct.unpack('c', msg[2:3])[0]
if msg_type == b'T':
timestamp = struct.unpack('L', msg[8:15] +
b'\x00')[0] # only use 7-bytes
packets.append(TimestampPacket(timestamp=timestamp))
elif msg_type == b'D':
io_chain = struct.unpack('B', msg[3:4])[0]
payload = msg[8:]
if len(payload) % 8 == 0:
for start_index in range(0, len(payload), 8):
packet_bytes = payload[start_index:start_index + 7]
packets.append(Packet(packet_bytes))
if key_generator:
packets[-1].chip_key = key_generator(
chip_id=packets[-1].chipid,
io_chain=io_chain,
**kwargs)
elif msg_type == b'H':
print('Heartbeat message: {}'.format(msg[3:]))
return packets
def test_record_deprecated_3_0_0(tmpdir):
logger = HDF5Logger(directory=str(tmpdir))
with pytest.warns(DeprecationWarning):
logger.open()
logger.record([Packet()])
assert len(logger._buffer['packets']) == 1
logger.record([TimestampPacket(timestamp=123)])
assert len(logger._buffer['packets']) == 2
def dataserver_message_decode(msgs, version=(1, 0), **kwargs):
r'''
Convert a list of larpix data server messages into packets. Additional packet meta data can be passed along via kwargs E.g.::
msg = b'\x01\x00D\x01\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00'
packets = dataserver_message_decode([msg], io_group=1)
packets[0] # Packet(b'\x04\x00\x00\x00\x00\x00\x00'), key of '1-1-1'
:param msgs: list of bytestream messages each starting with a single 8-byte header word, followed by N 8-byte data words
:param version: optional, message version to validate against, ``tuple`` of major, minor version numbers
:returns: list of ``larpix.Packet`` and ``larpix.TimestampPacket`` objects
'''
packets = []
for msg in msgs:
major, minor = struct.unpack('BB', msg[:2])
if (major, minor) != version:
warnings.warn(
'Message version mismatch! Expected {}, received {}'.format(
version, (major, minor)))
msg_type = struct.unpack('c', msg[2:3])[0]
if msg_type == b'T':
timestamp = struct.unpack('L', msg[8:15] +
b'\x00')[0] # only use 7-bytes
packets.append(TimestampPacket(timestamp=timestamp))
if kwargs:
for key, value in kwargs.items():
setattr(packets[-1], key, value)
elif msg_type == b'D':
io_chain = struct.unpack('B', msg[3:4])[0]
payload = msg[8:]
if len(payload) % 8 == 0:
for start_index in range(0, len(payload), 8):
packet_bytes = payload[start_index:start_index + 8]
if Packet == Packet_v1:
packets.append(Packet(packet_bytes[:-1]))
elif Packet == Packet_v2:
packets.append(Packet(packet_bytes))
packets[-1].io_channel = io_chain
if kwargs:
for key, value in kwargs.items():
setattr(packets[-1], key, value)
elif msg_type == b'H':
print('Heartbeat message: {}'.format(msg[3:]))
return packets
def _parse_packets_v2_0(row, message_dset, *args, **kwargs):
if row['packet_type'] == 4:
return TimestampPacket(row['timestamp'])
if row['packet_type'] == 5:
index = row['counter']
message_row = message_dset[index]
message = message_row['message'].decode()
timestamp = row['timestamp']
return MessagePacket(message, timestamp)
if row['packet_type'] < 4:
p = Packet_v2()
p.io_group = row['io_group']
p.io_channel = row['io_channel']
p.chip_id = row['chip_id']
p.packet_type = row['packet_type']
p.downstream_marker = row['downstream_marker']
p.parity = row['parity']
p.valid_parity = row['valid_parity']
p.direction = row['direction']
if p.packet_type == Packet_v2.DATA_PACKET:
p.channel_id = row['channel_id']
p.timestamp = row['timestamp']
p.dataword = row['dataword']
p.trigger_type = row['trigger_type']
p.local_fifo = row['local_fifo']
p.shared_fifo = row['shared_fifo']
if row['fifo_diagnostics_enabled'] != 0:
p.fifo_diagnostics_enabled = True
p.local_fifo = row['local_fifo_events']
p.shared_fifo = row['shared_fifo_events']
p.timestamp = row['timestamp']
elif p.packet_type in (Packet_v2.CONFIG_READ_PACKET,
Packet_v2.CONFIG_WRITE_PACKET):
p.register_address = row['register_address']
p.register_data = row['register_data']
return p
return None
def timestamp_packet():
p = TimestampPacket(timestamp=12345)
return p
def test_record(tmpdir):
logger = HDF5Logger(directory=str(tmpdir), enabled=True)
logger.record([Packet()])
assert len(logger._buffer['packets']) == 1
logger.record([TimestampPacket(timestamp=123)])
assert len(logger._buffer['packets']) == 2
def from_file(filename, version=None, start=None, end=None):
'''
Read the data from the given file into LArPix Packet objects.
:param filename: the name of the file to read
:param version: the format version. Specify this parameter to
enforce a version check. When a specific version such as
``'1.5'`` is specified, a ``RuntimeError`` will be raised if the
stored format version number is not an exact match. If a version
is prefixed with ``'~'`` such as ``'~1.5'``, a ``RuntimeError``
will be raised if the stored format version is *incompatible*
with the specified version. Compatible versions are those with
the same major version and at least the same minor version. E.g.
for ``'~1.5'``, versions between v1.5 and v2.0 are compatible.
If unspecified or ``None``, will use the stored format version.
:param start: the index of the first row to read
:param end: the index after the last row to read (same semantics as
Python ``range``)
:returns packet_dict: a dict with keys ``'packets'`` containing a
list of packet objects; and ``'created'``, ``'modified'``, and
``'version'``, containing the file metadata.
'''
with h5py.File(filename, 'r') as f:
file_version = f['_header'].attrs['version']
if version is None:
version = file_version
elif version[0] == '~':
file_major, _, file_minor = file_version.split('.')
version_major, _, version_minor = version.split('.')
version_major = version_major[1:]
if (file_major != version_major or file_minor < version_minor):
raise RuntimeError('Incompatible versions: existing: %s, '
'specified: %s' % (file_version, version))
else:
version = file_version
elif version == file_version:
pass
else:
raise RuntimeError('Incompatible versions: existing: %s, '
'specified: %s' % (file_version, version))
if version not in dtypes:
raise RuntimeError('Unknown version: %s' % version)
if version == '0.0':
dset_name = 'raw_packet'
else:
dset_name = 'packets'
message_dset_name = 'messages'
message_props = (
dtype_property_index_lookup[version][message_dset_name])
message_dset = f[message_dset_name]
props = dtype_property_index_lookup[version][dset_name]
packets = []
if start is None and end is None:
dset_iter = f[dset_name]
else:
dset_iter = f[dset_name][start:end]
for row in dset_iter:
if row[props['type']] == 4:
packets.append(TimestampPacket(row[props['timestamp']]))
continue
if row[props['type']] == 5:
index = row[props['counter']]
message_row = message_dset[index]
message = message_row[message_props['message']].decode()
timestamp = row[props['timestamp']]
packets.append(MessagePacket(message, timestamp))
continue
p = Packet()
p.chip_key = row[props['chip_key']]
p.packet_type = row[props['type']]
p.chipid = row[props['chipid']]
p.parity_bit_value = row[props['parity']]
if p.packet_type == Packet.DATA_PACKET:
p.channel = row[props['channel']]
p.timestamp = row[props['timestamp']]
p.dataword = row[props['adc_counts']]
p.fifo_half_flag = row[props['fifo_half']]
p.fifo_full_flag = row[props['fifo_full']]
elif p.packet_type == Packet.TEST_PACKET:
p.counter = row[props['counter']]
elif (p.packet_type == Packet.CONFIG_WRITE_PACKET
or p.packet_type == Packet.CONFIG_READ_PACKET):
p.register_address = row[props['register']]
p.register_data = row[props['value']]
if version != '0.0':
p.direction = row[props['direction']]
packets.append(p)
return {
'packets': packets,
'created': f['_header'].attrs['created'],
'modified': f['_header'].attrs['modified'],
'version': f['_header'].attrs['version'],
}
