def test_file_partial_read_v0_0(tmpfile, testdata):
io_groups, msgs = testdata
to_rawfile(tmpfile,
msgs=msgs,
msg_headers={'io_groups': io_groups},
version='0.0')
# skip reading msgs
rd = from_rawfile(tmpfile, msg_headers_only=True)
assert rd['msgs'] is None
assert len(rd['msg_headers']['io_groups']) == 3
# read from end
rd = from_rawfile(tmpfile, start=-1)
assert len(rd['msgs']) == 1
assert rd['msgs'][0] == msgs[-1]
assert len(rd['msg_headers']['io_groups']) == 1
assert rd['msg_headers']['io_groups'][0] == io_groups[-1]
# read from middle
rd = from_rawfile(tmpfile, start=1, end=3)
assert len(rd['msgs']) == 2
assert rd['msgs'][0] == msgs[1]
assert len(rd['msg_headers']['io_groups']) == 2
assert rd['msg_headers']['io_groups'][0] == io_groups[1]
# read with mask
rd = from_rawfile(tmpfile, mask=np.array([0, 1, 0]).astype(bool))
assert len(rd['msgs']) == 1
assert rd['msgs'][0] == msgs[1]
assert len(rd['msg_headers']['io_groups']) == 1
assert rd['msg_headers']['io_groups'][0] == io_groups[1]
def next_chunk(self):
"""Read and parse the next chunk of raw data, return formatted data"""
# Check if the file has been 'opened'
if not self._is_open:
print("You should open the file before reading!")
return None
# Check if at end of file
if self._current_read_index == self._raw_file_length:
# End of file
return None
# Set the chunk start and end indices
start = self._current_read_index
end = start + self._chunksize
if end > self._raw_file_length: end = self._raw_file_length
# Try to read the chunk
try:
rd = rh5.from_rawfile(self._filename, start=start, end=end)
self._current_read_index = end
except Exception as e:
print("Couldn't read file: ", self._filename)
return None
# Associate io group to packets, following Peter's example
pkts = list()
for io_group, msg in zip(rd['msg_headers']['io_groups'], rd['msgs']):
pkts.extend(parse(msg, io_group=io_group))
#pkts.extend(msg)
# Return the packets
return pkts
def test_incompatible_version_v0_0(tmpfile):
to_rawfile(tmpfile, version='0.0', io_version='0.0')
assert from_rawfile(tmpfile, version='0.0')['io_version'] == '0.0'
with pytest.raises(AssertionError):
from_rawfile(tmpfile, version='1.0')
pytest.fail('Should identify incompatible version')
with pytest.raises(AssertionError):
from_rawfile(tmpfile, version='0.1')
pytest.fail('Should identify incompatible version')
with pytest.raises(AssertionError):
to_rawfile(tmpfile, io_version='0.1')
pytest.fail('Should complain about incompatible io version')
with pytest.raises(AssertionError):
to_rawfile(tmpfile, io_version='1.0')
pytest.fail('Should complain about incompatible io version')
with pytest.raises(AssertionError):
from_rawfile(tmpfile, io_version='0.1')
pytest.fail('Should complain about incompatible io version')
with pytest.raises(AssertionError):
from_rawfile(tmpfile, io_version='1.0')
pytest.fail('Should complain about incompatible io version')
def test_file_full_v0_0(tmpfile, testdata):
io_groups, msgs = testdata
to_rawfile(tmpfile,
msgs=msgs,
msg_headers={'io_groups': io_groups},
version='0.0')
assert len_rawfile(tmpfile) == len(msgs)
rd = from_rawfile(tmpfile)
assert len(rd['msgs']) == len(msgs)
assert rd['msgs'][0] == msgs[0]
assert rd['msgs'][-1] == msgs[-1]
assert set(rd['msgs']) == set(msgs)
assert len(rd['msg_headers']['io_groups']) == len(io_groups)
assert rd['msg_headers']['io_groups'][0] == io_groups[0]
assert rd['msg_headers']['io_groups'][-1] == io_groups[-1]
assert set(rd['msg_headers']['io_groups']) == set(io_groups)
def main(input_filename, output_filename, block_size):
total_messages = len_rawfile(input_filename)
total_blocks = total_messages // block_size + 1
last = time.time()
for i_block in range(total_blocks):
start = i_block * block_size
end = min(start + block_size, total_messages)
if start == end: return
if time.time() > last + 1:
print('reading block {} of {}...\r'.format(i_block + 1,
total_blocks),
end='')
last = time.time()
rd = from_rawfile(input_filename, start=start, end=end)
pkts = list()
for i_msg, data in enumerate(
zip(rd['msg_headers']['io_groups'], rd['msgs'])):
io_group, msg = data
pkts.extend(parse(msg, io_group=io_group))
to_file(output_filename, packet_list=pkts)
print()