def test_file_partial_read_v0_0(tmpfile, testdata): io_groups, msgs = testdata to_rawfile(tmpfile, msgs=msgs, msg_headers={'io_groups': io_groups}, version='0.0') # skip reading msgs rd = from_rawfile(tmpfile, msg_headers_only=True) assert rd['msgs'] is None assert len(rd['msg_headers']['io_groups']) == 3 # read from end rd = from_rawfile(tmpfile, start=-1) assert len(rd['msgs']) == 1 assert rd['msgs'][0] == msgs[-1] assert len(rd['msg_headers']['io_groups']) == 1 assert rd['msg_headers']['io_groups'][0] == io_groups[-1] # read from middle rd = from_rawfile(tmpfile, start=1, end=3) assert len(rd['msgs']) == 2 assert rd['msgs'][0] == msgs[1] assert len(rd['msg_headers']['io_groups']) == 2 assert rd['msg_headers']['io_groups'][0] == io_groups[1] # read with mask rd = from_rawfile(tmpfile, mask=np.array([0, 1, 0]).astype(bool)) assert len(rd['msgs']) == 1 assert rd['msgs'][0] == msgs[1] assert len(rd['msg_headers']['io_groups']) == 1 assert rd['msg_headers']['io_groups'][0] == io_groups[1]
def next_chunk(self): """Read and parse the next chunk of raw data, return formatted data""" # Check if the file has been 'opened' if not self._is_open: print("You should open the file before reading!") return None # Check if at end of file if self._current_read_index == self._raw_file_length: # End of file return None # Set the chunk start and end indices start = self._current_read_index end = start + self._chunksize if end > self._raw_file_length: end = self._raw_file_length # Try to read the chunk try: rd = rh5.from_rawfile(self._filename, start=start, end=end) self._current_read_index = end except Exception as e: print("Couldn't read file: ", self._filename) return None # Associate io group to packets, following Peter's example pkts = list() for io_group, msg in zip(rd['msg_headers']['io_groups'], rd['msgs']): pkts.extend(parse(msg, io_group=io_group)) #pkts.extend(msg) # Return the packets return pkts
def test_incompatible_version_v0_0(tmpfile): to_rawfile(tmpfile, version='0.0', io_version='0.0') assert from_rawfile(tmpfile, version='0.0')['io_version'] == '0.0' with pytest.raises(AssertionError): from_rawfile(tmpfile, version='1.0') pytest.fail('Should identify incompatible version') with pytest.raises(AssertionError): from_rawfile(tmpfile, version='0.1') pytest.fail('Should identify incompatible version') with pytest.raises(AssertionError): to_rawfile(tmpfile, io_version='0.1') pytest.fail('Should complain about incompatible io version') with pytest.raises(AssertionError): to_rawfile(tmpfile, io_version='1.0') pytest.fail('Should complain about incompatible io version') with pytest.raises(AssertionError): from_rawfile(tmpfile, io_version='0.1') pytest.fail('Should complain about incompatible io version') with pytest.raises(AssertionError): from_rawfile(tmpfile, io_version='1.0') pytest.fail('Should complain about incompatible io version')
def test_file_full_v0_0(tmpfile, testdata): io_groups, msgs = testdata to_rawfile(tmpfile, msgs=msgs, msg_headers={'io_groups': io_groups}, version='0.0') assert len_rawfile(tmpfile) == len(msgs) rd = from_rawfile(tmpfile) assert len(rd['msgs']) == len(msgs) assert rd['msgs'][0] == msgs[0] assert rd['msgs'][-1] == msgs[-1] assert set(rd['msgs']) == set(msgs) assert len(rd['msg_headers']['io_groups']) == len(io_groups) assert rd['msg_headers']['io_groups'][0] == io_groups[0] assert rd['msg_headers']['io_groups'][-1] == io_groups[-1] assert set(rd['msg_headers']['io_groups']) == set(io_groups)
def main(input_filename, output_filename, block_size): total_messages = len_rawfile(input_filename) total_blocks = total_messages // block_size + 1 last = time.time() for i_block in range(total_blocks): start = i_block * block_size end = min(start + block_size, total_messages) if start == end: return if time.time() > last + 1: print('reading block {} of {}...\r'.format(i_block + 1, total_blocks), end='') last = time.time() rd = from_rawfile(input_filename, start=start, end=end) pkts = list() for i_msg, data in enumerate( zip(rd['msg_headers']['io_groups'], rd['msgs'])): io_group, msg = data pkts.extend(parse(msg, io_group=io_group)) to_file(output_filename, packet_list=pkts) print()