def read_entries(data, start_offset=0): # Inspired by: https://stackoverflow.com/questions/49699820/parsing-binary-messages-with-kaitai-struct-python stream = KaitaiStream(BytesIO(data)) stream.seek(start_offset) last = stream.pos() start = ModelSxLog(stream) log_entry = start.log_entry yield log_entry n_entries = 1 with tqdm(total=stream.size() - start_offset, unit='B', unit_scale=True, desc='Processing log') as pbar: while not stream.is_eof(): if n_entries % 1000 == 0: consumed = stream.pos() - last pbar.update(consumed) last = stream.pos() try: log_entry = ModelSxLog.Entry(stream, _root=start._root) if sum(log_entry.raw_bytes) % 256 != 0: print( f'Checksum error at {stream.pos()}, seeking to the next entry...' ) stream.read_bytes_term(0xaa, include_term=False, consume_term=False, eos_error=True) else: yield log_entry except ValidationNotEqualError: print( f'Encountered an error at {stream.pos()}, probably a corrupt entry, seeking to next one...' ) stream.read_bytes_term(0xaa, include_term=False, consume_term=False, eos_error=True) pass n_entries += 1 pbar.update(stream.pos() - last) stream.close()
def read_entries(data, max_num=1000): # Inspired by: https://stackoverflow.com/questions/49699820/parsing-binary-messages-with-kaitai-struct-python stream = KaitaiStream(BytesIO(data)) # print(f'{stream.pos():08x} ', end='') start = ModelSxLog(stream) # Initialize the parser on the root stream log_entry = start.log_entry yield log_entry num_entries = 1 while not stream.is_eof(): try: # print(f'{stream.pos():08x} ', end='') log_entry = ModelSxLog.Record(stream, _root=start._root) if sum(log_entry.raw_bytes) % 256 != 0: print( f'Checksum error at {stream.pos()}, seeking to the next entry...' ) stream.read_bytes_term(0xaa, include_term=False, consume_term=False, eos_error=True) else: yield log_entry except: # Unfortunately kaitaistruct does not specify the exception, assuming it's a wrong delimiter print( f'Encountered an error at {stream.pos()}, probably a corrupt entry, seeking to next one...' ) stream.read_bytes_term(0xaa, include_term=False, consume_term=False, eos_error=True) pass num_entries += 1 if num_entries > max_num: break stream.close()