def parse(self, stream: KaitaiStream, context: AST): offset = stream.pos() try: b = self.reader(stream) except EOFError as e: print(context.root.to_dict()) raise e return Integer(b, offset, self.bitwidth // 8)
def parse(self, stream: KaitaiStream, context: AST): offset = stream.pos() c = stream.read_bytes(len(self.contents)) if c != self.contents: print(context.root.to_dict()) raise RuntimeError( f"File offset {offset}: Expected {self.contents!r} but instead got {c!r}" ) return RawBytes(c, offset)
def read_entries(data, start_offset=0): # Inspired by: https://stackoverflow.com/questions/49699820/parsing-binary-messages-with-kaitai-struct-python stream = KaitaiStream(BytesIO(data)) stream.seek(start_offset) last = stream.pos() start = ModelSxLog(stream) log_entry = start.log_entry yield log_entry n_entries = 1 with tqdm(total=stream.size() - start_offset, unit='B', unit_scale=True, desc='Processing log') as pbar: while not stream.is_eof(): if n_entries % 1000 == 0: consumed = stream.pos() - last pbar.update(consumed) last = stream.pos() try: log_entry = ModelSxLog.Entry(stream, _root=start._root) if sum(log_entry.raw_bytes) % 256 != 0: print( f'Checksum error at {stream.pos()}, seeking to the next entry...' ) stream.read_bytes_term(0xaa, include_term=False, consume_term=False, eos_error=True) else: yield log_entry except ValidationNotEqualError: print( f'Encountered an error at {stream.pos()}, probably a corrupt entry, seeking to next one...' ) stream.read_bytes_term(0xaa, include_term=False, consume_term=False, eos_error=True) pass n_entries += 1 pbar.update(stream.pos() - last) stream.close()
def parse(self, stream: KaitaiStream, context: AST) -> RawBytes: offset = stream.pos() ret = bytearray() if self.size is None: size = None else: size = int(self.size.interpret(context)) while self.size is None or not self._size_met(ret, size): try: b = stream.read_bytes(1) except EOFError: if self.size_eos: break else: raise RuntimeError("Unexpected end of stream") ret.extend(b) if self.terminator is not None and b == self.terminator: break return RawBytes(bytes(ret), offset)
def gse_parse(file, outfile, matype_crib=int(0x4200), stream=False, tcp_hijack=False, tcp_hijack_ips=None, reliable=True): with open(outfile, 'wb') as pcap_file: io = KaitaiStream(open(file, 'rb')) pcap_writer = Writer() pcap_writer.create_header(pcap_file) bbframe_count = 1 pkt_count = 0 eof_count = 0 while True: try: # we record the last io position for kaitai so we can recover from EOF errors in streaming mode last_pos = io.pos() # prints the first BBframe we find at the current IO position # this throws EOF if there's no bytes left in the file current_bbframe = PureBb(io, matype_crib=matype_crib).bbframe if eof_count > 0: print("new frames found, continuing...") eof_count = 0 except EOFError: if not stream: # if we're reading from a static file EOFError is sufficient reason to stop break elif eof_count == 0: pass # otherwise we will wait progressively longer whenever there isn't data in the streamed file elif eof_count % 10000 == 0: time.sleep(1) elif eof_count > 1000000: time.sleep(10) elif eof_count > 1000600: # after an hour of no fresh bytes (plus a little bit more), gsextract will exit and clean up buffers # this normally means something has broken in the satellite hardware side print( "No new data received for at least 1 hour. Exiting gsextract." ) eof_count += 1 io.seek(last_pos) continue except: # we want to maximize recovery in the case of stream parsing errors so we will just keep trying continue bbframe_count += 1 # record stats on corrupt BBframes and then move to the next frame if hasattr(current_bbframe, 'corrupt_data'): counters['broken_bbframes'] += 1 print("BBFrame", bbframe_count, " contains corrupt data, (MA2:", current_bbframe.bbheader.matype_2, ") attempting to recover") else: # for valid BBFrames # next extract gse packets from the bbframe and try to make them into IP payloads gse_packets = get_gse_from_bbdata(current_bbframe.data_field) raw_packets = parse_gse_packet_array(gse_packets, bbframe_count, reliable=reliable) # if we get any IP packets, write them to a pcap file if len(raw_packets) > 0: pcap_writer.write(raw_packets, pcap_file) pkt_count += len(raw_packets) # print some progress stats if pkt_count % 10000 == 0: print(pkt_count, "packets parsed") print(counters) # Clean up any lingering fragments when GSExtract closes # these would be partially filled buffers from end of recording raw_packets = parse_gse_packet_array([], 0, cleanup=True, reliable=reliable) if len(raw_packets) > 0: pcap_writer.write(raw_packets, pcap_file) # Print some basic stats before finishing print(counters)