def value(self):
if hasattr(self, '_m_value'):
return self._m_value if hasattr(self, '_m_value') else None
io = self._root._io
_pos = io.pos()
io.seek(self.offset)
_on = self.tag
if _on == u"head":
self._raw__m_value = io.read_bytes(self.length)
io = KaitaiStream(BytesIO(self._raw__m_value))
self._m_value = self._root.Head(io, self, self._root)
elif _on == u"cvt ":
self._raw__m_value = io.read_bytes(self.length)
io = KaitaiStream(BytesIO(self._raw__m_value))
self._m_value = self._root.Cvt(io, self, self._root)
elif _on == u"prep":
self._raw__m_value = io.read_bytes(self.length)
io = KaitaiStream(BytesIO(self._raw__m_value))
self._m_value = self._root.Prep(io, self, self._root)
elif _on == u"kern":
self._raw__m_value = io.read_bytes(self.length)
io = KaitaiStream(BytesIO(self._raw__m_value))
self._m_value = self._root.Kern(io, self, self._root)
elif _on == u"hhea":
self._raw__m_value = io.read_bytes(self.length)
io = KaitaiStream(BytesIO(self._raw__m_value))
self._m_value = self._root.Hhea(io, self, self._root)
elif _on == u"post":
self._raw__m_value = io.read_bytes(self.length)
io = KaitaiStream(BytesIO(self._raw__m_value))
self._m_value = self._root.Post(io, self, self._root)
elif _on == u"OS/2":
self._raw__m_value = io.read_bytes(self.length)
io = KaitaiStream(BytesIO(self._raw__m_value))
self._m_value = self._root.Os2(io, self, self._root)
elif _on == u"name":
self._raw__m_value = io.read_bytes(self.length)
io = KaitaiStream(BytesIO(self._raw__m_value))
self._m_value = self._root.Name(io, self, self._root)
elif _on == u"maxp":
self._raw__m_value = io.read_bytes(self.length)
io = KaitaiStream(BytesIO(self._raw__m_value))
self._m_value = self._root.Maxp(io, self, self._root)
elif _on == u"glyf":
self._raw__m_value = io.read_bytes(self.length)
io = KaitaiStream(BytesIO(self._raw__m_value))
self._m_value = self._root.Glyf(io, self, self._root)
elif _on == u"fpgm":
self._raw__m_value = io.read_bytes(self.length)
io = KaitaiStream(BytesIO(self._raw__m_value))
self._m_value = self._root.Fpgm(io, self, self._root)
elif _on == u"cmap":
self._raw__m_value = io.read_bytes(self.length)
io = KaitaiStream(BytesIO(self._raw__m_value))
self._m_value = self._root.Cmap(io, self, self._root)
else:
self._m_value = io.read_bytes(self.length)
io.seek(_pos)
return self._m_value if hasattr(self, '_m_value') else None
def get_tstamp_at(data, offset):
stream = KaitaiStream(BytesIO(data))
stream.seek(offset)
entry = ModelSxLog(stream).log_entry
dt = datetime.utcfromtimestamp(
struct.unpack('>L', entry.body.value[:4])[0])
return dt
def as_dir(self):
if hasattr(self, '_m_as_dir'):
return self._m_as_dir if hasattr(self, '_m_as_dir') else None
io = self._root._io
_pos = io.pos()
io.seek(self.offset)
self._raw__m_as_dir = io.read_bytes(self.size)
io = KaitaiStream(BytesIO(self._raw__m_as_dir))
self._m_as_dir = self._root.DirInode(io, self, self._root)
io.seek(_pos)
return self._m_as_dir if hasattr(self, '_m_as_dir') else None
def body(self):
if hasattr(self, '_m_body'):
return self._m_body if hasattr(self, '_m_body') else None
io = self._root._io
_pos = io.pos()
io.seek(self.body_ofs)
self._raw__m_body = io.read_bytes(self.in_archive_length)
io = KaitaiStream(BytesIO(self._raw__m_body))
self._m_body = self._root.Body(io, self, self._root)
io.seek(_pos)
return self._m_body if hasattr(self, '_m_body') else None
def data(self):
if hasattr(self, '_m_data'):
return self._m_data if hasattr(self, '_m_data') else None
io = self._root._io
_pos = io.pos()
io.seek(self.ofs_data)
self._raw__m_data = io.read_bytes(self.len_data)
io = KaitaiStream(BytesIO(self._raw__m_data))
self._m_data = self._root.SubSection(16, io, self, self._root)
io.seek(_pos)
return self._m_data if hasattr(self, '_m_data') else None
def target(self):
if hasattr(self, '_m_target'):
return self._m_target if hasattr(self, '_m_target') else None
io = self._root._io
_pos = io.pos()
io.seek((self.val * self._root.block_size))
self._raw__m_target = io.read_bytes(self._root.block_size)
io = KaitaiStream(BytesIO(self._raw__m_target))
self._m_target = self._root.Obj(io, self, self._root)
io.seek(_pos)
return self._m_target if hasattr(self, '_m_target') else None
def extent_as_dir(self):
if hasattr(self, '_m_extent_as_dir'):
return self._m_extent_as_dir if hasattr(self, '_m_extent_as_dir') else None
if (self.file_flags & 2) != 0:
io = self._root._io
_pos = io.pos()
io.seek((self.lba_extent.le * self._root.sector_size))
self._raw__m_extent_as_dir = io.read_bytes(self.size_extent.le)
io = KaitaiStream(BytesIO(self._raw__m_extent_as_dir))
self._m_extent_as_dir = self._root.DirEntries(io, self, self._root)
io.seek(_pos)
return self._m_extent_as_dir if hasattr(self, '_m_extent_as_dir') else None
def contents(self):
if hasattr(self, '_m_contents'):
return self._m_contents if hasattr(self,
'_m_contents') else None
io = self._root._io
_pos = io.pos()
io.seek(self.offset)
_on = self.name
if _on == u"SECTORS":
self._raw__m_contents = io.read_bytes(self.size)
io = KaitaiStream(BytesIO(self._raw__m_contents))
self._m_contents = self._root.Sectors(io, self, self._root)
elif _on == u"TEXTURE1":
self._raw__m_contents = io.read_bytes(self.size)
io = KaitaiStream(BytesIO(self._raw__m_contents))
self._m_contents = self._root.Texture12(io, self, self._root)
elif _on == u"VERTEXES":
self._raw__m_contents = io.read_bytes(self.size)
io = KaitaiStream(BytesIO(self._raw__m_contents))
self._m_contents = self._root.Vertexes(io, self, self._root)
elif _on == u"BLOCKMAP":
self._raw__m_contents = io.read_bytes(self.size)
io = KaitaiStream(BytesIO(self._raw__m_contents))
self._m_contents = self._root.Blockmap(io, self, self._root)
elif _on == u"PNAMES":
self._raw__m_contents = io.read_bytes(self.size)
io = KaitaiStream(BytesIO(self._raw__m_contents))
self._m_contents = self._root.Pnames(io, self, self._root)
elif _on == u"TEXTURE2":
self._raw__m_contents = io.read_bytes(self.size)
io = KaitaiStream(BytesIO(self._raw__m_contents))
self._m_contents = self._root.Texture12(io, self, self._root)
elif _on == u"THINGS":
self._raw__m_contents = io.read_bytes(self.size)
io = KaitaiStream(BytesIO(self._raw__m_contents))
self._m_contents = self._root.Things(io, self, self._root)
elif _on == u"LINEDEFS":
self._raw__m_contents = io.read_bytes(self.size)
io = KaitaiStream(BytesIO(self._raw__m_contents))
self._m_contents = self._root.Linedefs(io, self, self._root)
elif _on == u"SIDEDEFS":
self._raw__m_contents = io.read_bytes(self.size)
io = KaitaiStream(BytesIO(self._raw__m_contents))
self._m_contents = self._root.Sidedefs(io, self, self._root)
else:
self._m_contents = io.read_bytes(self.size)
io.seek(_pos)
return self._m_contents if hasattr(self, '_m_contents') else None
def entries(self):
if hasattr(self, '_m_entries'):
return self._m_entries if hasattr(self, '_m_entries') else None
io = self._root._io
_pos = io.pos()
io.seek((self.entries_start * self._root.sector_size))
self._raw__m_entries = [None] * (self.entries_count)
self._m_entries = [None] * (self.entries_count)
for i in range(self.entries_count):
self._raw__m_entries[i] = io.read_bytes(self.entries_size)
io = KaitaiStream(BytesIO(self._raw__m_entries[i]))
self._m_entries[i] = self._root.PartitionEntry(io, self, self._root)
io.seek(_pos)
return self._m_entries if hasattr(self, '_m_entries') else None
def partition_lookup(self):
"""Every partition entry contains the number of partition entries.
We parse the first entry, to know how many to parse, including the first one.
No logic is given what to do if other entries have a different number.
"""
if hasattr(self, '_m_partition_lookup'):
return self._m_partition_lookup if hasattr(
self, '_m_partition_lookup') else None
io = self._root._io
_pos = io.pos()
io.seek(self._root.sector_size)
self._raw__m_partition_lookup = io.read_bytes(self.sector_size)
io = KaitaiStream(BytesIO(self._raw__m_partition_lookup))
self._m_partition_lookup = self._root.PartitionEntry(
io, self, self._root)
io.seek(_pos)
return self._m_partition_lookup if hasattr(
self, '_m_partition_lookup') else None
def read_entries(data, start_offset=0):
# Inspired by: https://stackoverflow.com/questions/49699820/parsing-binary-messages-with-kaitai-struct-python
stream = KaitaiStream(BytesIO(data))
stream.seek(start_offset)
last = stream.pos()
start = ModelSxLog(stream)
log_entry = start.log_entry
yield log_entry
n_entries = 1
with tqdm(total=stream.size() - start_offset,
unit='B',
unit_scale=True,
desc='Processing log') as pbar:
while not stream.is_eof():
if n_entries % 1000 == 0:
consumed = stream.pos() - last
pbar.update(consumed)
last = stream.pos()
try:
log_entry = ModelSxLog.Entry(stream, _root=start._root)
if sum(log_entry.raw_bytes) % 256 != 0:
print(
f'Checksum error at {stream.pos()}, seeking to the next entry...'
)
stream.read_bytes_term(0xaa,
include_term=False,
consume_term=False,
eos_error=True)
else:
yield log_entry
except ValidationNotEqualError:
print(
f'Encountered an error at {stream.pos()}, probably a corrupt entry, seeking to next one...'
)
stream.read_bytes_term(0xaa,
include_term=False,
consume_term=False,
eos_error=True)
pass
n_entries += 1
pbar.update(stream.pos() - last)
stream.close()
def partition_entries(self):
if hasattr(self, '_m_partition_entries'):
return self._m_partition_entries if hasattr(
self, '_m_partition_entries') else None
io = self._root._io
_pos = io.pos()
io.seek(self._root.sector_size)
self._raw__m_partition_entries = [None] * (
self._root.partition_lookup.number_of_partitions)
self._m_partition_entries = [None] * (
self._root.partition_lookup.number_of_partitions)
for i in range(self._root.partition_lookup.number_of_partitions):
self._raw__m_partition_entries[i] = io.read_bytes(self.sector_size)
io = KaitaiStream(BytesIO(self._raw__m_partition_entries[i]))
self._m_partition_entries[i] = self._root.PartitionEntry(
io, self, self._root)
io.seek(_pos)
return self._m_partition_entries if hasattr(
self, '_m_partition_entries') else None
def gse_parse(file,
outfile,
matype_crib=int(0x4200),
stream=False,
tcp_hijack=False,
tcp_hijack_ips=None,
reliable=True):
with open(outfile, 'wb') as pcap_file:
io = KaitaiStream(open(file, 'rb'))
pcap_writer = Writer()
pcap_writer.create_header(pcap_file)
bbframe_count = 1
pkt_count = 0
eof_count = 0
while True:
try:
# we record the last io position for kaitai so we can recover from EOF errors in streaming mode
last_pos = io.pos()
# prints the first BBframe we find at the current IO position
# this throws EOF if there's no bytes left in the file
current_bbframe = PureBb(io, matype_crib=matype_crib).bbframe
if eof_count > 0:
print("new frames found, continuing...")
eof_count = 0
except EOFError:
if not stream:
# if we're reading from a static file EOFError is sufficient reason to stop
break
elif eof_count == 0:
pass
# otherwise we will wait progressively longer whenever there isn't data in the streamed file
elif eof_count % 10000 == 0:
time.sleep(1)
elif eof_count > 1000000:
time.sleep(10)
elif eof_count > 1000600:
# after an hour of no fresh bytes (plus a little bit more), gsextract will exit and clean up buffers
# this normally means something has broken in the satellite hardware side
print(
"No new data received for at least 1 hour. Exiting gsextract."
)
eof_count += 1
io.seek(last_pos)
continue
except:
# we want to maximize recovery in the case of stream parsing errors so we will just keep trying
continue
bbframe_count += 1
# record stats on corrupt BBframes and then move to the next frame
if hasattr(current_bbframe, 'corrupt_data'):
counters['broken_bbframes'] += 1
print("BBFrame", bbframe_count,
" contains corrupt data, (MA2:",
current_bbframe.bbheader.matype_2,
") attempting to recover")
else:
# for valid BBFrames
# next extract gse packets from the bbframe and try to make them into IP payloads
gse_packets = get_gse_from_bbdata(current_bbframe.data_field)
raw_packets = parse_gse_packet_array(gse_packets,
bbframe_count,
reliable=reliable)
# if we get any IP packets, write them to a pcap file
if len(raw_packets) > 0:
pcap_writer.write(raw_packets, pcap_file)
pkt_count += len(raw_packets)
# print some progress stats
if pkt_count % 10000 == 0:
print(pkt_count, "packets parsed")
print(counters)
# Clean up any lingering fragments when GSExtract closes
# these would be partially filled buffers from end of recording
raw_packets = parse_gse_packet_array([],
0,
cleanup=True,
reliable=reliable)
if len(raw_packets) > 0:
pcap_writer.write(raw_packets, pcap_file)
# Print some basic stats before finishing
print(counters)
