class JokerContainerFile:
file = None
# Offset/sizes in bytes
identifier_offset = 0
identifier_size = 5
mode_offset = identifier_offset + identifier_size
mode_size = 2
data_offset = mode_offset + mode_size + 1 # Skips unknown byte 0x01 between mode and data
mode_f100_offset = data_offset + (7 * 8)
def __init__(self, f, read=True):
if read:
self.file = ConstBitStream(f.read())
else:
self.file = ConstBitStream(f)
self.file.pos = self.identifier_offset
identifier = self.file.peek(f'bytes:{self.identifier_size}')
if identifier != b'Joker':
raise ValueError('Input not a Joker container file')
@property
def mode(self):
self.file.pos = self.mode_offset * 8
return self.file.peek(f'uintle:{self.mode_size * 8}')
@property
def _data(self):
return self.file[self.data_offset * 8:]
@property
def data(self):
data = self._data
if self.mode == 0x0300 or self.mode == 0x0200:
# Compressed/encrypted
decompressed = zlib.decompress(data.tobytes())[:-1]
decoded = bytes.fromhex(base64.b64decode(decompressed, validate=True).decode())
data = JokerCipher.decrypt(decoded)
if self.mode == 0x300:
data = self._process_mode_300(data)
elif self.mode == 0xF100:
return data[self.mode_f100_offset:]
return data
@staticmethod
def _process_mode_300(data):
decoded = base64.decodebytes(data)
decrypted = DefaultCipher.decrypt(decoded)
decompressed = zlib.decompress(decrypted)
return decompressed
def lzwv_decode(in_array):
max_entries = 2**MAX_CODE_LEN
in_stream = ConstBitStream(in_array)
dictionary = {i: bytes([i]) for i in range(256)}
code_len = 9
out_array = bytearray()
while True:
try:
# Read a code
k = in_stream.read(f'uint:{code_len}')
# Look up code and emit value
v = dictionary[k]
out_array += v
# Add v + v_next[0] to dictionary
c = len(dictionary)
if c < max_entries:
if c == 2**code_len - 1:
code_len = min(MAX_CODE_LEN, code_len + 1)
k_next = in_stream.peek(f'uint:{code_len}')
if k_next < c:
v_next = dictionary[k_next]
dictionary[c] = v + v_next[:1]
else:
dictionary[c] = v + v[:1]
except ReadError:
break
return bytes(out_array)
def lzwf_decode(in_array, code_len=DEFAULT_CODE_LEN):
max_entries = 2**code_len
in_stream = ConstBitStream(in_array)
dictionary = {i: bytes([i]) for i in range(256)}
out_array = bytes()
while True:
try:
# Read a code
k = in_stream.read(f'uint:{code_len}')
# Look up code and emit value
v = dictionary[k]
out_array = out_array + v
# Add v + v_next[0] to dictionary
c = len(dictionary)
if c < max_entries:
k_next = in_stream.peek(f'uint:{code_len}')
if k_next < c:
v_next = dictionary[k_next]
dictionary[c] = v + v_next[:1]
else:
dictionary[c] = v + v[:1]
except ReadError:
break
return bytes(out_array)
def huffman_decode(data, decoded_len, serialized_tree, symbol_bits=8):
tree = deserialize_huffman_tree(serialized_tree, symbol_bits)
dictionary, min_code_len = make_decoding_dictionary(tree)
bits = ConstBitStream(data)
out = BitArray()
symbols_decoded = 0
try:
while symbols_decoded < decoded_len:
num_bits = min_code_len
code = bits.peek(f'bin:{num_bits}')
while code not in dictionary:
num_bits += 1
code = bits.peek(f'bin:{num_bits}')
out.append(Bits(uint=dictionary[code], length=symbol_bits))
symbols_decoded += 1
bits.read(f'bin:{num_bits}')
except ReadError:
pass
return out.tobytes()
def _load_le_instr(self, bitstream: bitstring.ConstBitStream,
numbits: int) -> str:
# THUMB mode instructions swap endianness every two bytes!
if (self.addr & 1) == 1 and numbits > 16:
chunk = ""
oldpos = bitstream.pos
for _ in range(0, numbits, 16):
chunk += bitstring.Bits(uint=bitstream.peek("uintle:%d" % 16),
length=16).bin
bitstream.pos += 16
bitstream.pos = oldpos
return chunk
return super()._load_le_instr(bitstream, numbits)
def process_tm_packet(binary_packet):
bits = ConstBitStream(binary_packet)
header = {
k: v
for k, v in zip(tm_header.keys(), bits.readlist(tm_header.values()))
}
if header['service_type'] in [3, 21
] and header['service_subtype'] in [25, 6]:
try:
ssid = bits.peek('uint:8')
header['ssid'] = ssid
except ReadError as e:
header['ssid'] = -1
if header['data_len'] + 7 != len(bits.hex) / 2:
raise ValueError('Expected and actual packet lentght do not match')
return header, bits.hex
def from_bytes(cls, bitstream):
'''
Look at the type of the message, instantiate the correct class and
let it parse the message.
'''
from pylisp.packet.lisp.control import type_registry
# Convert to ConstBitStream (if not already provided)
if not isinstance(bitstream, ConstBitStream):
if isinstance(bitstream, Bits):
bitstream = ConstBitStream(auto=bitstream)
else:
bitstream = ConstBitStream(bytes=bitstream)
# Peek at the bitstream to see which type it is
type_nr = bitstream.peek('uint:4')
# Look for the right class
type_class = type_registry.get_type_class(type_nr)
if not type_class:
raise ValueError("Can't handle message type {0}".format(type_nr))
# Let the specific class handle it from now on
return type_class.from_bytes(bitstream)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from bitstring import ConstBitStream
import sys
c = ConstBitStream(filename=sys.argv[1])
#Most log entry ends with 0A, this helps us to seek to the beginning of an entry
start = c.find('0x0A', bytealigned=True)[0]
#Seek 8 byte into the stream to skip EOL from previus log entry
start += 8
c.pos = start
while True:
#Read and print the binary log header
header = c.readlist('8*uintle:32')
print header
#Get the size in bits of the log message
msgSize = (header[0] * 8 - 256)
#Move pointer after message
c.pos += msgSize
#Check if EOL is present after message, if true, move pointer 8 byte
if c.peek(8).hex == '0a':
c.pos += 8
def from_bytes(cls, bitstream, decode_payload=True):
'''
Parse the given packet and update properties accordingly
'''
packet = cls()
# Convert to ConstBitStream (if not already provided)
if not isinstance(bitstream, ConstBitStream):
if isinstance(bitstream, Bits):
bitstream = ConstBitStream(auto=bitstream)
else:
bitstream = ConstBitStream(bytes=bitstream)
# Read the version
version = bitstream.read('uint:4')
if version != packet.version:
raise ValueError('Provided bytes do not contain an IPv4 packet')
# Read the header length
ihl = bitstream.read('uint:4')
if ihl < 5:
raise ValueError('Invalid IPv4 header length')
# Now that we know the length of the header we store it to be able
# to easily recalculate the header checksum later
remaining_header_bits = (ihl * 32) - 8
header = (BitStream('uint:4=4, uint:4=%d' % ihl) +
bitstream.peek(remaining_header_bits))
# Read the type of service
packet.tos = bitstream.read('uint:8')
# Read the total length
total_length = bitstream.read('uint:16')
if total_length < ihl * 4:
raise ValueError('Total length is shorter than the header')
# Read the identification
packet.identification = bitstream.read('uint:16')
# Read the flags
(reserved,
packet.dont_fragment,
packet.more_fragments) = bitstream.readlist('3*bool')
if reserved:
raise ValueError('Reserved flag must be 0')
# Read the fragment offset
packet.fragment_offset = bitstream.read('uint:13')
# Read the TTL
packet.ttl = bitstream.read('uint:8')
# Read the protocol number
packet.protocol = bitstream.read('uint:8')
# Read the header checksum
header_checksum = bitstream.read('uint:16')
# Set the checksum bits in the header to 0 and re-calculate
header[80:96] = BitStream(16)
my_checksum = checksum.ones_complement(header.bytes)
if my_checksum != header_checksum:
raise ValueError('Header checksum does not match')
# Read the source and destination addresses
packet.source = IPv4Address(bitstream.read('uint:32'))
packet.destination = IPv4Address(bitstream.read('uint:32'))
# Read the options
option_len = (ihl - 5) * 4
packet.options = bitstream.read('bytes:%d' % option_len)
# And the rest is payload
payload_bytes = (total_length) - (ihl * 4)
packet.payload = bitstream.read('bytes:%d' % payload_bytes)
if decode_payload:
payload_class = protocol_registry.get_type_class(packet.protocol)
if payload_class:
packet.payload = payload_class.from_bytes(packet.payload)
# There should be no remaining bits
if bitstream.pos != bitstream.len:
raise ValueError('Bits remaining after processing packet')
# Verify that the properties make sense
packet.sanitize()
return packet
def _load_le_instr(self, bitstream: bitstring.ConstBitStream, numbits: int) -> str:
return bitstring.Bits(uint=bitstream.peek("uintle:%d" % numbits), length=numbits).bin
class FileReader:
""" Some basic functionality for reading data from Binary files. """
def __init__(self, filename):
self.filename = filename
if isinstance(filename, str):
self.file = open(filename, 'rb')
self.bits = ConstBitStream(open(filename, 'rb'))
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.file.close()
def skip_bytes(self, count=1):
self.bits.read('pad:{0}'.format(count * 8))
def peek_int(self):
return self.bits.peek(32).uintle
def forward_to_first_non_zero_byte(self, start, end):
self.bits.pos = start
while self.bits.pos < end and self.bits.read(8).intle == 0:
pass
self.bits.pos -= 8
def read_strings_from_block(self, start, end, stopAtEmptyString=False):
self.bits.pos = start
r = []
while self.bits.pos < end:
s = self.read_string()
if s == "" and stopAtEmptyString:
return r
r.append(s)
return tuple(r)
def read_int(self):
""" Read a single little endian 4 byte integer """
return self.bits.read(32).intle
def findall(self, bs):
return self.bits.findall(bs, bytealigned=True)
def read_bytes(self, count):
return self.bits.read(count * 8)
def read_byte(self, skip=0):
i = self.bits.read(8).uint
if skip > 0:
self.skip_bytes(skip)
return i
def read_string_safe(self):
return self.bits.read('bytes:{0}'.format(
self.read_byte(skip=3))).decode("utf-8", 'replace')
def find(self, bs, start, end):
return self.bits.find(bs, start, end, True)
def find_first(self, bs):
return self.bits.find(bs)
def extract_compressed_payloads(self):
files = []
occ = self.findall('0x789C')
i = 0
readSize = 2**12
for pos in occ:
self.bits.pos = pos
#read the start of the stream into a buffer.
if (self.bits.length - self.pos) < 8 * 2**12:
readSize = int((self.bits.length - self.pos) / 8)
buf = self.bits.read('bytes:{0}'.format(readSize))
zo = zlib.decompressobj()
#start the decompression
try:
stream = zo.decompress(buf)
except zlib.error: # right magic number but not a zlib stream.
continue
while zo.unused_data == b'' and readSize >= 2**12:
if (self.bits.length - self.pos) < 8 * 2**12:
readSize = int((self.bits.length - self.pos) / 8)
block = self.bits.read('bytes:{0}'.format(readSize))
if len(block) > 0:
try:
stream += zo.decompress(block)
except zlib.error:
pass
else:
break # we've reached EOF
with open(self.filename + '_' + str(i) + '.decompressed',
'wb') as fh:
fh.write(stream)
files.append(self.filename + '_' + str(i) + '.decompressed')
i += 1
return files
@property
def pos(self):
return self.bits.pos
@pos.setter
def pos(self, value):
self.bits.pos = value
def read_string(self):
""" Read an undelimited string with the length given in the first 4 bytes """
return self.bits.read('bytes:{0}'.format(self.read_int())).decode(
"utf-8", 'replace')
class FileReader:
""" Some basic functionality for reading data from Binary files. """
def __init__(self, filename):
self.filename = filename
if isinstance(filename, str):
self.file = open(filename, 'rb')
self.bits = ConstBitStream(open(filename, 'rb'))
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.file.close()
def skip_bytes(self,count=1):
self.bits.read('pad:{0}'.format(count*8))
def peek_int(self):
return self.bits.peek(32).uintle
def forward_to_first_non_zero_byte(self, start, end):
self.bits.pos = start
while self.bits.pos < end and self.bits.read(8).intle == 0:
pass
self.bits.pos -= 8
def read_strings_from_block(self, start, end, stopAtEmptyString=False):
self.bits.pos = start
r = []
while self.bits.pos < end:
s = self.read_string()
if s == "" and stopAtEmptyString:
return r
r.append(s)
return tuple(r)
def read_int(self):
""" Read a single little endian 4 byte integer """
return self.bits.read(32).intle
def findall(self,bs):
return self.bits.findall(bs,bytealigned=True)
def read_bytes(self, count):
return self.bits.read(count*8)
def read_byte(self, skip = 0):
i = self.bits.read(8).uint
if skip > 0:
self.skip_bytes(skip)
return i
def read_string_safe(self):
return self.bits.read('bytes:{0}'.format(self.read_byte(skip=3))).decode("utf-8", 'replace')
def find(self, bs, start, end):
return self.bits.find(bs,start, end, True )
def find_first(self, bs):
return self.bits.find(bs)
def extract_compressed_payloads(self):
files = []
occ = self.findall('0x789C')
i = 0
readSize = 2**12
for pos in occ:
self.bits.pos = pos
#read the start of the stream into a buffer.
if (self.bits.length - self.pos) < 8*2**12:
readSize = int((self.bits.length - self.pos) / 8)
buf = self.bits.read('bytes:{0}'.format(readSize))
zo = zlib.decompressobj()
#start the decompression
try:
stream = zo.decompress(buf)
except zlib.error: # right magic number but not a zlib stream.
continue
while zo.unused_data == b'' and readSize >= 2**12:
if (self.bits.length - self.pos) < 8*2**12:
readSize = int((self.bits.length - self.pos) / 8)
block = self.bits.read('bytes:{0}'.format(readSize))
if len(block)> 0:
try:
stream += zo.decompress(block)
except zlib.error:
pass
else:
break # we've reached EOF
with open(self.filename + '_' + str(i) + '.decompressed', 'wb') as fh:
fh.write(stream)
files.append(self.filename + '_' + str(i) + '.decompressed')
i+=1
return files
@property
def pos(self):
return self.bits.pos
@pos.setter
def pos(self, value):
self.bits.pos = value
def read_string(self):
""" Read an undelimited string with the length given in the first 4 bytes """
return self.bits.read('bytes:{0}'.format(self.read_int())).decode("utf-8", 'replace')
