class JokerContainerFile: file = None # Offset/sizes in bytes identifier_offset = 0 identifier_size = 5 mode_offset = identifier_offset + identifier_size mode_size = 2 data_offset = mode_offset + mode_size + 1 # Skips unknown byte 0x01 between mode and data mode_f100_offset = data_offset + (7 * 8) def __init__(self, f, read=True): if read: self.file = ConstBitStream(f.read()) else: self.file = ConstBitStream(f) self.file.pos = self.identifier_offset identifier = self.file.peek(f'bytes:{self.identifier_size}') if identifier != b'Joker': raise ValueError('Input not a Joker container file') @property def mode(self): self.file.pos = self.mode_offset * 8 return self.file.peek(f'uintle:{self.mode_size * 8}') @property def _data(self): return self.file[self.data_offset * 8:] @property def data(self): data = self._data if self.mode == 0x0300 or self.mode == 0x0200: # Compressed/encrypted decompressed = zlib.decompress(data.tobytes())[:-1] decoded = bytes.fromhex(base64.b64decode(decompressed, validate=True).decode()) data = JokerCipher.decrypt(decoded) if self.mode == 0x300: data = self._process_mode_300(data) elif self.mode == 0xF100: return data[self.mode_f100_offset:] return data @staticmethod def _process_mode_300(data): decoded = base64.decodebytes(data) decrypted = DefaultCipher.decrypt(decoded) decompressed = zlib.decompress(decrypted) return decompressed
def lzwv_decode(in_array): max_entries = 2**MAX_CODE_LEN in_stream = ConstBitStream(in_array) dictionary = {i: bytes([i]) for i in range(256)} code_len = 9 out_array = bytearray() while True: try: # Read a code k = in_stream.read(f'uint:{code_len}') # Look up code and emit value v = dictionary[k] out_array += v # Add v + v_next[0] to dictionary c = len(dictionary) if c < max_entries: if c == 2**code_len - 1: code_len = min(MAX_CODE_LEN, code_len + 1) k_next = in_stream.peek(f'uint:{code_len}') if k_next < c: v_next = dictionary[k_next] dictionary[c] = v + v_next[:1] else: dictionary[c] = v + v[:1] except ReadError: break return bytes(out_array)
def lzwf_decode(in_array, code_len=DEFAULT_CODE_LEN): max_entries = 2**code_len in_stream = ConstBitStream(in_array) dictionary = {i: bytes([i]) for i in range(256)} out_array = bytes() while True: try: # Read a code k = in_stream.read(f'uint:{code_len}') # Look up code and emit value v = dictionary[k] out_array = out_array + v # Add v + v_next[0] to dictionary c = len(dictionary) if c < max_entries: k_next = in_stream.peek(f'uint:{code_len}') if k_next < c: v_next = dictionary[k_next] dictionary[c] = v + v_next[:1] else: dictionary[c] = v + v[:1] except ReadError: break return bytes(out_array)
def huffman_decode(data, decoded_len, serialized_tree, symbol_bits=8): tree = deserialize_huffman_tree(serialized_tree, symbol_bits) dictionary, min_code_len = make_decoding_dictionary(tree) bits = ConstBitStream(data) out = BitArray() symbols_decoded = 0 try: while symbols_decoded < decoded_len: num_bits = min_code_len code = bits.peek(f'bin:{num_bits}') while code not in dictionary: num_bits += 1 code = bits.peek(f'bin:{num_bits}') out.append(Bits(uint=dictionary[code], length=symbol_bits)) symbols_decoded += 1 bits.read(f'bin:{num_bits}') except ReadError: pass return out.tobytes()
def _load_le_instr(self, bitstream: bitstring.ConstBitStream, numbits: int) -> str: # THUMB mode instructions swap endianness every two bytes! if (self.addr & 1) == 1 and numbits > 16: chunk = "" oldpos = bitstream.pos for _ in range(0, numbits, 16): chunk += bitstring.Bits(uint=bitstream.peek("uintle:%d" % 16), length=16).bin bitstream.pos += 16 bitstream.pos = oldpos return chunk return super()._load_le_instr(bitstream, numbits)
def process_tm_packet(binary_packet): bits = ConstBitStream(binary_packet) header = { k: v for k, v in zip(tm_header.keys(), bits.readlist(tm_header.values())) } if header['service_type'] in [3, 21 ] and header['service_subtype'] in [25, 6]: try: ssid = bits.peek('uint:8') header['ssid'] = ssid except ReadError as e: header['ssid'] = -1 if header['data_len'] + 7 != len(bits.hex) / 2: raise ValueError('Expected and actual packet lentght do not match') return header, bits.hex
def from_bytes(cls, bitstream): ''' Look at the type of the message, instantiate the correct class and let it parse the message. ''' from pylisp.packet.lisp.control import type_registry # Convert to ConstBitStream (if not already provided) if not isinstance(bitstream, ConstBitStream): if isinstance(bitstream, Bits): bitstream = ConstBitStream(auto=bitstream) else: bitstream = ConstBitStream(bytes=bitstream) # Peek at the bitstream to see which type it is type_nr = bitstream.peek('uint:4') # Look for the right class type_class = type_registry.get_type_class(type_nr) if not type_class: raise ValueError("Can't handle message type {0}".format(type_nr)) # Let the specific class handle it from now on return type_class.from_bytes(bitstream)
#!/usr/bin/env python # -*- coding: utf-8 -*- from bitstring import ConstBitStream import sys c = ConstBitStream(filename=sys.argv[1]) #Most log entry ends with 0A, this helps us to seek to the beginning of an entry start = c.find('0x0A', bytealigned=True)[0] #Seek 8 byte into the stream to skip EOL from previus log entry start += 8 c.pos = start while True: #Read and print the binary log header header = c.readlist('8*uintle:32') print header #Get the size in bits of the log message msgSize = (header[0] * 8 - 256) #Move pointer after message c.pos += msgSize #Check if EOL is present after message, if true, move pointer 8 byte if c.peek(8).hex == '0a': c.pos += 8
def from_bytes(cls, bitstream, decode_payload=True): ''' Parse the given packet and update properties accordingly ''' packet = cls() # Convert to ConstBitStream (if not already provided) if not isinstance(bitstream, ConstBitStream): if isinstance(bitstream, Bits): bitstream = ConstBitStream(auto=bitstream) else: bitstream = ConstBitStream(bytes=bitstream) # Read the version version = bitstream.read('uint:4') if version != packet.version: raise ValueError('Provided bytes do not contain an IPv4 packet') # Read the header length ihl = bitstream.read('uint:4') if ihl < 5: raise ValueError('Invalid IPv4 header length') # Now that we know the length of the header we store it to be able # to easily recalculate the header checksum later remaining_header_bits = (ihl * 32) - 8 header = (BitStream('uint:4=4, uint:4=%d' % ihl) + bitstream.peek(remaining_header_bits)) # Read the type of service packet.tos = bitstream.read('uint:8') # Read the total length total_length = bitstream.read('uint:16') if total_length < ihl * 4: raise ValueError('Total length is shorter than the header') # Read the identification packet.identification = bitstream.read('uint:16') # Read the flags (reserved, packet.dont_fragment, packet.more_fragments) = bitstream.readlist('3*bool') if reserved: raise ValueError('Reserved flag must be 0') # Read the fragment offset packet.fragment_offset = bitstream.read('uint:13') # Read the TTL packet.ttl = bitstream.read('uint:8') # Read the protocol number packet.protocol = bitstream.read('uint:8') # Read the header checksum header_checksum = bitstream.read('uint:16') # Set the checksum bits in the header to 0 and re-calculate header[80:96] = BitStream(16) my_checksum = checksum.ones_complement(header.bytes) if my_checksum != header_checksum: raise ValueError('Header checksum does not match') # Read the source and destination addresses packet.source = IPv4Address(bitstream.read('uint:32')) packet.destination = IPv4Address(bitstream.read('uint:32')) # Read the options option_len = (ihl - 5) * 4 packet.options = bitstream.read('bytes:%d' % option_len) # And the rest is payload payload_bytes = (total_length) - (ihl * 4) packet.payload = bitstream.read('bytes:%d' % payload_bytes) if decode_payload: payload_class = protocol_registry.get_type_class(packet.protocol) if payload_class: packet.payload = payload_class.from_bytes(packet.payload) # There should be no remaining bits if bitstream.pos != bitstream.len: raise ValueError('Bits remaining after processing packet') # Verify that the properties make sense packet.sanitize() return packet
def _load_le_instr(self, bitstream: bitstring.ConstBitStream, numbits: int) -> str: return bitstring.Bits(uint=bitstream.peek("uintle:%d" % numbits), length=numbits).bin
class FileReader: """ Some basic functionality for reading data from Binary files. """ def __init__(self, filename): self.filename = filename if isinstance(filename, str): self.file = open(filename, 'rb') self.bits = ConstBitStream(open(filename, 'rb')) def __enter__(self): return self def __exit__(self, type, value, traceback): self.file.close() def skip_bytes(self, count=1): self.bits.read('pad:{0}'.format(count * 8)) def peek_int(self): return self.bits.peek(32).uintle def forward_to_first_non_zero_byte(self, start, end): self.bits.pos = start while self.bits.pos < end and self.bits.read(8).intle == 0: pass self.bits.pos -= 8 def read_strings_from_block(self, start, end, stopAtEmptyString=False): self.bits.pos = start r = [] while self.bits.pos < end: s = self.read_string() if s == "" and stopAtEmptyString: return r r.append(s) return tuple(r) def read_int(self): """ Read a single little endian 4 byte integer """ return self.bits.read(32).intle def findall(self, bs): return self.bits.findall(bs, bytealigned=True) def read_bytes(self, count): return self.bits.read(count * 8) def read_byte(self, skip=0): i = self.bits.read(8).uint if skip > 0: self.skip_bytes(skip) return i def read_string_safe(self): return self.bits.read('bytes:{0}'.format( self.read_byte(skip=3))).decode("utf-8", 'replace') def find(self, bs, start, end): return self.bits.find(bs, start, end, True) def find_first(self, bs): return self.bits.find(bs) def extract_compressed_payloads(self): files = [] occ = self.findall('0x789C') i = 0 readSize = 2**12 for pos in occ: self.bits.pos = pos #read the start of the stream into a buffer. if (self.bits.length - self.pos) < 8 * 2**12: readSize = int((self.bits.length - self.pos) / 8) buf = self.bits.read('bytes:{0}'.format(readSize)) zo = zlib.decompressobj() #start the decompression try: stream = zo.decompress(buf) except zlib.error: # right magic number but not a zlib stream. continue while zo.unused_data == b'' and readSize >= 2**12: if (self.bits.length - self.pos) < 8 * 2**12: readSize = int((self.bits.length - self.pos) / 8) block = self.bits.read('bytes:{0}'.format(readSize)) if len(block) > 0: try: stream += zo.decompress(block) except zlib.error: pass else: break # we've reached EOF with open(self.filename + '_' + str(i) + '.decompressed', 'wb') as fh: fh.write(stream) files.append(self.filename + '_' + str(i) + '.decompressed') i += 1 return files @property def pos(self): return self.bits.pos @pos.setter def pos(self, value): self.bits.pos = value def read_string(self): """ Read an undelimited string with the length given in the first 4 bytes """ return self.bits.read('bytes:{0}'.format(self.read_int())).decode( "utf-8", 'replace')
class FileReader: """ Some basic functionality for reading data from Binary files. """ def __init__(self, filename): self.filename = filename if isinstance(filename, str): self.file = open(filename, 'rb') self.bits = ConstBitStream(open(filename, 'rb')) def __enter__(self): return self def __exit__(self, type, value, traceback): self.file.close() def skip_bytes(self,count=1): self.bits.read('pad:{0}'.format(count*8)) def peek_int(self): return self.bits.peek(32).uintle def forward_to_first_non_zero_byte(self, start, end): self.bits.pos = start while self.bits.pos < end and self.bits.read(8).intle == 0: pass self.bits.pos -= 8 def read_strings_from_block(self, start, end, stopAtEmptyString=False): self.bits.pos = start r = [] while self.bits.pos < end: s = self.read_string() if s == "" and stopAtEmptyString: return r r.append(s) return tuple(r) def read_int(self): """ Read a single little endian 4 byte integer """ return self.bits.read(32).intle def findall(self,bs): return self.bits.findall(bs,bytealigned=True) def read_bytes(self, count): return self.bits.read(count*8) def read_byte(self, skip = 0): i = self.bits.read(8).uint if skip > 0: self.skip_bytes(skip) return i def read_string_safe(self): return self.bits.read('bytes:{0}'.format(self.read_byte(skip=3))).decode("utf-8", 'replace') def find(self, bs, start, end): return self.bits.find(bs,start, end, True ) def find_first(self, bs): return self.bits.find(bs) def extract_compressed_payloads(self): files = [] occ = self.findall('0x789C') i = 0 readSize = 2**12 for pos in occ: self.bits.pos = pos #read the start of the stream into a buffer. if (self.bits.length - self.pos) < 8*2**12: readSize = int((self.bits.length - self.pos) / 8) buf = self.bits.read('bytes:{0}'.format(readSize)) zo = zlib.decompressobj() #start the decompression try: stream = zo.decompress(buf) except zlib.error: # right magic number but not a zlib stream. continue while zo.unused_data == b'' and readSize >= 2**12: if (self.bits.length - self.pos) < 8*2**12: readSize = int((self.bits.length - self.pos) / 8) block = self.bits.read('bytes:{0}'.format(readSize)) if len(block)> 0: try: stream += zo.decompress(block) except zlib.error: pass else: break # we've reached EOF with open(self.filename + '_' + str(i) + '.decompressed', 'wb') as fh: fh.write(stream) files.append(self.filename + '_' + str(i) + '.decompressed') i+=1 return files @property def pos(self): return self.bits.pos @pos.setter def pos(self, value): self.bits.pos = value def read_string(self): """ Read an undelimited string with the length given in the first 4 bytes """ return self.bits.read('bytes:{0}'.format(self.read_int())).decode("utf-8", 'replace')