def decomp_worker(in_queue, pipe_filepath, time_queue, temp_overlay_file=None): start_time = time.time() data_size = 0 counter = 0 decompressor = LZMADecompressor() pipe = open(pipe_filepath, "w") while True: chunk = in_queue.get() if chunk == Synthesis_Const.END_OF_FILE: break data_size = data_size + len(chunk) decomp_chunk = decompressor.decompress(chunk) in_queue.task_done() pipe.write(decomp_chunk) if temp_overlay_file: temp_overlay_file.write(decomp_chunk) counter = counter + 1 decomp_chunk = decompressor.flush() pipe.write(decomp_chunk) pipe.close() if temp_overlay_file: temp_overlay_file.write(decomp_chunk) temp_overlay_file.close() end_time = time.time() time_queue.put({'start_time':start_time, 'end_time':end_time}) print "[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % \ (start_time, end_time, (end_time-start_time), counter, data_size)
def __init__(self, urlstate_data=None, encryption_key=None): """ If urlstate is empty a new empty state instance will be returned. If urlstate is not empty the constructor will rebuild the state attribute objects from the urlstate string. :type urlstate_data: str :type encryption_key: str :rtype: State :param encryption_key: The key to be used for encryption. :param urlstate_data: A string created by the method urlstate in this class. :return: An instance of this class. """ self._state_dict = {} self.delete = False if urlstate_data and not encryption_key: raise ValueError("If an 'urlstate_data' is supplied 'encrypt_key' must be specified.") if urlstate_data is not None: urlstate_data = urlstate_data.encode("utf-8") urlstate_data = base64.urlsafe_b64decode(urlstate_data) lzma = LZMADecompressor() urlstate_data = lzma.decompress(urlstate_data) urlstate_data = _AESCipher(encryption_key).decrypt(urlstate_data) lzma = LZMADecompressor() urlstate_data = lzma.decompress(urlstate_data) urlstate_data = urlstate_data.decode("UTF-8") self._state_dict = json.loads(urlstate_data)
def __init__(self, input_queue, output_path, time_queue, temp_overlay_file=None): self.input_queue = input_queue self.time_queue = time_queue self.output_path = output_path self.decompressor = LZMADecompressor() self.temp_overlay_file = temp_overlay_file Process.__init__(self, target=self.decompress_blobs)
def download(url, fi): d = LZMADecompressor() with get(url) as r: with open(fi + ".tmp", 'wb') as f: for chunk in r.iter_content(128): if not d.eof: f.write(d.decompress(chunk)) rename(fi + ".tmp", fi)
def main(): _, *grep_args, filename = sys.argv grep_cmd = ['egrep'] + grep_args gpg_cmd = ['gpg2', '--decrypt', '-o', '-', filename] fl = open(lock_path, 'wb') flock(fl, LOCK_EX) grep_process = None gpg_process = None try: grep_process = Popen(grep_cmd, stdin=PIPE) gpg_process = Popen(gpg_cmd, stdin=DEVNULL, stdout=PIPE, stderr=DEVNULL) head = b'' while True: if grep_process.returncode is not None: sys.exit('grep command has failed with returncode {}'.format(grep_process.returncode)) if gpg_process.returncode is not None: sys.exit('GPG command has failed with returncode {}'.format(gpg_process.returncode)) buf = gpg_process.stdout.read(65536) if buf == b'': if head == b'': sys.exit('Nothing received from gpg') break if fl is not None: fl.close() fl = None if head is not None: head += buf if len(head) < 100: continue else: if head.startswith(b'\xFD7zXZ\x00\x00'): decompressor = LZMADecompressor() else: decompressor = None buf = head head = None if decompressor is not None: buf = decompressor.decompress(buf) if decompressor.eof: assert not decompressor.unused_data try: grep_process.stdin.write(buf) except BrokenPipeError: sys.exit(1) gpg_process.wait() grep_process.stdin.close() grep_process.wait() grep_process = None finally: if gpg_process: gpg_process.terminate() if grep_process: grep_process.terminate() if gpg_process: gpg_process.wait() if grep_process: grep_process.wait()
def decompress( in_path, out_path ): with open( in_path, 'rb' ) as in_file, open( out_path, 'wb' ) as out_file: lzmad = LZMADecompressor( FORMAT_ALONE ) in_file.seek( 4, SEEK_SET ) data = in_file.read( 1024 ) while len( data ): out_file.write( lzmad.decompress( data ) ) data = in_file.read( 1024 )
def try_decompress_at(input_file: bytes, offset: int) -> bytes: decoded = None try: if Signature.check(input_file, offset, Signature.Compressed_GZIP): decoded = SingleGzipReader(BytesIO(input_file[offset:])).read( -1 ) # Will stop reading after the GZip footer thanks to our modification above. elif (Signature.check(input_file, offset, Signature.Compressed_XZ) or Signature.check(input_file, offset, Signature.Compressed_LZMA)): try: decoded = LZMADecompressor().decompress( input_file[offset:] ) # Will discard the extra bytes and put it an attribute. except Exception: decoded = LZMADecompressor().decompress( input_file[offset:offset + 5] + b'\xff' * 8 + input_file[offset + 5:]) # pylzma format compatibility elif Signature.check(input_file, offset, Signature.Compressed_BZ2): decoded = BZ2Decompressor().decompress( input_file[offset:] ) # Will discard the extra bytes and put it an attribute. elif Signature.check(input_file, offset, Signature.Compressed_LZ4): try: LZ4Decompressor = importlib.import_module('lz4.frame') except ModuleNotFoundError: logging.error('ERROR: This kernel requres LZ4 decompression.') logging.error( ' But "lz4" python package does not found.') logging.error( ' Example installation command: "sudo pip3 install lz4"' ) logging.error() return context = LZ4Decompressor.create_decompression_context() decoded, bytes_read, end_of_frame = LZ4Decompressor.decompress_chunk( context, input_file[offset:]) except Exception: pass if decoded and len(decoded) > 0x1000: logging.info(( '[+] Kernel successfully decompressed in-memory (the offsets that ' + 'follow will be given relative to the decompressed binary)')) return decoded
def data(self): if self.compression == 4: if self._data is None: decompressor = LZMADecompressor(format=FORMAT_XZ) offset = self.offset + 1 self._data = b"" while decompressor.needs_input: idata = self.buf[offset:offset + 1024] self._data += decompressor.decompress(idata) offset += 1024 return self._data, 0 else: return self.buf, self.offset + 1
class DecompStepProc(Process): def __init__(self, input_queue, output_path, time_queue, temp_overlay_file=None): self.input_queue = input_queue self.time_queue = time_queue self.output_path = output_path self.decompressor = LZMADecompressor() self.temp_overlay_file = temp_overlay_file Process.__init__(self, target=self.decompress_blobs) def exception_handler(self): LOG.error("decompress step error") @wrap_process_fault def decompress_blobs(self): self.output_queue = open(self.output_path, "w") start_time = time.time() data_size = 0 counter = 0 while True: chunk = self.input_queue.get() if chunk == Synthesis_Const.END_OF_FILE: break if chunk == Synthesis_Const.ERROR_OCCURED: break data_size = data_size + len(chunk) decomp_chunk = self.decompressor.decompress(chunk) self.input_queue.task_done() self.output_queue.write(decomp_chunk) counter = counter + 1 if self.temp_overlay_file: self.temp_overlay_file.write(decomp_chunk) decomp_chunk = self.decompressor.flush() self.output_queue.write(decomp_chunk) self.output_queue.close() if self.temp_overlay_file: self.temp_overlay_file.write(decomp_chunk) self.temp_overlay_file.close() end_time = time.time() self.time_queue.put({'start_time': start_time, 'end_time': end_time}) LOG.info("[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % \ (start_time, end_time, (end_time-start_time), counter, data_size))
def decompress(srcfd, destfd, bufsize=DEFAULT_BUFSIZE): setproctitle('mob decompression') log.debug("Starting decompression in process %d" % os.getpid()) decompressor = LZMADecompressor() src = os.fdopen(srcfd, 'rb') dest = os.fdopen(destfd, 'wb') while True: buf = src.read(bufsize) if not buf: break dest.write(decompressor.decompress(buf)) dest.write(decompressor.flush()) log.debug("Decompression finished")
class DecompStepProc(Process): def __init__(self, input_queue, output_path, time_queue, temp_overlay_file=None): self.input_queue = input_queue self.time_queue = time_queue self.output_path = output_path self.decompressor = LZMADecompressor() self.temp_overlay_file = temp_overlay_file Process.__init__(self, target=self.decompress_blobs) def exception_handler(self): LOG.error("decompress step error") @wrap_process_fault def decompress_blobs(self): self.output_queue = open(self.output_path, "w") start_time = time.time() data_size = 0 counter = 0 while True: chunk = self.input_queue.get() if chunk == Synthesis_Const.END_OF_FILE: break if chunk == Synthesis_Const.ERROR_OCCURED: break; data_size = data_size + len(chunk) decomp_chunk = self.decompressor.decompress(chunk) self.input_queue.task_done() self.output_queue.write(decomp_chunk) counter = counter + 1 if self.temp_overlay_file: self.temp_overlay_file.write(decomp_chunk) decomp_chunk = self.decompressor.flush() self.output_queue.write(decomp_chunk) self.output_queue.close() if self.temp_overlay_file: self.temp_overlay_file.write(decomp_chunk) self.temp_overlay_file.close() end_time = time.time() self.time_queue.put({'start_time':start_time, 'end_time':end_time}) LOG.info("[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % \ (start_time, end_time, (end_time-start_time), counter, data_size))
def decomp_overlay(meta, output_path): meta_dict = msgpack.unpackb(open(meta, "r").read()) decomp_start_time = time() comp_overlay_files = meta_dict[Const.META_OVERLAY_FILES] comp_overlay_files = [item[Const.META_OVERLAY_FILE_NAME] for item in comp_overlay_files] comp_overlay_files = [os.path.join(os.path.dirname(meta), item) for item in comp_overlay_files] overlay_file = open(output_path, "w+b") for comp_file in comp_overlay_files: decompressor = LZMADecompressor() comp_data = open(comp_file, "r").read() decomp_data = decompressor.decompress(comp_data) decomp_data += decompressor.flush() overlay_file.write(decomp_data) LOG.debug("Overlay decomp time for %d files: %f at %s\n" % \ (len(comp_overlay_files), (time()-decomp_start_time), output_path)) overlay_file.close() return meta_dict
class BlockRead: read_size = DEFAULT_BUFFER_SIZE def __init__( self, fileobj: IOAbstract, check: int, unpadded_size: int, uncompressed_size: int, ) -> None: self.length = uncompressed_size self.fileobj = IOCombiner( IOStatic(create_xz_header(check)), fileobj, IOStatic( create_xz_index_footer(check, [(unpadded_size, uncompressed_size)])), ) self.reset() def reset(self) -> None: self.fileobj.seek(0, SEEK_SET) self.pos = 0 self.decompressor = LZMADecompressor(format=FORMAT_XZ) def decompress(self, pos: int, size: int) -> bytes: if pos < self.pos: self.reset() skip_before = pos - self.pos # pylint: disable=using-constant-test if self.decompressor.eof: raise XZError("block: decompressor eof") if self.decompressor.needs_input: data_input = self.fileobj.read(self.read_size) if not data_input: raise XZError("block: data eof") else: data_input = b"" data_output = self.decompressor.decompress(data_input, skip_before + size) self.pos += len(data_output) if self.pos == self.length: # we reached the end of the block # according to the XZ specification, we must check the # remaining bytes of the block; this is mainly performed by the # decompressor itself when we consume it while not self.decompressor.eof: if self.decompress(self.pos, 1): raise LZMAError("Corrupt input data") return data_output[skip_before:]
def decompress_lzma(data): results = [] len(data) while True: decomp = LZMADecompressor(FORMAT_AUTO, None, None) try: res = decomp.decompress(data) except LZMAError: if results: break else: raise results.append(res) data = decomp.unused_data if not data: break if not decomp.eof: raise LZMAError("Compressed data ended before the end-of-stream marker was reached") return b"".join(results)
def decompress_lzma(data): results = [] len(data) while True: decomp = LZMADecompressor(FORMAT_AUTO, None, None) try: res = decomp.decompress(data) except LZMAError: if results: break # Leftover data is not a valid LZMA/XZ stream; ignore it. else: raise # Error on the first iteration; bail out. results.append(res) data = decomp.unused_data if not data: break if not decomp.eof: raise LZMAError("Compressed data ended before the end-of-stream marker was reached") return b"".join(results)
async def download(self, index): await self.update() if index not in self.block_files: raise FileNotFoundError block_file = self.block_files[index] if block_file.partial or not block_file.complete: raise FileNotFoundError loop = asyncio.get_running_loop() lz = LZMADecompressor() async for chunk in self._session.download(block_file.siapath): yield await loop.run_in_executor(None, lz.decompress, chunk)
def __init__(self, urlstate_data=None, encryption_key=None): """ If urlstate is empty a new empty state instance will be returned. If urlstate is not empty the constructor will rebuild the state attribute objects from the urlstate string. :type urlstate_data: str :type encryption_key: str :rtype: State :param encryption_key: The key to be used for encryption. :param urlstate_data: A string created by the method urlstate in this class. :return: An instance of this class. """ self._state_dict = {} self._delete = False if urlstate_data is not None: urlstate_data = urlstate_data.encode("utf-8") urlstate_data = base64.urlsafe_b64decode(urlstate_data) lzma = LZMADecompressor() urlstate_data = lzma.decompress(urlstate_data) urlstate_data = AESCipher(encryption_key).decrypt(urlstate_data) lzma = LZMADecompressor() urlstate_data = lzma.decompress(urlstate_data) urlstate_data = urlstate_data.decode("UTF-8") self._state_dict = json.loads(urlstate_data)
def __init__(self, urlstate_data=None, encryption_key=None): """ If urlstate is empty a new empty state instance will be returned. If urlstate is not empty the constructor will rebuild the state attribute objects from the urlstate string. :type urlstate_data: str :type encryption_key: str :rtype: State :param encryption_key: The key to be used for encryption. :param urlstate_data: A string created by the method urlstate in this class. :return: An instance of this class. """ self.delete = False if urlstate_data and not encryption_key: raise ValueError( "If an 'urlstate_data' is supplied 'encrypt_key' must be specified." ) if urlstate_data: urlstate_data = urlstate_data.encode("utf-8") urlstate_data = base64.urlsafe_b64decode(urlstate_data) lzma = LZMADecompressor() urlstate_data = lzma.decompress(urlstate_data) urlstate_data = _AESCipher(encryption_key).decrypt(urlstate_data) lzma = LZMADecompressor() urlstate_data = lzma.decompress(urlstate_data) urlstate_data = urlstate_data.decode("UTF-8") urlstate_data = json.loads(urlstate_data) super().__init__(urlstate_data or {})
def __lzma_decompress(self, compressed: bytes) -> bytes: """Decompress LZMA data.""" # Log the printf-friendly hex representation of the bytes to decompress hex = compressed.hex() hex = "\\x" + "\\x".join([hex[i:i + 2] for i in range(0, len(hex), 2)]) log.debug("Attempting LZMA decompression of bytes: {}".format(hex)) properties, dictionary_size, uncompressed_size = struct.unpack( "<BIQ", compressed[:13]) if properties > (4 * 5 + 4) * 9 + 8: log.warning("There seems to be an issue in the LZMA header") position_bits = properties // (9 * 5) literal_position_bits = (properties - position_bits * 9 * 5) // 9 literal_context_bits = ( properties - position_bits * 9 * 5) - literal_position_bits * 9 log.debug("LZMA dictionary_size={}".format(dictionary_size)) log.debug("LZMA uncompressed_size={}".format(uncompressed_size)) log.debug("LZMA literal_context_bits={}".format(literal_context_bits)) log.debug( "LZMA literal_position_bits={}".format(literal_position_bits)) log.debug("LZMA position_bits={}".format(position_bits)) if literal_context_bits + literal_position_bits > 4: log.warning( "literal_context_bits + litereal_position_bits > 4 which may indicate LZMA header issues" ) # lzma-js as used by PXT has a bug where the EOF marker is written incorrectly # disable it. # See: https://github.com/LZMA-JS/LZMA-JS/issues/44 # See: https://github.com/LZMA-JS/LZMA-JS/issues/54 decompressor = LZMADecompressor(lzma.FORMAT_ALONE, None, None) # This is sort of shady and may cause artefacts further down the road - # it does not smartly remove the end marker, but works for some tested # project files (also see workaround in __extract_sources) # it may have to be changed (6-7 seems to work well) return decompressor.decompress(compressed[:-6])
def get_debug_symbols(filename): print('Processing file:', filename) with open(filename, 'rb') as f: elffile = ELFFile(f) section_name = '.gnu_debugdata' debugdata = elffile.get_section_by_name(section_name) if not isinstance(debugdata, Section): print(' The file has no %s section' % section_name) return print(' Found %s section' % section_name) debugdata_filename = filename + '-symbols.elf' with open(debugdata_filename, 'wb') as f: print(' Extracting...') decompressor = LZMADecompressor(FORMAT_XZ) data = decompressor.decompress(debugdata.data()) f.write(data) symbols = [] with open(debugdata_filename, 'rb') as f: elffile = ELFFile(f) symbol_tables = [ s for s in elffile.iter_sections() if isinstance(s, SymbolTableSection) ] if not symbol_tables and elffile.num_sections() == 0: print(' INFO: No debug symbols.') for section in symbol_tables: symbols += [(symbol['st_value'], symbol.name) for symbol in section.iter_symbols() if len(symbol.name) != 0] remove(debugdata_filename) return symbols
def try_decompress_at(input_file: bytes, offset: int) -> bytes: decoded = None try: if input_file[offset:offset + 3] == b'\x1f\x8b\x08': # GZIP Signature decoded = SingleGzipReader(BytesIO(input_file[offset:])).read( -1 ) # Will stop reading after the GZip footer thanks to our modification above. elif input_file[offset:offset + 6] == b'\xfd7zXZ\x00' or input_file[ offset:offset + 3] == b']\x00\x00': # XZ/LZMA Signature try: decoded = LZMADecompressor().decompress( input_file[offset:] ) # Will discard the extra bytes and put it an attribute. except Exception: decoded = LZMADecompressor().decompress( input_file[offset:offset + 5] + b'\xff' * 8 + input_file[offset + 5:]) # pylzma format compatibility elif input_file[offset:offset + 3] == b'BZh': # BZ2 Signature decoded = BZ2Decompressor().decompress( input_file[offset:] ) # Will discard the extra bytes and put it an attribute. except Exception: pass if decoded and len(decoded) > 0x1000: print(( '[+] Kernel successfully decompressed in-memory (the offsets that ' + 'follow will be given relative to the decompressed binary)')) return decoded
def _decompress_lzma(self, data: bytes) -> bytes: ''' Correctly decompress LZMA data files. NOTE: https://stackoverflow.com/a/37400585/2895581 :params data: Bytes representation of response data. :returns outs: Bytes representation of decompressed LXMA data. ''' results: list = [] while True: decomp = LZMADecompressor(FORMAT_AUTO, None, None) try: res = decomp.decompress(data) # If there is leftover data, then it is not valid LZMA format and # we should ignore it. # If we encounter an error on the first iteration, bail out. except LZMAError: if results: break else: raise results.append(res) data = decomp.unused_data if not data: break if not decomp.eof: raise LZMAError( f'Compressed data ended before the end-of-stream marker was reached' ) return b"".join(results)
def __init__(self, urlstate_data=None, encryption_key=None): """ If urlstate is empty a new empty state instance will be returned. If urlstate is not empty the constructor will rebuild the state attribute objects from the urlstate string. :type urlstate_data: str :type encryption_key: str :rtype: State :param encryption_key: The key to be used for encryption. :param urlstate_data: A string created by the method urlstate in this class. :return: An instance of this class. """ self.delete = False urlstate_data = {} if urlstate_data is None else urlstate_data if urlstate_data and not encryption_key: raise ValueError("If an 'urlstate_data' is supplied 'encrypt_key' must be specified.") if urlstate_data: try: urlstate_data_bytes = urlstate_data.encode("utf-8") urlstate_data_b64decoded = base64.urlsafe_b64decode(urlstate_data_bytes) lzma = LZMADecompressor() urlstate_data_decompressed = lzma.decompress(urlstate_data_b64decoded) urlstate_data_decrypted = _AESCipher(encryption_key).decrypt( urlstate_data_decompressed ) lzma = LZMADecompressor() urlstate_data_decrypted_decompressed = lzma.decompress(urlstate_data_decrypted) urlstate_data_obj = json.loads(urlstate_data_decrypted_decompressed) except Exception as e: error_context = { "message": "Failed to load state data. Reinitializing empty state.", "reason": str(e), "urlstate_data": urlstate_data, } logger.warning(error_context) urlstate_data = {} else: urlstate_data = urlstate_data_obj session_id = ( urlstate_data[_SESSION_ID_KEY] if urlstate_data and _SESSION_ID_KEY in urlstate_data else uuid4().urn ) urlstate_data[_SESSION_ID_KEY] = session_id super().__init__(urlstate_data)
def __init__(self, fpatch, compression): if compression == 'lzma': self._decompressor = LZMADecompressor() elif compression == 'bz2': self._decompressor = BZ2Decompressor() elif compression == 'crle': self._decompressor = CrleDecompressor(patch_data_length(fpatch)) elif compression == 'none': self._decompressor = NoneDecompressor(patch_data_length(fpatch)) elif compression == 'heatshrink': self._decompressor = HeatshrinkDecompressor(patch_data_length(fpatch)) elif compression == 'zstd': self._decompressor = ZstdDecompressor(patch_data_length(fpatch)) elif compression == 'lz4': self._decompressor = Lz4Decompressor() else: raise Error(format_bad_compression_string(compression)) self._fpatch = fpatch
def decompress_raw(self) -> bytes: f = open(self.path, 'rb') magic_dict = { b'\x1f\x8b\x08': 'gzip', b'\x42\x5a\x68': 'bz2', b'\xfd\x37\x7a\x58\x5a\x00': 'lzma', b'\x5d\x00\x00': 'lzma', b'\x04\x22\x4d\x18': 'lz4.frame', b'\x28\xb5\x2f\xfd': 'zstandard', b'\x89\x4c\x5a\x4f\x00\x0d\x0a\x1a\x0a': 'lzo', } maxlen = max(len(x) for x in magic_dict) header = f.read(maxlen) f.seek(0) for magic, comp in magic_dict.items(): if header.startswith(magic): try: mod = importlib.import_module(comp) except ModuleNotFoundError: raise MissingDecompressorError( f'The Python module {comp!r} that is required ' f'to decompress kernel file {self.path} ' f'is not installed.') if comp == 'zstandard': # Technically a redundant import, this is just # to make your IDE happy :) import zstandard reader = zstandard.ZstdDecompressor().stream_reader(f) decomp = b'' while True: chunk = reader.read(1024 * 1024) if not chunk: break decomp += chunk return decomp elif comp == 'lzma': # Using .decompress() causes an error because of # no end-of-stream marker return LZMADecompressor().decompress(f.read()) else: return getattr(mod, 'decompress')(f.read()) return f.read()
def bdecompress(cb): if cb is None: return None com = LZMADecompressor() chunk = com.decompress(cb) return chunk
return delta_list if __name__ == "__main__": import random import string if sys.argv[1] == "comp": base = ''.join( random.choice(string.ascii_uppercase + string.digits) for x in range(2096)) compressor = LZMACompressor(LZMA_OPTION) comp = compressor.compress(base) comp += compressor.flush() decompressor = LZMADecompressor() decomp = decompressor.decompress(comp) decomp += decompressor.flush() if base != decomp: print "result is wrong" print "%d == %d" % (len(base), len(decomp)) sys.exit(1) print "success" elif sys.argv[1] == "xdelta": base = ''.join( random.choice(string.ascii_uppercase + string.digits) for x in range(4096)) modi = "~" * 4096 patch = diff_data(base, modi, len(base))
def reset(self) -> None: self.fileobj.seek(0, SEEK_SET) self.pos = 0 self.decompressor = LZMADecompressor(format=FORMAT_XZ)
fd.close() return delta_list if __name__ == "__main__": import random import string if sys.argv[1] == "comp": base = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(2096)) compressor = LZMACompressor(LZMA_OPTION) comp = compressor.compress(base) comp += compressor.flush() decompressor = LZMADecompressor() decomp = decompressor.decompress(comp) decomp += decompressor.flush() if base != decomp: print "result is wrong" print "%d == %d" % (len(base), len(decomp)) sys.exit(1) print "success" elif sys.argv[1] == "xdelta": base = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(4096)) modi = "~"*4096 patch = diff_data(base, modi, len(base)) recover = merge_data(base, patch, len(base))
def try_decompress_at(input_file: bytes, offset: int) -> bytes: decoded = None try: if Signature.check( input_file, offset, Signature.DTB_Appended_Qualcomm ): # Merely unpack a Qualcomm kernel file containing a magic and DTB offset at the start (so that offsets aren't wrong) dtb_offset_le = int.from_bytes(input_file[offset + 16:offset + 20], 'little') dtb_offset_be = int.from_bytes(input_file[offset + 16:offset + 20], 'big') decoded = input_file[offset + 20:offset + 20 + min(dtb_offset_le, dtb_offset_be)] elif Signature.check( input_file, offset, Signature.Android_Bootimg ): # Unpack an uncompressed Android Bootimg file, version 0, 1, 2 or 3 # See, for reference: # - https://github.com/osm0sis/mkbootimg/blob/master/unpackbootimg.c # - https://github.com/osm0sis/mkbootimg/blob/master/bootimg.h assert len(input_file) > 4096 header_version_raw = input_file[offset + 10 * 4:offset + 11 * 4] endianness = 'little' if header_version_raw in (b'\0\0\0\3', b'\3\0\0\0'): page_size = 4096 if header_version_raw == b'\0\0\0\3': endianness = 'big' else: page_size_raw = input_file[offset + 9 * 4:offset + 10 * 4] page_size_le = int.from_bytes(page_size_raw, 'little') page_size_be = int.from_bytes(page_size_raw, 'big') if page_size_le < page_size_be: page_size = page_size_le else: endianness = 'big' page_size = page_size_be kernel_size = int.from_bytes( input_file[offset + 2 * 4:offset + 3 * 4], endianness) assert len(input_file) > kernel_size > 0x1000 assert len(input_file) > page_size > 0x200 decoded = input_file[offset + page_size:offset + page_size + kernel_size] # Also try to re-unpack the output image in the case where the nested # kernel would start with a "UNCOMPRESSED_IMG" Qualcomm magic, for example decoded = try_decompress_at(decoded, 0) or decoded elif Signature.check(input_file, offset, Signature.Compressed_GZIP): decoded = SingleGzipReader(BytesIO(input_file[offset:])).read( -1 ) # GZIP - Will stop reading after the GZip footer thanks to our modification above. elif (Signature.check(input_file, offset, Signature.Compressed_XZ) or Signature.check(input_file, offset, Signature.Compressed_LZMA)): try: decoded = LZMADecompressor().decompress( input_file[offset:] ) # LZMA - Will discard the extra bytes and put it an attribute. except Exception: decoded = LZMADecompressor().decompress( input_file[offset:offset + 5] + b'\xff' * 8 + input_file[offset + 5:]) # pylzma format compatibility elif Signature.check(input_file, offset, Signature.Compressed_BZ2): decoded = BZ2Decompressor().decompress( input_file[offset:] ) # BZ2 - Will discard the extra bytes and put it an attribute. elif Signature.check(input_file, offset, Signature.Compressed_LZ4): # LZ4 support try: LZ4Decompressor = importlib.import_module('lz4.frame') except ModuleNotFoundError: logging.error('ERROR: This kernel requres LZ4 decompression.') logging.error(' But "lz4" python package was not found.') logging.error( ' Example installation command: "sudo pip3 install lz4"' ) logging.error() return context = LZ4Decompressor.create_decompression_context() decoded, bytes_read, end_of_frame = LZ4Decompressor.decompress_chunk( context, input_file[offset:]) elif Signature.check(input_file, offset, Signature.Compressed_LZ4_Legacy ): # LZ4 support (legacy format) try: from utils.lz4_legacy import decompress_lz4_buffer except ImportError: try: from vmlinux_to_elf.utils.lz4_legacy import decompress_lz4_buffer except ModuleNotFoundError: logging.error( 'ERROR: This kernel requres LZ4 decompression.') logging.error( ' But "lz4" python package was not found.') logging.error( ' Example installation command: "sudo pip3 install lz4"' ) logging.error() return decoded = decompress_lz4_buffer(BytesIO(input_file[offset:])) elif Signature.check(input_file, offset, Signature.Compressed_ZSTD): try: import zstandard as zstd except ModuleNotFoundError: logging.error('ERROR: This kernel requres ZSTD decompression.') logging.error( ' But "zstandard" python package was not found.') logging.error( ' Example installation command: "sudo pip3 install zstandard"' ) logging.error() return buf = BytesIO() context = zstd.ZstdDecompressor() for chunk in context.read_to_iter(BytesIO(input_file[offset:])): buf.write(chunk) buf.seek(0) decoded = buf.read() elif Signature.check(input_file, offset, Signature.Compressed_LZO): try: import lzo except ModuleNotFoundError: logging.error('ERROR: This kernel requres LZO decompression.') logging.error( ' But "python-lzo" python package was not found.') logging.error( ' Example installation command: "sudo pip3 install git+https://github.com/clubby789/python-lzo@b4e39df"' ) logging.error() return buf = BytesIO(input_file[offset:]) decoded = lzo.LzoFile(fileobj=buf, mode='rb').read() except Exception: pass if decoded and len(decoded) > 0x1000: logging.info(( '[+] Kernel successfully decompressed in-memory (the offsets that ' + 'follow will be given relative to the decompressed binary)')) return decoded