def __init__(self, urlstate_data=None, encryption_key=None): """ If urlstate is empty a new empty state instance will be returned. If urlstate is not empty the constructor will rebuild the state attribute objects from the urlstate string. :type urlstate_data: str :type encryption_key: str :rtype: State :param encryption_key: The key to be used for encryption. :param urlstate_data: A string created by the method urlstate in this class. :return: An instance of this class. """ self._state_dict = {} self.delete = False if urlstate_data and not encryption_key: raise ValueError("If an 'urlstate_data' is supplied 'encrypt_key' must be specified.") if urlstate_data is not None: urlstate_data = urlstate_data.encode("utf-8") urlstate_data = base64.urlsafe_b64decode(urlstate_data) lzma = LZMADecompressor() urlstate_data = lzma.decompress(urlstate_data) urlstate_data = _AESCipher(encryption_key).decrypt(urlstate_data) lzma = LZMADecompressor() urlstate_data = lzma.decompress(urlstate_data) urlstate_data = urlstate_data.decode("UTF-8") self._state_dict = json.loads(urlstate_data)
def __init__(self, urlstate_data=None, encryption_key=None): """ If urlstate is empty a new empty state instance will be returned. If urlstate is not empty the constructor will rebuild the state attribute objects from the urlstate string. :type urlstate_data: str :type encryption_key: str :rtype: State :param encryption_key: The key to be used for encryption. :param urlstate_data: A string created by the method urlstate in this class. :return: An instance of this class. """ self.delete = False if urlstate_data and not encryption_key: raise ValueError( "If an 'urlstate_data' is supplied 'encrypt_key' must be specified." ) if urlstate_data: urlstate_data = urlstate_data.encode("utf-8") urlstate_data = base64.urlsafe_b64decode(urlstate_data) lzma = LZMADecompressor() urlstate_data = lzma.decompress(urlstate_data) urlstate_data = _AESCipher(encryption_key).decrypt(urlstate_data) lzma = LZMADecompressor() urlstate_data = lzma.decompress(urlstate_data) urlstate_data = urlstate_data.decode("UTF-8") urlstate_data = json.loads(urlstate_data) super().__init__(urlstate_data or {})
def __init__(self, urlstate_data=None, encryption_key=None): """ If urlstate is empty a new empty state instance will be returned. If urlstate is not empty the constructor will rebuild the state attribute objects from the urlstate string. :type urlstate_data: str :type encryption_key: str :rtype: State :param encryption_key: The key to be used for encryption. :param urlstate_data: A string created by the method urlstate in this class. :return: An instance of this class. """ self._state_dict = {} self._delete = False if urlstate_data is not None: urlstate_data = urlstate_data.encode("utf-8") urlstate_data = base64.urlsafe_b64decode(urlstate_data) lzma = LZMADecompressor() urlstate_data = lzma.decompress(urlstate_data) urlstate_data = AESCipher(encryption_key).decrypt(urlstate_data) lzma = LZMADecompressor() urlstate_data = lzma.decompress(urlstate_data) urlstate_data = urlstate_data.decode("UTF-8") self._state_dict = json.loads(urlstate_data)
def __init__(self, urlstate_data=None, encryption_key=None): """ If urlstate is empty a new empty state instance will be returned. If urlstate is not empty the constructor will rebuild the state attribute objects from the urlstate string. :type urlstate_data: str :type encryption_key: str :rtype: State :param encryption_key: The key to be used for encryption. :param urlstate_data: A string created by the method urlstate in this class. :return: An instance of this class. """ self.delete = False urlstate_data = {} if urlstate_data is None else urlstate_data if urlstate_data and not encryption_key: raise ValueError("If an 'urlstate_data' is supplied 'encrypt_key' must be specified.") if urlstate_data: try: urlstate_data_bytes = urlstate_data.encode("utf-8") urlstate_data_b64decoded = base64.urlsafe_b64decode(urlstate_data_bytes) lzma = LZMADecompressor() urlstate_data_decompressed = lzma.decompress(urlstate_data_b64decoded) urlstate_data_decrypted = _AESCipher(encryption_key).decrypt( urlstate_data_decompressed ) lzma = LZMADecompressor() urlstate_data_decrypted_decompressed = lzma.decompress(urlstate_data_decrypted) urlstate_data_obj = json.loads(urlstate_data_decrypted_decompressed) except Exception as e: error_context = { "message": "Failed to load state data. Reinitializing empty state.", "reason": str(e), "urlstate_data": urlstate_data, } logger.warning(error_context) urlstate_data = {} else: urlstate_data = urlstate_data_obj session_id = ( urlstate_data[_SESSION_ID_KEY] if urlstate_data and _SESSION_ID_KEY in urlstate_data else uuid4().urn ) urlstate_data[_SESSION_ID_KEY] = session_id super().__init__(urlstate_data)
def decomp_worker(in_queue, pipe_filepath, time_queue, temp_overlay_file=None): start_time = time.time() data_size = 0 counter = 0 decompressor = LZMADecompressor() pipe = open(pipe_filepath, "w") while True: chunk = in_queue.get() if chunk == Synthesis_Const.END_OF_FILE: break data_size = data_size + len(chunk) decomp_chunk = decompressor.decompress(chunk) in_queue.task_done() pipe.write(decomp_chunk) if temp_overlay_file: temp_overlay_file.write(decomp_chunk) counter = counter + 1 decomp_chunk = decompressor.flush() pipe.write(decomp_chunk) pipe.close() if temp_overlay_file: temp_overlay_file.write(decomp_chunk) temp_overlay_file.close() end_time = time.time() time_queue.put({'start_time':start_time, 'end_time':end_time}) print "[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % \ (start_time, end_time, (end_time-start_time), counter, data_size)
def download(url, fi): d = LZMADecompressor() with get(url) as r: with open(fi + ".tmp", 'wb') as f: for chunk in r.iter_content(128): if not d.eof: f.write(d.decompress(chunk)) rename(fi + ".tmp", fi)
def main(): _, *grep_args, filename = sys.argv grep_cmd = ['egrep'] + grep_args gpg_cmd = ['gpg2', '--decrypt', '-o', '-', filename] fl = open(lock_path, 'wb') flock(fl, LOCK_EX) grep_process = None gpg_process = None try: grep_process = Popen(grep_cmd, stdin=PIPE) gpg_process = Popen(gpg_cmd, stdin=DEVNULL, stdout=PIPE, stderr=DEVNULL) head = b'' while True: if grep_process.returncode is not None: sys.exit('grep command has failed with returncode {}'.format(grep_process.returncode)) if gpg_process.returncode is not None: sys.exit('GPG command has failed with returncode {}'.format(gpg_process.returncode)) buf = gpg_process.stdout.read(65536) if buf == b'': if head == b'': sys.exit('Nothing received from gpg') break if fl is not None: fl.close() fl = None if head is not None: head += buf if len(head) < 100: continue else: if head.startswith(b'\xFD7zXZ\x00\x00'): decompressor = LZMADecompressor() else: decompressor = None buf = head head = None if decompressor is not None: buf = decompressor.decompress(buf) if decompressor.eof: assert not decompressor.unused_data try: grep_process.stdin.write(buf) except BrokenPipeError: sys.exit(1) gpg_process.wait() grep_process.stdin.close() grep_process.wait() grep_process = None finally: if gpg_process: gpg_process.terminate() if grep_process: grep_process.terminate() if gpg_process: gpg_process.wait() if grep_process: grep_process.wait()
class BlockRead: read_size = DEFAULT_BUFFER_SIZE def __init__( self, fileobj: IOAbstract, check: int, unpadded_size: int, uncompressed_size: int, ) -> None: self.length = uncompressed_size self.fileobj = IOCombiner( IOStatic(create_xz_header(check)), fileobj, IOStatic( create_xz_index_footer(check, [(unpadded_size, uncompressed_size)])), ) self.reset() def reset(self) -> None: self.fileobj.seek(0, SEEK_SET) self.pos = 0 self.decompressor = LZMADecompressor(format=FORMAT_XZ) def decompress(self, pos: int, size: int) -> bytes: if pos < self.pos: self.reset() skip_before = pos - self.pos # pylint: disable=using-constant-test if self.decompressor.eof: raise XZError("block: decompressor eof") if self.decompressor.needs_input: data_input = self.fileobj.read(self.read_size) if not data_input: raise XZError("block: data eof") else: data_input = b"" data_output = self.decompressor.decompress(data_input, skip_before + size) self.pos += len(data_output) if self.pos == self.length: # we reached the end of the block # according to the XZ specification, we must check the # remaining bytes of the block; this is mainly performed by the # decompressor itself when we consume it while not self.decompressor.eof: if self.decompress(self.pos, 1): raise LZMAError("Corrupt input data") return data_output[skip_before:]
def decompress( in_path, out_path ): with open( in_path, 'rb' ) as in_file, open( out_path, 'wb' ) as out_file: lzmad = LZMADecompressor( FORMAT_ALONE ) in_file.seek( 4, SEEK_SET ) data = in_file.read( 1024 ) while len( data ): out_file.write( lzmad.decompress( data ) ) data = in_file.read( 1024 )
def data(self): if self.compression == 4: if self._data is None: decompressor = LZMADecompressor(format=FORMAT_XZ) offset = self.offset + 1 self._data = b"" while decompressor.needs_input: idata = self.buf[offset:offset + 1024] self._data += decompressor.decompress(idata) offset += 1024 return self._data, 0 else: return self.buf, self.offset + 1
class DecompStepProc(Process): def __init__(self, input_queue, output_path, time_queue, temp_overlay_file=None): self.input_queue = input_queue self.time_queue = time_queue self.output_path = output_path self.decompressor = LZMADecompressor() self.temp_overlay_file = temp_overlay_file Process.__init__(self, target=self.decompress_blobs) def exception_handler(self): LOG.error("decompress step error") @wrap_process_fault def decompress_blobs(self): self.output_queue = open(self.output_path, "w") start_time = time.time() data_size = 0 counter = 0 while True: chunk = self.input_queue.get() if chunk == Synthesis_Const.END_OF_FILE: break if chunk == Synthesis_Const.ERROR_OCCURED: break data_size = data_size + len(chunk) decomp_chunk = self.decompressor.decompress(chunk) self.input_queue.task_done() self.output_queue.write(decomp_chunk) counter = counter + 1 if self.temp_overlay_file: self.temp_overlay_file.write(decomp_chunk) decomp_chunk = self.decompressor.flush() self.output_queue.write(decomp_chunk) self.output_queue.close() if self.temp_overlay_file: self.temp_overlay_file.write(decomp_chunk) self.temp_overlay_file.close() end_time = time.time() self.time_queue.put({'start_time': start_time, 'end_time': end_time}) LOG.info("[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % \ (start_time, end_time, (end_time-start_time), counter, data_size))
def decompress(srcfd, destfd, bufsize=DEFAULT_BUFSIZE): setproctitle('mob decompression') log.debug("Starting decompression in process %d" % os.getpid()) decompressor = LZMADecompressor() src = os.fdopen(srcfd, 'rb') dest = os.fdopen(destfd, 'wb') while True: buf = src.read(bufsize) if not buf: break dest.write(decompressor.decompress(buf)) dest.write(decompressor.flush()) log.debug("Decompression finished")
class DecompStepProc(Process): def __init__(self, input_queue, output_path, time_queue, temp_overlay_file=None): self.input_queue = input_queue self.time_queue = time_queue self.output_path = output_path self.decompressor = LZMADecompressor() self.temp_overlay_file = temp_overlay_file Process.__init__(self, target=self.decompress_blobs) def exception_handler(self): LOG.error("decompress step error") @wrap_process_fault def decompress_blobs(self): self.output_queue = open(self.output_path, "w") start_time = time.time() data_size = 0 counter = 0 while True: chunk = self.input_queue.get() if chunk == Synthesis_Const.END_OF_FILE: break if chunk == Synthesis_Const.ERROR_OCCURED: break; data_size = data_size + len(chunk) decomp_chunk = self.decompressor.decompress(chunk) self.input_queue.task_done() self.output_queue.write(decomp_chunk) counter = counter + 1 if self.temp_overlay_file: self.temp_overlay_file.write(decomp_chunk) decomp_chunk = self.decompressor.flush() self.output_queue.write(decomp_chunk) self.output_queue.close() if self.temp_overlay_file: self.temp_overlay_file.write(decomp_chunk) self.temp_overlay_file.close() end_time = time.time() self.time_queue.put({'start_time':start_time, 'end_time':end_time}) LOG.info("[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % \ (start_time, end_time, (end_time-start_time), counter, data_size))
def decomp_overlay(meta, output_path): meta_dict = msgpack.unpackb(open(meta, "r").read()) decomp_start_time = time() comp_overlay_files = meta_dict[Const.META_OVERLAY_FILES] comp_overlay_files = [item[Const.META_OVERLAY_FILE_NAME] for item in comp_overlay_files] comp_overlay_files = [os.path.join(os.path.dirname(meta), item) for item in comp_overlay_files] overlay_file = open(output_path, "w+b") for comp_file in comp_overlay_files: decompressor = LZMADecompressor() comp_data = open(comp_file, "r").read() decomp_data = decompressor.decompress(comp_data) decomp_data += decompressor.flush() overlay_file.write(decomp_data) LOG.debug("Overlay decomp time for %d files: %f at %s\n" % \ (len(comp_overlay_files), (time()-decomp_start_time), output_path)) overlay_file.close() return meta_dict
def decompress_lzma(data): results = [] len(data) while True: decomp = LZMADecompressor(FORMAT_AUTO, None, None) try: res = decomp.decompress(data) except LZMAError: if results: break else: raise results.append(res) data = decomp.unused_data if not data: break if not decomp.eof: raise LZMAError("Compressed data ended before the end-of-stream marker was reached") return b"".join(results)
def decompress_lzma(data): results = [] len(data) while True: decomp = LZMADecompressor(FORMAT_AUTO, None, None) try: res = decomp.decompress(data) except LZMAError: if results: break # Leftover data is not a valid LZMA/XZ stream; ignore it. else: raise # Error on the first iteration; bail out. results.append(res) data = decomp.unused_data if not data: break if not decomp.eof: raise LZMAError("Compressed data ended before the end-of-stream marker was reached") return b"".join(results)
def __lzma_decompress(self, compressed: bytes) -> bytes: """Decompress LZMA data.""" # Log the printf-friendly hex representation of the bytes to decompress hex = compressed.hex() hex = "\\x" + "\\x".join([hex[i:i + 2] for i in range(0, len(hex), 2)]) log.debug("Attempting LZMA decompression of bytes: {}".format(hex)) properties, dictionary_size, uncompressed_size = struct.unpack( "<BIQ", compressed[:13]) if properties > (4 * 5 + 4) * 9 + 8: log.warning("There seems to be an issue in the LZMA header") position_bits = properties // (9 * 5) literal_position_bits = (properties - position_bits * 9 * 5) // 9 literal_context_bits = ( properties - position_bits * 9 * 5) - literal_position_bits * 9 log.debug("LZMA dictionary_size={}".format(dictionary_size)) log.debug("LZMA uncompressed_size={}".format(uncompressed_size)) log.debug("LZMA literal_context_bits={}".format(literal_context_bits)) log.debug( "LZMA literal_position_bits={}".format(literal_position_bits)) log.debug("LZMA position_bits={}".format(position_bits)) if literal_context_bits + literal_position_bits > 4: log.warning( "literal_context_bits + litereal_position_bits > 4 which may indicate LZMA header issues" ) # lzma-js as used by PXT has a bug where the EOF marker is written incorrectly # disable it. # See: https://github.com/LZMA-JS/LZMA-JS/issues/44 # See: https://github.com/LZMA-JS/LZMA-JS/issues/54 decompressor = LZMADecompressor(lzma.FORMAT_ALONE, None, None) # This is sort of shady and may cause artefacts further down the road - # it does not smartly remove the end marker, but works for some tested # project files (also see workaround in __extract_sources) # it may have to be changed (6-7 seems to work well) return decompressor.decompress(compressed[:-6])
def get_debug_symbols(filename): print('Processing file:', filename) with open(filename, 'rb') as f: elffile = ELFFile(f) section_name = '.gnu_debugdata' debugdata = elffile.get_section_by_name(section_name) if not isinstance(debugdata, Section): print(' The file has no %s section' % section_name) return print(' Found %s section' % section_name) debugdata_filename = filename + '-symbols.elf' with open(debugdata_filename, 'wb') as f: print(' Extracting...') decompressor = LZMADecompressor(FORMAT_XZ) data = decompressor.decompress(debugdata.data()) f.write(data) symbols = [] with open(debugdata_filename, 'rb') as f: elffile = ELFFile(f) symbol_tables = [ s for s in elffile.iter_sections() if isinstance(s, SymbolTableSection) ] if not symbol_tables and elffile.num_sections() == 0: print(' INFO: No debug symbols.') for section in symbol_tables: symbols += [(symbol['st_value'], symbol.name) for symbol in section.iter_symbols() if len(symbol.name) != 0] remove(debugdata_filename) return symbols
def _decompress_lzma(self, data: bytes) -> bytes: ''' Correctly decompress LZMA data files. NOTE: https://stackoverflow.com/a/37400585/2895581 :params data: Bytes representation of response data. :returns outs: Bytes representation of decompressed LXMA data. ''' results: list = [] while True: decomp = LZMADecompressor(FORMAT_AUTO, None, None) try: res = decomp.decompress(data) # If there is leftover data, then it is not valid LZMA format and # we should ignore it. # If we encounter an error on the first iteration, bail out. except LZMAError: if results: break else: raise results.append(res) data = decomp.unused_data if not data: break if not decomp.eof: raise LZMAError( f'Compressed data ended before the end-of-stream marker was reached' ) return b"".join(results)
def bdecompress(cb): if cb is None: return None com = LZMADecompressor() chunk = com.decompress(cb) return chunk
fd.close() return delta_list if __name__ == "__main__": import random import string if sys.argv[1] == "comp": base = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(2096)) compressor = LZMACompressor(LZMA_OPTION) comp = compressor.compress(base) comp += compressor.flush() decompressor = LZMADecompressor() decomp = decompressor.decompress(comp) decomp += decompressor.flush() if base != decomp: print "result is wrong" print "%d == %d" % (len(base), len(decomp)) sys.exit(1) print "success" elif sys.argv[1] == "xdelta": base = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(4096)) modi = "~"*4096 patch = diff_data(base, modi, len(base)) recover = merge_data(base, patch, len(base)) if sha256(modi).digest() == sha256(recover).digest():
if __name__ == "__main__": import random import string if sys.argv[1] == "comp": base = ''.join( random.choice(string.ascii_uppercase + string.digits) for x in range(2096)) compressor = LZMACompressor(LZMA_OPTION) comp = compressor.compress(base) comp += compressor.flush() decompressor = LZMADecompressor() decomp = decompressor.decompress(comp) decomp += decompressor.flush() if base != decomp: print "result is wrong" print "%d == %d" % (len(base), len(decomp)) sys.exit(1) print "success" elif sys.argv[1] == "xdelta": base = ''.join( random.choice(string.ascii_uppercase + string.digits) for x in range(4096)) modi = "~" * 4096 patch = diff_data(base, modi, len(base)) recover = merge_data(base, patch, len(base))