def decomp_worker(in_queue, pipe_filepath, time_queue, temp_overlay_file=None): start_time = time.time() data_size = 0 counter = 0 decompressor = LZMADecompressor() pipe = open(pipe_filepath, "w") while True: chunk = in_queue.get() if chunk == Synthesis_Const.END_OF_FILE: break data_size = data_size + len(chunk) decomp_chunk = decompressor.decompress(chunk) in_queue.task_done() pipe.write(decomp_chunk) if temp_overlay_file: temp_overlay_file.write(decomp_chunk) counter = counter + 1 decomp_chunk = decompressor.flush() pipe.write(decomp_chunk) pipe.close() if temp_overlay_file: temp_overlay_file.write(decomp_chunk) temp_overlay_file.close() end_time = time.time() time_queue.put({'start_time':start_time, 'end_time':end_time}) print "[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % \ (start_time, end_time, (end_time-start_time), counter, data_size)
class DecompStepProc(Process): def __init__(self, input_queue, output_path, time_queue, temp_overlay_file=None): self.input_queue = input_queue self.time_queue = time_queue self.output_path = output_path self.decompressor = LZMADecompressor() self.temp_overlay_file = temp_overlay_file Process.__init__(self, target=self.decompress_blobs) def exception_handler(self): LOG.error("decompress step error") @wrap_process_fault def decompress_blobs(self): self.output_queue = open(self.output_path, "w") start_time = time.time() data_size = 0 counter = 0 while True: chunk = self.input_queue.get() if chunk == Synthesis_Const.END_OF_FILE: break if chunk == Synthesis_Const.ERROR_OCCURED: break data_size = data_size + len(chunk) decomp_chunk = self.decompressor.decompress(chunk) self.input_queue.task_done() self.output_queue.write(decomp_chunk) counter = counter + 1 if self.temp_overlay_file: self.temp_overlay_file.write(decomp_chunk) decomp_chunk = self.decompressor.flush() self.output_queue.write(decomp_chunk) self.output_queue.close() if self.temp_overlay_file: self.temp_overlay_file.write(decomp_chunk) self.temp_overlay_file.close() end_time = time.time() self.time_queue.put({'start_time': start_time, 'end_time': end_time}) LOG.info("[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % \ (start_time, end_time, (end_time-start_time), counter, data_size))
def decompress(srcfd, destfd, bufsize=DEFAULT_BUFSIZE): setproctitle('mob decompression') log.debug("Starting decompression in process %d" % os.getpid()) decompressor = LZMADecompressor() src = os.fdopen(srcfd, 'rb') dest = os.fdopen(destfd, 'wb') while True: buf = src.read(bufsize) if not buf: break dest.write(decompressor.decompress(buf)) dest.write(decompressor.flush()) log.debug("Decompression finished")
class DecompStepProc(Process): def __init__(self, input_queue, output_path, time_queue, temp_overlay_file=None): self.input_queue = input_queue self.time_queue = time_queue self.output_path = output_path self.decompressor = LZMADecompressor() self.temp_overlay_file = temp_overlay_file Process.__init__(self, target=self.decompress_blobs) def exception_handler(self): LOG.error("decompress step error") @wrap_process_fault def decompress_blobs(self): self.output_queue = open(self.output_path, "w") start_time = time.time() data_size = 0 counter = 0 while True: chunk = self.input_queue.get() if chunk == Synthesis_Const.END_OF_FILE: break if chunk == Synthesis_Const.ERROR_OCCURED: break; data_size = data_size + len(chunk) decomp_chunk = self.decompressor.decompress(chunk) self.input_queue.task_done() self.output_queue.write(decomp_chunk) counter = counter + 1 if self.temp_overlay_file: self.temp_overlay_file.write(decomp_chunk) decomp_chunk = self.decompressor.flush() self.output_queue.write(decomp_chunk) self.output_queue.close() if self.temp_overlay_file: self.temp_overlay_file.write(decomp_chunk) self.temp_overlay_file.close() end_time = time.time() self.time_queue.put({'start_time':start_time, 'end_time':end_time}) LOG.info("[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % \ (start_time, end_time, (end_time-start_time), counter, data_size))
def decomp_overlay(meta, output_path): meta_dict = msgpack.unpackb(open(meta, "r").read()) decomp_start_time = time() comp_overlay_files = meta_dict[Const.META_OVERLAY_FILES] comp_overlay_files = [item[Const.META_OVERLAY_FILE_NAME] for item in comp_overlay_files] comp_overlay_files = [os.path.join(os.path.dirname(meta), item) for item in comp_overlay_files] overlay_file = open(output_path, "w+b") for comp_file in comp_overlay_files: decompressor = LZMADecompressor() comp_data = open(comp_file, "r").read() decomp_data = decompressor.decompress(comp_data) decomp_data += decompressor.flush() overlay_file.write(decomp_data) LOG.debug("Overlay decomp time for %d files: %f at %s\n" % \ (len(comp_overlay_files), (time()-decomp_start_time), output_path)) overlay_file.close() return meta_dict
return delta_list if __name__ == "__main__": import random import string if sys.argv[1] == "comp": base = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(2096)) compressor = LZMACompressor(LZMA_OPTION) comp = compressor.compress(base) comp += compressor.flush() decompressor = LZMADecompressor() decomp = decompressor.decompress(comp) decomp += decompressor.flush() if base != decomp: print "result is wrong" print "%d == %d" % (len(base), len(decomp)) sys.exit(1) print "success" elif sys.argv[1] == "xdelta": base = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(4096)) modi = "~"*4096 patch = diff_data(base, modi, len(base)) recover = merge_data(base, patch, len(base)) if sha256(modi).digest() == sha256(recover).digest(): print "SUCCESS"
if __name__ == "__main__": import random import string if sys.argv[1] == "comp": base = ''.join( random.choice(string.ascii_uppercase + string.digits) for x in range(2096)) compressor = LZMACompressor(LZMA_OPTION) comp = compressor.compress(base) comp += compressor.flush() decompressor = LZMADecompressor() decomp = decompressor.decompress(comp) decomp += decompressor.flush() if base != decomp: print "result is wrong" print "%d == %d" % (len(base), len(decomp)) sys.exit(1) print "success" elif sys.argv[1] == "xdelta": base = ''.join( random.choice(string.ascii_uppercase + string.digits) for x in range(4096)) modi = "~" * 4096 patch = diff_data(base, modi, len(base)) recover = merge_data(base, patch, len(base))