def decomp_worker(in_queue, pipe_filepath, time_queue, temp_overlay_file=None):
start_time = time.time()
data_size = 0
counter = 0
decompressor = LZMADecompressor()
pipe = open(pipe_filepath, "w")
while True:
chunk = in_queue.get()
if chunk == Synthesis_Const.END_OF_FILE:
break
data_size = data_size + len(chunk)
decomp_chunk = decompressor.decompress(chunk)
in_queue.task_done()
pipe.write(decomp_chunk)
if temp_overlay_file:
temp_overlay_file.write(decomp_chunk)
counter = counter + 1
decomp_chunk = decompressor.flush()
pipe.write(decomp_chunk)
pipe.close()
if temp_overlay_file:
temp_overlay_file.write(decomp_chunk)
temp_overlay_file.close()
end_time = time.time()
time_queue.put({'start_time':start_time, 'end_time':end_time})
print "[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % \
(start_time, end_time, (end_time-start_time),
counter, data_size)
class DecompStepProc(Process):
def __init__(self,
input_queue,
output_path,
time_queue,
temp_overlay_file=None):
self.input_queue = input_queue
self.time_queue = time_queue
self.output_path = output_path
self.decompressor = LZMADecompressor()
self.temp_overlay_file = temp_overlay_file
Process.__init__(self, target=self.decompress_blobs)
def exception_handler(self):
LOG.error("decompress step error")
@wrap_process_fault
def decompress_blobs(self):
self.output_queue = open(self.output_path, "w")
start_time = time.time()
data_size = 0
counter = 0
while True:
chunk = self.input_queue.get()
if chunk == Synthesis_Const.END_OF_FILE:
break
if chunk == Synthesis_Const.ERROR_OCCURED:
break
data_size = data_size + len(chunk)
decomp_chunk = self.decompressor.decompress(chunk)
self.input_queue.task_done()
self.output_queue.write(decomp_chunk)
counter = counter + 1
if self.temp_overlay_file:
self.temp_overlay_file.write(decomp_chunk)
decomp_chunk = self.decompressor.flush()
self.output_queue.write(decomp_chunk)
self.output_queue.close()
if self.temp_overlay_file:
self.temp_overlay_file.write(decomp_chunk)
self.temp_overlay_file.close()
end_time = time.time()
self.time_queue.put({'start_time': start_time, 'end_time': end_time})
LOG.info("[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % \
(start_time, end_time, (end_time-start_time),
counter, data_size))
def decompress(srcfd, destfd, bufsize=DEFAULT_BUFSIZE):
setproctitle('mob decompression')
log.debug("Starting decompression in process %d" % os.getpid())
decompressor = LZMADecompressor()
src = os.fdopen(srcfd, 'rb')
dest = os.fdopen(destfd, 'wb')
while True:
buf = src.read(bufsize)
if not buf:
break
dest.write(decompressor.decompress(buf))
dest.write(decompressor.flush())
log.debug("Decompression finished")
class DecompStepProc(Process):
def __init__(self, input_queue, output_path, time_queue, temp_overlay_file=None):
self.input_queue = input_queue
self.time_queue = time_queue
self.output_path = output_path
self.decompressor = LZMADecompressor()
self.temp_overlay_file = temp_overlay_file
Process.__init__(self, target=self.decompress_blobs)
def exception_handler(self):
LOG.error("decompress step error")
@wrap_process_fault
def decompress_blobs(self):
self.output_queue = open(self.output_path, "w")
start_time = time.time()
data_size = 0
counter = 0
while True:
chunk = self.input_queue.get()
if chunk == Synthesis_Const.END_OF_FILE:
break
if chunk == Synthesis_Const.ERROR_OCCURED:
break;
data_size = data_size + len(chunk)
decomp_chunk = self.decompressor.decompress(chunk)
self.input_queue.task_done()
self.output_queue.write(decomp_chunk)
counter = counter + 1
if self.temp_overlay_file:
self.temp_overlay_file.write(decomp_chunk)
decomp_chunk = self.decompressor.flush()
self.output_queue.write(decomp_chunk)
self.output_queue.close()
if self.temp_overlay_file:
self.temp_overlay_file.write(decomp_chunk)
self.temp_overlay_file.close()
end_time = time.time()
self.time_queue.put({'start_time':start_time, 'end_time':end_time})
LOG.info("[Decomp] : (%s)-(%s)=(%s) (%d loop, %d bytes)" % \
(start_time, end_time, (end_time-start_time),
counter, data_size))
def decomp_overlay(meta, output_path):
meta_dict = msgpack.unpackb(open(meta, "r").read())
decomp_start_time = time()
comp_overlay_files = meta_dict[Const.META_OVERLAY_FILES]
comp_overlay_files = [item[Const.META_OVERLAY_FILE_NAME] for item in comp_overlay_files]
comp_overlay_files = [os.path.join(os.path.dirname(meta), item) for item in comp_overlay_files]
overlay_file = open(output_path, "w+b")
for comp_file in comp_overlay_files:
decompressor = LZMADecompressor()
comp_data = open(comp_file, "r").read()
decomp_data = decompressor.decompress(comp_data)
decomp_data += decompressor.flush()
overlay_file.write(decomp_data)
LOG.debug("Overlay decomp time for %d files: %f at %s\n" % \
(len(comp_overlay_files), (time()-decomp_start_time), output_path))
overlay_file.close()
return meta_dict
def decomp_overlay(meta, output_path):
meta_dict = msgpack.unpackb(open(meta, "r").read())
decomp_start_time = time()
comp_overlay_files = meta_dict[Const.META_OVERLAY_FILES]
comp_overlay_files = [item[Const.META_OVERLAY_FILE_NAME] for item in comp_overlay_files]
comp_overlay_files = [os.path.join(os.path.dirname(meta), item) for item in comp_overlay_files]
overlay_file = open(output_path, "w+b")
for comp_file in comp_overlay_files:
decompressor = LZMADecompressor()
comp_data = open(comp_file, "r").read()
decomp_data = decompressor.decompress(comp_data)
decomp_data += decompressor.flush()
overlay_file.write(decomp_data)
LOG.debug("Overlay decomp time for %d files: %f at %s\n" % \
(len(comp_overlay_files), (time()-decomp_start_time), output_path))
overlay_file.close()
return meta_dict
return delta_list
if __name__ == "__main__":
import random
import string
if sys.argv[1] == "comp":
base = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(2096))
compressor = LZMACompressor(LZMA_OPTION)
comp = compressor.compress(base)
comp += compressor.flush()
decompressor = LZMADecompressor()
decomp = decompressor.decompress(comp)
decomp += decompressor.flush()
if base != decomp:
print "result is wrong"
print "%d == %d" % (len(base), len(decomp))
sys.exit(1)
print "success"
elif sys.argv[1] == "xdelta":
base = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(4096))
modi = "~"*4096
patch = diff_data(base, modi, len(base))
recover = merge_data(base, patch, len(base))
if sha256(modi).digest() == sha256(recover).digest():
print "SUCCESS"
if __name__ == "__main__":
import random
import string
if sys.argv[1] == "comp":
base = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for x in range(2096))
compressor = LZMACompressor(LZMA_OPTION)
comp = compressor.compress(base)
comp += compressor.flush()
decompressor = LZMADecompressor()
decomp = decompressor.decompress(comp)
decomp += decompressor.flush()
if base != decomp:
print "result is wrong"
print "%d == %d" % (len(base), len(decomp))
sys.exit(1)
print "success"
elif sys.argv[1] == "xdelta":
base = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for x in range(4096))
modi = "~" * 4096
patch = diff_data(base, modi, len(base))
recover = merge_data(base, patch, len(base))