def decomp_overlay(meta, output_path):
meta_dict = msgpack.unpackb(open(meta, "r").read())
decomp_start_time = time()
comp_overlay_files = meta_dict[Const.META_OVERLAY_FILES]
comp_overlay_files = [item[Const.META_OVERLAY_FILE_NAME] for item in comp_overlay_files]
comp_overlay_files = [os.path.join(os.path.dirname(meta), item) for item in comp_overlay_files]
overlay_file = open(output_path, "w+b")
for comp_file in comp_overlay_files:
decompressor = LZMADecompressor()
comp_data = open(comp_file, "r").read()
decomp_data = decompressor.decompress(comp_data)
decomp_data += decompressor.flush()
overlay_file.write(decomp_data)
LOG.debug("Overlay decomp time for %d files: %f at %s\n" % \
(len(comp_overlay_files), (time()-decomp_start_time), output_path))
overlay_file.close()
return meta_dict
def piping_synthesis(overlay_url, base_path):
# check_base VM
start_time = time.time()
meta_stream = urllib2.urlopen(overlay_url)
meta_raw = read_all(meta_stream)
meta_info = msgpack.unpackb(meta_raw)
url_manager = Manager()
overlay_urls = url_manager.list()
url_prefix = os.path.dirname(overlay_url)
for blob in meta_info[Const.META_OVERLAY_FILES]:
blob_filename = os.path.basename(blob[Const.META_OVERLAY_FILE_NAME])
url = os.path.join(url_prefix, blob_filename)
overlay_urls.append(url)
(base_diskmeta, base_mem, base_memmeta) = \
Const.get_basepath(base_path, check_exist=True)
# read overlay files
# create named pipe to convert queue to stream
time_transfer = Queue(); time_decomp = Queue();
time_delta = Queue(); time_fuse = Queue();
tmp_dir = tempfile.mkdtemp()
temp_overlay_filepath = os.path.join(tmp_dir, "overlay_file")
temp_overlay_file = open(temp_overlay_filepath, "w+b")
overlay_pipe = os.path.join(tmp_dir, 'overlay_pipe')
os.mkfifo(overlay_pipe)
# overlay
demanding_queue = Queue()
download_queue = JoinableQueue()
download_process = Process(target=synthesis_server.network_worker,
args=(
overlay_urls, demanding_queue, download_queue, time_transfer, CHUNK_SIZE,
)
)
decomp_process = Process(target=synthesis_server.decomp_worker,
args=(
download_queue, overlay_pipe, time_decomp, temp_overlay_file,
)
)
modified_img, modified_mem, fuse, delta_proc, fuse_thread = \
synthesis.recover_launchVM(base_path, meta_info, overlay_pipe,
log=sys.stdout, demanding_queue=demanding_queue)
delta_proc.time_queue = time_delta
fuse_thread.time_queue = time_fuse
# start processes
download_process.start()
decomp_process.start()
delta_proc.start()
fuse_thread.start()
# wait for end
delta_proc.join()
fuse_thread.join()
# printout result
end_time = time.time()
total_time = (end_time-start_time)
synthesis_server.SynthesisTCPHandler.print_statistics(start_time, end_time, \
time_transfer, time_decomp, time_delta, time_fuse, \
print_out=sys.stdout)
delta_proc.finish()
if os.path.exists(overlay_pipe):
os.unlink(overlay_pipe)
shutil.rmtree(tmp_dir)
print "\n[Time] Total Time for synthesis(including download) : %f" % (total_time)
return fuse
def decoding(data):
return msgpack.unpackb(data)
def decoding(data):
return msgpack.unpackb(data)