def ret_fail(self, message): LOG.error("%s" % str(message)) message = NetworkUtil.encoding({ Protocol.KEY_COMMAND : Protocol.MESSAGE_COMMAND_FAIELD, Protocol.KEY_FAILED_REASON : message }) message_size = struct.pack("!I", len(message)) self.request.send(message_size) self.wfile.write(message)
def ret_fail(self, message): LOG.error("%s" % str(message)) message = NetworkUtil.encoding({ Protocol.KEY_COMMAND: Protocol.MESSAGE_COMMAND_FAILED, Protocol.KEY_FAILED_REASON: message }) message_size = struct.pack("!I", len(message)) self.request.send(message_size) self.wfile.write(message)
def send_synthesis_done(self): message = NetworkUtil.encoding({ Protocol.KEY_COMMAND : Protocol.MESSAGE_COMMAND_SYNTHESIS_DONE, }) LOG.info("SUCCESS to launch VM") try: message_size = struct.pack("!I", len(message)) self.request.send(message_size) self.wfile.write(message) except socket.error as e: pass
def send_synthesis_done(self): message = NetworkUtil.encoding({ Protocol.KEY_COMMAND: Protocol.MESSAGE_COMMAND_SYNTHESIS_DONE, }) LOG.info("SUCCESS to launch VM") try: message_size = struct.pack("!I", len(message)) self.request.send(message_size) self.wfile.write(message) except socket.error as e: pass
def ret_success(self, req_command, payload=None): send_message = { Protocol.KEY_COMMAND: Protocol.MESSAGE_COMMAND_SUCCESS, Protocol.KEY_REQUESTED_COMMAND: req_command, } if payload: send_message.update(payload) message = NetworkUtil.encoding(send_message) message_size = struct.pack("!I", len(message)) self.request.send(message_size) self.wfile.write(message) self.wfile.flush()
def ret_success(self, req_command, payload=None): send_message = { Protocol.KEY_COMMAND : Protocol.MESSAGE_COMMAND_SUCCESS, Protocol.KEY_REQUESTED_COMMAND : req_command, } if payload: send_message.update(payload) message = NetworkUtil.encoding(send_message) message_size = struct.pack("!I", len(message)) self.request.send(message_size) self.wfile.write(message) self.wfile.flush()
def transfer(self): # connect address = (self.remote_addr, self.remote_port) sock = None for index in range(5): LOG.info("Connecting to (%s).." % str(address)) try: sock = socket.create_connection(address, 10) break except Exception as e: time.sleep(1) pass if sock == None: msg = "failed to connect to %s" % str(address) raise StreamSynthesisClientError(msg) sock.setblocking(True) self.blob_sent_time_dict = dict() self.receive_thread = NetworkMeasurementThread(sock, self.blob_sent_time_dict, self.monitor_network_bw, self.vm_resume_time_at_dest) self.receive_thread.start() # send header header_dict = { Protocol.KEY_SYNTHESIS_OPTION: None, } header_dict.update(self.metadata) header = NetworkUtil.encoding(header_dict) sock.sendall(struct.pack("!I", len(header))) sock.sendall(header) # stream blob blob_counter = 0 while True: comp_task = self.compdata_queue.get() if self.is_first_recv == False: self.is_first_recv = True self.time_first_recv = time.time() LOG.debug("[time] Transfer first input at : %f" % (self.time_first_recv)) transfer_size = 0 if comp_task == Const.QUEUE_SUCCESS_MESSAGE: break if comp_task == Const.QUEUE_FAILED_MESSAGE: sys.stderr.write("Failed to get compressed data\n") break (blob_comp_type, compdata, disk_chunks, memory_chunks) = comp_task blob_header_dict = { Const.META_OVERLAY_FILE_COMPRESSION: blob_comp_type, Const.META_OVERLAY_FILE_SIZE:len(compdata), Const.META_OVERLAY_FILE_DISK_CHUNKS: disk_chunks, Const.META_OVERLAY_FILE_MEMORY_CHUNKS: memory_chunks } # send header = NetworkUtil.encoding(blob_header_dict) sock.sendall(struct.pack("!I", len(header))) sock.sendall(header) self.blob_sent_time_dict[blob_counter] = (time.time(), len(compdata)) sock.sendall(compdata) transfer_size += (4+len(header)+len(compdata)) blob_counter += 1 # end message end_header = { "blob_type": "blob", Const.META_OVERLAY_FILE_SIZE:0 } header = NetworkUtil.encoding(end_header) sock.sendall(struct.pack("!I", len(header))) sock.sendall(header) self.is_processing_alive.value = False self.time_finish_transmission.value = time.time() sys.stdout.write("Finish transmission. Waiting for finishing migration\n") self.receive_thread.join() sock.close()
def transfer(self): # connect address = (self.remote_addr, self.remote_port) sock = None for _ in range(5): LOG.info("Attempting connection to (%s)..", str(address)) try: sock = socket.create_connection(address, 10) break except Exception as ex: time.sleep(1) LOG.error(ex) if sock is None: msg = "Failed to connect to %s" % str(address) LOG.error(msg) raise StreamSynthesisClientError(msg) else: sock.setblocking(True) self.blob_sent_time_dict = dict() self.receive_thread = NetworkMeasurementThread( sock, self.blob_sent_time_dict, self.monitor_network_bw, self.vm_resume_time_at_dest) self.receive_thread.start() # send header header_dict = { Protocol.KEY_SYNTHESIS_OPTION: None, } header_dict.update(self.metadata) header = NetworkUtil.encoding(header_dict) sock.sendall(struct.pack("!I", len(header))) sock.sendall(header) # stream blob blob_counter = 0 while True: comp_task = self.compdata_queue.get() if self.is_first_recv == False: self.is_first_recv = True self.time_first_recv = time.time() LOG.debug("[time] Transfer first input at : %f", (self.time_first_recv)) transfer_size = 0 if comp_task == Const.QUEUE_SUCCESS_MESSAGE: LOG.debug( "Received SUCCESS message from compression task. Breaking..." ) break if comp_task == Const.QUEUE_FAILED_MESSAGE: LOG.error( "Failed to get compressed data from compression thread!" ) break (blob_comp_type, compdata, disk_chunks, memory_chunks) = comp_task blob_header_dict = { Const.META_OVERLAY_FILE_COMPRESSION: blob_comp_type, Const.META_OVERLAY_FILE_SIZE: len(compdata), Const.META_OVERLAY_FILE_DISK_CHUNKS: disk_chunks, Const.META_OVERLAY_FILE_MEMORY_CHUNKS: memory_chunks } # send header = NetworkUtil.encoding(blob_header_dict) sock.sendall(struct.pack("!I", len(header))) sock.sendall(header) self.blob_sent_time_dict[blob_counter] = (time.time(), len(compdata)) sock.sendall(compdata) transfer_size += (4 + len(header) + len(compdata)) blob_counter += 1 #send the current iteration number for use at the destination sock.sendall( struct.pack( "!I", self.process_controller.get_migration_iteration_count( ))) # end message end_header = {"blob_type": "blob", Const.META_OVERLAY_FILE_SIZE: 0} header = NetworkUtil.encoding(end_header) sock.sendall(struct.pack("!I", len(header))) sock.sendall(header) self.is_processing_alive.value = False self.time_finish_transmission.value = time.time() LOG.info( "Finished data transmission. Waiting for response that deltas have been applied..." ) self.receive_thread.join() sock.close()
def handle(self): '''Handle request from the client Each request follows this format: | header size | header | blob header size | blob header | blob data | | (4 bytes) | (var) | (4 bytes) | (var bytes) | (var bytes)| ''' # variable self.total_recved_size_cur = 0 self.total_recved_size_prev = 0 # get header data = self._recv_all(4) if data is None or len(data) != 4: raise StreamSynthesisError( "Failed to receive first byte of header") message_size = struct.unpack("!I", data)[0] msgpack_data = self._recv_all(message_size) metadata = NetworkUtil.decoding(msgpack_data) launch_disk_size = metadata[Cloudlet_Const.META_RESUME_VM_DISK_SIZE] launch_memory_size = metadata[ Cloudlet_Const.META_RESUME_VM_MEMORY_SIZE] analysis_mq = multiprocessing.Queue() analysis_proc = HandoffAnalysisProc(handoff_url=self.client_address[0], message_queue=analysis_mq, disk_size=launch_disk_size, mem_size=launch_memory_size) analysis_proc.start() analysis_mq.put("=" * 50) analysis_mq.put("Adaptive VM Handoff Initiated") analysis_mq.put( "Client Connection - %s:%d" % (self.client_address[0], self.client_address[1])) #client_address is a tuple (ip, port) if self.server.handoff_data is not None: analysis_mq.put("Handoff via OpenStack") via_openstack = True else: analysis_mq.put("Handoff via cloudlet CLI") via_openstack = False synthesis_option, base_diskpath = self._check_validity(metadata) if base_diskpath is None: raise StreamSynthesisError("No matching base VM") if via_openstack: base_diskpath, base_mempath, base_diskmeta, base_memmeta = self.server.handoff_data.base_vm_paths else: (base_diskmeta, base_mempath, base_memmeta) = Cloudlet_Const.get_basepath(base_diskpath, check_exist=True) analysis_mq.put("Synthesis Options %s" % str(pformat(self.synthesis_option))) analysis_mq.put("Base VM Path: %s" % base_diskpath) analysis_mq.put("Image Disk Size: %d" % launch_disk_size) analysis_mq.put("Image Memory Size: %d" % launch_memory_size) analysis_mq.put("=" * 50) # variables for FUSE if via_openstack: launch_disk = self.server.handoff_data.launch_diskpath launch_mem = self.server.handoff_data.launch_memorypath else: temp_synthesis_dir = tempfile.mkdtemp(prefix="cloudlet-comp-") launch_disk = os.path.join(temp_synthesis_dir, "launch-disk") launch_mem = os.path.join(temp_synthesis_dir, "launch-mem") memory_chunk_all = set() disk_chunk_all = set() # start pipelining processes network_out_queue = multiprocessing.Queue() decomp_queue = multiprocessing.Queue() fuse_info_queue = multiprocessing.Queue() decomp_proc = DecompProc(network_out_queue, decomp_queue, num_proc=4, analysis_queue=analysis_mq) decomp_proc.start() analysis_mq.put("Starting (%d) decompression processes..." % (decomp_proc.num_proc)) delta_proc = RecoverDeltaProc(base_diskpath, base_mempath, decomp_queue, launch_mem, launch_disk, Cloudlet_Const.CHUNK_SIZE, fuse_info_queue, analysis_mq) delta_proc.start() analysis_mq.put("Starting delta recovery process...") # get each blob recv_blob_counter = 0 while True: data = self._recv_all(4) if data is None or len(data) != 4: raise StreamSynthesisError( "Failed to receive first byte of header") blob_header_size = struct.unpack("!I", data)[0] blob_header_raw = self._recv_all(blob_header_size) blob_header = NetworkUtil.decoding(blob_header_raw) blob_size = blob_header.get(Cloudlet_Const.META_OVERLAY_FILE_SIZE) if blob_size is None: raise StreamSynthesisError("Failed to receive blob") if blob_size == 0: analysis_mq.put("End of stream received from client at %f)" % (time.time())) break blob_comp_type = blob_header.get( Cloudlet_Const.META_OVERLAY_FILE_COMPRESSION) blob_disk_chunk = blob_header.get( Cloudlet_Const.META_OVERLAY_FILE_DISK_CHUNKS) blob_memory_chunk = blob_header.get( Cloudlet_Const.META_OVERLAY_FILE_MEMORY_CHUNKS) # send ack right before getting the blob ack_data = struct.pack("!Q", 0x01) self.request.send(ack_data) compressed_blob = self._recv_all(blob_size, ack_size=200 * 1024) # send ack right after getting the blob ack_data = struct.pack("!Q", 0x02) self.request.send(ack_data) network_out_queue.put((blob_comp_type, compressed_blob)) #TODO: remove the interweaving of the valid bit here #TODO: and change the code path in cloudlet_driver.py so that #TODO: it uses the chunk sets in favor of the tuples if via_openstack: memory_chunk_set = set( ["%ld:1" % item for item in blob_memory_chunk]) disk_chunk_set = set( ["%ld:1" % item for item in blob_disk_chunk]) memory_chunk_all.update(memory_chunk_set) disk_chunk_all.update(disk_chunk_set) else: memory_chunk_all.update(blob_memory_chunk) disk_chunk_all.update(blob_disk_chunk) recv_blob_counter += 1 analysis_mq.put("B,R,%d" % (recv_blob_counter)) network_out_queue.put(Cloudlet_Const.QUEUE_SUCCESS_MESSAGE) delta_proc.join() LOG.debug("%f\tdeltaproc join" % (time.time())) analysis_mq.put("Adaptive VM Handoff Complete!") analysis_mq.put("=" * 50) analysis_mq.put("!E_O_Q!") analysis_proc.join() if via_openstack: ack_data = struct.pack("!Qd", 0x10, time.time()) LOG.info("send ack to client: %d" % len(ack_data)) self.request.sendall(ack_data) disk_overlay_map = ','.join(disk_chunk_all) memory_overlay_map = ','.join(memory_chunk_all) # NOTE: fuse and synthesis take place in cloudlet_driver.py when launched from openstack but #this data must be written to stdout so the pipe connected to cloudlet_driver.py can finish the handoff #TODO: instead of sending this stdout buffer over the pipe to cloudlet_driver.py, we should probably #TODO: move to multiprocessing.Pipe or Queue to avoid issues with other items being dumped to stdout #TODO: and causing problems with this data being sent back; i.e. anything written via LOG #TODO: after this will end up in stdout because the logger has a StreamHandler configured to use stdout sys.stdout.write("openstack\t%s\t%s\t%s\t%s" % (launch_disk_size, launch_memory_size, disk_overlay_map, memory_overlay_map)) else: # We told to FUSE that we have everything ready, so we need to wait # until delta_proc finishes. we cannot start VM before delta_proc # finishes, because we don't know what will be modified in the future time_fuse_start = time.time() fuse = run_fuse(Cloudlet_Const.CLOUDLETFS_PATH, Cloudlet_Const.CHUNK_SIZE, base_diskpath, launch_disk_size, base_mempath, launch_memory_size, resumed_disk=launch_disk, disk_chunks=disk_chunk_all, resumed_memory=launch_mem, memory_chunks=memory_chunk_all, valid_bit=1) time_fuse_end = time.time() synthesized_vm = SynthesizedVM(launch_disk, launch_mem, fuse) synthesized_vm.start() synthesized_vm.join() # since libvirt does not return immediately after resuming VM, we # measure resume time directly from QEMU actual_resume_time = 0 splited_log = open("/tmp/qemu_debug_messages", "r").read().split("\n") for line in splited_log: if line.startswith("INCOMING_FINISH"): actual_resume_time = float(line.split(" ")[-1]) LOG.info("[time] non-pipelined time %f (%f ~ %f ~ %f)" % ( actual_resume_time - time_fuse_start, time_fuse_start, time_fuse_end, actual_resume_time, )) ack_data = struct.pack("!Qd", 0x10, actual_resume_time) LOG.info("send ack to client: %d" % len(ack_data)) self.request.sendall(ack_data) connect_vnc(synthesized_vm.machine, True) signal.signal(signal.SIGUSR1, handlesig) signal.pause() synthesized_vm.monitor.terminate() synthesized_vm.monitor.join() synthesized_vm.terminate()
def handle(self): '''Handle request from the client Each request follows this format: | header size | header | blob header size | blob header | blob data | | (4 bytes) | (var) | (4 bytes) | (var bytes) | (var bytes)| ''' if self.server.handoff_data is not None: LOG.debug("VM synthesis using OpenStack") else: LOG.debug("VM synthesis as standalone") # variable self.total_recved_size_cur = 0 self.total_recved_size_prev = 0 # get header data = self._recv_all(4) if data == None or len(data) != 4: raise StreamSynthesisError("Failed to receive first byte of header") message_size = struct.unpack("!I", data)[0] msgpack_data = self._recv_all(message_size) metadata = NetworkUtil.decoding(msgpack_data) launch_disk_size = metadata[Cloudlet_Const.META_RESUME_VM_DISK_SIZE] launch_memory_size = metadata[Cloudlet_Const.META_RESUME_VM_MEMORY_SIZE] synthesis_option, base_diskpath = self._check_validity(metadata) if base_diskpath == None: raise StreamSynthesisError("No matching base VM") if self.server.handoff_data: base_diskpath, base_diskmeta, base_mempath, base_memmeta =\ self.server.handoff_data.base_vm_paths else: (base_diskmeta, base_mempath, base_memmeta) = \ Cloudlet_Const.get_basepath(base_diskpath, check_exist=True) LOG.info(" - %s" % str(pformat(self.synthesis_option))) LOG.info(" - Base VM : %s" % base_diskpath) # variables for FUSE temp_synthesis_dir = tempfile.mkdtemp(prefix="cloudlet-comp-") launch_disk = os.path.join(temp_synthesis_dir, "launch-disk") launch_mem = os.path.join(temp_synthesis_dir, "launch-mem") memory_chunk_all = set() disk_chunk_all = set() # start pipelining processes network_out_queue = multiprocessing.Queue() decomp_queue = multiprocessing.Queue() fuse_info_queue = multiprocessing.Queue() decomp_proc = DecompProc(network_out_queue, decomp_queue, num_proc=4) decomp_proc.start() LOG.info("Start Decompression process") delta_proc = RecoverDeltaProc(base_diskpath, base_mempath, decomp_queue, launch_mem, launch_disk, Cloudlet_Const.CHUNK_SIZE, fuse_info_queue) delta_proc.start() LOG.info("Start Synthesis process") # get each blob recv_blob_counter = 0 while True: data = self._recv_all(4) if data == None or len(data) != 4: raise StreamSynthesisError("Failed to receive first byte of header") break blob_header_size = struct.unpack("!I", data)[0] blob_header_raw = self._recv_all(blob_header_size) blob_header = NetworkUtil.decoding(blob_header_raw) blob_size = blob_header.get(Cloudlet_Const.META_OVERLAY_FILE_SIZE) if blob_size == None: raise StreamSynthesisError("Failed to receive blob") if blob_size == 0: LOG.debug("%f\tend of stream" % (time.time())) break blob_comp_type = blob_header.get(Cloudlet_Const.META_OVERLAY_FILE_COMPRESSION) blob_disk_chunk = blob_header.get(Cloudlet_Const.META_OVERLAY_FILE_DISK_CHUNKS) blob_memory_chunk = blob_header.get(Cloudlet_Const.META_OVERLAY_FILE_MEMORY_CHUNKS) # send ack right before getting the blob ack_data = struct.pack("!Q", 0x01) self.request.send(ack_data) compressed_blob = self._recv_all(blob_size, ack_size=200*1024) # send ack right after getting the blob ack_data = struct.pack("!Q", 0x02) self.request.send(ack_data) network_out_queue.put((blob_comp_type, compressed_blob)) memory_chunk_set = set(["%ld:1" % item for item in blob_memory_chunk]) disk_chunk_set = set(["%ld:1" % item for item in blob_disk_chunk]) memory_chunk_all.update(memory_chunk_set) disk_chunk_all.update(disk_chunk_set) LOG.debug("%f\treceive one blob" % (time.time())) recv_blob_counter += 1 network_out_queue.put(Cloudlet_Const.QUEUE_SUCCESS_MESSAGE) delta_proc.join() LOG.debug("%f\tdeltaproc join" % (time.time())) # We told to FUSE that we have everything ready, so we need to wait # until delta_proc fininshes. we cannot start VM before delta_proc # finishes, because we don't know what will be modified in the future time_fuse_start = time.time() disk_overlay_map = ','.join(disk_chunk_all) memory_overlay_map = ','.join(memory_chunk_all) fuse = run_fuse(Cloudlet_Const.CLOUDLETFS_PATH, Cloudlet_Const.CHUNK_SIZE, base_diskpath, launch_disk_size, base_mempath, launch_memory_size, resumed_disk=launch_disk, disk_overlay_map=disk_overlay_map, resumed_memory=launch_mem, memory_overlay_map=memory_overlay_map) time_fuse_end = time.time() memory_path = os.path.join(fuse.mountpoint, 'memory', 'image') if self.server.handoff_data: synthesized_vm = SynthesizedVM( launch_disk, launch_mem, fuse, disk_only=False, qemu_args=None, nova_xml=self.server.handoff_data.libvirt_xml, nova_conn=self.server.handoff_data._conn, nova_util=self.server.handoff_data._libvirt_utils ) else: synthesized_vm = SynthesizedVM(launch_disk, launch_mem, fuse) synthesized_vm.start() synthesized_vm.join() # to be delete #libvirt_xml = synthesized_vm.new_xml_str #vmpaths = [base_diskpath, base_diskmeta, base_mempath, base_memmeta] #base_hashvalue = metadata.get(Cloudlet_Const.META_BASE_VM_SHA256, None) #ds = HandoffDataRecv() #ds.save_data(vmpaths, base_hashvalue, libvirt_xml, "qemu:///session") #ds.to_file("/home/stack/cloudlet/provisioning/handff_recv_data") # since libvirt does not return immediately after resuming VM, we # measure resume time directly from QEMU actual_resume_time = 0 splited_log = open("/tmp/qemu_debug_messages", "r").read().split("\n") for line in splited_log: if line.startswith("INCOMING_FINISH"): actual_resume_time = float(line.split(" ")[-1]) time_resume_end = time.time() LOG.info("[time] non-pipelined time %f (%f ~ %f ~ %f)" % ( actual_resume_time-time_fuse_start, time_fuse_start, time_fuse_end, actual_resume_time, )) if self.server.handoff_data == None: # for a standalone version, terminate a VM for the next testing #connect_vnc(synthesized_vm.machine) LOG.debug("Finishing VM in 3 seconds") time.sleep(3) synthesized_vm.monitor.terminate() synthesized_vm.monitor.join() synthesized_vm.terminate() # send end message ack_data = struct.pack("!Qd", 0x10, actual_resume_time) LOG.info("send ack to client: %d" % len(ack_data)) self.request.sendall(ack_data) LOG.info("finished")
def handle(self): '''Handle request from the client Each request follows this format: | header size | header | blob header size | blob header | blob data | | (4 bytes) | (var) | (4 bytes) | (var bytes) | (var bytes)| ''' if self.server.handoff_data is not None: LOG.debug("VM synthesis using OpenStack") else: LOG.debug("VM synthesis as standalone") # variable self.total_recved_size_cur = 0 self.total_recved_size_prev = 0 # get header data = self._recv_all(4) if data == None or len(data) != 4: raise StreamSynthesisError( "Failed to receive first byte of header") message_size = struct.unpack("!I", data)[0] msgpack_data = self._recv_all(message_size) metadata = NetworkUtil.decoding(msgpack_data) launch_disk_size = metadata[Cloudlet_Const.META_RESUME_VM_DISK_SIZE] launch_memory_size = metadata[ Cloudlet_Const.META_RESUME_VM_MEMORY_SIZE] synthesis_option, base_diskpath = self._check_validity(metadata) if base_diskpath == None: raise StreamSynthesisError("No matching base VM") if self.server.handoff_data: base_diskpath, base_diskmeta, base_mempath, base_memmeta =\ self.server.handoff_data.base_vm_paths else: (base_diskmeta, base_mempath, base_memmeta) = \ Cloudlet_Const.get_basepath(base_diskpath, check_exist=True) LOG.info(" - %s" % str(pformat(self.synthesis_option))) LOG.info(" - Base VM : %s" % base_diskpath) # variables for FUSE temp_synthesis_dir = tempfile.mkdtemp(prefix="cloudlet-comp-") launch_disk = os.path.join(temp_synthesis_dir, "launch-disk") launch_mem = os.path.join(temp_synthesis_dir, "launch-mem") memory_chunk_all = set() disk_chunk_all = set() # start pipelining processes network_out_queue = multiprocessing.Queue() decomp_queue = multiprocessing.Queue() fuse_info_queue = multiprocessing.Queue() decomp_proc = DecompProc(network_out_queue, decomp_queue, num_proc=4) decomp_proc.start() LOG.info("Start Decompression process") delta_proc = RecoverDeltaProc(base_diskpath, base_mempath, decomp_queue, launch_mem, launch_disk, Cloudlet_Const.CHUNK_SIZE, fuse_info_queue) delta_proc.start() LOG.info("Start Synthesis process") # get each blob recv_blob_counter = 0 while True: data = self._recv_all(4) if data == None or len(data) != 4: raise StreamSynthesisError( "Failed to receive first byte of header") break blob_header_size = struct.unpack("!I", data)[0] blob_header_raw = self._recv_all(blob_header_size) blob_header = NetworkUtil.decoding(blob_header_raw) blob_size = blob_header.get(Cloudlet_Const.META_OVERLAY_FILE_SIZE) if blob_size == None: raise StreamSynthesisError("Failed to receive blob") if blob_size == 0: LOG.debug("%f\tend of stream" % (time.time())) break blob_comp_type = blob_header.get( Cloudlet_Const.META_OVERLAY_FILE_COMPRESSION) blob_disk_chunk = blob_header.get( Cloudlet_Const.META_OVERLAY_FILE_DISK_CHUNKS) blob_memory_chunk = blob_header.get( Cloudlet_Const.META_OVERLAY_FILE_MEMORY_CHUNKS) # send ack right before getting the blob ack_data = struct.pack("!Q", 0x01) self.request.send(ack_data) compressed_blob = self._recv_all(blob_size, ack_size=200 * 1024) # send ack right after getting the blob ack_data = struct.pack("!Q", 0x02) self.request.send(ack_data) network_out_queue.put((blob_comp_type, compressed_blob)) memory_chunk_set = set( ["%ld:1" % item for item in blob_memory_chunk]) disk_chunk_set = set(["%ld:1" % item for item in blob_disk_chunk]) memory_chunk_all.update(memory_chunk_set) disk_chunk_all.update(disk_chunk_set) LOG.debug("%f\treceive one blob" % (time.time())) recv_blob_counter += 1 network_out_queue.put(Cloudlet_Const.QUEUE_SUCCESS_MESSAGE) delta_proc.join() LOG.debug("%f\tdeltaproc join" % (time.time())) # We told to FUSE that we have everything ready, so we need to wait # until delta_proc fininshes. we cannot start VM before delta_proc # finishes, because we don't know what will be modified in the future time_fuse_start = time.time() disk_overlay_map = ','.join(disk_chunk_all) memory_overlay_map = ','.join(memory_chunk_all) fuse = run_fuse(Cloudlet_Const.CLOUDLETFS_PATH, Cloudlet_Const.CHUNK_SIZE, base_diskpath, launch_disk_size, base_mempath, launch_memory_size, resumed_disk=launch_disk, disk_overlay_map=disk_overlay_map, resumed_memory=launch_mem, memory_overlay_map=memory_overlay_map) time_fuse_end = time.time() memory_path = os.path.join(fuse.mountpoint, 'memory', 'image') if self.server.handoff_data: synthesized_vm = SynthesizedVM( launch_disk, launch_mem, fuse, disk_only=False, qemu_args=None, nova_xml=self.server.handoff_data.libvirt_xml, nova_conn=self.server.handoff_data._conn, nova_util=self.server.handoff_data._libvirt_utils) else: synthesized_vm = SynthesizedVM(launch_disk, launch_mem, fuse) synthesized_vm.start() synthesized_vm.join() # to be delete #libvirt_xml = synthesized_vm.new_xml_str #vmpaths = [base_diskpath, base_diskmeta, base_mempath, base_memmeta] #base_hashvalue = metadata.get(Cloudlet_Const.META_BASE_VM_SHA256, None) #ds = HandoffDataRecv() #ds.save_data(vmpaths, base_hashvalue, libvirt_xml, "qemu:///session") #ds.to_file("/home/stack/cloudlet/provisioning/handff_recv_data") # since libvirt does not return immediately after resuming VM, we # measure resume time directly from QEMU actual_resume_time = 0 splited_log = open("/tmp/qemu_debug_messages", "r").read().split("\n") for line in splited_log: if line.startswith("INCOMING_FINISH"): actual_resume_time = float(line.split(" ")[-1]) time_resume_end = time.time() LOG.info("[time] non-pipelined time %f (%f ~ %f ~ %f)" % ( actual_resume_time - time_fuse_start, time_fuse_start, time_fuse_end, actual_resume_time, )) if self.server.handoff_data == None: # for a standalone version, terminate a VM for the next testing #connect_vnc(synthesized_vm.machine) LOG.debug("Finishing VM in 3 seconds") time.sleep(3) synthesized_vm.monitor.terminate() synthesized_vm.monitor.join() synthesized_vm.terminate() # send end message ack_data = struct.pack("!Qd", 0x10, actual_resume_time) LOG.info("send ack to client: %d" % len(ack_data)) self.request.sendall(ack_data) LOG.info("finished")
def handle(self): '''Handle request from the client Each request follows this format: | header size | header | blob header size | blob header | blob data | | (4 bytes) | (var) | (4 bytes) | (var bytes) | (var bytes)| ''' # variable self.total_recved_size_cur = 0 self.total_recved_size_prev = 0 # get header data = self._recv_all(4) if data is None or len(data) != 4: raise StreamSynthesisError("Failed to receive first byte of header") message_size = struct.unpack("!I", data)[0] msgpack_data = self._recv_all(message_size) metadata = NetworkUtil.decoding(msgpack_data) launch_disk_size = metadata[Cloudlet_Const.META_RESUME_VM_DISK_SIZE] launch_memory_size = metadata[Cloudlet_Const.META_RESUME_VM_MEMORY_SIZE] analysis_mq = multiprocessing.Queue() analysis_proc = HandoffAnalysisProc(handoff_url=self.client_address[0],message_queue=analysis_mq, disk_size=launch_disk_size, mem_size=launch_memory_size) analysis_proc.start() analysis_mq.put("=" * 50) analysis_mq.put("Adaptive VM Handoff Initiated") analysis_mq.put("Client Connection - %s:%d" % (self.client_address[0], self.client_address[1])) #client_address is a tuple (ip, port) if self.server.handoff_data is not None: analysis_mq.put("Handoff via OpenStack") via_openstack = True else: analysis_mq.put("Handoff via cloudlet CLI") via_openstack = False synthesis_option, base_diskpath = self._check_validity(metadata) if base_diskpath is None: raise StreamSynthesisError("No matching base VM") if via_openstack: base_diskpath, base_mempath, base_diskmeta, base_memmeta = self.server.handoff_data.base_vm_paths else: (base_diskmeta, base_mempath, base_memmeta) = Cloudlet_Const.get_basepath(base_diskpath, check_exist=True) analysis_mq.put("Synthesis Options %s" % str(pformat(self.synthesis_option))) analysis_mq.put("Base VM Path: %s" % base_diskpath) analysis_mq.put("Image Disk Size: %d" % launch_disk_size) analysis_mq.put("Image Memory Size: %d" % launch_memory_size) analysis_mq.put("=" * 50) # variables for FUSE if via_openstack: launch_disk = self.server.handoff_data.launch_diskpath launch_mem = self.server.handoff_data.launch_memorypath else: temp_synthesis_dir = tempfile.mkdtemp(prefix="cloudlet-comp-") launch_disk = os.path.join(temp_synthesis_dir, "launch-disk") launch_mem = os.path.join(temp_synthesis_dir, "launch-mem") memory_chunk_all = set() disk_chunk_all = set() # start pipelining processes network_out_queue = multiprocessing.Queue() decomp_queue = multiprocessing.Queue() fuse_info_queue = multiprocessing.Queue() decomp_proc = DecompProc(network_out_queue, decomp_queue, num_proc=4, analysis_queue=analysis_mq) decomp_proc.start() analysis_mq.put("Starting (%d) decompression processes..." % (decomp_proc.num_proc)) delta_proc = RecoverDeltaProc(base_diskpath, base_mempath, decomp_queue, launch_mem, launch_disk, Cloudlet_Const.CHUNK_SIZE, fuse_info_queue, analysis_mq) delta_proc.start() analysis_mq.put("Starting delta recovery process...") # get each blob recv_blob_counter = 0 while True: data = self._recv_all(4) if data is None or len(data) != 4: raise StreamSynthesisError("Failed to receive first byte of header") blob_header_size = struct.unpack("!I", data)[0] blob_header_raw = self._recv_all(blob_header_size) blob_header = NetworkUtil.decoding(blob_header_raw) blob_size = blob_header.get(Cloudlet_Const.META_OVERLAY_FILE_SIZE) if blob_size is None: raise StreamSynthesisError("Failed to receive blob") if blob_size == 0: analysis_mq.put("End of stream received from client at %f)" % (time.time())) break blob_comp_type = blob_header.get(Cloudlet_Const.META_OVERLAY_FILE_COMPRESSION) blob_disk_chunk = blob_header.get(Cloudlet_Const.META_OVERLAY_FILE_DISK_CHUNKS) blob_memory_chunk = blob_header.get(Cloudlet_Const.META_OVERLAY_FILE_MEMORY_CHUNKS) # send ack right before getting the blob ack_data = struct.pack("!Q", 0x01) self.request.send(ack_data) compressed_blob = self._recv_all(blob_size, ack_size=200*1024) # send ack right after getting the blob ack_data = struct.pack("!Q", 0x02) self.request.send(ack_data) network_out_queue.put((blob_comp_type, compressed_blob)) #TODO: remove the interweaving of the valid bit here #TODO: and change the code path in cloudlet_driver.py so that #TODO: it uses the chunk sets in favor of the tuples if via_openstack: memory_chunk_set = set(["%ld:1" % item for item in blob_memory_chunk]) disk_chunk_set = set(["%ld:1" % item for item in blob_disk_chunk]) memory_chunk_all.update(memory_chunk_set) disk_chunk_all.update(disk_chunk_set) else: memory_chunk_all.update(blob_memory_chunk) disk_chunk_all.update(blob_disk_chunk) recv_blob_counter += 1 analysis_mq.put("B,R,%d" % (recv_blob_counter)) data = self._recv_all(4) iter = struct.unpack("!I", data)[0] analysis_mq.put("iter,%d" % (iter)) network_out_queue.put(Cloudlet_Const.QUEUE_SUCCESS_MESSAGE) delta_proc.join() LOG.debug("%f\tdeltaproc join" % (time.time())) analysis_mq.put("Adaptive VM Handoff Complete!") analysis_mq.put("=" * 50) analysis_mq.put("!E_O_Q!") analysis_proc.join() if via_openstack: ack_data = struct.pack("!Qd", 0x10, time.time()) LOG.info("send ack to client: %d" % len(ack_data)) self.request.sendall(ack_data) disk_overlay_map = ','.join(disk_chunk_all) memory_overlay_map = ','.join(memory_chunk_all) # NOTE: fuse and synthesis take place in cloudlet_driver.py when launched from openstack but #this data must be written to stdout so the pipe connected to cloudlet_driver.py can finish the handoff #TODO: instead of sending this stdout buffer over the pipe to cloudlet_driver.py, we should probably #TODO: move to multiprocessing.Pipe or Queue to avoid issues with other items being dumped to stdout #TODO: and causing problems with this data being sent back; i.e. anything written via LOG #TODO: after this will end up in stdout because the logger has a StreamHandler configured to use stdout sys.stdout.write("openstack\t%s\t%s\t%s\t%s" % (launch_disk_size, launch_memory_size, disk_overlay_map, memory_overlay_map)) else: # We told to FUSE that we have everything ready, so we need to wait # until delta_proc finishes. we cannot start VM before delta_proc # finishes, because we don't know what will be modified in the future time_fuse_start = time.time() fuse = run_fuse(Cloudlet_Const.CLOUDLETFS_PATH, Cloudlet_Const.CHUNK_SIZE, base_diskpath, launch_disk_size, base_mempath, launch_memory_size, resumed_disk=launch_disk, disk_chunks=disk_chunk_all, resumed_memory=launch_mem, memory_chunks=memory_chunk_all, valid_bit=1) time_fuse_end = time.time() synthesized_vm = SynthesizedVM(launch_disk, launch_mem, fuse) synthesized_vm.start() synthesized_vm.join() # since libvirt does not return immediately after resuming VM, we # measure resume time directly from QEMU actual_resume_time = 0 splited_log = open("/tmp/qemu_debug_messages", "r").read().split("\n") for line in splited_log: if line.startswith("INCOMING_FINISH"): actual_resume_time = float(line.split(" ")[-1]) LOG.info("[time] non-pipelined time %f (%f ~ %f ~ %f)" % ( actual_resume_time-time_fuse_start, time_fuse_start, time_fuse_end, actual_resume_time, )) ack_data = struct.pack("!Qd", 0x10, actual_resume_time) LOG.info("send ack to client: %d" % len(ack_data)) self.request.sendall(ack_data) connect_vnc(synthesized_vm.machine, True) signal.signal(signal.SIGUSR1, handlesig) signal.pause() synthesized_vm.monitor.terminate() synthesized_vm.monitor.join() synthesized_vm.terminate()