def start_as_client(self): print 'I am the client!' sleep(0.5) real_members = [ host.split('_')[0] for host in sorted(self.membList.keys()) ] print 'All members: {}'.format(real_members) self.masters_workers = [ socket.gethostbyname(host) for host in real_members if host != self.host_name ] for host in self.masters_workers: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((host, self.port)) send_all_encrypted(sock, self.message_input) send_all_from_file(sock, self.app_file, self.messageInterval) send_all_encrypted(sock, self.app_args) send_all_encrypted(sock, self.masters_workers) send_all_encrypted(sock, self.is_undirected) if host in self.masters_workers[0:2]: send_all_from_file(sock, self.filename_pair[0], self.messageInterval) send_all_encrypted(sock, self.filename_pair[1])
def broadCastFile(self, targets, filename): for target in targets: target_host, target_nodeName, sock = self.getParams(target) send_all_encrypted(sock, self.message_file) if target == self.groupID: send_all_encrypted(sock, filename) else: send_all_from_file(sock, filename, self.messageInterval) logging.debug( stampedMsg('{} pushing file {} to node {}'.format( self.nodeName, filename, target_nodeName)))
def preprocess(self): sleep(0.5) self.server_task = Thread(target=self.background_server) self.server_task.daemon = True self.server_task.start() print('I have {} workers!'.format(self.num_workers)) self.v_to_m_dict, self.num_vertices = parse_file( self.input_filename, self.num_workers, self.masters_workers) print('num_vertices: ', self.num_vertices) for ix in range(self.num_workers): sock = self.send_to_worker([ Commons.request_preprocess, self.input_filename, self.v_to_m_dict, self.num_vertices ], self.masters_workers[ix + 2]) send_all_from_file(sock, self.input_filename, 0.001) while (self.num_preprocess_done < self.num_workers): sleep(1)
def collect_results(self): self.result_files = [0] * self.num_workers for ix in range(self.num_workers): worker = self.alive_workers[ix] self.result_files[ix] = 'file_piece_' + str(ix) + '_out' self.send_to_worker( [Commons.request_result, self.result_files[ix]], worker) while (self.num_process_done < self.num_workers): sleep(1) for ix in range(self.num_workers): dfsWrapper(self.dfs.getFile, self.result_files[ix]) combine_files(self.output_filename, self.result_files) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((self.client_ip, self.driver_port)) send_all_encrypted(sock, self.client_message) send_all_from_file(sock, self.output_filename, 0.001) self.remote_end_tasks()
def server_task(self): #first, start local timer, the rest of the process follows this timer self.timer.tic() # a monitor receive message, check and response, also multicase failure message self.monitor = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.monitor.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.monitor.bind((self.host, self.port)) self.monitor.listen(5) # self.monitor.listen(10) # UDP doesn't support this logging.info(stampedMsg('FS Monitoring process opens.')) # keep receiving msgs from other VMs # receiving heartbeat and other messages # pdb.set_trace() while True: try: conn, addr = self.monitor.accept() rmtHost = socket.gethostbyaddr(addr[0])[0] logging.debug( stampedMsg('FS Monitor recieve instruction from {}'). format(rmtHost)) except socket.error, e: logging.warning("Caught exception socket.error : %s" % e) logging.warning( stampedMsg('Fail to receive signal from clients {}'.format( rmtHost))) break #TODO: should we break listening if UDP reception has troubles? message = receive_all_decrypted(conn) # the instruction if not message: # possibly never called in UDP logging.info( stampedMsg('Receiving stop signal from clients {}'.format( rmtHost))) break # log whatever recieved logging.debug(stampedMsg(message)) if message == self.message_file: # if receive leave signal # include filename info for debugging purposes if rmtHost == self.hostName: filename = str(receive_all_decrypted(conn)) else: filename, _ = receive_all_to_target( conn, self.messageInterval) logging.info( stampedMsg('receiving file {} from {}'.format( filename, rmtHost))) self.local_file_info[filename] = datetime.datetime.now( ).isoformat() if filename in self.global_file_info: self.global_file_info[filename][0] = self.timer.toc() elif message == self.message_data: # .... filename, file_nodes = receive_all_decrypted(conn) filename = str( filename) # get rid of annoying utf-encoding prefix file_nodes = list(map(str, file_nodes)) self.global_file_info[filename] = [ self.timer.toc(), file_nodes ] elif message == self.message_ask_time: filename = receive_all_decrypted(conn) send_all_encrypted(conn, self.local_file_info[filename]) elif message == self.message_ask_file: filename = receive_all_decrypted(conn) send_all_from_file(conn, filename, self.messageInterval) elif message == self.message_delete_data: filename = receive_all_decrypted(conn) if filename in self.global_file_info: del self.global_file_info[filename] elif message == self.message_delete_file: filename = receive_all_decrypted(conn) if filename in self.local_file_info: del self.local_file_info[filename] try: os.remove(filename) except: logging.debug( stampedMsg( 'Deleting file {} failed'.format(filename)))