def start_as_client(self):
print 'I am the client!'
sleep(0.5)
real_members = [
host.split('_')[0] for host in sorted(self.membList.keys())
]
print 'All members: {}'.format(real_members)
self.masters_workers = [
socket.gethostbyname(host) for host in real_members
if host != self.host_name
]
for host in self.masters_workers:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, self.port))
send_all_encrypted(sock, self.message_input)
send_all_from_file(sock, self.app_file, self.messageInterval)
send_all_encrypted(sock, self.app_args)
send_all_encrypted(sock, self.masters_workers)
send_all_encrypted(sock, self.is_undirected)
if host in self.masters_workers[0:2]:
send_all_from_file(sock, self.filename_pair[0],
self.messageInterval)
send_all_encrypted(sock, self.filename_pair[1])
def broadCastFile(self, targets, filename):
for target in targets:
target_host, target_nodeName, sock = self.getParams(target)
send_all_encrypted(sock, self.message_file)
if target == self.groupID:
send_all_encrypted(sock, filename)
else:
send_all_from_file(sock, filename, self.messageInterval)
logging.debug(
stampedMsg('{} pushing file {} to node {}'.format(
self.nodeName, filename, target_nodeName)))
def preprocess(self):
sleep(0.5)
self.server_task = Thread(target=self.background_server)
self.server_task.daemon = True
self.server_task.start()
print('I have {} workers!'.format(self.num_workers))
self.v_to_m_dict, self.num_vertices = parse_file(
self.input_filename, self.num_workers, self.masters_workers)
print('num_vertices: ', self.num_vertices)
for ix in range(self.num_workers):
sock = self.send_to_worker([
Commons.request_preprocess, self.input_filename,
self.v_to_m_dict, self.num_vertices
], self.masters_workers[ix + 2])
send_all_from_file(sock, self.input_filename, 0.001)
while (self.num_preprocess_done < self.num_workers):
sleep(1)
def collect_results(self):
self.result_files = [0] * self.num_workers
for ix in range(self.num_workers):
worker = self.alive_workers[ix]
self.result_files[ix] = 'file_piece_' + str(ix) + '_out'
self.send_to_worker(
[Commons.request_result, self.result_files[ix]], worker)
while (self.num_process_done < self.num_workers):
sleep(1)
for ix in range(self.num_workers):
dfsWrapper(self.dfs.getFile, self.result_files[ix])
combine_files(self.output_filename, self.result_files)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.client_ip, self.driver_port))
send_all_encrypted(sock, self.client_message)
send_all_from_file(sock, self.output_filename, 0.001)
self.remote_end_tasks()
def server_task(self):
#first, start local timer, the rest of the process follows this timer
self.timer.tic()
# a monitor receive message, check and response, also multicase failure message
self.monitor = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.monitor.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.monitor.bind((self.host, self.port))
self.monitor.listen(5)
# self.monitor.listen(10) # UDP doesn't support this
logging.info(stampedMsg('FS Monitoring process opens.'))
# keep receiving msgs from other VMs
# receiving heartbeat and other messages
# pdb.set_trace()
while True:
try:
conn, addr = self.monitor.accept()
rmtHost = socket.gethostbyaddr(addr[0])[0]
logging.debug(
stampedMsg('FS Monitor recieve instruction from {}').
format(rmtHost))
except socket.error, e:
logging.warning("Caught exception socket.error : %s" % e)
logging.warning(
stampedMsg('Fail to receive signal from clients {}'.format(
rmtHost)))
break #TODO: should we break listening if UDP reception has troubles?
message = receive_all_decrypted(conn) # the instruction
if not message: # possibly never called in UDP
logging.info(
stampedMsg('Receiving stop signal from clients {}'.format(
rmtHost)))
break
# log whatever recieved
logging.debug(stampedMsg(message))
if message == self.message_file: # if receive leave signal
# include filename info for debugging purposes
if rmtHost == self.hostName:
filename = str(receive_all_decrypted(conn))
else:
filename, _ = receive_all_to_target(
conn, self.messageInterval)
logging.info(
stampedMsg('receiving file {} from {}'.format(
filename, rmtHost)))
self.local_file_info[filename] = datetime.datetime.now(
).isoformat()
if filename in self.global_file_info:
self.global_file_info[filename][0] = self.timer.toc()
elif message == self.message_data: # ....
filename, file_nodes = receive_all_decrypted(conn)
filename = str(
filename) # get rid of annoying utf-encoding prefix
file_nodes = list(map(str, file_nodes))
self.global_file_info[filename] = [
self.timer.toc(), file_nodes
]
elif message == self.message_ask_time:
filename = receive_all_decrypted(conn)
send_all_encrypted(conn, self.local_file_info[filename])
elif message == self.message_ask_file:
filename = receive_all_decrypted(conn)
send_all_from_file(conn, filename, self.messageInterval)
elif message == self.message_delete_data:
filename = receive_all_decrypted(conn)
if filename in self.global_file_info:
del self.global_file_info[filename]
elif message == self.message_delete_file:
filename = receive_all_decrypted(conn)
if filename in self.local_file_info:
del self.local_file_info[filename]
try:
os.remove(filename)
except:
logging.debug(
stampedMsg(
'Deleting file {} failed'.format(filename)))