def background_server(self):
self.server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server_sock.bind((self.host, self.master_port))
self.server_sock.listen(5)
while True:
conn, addr = self.server_sock.accept()
rmtHost = socket.gethostbyaddr(addr[0])[0]
try:
message = receive_all_decrypted(conn)
except (socket.error, ValueError) as e:
continue
if message == Commons.ack_preprocess:
self.num_preprocess_done += 1
elif message == Commons.finish_compute:
halt = receive_all_decrypted(conn)
self.all_done.append(halt)
elif message == Commons.ack_result:
self.num_process_done += 1
elif message == self.fail_message:
self.failures.append(receive_all_decrypted(conn))
elif message == Commons.new_master:
superstep, halt = receive_all_decrypted(conn)
assert (self.superstep == 0 or self.superstep == superstep)
self.superstep = superstep
self.all_done.append(halt)
def change_work(self, conn, addr):
print('receive request to change work')
superstep, self.alive_workers, vertices_info, v_to_m_dict = receive_all_decrypted(
conn)
self.curr_thread.join()
self.reinit_vars()
self.superstep, self.v_to_m_dict = superstep, v_to_m_dict
self.hasFailure = True
file_edges = checkpt_file_name(self.machine_ix, 0)
file_vals = checkpt_file_name(self.machine_ix, superstep)
file_messages = checkpt_message_file_name(self.machine_ix, superstep)
collect_vertices_info(file_edges, file_vals, file_messages,
vertices_info)
self.vertices = {}
self.first_len_message = defaultdict(int)
self.vertex_to_messages = defaultdict(list)
for v in vertices_info:
neighbors, value, first_len, messages = vertices_info[v]
assert (v not in self.vertices)
self.init_vertex(v)
for n in neighbors:
self.vertices[v].neighbors.append([n, 1, self.gethost(n)])
self.vertices[v].value = value
self.first_len_message[v] = int(first_len)
self.vertex_to_messages[v] = map(float, messages)
self.sorted_vertices = map(str, sorted(map(int, self.vertices.keys())))
print('Now we have {} vertices~'.format(len(self.sorted_vertices)))
def return_result_file(self, conn, addr):
self.output_filename, = receive_all_decrypted(conn)
self.load_to_file(self.output_filename)
dfsWrapper(self.dfs.putFile, self.output_filename)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.addr, self.master_port))
send_all_encrypted(sock, Commons.ack_result)
def start_main_server(self):
print('I am worker No.{}!'.format(self.machine_ix))
self.monitor = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.monitor.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.monitor.bind((self.host, self.worker_port))
self.monitor.listen(5)
while True:
try:
conn, addr = self.monitor.accept()
message = receive_all_decrypted(conn)
if message == Commons.request_preprocess:
self.load_and_preprocess(conn, addr)
elif message == Commons.request_compute:
superstep, checkpt = receive_all_decrypted(conn)
self.curr_thread = threading.Thread(target=self.compute,
args=(superstep,
checkpt))
self.curr_thread.daemon = True
self.curr_thread.start()
elif message == Commons.request_result: # final step
self.return_result_file(conn, addr)
elif message == Commons.end_now:
sys.exit()
elif message == None: # for inner vertex communication
self.new_thread_queue(receive_all_decrypted(conn), addr)
elif message == 'buffer_count':
self.receive_buffer_count[addr[0]] = receive_all_decrypted(
conn)
if message == Commons.new_master:
threading.Thread(target=self.new_master,
args=(addr, )).start()
elif message == Commons.work_change:
self.change_work(conn, addr)
except (socket.error, ValueError) as e:
print(e)
continue
def background_server(self, queue):
conn, addr = self.server_sock.accept()
rmtHost = socket.gethostbyaddr(addr[0])[0]
message = receive_all_decrypted(conn) # the instruction
if message == self.message_input:
self.input_task.terminate()
print
self.app_file, _ = receive_all_to_target(conn,
self.messageInterval)
self.app_args = receive_all_decrypted(conn)
self.masters_workers = receive_all_decrypted(conn)
self.is_undirected = receive_all_decrypted(conn)
if self.host in self.masters_workers[0:2]:
self.role = 'master'
self.filename_pair[0], _ = receive_all_to_target(
conn, self.messageInterval)
self.filename_pair[1] = receive_all_decrypted(conn)
if self.host == self.masters_workers[1]:
self.role = 'standby'
print 'I am the standby master!'
else:
self.role = 'worker'
queue.put(
(self.app_file, self.app_args, self.filename_pair, self.role,
addr[0], self.masters_workers, self.is_undirected))
return
elif message == self.message_output: # for client and standby
if self.role != 'standby': # a hack since self.role not updated in this process
filename, _ = receive_all_to_target(conn, self.messageInterval)
assert (filename == self.result_file)
print 'Task done, result is published to {}'.format(filename)
elif message == self.message_congrats:
print('Hehe, now it is my turn')
self.role = 'master'
queue.put(self.role)
def mostRecentNode(self, targets, filename):
max_time, max_target = None, None
for target in targets:
target_host, target_nodeName, sock = self.getParams(target)
send_all_encrypted(sock, self.message_ask_time)
send_all_encrypted(sock, filename)
timestamp = receive_all_decrypted(sock)
if max_time == None or timestamp > max_time:
max_time, max_target = timestamp, target
return max_target
def load_and_preprocess(self, conn, addr):
start_time = time.time()
print('receive command to load file')
self.addr = addr[0]
self.input_filename, self.v_to_m_dict, self.num_vertices = receive_all_decrypted(
conn)
self.input_filename, _ = receive_all_to_target(conn, 0.001)
self.preprocess(self.input_filename)
print('preprocess done after {} seconds'.format(time.time() -
start_time))
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.addr, self.master_port))
send_all_encrypted(sock, Commons.ack_preprocess)
def server_task(self):
#first, start local timer, the rest of the process follows this timer
self.timer.tic()
# a monitor receive message, check and response, also multicase failure message
self.monitor = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.monitor.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.monitor.bind((self.host, self.port))
self.monitor.listen(5)
# self.monitor.listen(10) # UDP doesn't support this
logging.info(stampedMsg('FS Monitoring process opens.'))
# keep receiving msgs from other VMs
# receiving heartbeat and other messages
# pdb.set_trace()
while True:
try:
conn, addr = self.monitor.accept()
rmtHost = socket.gethostbyaddr(addr[0])[0]
logging.debug(
stampedMsg('FS Monitor recieve instruction from {}').
format(rmtHost))
except socket.error, e:
logging.warning("Caught exception socket.error : %s" % e)
logging.warning(
stampedMsg('Fail to receive signal from clients {}'.format(
rmtHost)))
break #TODO: should we break listening if UDP reception has troubles?
message = receive_all_decrypted(conn) # the instruction
if not message: # possibly never called in UDP
logging.info(
stampedMsg('Receiving stop signal from clients {}'.format(
rmtHost)))
break
# log whatever recieved
logging.debug(stampedMsg(message))
if message == self.message_file: # if receive leave signal
# include filename info for debugging purposes
if rmtHost == self.hostName:
filename = str(receive_all_decrypted(conn))
else:
filename, _ = receive_all_to_target(
conn, self.messageInterval)
logging.info(
stampedMsg('receiving file {} from {}'.format(
filename, rmtHost)))
self.local_file_info[filename] = datetime.datetime.now(
).isoformat()
if filename in self.global_file_info:
self.global_file_info[filename][0] = self.timer.toc()
elif message == self.message_data: # ....
filename, file_nodes = receive_all_decrypted(conn)
filename = str(
filename) # get rid of annoying utf-encoding prefix
file_nodes = list(map(str, file_nodes))
self.global_file_info[filename] = [
self.timer.toc(), file_nodes
]
elif message == self.message_ask_time:
filename = receive_all_decrypted(conn)
send_all_encrypted(conn, self.local_file_info[filename])
elif message == self.message_ask_file:
filename = receive_all_decrypted(conn)
send_all_from_file(conn, filename, self.messageInterval)
elif message == self.message_delete_data:
filename = receive_all_decrypted(conn)
if filename in self.global_file_info:
del self.global_file_info[filename]
elif message == self.message_delete_file:
filename = receive_all_decrypted(conn)
if filename in self.local_file_info:
del self.local_file_info[filename]
try:
os.remove(filename)
except:
logging.debug(
stampedMsg(
'Deleting file {} failed'.format(filename)))