class Worker:
def __init__(self, recv_queue, send_queue, master_queue, status_queue,
data_cluster_write, data_cluster_read, my_id):
self.my_id = my_id
self.global_saver = StateSaver(GLOBAL_STAGE, data_cluster_write,
data_cluster_read)
self.single_saver = StateSaver(my_id, data_cluster_write,
data_cluster_read)
self.date_redirector = DateRedirector(recv_queue, send_queue,
master_queue, status_queue,
self.global_saver,
self.single_saver, self.my_id)
self.coordinator = Coordinator(self.my_id, data_cluster_write,
data_cluster_read)
def start(self):
state = self.single_saver.load_state("STATE")
if state == "WAITING":
#Wait for coordinator
print("Waiting for coordinator")
self.coordinator.wait_to_work()
self.single_saver.save_state("STATE", "", "READY")
else:
print("Starting to receive")
self.date_redirector.start()
class Protocol:
def __init__(self, recv_queue, send_queue, total_workers, status_queue,
worker_id, data_cluster_write, data_cluster_read):
self.connection = Connection()
self.worker_id = worker_id
self.total_workers = total_workers
self.pending_connections = total_workers
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.send_queue = send_queue
self.state_saver = StateSaver(self.worker_id, data_cluster_write,
data_cluster_read)
self.connection_id = None
def start_connection(self):
self.receiver.start_receiving(self.data_read)
self.connection.close()
def data_read(self, msg_type, msg):
print("Msg received: {}".format(msg))
if msg_type == STOP:
self.receiver.close()
self.sender.send(STOP, '')
self.status_sender.send(FINISHED, FINISHED)
return
data_recv = json.loads(msg)
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated(connection_id, message_id):
print("Duplicated message: {}".format(msg))
return
if connection_id != self.connection_id:
old_data = self.state_saver.load_state(connection_id)
if old_data is not None:
self.pending_connections = old_data
else:
self.pending_connections = self.total_workers
self.connection_id = connection_id
if msg_type == EOF:
self.pending_connections -= 1
if self.pending_connections == 0:
data_recv[1] = RESUME_MSG_ID
self.sender.send(EOF, json.dumps(data_recv))
print("Ended processing")
data_to_save = self.pending_connections
self.state_saver.save_state(connection_id, message_id, data_to_save)
def __init__(self, recv_queue, send_queues, data_cluster_write,
data_cluster_read, place_manager_queue):
self.state_saver = StateSaver(STAGE, data_cluster_write,
data_cluster_read)
self.protocol = Protocol(recv_queue, send_queues, self.state_saver,
place_manager_queue)
self.working = False
self.actual_conn = None
def __init__(self, recv_queue, send_queue, status_queue,
data_cluster_write, data_cluster_read):
self.connection = Connection()
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.state_saver = StateSaver(STAGE, data_cluster_write,
data_cluster_read)
self.connection_id = None
def __init__(self, recv_queue, recv_request_queue, cluster_w_dir, cluster_r_dir):
self.cluster_w_dir = cluster_w_dir
self.cluster_r_dir = cluster_r_dir
self.recv_request_queue = recv_request_queue
self.state_saver = StateSaver(STAGE, cluster_w_dir, cluster_r_dir)
self.protocol = ReceiverProtocol(recv_queue, self.state_saver)
self.cluster_reader = SecureData(cluster_w_dir, cluster_r_dir)
self.places = {}
class Worker:
def __init__(self, recv_queue, send_queue, master_queue, status_queue,
data_cluster_write, data_cluster_read, worker_id):
self.global_saver = StateSaver(GLOBAL_STAGE, data_cluster_write,
data_cluster_read)
self.single_saver = StateSaver(worker_id, data_cluster_write,
data_cluster_read)
self.redirector = CountRedirector(recv_queue, send_queue, master_queue,
self.data_received,
self.no_more_data, status_queue,
self.global_saver, self.single_saver,
worker_id, self.load_data,
self.reset_data, self.save_data)
self.total_deceduti = 0
self.total_positivi = 0
self.coordinator = Coordinator(worker_id, data_cluster_write,
data_cluster_read)
def start(self):
state = self.single_saver.load_state("STATE")
if state == "WAITING":
#Wait for coordinator
print("Waiting for coordinator")
self.coordinator.wait_to_work()
self.single_saver.save_state("STATE", "", "READY")
else:
self.redirector.start()
def no_more_data(self):
self.redirector.send_data(self.total_positivi, self.total_deceduti)
def data_received(self, result):
if result == "positivi":
self.total_positivi += 1
else:
self.total_deceduti += 1
def load_data(self, data):
self.total_positivi = data[0]
self.total_deceduti = data[1]
def reset_data(self):
self.total_deceduti = 0
self.total_positivi = 0
def save_data(self):
return [self.total_positivi, self.total_deceduti]
def __init__(self, recv_queue, send_queue, master_queue, status_queue,
data_cluster_write, data_cluster_read, my_id):
self.my_id = my_id
self.global_saver = StateSaver(GLOBAL_STAGE, data_cluster_write,
data_cluster_read)
self.single_saver = StateSaver(my_id, data_cluster_write,
data_cluster_read)
self.date_redirector = DateRedirector(recv_queue, send_queue,
master_queue, status_queue,
self.global_saver,
self.single_saver, self.my_id)
self.coordinator = Coordinator(self.my_id, data_cluster_write,
data_cluster_read)
def __init__(self, recv_queue, send_queue, total_workers, status_queue,
worker_id, data_cluster_write, data_cluster_read):
self.connection = Connection()
self.worker_id = worker_id
self.total_workers = total_workers
self.pending_connections = total_workers
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.send_queue = send_queue
self.state_saver = StateSaver(self.worker_id, data_cluster_write,
data_cluster_read)
self.connection_id = None
def __init__(self, recv_queue, send_queue, master_queue, status_queue,
data_cluster_write, data_cluster_read, worker_id):
self.global_saver = StateSaver(GLOBAL_STAGE, data_cluster_write,
data_cluster_read)
self.single_saver = StateSaver(worker_id, data_cluster_write,
data_cluster_read)
self.redirector = CountRedirector(recv_queue, send_queue, master_queue,
self.data_received,
self.no_more_data, status_queue,
self.global_saver, self.single_saver,
worker_id, self.load_data,
self.reset_data, self.save_data)
self.total_deceduti = 0
self.total_positivi = 0
self.coordinator = Coordinator(worker_id, data_cluster_write,
data_cluster_read)
def __init__(
self,
recv_queue,
status_queue,
data_cluster_write,
data_cluster_read,
coordinator_queue
):
self.connection = Connection()
self.expected = EXPECTED_EOF
self.actual = 0
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.coordinator_sender = SecureRpcSender(coordinator_queue, Connection())
self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read)
self.connection_id = None
def __init__(
self,
recv_queue,
send_queue,
master_queue,
recv_init_queue,
status_queue,
data_cluster_write,
data_cluster_read,
my_id
):
self.global_saver = StateSaver(GLOBAL_STAGE, data_cluster_write, data_cluster_read)
self.single_saver = StateSaver(my_id, data_cluster_write, data_cluster_read)
self.map_controller = MapController(
recv_queue,
send_queue,
master_queue,
self.process_data,
status_queue,
self.global_saver,
self.single_saver,
my_id
)
self.initialize_protocol = ProtocolInitialize(
recv_init_queue,
self.process_places
)
self.cluster_reader = SecureData(data_cluster_write, data_cluster_read)
self.coordinator = Coordinator(
my_id,
data_cluster_write,
data_cluster_read
)
class Coordinator:
def __init__(self, recv_queue, data_cluster_write, data_cluster_read):
self.connection = Connection()
self.state_saver = StateSaver("coordinator_" + recv_queue,
data_cluster_write, data_cluster_read)
self.recv_queue = recv_queue
def wait_to_work(self):
self.receiver = SecureDirectReceiver(self.recv_queue, self.connection)
self.receiver.start_receiving(self.receive_to_work)
def receive_to_work(self, msg_type, conn_id):
#Check duplicates
if self.state_saver.is_duplicated(conn_id, msg_type):
print("Duplicated message: {} {}".format(conn_id, msg_type))
return
print("Receive to start working")
if msg_type == RESTART:
self.receiver.close()
self.state_saver.save_state(conn_id, msg_type, ".")
class CoordinatorManager:
def __init__(self, recv_queue, send_queues, data_cluster_write,
data_cluster_read, place_manager_queue):
self.state_saver = StateSaver(STAGE, data_cluster_write,
data_cluster_read)
self.protocol = Protocol(recv_queue, send_queues, self.state_saver,
place_manager_queue)
self.working = False
self.actual_conn = None
def start(self):
load = self.state_saver.load_state("STATE")
if load != None:
[conn_id, state] = json.loads(load)
self.actual_conn = conn_id
if state == "RESTART":
self.work_finished(conn_id)
elif state == "BLOCKED":
self.working = True
self.protocol.start_receiving(self.work_finished, self.new_client)
def new_client(self, conn_id):
if conn_id == self.actual_conn:
return READY
if self.working:
return NOT_READY
self.working = True
self.actual_conn = conn_id
return READY
def work_finished(self, conn_id):
#Force place manager to restart its requester
#and force every worker to restart
self.protocol.restart_all_senders(conn_id)
self.working = False
class Protocol:
def __init__(self, recv_queue, send_queue, status_queue,
data_cluster_write, data_cluster_read):
self.connection = Connection()
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.connection_id = None
self.state_saver = StateSaver(STAGE, data_cluster_write,
data_cluster_read)
def start_connection(self, callback, callback_eof, callback_load,
callback_reset, callback_save):
self.callback = callback
self.callback_eof = callback_eof
self.callback_load = callback_load
self.callback_reset = callback_reset
self.callback_save = callback_save
self.receiver.start_receiving(self.data_read)
def data_read(self, msg_type, msg):
if msg_type == STOP:
self.receiver.close()
self.sender.send(STOP, '')
self.status_sender.send(FINISHED, FINISHED)
return
data_recv = json.loads(msg)
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated(connection_id, message_id):
print("Duplicated message: {}".format(msg))
return
if connection_id != self.connection_id:
old_data = self.state_saver.load_state(connection_id)
if old_data is not None:
self.callback_load(old_data)
else:
self.callback_reset()
self.connection_id = connection_id
if msg_type == EOF:
print("Ended processing")
self.callback_eof()
else:
[positivi, deceduti] = data_recv[2:]
self.callback(positivi, deceduti)
data_to_save = self.callback_save()
self.state_saver.save_state(connection_id, message_id, data_to_save)
def send_data(self, data):
data_to_send = [self.connection_id, COUNT_MSG_ID, data]
eof_to_send = [self.connection_id, EOF_MSG_ID]
self.sender.send(TOTAL_COUNT, json.dumps(data_to_send))
self.sender.send(EOF, json.dumps(eof_to_send))
class Protocol:
def __init__(
self,
recv_queue,
status_queue,
data_cluster_write,
data_cluster_read,
coordinator_queue
):
self.connection = Connection()
self.expected = EXPECTED_EOF
self.actual = 0
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.coordinator_sender = SecureRpcSender(coordinator_queue, Connection())
self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read)
self.connection_id = None
def start_connection(
self,
callback_top,
callback_date,
callback_count,
callback_all_data,
callback_load,
callback_reset,
callback_save
):
self.callback_top = callback_top
self.callback_date = callback_date
self.callback_count = callback_count
self.callback_all_data = callback_all_data
self.callback_load = callback_load
self.callback_reset = callback_reset
self.callback_save = callback_save
if self.actual < self.expected:
self.receiver.start_receiving(self.data_read)
self.connection.close()
def data_read(self, msg_type, msg):
print("Msg received" + msg)
if msg_type == STOP:
self.receiver.close()
self.status_sender.send(FINISHED, FINISHED)
return
data_recv = json.loads(msg)
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated(connection_id, message_id):
print("Duplicated message: {}".format(msg))
return
if connection_id != self.connection_id:
last_state = self.state_saver.load_state(connection_id)
if last_state is not None:
[old_data, actual_conns] = last_state
self.actual = actual_conns
self.callback_load(old_data)
else:
self.actual = 0
self.callback_reset()
self.connection_id = connection_id
if msg_type == EOF:
self.actual += 1
if self.actual == self.expected:
self.callback_all_data(self.connection_id)
self.reset_coordinator()
print("Ended processing")
elif msg_type == TOP_CITIES:
print("Received TOP CITIES")
[data] = data_recv[2:]
self.callback_top(json.loads(data))
elif msg_type == DATE_RESULTS:
print("Received DATE RESULTS")
[data] = data_recv[2:]
self.callback_date(json.loads(data))
elif msg_type == TOTAL_COUNT:
print("Received COUNT TOTAL")
[data] = data_recv[2:]
self.callback_count(data)
data_to_save = [self.callback_save(), self.actual]
self.state_saver.save_state(connection_id, message_id, data_to_save)
def reset_coordinator(self):
_tmp = self.coordinator_sender.send(json.dumps([self.connection_id, RESTART]))
print("Received: {}".format(_tmp))
class Protocol:
def __init__(self, recv_queue, send_queue, total_workers, status_queue,
data_cluster_write, data_cluster_read):
self.connection = Connection()
self.total_workers = total_workers
self.pending_connections = total_workers
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.state_saver = StateSaver(STAGE, data_cluster_write,
data_cluster_read)
self.connection_id = None
def start_connection(self, callback, callback_eof, callback_load,
callback_reset, callback_save):
self.callback = callback
self.callback_eof = callback_eof
self.callback_load = callback_load
self.callback_reset = callback_reset
self.callback_save = callback_save
self.receiver.start_receiving(self.data_read)
def data_read(self, msg_type, msg):
print("Msg received" + msg)
if msg_type == STOP:
self.receiver.close()
self.sender.send(STOP, '')
self.status_sender.send(FINISHED, FINISHED)
return
data_recv = json.loads(msg)
print("DATE SORTER: {}".format(msg))
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated(connection_id, message_id):
print("Duplicated message: {}".format(msg))
return
if connection_id != self.connection_id:
last_state = self.state_saver.load_state(connection_id)
if last_state is not None:
[old_data, pending_conns] = last_state
self.pending_connections = pending_conns
self.callback_load(old_data)
else:
self.pending_connections = self.total_workers
self.callback_reset()
self.connection_id = connection_id
if msg_type == EOF:
self.pending_connections -= 1
if self.pending_connections == 0:
self.callback_eof()
print("Ended processing")
else:
[data] = data_recv[2:]
self.callback(json.loads(data))
data_to_save = [self.callback_save(), self.pending_connections]
self.state_saver.save_state(connection_id, message_id, data_to_save)
def send_data(self, data):
print("Sending Dates results")
new_data = [self.connection_id, DATE_MSG_ID, json.dumps(data)]
eof_to_send = [self.connection_id, EOF_MSG_ID]
self.sender.send(DATE_RESULTS, json.dumps(new_data))
self.sender.send(EOF, json.dumps(eof_to_send))
class Worker:
def __init__(
self,
recv_queue,
send_queue,
master_queue,
recv_init_queue,
status_queue,
data_cluster_write,
data_cluster_read,
my_id
):
self.global_saver = StateSaver(GLOBAL_STAGE, data_cluster_write, data_cluster_read)
self.single_saver = StateSaver(my_id, data_cluster_write, data_cluster_read)
self.map_controller = MapController(
recv_queue,
send_queue,
master_queue,
self.process_data,
status_queue,
self.global_saver,
self.single_saver,
my_id
)
self.initialize_protocol = ProtocolInitialize(
recv_init_queue,
self.process_places
)
self.cluster_reader = SecureData(data_cluster_write, data_cluster_read)
self.coordinator = Coordinator(
my_id,
data_cluster_write,
data_cluster_read
)
def process_places(self, region, longitude, latitude):
point = NamedPoint(region, longitude, latitude)
self.places.append(point)
def read_places(self, conn_id):
print("Reading places from {}".format(conn_id))
result = self.cluster_reader.read_file(conn_id, "places.txt")
places = json.loads(result)
# Restart places
self.places = []
for place in places:
print("{} {} {}".format(place, float(places[place][0]), float(places[place][1])))
self.process_places(place, float(places[place][0]), float(places[place][1]))
def start(self):
self.places = []
state = self.single_saver.load_state("STATE")
if state == "WAITING":
#Wait for coordinator
print("Waiting for coordinator")
self.coordinator.wait_to_work()
self.single_saver.save_state("STATE", "", "READY")
else:
#Block until places has been saved
conn_id = self.initialize_protocol.start_connection()
#Read places
self.read_places(conn_id)
print("All places received")
self.map_controller.start()
def process_data(self, latitude, longitude):
point = Point(longitude, latitude)
return point.closest_point(self.places).name
class PlaceReceiver:
def __init__(self, recv_queue, recv_request_queue, cluster_w_dir, cluster_r_dir):
self.cluster_w_dir = cluster_w_dir
self.cluster_r_dir = cluster_r_dir
self.recv_request_queue = recv_request_queue
self.state_saver = StateSaver(STAGE, cluster_w_dir, cluster_r_dir)
self.protocol = ReceiverProtocol(recv_queue, self.state_saver)
self.cluster_reader = SecureData(cluster_w_dir, cluster_r_dir)
self.places = {}
def start(self):
load = self.state_saver.load_state("STATE")
if load != None:
[conn_id, state] = json.loads(load)
if state == "RESTART":
print("Starting to receive places")
conn_id = self.protocol.start_connection(
self.data_read,
self.load_data,
self.reset_data,
self.save_data
)
print("Starting to accept requests")
self.protocol_requester = RequesterProtocol(
self.recv_request_queue,
self.state_saver
)
self.protocol_requester.start_connection(
conn_id
)
elif state == "REQUESTER":
print("Starting to accept requests")
self.protocol_requester = RequesterProtocol(
self.recv_request_queue,
self.state_saver
)
self.protocol_requester.start_connection(
conn_id
)
else:
print("Starting to receive places")
conn_id = self.protocol.start_connection(
self.data_read,
self.load_data,
self.reset_data,
self.save_data
)
print("Starting to accept requests")
self.protocol_requester = RequesterProtocol(
self.recv_request_queue,
self.state_saver
)
self.protocol_requester.start_connection(
conn_id
)
def load_data(self, places):
self.places = places
def reset_data(self):
self.places = {}
def save_data(self):
return self.places
def data_read(self, conn_id, place, latitude, longitude):
self.places[place] = (latitude, longitude)
self.cluster_reader.write_to_file(conn_id, "places.txt", json.dumps(self.places))
print("Write finished")
class Protocol:
def __init__(self, recv_queue, send_queue, status_queue,
data_cluster_write, data_cluster_read):
self.connection = Connection()
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.state_saver = StateSaver(STAGE, data_cluster_write,
data_cluster_read)
self.connection_id = None
def start_connection(self, callback, callback_eof, callback_load,
callback_reset, callback_save):
self.callback = callback
self.callback_eof = callback_eof
self.callback_load = callback_load
self.callback_reset = callback_reset
self.callback_save = callback_save
self.receiver.start_receiving(self.data_read)
def send_data(self, data):
#Unique message so that if this fails, the next one that raises will
#send the same id
data_to_send = [self.connection_id, PLACE_MSG_ID, json.dumps(data)]
eof_to_send = [self.connection_id, EOF_MSG_ID]
self.sender.send(NORMAL, json.dumps(data_to_send))
self.sender.send(EOF, json.dumps(eof_to_send))
def data_read(self, msg_type, msg):
if msg_type == STOP:
self.receiver.close()
self.sender.send(STOP, STOP)
self.status_sender.send(FINISHED, FINISHED)
return
data_recv = json.loads(msg)
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated(connection_id, message_id):
print("Duplicated message: {}".format(msg))
return
if connection_id != self.connection_id:
old_data = self.state_saver.load_state(connection_id)
if old_data is not None:
self.callback_load(old_data)
else:
self.callback_reset()
self.connection_id = connection_id
if msg_type == EOF:
print("Ended processing")
self.callback_eof()
else:
[place] = data_recv[2:]
self.callback(place)
data_to_save = self.callback_save()
self.state_saver.save_state(connection_id, message_id, data_to_save)
def __init__(self, recv_queue, data_cluster_write, data_cluster_read):
self.connection = Connection()
self.state_saver = StateSaver("coordinator_" + recv_queue,
data_cluster_write, data_cluster_read)
self.recv_queue = recv_queue