def __init__(self, recv_queue, send_queue, total_workers, status_queue):
self.connection = Connection()
print("Connected to RabbitMQ")
self.total_workers = total_workers
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDistributedSender(send_queue, self.connection)
class Protocol:
def __init__(self, send_queue):
self.connection = Connection()
self.sender = SecureDirectSender(send_queue, self.connection)
def send_status(self, status, worker_id, worker_type):
msg = status + "," + worker_id + "," + worker_type
self.sender.send(STATUS, msg)
def __init__(self, recv_queue, send_queue, status_queue,
data_cluster_write, data_cluster_read):
self.connection = Connection()
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.state_saver = StateSaver(STAGE, data_cluster_write,
data_cluster_read)
self.connection_id = None
def __init__(self, recv_queue, send_queues, master_send_queue, status_queue, state_saver, my_id):
self.connection = Connection()
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.senders = {}
self.recv_queue = recv_queue
for queue in send_queues:
self.senders[queue] = SecureDirectSender(queue, self.connection)
self.master_sender = SecureDirectSender(master_send_queue, self.connection)
self.state_saver = state_saver
self.my_id = my_id
print("Creating RabbitMQ Queue: {}".format(self.recv_queue))
self.receiver = SecureDistributedReceiver(self.recv_queue, self.connection)
def __init__(self, recv_queue, send_queue, total_workers, status_queue,
worker_id, data_cluster_write, data_cluster_read):
self.connection = Connection()
self.worker_id = worker_id
self.total_workers = total_workers
self.pending_connections = total_workers
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.send_queue = send_queue
self.state_saver = StateSaver(self.worker_id, data_cluster_write,
data_cluster_read)
self.connection_id = None
def __init__(
self,
recv_queue,
status_queue,
data_cluster_write,
data_cluster_read,
coordinator_queue
):
self.connection = Connection()
self.expected = EXPECTED_EOF
self.actual = 0
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.coordinator_sender = SecureRpcSender(coordinator_queue, Connection())
self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read)
self.connection_id = None
class Protocol:
def __init__(self, recv_queue, send_queue, total_workers, status_queue,
worker_id, data_cluster_write, data_cluster_read):
self.connection = Connection()
self.worker_id = worker_id
self.total_workers = total_workers
self.pending_connections = total_workers
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.send_queue = send_queue
self.state_saver = StateSaver(self.worker_id, data_cluster_write,
data_cluster_read)
self.connection_id = None
def start_connection(self):
self.receiver.start_receiving(self.data_read)
self.connection.close()
def data_read(self, msg_type, msg):
print("Msg received: {}".format(msg))
if msg_type == STOP:
self.receiver.close()
self.sender.send(STOP, '')
self.status_sender.send(FINISHED, FINISHED)
return
data_recv = json.loads(msg)
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated(connection_id, message_id):
print("Duplicated message: {}".format(msg))
return
if connection_id != self.connection_id:
old_data = self.state_saver.load_state(connection_id)
if old_data is not None:
self.pending_connections = old_data
else:
self.pending_connections = self.total_workers
self.connection_id = connection_id
if msg_type == EOF:
self.pending_connections -= 1
if self.pending_connections == 0:
data_recv[1] = RESUME_MSG_ID
self.sender.send(EOF, json.dumps(data_recv))
print("Ended processing")
data_to_save = self.pending_connections
self.state_saver.save_state(connection_id, message_id, data_to_save)
class Protocol:
def __init__(self, recv_queue, send_queue, total_workers, status_queue):
self.connection = Connection()
print("Connected to RabbitMQ")
self.total_workers = total_workers
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDistributedSender(send_queue, self.connection)
def start_connection(self):
self.receiver.start_receiving(self.data_read)
def data_read(self, msg_type, msg):
print("Got a message!")
if msg_type == STOP:
self.send_stop()
self.status_sender.send(FINISHED, FINISHED)
self.close()
elif msg_type == EOF:
self.send_eof(msg)
def send_eof(self, msg):
print("Sending EOF to workers: {}".format(msg))
[conn_id, msg_id, eof] = msg.split(',')
new_msg = [conn_id, msg_id, eof]
for i in range(0, self.total_workers):
self.sender.send(EOF, json.dumps(new_msg))
def send_stop(self):
for i in range(0, self.total_workers):
self.sender.send(STOP, '')
def close(self):
self.receiver.close()
self.connection.close()
class Protocol:
def __init__(self, recv_queue, send_queues, master_send_queue, status_queue, state_saver, my_id):
self.connection = Connection()
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.senders = {}
self.recv_queue = recv_queue
for queue in send_queues:
self.senders[queue] = SecureDirectSender(queue, self.connection)
self.master_sender = SecureDirectSender(master_send_queue, self.connection)
self.state_saver = state_saver
self.my_id = my_id
print("Creating RabbitMQ Queue: {}".format(self.recv_queue))
self.receiver = SecureDistributedReceiver(self.recv_queue, self.connection)
def start_connection(self, callback, callback_eof):
self.callback = callback
self.callback_eof = callback_eof
self.receiver.start_receiving(self.data_read)
def send_data(self, data, where):
self.senders[where].send(NORMAL, data)
def send_master_ended(self, msg):
data_recv = json.loads(msg)
data_recv[1] = self.my_id
self.master_sender.send(EOF, json.dumps(data_recv))
def send_master_stop(self):
self.master_sender.send(STOP, "")
def data_read(self, msg_type, msg):
print("Received: {}".format(msg))
if msg_type == STOP:
self.receiver.close()
self.send_master_stop()
self.status_sender.send(FINISHED, FINISHED)
return
if msg_type == EOF:
data_recv = json.loads(msg)
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated("STATE", connection_id):
print("Duplicated message: {}".format(connection_id))
return
self.callback_eof(msg)
self.send_master_ended(msg)
print("Ended processing")
self.state_saver.save_state("STATE", connection_id, "WAITING")
self.receiver.close()
else:
self.callback(msg)
class Protocol:
def __init__(self, recv_queue, send_queue, status_queue,
data_cluster_write, data_cluster_read):
self.connection = Connection()
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.connection_id = None
self.state_saver = StateSaver(STAGE, data_cluster_write,
data_cluster_read)
def start_connection(self, callback, callback_eof, callback_load,
callback_reset, callback_save):
self.callback = callback
self.callback_eof = callback_eof
self.callback_load = callback_load
self.callback_reset = callback_reset
self.callback_save = callback_save
self.receiver.start_receiving(self.data_read)
def data_read(self, msg_type, msg):
if msg_type == STOP:
self.receiver.close()
self.sender.send(STOP, '')
self.status_sender.send(FINISHED, FINISHED)
return
data_recv = json.loads(msg)
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated(connection_id, message_id):
print("Duplicated message: {}".format(msg))
return
if connection_id != self.connection_id:
old_data = self.state_saver.load_state(connection_id)
if old_data is not None:
self.callback_load(old_data)
else:
self.callback_reset()
self.connection_id = connection_id
if msg_type == EOF:
print("Ended processing")
self.callback_eof()
else:
[positivi, deceduti] = data_recv[2:]
self.callback(positivi, deceduti)
data_to_save = self.callback_save()
self.state_saver.save_state(connection_id, message_id, data_to_save)
def send_data(self, data):
data_to_send = [self.connection_id, COUNT_MSG_ID, data]
eof_to_send = [self.connection_id, EOF_MSG_ID]
self.sender.send(TOTAL_COUNT, json.dumps(data_to_send))
self.sender.send(EOF, json.dumps(eof_to_send))
class Protocol:
def __init__(
self,
recv_queue,
status_queue,
data_cluster_write,
data_cluster_read,
coordinator_queue
):
self.connection = Connection()
self.expected = EXPECTED_EOF
self.actual = 0
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.coordinator_sender = SecureRpcSender(coordinator_queue, Connection())
self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read)
self.connection_id = None
def start_connection(
self,
callback_top,
callback_date,
callback_count,
callback_all_data,
callback_load,
callback_reset,
callback_save
):
self.callback_top = callback_top
self.callback_date = callback_date
self.callback_count = callback_count
self.callback_all_data = callback_all_data
self.callback_load = callback_load
self.callback_reset = callback_reset
self.callback_save = callback_save
if self.actual < self.expected:
self.receiver.start_receiving(self.data_read)
self.connection.close()
def data_read(self, msg_type, msg):
print("Msg received" + msg)
if msg_type == STOP:
self.receiver.close()
self.status_sender.send(FINISHED, FINISHED)
return
data_recv = json.loads(msg)
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated(connection_id, message_id):
print("Duplicated message: {}".format(msg))
return
if connection_id != self.connection_id:
last_state = self.state_saver.load_state(connection_id)
if last_state is not None:
[old_data, actual_conns] = last_state
self.actual = actual_conns
self.callback_load(old_data)
else:
self.actual = 0
self.callback_reset()
self.connection_id = connection_id
if msg_type == EOF:
self.actual += 1
if self.actual == self.expected:
self.callback_all_data(self.connection_id)
self.reset_coordinator()
print("Ended processing")
elif msg_type == TOP_CITIES:
print("Received TOP CITIES")
[data] = data_recv[2:]
self.callback_top(json.loads(data))
elif msg_type == DATE_RESULTS:
print("Received DATE RESULTS")
[data] = data_recv[2:]
self.callback_date(json.loads(data))
elif msg_type == TOTAL_COUNT:
print("Received COUNT TOTAL")
[data] = data_recv[2:]
self.callback_count(data)
data_to_save = [self.callback_save(), self.actual]
self.state_saver.save_state(connection_id, message_id, data_to_save)
def reset_coordinator(self):
_tmp = self.coordinator_sender.send(json.dumps([self.connection_id, RESTART]))
print("Received: {}".format(_tmp))
class Protocol:
def __init__(self, recv_queue, send_queue, total_workers, status_queue,
data_cluster_write, data_cluster_read):
self.connection = Connection()
self.total_workers = total_workers
self.pending_connections = total_workers
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.state_saver = StateSaver(STAGE, data_cluster_write,
data_cluster_read)
self.connection_id = None
def start_connection(self, callback, callback_eof, callback_load,
callback_reset, callback_save):
self.callback = callback
self.callback_eof = callback_eof
self.callback_load = callback_load
self.callback_reset = callback_reset
self.callback_save = callback_save
self.receiver.start_receiving(self.data_read)
def data_read(self, msg_type, msg):
print("Msg received" + msg)
if msg_type == STOP:
self.receiver.close()
self.sender.send(STOP, '')
self.status_sender.send(FINISHED, FINISHED)
return
data_recv = json.loads(msg)
print("DATE SORTER: {}".format(msg))
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated(connection_id, message_id):
print("Duplicated message: {}".format(msg))
return
if connection_id != self.connection_id:
last_state = self.state_saver.load_state(connection_id)
if last_state is not None:
[old_data, pending_conns] = last_state
self.pending_connections = pending_conns
self.callback_load(old_data)
else:
self.pending_connections = self.total_workers
self.callback_reset()
self.connection_id = connection_id
if msg_type == EOF:
self.pending_connections -= 1
if self.pending_connections == 0:
self.callback_eof()
print("Ended processing")
else:
[data] = data_recv[2:]
self.callback(json.loads(data))
data_to_save = [self.callback_save(), self.pending_connections]
self.state_saver.save_state(connection_id, message_id, data_to_save)
def send_data(self, data):
print("Sending Dates results")
new_data = [self.connection_id, DATE_MSG_ID, json.dumps(data)]
eof_to_send = [self.connection_id, EOF_MSG_ID]
self.sender.send(DATE_RESULTS, json.dumps(new_data))
self.sender.send(EOF, json.dumps(eof_to_send))
class Protocol:
def __init__(self, recv_queue, send_queue, status_queue,
data_cluster_write, data_cluster_read):
self.connection = Connection()
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.state_saver = StateSaver(STAGE, data_cluster_write,
data_cluster_read)
self.connection_id = None
def start_connection(self, callback, callback_eof, callback_load,
callback_reset, callback_save):
self.callback = callback
self.callback_eof = callback_eof
self.callback_load = callback_load
self.callback_reset = callback_reset
self.callback_save = callback_save
self.receiver.start_receiving(self.data_read)
def send_data(self, data):
#Unique message so that if this fails, the next one that raises will
#send the same id
data_to_send = [self.connection_id, PLACE_MSG_ID, json.dumps(data)]
eof_to_send = [self.connection_id, EOF_MSG_ID]
self.sender.send(NORMAL, json.dumps(data_to_send))
self.sender.send(EOF, json.dumps(eof_to_send))
def data_read(self, msg_type, msg):
if msg_type == STOP:
self.receiver.close()
self.sender.send(STOP, STOP)
self.status_sender.send(FINISHED, FINISHED)
return
data_recv = json.loads(msg)
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated(connection_id, message_id):
print("Duplicated message: {}".format(msg))
return
if connection_id != self.connection_id:
old_data = self.state_saver.load_state(connection_id)
if old_data is not None:
self.callback_load(old_data)
else:
self.callback_reset()
self.connection_id = connection_id
if msg_type == EOF:
print("Ended processing")
self.callback_eof()
else:
[place] = data_recv[2:]
self.callback(place)
data_to_save = self.callback_save()
self.state_saver.save_state(connection_id, message_id, data_to_save)
def __init__(self, send_queue):
self.connection = Connection()
self.sender = SecureDirectSender(send_queue, self.connection)