class Protocol:
def __init__(self, recv_queue):
self.connection = Connection()
self.expected = 3
self.actual = 0
self.receiver = self.connection.create_direct_receiver(recv_queue)
def start_connection(self, callback_top, callback_date, callback_count):
self.callback_top = callback_top
self.callback_date = callback_date
self.callback_count = callback_count
self.receiver.start_receiving(self.data_read)
def data_read(self, msg_type, msg):
if msg_type == EOF:
self.actual += 1
if self.actual == self.expected:
self.connection.close()
elif msg_type == TOP_CITIES:
self.callback_top(json.loads(msg))
elif msg_type == DATE_RESULTS:
self.callback_date(json.loads(msg))
elif msg_type == TOTAL_COUNT:
self.callback_count(float(msg))
class Protocol:
def __init__(self, recv_queue, send_queues, master_send_queue):
self.connection = Connection()
self.receiver = self.connection.create_distributed_work_receiver(
recv_queue)
self.senders = {}
for queue in send_queues:
self.senders[queue] = self.connection.create_direct_sender(queue)
self.master_sender = self.connection.create_direct_sender(
master_send_queue)
def start_connection(self, callback, callback_eof):
self.callback = callback
self.callback_eof = callback_eof
self.receiver.start_receiving(self.data_read)
def send_data(self, data, where):
self.senders[where].send(NORMAL, data)
def send_master_ended(self):
self.master_sender.send(EOF, "")
def data_read(self, method, msg_type, msg):
if msg_type == EOF:
self.callback_eof()
self.receiver.send_ack(method)
self.receiver.close()
self.send_master_ended()
self.connection.close()
else:
self.callback(msg)
self.receiver.send_ack(method)
class Protocol:
def __init__(self, recv_queue, send_queue, total_workers, status_queue,
worker_id, data_cluster_write, data_cluster_read):
self.connection = Connection()
self.worker_id = worker_id
self.total_workers = total_workers
self.pending_connections = total_workers
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDirectSender(send_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.send_queue = send_queue
self.state_saver = StateSaver(self.worker_id, data_cluster_write,
data_cluster_read)
self.connection_id = None
def start_connection(self):
self.receiver.start_receiving(self.data_read)
self.connection.close()
def data_read(self, msg_type, msg):
print("Msg received: {}".format(msg))
if msg_type == STOP:
self.receiver.close()
self.sender.send(STOP, '')
self.status_sender.send(FINISHED, FINISHED)
return
data_recv = json.loads(msg)
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated(connection_id, message_id):
print("Duplicated message: {}".format(msg))
return
if connection_id != self.connection_id:
old_data = self.state_saver.load_state(connection_id)
if old_data is not None:
self.pending_connections = old_data
else:
self.pending_connections = self.total_workers
self.connection_id = connection_id
if msg_type == EOF:
self.pending_connections -= 1
if self.pending_connections == 0:
data_recv[1] = RESUME_MSG_ID
self.sender.send(EOF, json.dumps(data_recv))
print("Ended processing")
data_to_save = self.pending_connections
self.state_saver.save_state(connection_id, message_id, data_to_save)
class Protocol:
def __init__(self, recv_queue, send_queue, total_workers):
self.connection = Connection()
self.total_workers = total_workers
self.receiver = self.connection.create_direct_receiver(recv_queue)
self.sender = self.connection.create_distributed_work_sender(
send_queue)
def start_connection(self): #, callback):
#self.callback = callback
self.receiver.start_receiving(self.data_read)
#def send_data(self, data):
# self.sender.send(NORMAL, data)
def data_read(self, msg_type, msg):
self.receiver.close()
self.send_eof()
self.connection.close()
'''if msg_type == "EOF":
self.receiver.close()
self.send_eof()
self.connection.close()
else:
self.callback(msg)'''
def send_eof(self):
for i in range(0, self.total_workers):
self.sender.send(EOF, '')
class ProtocolInitialize:
def __init__(self, recv_queue, callback):
self.connection = Connection()
self.callback = callback
self.receiver = self.connection.create_topic_receiver(recv_queue)
def start_connection(self):
print("Starting to receive places")
self.receiver.start_receiving(self.data_read)
def data_read(self, msg_type, msg):
if msg_type == EOF:
self.connection.close()
else:
[region, latitude, longitude] = msg.split(",")
self.callback(region, float(latitude), float(longitude))
class Protocol:
def __init__(self, queue_map, queue_date, queue_count, eof_map, eof_date,
eof_count, topic_places):
self.connection = Connection()
self.sender_map = self.connection.create_distributed_work_sender(
queue_map)
self.sender_date = self.connection.create_distributed_work_sender(
queue_date)
self.sender_count = self.connection.create_distributed_work_sender(
queue_count)
self.sender_places = self.connection.create_topic_sender(topic_places)
self.eof_map = self.connection.create_direct_sender(eof_map)
self.eof_date = self.connection.create_direct_sender(eof_date)
self.eof_count = self.connection.create_direct_sender(eof_count)
def process_places(self, region, latitude, longitude):
message = region + "," + latitude + "," + longitude
self.sender_places.send(NORMAL, message)
def send_no_more_places(self):
self.sender_places.send(EOF, "")
def process(self, date, latitude, longitude, result):
message = date + "," + latitude + "," + longitude + "," + result
self.sender_map.send(NORMAL, message)
self.sender_date.send(NORMAL, message)
self.sender_count.send(NORMAL, message)
def close(self):
#self.sender_map.send(EOF, "")
#self.sender_date.send(EOF, "")
#self.sender_count.send(EOF, "")
self.eof_map.send(EOF, "")
self.eof_date.send(EOF, "")
self.eof_count.send(EOF, "")
self.connection.close()
class Protocol:
def __init__(self, recv_queue, send_queue, total_workers):
self.connection = Connection()
self.pending_connections = total_workers
self.receiver = self.connection.create_direct_receiver(recv_queue)
self.sender = self.connection.create_direct_sender(send_queue)
self.send_queue = send_queue
def start_connection(self):
self.receiver.start_receiving(self.data_read)
def data_read(self, msg_type, msg):
if msg_type == EOF:
self.pending_connections -= 1
if self.pending_connections == 0:
self.receiver.close()
self.sender.send(EOF, '')
self.connection.close()
class Protocol:
def __init__(self, recv_queue, send_queue, total_workers, status_queue):
self.connection = Connection()
print("Connected to RabbitMQ")
self.total_workers = total_workers
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.sender = SecureDistributedSender(send_queue, self.connection)
def start_connection(self):
self.receiver.start_receiving(self.data_read)
def data_read(self, msg_type, msg):
print("Got a message!")
if msg_type == STOP:
self.send_stop()
self.status_sender.send(FINISHED, FINISHED)
self.close()
elif msg_type == EOF:
self.send_eof(msg)
def send_eof(self, msg):
print("Sending EOF to workers: {}".format(msg))
[conn_id, msg_id, eof] = msg.split(',')
new_msg = [conn_id, msg_id, eof]
for i in range(0, self.total_workers):
self.sender.send(EOF, json.dumps(new_msg))
def send_stop(self):
for i in range(0, self.total_workers):
self.sender.send(STOP, '')
def close(self):
self.receiver.close()
self.connection.close()
class Protocol:
def __init__(
self,
recv_queue,
status_queue,
data_cluster_write,
data_cluster_read,
coordinator_queue
):
self.connection = Connection()
self.expected = EXPECTED_EOF
self.actual = 0
self.receiver = SecureDirectReceiver(recv_queue, self.connection)
self.status_sender = SecureDirectSender(status_queue, self.connection)
self.coordinator_sender = SecureRpcSender(coordinator_queue, Connection())
self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read)
self.connection_id = None
def start_connection(
self,
callback_top,
callback_date,
callback_count,
callback_all_data,
callback_load,
callback_reset,
callback_save
):
self.callback_top = callback_top
self.callback_date = callback_date
self.callback_count = callback_count
self.callback_all_data = callback_all_data
self.callback_load = callback_load
self.callback_reset = callback_reset
self.callback_save = callback_save
if self.actual < self.expected:
self.receiver.start_receiving(self.data_read)
self.connection.close()
def data_read(self, msg_type, msg):
print("Msg received" + msg)
if msg_type == STOP:
self.receiver.close()
self.status_sender.send(FINISHED, FINISHED)
return
data_recv = json.loads(msg)
[connection_id, message_id] = data_recv[:2]
if self.state_saver.is_duplicated(connection_id, message_id):
print("Duplicated message: {}".format(msg))
return
if connection_id != self.connection_id:
last_state = self.state_saver.load_state(connection_id)
if last_state is not None:
[old_data, actual_conns] = last_state
self.actual = actual_conns
self.callback_load(old_data)
else:
self.actual = 0
self.callback_reset()
self.connection_id = connection_id
if msg_type == EOF:
self.actual += 1
if self.actual == self.expected:
self.callback_all_data(self.connection_id)
self.reset_coordinator()
print("Ended processing")
elif msg_type == TOP_CITIES:
print("Received TOP CITIES")
[data] = data_recv[2:]
self.callback_top(json.loads(data))
elif msg_type == DATE_RESULTS:
print("Received DATE RESULTS")
[data] = data_recv[2:]
self.callback_date(json.loads(data))
elif msg_type == TOTAL_COUNT:
print("Received COUNT TOTAL")
[data] = data_recv[2:]
self.callback_count(data)
data_to_save = [self.callback_save(), self.actual]
self.state_saver.save_state(connection_id, message_id, data_to_save)
def reset_coordinator(self):
_tmp = self.coordinator_sender.send(json.dumps([self.connection_id, RESTART]))
print("Received: {}".format(_tmp))
