class Protocol: def __init__(self, recv_queue, send_queue, total_workers, status_queue, worker_id, data_cluster_write, data_cluster_read): self.connection = Connection() self.worker_id = worker_id self.total_workers = total_workers self.pending_connections = total_workers self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDirectSender(send_queue, self.connection) self.status_sender = SecureDirectSender(status_queue, self.connection) self.send_queue = send_queue self.state_saver = StateSaver(self.worker_id, data_cluster_write, data_cluster_read) self.connection_id = None def start_connection(self): self.receiver.start_receiving(self.data_read) self.connection.close() def data_read(self, msg_type, msg): print("Msg received: {}".format(msg)) if msg_type == STOP: self.receiver.close() self.sender.send(STOP, '') self.status_sender.send(FINISHED, FINISHED) return data_recv = json.loads(msg) [connection_id, message_id] = data_recv[:2] if self.state_saver.is_duplicated(connection_id, message_id): print("Duplicated message: {}".format(msg)) return if connection_id != self.connection_id: old_data = self.state_saver.load_state(connection_id) if old_data is not None: self.pending_connections = old_data else: self.pending_connections = self.total_workers self.connection_id = connection_id if msg_type == EOF: self.pending_connections -= 1 if self.pending_connections == 0: data_recv[1] = RESUME_MSG_ID self.sender.send(EOF, json.dumps(data_recv)) print("Ended processing") data_to_save = self.pending_connections self.state_saver.save_state(connection_id, message_id, data_to_save)
def __init__(self, recv_queue, state_saver): self.connection = Connection() self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.state_saver = state_saver self.connection_id = None
def __init__(self, recv_queue, send_queue, total_workers, status_queue): self.connection = Connection() print("Connected to RabbitMQ") self.total_workers = total_workers self.status_sender = SecureDirectSender(status_queue, self.connection) self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDistributedSender(send_queue, self.connection)
def __init__(self, recv_queue, send_queue, status_queue, data_cluster_write, data_cluster_read): self.connection = Connection() self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDirectSender(send_queue, self.connection) self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read) self.connection_id = None
class ReceiverProtocol: def __init__(self, recv_queue, state_saver): self.connection = Connection() self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.state_saver = state_saver self.connection_id = None def start_connection(self, callback, callback_load, callback_reset, callback_save): print("Starting to receive places") self.callback = callback self.callback_load = callback_load self.callback_reset = callback_reset self.callback_save = callback_save self.receiver.start_receiving(self.data_read) return self.connection_id def data_read(self, msg_type, msg): print("Msg received" + msg) if msg_type == NORMAL: [connection_id, message_id, region, latitude, longitude] = msg.split(",") else: [connection_id, message_id, eof] = msg.split(",") if self.state_saver.is_duplicated(connection_id, message_id): print("Duplicated message: {}".format(msg)) return if connection_id != self.connection_id: last_state = self.state_saver.load_state(connection_id) if last_state is not None: self.callback_load(json.loads(last_state)) else: self.callback_reset() self.connection_id = connection_id if msg_type == EOF: self.receiver.close() self.state_saver.save_state( "STATE", "", json.dumps([self.connection_id, "REQUESTER"])) else: self.callback(connection_id, region, latitude, longitude) data_to_save = self.callback_save() self.state_saver.save_state(connection_id, message_id, data_to_save)
class Protocol: def __init__(self, recv_queue): self.connection = Connection() self.receiver = SecureDirectReceiver(recv_queue, self.connection) def start_receiving(self, callback): self.callback = callback self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): [status, worker_id, worker_type] = msg.split(',') self.callback(status, worker_id, worker_type)
def __init__(self, recv_queue, send_queue, total_workers, status_queue, worker_id, data_cluster_write, data_cluster_read): self.connection = Connection() self.worker_id = worker_id self.total_workers = total_workers self.pending_connections = total_workers self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDirectSender(send_queue, self.connection) self.status_sender = SecureDirectSender(status_queue, self.connection) self.send_queue = send_queue self.state_saver = StateSaver(self.worker_id, data_cluster_write, data_cluster_read) self.connection_id = None
def __init__( self, recv_queue, status_queue, data_cluster_write, data_cluster_read, coordinator_queue ): self.connection = Connection() self.expected = EXPECTED_EOF self.actual = 0 self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.status_sender = SecureDirectSender(status_queue, self.connection) self.coordinator_sender = SecureRpcSender(coordinator_queue, Connection()) self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read) self.connection_id = None
class Protocol: def __init__(self, recv_queue, send_queue, total_workers, status_queue): self.connection = Connection() print("Connected to RabbitMQ") self.total_workers = total_workers self.status_sender = SecureDirectSender(status_queue, self.connection) self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDistributedSender(send_queue, self.connection) def start_connection(self): self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): print("Got a message!") if msg_type == STOP: self.send_stop() self.status_sender.send(FINISHED, FINISHED) self.close() elif msg_type == EOF: self.send_eof(msg) def send_eof(self, msg): print("Sending EOF to workers: {}".format(msg)) [conn_id, msg_id, eof] = msg.split(',') new_msg = [conn_id, msg_id, eof] for i in range(0, self.total_workers): self.sender.send(EOF, json.dumps(new_msg)) def send_stop(self): for i in range(0, self.total_workers): self.sender.send(STOP, '') def close(self): self.receiver.close() self.connection.close()
class Coordinator: def __init__(self, recv_queue, data_cluster_write, data_cluster_read): self.connection = Connection() self.state_saver = StateSaver("coordinator_" + recv_queue, data_cluster_write, data_cluster_read) self.recv_queue = recv_queue def wait_to_work(self): self.receiver = SecureDirectReceiver(self.recv_queue, self.connection) self.receiver.start_receiving(self.receive_to_work) def receive_to_work(self, msg_type, conn_id): #Check duplicates if self.state_saver.is_duplicated(conn_id, msg_type): print("Duplicated message: {} {}".format(conn_id, msg_type)) return print("Receive to start working") if msg_type == RESTART: self.receiver.close() self.state_saver.save_state(conn_id, msg_type, ".")
class Protocol: def __init__(self, recv_queue, send_queue, status_queue, data_cluster_write, data_cluster_read): self.connection = Connection() self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDirectSender(send_queue, self.connection) self.status_sender = SecureDirectSender(status_queue, self.connection) self.connection_id = None self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read) def start_connection(self, callback, callback_eof, callback_load, callback_reset, callback_save): self.callback = callback self.callback_eof = callback_eof self.callback_load = callback_load self.callback_reset = callback_reset self.callback_save = callback_save self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): if msg_type == STOP: self.receiver.close() self.sender.send(STOP, '') self.status_sender.send(FINISHED, FINISHED) return data_recv = json.loads(msg) [connection_id, message_id] = data_recv[:2] if self.state_saver.is_duplicated(connection_id, message_id): print("Duplicated message: {}".format(msg)) return if connection_id != self.connection_id: old_data = self.state_saver.load_state(connection_id) if old_data is not None: self.callback_load(old_data) else: self.callback_reset() self.connection_id = connection_id if msg_type == EOF: print("Ended processing") self.callback_eof() else: [positivi, deceduti] = data_recv[2:] self.callback(positivi, deceduti) data_to_save = self.callback_save() self.state_saver.save_state(connection_id, message_id, data_to_save) def send_data(self, data): data_to_send = [self.connection_id, COUNT_MSG_ID, data] eof_to_send = [self.connection_id, EOF_MSG_ID] self.sender.send(TOTAL_COUNT, json.dumps(data_to_send)) self.sender.send(EOF, json.dumps(eof_to_send))
class Protocol: def __init__( self, recv_queue, status_queue, data_cluster_write, data_cluster_read, coordinator_queue ): self.connection = Connection() self.expected = EXPECTED_EOF self.actual = 0 self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.status_sender = SecureDirectSender(status_queue, self.connection) self.coordinator_sender = SecureRpcSender(coordinator_queue, Connection()) self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read) self.connection_id = None def start_connection( self, callback_top, callback_date, callback_count, callback_all_data, callback_load, callback_reset, callback_save ): self.callback_top = callback_top self.callback_date = callback_date self.callback_count = callback_count self.callback_all_data = callback_all_data self.callback_load = callback_load self.callback_reset = callback_reset self.callback_save = callback_save if self.actual < self.expected: self.receiver.start_receiving(self.data_read) self.connection.close() def data_read(self, msg_type, msg): print("Msg received" + msg) if msg_type == STOP: self.receiver.close() self.status_sender.send(FINISHED, FINISHED) return data_recv = json.loads(msg) [connection_id, message_id] = data_recv[:2] if self.state_saver.is_duplicated(connection_id, message_id): print("Duplicated message: {}".format(msg)) return if connection_id != self.connection_id: last_state = self.state_saver.load_state(connection_id) if last_state is not None: [old_data, actual_conns] = last_state self.actual = actual_conns self.callback_load(old_data) else: self.actual = 0 self.callback_reset() self.connection_id = connection_id if msg_type == EOF: self.actual += 1 if self.actual == self.expected: self.callback_all_data(self.connection_id) self.reset_coordinator() print("Ended processing") elif msg_type == TOP_CITIES: print("Received TOP CITIES") [data] = data_recv[2:] self.callback_top(json.loads(data)) elif msg_type == DATE_RESULTS: print("Received DATE RESULTS") [data] = data_recv[2:] self.callback_date(json.loads(data)) elif msg_type == TOTAL_COUNT: print("Received COUNT TOTAL") [data] = data_recv[2:] self.callback_count(data) data_to_save = [self.callback_save(), self.actual] self.state_saver.save_state(connection_id, message_id, data_to_save) def reset_coordinator(self): _tmp = self.coordinator_sender.send(json.dumps([self.connection_id, RESTART])) print("Received: {}".format(_tmp))
class Protocol: def __init__(self, recv_queue, send_queue, total_workers, status_queue, data_cluster_write, data_cluster_read): self.connection = Connection() self.total_workers = total_workers self.pending_connections = total_workers self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDirectSender(send_queue, self.connection) self.status_sender = SecureDirectSender(status_queue, self.connection) self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read) self.connection_id = None def start_connection(self, callback, callback_eof, callback_load, callback_reset, callback_save): self.callback = callback self.callback_eof = callback_eof self.callback_load = callback_load self.callback_reset = callback_reset self.callback_save = callback_save self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): print("Msg received" + msg) if msg_type == STOP: self.receiver.close() self.sender.send(STOP, '') self.status_sender.send(FINISHED, FINISHED) return data_recv = json.loads(msg) print("DATE SORTER: {}".format(msg)) [connection_id, message_id] = data_recv[:2] if self.state_saver.is_duplicated(connection_id, message_id): print("Duplicated message: {}".format(msg)) return if connection_id != self.connection_id: last_state = self.state_saver.load_state(connection_id) if last_state is not None: [old_data, pending_conns] = last_state self.pending_connections = pending_conns self.callback_load(old_data) else: self.pending_connections = self.total_workers self.callback_reset() self.connection_id = connection_id if msg_type == EOF: self.pending_connections -= 1 if self.pending_connections == 0: self.callback_eof() print("Ended processing") else: [data] = data_recv[2:] self.callback(json.loads(data)) data_to_save = [self.callback_save(), self.pending_connections] self.state_saver.save_state(connection_id, message_id, data_to_save) def send_data(self, data): print("Sending Dates results") new_data = [self.connection_id, DATE_MSG_ID, json.dumps(data)] eof_to_send = [self.connection_id, EOF_MSG_ID] self.sender.send(DATE_RESULTS, json.dumps(new_data)) self.sender.send(EOF, json.dumps(eof_to_send))
class Protocol: def __init__(self, recv_queue, send_queue, status_queue, data_cluster_write, data_cluster_read): self.connection = Connection() self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDirectSender(send_queue, self.connection) self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read) self.connection_id = None def start_connection(self, callback, callback_eof, callback_load, callback_reset, callback_save): self.callback = callback self.callback_eof = callback_eof self.callback_load = callback_load self.callback_reset = callback_reset self.callback_save = callback_save self.receiver.start_receiving(self.data_read) def send_data(self, data): #Unique message so that if this fails, the next one that raises will #send the same id data_to_send = [self.connection_id, PLACE_MSG_ID, json.dumps(data)] eof_to_send = [self.connection_id, EOF_MSG_ID] self.sender.send(NORMAL, json.dumps(data_to_send)) self.sender.send(EOF, json.dumps(eof_to_send)) def data_read(self, msg_type, msg): if msg_type == STOP: self.receiver.close() self.sender.send(STOP, STOP) self.status_sender.send(FINISHED, FINISHED) return data_recv = json.loads(msg) [connection_id, message_id] = data_recv[:2] if self.state_saver.is_duplicated(connection_id, message_id): print("Duplicated message: {}".format(msg)) return if connection_id != self.connection_id: old_data = self.state_saver.load_state(connection_id) if old_data is not None: self.callback_load(old_data) else: self.callback_reset() self.connection_id = connection_id if msg_type == EOF: print("Ended processing") self.callback_eof() else: [place] = data_recv[2:] self.callback(place) data_to_save = self.callback_save() self.state_saver.save_state(connection_id, message_id, data_to_save)
def wait_to_work(self): self.receiver = SecureDirectReceiver(self.recv_queue, self.connection) self.receiver.start_receiving(self.receive_to_work)
def __init__(self, recv_queue): self.connection = Connection() self.receiver = SecureDirectReceiver(recv_queue, self.connection)