class Protocol: def __init__(self, recv_queue): self.connection = Connection() self.expected = 3 self.actual = 0 self.receiver = self.connection.create_direct_receiver(recv_queue) def start_connection(self, callback_top, callback_date, callback_count): self.callback_top = callback_top self.callback_date = callback_date self.callback_count = callback_count self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): if msg_type == EOF: self.actual += 1 if self.actual == self.expected: self.connection.close() elif msg_type == TOP_CITIES: self.callback_top(json.loads(msg)) elif msg_type == DATE_RESULTS: self.callback_date(json.loads(msg)) elif msg_type == TOTAL_COUNT: self.callback_count(float(msg))
class Protocol: def __init__(self, recv_queue, send_queues, master_send_queue): self.connection = Connection() self.receiver = self.connection.create_distributed_work_receiver( recv_queue) self.senders = {} for queue in send_queues: self.senders[queue] = self.connection.create_direct_sender(queue) self.master_sender = self.connection.create_direct_sender( master_send_queue) def start_connection(self, callback, callback_eof): self.callback = callback self.callback_eof = callback_eof self.receiver.start_receiving(self.data_read) def send_data(self, data, where): self.senders[where].send(NORMAL, data) def send_master_ended(self): self.master_sender.send(EOF, "") def data_read(self, method, msg_type, msg): if msg_type == EOF: self.callback_eof() self.receiver.send_ack(method) self.receiver.close() self.send_master_ended() self.connection.close() else: self.callback(msg) self.receiver.send_ack(method)
class Protocol: def __init__(self, recv_queue, send_queue, total_workers, status_queue, worker_id, data_cluster_write, data_cluster_read): self.connection = Connection() self.worker_id = worker_id self.total_workers = total_workers self.pending_connections = total_workers self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDirectSender(send_queue, self.connection) self.status_sender = SecureDirectSender(status_queue, self.connection) self.send_queue = send_queue self.state_saver = StateSaver(self.worker_id, data_cluster_write, data_cluster_read) self.connection_id = None def start_connection(self): self.receiver.start_receiving(self.data_read) self.connection.close() def data_read(self, msg_type, msg): print("Msg received: {}".format(msg)) if msg_type == STOP: self.receiver.close() self.sender.send(STOP, '') self.status_sender.send(FINISHED, FINISHED) return data_recv = json.loads(msg) [connection_id, message_id] = data_recv[:2] if self.state_saver.is_duplicated(connection_id, message_id): print("Duplicated message: {}".format(msg)) return if connection_id != self.connection_id: old_data = self.state_saver.load_state(connection_id) if old_data is not None: self.pending_connections = old_data else: self.pending_connections = self.total_workers self.connection_id = connection_id if msg_type == EOF: self.pending_connections -= 1 if self.pending_connections == 0: data_recv[1] = RESUME_MSG_ID self.sender.send(EOF, json.dumps(data_recv)) print("Ended processing") data_to_save = self.pending_connections self.state_saver.save_state(connection_id, message_id, data_to_save)
class Protocol: def __init__(self, recv_queue, send_queue, total_workers): self.connection = Connection() self.total_workers = total_workers self.receiver = self.connection.create_direct_receiver(recv_queue) self.sender = self.connection.create_distributed_work_sender( send_queue) def start_connection(self): #, callback): #self.callback = callback self.receiver.start_receiving(self.data_read) #def send_data(self, data): # self.sender.send(NORMAL, data) def data_read(self, msg_type, msg): self.receiver.close() self.send_eof() self.connection.close() '''if msg_type == "EOF": self.receiver.close() self.send_eof() self.connection.close() else: self.callback(msg)''' def send_eof(self): for i in range(0, self.total_workers): self.sender.send(EOF, '')
class ProtocolInitialize: def __init__(self, recv_queue, callback): self.connection = Connection() self.callback = callback self.receiver = self.connection.create_topic_receiver(recv_queue) def start_connection(self): print("Starting to receive places") self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): if msg_type == EOF: self.connection.close() else: [region, latitude, longitude] = msg.split(",") self.callback(region, float(latitude), float(longitude))
class Protocol: def __init__(self, queue_map, queue_date, queue_count, eof_map, eof_date, eof_count, topic_places): self.connection = Connection() self.sender_map = self.connection.create_distributed_work_sender( queue_map) self.sender_date = self.connection.create_distributed_work_sender( queue_date) self.sender_count = self.connection.create_distributed_work_sender( queue_count) self.sender_places = self.connection.create_topic_sender(topic_places) self.eof_map = self.connection.create_direct_sender(eof_map) self.eof_date = self.connection.create_direct_sender(eof_date) self.eof_count = self.connection.create_direct_sender(eof_count) def process_places(self, region, latitude, longitude): message = region + "," + latitude + "," + longitude self.sender_places.send(NORMAL, message) def send_no_more_places(self): self.sender_places.send(EOF, "") def process(self, date, latitude, longitude, result): message = date + "," + latitude + "," + longitude + "," + result self.sender_map.send(NORMAL, message) self.sender_date.send(NORMAL, message) self.sender_count.send(NORMAL, message) def close(self): #self.sender_map.send(EOF, "") #self.sender_date.send(EOF, "") #self.sender_count.send(EOF, "") self.eof_map.send(EOF, "") self.eof_date.send(EOF, "") self.eof_count.send(EOF, "") self.connection.close()
class Protocol: def __init__(self, recv_queue, send_queue, total_workers): self.connection = Connection() self.pending_connections = total_workers self.receiver = self.connection.create_direct_receiver(recv_queue) self.sender = self.connection.create_direct_sender(send_queue) self.send_queue = send_queue def start_connection(self): self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): if msg_type == EOF: self.pending_connections -= 1 if self.pending_connections == 0: self.receiver.close() self.sender.send(EOF, '') self.connection.close()
class Protocol: def __init__(self, recv_queue, send_queue, total_workers, status_queue): self.connection = Connection() print("Connected to RabbitMQ") self.total_workers = total_workers self.status_sender = SecureDirectSender(status_queue, self.connection) self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDistributedSender(send_queue, self.connection) def start_connection(self): self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): print("Got a message!") if msg_type == STOP: self.send_stop() self.status_sender.send(FINISHED, FINISHED) self.close() elif msg_type == EOF: self.send_eof(msg) def send_eof(self, msg): print("Sending EOF to workers: {}".format(msg)) [conn_id, msg_id, eof] = msg.split(',') new_msg = [conn_id, msg_id, eof] for i in range(0, self.total_workers): self.sender.send(EOF, json.dumps(new_msg)) def send_stop(self): for i in range(0, self.total_workers): self.sender.send(STOP, '') def close(self): self.receiver.close() self.connection.close()
class Protocol: def __init__( self, recv_queue, status_queue, data_cluster_write, data_cluster_read, coordinator_queue ): self.connection = Connection() self.expected = EXPECTED_EOF self.actual = 0 self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.status_sender = SecureDirectSender(status_queue, self.connection) self.coordinator_sender = SecureRpcSender(coordinator_queue, Connection()) self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read) self.connection_id = None def start_connection( self, callback_top, callback_date, callback_count, callback_all_data, callback_load, callback_reset, callback_save ): self.callback_top = callback_top self.callback_date = callback_date self.callback_count = callback_count self.callback_all_data = callback_all_data self.callback_load = callback_load self.callback_reset = callback_reset self.callback_save = callback_save if self.actual < self.expected: self.receiver.start_receiving(self.data_read) self.connection.close() def data_read(self, msg_type, msg): print("Msg received" + msg) if msg_type == STOP: self.receiver.close() self.status_sender.send(FINISHED, FINISHED) return data_recv = json.loads(msg) [connection_id, message_id] = data_recv[:2] if self.state_saver.is_duplicated(connection_id, message_id): print("Duplicated message: {}".format(msg)) return if connection_id != self.connection_id: last_state = self.state_saver.load_state(connection_id) if last_state is not None: [old_data, actual_conns] = last_state self.actual = actual_conns self.callback_load(old_data) else: self.actual = 0 self.callback_reset() self.connection_id = connection_id if msg_type == EOF: self.actual += 1 if self.actual == self.expected: self.callback_all_data(self.connection_id) self.reset_coordinator() print("Ended processing") elif msg_type == TOP_CITIES: print("Received TOP CITIES") [data] = data_recv[2:] self.callback_top(json.loads(data)) elif msg_type == DATE_RESULTS: print("Received DATE RESULTS") [data] = data_recv[2:] self.callback_date(json.loads(data)) elif msg_type == TOTAL_COUNT: print("Received COUNT TOTAL") [data] = data_recv[2:] self.callback_count(data) data_to_save = [self.callback_save(), self.actual] self.state_saver.save_state(connection_id, message_id, data_to_save) def reset_coordinator(self): _tmp = self.coordinator_sender.send(json.dumps([self.connection_id, RESTART])) print("Received: {}".format(_tmp))