class Protocol: def __init__(self, recv_queue, send_queues, master_send_queue): self.connection = Connection() self.receiver = self.connection.create_distributed_work_receiver( recv_queue) self.senders = {} for queue in send_queues: self.senders[queue] = self.connection.create_direct_sender(queue) self.master_sender = self.connection.create_direct_sender( master_send_queue) def start_connection(self, callback, callback_eof): self.callback = callback self.callback_eof = callback_eof self.receiver.start_receiving(self.data_read) def send_data(self, data, where): self.senders[where].send(NORMAL, data) def send_master_ended(self): self.master_sender.send(EOF, "") def data_read(self, method, msg_type, msg): if msg_type == EOF: self.callback_eof() self.receiver.send_ack(method) self.receiver.close() self.send_master_ended() self.connection.close() else: self.callback(msg) self.receiver.send_ack(method)
class Protocol: def __init__(self, recv_queue, send_queue, total_workers): self.connection = Connection() self.pending_connections = total_workers self.receiver = self.connection.create_direct_receiver(recv_queue) self.sender = self.connection.create_direct_sender(send_queue) def start_connection(self, callback): self.callback = callback self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): if msg_type == EOF: #self.receiver.close() self.pending_connections -= 1 if self.pending_connections == 0: self.receiver.close() else: self.callback(json.loads(msg)) def send_data(self, top_cities): self.sender.send(TOP_CITIES, json.dumps(top_cities)) self.sender.send(EOF, '')
def __init__(self, recv_queue, send_queue, total_workers): self.connection = Connection() self.pending_connections = total_workers self.receiver = self.connection.create_direct_receiver(recv_queue) self.sender = self.connection.create_direct_sender(send_queue)
class Protocol: def __init__(self, recv_queue): self.connection = Connection() self.expected = 3 self.actual = 0 self.receiver = self.connection.create_direct_receiver(recv_queue) def start_connection(self, callback_top, callback_date, callback_count): self.callback_top = callback_top self.callback_date = callback_date self.callback_count = callback_count self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): if msg_type == EOF: self.actual += 1 if self.actual == self.expected: self.connection.close() elif msg_type == TOP_CITIES: self.callback_top(json.loads(msg)) elif msg_type == DATE_RESULTS: self.callback_date(json.loads(msg)) elif msg_type == TOTAL_COUNT: self.callback_count(float(msg))
class Protocol: def __init__(self, recv_queue, send_queue, total_workers, status_queue, worker_id, data_cluster_write, data_cluster_read): self.connection = Connection() self.worker_id = worker_id self.total_workers = total_workers self.pending_connections = total_workers self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDirectSender(send_queue, self.connection) self.status_sender = SecureDirectSender(status_queue, self.connection) self.send_queue = send_queue self.state_saver = StateSaver(self.worker_id, data_cluster_write, data_cluster_read) self.connection_id = None def start_connection(self): self.receiver.start_receiving(self.data_read) self.connection.close() def data_read(self, msg_type, msg): print("Msg received: {}".format(msg)) if msg_type == STOP: self.receiver.close() self.sender.send(STOP, '') self.status_sender.send(FINISHED, FINISHED) return data_recv = json.loads(msg) [connection_id, message_id] = data_recv[:2] if self.state_saver.is_duplicated(connection_id, message_id): print("Duplicated message: {}".format(msg)) return if connection_id != self.connection_id: old_data = self.state_saver.load_state(connection_id) if old_data is not None: self.pending_connections = old_data else: self.pending_connections = self.total_workers self.connection_id = connection_id if msg_type == EOF: self.pending_connections -= 1 if self.pending_connections == 0: data_recv[1] = RESUME_MSG_ID self.sender.send(EOF, json.dumps(data_recv)) print("Ended processing") data_to_save = self.pending_connections self.state_saver.save_state(connection_id, message_id, data_to_save)
def __init__(self, recv_queue): self.connection = Connection() self.expected = 3 self.actual = 0 self.receiver = self.connection.create_direct_receiver(recv_queue)
class Protocol: def __init__(self, recv_queue, send_queue, total_workers): self.connection = Connection() self.total_workers = total_workers self.receiver = self.connection.create_direct_receiver(recv_queue) self.sender = self.connection.create_distributed_work_sender( send_queue) def start_connection(self): #, callback): #self.callback = callback self.receiver.start_receiving(self.data_read) #def send_data(self, data): # self.sender.send(NORMAL, data) def data_read(self, msg_type, msg): self.receiver.close() self.send_eof() self.connection.close() '''if msg_type == "EOF": self.receiver.close() self.send_eof() self.connection.close() else: self.callback(msg)''' def send_eof(self): for i in range(0, self.total_workers): self.sender.send(EOF, '')
def __init__(self, recv_queue, send_queue, total_workers): self.connection = Connection() self.total_workers = total_workers self.receiver = self.connection.create_direct_receiver(recv_queue) self.sender = self.connection.create_distributed_work_sender( send_queue)
def __init__(self, recv_queue, send_queue, total_workers, status_queue): self.connection = Connection() print("Connected to RabbitMQ") self.total_workers = total_workers self.status_sender = SecureDirectSender(status_queue, self.connection) self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDistributedSender(send_queue, self.connection)
def __init__(self, recv_queue, send_queues): self.connection = Connection() self.receiver = SecureRpcReceiver(recv_queue, self.connection) self.senders = [] for queue in send_queues: print("Linking senders to: {}".format(queue)) self.senders.append(SecureRpcSender(queue, Connection()))
def __init__(self, recv_queue, send_queues, master_send_queue): self.connection = Connection() self.receiver = self.connection.create_distributed_work_receiver( recv_queue) self.senders = {} for queue in send_queues: self.senders[queue] = self.connection.create_direct_sender(queue) self.master_sender = self.connection.create_direct_sender( master_send_queue)
def __init__(self, queue_map, queue_date, queue_count, eof_map, eof_date, eof_count, topic_places): self.connection = Connection() self.sender_map = self.connection.create_distributed_work_sender( queue_map) self.sender_date = self.connection.create_distributed_work_sender( queue_date) self.sender_count = self.connection.create_distributed_work_sender( queue_count) self.sender_places = self.connection.create_topic_sender(topic_places) self.eof_map = self.connection.create_direct_sender(eof_map) self.eof_date = self.connection.create_direct_sender(eof_date) self.eof_count = self.connection.create_direct_sender(eof_count)
def __init__(self, recv_queue, state_saver): self.connection = Connection() self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.state_saver = state_saver self.connection_id = None
def __init__(self, recv_queue, send_queue, total_workers, status_queue, worker_id, data_cluster_write, data_cluster_read): self.connection = Connection() self.worker_id = worker_id self.total_workers = total_workers self.pending_connections = total_workers self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDirectSender(send_queue, self.connection) self.status_sender = SecureDirectSender(status_queue, self.connection) self.send_queue = send_queue self.state_saver = StateSaver(self.worker_id, data_cluster_write, data_cluster_read) self.connection_id = None
class ProtocolInitialize: def __init__(self, recv_queue, callback): self.connection = Connection() self.callback = callback self.receiver = self.connection.create_topic_receiver(recv_queue) def start_connection(self): print("Starting to receive places") self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): if msg_type == EOF: self.connection.close() else: [region, latitude, longitude] = msg.split(",") self.callback(region, float(latitude), float(longitude))
def __init__(self, recv_queue, send_queue, status_queue, data_cluster_write, data_cluster_read): self.connection = Connection() self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDirectSender(send_queue, self.connection) self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read) self.connection_id = None
class Protocol: def __init__(self, recv_queue, send_queue): self.connection = Connection() self.receiver = self.connection.create_direct_receiver(recv_queue) self.sender = self.connection.create_direct_sender(send_queue) def start_connection(self, callback): self.callback = callback self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): if msg_type == EOF: self.receiver.close() else: [positivi, deceduti] = msg.split(',') self.callback(int(positivi), int(deceduti)) def send_data(self, percentage): self.sender.send(TOTAL_COUNT, str(percentage)) self.sender.send(EOF, '')
def __init__( self, recv_queue, status_queue, data_cluster_write, data_cluster_read, coordinator_queue ): self.connection = Connection() self.expected = EXPECTED_EOF self.actual = 0 self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.status_sender = SecureDirectSender(status_queue, self.connection) self.coordinator_sender = SecureRpcSender(coordinator_queue, Connection()) self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read) self.connection_id = None
class Protocol: def __init__(self, recv_queue, send_queue, total_workers): self.connection = Connection() self.pending_connections = total_workers self.receiver = self.connection.create_direct_receiver(recv_queue) self.sender = self.connection.create_direct_sender(send_queue) self.send_queue = send_queue def start_connection(self): self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): if msg_type == EOF: self.pending_connections -= 1 if self.pending_connections == 0: self.receiver.close() self.sender.send(EOF, '') self.connection.close()
class Protocol: def __init__(self, recv_queue, send_queue, master_queue): self.connection = Connection() self.receiver = self.connection.create_direct_receiver(recv_queue) self.sender = self.connection.create_direct_sender(send_queue) self.master_sender = self.connection.create_direct_sender(master_queue) def start_connection(self, callback): self.callback = callback self.receiver.start_receiving(self.data_read) def send_data(self, data): self.sender.send(NORMAL, json.dumps(data)) self.sender.send(EOF, "") #self.master_sender.send(EOF, "") def data_read(self, msg_type, msg): if msg_type == EOF: self.receiver.close() else: self.callback(msg)
class Protocol: def __init__(self, recv_queue, send_queue, total_workers, status_queue): self.connection = Connection() print("Connected to RabbitMQ") self.total_workers = total_workers self.status_sender = SecureDirectSender(status_queue, self.connection) self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.sender = SecureDistributedSender(send_queue, self.connection) def start_connection(self): self.receiver.start_receiving(self.data_read) def data_read(self, msg_type, msg): print("Got a message!") if msg_type == STOP: self.send_stop() self.status_sender.send(FINISHED, FINISHED) self.close() elif msg_type == EOF: self.send_eof(msg) def send_eof(self, msg): print("Sending EOF to workers: {}".format(msg)) [conn_id, msg_id, eof] = msg.split(',') new_msg = [conn_id, msg_id, eof] for i in range(0, self.total_workers): self.sender.send(EOF, json.dumps(new_msg)) def send_stop(self): for i in range(0, self.total_workers): self.sender.send(STOP, '') def close(self): self.receiver.close() self.connection.close()
class Protocol: def __init__(self, queue_map, queue_date, queue_count, eof_map, eof_date, eof_count, topic_places): self.connection = Connection() self.sender_map = self.connection.create_distributed_work_sender( queue_map) self.sender_date = self.connection.create_distributed_work_sender( queue_date) self.sender_count = self.connection.create_distributed_work_sender( queue_count) self.sender_places = self.connection.create_topic_sender(topic_places) self.eof_map = self.connection.create_direct_sender(eof_map) self.eof_date = self.connection.create_direct_sender(eof_date) self.eof_count = self.connection.create_direct_sender(eof_count) def process_places(self, region, latitude, longitude): message = region + "," + latitude + "," + longitude self.sender_places.send(NORMAL, message) def send_no_more_places(self): self.sender_places.send(EOF, "") def process(self, date, latitude, longitude, result): message = date + "," + latitude + "," + longitude + "," + result self.sender_map.send(NORMAL, message) self.sender_date.send(NORMAL, message) self.sender_count.send(NORMAL, message) def close(self): #self.sender_map.send(EOF, "") #self.sender_date.send(EOF, "") #self.sender_count.send(EOF, "") self.eof_map.send(EOF, "") self.eof_date.send(EOF, "") self.eof_count.send(EOF, "") self.connection.close()
def __init__(self, recv_queue, send_queues, master_send_queue, status_queue, state_saver, my_id): self.connection = Connection() self.status_sender = SecureDirectSender(status_queue, self.connection) self.senders = {} self.recv_queue = recv_queue for queue in send_queues: self.senders[queue] = SecureDirectSender(queue, self.connection) self.master_sender = SecureDirectSender(master_send_queue, self.connection) self.state_saver = state_saver self.my_id = my_id print("Creating RabbitMQ Queue: {}".format(self.recv_queue)) self.receiver = SecureDistributedReceiver(self.recv_queue, self.connection)
class Protocol: def __init__( self, recv_queue, status_queue, data_cluster_write, data_cluster_read, coordinator_queue ): self.connection = Connection() self.expected = EXPECTED_EOF self.actual = 0 self.receiver = SecureDirectReceiver(recv_queue, self.connection) self.status_sender = SecureDirectSender(status_queue, self.connection) self.coordinator_sender = SecureRpcSender(coordinator_queue, Connection()) self.state_saver = StateSaver(STAGE, data_cluster_write, data_cluster_read) self.connection_id = None def start_connection( self, callback_top, callback_date, callback_count, callback_all_data, callback_load, callback_reset, callback_save ): self.callback_top = callback_top self.callback_date = callback_date self.callback_count = callback_count self.callback_all_data = callback_all_data self.callback_load = callback_load self.callback_reset = callback_reset self.callback_save = callback_save if self.actual < self.expected: self.receiver.start_receiving(self.data_read) self.connection.close() def data_read(self, msg_type, msg): print("Msg received" + msg) if msg_type == STOP: self.receiver.close() self.status_sender.send(FINISHED, FINISHED) return data_recv = json.loads(msg) [connection_id, message_id] = data_recv[:2] if self.state_saver.is_duplicated(connection_id, message_id): print("Duplicated message: {}".format(msg)) return if connection_id != self.connection_id: last_state = self.state_saver.load_state(connection_id) if last_state is not None: [old_data, actual_conns] = last_state self.actual = actual_conns self.callback_load(old_data) else: self.actual = 0 self.callback_reset() self.connection_id = connection_id if msg_type == EOF: self.actual += 1 if self.actual == self.expected: self.callback_all_data(self.connection_id) self.reset_coordinator() print("Ended processing") elif msg_type == TOP_CITIES: print("Received TOP CITIES") [data] = data_recv[2:] self.callback_top(json.loads(data)) elif msg_type == DATE_RESULTS: print("Received DATE RESULTS") [data] = data_recv[2:] self.callback_date(json.loads(data)) elif msg_type == TOTAL_COUNT: print("Received COUNT TOTAL") [data] = data_recv[2:] self.callback_count(data) data_to_save = [self.callback_save(), self.actual] self.state_saver.save_state(connection_id, message_id, data_to_save) def reset_coordinator(self): _tmp = self.coordinator_sender.send(json.dumps([self.connection_id, RESTART])) print("Received: {}".format(_tmp))
def __init__(self, recv_queue): self.connection = Connection() self.receiver = SecureRpcReceiver(recv_queue, self.connection)
def __init__(self, recv_queue, callback): self.connection = Connection() self.callback = callback self.receiver = self.connection.create_topic_receiver(recv_queue)
def __init__(self, recv_queue, data_cluster_write, data_cluster_read): self.connection = Connection() self.state_saver = StateSaver("coordinator_" + recv_queue, data_cluster_write, data_cluster_read) self.recv_queue = recv_queue
def __init__(self, send_queue, callback): self.connection = Connection() self.callback = callback self.sender = SecureRpcSender(send_queue, self.connection)
def __init__(self, cluster_w_dir, cluster_r_dir): self.connection = Connection() self.sender_write = SecureRpcSender(cluster_w_dir, self.connection) self.sender_read = SecureRpcSender(cluster_r_dir, self.connection)
def __init__(self, recv_queue, send_queue): self.connection = Connection() self.receiver = self.connection.create_direct_receiver(recv_queue) self.sender = self.connection.create_direct_sender(send_queue)