class SurfaceDispatcher: def __init__(self): self.in_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE, consumer=True, queue_name=MATCHES_QUEUE) self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.in_queue.consume(self.dispatch) def dispatch(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.in_queue.cancel() return data = body.decode().split(',') surface = data[3] minutes = data[9] if minutes == '' or surface in ('', 'None'): return self.out_queue.publish(minutes, surface) logging.info('Sent %s minutes to %s accumulator' % (minutes, surface))
class Accumulator: def __init__(self, routing_key, exchange, output_exchange): self.routing_key = routing_key self.total = 0 self.amount = 0.0 self.in_queue = RabbitMQQueue(exchange=exchange, exchange_type='direct', consumer=True, exclusive=True, routing_keys=routing_key.split('-')) self.out_queue = RabbitMQQueue(exchange=output_exchange) def run(self): self.in_queue.consume(self.add) def add(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: body = ','.join( [self.routing_key, str(self.amount), str(self.total)]) self.out_queue.publish(body) self.in_queue.cancel() return self.total += float(body.decode()) self.amount += 1 logging.debug('Current total: %f' % self.total) logging.debug('Current amount: %f' % self.amount)
class Database: def __init__(self): self.count = 0 self.in_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct', consumer=True, exclusive=True, routing_keys=FILES) self.out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE) def run(self): self.in_queue.consume(self.persist) def persist(self, ch, method, properties, body): logging.info('Received %r' % body) result = body.decode() if result == END: self.count += 1 if self.count != 3: return for filename in FILES: file = open(filename, 'r') response = file.read() file.close() self.out_queue.publish(response) self.in_queue.cancel() return file = open(method.routing_key, 'a+') file.write(result + '\n') file.close()
class DifferentHandsFilter: def __init__(self): self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, consumer=True, queue_name=JOINED_QUEUE) self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.in_queue.consume(self.filter) def filter(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.in_queue.cancel() return data = body.decode().split(',') winner_hand = data[3] loser_hand = data[7] if winner_hand in HANDS and loser_hand != winner_hand: self.out_queue.publish('1', winner_hand) logging.info('Sent 1 to %s accumulator' % winner_hand)
class Terminator: def __init__(self, processes_number, in_exchange, group_exchange, next_exchange, next_exchange_type, next_routing_keys): self.processes_number = processes_number self.next_routing_keys = next_routing_keys self.closed = 0 self.in_queue = RabbitMQQueue(exchange=in_exchange, consumer=True, exclusive=True) self.group_queue = RabbitMQQueue(exchange=group_exchange) self.next_queue = RabbitMQQueue(exchange=next_exchange, exchange_type=next_exchange_type) def run(self): self.in_queue.consume(self.close) def close(self, ch, method, properties, body): if body == END_ENCODED: for i in range(self.processes_number): self.group_queue.publish(CLOSE) return if body == OK_ENCODED: self.closed += 1 if self.closed == self.processes_number: for routing_key in self.next_routing_keys.split('-'): self.next_queue.publish(END, routing_key) self.in_queue.cancel()
class PercentageCalculator: def __init__(self): self.left = None self.right = None self.in_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') def run(self): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) [hand, amount, total] = body.decode().split(',') if hand == RIGHT: self.right = float(amount) if self.left is None: return if hand == NO_RIGHT: self.left = float(amount) if self.right is None: return right_percentage = 100 * self.right / (self.left + self.right) left_percentage = 100 - right_percentage right_response = 'R Victories: {}%'.format(right_percentage) left_response = 'L Victories: {}%'.format(left_percentage) self.out_queue.publish(right_response, ROUTING_KEY) self.out_queue.publish(left_response, ROUTING_KEY) self.out_queue.publish(END, ROUTING_KEY) self.in_queue.cancel()
class AgeDifferenceFilter: def __init__(self): self.in_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE, consumer=True, queue_name=AGE_DIFFERENCE_FILTER_QUEUE) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.in_queue.consume(self.filter) def filter(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.in_queue.cancel() return data = body.decode().split(',') winner_age = int(data[4]) loser_age = int(data[8]) if winner_age - loser_age >= 20: winner_name = ' '.join([data[1], data[2]]) loser_name = ' '.join([data[5], data[6]]) result = '{}\t{}\t{}\t{}'.format(winner_age, winner_name, loser_age, loser_name) self.out_queue.publish(result, ROUTING_KEY) logging.info('Sent %s' % result)
class Client: def __init__(self, argv): self.id = str(uuid1()) self.metadata = [FROM_DEFAULT, TO_DEFAULT, self.id] self.parse_args(argv) self.results = 0 self.in_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' + self.id, consumer=True, exclusive=True) self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE) def parse_args(self, argv): try: options, args = getopt.getopt(argv, "f:t:", ["from=", "to="]) except getopt.GetoptError: print("Usage: python3 client.py [--from=YYYYMMDD] [--to=YYYYMMDD]") sys.exit(2) for option, arg in options: if option in ("-f", "--from"): self.metadata[0] = arg else: self.metadata[1] = arg def run(self): self.send_matches_data() self.in_queue.consume(self.print_response) def send_matches_data(self): for filename in glob(MATCHES_DATA): with open(filename, 'r') as file: file.readline() for line in iter(file.readline, ''): body = ','.join(self.metadata) + ',' + line self.matches_queue.publish(body) logging.info('Sent %s' % body) end = ','.join([self.id, END]) self.matches_queue.publish(end) logging.info('Sent %s' % end) def print_response(self, ch, method, properties, body): print(body.decode()) self.results += 1 ch.basic_ack(delivery_tag=method.delivery_tag) if self.results == 3: self.in_queue.cancel()
class AgeCalculator: def __init__(self): self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, consumer=True, queue_name=AGE_CALCULATOR_QUEUE) self.out_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE) self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.in_queue.cancel() return data = body.decode().split(',') tourney_date = data[0] winner_birthdate = data[4] loser_birthdate = data[8] if winner_birthdate == '' or loser_birthdate == '': return tourney_date = datetime.strptime(tourney_date, '%Y%m%d') winner_age = self._compute_age( datetime.strptime(winner_birthdate, '%Y%m%d'), tourney_date) loser_age = self._compute_age( datetime.strptime(loser_birthdate, '%Y%m%d'), tourney_date) data[4] = str(winner_age) data[8] = str(loser_age) body = ','.join(data) self.out_queue.publish(body) logging.info('Sent %s' % body) def _compute_age(self, birthdate, tourney_date): years = tourney_date.year - birthdate.year if tourney_date.month < birthdate.month or \ (tourney_date.month == birthdate.month and tourney_date.day < birthdate.day): years -= 1 return years
class Joiner: def __init__(self): self.players = {} self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE, consumer=True, queue_name=MATCHES_QUEUE) self.players_queue = RabbitMQQueue(exchange=PLAYERS_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE) self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.players_queue.consume(self.save_player) self.matches_queue.consume(self.join) def save_player(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.players_queue.cancel() return data = body.decode().split(',') self.players[data[0]] = data[1:5] def join(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.matches_queue.cancel() return data = body.decode().split(',') winner_id = data[4] loser_id = data[5] data = [data[2]] + self.players[winner_id] + self.players[loser_id] body = ','.join(data) self.out_queue.publish(body) logging.info('Sent %s' % body)
class AverageCalculator: def __init__(self): self.count = 0 self.in_queue = RabbitMQQueue(exchange=AVERAGE_CALCULATOR_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') def run(self): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) [surface, amount, total] = body.decode().split(',') avg = float(total) / float(amount) result = '{}: {} minutes'.format(surface, avg) self.out_queue.publish(result, ROUTING_KEY) self.count += 1 if self.count == 3: self.out_queue.publish(END, ROUTING_KEY) self.in_queue.cancel() logging.info('Sent %s' % result)
class Client: def __init__(self): self.results = 0 self.in_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE, consumer=True, exclusive=True) self.players_queue = RabbitMQQueue(exchange=PLAYERS_EXCHANGE) self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE) def run(self): self.send_players_data() self.send_matches_data() self.in_queue.consume(self.print_response) def send_players_data(self): with open(PLAYERS_DATA, 'r') as file: file.readline() for line in iter(file.readline, ''): self.players_queue.publish(line) logging.info('Sent %s' % line) self.players_queue.publish(END) def send_matches_data(self): for filename in glob(MATCHES_DATA): with open(filename, 'r') as file: file.readline() for line in iter(file.readline, ''): self.matches_queue.publish(line) logging.info('Sent %s' % line) self.matches_queue.publish(END) def print_response(self, ch, method, properties, body): print(body.decode()) self.results += 1 if self.results == 3: self.in_queue.cancel()
class Database: def __init__(self): self.hostname = os.environ['HOSTNAME'] self.count = {} self.in_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct', consumer=True, queue_name='{}_queue'.format( self.hostname), routing_keys=FILES) self.storage_queue = RabbitMQQueue(exchange='storage_input') self.data_queue = RabbitMQQueue(exchange='storage_output', exchange_type='direct', consumer=True, exclusive=True, routing_keys=[self.hostname]) def run(self, _): cmd = ';'.join(['CATCHUP', self.hostname]) self.storage_queue.publish(cmd) logging.info('Sent %s to storage' % cmd) self.data_queue.consume(self.update_count) self.in_queue.consume(self.persist) def update_count(self, ch, method, properties, body): data = body.decode() if data == 'END': logging.info('State of %s updated' % self.hostname) self.data_queue.cancel() ch.basic_ack(delivery_tag=method.delivery_tag) return params = data.split(';') id = params[-1] count = int(params[2]) self.count[id] = count logging.info('Count of %s updated: %d' % (id, count)) ch.basic_ack(delivery_tag=method.delivery_tag) def persist(self, ch, method, properties, body): logging.info('Received %r from %s' % (body, method.routing_key)) data = body.decode().split(',') id = data[0] result = data[1] if result == END: self.count[id] = self.count.get(id, 0) + 1 cmd = ';'.join(['WRITE', self.hostname, str(self.count[id]), id]) self.storage_queue.publish(cmd) if self.count[id] != 3: ch.basic_ack(delivery_tag=method.delivery_tag) return for filename in FILES: try: file = open(filename + id, 'r') response = file.read() file.close() except FileNotFoundError: response = '%s: No results' % filename out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' + id) out_queue.publish(response) logging.info('Sent %s' % response) ch.basic_ack(delivery_tag=method.delivery_tag) return file = open(method.routing_key + id, 'a+') file.write(result + '\n') file.close() ch.basic_ack(delivery_tag=method.delivery_tag)
class Accumulator: def __init__(self, routing_key, exchange, output_exchange): self.hostname = os.environ['HOSTNAME'] self.routing_key = routing_key self.values = {} self.in_queue = RabbitMQQueue(exchange=exchange, exchange_type='direct', consumer=True, queue_name='{}_queue'.format(self.hostname), routing_keys=routing_key.split('-')) self.out_queue = RabbitMQQueue(exchange=output_exchange) self.storage_queue = RabbitMQQueue(exchange='storage_input') self.data_queue = RabbitMQQueue(exchange='storage_output', exchange_type='direct', consumer=True, exclusive=True, routing_keys=[self.hostname]) def run(self, _): cmd = ';'.join(['CATCHUP', self.hostname]) self.storage_queue.publish(cmd) logging.info('Sent %s to storage' % cmd) self.data_queue.consume(self.update_values) self.in_queue.consume(self.add) def update_values(self, ch, method, properties, body): data = body.decode() if data == 'END': logging.info('State of %s updated' % self.hostname) self.data_queue.cancel() ch.basic_ack(delivery_tag=method.delivery_tag) return params = data.split(';') id = params[-1] values = params[2].split(',') amount = int(values[0]) total = float(values[1]) self.values[id] = [amount, total] logging.info('Values of %s updated: [%d, %f]' % (id, amount, total)) ch.basic_ack(delivery_tag=method.delivery_tag) def add(self, ch, method, properties, body): logging.info('Received %r' % body) data = body.decode().split(',') id = data[0] if data[1] == END: [amount, total] = self.values[id] body = ','.join([id, self.routing_key, str(amount), str(total)]) self.out_queue.publish(body) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return if not id in self.values: self.values[id] = [0, 0] self.values[id][0] += 1 amount = self.values[id][0] logging.debug('Current amount: %f' % amount) self.values[id][1] += float(data[1]) total = self.values[id][1] logging.debug('Current total: %f' % total) values = ','.join([str(amount), str(total)]) cmd = ';'.join(['WRITE', self.hostname, values, id]) self.storage_queue.publish(cmd) ch.basic_ack(delivery_tag=method.delivery_tag)
class Storage: def __init__(self, pid): self.pid = pid self.role = SLAVE_ROLE self.input_queue = RabbitMQQueue( exchange='storage_slave', consumer=True, queue_name='slave{}_queue'.format(pid)) self.output_queue = RabbitMQQueue(exchange='storage_output', exchange_type='direct') self.master_queue = RabbitMQQueue(exchange='storage_input', consumer=True, queue_name='master_queue') self.instance_queue = RabbitMQQueue( exchange='storage_internal_{}'.format(pid), consumer=True, queue_name='storage_internal_{}'.format(pid)) self.heartbeatproc = None def run(self, heartbeatproc): self.heartbeatproc = heartbeatproc self.heartbeatproc.metadata = self.role self.input_queue.consume(self.process) self.instance_queue.consume(self.listen) def process(self, ch, method, properties, body): if self.role == SLAVE_ROLE: logging.info('[SLAVE] Received %r' % body) msg = body.decode() parts = msg.split(",") if parts[0] == MASTER_NEEDED_MSG: # New master message if int(parts[1]) == int(self.pid): # and I am the new master logging.info( '[SLAVE] I was asked to be the new Storage Master') self.role = MASTER_ROLE # start sending new role in heartbeats self.heartbeatproc.metadata = self.role ch.basic_ack(delivery_tag=method.delivery_tag) self.instance_queue.publish(MASTER_NEEDED_MSG) self.input_queue.cancel() else: logging.info( '[SLAVE] I received a master message but I was not the target' ) # I am not the new master, discard it ch.basic_ack(delivery_tag=method.delivery_tag) return logging.info('[SLAVE] Saving message') self.persistState(body) if ch.is_open: ch.basic_ack(delivery_tag=method.delivery_tag) def listen(self, ch, method, properties, body): logging.info('[MASTER] I am consuming from storage_input') self.master_queue.consume(self.processMaster) def processMaster(self, ch, method, properties, body): logging.info('[MASTER] Received %r' % body) if self.isReadRequest(body): self.processRead(body) if self.isWriteRequest(body): self.persistState(body) self.input_queue.publish(body) if ch.is_open: ch.basic_ack(delivery_tag=method.delivery_tag) def persistState(self, body): state = body.decode() logging.info('Persisting to disk {%r}' % state) # MSG = CMD;tipoNodo_nroNodo;estado;job_id # EJ: WRITE;joiner_3;93243;123123 params = state.split(';') # path = "/storage/nodeType_nodeNumber/" path = BASE_PATH + params[1] + "/" pathlib.Path(path).mkdir(parents=True, exist_ok=True) # filename = "job_id" filename = params[-1] + ".state" f = open(path + filename, "w+") # Truncates previous state & writes f.write(str(state)) f.close() logging.info('Persisted {%r}' % state) def isReadRequest(self, b): if CATCHUP_COMMAND in b.decode(): return True return False def isWriteRequest(self, b): if WRITE_COMMAND in b.decode(): return True return False def processRead(self, msg): # MSG = CMD;tipoNodo_nroNodo # EJ: READ;joiner_3 params = msg.decode().split(';') path = BASE_PATH + params[1] + "/" client_routing_key = params[1] for filename in glob(path + "*.state"): with open(filename, 'r') as file: contents = file.read() self.output_queue.publish(contents, client_routing_key) self.output_queue.publish('END', client_routing_key)