class Database: def __init__(self): self.count = 0 self.in_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct', consumer=True, exclusive=True, routing_keys=FILES) self.out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE) def run(self): self.in_queue.consume(self.persist) def persist(self, ch, method, properties, body): logging.info('Received %r' % body) result = body.decode() if result == END: self.count += 1 if self.count != 3: return for filename in FILES: file = open(filename, 'r') response = file.read() file.close() self.out_queue.publish(response) self.in_queue.cancel() return file = open(method.routing_key, 'a+') file.write(result + '\n') file.close()
class AverageCalculator: def __init__(self): self.count = {} self.in_queue = RabbitMQQueue(exchange=AVERAGE_CALCULATOR_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') def run(self, _): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) [id, surface, amount, total] = body.decode().split(',') avg = float(total) / float(amount) result = '{}: {} minutes'.format(surface, avg) body = ','.join([id, result]) self.out_queue.publish(body, ROUTING_KEY) logging.info('Sent %s' % body) self.count[id] = self.count.get(id, 0) + 1 if self.count[id] == 3: end = ','.join([id, END]) self.out_queue.publish(end, ROUTING_KEY) logging.info('Sent %s' % end) ch.basic_ack(delivery_tag=method.delivery_tag)
class Terminator: def __init__(self, processes_number, in_exchange, group_exchange, next_exchange, next_exchange_type, next_routing_keys): self.processes_number = processes_number self.next_routing_keys = next_routing_keys self.closed = 0 self.in_queue = RabbitMQQueue(exchange=in_exchange, consumer=True, exclusive=True) self.group_queue = RabbitMQQueue(exchange=group_exchange) self.next_queue = RabbitMQQueue(exchange=next_exchange, exchange_type=next_exchange_type) def run(self): self.in_queue.consume(self.close) def close(self, ch, method, properties, body): if body == END_ENCODED: for i in range(self.processes_number): self.group_queue.publish(CLOSE) return if body == OK_ENCODED: self.closed += 1 if self.closed == self.processes_number: for routing_key in self.next_routing_keys.split('-'): self.next_queue.publish(END, routing_key) self.in_queue.cancel()
def persist(self, ch, method, properties, body): logging.info('Received %r from %s' % (body, method.routing_key)) data = body.decode().split(',') id = data[0] result = data[1] if result == END: self.count[id] = self.count.get(id, 0) + 1 cmd = ';'.join(['WRITE', self.hostname, str(self.count[id]), id]) self.storage_queue.publish(cmd) if self.count[id] != 3: ch.basic_ack(delivery_tag=method.delivery_tag) return for filename in FILES: try: file = open(filename + id, 'r') response = file.read() file.close() except FileNotFoundError: response = '%s: No results' % filename out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' + id) out_queue.publish(response) logging.info('Sent %s' % response) ch.basic_ack(delivery_tag=method.delivery_tag) return file = open(method.routing_key + id, 'a+') file.write(result + '\n') file.close() ch.basic_ack(delivery_tag=method.delivery_tag)
class Accumulator: def __init__(self, routing_key, exchange, output_exchange): self.routing_key = routing_key self.total = 0 self.amount = 0.0 self.in_queue = RabbitMQQueue(exchange=exchange, exchange_type='direct', consumer=True, exclusive=True, routing_keys=routing_key.split('-')) self.out_queue = RabbitMQQueue(exchange=output_exchange) def run(self): self.in_queue.consume(self.add) def add(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: body = ','.join( [self.routing_key, str(self.amount), str(self.total)]) self.out_queue.publish(body) self.in_queue.cancel() return self.total += float(body.decode()) self.amount += 1 logging.debug('Current total: %f' % self.total) logging.debug('Current amount: %f' % self.amount)
class Joiner: def __init__(self): self.acked = set() self.players = {} self.matches_queue = RabbitMQQueue(exchange=FILTERED_EXCHANGE, exchange_type='direct', consumer=True, queue_name=FILTERED_QUEUE, routing_keys=['joiner']) self.out_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self, _): self.save_players() self.matches_queue.consume(self.join) def save_players(self): with open(PLAYERS_DATA, 'r') as file: file.readline() for line in iter(file.readline, ''): data = line.split(',') self.players[data[0]] = data[1:5] def join(self, ch, method, properties, body): logging.info('Received %r' % body) data = body.decode().split(',') id = data[0] if data[1] == END: self.terminator_queue.publish(body) logging.info('Sent %r' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return if data[1] == CLOSE: if not id in self.acked: body = ','.join([id, OK]) self.terminator_queue.publish(body) self.acked.add(id) else: self.matches_queue.publish(body) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return winner_id = data[5] loser_id = data[6] data = [data[0], data[3] ] + self.players[winner_id] + self.players[loser_id] body = ','.join(data) self.out_queue.publish(body, 'filter') self.out_queue.publish(body, 'calculator') logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag)
class Client: def __init__(self, argv): self.id = str(uuid1()) self.metadata = [FROM_DEFAULT, TO_DEFAULT, self.id] self.parse_args(argv) self.results = 0 self.in_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' + self.id, consumer=True, exclusive=True) self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE) def parse_args(self, argv): try: options, args = getopt.getopt(argv, "f:t:", ["from=", "to="]) except getopt.GetoptError: print("Usage: python3 client.py [--from=YYYYMMDD] [--to=YYYYMMDD]") sys.exit(2) for option, arg in options: if option in ("-f", "--from"): self.metadata[0] = arg else: self.metadata[1] = arg def run(self): self.send_matches_data() self.in_queue.consume(self.print_response) def send_matches_data(self): for filename in glob(MATCHES_DATA): with open(filename, 'r') as file: file.readline() for line in iter(file.readline, ''): body = ','.join(self.metadata) + ',' + line self.matches_queue.publish(body) logging.info('Sent %s' % body) end = ','.join([self.id, END]) self.matches_queue.publish(end) logging.info('Sent %s' % end) def print_response(self, ch, method, properties, body): print(body.decode()) self.results += 1 ch.basic_ack(delivery_tag=method.delivery_tag) if self.results == 3: self.in_queue.cancel()
class DateFilter: def __init__(self): self.acked = set() self.in_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE, consumer=True, queue_name=MATCHES_QUEUE) self.out_queue = RabbitMQQueue(exchange=FILTERED_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self, _): self.in_queue.consume(self.filter) def filter(self, ch, method, properties, body): logging.info('Received %r' % body) match = body.decode().split(',') if match[1] == END: self.terminator_queue.publish(body) logging.info('Sent %r' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return if match[1] == CLOSE: id = match[0] if not id in self.acked: body = ','.join([id, OK]) self.terminator_queue.publish(body) self.acked.add(id) else: self.in_queue.publish(body) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return date_from = match[0] date_to = match[1] tourney_date = match[5] if tourney_date < date_from or tourney_date > date_to: ch.basic_ack(delivery_tag=method.delivery_tag) return data = ','.join(match[2:]) self.out_queue.publish(data, 'joiner') self.out_queue.publish(data, 'dispatcher') logging.info('Sent %s' % data) ch.basic_ack(delivery_tag=method.delivery_tag)
class Terminator: def __init__(self, processes_number, in_exchange, group_exchange, \ group_exchange_type, group_routing_key, next_exchange, \ next_exchange_type, next_routing_keys): self.processes_number = processes_number self.next_routing_keys = next_routing_keys self.group_routing_key = group_routing_key self.closed = {} self.in_queue = RabbitMQQueue(exchange=in_exchange, consumer=True, exclusive=True) self.group_queue = RabbitMQQueue(exchange=group_exchange, exchange_type=group_exchange_type) self.next_queue = RabbitMQQueue(exchange=next_exchange, exchange_type=next_exchange_type) def run(self, _): self.in_queue.consume(self.close) def close(self, ch, method, properties, body): logging.info('Received %r' % body) data = body.decode().split(',') if data[1] == END: for i in range(self.processes_number): body = ','.join([data[0], CLOSE]) self.group_queue.publish(body, self.group_routing_key) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return if data[1] == OK: id = data[0] self.closed[id] = self.closed.get(id, 0) + 1 if self.closed[id] == self.processes_number: for routing_key in self.next_routing_keys.split('-'): body = ','.join([id, END]) self.next_queue.publish(body, routing_key) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag)
class PercentageCalculator: def __init__(self): self.left = None self.right = None self.in_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') def run(self): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) [hand, amount, total] = body.decode().split(',') if hand == RIGHT: self.right = float(amount) if self.left is None: return if hand == NO_RIGHT: self.left = float(amount) if self.right is None: return right_percentage = 100 * self.right / (self.left + self.right) left_percentage = 100 - right_percentage right_response = 'R Victories: {}%'.format(right_percentage) left_response = 'L Victories: {}%'.format(left_percentage) self.out_queue.publish(right_response, ROUTING_KEY) self.out_queue.publish(left_response, ROUTING_KEY) self.out_queue.publish(END, ROUTING_KEY) self.in_queue.cancel()
class AgeDifferenceFilter: def __init__(self): self.in_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE, consumer=True, queue_name=AGE_DIFFERENCE_FILTER_QUEUE) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.in_queue.consume(self.filter) def filter(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.in_queue.cancel() return data = body.decode().split(',') winner_age = int(data[4]) loser_age = int(data[8]) if winner_age - loser_age >= 20: winner_name = ' '.join([data[1], data[2]]) loser_name = ' '.join([data[5], data[6]]) result = '{}\t{}\t{}\t{}'.format(winner_age, winner_name, loser_age, loser_name) self.out_queue.publish(result, ROUTING_KEY) logging.info('Sent %s' % result)
class DifferentHandsFilter: def __init__(self): self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, consumer=True, queue_name=JOINED_QUEUE) self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.in_queue.consume(self.filter) def filter(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.in_queue.cancel() return data = body.decode().split(',') winner_hand = data[3] loser_hand = data[7] if winner_hand in HANDS and loser_hand != winner_hand: self.out_queue.publish('1', winner_hand) logging.info('Sent 1 to %s accumulator' % winner_hand)
class SurfaceDispatcher: def __init__(self): self.in_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE, consumer=True, queue_name=MATCHES_QUEUE) self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.in_queue.consume(self.dispatch) def dispatch(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.in_queue.cancel() return data = body.decode().split(',') surface = data[3] minutes = data[9] if minutes == '' or surface in ('', 'None'): return self.out_queue.publish(minutes, surface) logging.info('Sent %s minutes to %s accumulator' % (minutes, surface))
class SurfaceDispatcher: def __init__(self): self.acked = set() self.in_queue = RabbitMQQueue(exchange=FILTERED_EXCHANGE, exchange_type='direct', consumer=True, queue_name=FILTERED_QUEUE, routing_keys=['dispatcher']) self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self, _): self.in_queue.consume(self.dispatch) def dispatch(self, ch, method, properties, body): logging.info('Received %r' % body) data = body.decode().split(',') id = data[0] if data[1] == END: self.terminator_queue.publish(body) logging.info('Sent %r' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return if data[1] == CLOSE: if not id in self.acked: body = ','.join([id, OK]) self.terminator_queue.publish(body) self.acked.add(id) else: self.in_queue.publish(body) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return surface = data[4] minutes = data[10] if minutes == '' or surface in ('', 'None'): ch.basic_ack(delivery_tag=method.delivery_tag) return body = ','.join([id, minutes]) self.out_queue.publish(body, surface) logging.info('Sent %s to %s accumulator' % (body, surface)) ch.basic_ack(delivery_tag=method.delivery_tag)
class AverageCalculator: def __init__(self): self.count = 0 self.in_queue = RabbitMQQueue(exchange=AVERAGE_CALCULATOR_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') def run(self): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) [surface, amount, total] = body.decode().split(',') avg = float(total) / float(amount) result = '{}: {} minutes'.format(surface, avg) self.out_queue.publish(result, ROUTING_KEY) self.count += 1 if self.count == 3: self.out_queue.publish(END, ROUTING_KEY) self.in_queue.cancel() logging.info('Sent %s' % result)
class AgeDifferenceFilter: def __init__(self): self.acked = set() self.in_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE, consumer=True, queue_name=AGE_DIFFERENCE_FILTER_QUEUE) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self, _): self.in_queue.consume(self.filter) def filter(self, ch, method, properties, body): logging.info('Received %r' % body) data = body.decode().split(',') id = data[0] if data[1] == END: self.terminator_queue.publish(body) logging.info('Sent %r' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return if data[1] == CLOSE: if not id in self.acked: body = ','.join([id, OK]) self.terminator_queue.publish(body) self.acked.add(id) else: self.in_queue.publish(body) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return winner_age = int(data[5]) loser_age = int(data[9]) if winner_age - loser_age >= 20: winner_name = ' '.join([data[2], data[3]]) loser_name = ' '.join([data[6], data[7]]) result = '{}\t{}\t{}\t{}'.format(winner_age, winner_name, loser_age, loser_name) body = ','.join([id, result]) self.out_queue.publish(body, ROUTING_KEY) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag)
class DifferentHandsFilter: def __init__(self): self.acked = set() self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, exchange_type='direct', consumer=True, queue_name=JOINED_QUEUE, routing_keys=['filter']) self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self, _): self.in_queue.consume(self.filter) def filter(self, ch, method, properties, body): logging.info('Received %r' % body) data = body.decode().split(',') id = data[0] if data[1] == END: self.terminator_queue.publish(body) logging.info('Sent %r' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return if data[1] == CLOSE: if not id in self.acked: body = ','.join([data[0], OK]) self.terminator_queue.publish(body) self.acked.add(id) else: self.in_queue.publish(body) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return winner_hand = data[4] loser_hand = data[8] if winner_hand in HANDS and loser_hand != winner_hand: body = ','.join([id, '1']) self.out_queue.publish(body, winner_hand) logging.info('Sent %s to %s accumulator' % (body, winner_hand)) ch.basic_ack(delivery_tag=method.delivery_tag)
class PercentageCalculator: def __init__(self): self.hands = {} self.in_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') def run(self, _): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) [id, hand, amount, total] = body.decode().split(',') if not id in self.hands: self.hands[id] = [None, None] left = self.hands[id][1] if hand == RIGHT: self.hands[id][0] = float(amount) if self.hands[id][1] is None: ch.basic_ack(delivery_tag=method.delivery_tag) return if hand == NO_RIGHT: self.hands[id][1] = float(amount) if self.hands[id][0] is None: ch.basic_ack(delivery_tag=method.delivery_tag) return right_percentage = 100 * self.hands[id][0] / (self.hands[id][0] + self.hands[id][1]) left_percentage = 100 - right_percentage right_response = 'R Victories: {}%'.format(right_percentage) left_response = 'L Victories: {}%'.format(left_percentage) body = ','.join([id, right_response]) self.out_queue.publish(body, ROUTING_KEY) logging.info('Sent %s' % body) body = ','.join([id, left_response]) self.out_queue.publish(body, ROUTING_KEY) logging.info('Sent %s' % body) body = ','.join([id, END]) self.out_queue.publish(body, ROUTING_KEY) ch.basic_ack(delivery_tag=method.delivery_tag)
class Client: def __init__(self): self.results = 0 self.in_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE, consumer=True, exclusive=True) self.players_queue = RabbitMQQueue(exchange=PLAYERS_EXCHANGE) self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE) def run(self): self.send_players_data() self.send_matches_data() self.in_queue.consume(self.print_response) def send_players_data(self): with open(PLAYERS_DATA, 'r') as file: file.readline() for line in iter(file.readline, ''): self.players_queue.publish(line) logging.info('Sent %s' % line) self.players_queue.publish(END) def send_matches_data(self): for filename in glob(MATCHES_DATA): with open(filename, 'r') as file: file.readline() for line in iter(file.readline, ''): self.matches_queue.publish(line) logging.info('Sent %s' % line) self.matches_queue.publish(END) def print_response(self, ch, method, properties, body): print(body.decode()) self.results += 1 if self.results == 3: self.in_queue.cancel()
class AgeCalculator: def __init__(self): self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, consumer=True, queue_name=AGE_CALCULATOR_QUEUE) self.out_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE) self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.in_queue.cancel() return data = body.decode().split(',') tourney_date = data[0] winner_birthdate = data[4] loser_birthdate = data[8] if winner_birthdate == '' or loser_birthdate == '': return tourney_date = datetime.strptime(tourney_date, '%Y%m%d') winner_age = self._compute_age( datetime.strptime(winner_birthdate, '%Y%m%d'), tourney_date) loser_age = self._compute_age( datetime.strptime(loser_birthdate, '%Y%m%d'), tourney_date) data[4] = str(winner_age) data[8] = str(loser_age) body = ','.join(data) self.out_queue.publish(body) logging.info('Sent %s' % body) def _compute_age(self, birthdate, tourney_date): years = tourney_date.year - birthdate.year if tourney_date.month < birthdate.month or \ (tourney_date.month == birthdate.month and tourney_date.day < birthdate.day): years -= 1 return years
class Joiner: def __init__(self): self.players = {} self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE, consumer=True, queue_name=MATCHES_QUEUE) self.players_queue = RabbitMQQueue(exchange=PLAYERS_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE) self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.players_queue.consume(self.save_player) self.matches_queue.consume(self.join) def save_player(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.players_queue.cancel() return data = body.decode().split(',') self.players[data[0]] = data[1:5] def join(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.matches_queue.cancel() return data = body.decode().split(',') winner_id = data[4] loser_id = data[5] data = [data[2]] + self.players[winner_id] + self.players[loser_id] body = ','.join(data) self.out_queue.publish(body) logging.info('Sent %s' % body)
class AgeCalculator: def __init__(self): self.acked = set() self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, exchange_type='direct', consumer=True, queue_name=AGE_CALCULATOR_QUEUE, routing_keys=['calculator']) self.out_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE) self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self, _): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) data = body.decode().split(',') id = data[0] if data[1] == END: self.terminator_queue.publish(body) logging.info('Sent %r' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return if data[1] == CLOSE: if not id in self.acked: body = ','.join([id, OK]) self.terminator_queue.publish(body) self.acked.add(id) else: self.in_queue.publish(body) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return tourney_date = data[1] winner_birthdate = data[5] loser_birthdate = data[9] if winner_birthdate == '' or loser_birthdate == '': ch.basic_ack(delivery_tag=method.delivery_tag) return tourney_date = datetime.strptime(tourney_date, '%Y%m%d') winner_age = self._compute_age( datetime.strptime(winner_birthdate, '%Y%m%d'), tourney_date) loser_age = self._compute_age( datetime.strptime(loser_birthdate, '%Y%m%d'), tourney_date) data[5] = str(winner_age) data[9] = str(loser_age) body = ','.join(data) self.out_queue.publish(body) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) def _compute_age(self, birthdate, tourney_date): years = tourney_date.year - birthdate.year if tourney_date.month < birthdate.month or \ (tourney_date.month == birthdate.month and tourney_date.day < birthdate.day): years -= 1 return years
class Storage: def __init__(self, pid): self.pid = pid self.role = SLAVE_ROLE self.input_queue = RabbitMQQueue( exchange='storage_slave', consumer=True, queue_name='slave{}_queue'.format(pid)) self.output_queue = RabbitMQQueue(exchange='storage_output', exchange_type='direct') self.master_queue = RabbitMQQueue(exchange='storage_input', consumer=True, queue_name='master_queue') self.instance_queue = RabbitMQQueue( exchange='storage_internal_{}'.format(pid), consumer=True, queue_name='storage_internal_{}'.format(pid)) self.heartbeatproc = None def run(self, heartbeatproc): self.heartbeatproc = heartbeatproc self.heartbeatproc.metadata = self.role self.input_queue.consume(self.process) self.instance_queue.consume(self.listen) def process(self, ch, method, properties, body): if self.role == SLAVE_ROLE: logging.info('[SLAVE] Received %r' % body) msg = body.decode() parts = msg.split(",") if parts[0] == MASTER_NEEDED_MSG: # New master message if int(parts[1]) == int(self.pid): # and I am the new master logging.info( '[SLAVE] I was asked to be the new Storage Master') self.role = MASTER_ROLE # start sending new role in heartbeats self.heartbeatproc.metadata = self.role ch.basic_ack(delivery_tag=method.delivery_tag) self.instance_queue.publish(MASTER_NEEDED_MSG) self.input_queue.cancel() else: logging.info( '[SLAVE] I received a master message but I was not the target' ) # I am not the new master, discard it ch.basic_ack(delivery_tag=method.delivery_tag) return logging.info('[SLAVE] Saving message') self.persistState(body) if ch.is_open: ch.basic_ack(delivery_tag=method.delivery_tag) def listen(self, ch, method, properties, body): logging.info('[MASTER] I am consuming from storage_input') self.master_queue.consume(self.processMaster) def processMaster(self, ch, method, properties, body): logging.info('[MASTER] Received %r' % body) if self.isReadRequest(body): self.processRead(body) if self.isWriteRequest(body): self.persistState(body) self.input_queue.publish(body) if ch.is_open: ch.basic_ack(delivery_tag=method.delivery_tag) def persistState(self, body): state = body.decode() logging.info('Persisting to disk {%r}' % state) # MSG = CMD;tipoNodo_nroNodo;estado;job_id # EJ: WRITE;joiner_3;93243;123123 params = state.split(';') # path = "/storage/nodeType_nodeNumber/" path = BASE_PATH + params[1] + "/" pathlib.Path(path).mkdir(parents=True, exist_ok=True) # filename = "job_id" filename = params[-1] + ".state" f = open(path + filename, "w+") # Truncates previous state & writes f.write(str(state)) f.close() logging.info('Persisted {%r}' % state) def isReadRequest(self, b): if CATCHUP_COMMAND in b.decode(): return True return False def isWriteRequest(self, b): if WRITE_COMMAND in b.decode(): return True return False def processRead(self, msg): # MSG = CMD;tipoNodo_nroNodo # EJ: READ;joiner_3 params = msg.decode().split(';') path = BASE_PATH + params[1] + "/" client_routing_key = params[1] for filename in glob(path + "*.state"): with open(filename, 'r') as file: contents = file.read() self.output_queue.publish(contents, client_routing_key) self.output_queue.publish('END', client_routing_key)
class Database: def __init__(self): self.hostname = os.environ['HOSTNAME'] self.count = {} self.in_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct', consumer=True, queue_name='{}_queue'.format( self.hostname), routing_keys=FILES) self.storage_queue = RabbitMQQueue(exchange='storage_input') self.data_queue = RabbitMQQueue(exchange='storage_output', exchange_type='direct', consumer=True, exclusive=True, routing_keys=[self.hostname]) def run(self, _): cmd = ';'.join(['CATCHUP', self.hostname]) self.storage_queue.publish(cmd) logging.info('Sent %s to storage' % cmd) self.data_queue.consume(self.update_count) self.in_queue.consume(self.persist) def update_count(self, ch, method, properties, body): data = body.decode() if data == 'END': logging.info('State of %s updated' % self.hostname) self.data_queue.cancel() ch.basic_ack(delivery_tag=method.delivery_tag) return params = data.split(';') id = params[-1] count = int(params[2]) self.count[id] = count logging.info('Count of %s updated: %d' % (id, count)) ch.basic_ack(delivery_tag=method.delivery_tag) def persist(self, ch, method, properties, body): logging.info('Received %r from %s' % (body, method.routing_key)) data = body.decode().split(',') id = data[0] result = data[1] if result == END: self.count[id] = self.count.get(id, 0) + 1 cmd = ';'.join(['WRITE', self.hostname, str(self.count[id]), id]) self.storage_queue.publish(cmd) if self.count[id] != 3: ch.basic_ack(delivery_tag=method.delivery_tag) return for filename in FILES: try: file = open(filename + id, 'r') response = file.read() file.close() except FileNotFoundError: response = '%s: No results' % filename out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' + id) out_queue.publish(response) logging.info('Sent %s' % response) ch.basic_ack(delivery_tag=method.delivery_tag) return file = open(method.routing_key + id, 'a+') file.write(result + '\n') file.close() ch.basic_ack(delivery_tag=method.delivery_tag)
class Accumulator: def __init__(self, routing_key, exchange, output_exchange): self.hostname = os.environ['HOSTNAME'] self.routing_key = routing_key self.values = {} self.in_queue = RabbitMQQueue(exchange=exchange, exchange_type='direct', consumer=True, queue_name='{}_queue'.format(self.hostname), routing_keys=routing_key.split('-')) self.out_queue = RabbitMQQueue(exchange=output_exchange) self.storage_queue = RabbitMQQueue(exchange='storage_input') self.data_queue = RabbitMQQueue(exchange='storage_output', exchange_type='direct', consumer=True, exclusive=True, routing_keys=[self.hostname]) def run(self, _): cmd = ';'.join(['CATCHUP', self.hostname]) self.storage_queue.publish(cmd) logging.info('Sent %s to storage' % cmd) self.data_queue.consume(self.update_values) self.in_queue.consume(self.add) def update_values(self, ch, method, properties, body): data = body.decode() if data == 'END': logging.info('State of %s updated' % self.hostname) self.data_queue.cancel() ch.basic_ack(delivery_tag=method.delivery_tag) return params = data.split(';') id = params[-1] values = params[2].split(',') amount = int(values[0]) total = float(values[1]) self.values[id] = [amount, total] logging.info('Values of %s updated: [%d, %f]' % (id, amount, total)) ch.basic_ack(delivery_tag=method.delivery_tag) def add(self, ch, method, properties, body): logging.info('Received %r' % body) data = body.decode().split(',') id = data[0] if data[1] == END: [amount, total] = self.values[id] body = ','.join([id, self.routing_key, str(amount), str(total)]) self.out_queue.publish(body) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return if not id in self.values: self.values[id] = [0, 0] self.values[id][0] += 1 amount = self.values[id][0] logging.debug('Current amount: %f' % amount) self.values[id][1] += float(data[1]) total = self.values[id][1] logging.debug('Current total: %f' % total) values = ','.join([str(amount), str(total)]) cmd = ';'.join(['WRITE', self.hostname, values, id]) self.storage_queue.publish(cmd) ch.basic_ack(delivery_tag=method.delivery_tag)
class ElectableProcess: """This class factorizes the leader election protocol. When this process is chosen as a leader this class runs callback on a separate thread.""" def __init__(self, pid, pid_list, callback): """Starts the election protocol. Receives this process' pid number, a list of other pids and a callback function used when this process is elected.""" self.pid = pid self.pid_list = pid_list self.onleader_callback = callback self.received_answer = False self.leader = None logging.basicConfig( format='%(asctime)s [PID {}] %(message)s'.format(self.pid)) def run(self, _): self.setup_queues() logging.info( "Waiting {}s for starting everything.".format(INITIAL_SLEEP)) time.sleep(INITIAL_SLEEP) logging.info("Starting off as listening to heartbeats..") self.follow_leader() def setup_queues(self): self.exchange = RabbitMQQueue(exchange=ELECTION_EXCHANGE, consumer=False, exchange_type="direct") self.process_queue = RabbitMQQueue(exchange=ELECTION_EXCHANGE, consumer=True, exchange_type="direct", routing_keys=[str(self.pid)]) self.leader_ex = RabbitMQQueue(exchange=LEADER_EXCHANGE, consumer=False, durable=False) self.leader_queue = RabbitMQQueue(exchange=LEADER_EXCHANGE, consumer=True, durable=False) def process_message(self, ch, method, properties, body): data = body.decode().split(',') logging.info("Received {}".format(body.decode())) if data[0] == "election": logging.info("Sending answer,{} to {}".format(self.pid, data[1])) self.exchange.publish("answer,{}".format(self.pid), data[1]) elif data[0] == "answer": self.received_answer = True # atomic elif data[0] == "coordinator": self.leader = int(data[1]) # atomic def start_election(self): self.received_answer = False self.leader = None logging.info("Starting election -- sending ELECTION") for remote_pid in self.pid_list: if remote_pid > self.pid: logging.info("Sending election,{} to {}".format( self.pid, remote_pid)) self.exchange.publish("election,{}".format(self.pid), str(remote_pid)) # If we do not receive an answer within the timeout, # we are now the coordinator. # This processes requests in another thread, logging.info("Consuming from queue") self.process_queue.async_consume(self.process_message, auto_ack=True) logging.info("Sleeping for {}s".format(TIMEOUT_ANSWER)) time.sleep(TIMEOUT_ANSWER) logging.info("Woke up from timeout") if self.received_answer: logging.info("Answer received!") time.sleep(TIMEOUT_COORD) logging.info("Woke up from coord timeout") # same as before => factorize in a function? if self.leader is None: logging.info("NO leader still! Rebooting election") self.start_election() logging.info("NEW leader! id is: {}".format(self.leader)) else: logging.info( "NO answer received! Therefore I am the leader. Sending coord." ) # No answer received in timeout, therefore this process is leader for remote_pid in self.pid_list: if remote_pid < self.pid: logging.info("->coordinator,{} to key {}".format( self.pid, str(remote_pid))) self.exchange.publish("coordinator,{}".format(self.pid), str(remote_pid)) logging.info("Setting leader pid") self.leader = self.pid if self.leader == self.pid: logging.info("Starting leader logic in this process") self.start_leader() else: logging.info("Following leader...") self.follow_leader() def start_leader(self): self.logic_process = multiprocessing.Process( target=self.onleader_callback) self.logic_process.start() # set up leader heartbeat while True: self.leader_ex.publish("heartbeat") time.sleep(HEARTBEAT_INTERVAL) def process_heartbeat(self, ch, method, properties, body): self.last_heartbeat = time.time() def follow_leader(self): # in another process update the counter with the time # this one just wakes up every fraction of a second updating # the count self.last_heartbeat = time.time() - 0.01 self.leader_queue.async_consume(self.process_heartbeat, auto_ack=True) diff = time.time() - self.last_heartbeat while diff < TIMEOUT_HEARTBEAT: #logging.info("Sleeping for {}s".format(TIMEOUT_HEARTBEAT - diff + 0.1)) time.sleep(TIMEOUT_HEARTBEAT - diff + 0.1) # So we don't over-wait diff = time.time() - self.last_heartbeat logging.info("Timed out leader. About to start a new election") self.start_election()
class HeartbeatProcess: """This class factorizes the heartbeat protocol. Receives a hostname, some metadata and a callback. Starts a thread sending heartbeats to the watchdog exchange and then calls the callback to execute business logic. Metadata can be used to provide additional information to the watchdog, such as the specific role the process currently has. Therefore the callback also receives this object, so it can change the metadata when it changes its role during runtime. Parameters ---------- hostname: string metadata: string A string without commas so that it can be passed during heartbeat. callback: function Receives this object as argument. Useful for altering metadata during runtime.""" def __init__(self, hostname, metadata, callback): self.hostname = hostname self.metadata = metadata self.callback = callback self.exchange = RabbitMQQueue(exchange=HEARTBEAT_EXCHANGE, consumer=False, exchange_type="fanout", durable=False) logging.basicConfig(format='%(asctime)s [PID {}] %(message)s'.format( self.hostname), level=logging.INFO) logging.info( "Instancing heartbeat process for hostname {}".format(hostname)) @staticmethod def setup(classname, *args, **kwargs): """Given a class with a run() method, tries to start a heartbeat process loading the hostname from the environment variable HOSTNAME. Starts with no metadata.""" def run_logic(hbproc): obj = classname(*args, **kwargs) obj.run(hbproc) return HeartbeatProcess(os.getenv("HOSTNAME", "-1"), "", run_logic) def start_heartbeat_thread(self): logging.info("Starting heartbeat thread..") self.thread = threading.Thread(target=self.periodic_heartbeat) self.thread.start() def periodic_heartbeat(self): while True: logging.debug("Sending heartbeat -- message is {}".format( "heartbeat,{},{}".format(self.hostname, self.metadata))) self.exchange.publish("heartbeat,{},{}".format( self.hostname, self.metadata)) time.sleep(HEARTBEAT_INTERVAL) def run(self): """Starts the thread and runs the callback""" logging.info("Starting thread and running the callback") self.start_heartbeat_thread() self.callback(self)
class WatchdogProcess: """This process receives heartbeats from other containers and launches new instances when it detects that a process has failed. Since this particular process can be started while the cluster is already running, in case of a leader failure for example, it needs to know the a priori configuration of instances. It can be the case that there already are failed processes when this process starts.""" def __init__(self, hostname): self.hostname = hostname self.basedirname = os.getenv("BASEDIRNAME", "error") self.to_start = [] self.to_stop = [] self.reassigning = False self.load_default_config() def run(self): logging.info("Starting Watchdog Logic") logging.info("Setting up queues") self.setup_queues() logging.info("Launching receiver thread") self.launch_receiver_thread() logging.info("Launching spawner threads") self.launch_spawner_threads() logging.info("Launching checker") self.launch_checker() def setup_queues(self): self.leader_queue = RabbitMQQueue(exchange=EXCHANGE, consumer=True, durable=False) self.storage_exchange = RabbitMQQueue(exchange='storage_slave', consumer=False) def load_default_config(self): self.default_config = load_yaml() self.last_timeout = {} self.storage_roles = {} for key in self.default_config.keys(): if key == "client": continue structure_single_key = { "{}_{}".format(key, i): time.time() + STARTING_TOLERANCE for i in range(self.default_config[key]) } self.last_timeout.update(structure_single_key) if key == "storage": # TODO self.storage_roles.update( {k: SLAVE_ROLE for k in structure_single_key.keys()}) self.storage_master_heartbeat = 0 # instant timeout # Special structure to indicate that a container is being restarted and # should not be checked for timeouts. self.respawning = {k: False for k in self.last_timeout.keys()} def process_heartbeat(self, ch, method, properties, body): logging.debug("Received heartbeat message is {}".format(str(body))) data = body.decode().split(',') recv_hostname = data[1] recv_metadata = data[2] self.last_timeout[recv_hostname] = time.time() if recv_hostname.split("_")[0] == "storage": self.storage_roles[recv_hostname] = recv_metadata if self.storage_roles[recv_hostname] == MASTER_ROLE: self.storage_master_heartbeat = self.last_timeout[ recv_hostname] def mkimgname(self, imgid): return "{}_{}_1".format(self.basedirname, imgid) def stop_container(self, imgid): imgid = self.mkimgname(imgid) failed = 0 result = subprocess.run(['docker', 'stop', "-t", "1", imgid], check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) logging.info( 'Stop command executed. Result={}. Output={}. Error={}'.format( result.returncode, result.stdout, result.stderr)) while result.returncode != 0 and failed < MAXFAILED: failed += 1 logging.error( "Return code was not zero. Trying again.. ({}/{} attempts)". format(failed, MAXFAILED)) result = subprocess.run(['docker', 'stop', "-t", "1", imgid], check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if failed >= MAXFAILED: raise Exception("Surpassed maximum retries for docker stop") def launch_container(self, imgid): imgid = self.mkimgname(imgid) cmd = ['docker', 'start', imgid] result = subprocess.run(cmd, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) logging.info( 'Start command executed. CMD={} Result={}. Output={}. Error={}'. format(" ".join(cmd), result.returncode, result.stdout, result.stderr)) def container_is_running(self, imgid): result = subprocess.run(['docker', 'ps'], check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return imgid in str(result.stdout) def respawn_stop_loop(self): """ This thread stops containers listed in self.to_pop and sends them onto the respawn_start_thread via the self.to_start list. """ while True: if len(self.to_stop) == 0: time.sleep(POLL_INTERVAL) continue imgid = self.to_stop.pop() try: self.stop_container(imgid) self.to_start.append(imgid) except Exception as e: # If we find an error stopping the container then we unmark #it and hope next time there is no error. logging.error("Exception on respawn_stop_thread: " + str(e)) self.respawning[imgid] = False def respawn_start_loop(self): """ This thread starts containers listed in self.to_start after a START_SLEEP interval and removes the respawning flag. The first heartbeat timeout is set with a STARTING_TOLERANCE offset. """ waiting = [] # internal list that we don't have race conditions on while True: current_time = time.time() if len(self.to_start) == 0 and len(waiting) == 0: time.sleep(POLL_INTERVAL) continue if len(self.to_start) > 0: imgid = self.to_start.pop() #atomic waiting.append((imgid, current_time + START_SLEEP)) # launch a container if we have passed the scheduled start time for imgid, scheduled_time in waiting: if scheduled_time <= current_time: try: self.launch_container(imgid) except Exception as e: logging.error("Exception on respawn_start_thread: " + str(e)) finally: self.last_timeout[imgid] = time.time( ) + STARTING_TOLERANCE self.respawning[imgid] = False # filter launched container waiting = [(i, t) for i, t in waiting if current_time < t] def launch_spawner_threads(self): respawn_start = threading.Thread(target=self.respawn_stop_loop) respawn_start.start() respawn_stop = threading.Thread(target=self.respawn_start_loop) respawn_stop.start() def launch_receiver_thread(self): self.leader_queue.async_consume(self.process_heartbeat, auto_ack=True) def reassign_storage_master(self): logging.info("Watchdog Leader: reassigning master") #TODO: what about the first time?all responding but no master => por esto el flag adicional y eso. # setear el master como slave y agarrar un nodo vivo logging.info("Storage settings right now: {}".format( self.storage_roles)) current_time = time.time() for storage_node in self.storage_roles.keys(): self.storage_roles[storage_node] = SLAVE_ROLE if current_time - self.last_timeout[ storage_node] < HEARTBEAT_TIMEOUT: new_master = storage_node logging.info( "Watchdog Leader: new master should be {}, current role is {}". format(new_master, self.storage_roles[new_master])) self.storage_exchange.publish("{},{}".format( NEW_MASTER_MSG, new_master.split("_")[-1] # PID )) # wait until node has processed the message and changed roles while self.storage_roles[new_master] == SLAVE_ROLE: logging.info( "Watchdog Leader: waiting for {} to change role".format( new_master)) time.sleep(POLL_INTERVAL) logging.info( "Watchdog Leader: {} has changed role to {}, moving away".format( new_master, self.storage_roles[new_master])) logging.info("Storage settings after reassigning: {}".format( self.storage_roles)) self.reassigning = False def launch_checker(self): i = 0 while True: i = (i + 1) % 5 current_time = time.time() for hostname, last_heartbeat in self.last_timeout.items(): if current_time - last_heartbeat > HEARTBEAT_TIMEOUT and not self.respawning[ hostname]: logging.info( "Detected timeout of hostname {}".format(hostname)) self.respawning[hostname] = True self.to_stop.append( hostname) # another thread reads this queue if current_time - self.storage_master_heartbeat > HEARTBEAT_TIMEOUT and not self.reassigning: self.reassigning = True reassign_thread = threading.Thread( target=self.reassign_storage_master) reassign_thread.start() if i == 1: logging.info("Storage settings right now: {}".format( self.storage_roles)) time.sleep(CHECK_INTERVAL)