class PercentageCalculator: def __init__(self): self.left = None self.right = None self.in_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') def run(self): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) [hand, amount, total] = body.decode().split(',') if hand == RIGHT: self.right = float(amount) if self.left is None: return if hand == NO_RIGHT: self.left = float(amount) if self.right is None: return right_percentage = 100 * self.right / (self.left + self.right) left_percentage = 100 - right_percentage right_response = 'R Victories: {}%'.format(right_percentage) left_response = 'L Victories: {}%'.format(left_percentage) self.out_queue.publish(right_response, ROUTING_KEY) self.out_queue.publish(left_response, ROUTING_KEY) self.out_queue.publish(END, ROUTING_KEY) self.in_queue.cancel()
class DifferentHandsFilter: def __init__(self): self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, consumer=True, queue_name=JOINED_QUEUE) self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.in_queue.consume(self.filter) def filter(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.in_queue.cancel() return data = body.decode().split(',') winner_hand = data[3] loser_hand = data[7] if winner_hand in HANDS and loser_hand != winner_hand: self.out_queue.publish('1', winner_hand) logging.info('Sent 1 to %s accumulator' % winner_hand)
def __init__(self): self.count = 0 self.in_queue = RabbitMQQueue(exchange=AVERAGE_CALCULATOR_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct')
class SurfaceDispatcher: def __init__(self): self.in_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE, consumer=True, queue_name=MATCHES_QUEUE) self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.in_queue.consume(self.dispatch) def dispatch(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.in_queue.cancel() return data = body.decode().split(',') surface = data[3] minutes = data[9] if minutes == '' or surface in ('', 'None'): return self.out_queue.publish(minutes, surface) logging.info('Sent %s minutes to %s accumulator' % (minutes, surface))
def __init__(self): self.hands = {} self.in_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct')
class AgeDifferenceFilter: def __init__(self): self.in_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE, consumer=True, queue_name=AGE_DIFFERENCE_FILTER_QUEUE) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.in_queue.consume(self.filter) def filter(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.in_queue.cancel() return data = body.decode().split(',') winner_age = int(data[4]) loser_age = int(data[8]) if winner_age - loser_age >= 20: winner_name = ' '.join([data[1], data[2]]) loser_name = ' '.join([data[5], data[6]]) result = '{}\t{}\t{}\t{}'.format(winner_age, winner_name, loser_age, loser_name) self.out_queue.publish(result, ROUTING_KEY) logging.info('Sent %s' % result)
def persist(self, ch, method, properties, body): logging.info('Received %r from %s' % (body, method.routing_key)) data = body.decode().split(',') id = data[0] result = data[1] if result == END: self.count[id] = self.count.get(id, 0) + 1 cmd = ';'.join(['WRITE', self.hostname, str(self.count[id]), id]) self.storage_queue.publish(cmd) if self.count[id] != 3: ch.basic_ack(delivery_tag=method.delivery_tag) return for filename in FILES: try: file = open(filename + id, 'r') response = file.read() file.close() except FileNotFoundError: response = '%s: No results' % filename out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' + id) out_queue.publish(response) logging.info('Sent %s' % response) ch.basic_ack(delivery_tag=method.delivery_tag) return file = open(method.routing_key + id, 'a+') file.write(result + '\n') file.close() ch.basic_ack(delivery_tag=method.delivery_tag)
def __init__(self): self.count = 0 self.in_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct', consumer=True, exclusive=True, routing_keys=FILES) self.out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE)
def __init__(self, argv): self.id = str(uuid1()) self.metadata = [FROM_DEFAULT, TO_DEFAULT, self.id] self.parse_args(argv) self.results = 0 self.in_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' + self.id, consumer=True, exclusive=True) self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE)
def __init__(self, routing_key, exchange, output_exchange): self.routing_key = routing_key self.total = 0 self.amount = 0.0 self.in_queue = RabbitMQQueue(exchange=exchange, exchange_type='direct', consumer=True, exclusive=True, routing_keys=routing_key.split('-')) self.out_queue = RabbitMQQueue(exchange=output_exchange)
def __init__(self, hostname, metadata, callback): self.hostname = hostname self.metadata = metadata self.callback = callback self.exchange = RabbitMQQueue(exchange=HEARTBEAT_EXCHANGE, consumer=False, exchange_type="fanout", durable=False) logging.basicConfig(format='%(asctime)s [PID {}] %(message)s'.format( self.hostname), level=logging.INFO) logging.info( "Instancing heartbeat process for hostname {}".format(hostname))
class Terminator: def __init__(self, processes_number, in_exchange, group_exchange, next_exchange, next_exchange_type, next_routing_keys): self.processes_number = processes_number self.next_routing_keys = next_routing_keys self.closed = 0 self.in_queue = RabbitMQQueue(exchange=in_exchange, consumer=True, exclusive=True) self.group_queue = RabbitMQQueue(exchange=group_exchange) self.next_queue = RabbitMQQueue(exchange=next_exchange, exchange_type=next_exchange_type) def run(self): self.in_queue.consume(self.close) def close(self, ch, method, properties, body): if body == END_ENCODED: for i in range(self.processes_number): self.group_queue.publish(CLOSE) return if body == OK_ENCODED: self.closed += 1 if self.closed == self.processes_number: for routing_key in self.next_routing_keys.split('-'): self.next_queue.publish(END, routing_key) self.in_queue.cancel()
class SurfaceDispatcher: def __init__(self): self.acked = set() self.in_queue = RabbitMQQueue(exchange=FILTERED_EXCHANGE, exchange_type='direct', consumer=True, queue_name=FILTERED_QUEUE, routing_keys=['dispatcher']) self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self, _): self.in_queue.consume(self.dispatch) def dispatch(self, ch, method, properties, body): logging.info('Received %r' % body) data = body.decode().split(',') id = data[0] if data[1] == END: self.terminator_queue.publish(body) logging.info('Sent %r' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return if data[1] == CLOSE: if not id in self.acked: body = ','.join([id, OK]) self.terminator_queue.publish(body) self.acked.add(id) else: self.in_queue.publish(body) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return surface = data[4] minutes = data[10] if minutes == '' or surface in ('', 'None'): ch.basic_ack(delivery_tag=method.delivery_tag) return body = ','.join([id, minutes]) self.out_queue.publish(body, surface) logging.info('Sent %s to %s accumulator' % (body, surface)) ch.basic_ack(delivery_tag=method.delivery_tag)
class AgeCalculator: def __init__(self): self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, consumer=True, queue_name=AGE_CALCULATOR_QUEUE) self.out_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE) self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: self.terminator_queue.publish(END) return if body == CLOSE_ENCODED: self.terminator_queue.publish(OK) self.in_queue.cancel() return data = body.decode().split(',') tourney_date = data[0] winner_birthdate = data[4] loser_birthdate = data[8] if winner_birthdate == '' or loser_birthdate == '': return tourney_date = datetime.strptime(tourney_date, '%Y%m%d') winner_age = self._compute_age( datetime.strptime(winner_birthdate, '%Y%m%d'), tourney_date) loser_age = self._compute_age( datetime.strptime(loser_birthdate, '%Y%m%d'), tourney_date) data[4] = str(winner_age) data[8] = str(loser_age) body = ','.join(data) self.out_queue.publish(body) logging.info('Sent %s' % body) def _compute_age(self, birthdate, tourney_date): years = tourney_date.year - birthdate.year if tourney_date.month < birthdate.month or \ (tourney_date.month == birthdate.month and tourney_date.day < birthdate.day): years -= 1 return years
def __init__(self): self.players = {} self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE, consumer=True, queue_name=MATCHES_QUEUE) self.players_queue = RabbitMQQueue(exchange=PLAYERS_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE) self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def __init__(self, routing_key, exchange, output_exchange): self.hostname = os.environ['HOSTNAME'] self.routing_key = routing_key self.values = {} self.in_queue = RabbitMQQueue(exchange=exchange, exchange_type='direct', consumer=True, queue_name='{}_queue'.format(self.hostname), routing_keys=routing_key.split('-')) self.out_queue = RabbitMQQueue(exchange=output_exchange) self.storage_queue = RabbitMQQueue(exchange='storage_input') self.data_queue = RabbitMQQueue(exchange='storage_output', exchange_type='direct', consumer=True, exclusive=True, routing_keys=[self.hostname])
class AgeDifferenceFilter: def __init__(self): self.acked = set() self.in_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE, consumer=True, queue_name=AGE_DIFFERENCE_FILTER_QUEUE) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self, _): self.in_queue.consume(self.filter) def filter(self, ch, method, properties, body): logging.info('Received %r' % body) data = body.decode().split(',') id = data[0] if data[1] == END: self.terminator_queue.publish(body) logging.info('Sent %r' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return if data[1] == CLOSE: if not id in self.acked: body = ','.join([id, OK]) self.terminator_queue.publish(body) self.acked.add(id) else: self.in_queue.publish(body) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return winner_age = int(data[5]) loser_age = int(data[9]) if winner_age - loser_age >= 20: winner_name = ' '.join([data[2], data[3]]) loser_name = ' '.join([data[6], data[7]]) result = '{}\t{}\t{}\t{}'.format(winner_age, winner_name, loser_age, loser_name) body = ','.join([id, result]) self.out_queue.publish(body, ROUTING_KEY) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag)
class DifferentHandsFilter: def __init__(self): self.acked = set() self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, exchange_type='direct', consumer=True, queue_name=JOINED_QUEUE, routing_keys=['filter']) self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE) def run(self, _): self.in_queue.consume(self.filter) def filter(self, ch, method, properties, body): logging.info('Received %r' % body) data = body.decode().split(',') id = data[0] if data[1] == END: self.terminator_queue.publish(body) logging.info('Sent %r' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return if data[1] == CLOSE: if not id in self.acked: body = ','.join([data[0], OK]) self.terminator_queue.publish(body) self.acked.add(id) else: self.in_queue.publish(body) logging.info('Sent %s' % body) ch.basic_ack(delivery_tag=method.delivery_tag) return winner_hand = data[4] loser_hand = data[8] if winner_hand in HANDS and loser_hand != winner_hand: body = ','.join([id, '1']) self.out_queue.publish(body, winner_hand) logging.info('Sent %s to %s accumulator' % (body, winner_hand)) ch.basic_ack(delivery_tag=method.delivery_tag)
def __init__(self): self.acked = set() self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, exchange_type='direct', consumer=True, queue_name=JOINED_QUEUE, routing_keys=['filter']) self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
class Accumulator: def __init__(self, routing_key, exchange, output_exchange): self.routing_key = routing_key self.total = 0 self.amount = 0.0 self.in_queue = RabbitMQQueue(exchange=exchange, exchange_type='direct', consumer=True, exclusive=True, routing_keys=routing_key.split('-')) self.out_queue = RabbitMQQueue(exchange=output_exchange) def run(self): self.in_queue.consume(self.add) def add(self, ch, method, properties, body): logging.info('Received %r' % body) if body == END_ENCODED: body = ','.join( [self.routing_key, str(self.amount), str(self.total)]) self.out_queue.publish(body) self.in_queue.cancel() return self.total += float(body.decode()) self.amount += 1 logging.debug('Current total: %f' % self.total) logging.debug('Current amount: %f' % self.amount)
class AverageCalculator: def __init__(self): self.count = {} self.in_queue = RabbitMQQueue(exchange=AVERAGE_CALCULATOR_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') def run(self, _): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) [id, surface, amount, total] = body.decode().split(',') avg = float(total) / float(amount) result = '{}: {} minutes'.format(surface, avg) body = ','.join([id, result]) self.out_queue.publish(body, ROUTING_KEY) logging.info('Sent %s' % body) self.count[id] = self.count.get(id, 0) + 1 if self.count[id] == 3: end = ','.join([id, END]) self.out_queue.publish(end, ROUTING_KEY) logging.info('Sent %s' % end) ch.basic_ack(delivery_tag=method.delivery_tag)
def __init__(self): self.in_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE, consumer=True, queue_name=MATCHES_QUEUE) self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
class Database: def __init__(self): self.count = 0 self.in_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct', consumer=True, exclusive=True, routing_keys=FILES) self.out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE) def run(self): self.in_queue.consume(self.persist) def persist(self, ch, method, properties, body): logging.info('Received %r' % body) result = body.decode() if result == END: self.count += 1 if self.count != 3: return for filename in FILES: file = open(filename, 'r') response = file.read() file.close() self.out_queue.publish(response) self.in_queue.cancel() return file = open(method.routing_key, 'a+') file.write(result + '\n') file.close()
def __init__(self): self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, consumer=True, queue_name=JOINED_QUEUE) self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def setup_queues(self): self.exchange = RabbitMQQueue(exchange=ELECTION_EXCHANGE, consumer=False, exchange_type="direct") self.process_queue = RabbitMQQueue(exchange=ELECTION_EXCHANGE, consumer=True, exchange_type="direct", routing_keys=[str(self.pid)]) self.leader_ex = RabbitMQQueue(exchange=LEADER_EXCHANGE, consumer=False, durable=False) self.leader_queue = RabbitMQQueue(exchange=LEADER_EXCHANGE, consumer=True, durable=False)
class Client: def __init__(self, argv): self.id = str(uuid1()) self.metadata = [FROM_DEFAULT, TO_DEFAULT, self.id] self.parse_args(argv) self.results = 0 self.in_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' + self.id, consumer=True, exclusive=True) self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE) def parse_args(self, argv): try: options, args = getopt.getopt(argv, "f:t:", ["from=", "to="]) except getopt.GetoptError: print("Usage: python3 client.py [--from=YYYYMMDD] [--to=YYYYMMDD]") sys.exit(2) for option, arg in options: if option in ("-f", "--from"): self.metadata[0] = arg else: self.metadata[1] = arg def run(self): self.send_matches_data() self.in_queue.consume(self.print_response) def send_matches_data(self): for filename in glob(MATCHES_DATA): with open(filename, 'r') as file: file.readline() for line in iter(file.readline, ''): body = ','.join(self.metadata) + ',' + line self.matches_queue.publish(body) logging.info('Sent %s' % body) end = ','.join([self.id, END]) self.matches_queue.publish(end) logging.info('Sent %s' % end) def print_response(self, ch, method, properties, body): print(body.decode()) self.results += 1 ch.basic_ack(delivery_tag=method.delivery_tag) if self.results == 3: self.in_queue.cancel()
class PercentageCalculator: def __init__(self): self.hands = {} self.in_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, consumer=True, exclusive=True) self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct') def run(self, _): self.in_queue.consume(self.calculate) def calculate(self, ch, method, properties, body): logging.info('Received %r' % body) [id, hand, amount, total] = body.decode().split(',') if not id in self.hands: self.hands[id] = [None, None] left = self.hands[id][1] if hand == RIGHT: self.hands[id][0] = float(amount) if self.hands[id][1] is None: ch.basic_ack(delivery_tag=method.delivery_tag) return if hand == NO_RIGHT: self.hands[id][1] = float(amount) if self.hands[id][0] is None: ch.basic_ack(delivery_tag=method.delivery_tag) return right_percentage = 100 * self.hands[id][0] / (self.hands[id][0] + self.hands[id][1]) left_percentage = 100 - right_percentage right_response = 'R Victories: {}%'.format(right_percentage) left_response = 'L Victories: {}%'.format(left_percentage) body = ','.join([id, right_response]) self.out_queue.publish(body, ROUTING_KEY) logging.info('Sent %s' % body) body = ','.join([id, left_response]) self.out_queue.publish(body, ROUTING_KEY) logging.info('Sent %s' % body) body = ','.join([id, END]) self.out_queue.publish(body, ROUTING_KEY) ch.basic_ack(delivery_tag=method.delivery_tag)
def __init__(self): self.acked = set() self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, exchange_type='direct', consumer=True, queue_name=AGE_CALCULATOR_QUEUE, routing_keys=['calculator']) self.out_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE) self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def __init__(self, pid): self.pid = pid self.role = SLAVE_ROLE self.input_queue = RabbitMQQueue( exchange='storage_slave', consumer=True, queue_name='slave{}_queue'.format(pid)) self.output_queue = RabbitMQQueue(exchange='storage_output', exchange_type='direct') self.master_queue = RabbitMQQueue(exchange='storage_input', consumer=True, queue_name='master_queue') self.instance_queue = RabbitMQQueue( exchange='storage_internal_{}'.format(pid), consumer=True, queue_name='storage_internal_{}'.format(pid)) self.heartbeatproc = None
def __init__(self): self.acked = set() self.in_queue = RabbitMQQueue(exchange=FILTERED_EXCHANGE, exchange_type='direct', consumer=True, queue_name=FILTERED_QUEUE, routing_keys=['dispatcher']) self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE, exchange_type='direct') self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)