class PercentageCalculator:
def __init__(self):
self.left = None
self.right = None
self.in_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE,
consumer=True,
exclusive=True)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct')
def run(self):
self.in_queue.consume(self.calculate)
def calculate(self, ch, method, properties, body):
logging.info('Received %r' % body)
[hand, amount, total] = body.decode().split(',')
if hand == RIGHT:
self.right = float(amount)
if self.left is None:
return
if hand == NO_RIGHT:
self.left = float(amount)
if self.right is None:
return
right_percentage = 100 * self.right / (self.left + self.right)
left_percentage = 100 - right_percentage
right_response = 'R Victories: {}%'.format(right_percentage)
left_response = 'L Victories: {}%'.format(left_percentage)
self.out_queue.publish(right_response, ROUTING_KEY)
self.out_queue.publish(left_response, ROUTING_KEY)
self.out_queue.publish(END, ROUTING_KEY)
self.in_queue.cancel()
class DifferentHandsFilter:
def __init__(self):
self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE,
consumer=True,
queue_name=JOINED_QUEUE)
self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE,
exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self):
self.in_queue.consume(self.filter)
def filter(self, ch, method, properties, body):
logging.info('Received %r' % body)
if body == END_ENCODED:
self.terminator_queue.publish(END)
return
if body == CLOSE_ENCODED:
self.terminator_queue.publish(OK)
self.in_queue.cancel()
return
data = body.decode().split(',')
winner_hand = data[3]
loser_hand = data[7]
if winner_hand in HANDS and loser_hand != winner_hand:
self.out_queue.publish('1', winner_hand)
logging.info('Sent 1 to %s accumulator' % winner_hand)
def __init__(self):
self.count = 0
self.in_queue = RabbitMQQueue(exchange=AVERAGE_CALCULATOR_EXCHANGE,
consumer=True,
exclusive=True)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct')
class SurfaceDispatcher:
def __init__(self):
self.in_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE,
consumer=True,
queue_name=MATCHES_QUEUE)
self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE,
exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self):
self.in_queue.consume(self.dispatch)
def dispatch(self, ch, method, properties, body):
logging.info('Received %r' % body)
if body == END_ENCODED:
self.terminator_queue.publish(END)
return
if body == CLOSE_ENCODED:
self.terminator_queue.publish(OK)
self.in_queue.cancel()
return
data = body.decode().split(',')
surface = data[3]
minutes = data[9]
if minutes == '' or surface in ('', 'None'):
return
self.out_queue.publish(minutes, surface)
logging.info('Sent %s minutes to %s accumulator' % (minutes, surface))
def __init__(self):
self.hands = {}
self.in_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE,
consumer=True,
exclusive=True)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct')
class AgeDifferenceFilter:
def __init__(self):
self.in_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE, consumer=True,
queue_name=AGE_DIFFERENCE_FILTER_QUEUE)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self):
self.in_queue.consume(self.filter)
def filter(self, ch, method, properties, body):
logging.info('Received %r' % body)
if body == END_ENCODED:
self.terminator_queue.publish(END)
return
if body == CLOSE_ENCODED:
self.terminator_queue.publish(OK)
self.in_queue.cancel()
return
data = body.decode().split(',')
winner_age = int(data[4])
loser_age = int(data[8])
if winner_age - loser_age >= 20:
winner_name = ' '.join([data[1], data[2]])
loser_name = ' '.join([data[5], data[6]])
result = '{}\t{}\t{}\t{}'.format(winner_age, winner_name, loser_age, loser_name)
self.out_queue.publish(result, ROUTING_KEY)
logging.info('Sent %s' % result)
def persist(self, ch, method, properties, body):
logging.info('Received %r from %s' % (body, method.routing_key))
data = body.decode().split(',')
id = data[0]
result = data[1]
if result == END:
self.count[id] = self.count.get(id, 0) + 1
cmd = ';'.join(['WRITE', self.hostname, str(self.count[id]), id])
self.storage_queue.publish(cmd)
if self.count[id] != 3:
ch.basic_ack(delivery_tag=method.delivery_tag)
return
for filename in FILES:
try:
file = open(filename + id, 'r')
response = file.read()
file.close()
except FileNotFoundError:
response = '%s: No results' % filename
out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' +
id)
out_queue.publish(response)
logging.info('Sent %s' % response)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
file = open(method.routing_key + id, 'a+')
file.write(result + '\n')
file.close()
ch.basic_ack(delivery_tag=method.delivery_tag)
def __init__(self):
self.count = 0
self.in_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct',
consumer=True,
exclusive=True,
routing_keys=FILES)
self.out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE)
def __init__(self, argv):
self.id = str(uuid1())
self.metadata = [FROM_DEFAULT, TO_DEFAULT, self.id]
self.parse_args(argv)
self.results = 0
self.in_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' +
self.id,
consumer=True,
exclusive=True)
self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE)
def __init__(self, routing_key, exchange, output_exchange):
self.routing_key = routing_key
self.total = 0
self.amount = 0.0
self.in_queue = RabbitMQQueue(exchange=exchange,
exchange_type='direct',
consumer=True,
exclusive=True,
routing_keys=routing_key.split('-'))
self.out_queue = RabbitMQQueue(exchange=output_exchange)
def __init__(self, hostname, metadata, callback):
self.hostname = hostname
self.metadata = metadata
self.callback = callback
self.exchange = RabbitMQQueue(exchange=HEARTBEAT_EXCHANGE,
consumer=False,
exchange_type="fanout",
durable=False)
logging.basicConfig(format='%(asctime)s [PID {}] %(message)s'.format(
self.hostname),
level=logging.INFO)
logging.info(
"Instancing heartbeat process for hostname {}".format(hostname))
class Terminator:
def __init__(self, processes_number, in_exchange, group_exchange,
next_exchange, next_exchange_type, next_routing_keys):
self.processes_number = processes_number
self.next_routing_keys = next_routing_keys
self.closed = 0
self.in_queue = RabbitMQQueue(exchange=in_exchange,
consumer=True,
exclusive=True)
self.group_queue = RabbitMQQueue(exchange=group_exchange)
self.next_queue = RabbitMQQueue(exchange=next_exchange,
exchange_type=next_exchange_type)
def run(self):
self.in_queue.consume(self.close)
def close(self, ch, method, properties, body):
if body == END_ENCODED:
for i in range(self.processes_number):
self.group_queue.publish(CLOSE)
return
if body == OK_ENCODED:
self.closed += 1
if self.closed == self.processes_number:
for routing_key in self.next_routing_keys.split('-'):
self.next_queue.publish(END, routing_key)
self.in_queue.cancel()
class SurfaceDispatcher:
def __init__(self):
self.acked = set()
self.in_queue = RabbitMQQueue(exchange=FILTERED_EXCHANGE,
exchange_type='direct',
consumer=True,
queue_name=FILTERED_QUEUE,
routing_keys=['dispatcher'])
self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE,
exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self, _):
self.in_queue.consume(self.dispatch)
def dispatch(self, ch, method, properties, body):
logging.info('Received %r' % body)
data = body.decode().split(',')
id = data[0]
if data[1] == END:
self.terminator_queue.publish(body)
logging.info('Sent %r' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if data[1] == CLOSE:
if not id in self.acked:
body = ','.join([id, OK])
self.terminator_queue.publish(body)
self.acked.add(id)
else:
self.in_queue.publish(body)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
surface = data[4]
minutes = data[10]
if minutes == '' or surface in ('', 'None'):
ch.basic_ack(delivery_tag=method.delivery_tag)
return
body = ','.join([id, minutes])
self.out_queue.publish(body, surface)
logging.info('Sent %s to %s accumulator' % (body, surface))
ch.basic_ack(delivery_tag=method.delivery_tag)
class AgeCalculator:
def __init__(self):
self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE,
consumer=True,
queue_name=AGE_CALCULATOR_QUEUE)
self.out_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE)
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self):
self.in_queue.consume(self.calculate)
def calculate(self, ch, method, properties, body):
logging.info('Received %r' % body)
if body == END_ENCODED:
self.terminator_queue.publish(END)
return
if body == CLOSE_ENCODED:
self.terminator_queue.publish(OK)
self.in_queue.cancel()
return
data = body.decode().split(',')
tourney_date = data[0]
winner_birthdate = data[4]
loser_birthdate = data[8]
if winner_birthdate == '' or loser_birthdate == '':
return
tourney_date = datetime.strptime(tourney_date, '%Y%m%d')
winner_age = self._compute_age(
datetime.strptime(winner_birthdate, '%Y%m%d'), tourney_date)
loser_age = self._compute_age(
datetime.strptime(loser_birthdate, '%Y%m%d'), tourney_date)
data[4] = str(winner_age)
data[8] = str(loser_age)
body = ','.join(data)
self.out_queue.publish(body)
logging.info('Sent %s' % body)
def _compute_age(self, birthdate, tourney_date):
years = tourney_date.year - birthdate.year
if tourney_date.month < birthdate.month or \
(tourney_date.month == birthdate.month and tourney_date.day < birthdate.day):
years -= 1
return years
def __init__(self):
self.players = {}
self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE,
consumer=True,
queue_name=MATCHES_QUEUE)
self.players_queue = RabbitMQQueue(exchange=PLAYERS_EXCHANGE,
consumer=True,
exclusive=True)
self.out_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE)
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def __init__(self, routing_key, exchange, output_exchange):
self.hostname = os.environ['HOSTNAME']
self.routing_key = routing_key
self.values = {}
self.in_queue = RabbitMQQueue(exchange=exchange, exchange_type='direct',
consumer=True, queue_name='{}_queue'.format(self.hostname),
routing_keys=routing_key.split('-'))
self.out_queue = RabbitMQQueue(exchange=output_exchange)
self.storage_queue = RabbitMQQueue(exchange='storage_input')
self.data_queue = RabbitMQQueue(exchange='storage_output', exchange_type='direct',
consumer=True, exclusive=True,
routing_keys=[self.hostname])
class AgeDifferenceFilter:
def __init__(self):
self.acked = set()
self.in_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE, consumer=True,
queue_name=AGE_DIFFERENCE_FILTER_QUEUE)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self, _):
self.in_queue.consume(self.filter)
def filter(self, ch, method, properties, body):
logging.info('Received %r' % body)
data = body.decode().split(',')
id = data[0]
if data[1] == END:
self.terminator_queue.publish(body)
logging.info('Sent %r' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if data[1] == CLOSE:
if not id in self.acked:
body = ','.join([id, OK])
self.terminator_queue.publish(body)
self.acked.add(id)
else:
self.in_queue.publish(body)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
winner_age = int(data[5])
loser_age = int(data[9])
if winner_age - loser_age >= 20:
winner_name = ' '.join([data[2], data[3]])
loser_name = ' '.join([data[6], data[7]])
result = '{}\t{}\t{}\t{}'.format(winner_age, winner_name, loser_age, loser_name)
body = ','.join([id, result])
self.out_queue.publish(body, ROUTING_KEY)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
class DifferentHandsFilter:
def __init__(self):
self.acked = set()
self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, exchange_type='direct',
consumer=True, queue_name=JOINED_QUEUE,
routing_keys=['filter'])
self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self, _):
self.in_queue.consume(self.filter)
def filter(self, ch, method, properties, body):
logging.info('Received %r' % body)
data = body.decode().split(',')
id = data[0]
if data[1] == END:
self.terminator_queue.publish(body)
logging.info('Sent %r' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if data[1] == CLOSE:
if not id in self.acked:
body = ','.join([data[0], OK])
self.terminator_queue.publish(body)
self.acked.add(id)
else:
self.in_queue.publish(body)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
winner_hand = data[4]
loser_hand = data[8]
if winner_hand in HANDS and loser_hand != winner_hand:
body = ','.join([id, '1'])
self.out_queue.publish(body, winner_hand)
logging.info('Sent %s to %s accumulator' % (body, winner_hand))
ch.basic_ack(delivery_tag=method.delivery_tag)
def __init__(self):
self.acked = set()
self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, exchange_type='direct',
consumer=True, queue_name=JOINED_QUEUE,
routing_keys=['filter'])
self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
class Accumulator:
def __init__(self, routing_key, exchange, output_exchange):
self.routing_key = routing_key
self.total = 0
self.amount = 0.0
self.in_queue = RabbitMQQueue(exchange=exchange,
exchange_type='direct',
consumer=True,
exclusive=True,
routing_keys=routing_key.split('-'))
self.out_queue = RabbitMQQueue(exchange=output_exchange)
def run(self):
self.in_queue.consume(self.add)
def add(self, ch, method, properties, body):
logging.info('Received %r' % body)
if body == END_ENCODED:
body = ','.join(
[self.routing_key,
str(self.amount),
str(self.total)])
self.out_queue.publish(body)
self.in_queue.cancel()
return
self.total += float(body.decode())
self.amount += 1
logging.debug('Current total: %f' % self.total)
logging.debug('Current amount: %f' % self.amount)
class AverageCalculator:
def __init__(self):
self.count = {}
self.in_queue = RabbitMQQueue(exchange=AVERAGE_CALCULATOR_EXCHANGE,
consumer=True,
exclusive=True)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct')
def run(self, _):
self.in_queue.consume(self.calculate)
def calculate(self, ch, method, properties, body):
logging.info('Received %r' % body)
[id, surface, amount, total] = body.decode().split(',')
avg = float(total) / float(amount)
result = '{}: {} minutes'.format(surface, avg)
body = ','.join([id, result])
self.out_queue.publish(body, ROUTING_KEY)
logging.info('Sent %s' % body)
self.count[id] = self.count.get(id, 0) + 1
if self.count[id] == 3:
end = ','.join([id, END])
self.out_queue.publish(end, ROUTING_KEY)
logging.info('Sent %s' % end)
ch.basic_ack(delivery_tag=method.delivery_tag)
def __init__(self):
self.in_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE,
consumer=True,
queue_name=MATCHES_QUEUE)
self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE,
exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
class Database:
def __init__(self):
self.count = 0
self.in_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct',
consumer=True,
exclusive=True,
routing_keys=FILES)
self.out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE)
def run(self):
self.in_queue.consume(self.persist)
def persist(self, ch, method, properties, body):
logging.info('Received %r' % body)
result = body.decode()
if result == END:
self.count += 1
if self.count != 3:
return
for filename in FILES:
file = open(filename, 'r')
response = file.read()
file.close()
self.out_queue.publish(response)
self.in_queue.cancel()
return
file = open(method.routing_key, 'a+')
file.write(result + '\n')
file.close()
def __init__(self):
self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE,
consumer=True,
queue_name=JOINED_QUEUE)
self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE,
exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def setup_queues(self):
self.exchange = RabbitMQQueue(exchange=ELECTION_EXCHANGE,
consumer=False,
exchange_type="direct")
self.process_queue = RabbitMQQueue(exchange=ELECTION_EXCHANGE,
consumer=True,
exchange_type="direct",
routing_keys=[str(self.pid)])
self.leader_ex = RabbitMQQueue(exchange=LEADER_EXCHANGE,
consumer=False,
durable=False)
self.leader_queue = RabbitMQQueue(exchange=LEADER_EXCHANGE,
consumer=True,
durable=False)
class Client:
def __init__(self, argv):
self.id = str(uuid1())
self.metadata = [FROM_DEFAULT, TO_DEFAULT, self.id]
self.parse_args(argv)
self.results = 0
self.in_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' +
self.id,
consumer=True,
exclusive=True)
self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE)
def parse_args(self, argv):
try:
options, args = getopt.getopt(argv, "f:t:", ["from=", "to="])
except getopt.GetoptError:
print("Usage: python3 client.py [--from=YYYYMMDD] [--to=YYYYMMDD]")
sys.exit(2)
for option, arg in options:
if option in ("-f", "--from"):
self.metadata[0] = arg
else:
self.metadata[1] = arg
def run(self):
self.send_matches_data()
self.in_queue.consume(self.print_response)
def send_matches_data(self):
for filename in glob(MATCHES_DATA):
with open(filename, 'r') as file:
file.readline()
for line in iter(file.readline, ''):
body = ','.join(self.metadata) + ',' + line
self.matches_queue.publish(body)
logging.info('Sent %s' % body)
end = ','.join([self.id, END])
self.matches_queue.publish(end)
logging.info('Sent %s' % end)
def print_response(self, ch, method, properties, body):
print(body.decode())
self.results += 1
ch.basic_ack(delivery_tag=method.delivery_tag)
if self.results == 3:
self.in_queue.cancel()
class PercentageCalculator:
def __init__(self):
self.hands = {}
self.in_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE,
consumer=True,
exclusive=True)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct')
def run(self, _):
self.in_queue.consume(self.calculate)
def calculate(self, ch, method, properties, body):
logging.info('Received %r' % body)
[id, hand, amount, total] = body.decode().split(',')
if not id in self.hands:
self.hands[id] = [None, None]
left = self.hands[id][1]
if hand == RIGHT:
self.hands[id][0] = float(amount)
if self.hands[id][1] is None:
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if hand == NO_RIGHT:
self.hands[id][1] = float(amount)
if self.hands[id][0] is None:
ch.basic_ack(delivery_tag=method.delivery_tag)
return
right_percentage = 100 * self.hands[id][0] / (self.hands[id][0] +
self.hands[id][1])
left_percentage = 100 - right_percentage
right_response = 'R Victories: {}%'.format(right_percentage)
left_response = 'L Victories: {}%'.format(left_percentage)
body = ','.join([id, right_response])
self.out_queue.publish(body, ROUTING_KEY)
logging.info('Sent %s' % body)
body = ','.join([id, left_response])
self.out_queue.publish(body, ROUTING_KEY)
logging.info('Sent %s' % body)
body = ','.join([id, END])
self.out_queue.publish(body, ROUTING_KEY)
ch.basic_ack(delivery_tag=method.delivery_tag)
def __init__(self):
self.acked = set()
self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE,
exchange_type='direct',
consumer=True,
queue_name=AGE_CALCULATOR_QUEUE,
routing_keys=['calculator'])
self.out_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE)
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def __init__(self, pid):
self.pid = pid
self.role = SLAVE_ROLE
self.input_queue = RabbitMQQueue(
exchange='storage_slave',
consumer=True,
queue_name='slave{}_queue'.format(pid))
self.output_queue = RabbitMQQueue(exchange='storage_output',
exchange_type='direct')
self.master_queue = RabbitMQQueue(exchange='storage_input',
consumer=True,
queue_name='master_queue')
self.instance_queue = RabbitMQQueue(
exchange='storage_internal_{}'.format(pid),
consumer=True,
queue_name='storage_internal_{}'.format(pid))
self.heartbeatproc = None
def __init__(self):
self.acked = set()
self.in_queue = RabbitMQQueue(exchange=FILTERED_EXCHANGE,
exchange_type='direct',
consumer=True,
queue_name=FILTERED_QUEUE,
routing_keys=['dispatcher'])
self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE,
exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
