class DifferentHandsFilter:
def __init__(self):
self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE,
consumer=True,
queue_name=JOINED_QUEUE)
self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE,
exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self):
self.in_queue.consume(self.filter)
def filter(self, ch, method, properties, body):
logging.info('Received %r' % body)
if body == END_ENCODED:
self.terminator_queue.publish(END)
return
if body == CLOSE_ENCODED:
self.terminator_queue.publish(OK)
self.in_queue.cancel()
return
data = body.decode().split(',')
winner_hand = data[3]
loser_hand = data[7]
if winner_hand in HANDS and loser_hand != winner_hand:
self.out_queue.publish('1', winner_hand)
logging.info('Sent 1 to %s accumulator' % winner_hand)
class Accumulator:
def __init__(self, routing_key, exchange, output_exchange):
self.routing_key = routing_key
self.total = 0
self.amount = 0.0
self.in_queue = RabbitMQQueue(exchange=exchange,
exchange_type='direct',
consumer=True,
exclusive=True,
routing_keys=routing_key.split('-'))
self.out_queue = RabbitMQQueue(exchange=output_exchange)
def run(self):
self.in_queue.consume(self.add)
def add(self, ch, method, properties, body):
logging.info('Received %r' % body)
if body == END_ENCODED:
body = ','.join(
[self.routing_key,
str(self.amount),
str(self.total)])
self.out_queue.publish(body)
self.in_queue.cancel()
return
self.total += float(body.decode())
self.amount += 1
logging.debug('Current total: %f' % self.total)
logging.debug('Current amount: %f' % self.amount)
class Terminator:
def __init__(self, processes_number, in_exchange, group_exchange,
next_exchange, next_exchange_type, next_routing_keys):
self.processes_number = processes_number
self.next_routing_keys = next_routing_keys
self.closed = 0
self.in_queue = RabbitMQQueue(exchange=in_exchange,
consumer=True,
exclusive=True)
self.group_queue = RabbitMQQueue(exchange=group_exchange)
self.next_queue = RabbitMQQueue(exchange=next_exchange,
exchange_type=next_exchange_type)
def run(self):
self.in_queue.consume(self.close)
def close(self, ch, method, properties, body):
if body == END_ENCODED:
for i in range(self.processes_number):
self.group_queue.publish(CLOSE)
return
if body == OK_ENCODED:
self.closed += 1
if self.closed == self.processes_number:
for routing_key in self.next_routing_keys.split('-'):
self.next_queue.publish(END, routing_key)
self.in_queue.cancel()
class Database:
def __init__(self):
self.count = 0
self.in_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct',
consumer=True,
exclusive=True,
routing_keys=FILES)
self.out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE)
def run(self):
self.in_queue.consume(self.persist)
def persist(self, ch, method, properties, body):
logging.info('Received %r' % body)
result = body.decode()
if result == END:
self.count += 1
if self.count != 3:
return
for filename in FILES:
file = open(filename, 'r')
response = file.read()
file.close()
self.out_queue.publish(response)
self.in_queue.cancel()
return
file = open(method.routing_key, 'a+')
file.write(result + '\n')
file.close()
class PercentageCalculator:
def __init__(self):
self.left = None
self.right = None
self.in_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE,
consumer=True,
exclusive=True)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct')
def run(self):
self.in_queue.consume(self.calculate)
def calculate(self, ch, method, properties, body):
logging.info('Received %r' % body)
[hand, amount, total] = body.decode().split(',')
if hand == RIGHT:
self.right = float(amount)
if self.left is None:
return
if hand == NO_RIGHT:
self.left = float(amount)
if self.right is None:
return
right_percentage = 100 * self.right / (self.left + self.right)
left_percentage = 100 - right_percentage
right_response = 'R Victories: {}%'.format(right_percentage)
left_response = 'L Victories: {}%'.format(left_percentage)
self.out_queue.publish(right_response, ROUTING_KEY)
self.out_queue.publish(left_response, ROUTING_KEY)
self.out_queue.publish(END, ROUTING_KEY)
self.in_queue.cancel()
class AgeDifferenceFilter:
def __init__(self):
self.in_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE, consumer=True,
queue_name=AGE_DIFFERENCE_FILTER_QUEUE)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self):
self.in_queue.consume(self.filter)
def filter(self, ch, method, properties, body):
logging.info('Received %r' % body)
if body == END_ENCODED:
self.terminator_queue.publish(END)
return
if body == CLOSE_ENCODED:
self.terminator_queue.publish(OK)
self.in_queue.cancel()
return
data = body.decode().split(',')
winner_age = int(data[4])
loser_age = int(data[8])
if winner_age - loser_age >= 20:
winner_name = ' '.join([data[1], data[2]])
loser_name = ' '.join([data[5], data[6]])
result = '{}\t{}\t{}\t{}'.format(winner_age, winner_name, loser_age, loser_name)
self.out_queue.publish(result, ROUTING_KEY)
logging.info('Sent %s' % result)
class SurfaceDispatcher:
def __init__(self):
self.in_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE,
consumer=True,
queue_name=MATCHES_QUEUE)
self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE,
exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self):
self.in_queue.consume(self.dispatch)
def dispatch(self, ch, method, properties, body):
logging.info('Received %r' % body)
if body == END_ENCODED:
self.terminator_queue.publish(END)
return
if body == CLOSE_ENCODED:
self.terminator_queue.publish(OK)
self.in_queue.cancel()
return
data = body.decode().split(',')
surface = data[3]
minutes = data[9]
if minutes == '' or surface in ('', 'None'):
return
self.out_queue.publish(minutes, surface)
logging.info('Sent %s minutes to %s accumulator' % (minutes, surface))
class AverageCalculator:
def __init__(self):
self.count = {}
self.in_queue = RabbitMQQueue(exchange=AVERAGE_CALCULATOR_EXCHANGE,
consumer=True,
exclusive=True)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct')
def run(self, _):
self.in_queue.consume(self.calculate)
def calculate(self, ch, method, properties, body):
logging.info('Received %r' % body)
[id, surface, amount, total] = body.decode().split(',')
avg = float(total) / float(amount)
result = '{}: {} minutes'.format(surface, avg)
body = ','.join([id, result])
self.out_queue.publish(body, ROUTING_KEY)
logging.info('Sent %s' % body)
self.count[id] = self.count.get(id, 0) + 1
if self.count[id] == 3:
end = ','.join([id, END])
self.out_queue.publish(end, ROUTING_KEY)
logging.info('Sent %s' % end)
ch.basic_ack(delivery_tag=method.delivery_tag)
class Joiner:
def __init__(self):
self.acked = set()
self.players = {}
self.matches_queue = RabbitMQQueue(exchange=FILTERED_EXCHANGE,
exchange_type='direct',
consumer=True,
queue_name=FILTERED_QUEUE,
routing_keys=['joiner'])
self.out_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE,
exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self, _):
self.save_players()
self.matches_queue.consume(self.join)
def save_players(self):
with open(PLAYERS_DATA, 'r') as file:
file.readline()
for line in iter(file.readline, ''):
data = line.split(',')
self.players[data[0]] = data[1:5]
def join(self, ch, method, properties, body):
logging.info('Received %r' % body)
data = body.decode().split(',')
id = data[0]
if data[1] == END:
self.terminator_queue.publish(body)
logging.info('Sent %r' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if data[1] == CLOSE:
if not id in self.acked:
body = ','.join([id, OK])
self.terminator_queue.publish(body)
self.acked.add(id)
else:
self.matches_queue.publish(body)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
winner_id = data[5]
loser_id = data[6]
data = [data[0], data[3]
] + self.players[winner_id] + self.players[loser_id]
body = ','.join(data)
self.out_queue.publish(body, 'filter')
self.out_queue.publish(body, 'calculator')
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
class PercentageCalculator:
def __init__(self):
self.hands = {}
self.in_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE,
consumer=True,
exclusive=True)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct')
def run(self, _):
self.in_queue.consume(self.calculate)
def calculate(self, ch, method, properties, body):
logging.info('Received %r' % body)
[id, hand, amount, total] = body.decode().split(',')
if not id in self.hands:
self.hands[id] = [None, None]
left = self.hands[id][1]
if hand == RIGHT:
self.hands[id][0] = float(amount)
if self.hands[id][1] is None:
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if hand == NO_RIGHT:
self.hands[id][1] = float(amount)
if self.hands[id][0] is None:
ch.basic_ack(delivery_tag=method.delivery_tag)
return
right_percentage = 100 * self.hands[id][0] / (self.hands[id][0] +
self.hands[id][1])
left_percentage = 100 - right_percentage
right_response = 'R Victories: {}%'.format(right_percentage)
left_response = 'L Victories: {}%'.format(left_percentage)
body = ','.join([id, right_response])
self.out_queue.publish(body, ROUTING_KEY)
logging.info('Sent %s' % body)
body = ','.join([id, left_response])
self.out_queue.publish(body, ROUTING_KEY)
logging.info('Sent %s' % body)
body = ','.join([id, END])
self.out_queue.publish(body, ROUTING_KEY)
ch.basic_ack(delivery_tag=method.delivery_tag)
class Client:
def __init__(self, argv):
self.id = str(uuid1())
self.metadata = [FROM_DEFAULT, TO_DEFAULT, self.id]
self.parse_args(argv)
self.results = 0
self.in_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' +
self.id,
consumer=True,
exclusive=True)
self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE)
def parse_args(self, argv):
try:
options, args = getopt.getopt(argv, "f:t:", ["from=", "to="])
except getopt.GetoptError:
print("Usage: python3 client.py [--from=YYYYMMDD] [--to=YYYYMMDD]")
sys.exit(2)
for option, arg in options:
if option in ("-f", "--from"):
self.metadata[0] = arg
else:
self.metadata[1] = arg
def run(self):
self.send_matches_data()
self.in_queue.consume(self.print_response)
def send_matches_data(self):
for filename in glob(MATCHES_DATA):
with open(filename, 'r') as file:
file.readline()
for line in iter(file.readline, ''):
body = ','.join(self.metadata) + ',' + line
self.matches_queue.publish(body)
logging.info('Sent %s' % body)
end = ','.join([self.id, END])
self.matches_queue.publish(end)
logging.info('Sent %s' % end)
def print_response(self, ch, method, properties, body):
print(body.decode())
self.results += 1
ch.basic_ack(delivery_tag=method.delivery_tag)
if self.results == 3:
self.in_queue.cancel()
class SurfaceDispatcher:
def __init__(self):
self.acked = set()
self.in_queue = RabbitMQQueue(exchange=FILTERED_EXCHANGE,
exchange_type='direct',
consumer=True,
queue_name=FILTERED_QUEUE,
routing_keys=['dispatcher'])
self.out_queue = RabbitMQQueue(exchange=SURFACE_EXCHANGE,
exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self, _):
self.in_queue.consume(self.dispatch)
def dispatch(self, ch, method, properties, body):
logging.info('Received %r' % body)
data = body.decode().split(',')
id = data[0]
if data[1] == END:
self.terminator_queue.publish(body)
logging.info('Sent %r' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if data[1] == CLOSE:
if not id in self.acked:
body = ','.join([id, OK])
self.terminator_queue.publish(body)
self.acked.add(id)
else:
self.in_queue.publish(body)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
surface = data[4]
minutes = data[10]
if minutes == '' or surface in ('', 'None'):
ch.basic_ack(delivery_tag=method.delivery_tag)
return
body = ','.join([id, minutes])
self.out_queue.publish(body, surface)
logging.info('Sent %s to %s accumulator' % (body, surface))
ch.basic_ack(delivery_tag=method.delivery_tag)
class AgeCalculator:
def __init__(self):
self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE,
consumer=True,
queue_name=AGE_CALCULATOR_QUEUE)
self.out_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE)
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self):
self.in_queue.consume(self.calculate)
def calculate(self, ch, method, properties, body):
logging.info('Received %r' % body)
if body == END_ENCODED:
self.terminator_queue.publish(END)
return
if body == CLOSE_ENCODED:
self.terminator_queue.publish(OK)
self.in_queue.cancel()
return
data = body.decode().split(',')
tourney_date = data[0]
winner_birthdate = data[4]
loser_birthdate = data[8]
if winner_birthdate == '' or loser_birthdate == '':
return
tourney_date = datetime.strptime(tourney_date, '%Y%m%d')
winner_age = self._compute_age(
datetime.strptime(winner_birthdate, '%Y%m%d'), tourney_date)
loser_age = self._compute_age(
datetime.strptime(loser_birthdate, '%Y%m%d'), tourney_date)
data[4] = str(winner_age)
data[8] = str(loser_age)
body = ','.join(data)
self.out_queue.publish(body)
logging.info('Sent %s' % body)
def _compute_age(self, birthdate, tourney_date):
years = tourney_date.year - birthdate.year
if tourney_date.month < birthdate.month or \
(tourney_date.month == birthdate.month and tourney_date.day < birthdate.day):
years -= 1
return years
class DateFilter:
def __init__(self):
self.acked = set()
self.in_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE, consumer=True,
queue_name=MATCHES_QUEUE)
self.out_queue = RabbitMQQueue(exchange=FILTERED_EXCHANGE, exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self, _):
self.in_queue.consume(self.filter)
def filter(self, ch, method, properties, body):
logging.info('Received %r' % body)
match = body.decode().split(',')
if match[1] == END:
self.terminator_queue.publish(body)
logging.info('Sent %r' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if match[1] == CLOSE:
id = match[0]
if not id in self.acked:
body = ','.join([id, OK])
self.terminator_queue.publish(body)
self.acked.add(id)
else:
self.in_queue.publish(body)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
date_from = match[0]
date_to = match[1]
tourney_date = match[5]
if tourney_date < date_from or tourney_date > date_to:
ch.basic_ack(delivery_tag=method.delivery_tag)
return
data = ','.join(match[2:])
self.out_queue.publish(data, 'joiner')
self.out_queue.publish(data, 'dispatcher')
logging.info('Sent %s' % data)
ch.basic_ack(delivery_tag=method.delivery_tag)
class AgeDifferenceFilter:
def __init__(self):
self.acked = set()
self.in_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE, consumer=True,
queue_name=AGE_DIFFERENCE_FILTER_QUEUE)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE, exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self, _):
self.in_queue.consume(self.filter)
def filter(self, ch, method, properties, body):
logging.info('Received %r' % body)
data = body.decode().split(',')
id = data[0]
if data[1] == END:
self.terminator_queue.publish(body)
logging.info('Sent %r' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if data[1] == CLOSE:
if not id in self.acked:
body = ','.join([id, OK])
self.terminator_queue.publish(body)
self.acked.add(id)
else:
self.in_queue.publish(body)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
winner_age = int(data[5])
loser_age = int(data[9])
if winner_age - loser_age >= 20:
winner_name = ' '.join([data[2], data[3]])
loser_name = ' '.join([data[6], data[7]])
result = '{}\t{}\t{}\t{}'.format(winner_age, winner_name, loser_age, loser_name)
body = ','.join([id, result])
self.out_queue.publish(body, ROUTING_KEY)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
class Joiner:
def __init__(self):
self.players = {}
self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE,
consumer=True,
queue_name=MATCHES_QUEUE)
self.players_queue = RabbitMQQueue(exchange=PLAYERS_EXCHANGE,
consumer=True,
exclusive=True)
self.out_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE)
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self):
self.players_queue.consume(self.save_player)
self.matches_queue.consume(self.join)
def save_player(self, ch, method, properties, body):
logging.info('Received %r' % body)
if body == END_ENCODED:
self.players_queue.cancel()
return
data = body.decode().split(',')
self.players[data[0]] = data[1:5]
def join(self, ch, method, properties, body):
logging.info('Received %r' % body)
if body == END_ENCODED:
self.terminator_queue.publish(END)
return
if body == CLOSE_ENCODED:
self.terminator_queue.publish(OK)
self.matches_queue.cancel()
return
data = body.decode().split(',')
winner_id = data[4]
loser_id = data[5]
data = [data[2]] + self.players[winner_id] + self.players[loser_id]
body = ','.join(data)
self.out_queue.publish(body)
logging.info('Sent %s' % body)
class Terminator:
def __init__(self, processes_number, in_exchange, group_exchange, \
group_exchange_type, group_routing_key, next_exchange, \
next_exchange_type, next_routing_keys):
self.processes_number = processes_number
self.next_routing_keys = next_routing_keys
self.group_routing_key = group_routing_key
self.closed = {}
self.in_queue = RabbitMQQueue(exchange=in_exchange,
consumer=True,
exclusive=True)
self.group_queue = RabbitMQQueue(exchange=group_exchange,
exchange_type=group_exchange_type)
self.next_queue = RabbitMQQueue(exchange=next_exchange,
exchange_type=next_exchange_type)
def run(self, _):
self.in_queue.consume(self.close)
def close(self, ch, method, properties, body):
logging.info('Received %r' % body)
data = body.decode().split(',')
if data[1] == END:
for i in range(self.processes_number):
body = ','.join([data[0], CLOSE])
self.group_queue.publish(body, self.group_routing_key)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if data[1] == OK:
id = data[0]
self.closed[id] = self.closed.get(id, 0) + 1
if self.closed[id] == self.processes_number:
for routing_key in self.next_routing_keys.split('-'):
body = ','.join([id, END])
self.next_queue.publish(body, routing_key)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
class DifferentHandsFilter:
def __init__(self):
self.acked = set()
self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE, exchange_type='direct',
consumer=True, queue_name=JOINED_QUEUE,
routing_keys=['filter'])
self.out_queue = RabbitMQQueue(exchange=HANDS_EXCHANGE, exchange_type='direct')
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self, _):
self.in_queue.consume(self.filter)
def filter(self, ch, method, properties, body):
logging.info('Received %r' % body)
data = body.decode().split(',')
id = data[0]
if data[1] == END:
self.terminator_queue.publish(body)
logging.info('Sent %r' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if data[1] == CLOSE:
if not id in self.acked:
body = ','.join([data[0], OK])
self.terminator_queue.publish(body)
self.acked.add(id)
else:
self.in_queue.publish(body)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
winner_hand = data[4]
loser_hand = data[8]
if winner_hand in HANDS and loser_hand != winner_hand:
body = ','.join([id, '1'])
self.out_queue.publish(body, winner_hand)
logging.info('Sent %s to %s accumulator' % (body, winner_hand))
ch.basic_ack(delivery_tag=method.delivery_tag)
class AverageCalculator:
def __init__(self):
self.count = 0
self.in_queue = RabbitMQQueue(exchange=AVERAGE_CALCULATOR_EXCHANGE,
consumer=True,
exclusive=True)
self.out_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct')
def run(self):
self.in_queue.consume(self.calculate)
def calculate(self, ch, method, properties, body):
logging.info('Received %r' % body)
[surface, amount, total] = body.decode().split(',')
avg = float(total) / float(amount)
result = '{}: {} minutes'.format(surface, avg)
self.out_queue.publish(result, ROUTING_KEY)
self.count += 1
if self.count == 3:
self.out_queue.publish(END, ROUTING_KEY)
self.in_queue.cancel()
logging.info('Sent %s' % result)
class Client:
def __init__(self):
self.results = 0
self.in_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE, consumer=True,
exclusive=True)
self.players_queue = RabbitMQQueue(exchange=PLAYERS_EXCHANGE)
self.matches_queue = RabbitMQQueue(exchange=MATCHES_EXCHANGE)
def run(self):
self.send_players_data()
self.send_matches_data()
self.in_queue.consume(self.print_response)
def send_players_data(self):
with open(PLAYERS_DATA, 'r') as file:
file.readline()
for line in iter(file.readline, ''):
self.players_queue.publish(line)
logging.info('Sent %s' % line)
self.players_queue.publish(END)
def send_matches_data(self):
for filename in glob(MATCHES_DATA):
with open(filename, 'r') as file:
file.readline()
for line in iter(file.readline, ''):
self.matches_queue.publish(line)
logging.info('Sent %s' % line)
self.matches_queue.publish(END)
def print_response(self, ch, method, properties, body):
print(body.decode())
self.results += 1
if self.results == 3:
self.in_queue.cancel()
class AgeCalculator:
def __init__(self):
self.acked = set()
self.in_queue = RabbitMQQueue(exchange=OUT_JOINER_EXCHANGE,
exchange_type='direct',
consumer=True,
queue_name=AGE_CALCULATOR_QUEUE,
routing_keys=['calculator'])
self.out_queue = RabbitMQQueue(exchange=OUT_AGE_CALCULATOR_EXCHANGE)
self.terminator_queue = RabbitMQQueue(exchange=TERMINATOR_EXCHANGE)
def run(self, _):
self.in_queue.consume(self.calculate)
def calculate(self, ch, method, properties, body):
logging.info('Received %r' % body)
data = body.decode().split(',')
id = data[0]
if data[1] == END:
self.terminator_queue.publish(body)
logging.info('Sent %r' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if data[1] == CLOSE:
if not id in self.acked:
body = ','.join([id, OK])
self.terminator_queue.publish(body)
self.acked.add(id)
else:
self.in_queue.publish(body)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
tourney_date = data[1]
winner_birthdate = data[5]
loser_birthdate = data[9]
if winner_birthdate == '' or loser_birthdate == '':
ch.basic_ack(delivery_tag=method.delivery_tag)
return
tourney_date = datetime.strptime(tourney_date, '%Y%m%d')
winner_age = self._compute_age(
datetime.strptime(winner_birthdate, '%Y%m%d'), tourney_date)
loser_age = self._compute_age(
datetime.strptime(loser_birthdate, '%Y%m%d'), tourney_date)
data[5] = str(winner_age)
data[9] = str(loser_age)
body = ','.join(data)
self.out_queue.publish(body)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
def _compute_age(self, birthdate, tourney_date):
years = tourney_date.year - birthdate.year
if tourney_date.month < birthdate.month or \
(tourney_date.month == birthdate.month and tourney_date.day < birthdate.day):
years -= 1
return years
class Database:
def __init__(self):
self.hostname = os.environ['HOSTNAME']
self.count = {}
self.in_queue = RabbitMQQueue(exchange=DATABASE_EXCHANGE,
exchange_type='direct',
consumer=True,
queue_name='{}_queue'.format(
self.hostname),
routing_keys=FILES)
self.storage_queue = RabbitMQQueue(exchange='storage_input')
self.data_queue = RabbitMQQueue(exchange='storage_output',
exchange_type='direct',
consumer=True,
exclusive=True,
routing_keys=[self.hostname])
def run(self, _):
cmd = ';'.join(['CATCHUP', self.hostname])
self.storage_queue.publish(cmd)
logging.info('Sent %s to storage' % cmd)
self.data_queue.consume(self.update_count)
self.in_queue.consume(self.persist)
def update_count(self, ch, method, properties, body):
data = body.decode()
if data == 'END':
logging.info('State of %s updated' % self.hostname)
self.data_queue.cancel()
ch.basic_ack(delivery_tag=method.delivery_tag)
return
params = data.split(';')
id = params[-1]
count = int(params[2])
self.count[id] = count
logging.info('Count of %s updated: %d' % (id, count))
ch.basic_ack(delivery_tag=method.delivery_tag)
def persist(self, ch, method, properties, body):
logging.info('Received %r from %s' % (body, method.routing_key))
data = body.decode().split(',')
id = data[0]
result = data[1]
if result == END:
self.count[id] = self.count.get(id, 0) + 1
cmd = ';'.join(['WRITE', self.hostname, str(self.count[id]), id])
self.storage_queue.publish(cmd)
if self.count[id] != 3:
ch.basic_ack(delivery_tag=method.delivery_tag)
return
for filename in FILES:
try:
file = open(filename + id, 'r')
response = file.read()
file.close()
except FileNotFoundError:
response = '%s: No results' % filename
out_queue = RabbitMQQueue(exchange=RESPONSE_EXCHANGE + ':' +
id)
out_queue.publish(response)
logging.info('Sent %s' % response)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
file = open(method.routing_key + id, 'a+')
file.write(result + '\n')
file.close()
ch.basic_ack(delivery_tag=method.delivery_tag)
class Accumulator:
def __init__(self, routing_key, exchange, output_exchange):
self.hostname = os.environ['HOSTNAME']
self.routing_key = routing_key
self.values = {}
self.in_queue = RabbitMQQueue(exchange=exchange, exchange_type='direct',
consumer=True, queue_name='{}_queue'.format(self.hostname),
routing_keys=routing_key.split('-'))
self.out_queue = RabbitMQQueue(exchange=output_exchange)
self.storage_queue = RabbitMQQueue(exchange='storage_input')
self.data_queue = RabbitMQQueue(exchange='storage_output', exchange_type='direct',
consumer=True, exclusive=True,
routing_keys=[self.hostname])
def run(self, _):
cmd = ';'.join(['CATCHUP', self.hostname])
self.storage_queue.publish(cmd)
logging.info('Sent %s to storage' % cmd)
self.data_queue.consume(self.update_values)
self.in_queue.consume(self.add)
def update_values(self, ch, method, properties, body):
data = body.decode()
if data == 'END':
logging.info('State of %s updated' % self.hostname)
self.data_queue.cancel()
ch.basic_ack(delivery_tag=method.delivery_tag)
return
params = data.split(';')
id = params[-1]
values = params[2].split(',')
amount = int(values[0])
total = float(values[1])
self.values[id] = [amount, total]
logging.info('Values of %s updated: [%d, %f]' % (id, amount, total))
ch.basic_ack(delivery_tag=method.delivery_tag)
def add(self, ch, method, properties, body):
logging.info('Received %r' % body)
data = body.decode().split(',')
id = data[0]
if data[1] == END:
[amount, total] = self.values[id]
body = ','.join([id, self.routing_key, str(amount), str(total)])
self.out_queue.publish(body)
logging.info('Sent %s' % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
return
if not id in self.values:
self.values[id] = [0, 0]
self.values[id][0] += 1
amount = self.values[id][0]
logging.debug('Current amount: %f' % amount)
self.values[id][1] += float(data[1])
total = self.values[id][1]
logging.debug('Current total: %f' % total)
values = ','.join([str(amount), str(total)])
cmd = ';'.join(['WRITE', self.hostname, values, id])
self.storage_queue.publish(cmd)
ch.basic_ack(delivery_tag=method.delivery_tag)
class Storage:
def __init__(self, pid):
self.pid = pid
self.role = SLAVE_ROLE
self.input_queue = RabbitMQQueue(
exchange='storage_slave',
consumer=True,
queue_name='slave{}_queue'.format(pid))
self.output_queue = RabbitMQQueue(exchange='storage_output',
exchange_type='direct')
self.master_queue = RabbitMQQueue(exchange='storage_input',
consumer=True,
queue_name='master_queue')
self.instance_queue = RabbitMQQueue(
exchange='storage_internal_{}'.format(pid),
consumer=True,
queue_name='storage_internal_{}'.format(pid))
self.heartbeatproc = None
def run(self, heartbeatproc):
self.heartbeatproc = heartbeatproc
self.heartbeatproc.metadata = self.role
self.input_queue.consume(self.process)
self.instance_queue.consume(self.listen)
def process(self, ch, method, properties, body):
if self.role == SLAVE_ROLE:
logging.info('[SLAVE] Received %r' % body)
msg = body.decode()
parts = msg.split(",")
if parts[0] == MASTER_NEEDED_MSG:
# New master message
if int(parts[1]) == int(self.pid):
# and I am the new master
logging.info(
'[SLAVE] I was asked to be the new Storage Master')
self.role = MASTER_ROLE
# start sending new role in heartbeats
self.heartbeatproc.metadata = self.role
ch.basic_ack(delivery_tag=method.delivery_tag)
self.instance_queue.publish(MASTER_NEEDED_MSG)
self.input_queue.cancel()
else:
logging.info(
'[SLAVE] I received a master message but I was not the target'
)
# I am not the new master, discard it
ch.basic_ack(delivery_tag=method.delivery_tag)
return
logging.info('[SLAVE] Saving message')
self.persistState(body)
if ch.is_open:
ch.basic_ack(delivery_tag=method.delivery_tag)
def listen(self, ch, method, properties, body):
logging.info('[MASTER] I am consuming from storage_input')
self.master_queue.consume(self.processMaster)
def processMaster(self, ch, method, properties, body):
logging.info('[MASTER] Received %r' % body)
if self.isReadRequest(body):
self.processRead(body)
if self.isWriteRequest(body):
self.persistState(body)
self.input_queue.publish(body)
if ch.is_open:
ch.basic_ack(delivery_tag=method.delivery_tag)
def persistState(self, body):
state = body.decode()
logging.info('Persisting to disk {%r}' % state)
# MSG = CMD;tipoNodo_nroNodo;estado;job_id
# EJ: WRITE;joiner_3;93243;123123
params = state.split(';')
# path = "/storage/nodeType_nodeNumber/"
path = BASE_PATH + params[1] + "/"
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
# filename = "job_id"
filename = params[-1] + ".state"
f = open(path + filename, "w+") # Truncates previous state & writes
f.write(str(state))
f.close()
logging.info('Persisted {%r}' % state)
def isReadRequest(self, b):
if CATCHUP_COMMAND in b.decode():
return True
return False
def isWriteRequest(self, b):
if WRITE_COMMAND in b.decode():
return True
return False
def processRead(self, msg):
# MSG = CMD;tipoNodo_nroNodo
# EJ: READ;joiner_3
params = msg.decode().split(';')
path = BASE_PATH + params[1] + "/"
client_routing_key = params[1]
for filename in glob(path + "*.state"):
with open(filename, 'r') as file:
contents = file.read()
self.output_queue.publish(contents, client_routing_key)
self.output_queue.publish('END', client_routing_key)
