class RegionReader(object):
def __init__(self, file_path, rabbitmq_queues):
self.file_path = file_path
self.queues = rabbitmq_queues
def start(self):
region_number = 0
with open(self.file_path, "r") as regions:
next(regions) #skip headers
for region in regions:
self.queues.send_all(region)
region_number += 1
logging.info("Cases sended: {}".format(region_number))
logging.info("Sending EOM")
self.queues.send_eom()
if __name__ == '__main__':
config_log("INIT REGIONS")
rabbitmq_host = os.environ['HOST']
distance_workers = int(os.environ['DISTANCE_WORKERS'])
rabbitmq_queues = Queues(SEND_QUEUE_NAME, rabbitmq_host, distance_workers)
worker = RegionReader(FILE, rabbitmq_queues)
logging.info("Worker created, started running")
worker.start()
logging.info("Worker finished, exiting")
class CasesReader(object):
def __init__(self, file_path, rabbitmq_queues):
self.file_path = file_path
self.queues = rabbitmq_queues
def start(self):
case_number = 0
with open(self.file_path, "r") as cases:
next(cases) #skip headers
for case in cases:
self.queues.send(case)
case_number += 1
if (case_number % LOG_FREQUENCY == 0):
logging.info("Sending case [%d] %s", case_number, case)
time.sleep(5)
logging.info("Cases sended: {}".format(case_number))
logging.info("Sending EOM")
self.queues.send_eom()
if __name__ == '__main__':
config_log("INIT")
rabbitmq_host = os.environ['HOST']
filter_parser_workers = int(os.environ['FILTER_PARSER_WORKERS'])
rabbitmq_queues = Queues(SEND_QUEUE_NAME, rabbitmq_host, filter_parser_workers)
worker = CasesReader(FILE, rabbitmq_queues)
logging.info("Worker created, started running")
worker.start()
logging.info("Worker finished, exiting")
def _callback(self, ch, method, properties, body):
decoded_body = body.decode('UTF-8')
body_values = decoded_body.split(',')
region = body_values[REGION]
positives = int(body_values[POSITIVES])
self.total[region] = self.total.get(region, 0) + positives
def _calculate_three_regions(self):
items = self.total.items();
items_ordered = sorted(items, key=lambda c: c[1], reverse=True)
return items_ordered[0:3]
if __name__ == '__main__':
host = os.environ['HOST']
worker_id = int(os.environ['SERVICE_ID'])
counter_by_region_workers = int(os.environ['COUNTER_BY_REGION_WORKERS'])
config_log("TOTAL BY REGION {}".format(worker_id))
send_queue = Queue(SEND_QUEUE_NAME, host)
receive_queue = Queue(RECEIVE_QUEUE_NAME, host, counter_by_region_workers)
worker = ReducerRegions(receive_queue, send_queue)
logging.info("Worker created, started running")
worker.run()
logging.info("Worker finished, exiting")
self.send_queue.send("Percentage: {}".format(percentage))
logging.info("Sending EOM to queues")
self.send_queue.send_eom()
logging.info("Finish")
def _callback(self, ch, method, properties, body):
decoded_body = body.decode('UTF-8')
positives, deceases = decoded_body.split(',')
self.positive_cases += int(positives)
self.decease_cases += int(deceases)
self.log_counter += 1
if __name__ == '__main__':
host = os.environ['HOST']
worker_id = int(os.environ['SERVICE_ID'])
counter_workers = int(os.environ['COUNTER_WORKERS'])
config_log("PERCENTAGE")
send_queue = Queue(SEND_QUEUE_NAME, host)
receive_queue = Queue(RECEIVE_QUEUE_NAME, host, counter_workers)
worker = ReducerPercentage(receive_queue, send_queue)
logging.info("Worker created, started running")
worker.run()
logging.info("Worker finished, exiting")
return
self.log_counter += 1
if self.log_counter % LOG_FREQUENCY == 0:
logging.info("Received line [%d] Case %s", self.log_counter,
decoded_body)
logging.info("Sending %d,%d", self.positive_cases,
self.decease_cases)
self.send_queue.send('{},{}'.format(self.positive_cases,
self.decease_cases))
self.positive_cases = 0
self.decease_cases = 0
if __name__ == '__main__':
host = os.environ['HOST']
filter_workers = int(os.environ['FILTER_PARSER_WORKERS'])
worker_id = int(os.environ['SERVICE_ID'])
config_log("COUNTER {}".format(worker_id))
send_queue = Queue(SEND_QUEUE_NAME, host)
receive_queue = Queue("{}_{}".format(RECEIVE_QUEUE_NAME, worker_id), host,
filter_workers)
worker = CounterCases(receive_queue, send_queue)
logging.info("Worker created, started running")
worker.run()
logging.info("Worker finished, exiting")
self.log_counter += 1
lat = float(body_values[LAT])
long = float(body_values[LONG])
region = self.haversine.calculate_location(lat, long)
if self.log_counter % LOG_FREQUENCY == 0:
logging.info("Received line [%d] Lat %s, Long %s", self.log_counter, lat, long)
self.send_queues.send(region)
if __name__ == '__main__':
rabbitmq_host = os.environ['HOST']
filter_parser_workers = int(os.environ['FILTER_PARSER_WORKERS'])
# distance_workers = int(os.environ['DISTANCE_WORKERS'])
counter_by_region_workers = int(os.environ['COUNTER_BY_REGION_WORKERS'])
worker_id = int(os.environ['SERVICE_ID'])
config_log("DISTANCE CALCULATOR {}".format(worker_id))
regions_queue = Queue("{}_{}".format(REGIONS_QUEUE_NAME, worker_id), rabbitmq_host)
receive_queue = Queue("{}_{}".format(RECEIVE_QUEUE_NAME, worker_id), rabbitmq_host, filter_parser_workers)
send_queues = Queues(SEND_QUEUE_NAME, rabbitmq_host, counter_by_region_workers)
worker = DistanceCalculator(regions_queue, receive_queue, send_queues)
logging.info("Worker created, started running")
worker.run()
logging.info("Worker finished, exiting")
def _callback(self, ch, method, properties, body):
region = body.decode('UTF-8')
self.log_counter += 1
self.counter[region] = self.counter.get(region, 0) + 1
if self.log_counter % LOG_FREQUENCY == 0:
logging.info("Received line [%d] Region %s", self.log_counter, region)
self._send()
def _send(self):
for region, cases in self.counter.items():
self.send_queue.send('{},{}'.format(region,cases))
self.counter = {}
if __name__ == '__main__':
host = os.environ['HOST']
worker_id = int(os.environ['SERVICE_ID'])
config_log("COUNTER BY REGION {}".format(worker_id))
distance_workers = int(os.environ['DISTANCE_WORKERS'])
send_queue = Queue(SEND_QUEUE_NAME, host)
receive_queue = Queue("{}_{}".format(RECEIVE_QUEUE_NAME, worker_id), host, distance_workers)
worker = CounterByRegion(receive_queue, send_queue)
logging.info("Worker created, started running")
worker.run()
logging.info("Worker finished, exiting")
else:
return
if self.log_counter % LOG_FREQUENCY == 0:
logging.info("Received line [%d] Date %s Case %s",
self.log_counter, date, case)
self._send()
def _send(self):
for date, cases in self.counter.items():
self.send_queue.send('{},{},{}'.format(date,
cases['positive_cases'],
cases['decease_cases']))
self.counter = {}
if __name__ == '__main__':
host = os.environ['HOST']
worker_id = int(os.environ['SERVICE_ID'])
config_log("COUNTER BY DATE {}".format(worker_id))
filter_workers = int(os.environ['FILTER_PARSER_WORKERS'])
send_queue = Queue(SEND_QUEUE_NAME, host)
receive_queue = Queue("{}_{}".format(RECEIVE_QUEUE_NAME, worker_id), host,
filter_workers)
worker = CounterCases(receive_queue, send_queue)
logging.info("Worker created, started running")
worker.run()
logging.info("Worker finished, exiting")
class End(object):
def __init__(self, file_path, rabbitmq_queue):
self.file_path = file_path
self.queue = rabbitmq_queue
def run(self):
logging.info("Start consuming")
self.queue.consume(self._callback)
logging.info("Finish")
def _callback(self, ch, method, properties, body):
decoded_body = body.decode('UTF-8')
with open(self.file_path, "a") as file:
file.write(decoded_body)
file.write('\n')
if __name__ == '__main__':
config_log("END")
rabbitmq_host = os.environ['HOST']
reducer_workers = int(os.environ['REDUCER_WORKERS'])
rabbitmq_queue = Queue(RECEIVE_QUEUE_NAME, rabbitmq_host, reducer_workers)
reporter = End(FILE, rabbitmq_queue)
logging.info("Worker created, started running")
reporter.run()
logging.info("Worker finished, exiting")
self.counter_queues.send(case)
self.counter_by_date_queues.send("{},{}".format(date, case))
if case == POSITIVE:
self.distance_queues.send("{},{}".format(lat, long))
if __name__ == '__main__':
rabbitmq_host = os.environ['HOST']
filter_parser_workers = int(os.environ['FILTER_PARSER_WORKERS'])
counter_workers = int(os.environ['COUNTER_WORKERS'])
counter_by_date_workers = int(os.environ['COUNTER_WORKERS'])
distance_workers = int(os.environ['DISTANCE_WORKERS'])
worker_id = int(os.environ['SERVICE_ID'])
config_log("FILTER PARSER {}".format(worker_id))
counter_queues = Queues(COUNTER_QUEUE_NAME, rabbitmq_host, counter_workers)
counter_by_date_queues = Queues(COUNTER_DATE_QUEUE_NAME, rabbitmq_host,
counter_by_date_workers)
distance_queues = Queues(DISTANCE_NAME_QUEUE, rabbitmq_host,
distance_workers)
receive_queue = Queue("{}_{}".format(RECEIVE_QUEUE_NAME, worker_id),
rabbitmq_host)
worker = FilterParser(receive_queue, counter_queues,
counter_by_date_queues, distance_queues)
logging.info("Worker created, started running")
worker.run()
logging.info("Worker finished, exiting")
logging.info("Finish")
def _callback(self, ch, method, properties, body):
decoded_body = body.decode('UTF-8')
body_values = decoded_body.split(',')
date = body_values[DATE]
positives = int(body_values[POSITIVES])
deceases = int(body_values[DECEASES])
if date not in self.total:
self.total[date] = {'positive_cases': 0, 'decease_cases': 0}
self.total[date]['positive_cases'] += positives
self.total[date]['decease_cases'] += deceases
if __name__ == '__main__':
host = os.environ['HOST']
worker_id = int(os.environ['SERVICE_ID'])
counter_workers = int(os.environ['COUNTER_BY_DATE_WORKERS'])
config_log("TOTAL BY DATE {}".format(worker_id))
send_queue = Queue(SEND_QUEUE_NAME, host)
receive_queue = Queue(RECEIVE_QUEUE_NAME, host, counter_workers)
worker = ReducerByDate(receive_queue, send_queue)
logging.info("Worker created, started running")
worker.run()
logging.info("Worker finished, exiting")