def handler(event, contest): logger.info("Start!") executor = Pool(100) cal = Sum() queue_url = event['queueUrl'] message_count = event['messageCount'] queue = sqs.Queue(queue_url) num_of_calls = message_count // batch_count queues = [] for i in range(num_of_calls): queues.append(queue) message_count = 0 responses = executor.map(one_request, queues) for response in responses: message_count += len(response) for msg in response: cal.add(msg) logger.info("Receive API count: {}".format(num_of_calls)) logger.info("Fetched messages: {}".format(message_count)) executor.close()
def handler(event, context): logger.info("Start!") poll = Poll() cal = Sum() update = Update() delete = Delete() table = event['table'] queue_url = event['queueUrl'] message_count = event['messageCount'] def on_error(e): raise e def on_poll_completed(): logger.info("Receive API count: {}".format(poll.fetch_count)) logger.info("Fetched messages: {}".format(poll.message_count)) update_and_delete() def update_and_delete_one(key): updated_message_ids = update.execute(db, table, cal.stats[key]) return delete.execute(sqs_client, queue_url, updated_message_ids) def update_and_delete(): delete_results = [] async_one = rx.Observable.to_async(update_and_delete_one) for key in cal.stats: delete_results.append(async_one(key)) rx.Observable.merge(delete_results).to_blocking().for_each(block) on_next_message = cal.add messages = poll.messages(sqs, queue_url, message_count).to_blocking() messages_observer = rx.Observer(on_next_message, on_error, on_poll_completed) messages.subscribe(messages_observer) messages.for_each(block) logger.info("Update API count: {}".format(update.update_count)) logger.info("Delete API count: {}".format(delete.delete_count)) logger.info("Delete Message count: {}".format(delete.message_count))
def __init__(self, loop): self.fetch_count = 0 self.message_count = 0 self.loop = loop self.cal = Sum()