class Poll(object): def __init__(self, loop): self.fetch_count = 0 self.message_count = 0 self.loop = loop self.cal = Sum() @asyncio.coroutine def _one_request(self, queue): self.fetch_count += 1 result = yield From(self.loop.run_in_executor( None, one_request, queue)) self.message_count += len(result) for msg in result: self.cal.add(msg) def messages(self, sqs, queue_url, message_count): queue = sqs.Queue(queue_url) num_of_calls = message_count // batch_count tasks = [] for i in range(num_of_calls): tasks.append(self._one_request(queue)) self.loop.run_until_complete(asyncio.wait(tasks))
class Poll(object): def __init__(self, loop): self.fetch_count = 0 self.message_count = 0 self.loop = loop self.cal = Sum() @asyncio.coroutine def _one_request(self, queue): self.fetch_count += 1 result = yield From(self.loop.run_in_executor(None, one_request, queue)) self.message_count += len(result) for msg in result: self.cal.add(msg) def messages(self, sqs, queue_url, message_count): queue = sqs.Queue(queue_url) num_of_calls = message_count // batch_count tasks = [] for i in range(num_of_calls): tasks.append(self._one_request(queue)) self.loop.run_until_complete(asyncio.wait(tasks))
def handler(event, contest): logger.info("Start!") executor = Pool(100) cal = Sum() queue_url = event['queueUrl'] message_count = event['messageCount'] queue = sqs.Queue(queue_url) num_of_calls = message_count // batch_count queues = [] for i in range(num_of_calls): queues.append(queue) message_count = 0 responses = executor.map(one_request, queues) for response in responses: message_count += len(response) for msg in response: cal.add(msg) logger.info("Receive API count: {}".format(num_of_calls)) logger.info("Fetched messages: {}".format(message_count)) executor.close()
def handler(event, contest): logger.info("Start!") executor = ThreadPoolExecutor(max_workers=100) cal = Sum() queue_url = event['queueUrl'] message_count = event['messageCount'] queue = sqs.Queue(queue_url) num_of_calls = message_count // batch_count queues = [] for i in range(num_of_calls): queues.append(queue) message_count = 0 responses = executor.map(one_request, queues) for response in responses: message_count += len(response) for msg in response: cal.add(msg) logger.info("Receive API count: {}".format(num_of_calls)) logger.info("Fetched messages: {}".format(message_count)) executor.shutdown()
class Poll(object): def __init__(self, loop): self.fetch_count = 0 self.message_count = 0 self.loop = loop self.cal = Sum() @gen.coroutine def _one_request(self, sqs_receive_message, queue_url): self.fetch_count += 1 resp = yield gen.Task(sqs_receive_message.call, QueueUrl=queue_url, MaxNumberOfMessages=batch_count, WaitTimeSeconds=wait_time) messages = resp['Messages'] self.message_count += len(messages) for msg in messages: self.cal.add(msg) def messages(self, sqs_receive_message, queue_url, message_count): num_of_calls = message_count // batch_count @gen.coroutine def poll(): r = [] for i in range(num_of_calls): r.append(self._one_request(sqs_receive_message, queue_url)) yield r self.loop.run_sync(poll)
def handler(event, context): logger.info("Start!") poll = Poll() cal = Sum() update = Update() delete = Delete() table = event['table'] queue_url = event['queueUrl'] message_count = event['messageCount'] def on_error(e): raise e def on_poll_completed(): logger.info("Receive API count: {}".format(poll.fetch_count)) logger.info("Fetched messages: {}".format(poll.message_count)) update_and_delete() def update_and_delete_one(key): updated_message_ids = update.execute(db, table, cal.stats[key]) return delete.execute(sqs_client, queue_url, updated_message_ids) def update_and_delete(): delete_results = [] async_one = rx.Observable.to_async(update_and_delete_one) for key in cal.stats: delete_results.append(async_one(key)) rx.Observable.merge(delete_results).to_blocking().for_each(block) on_next_message = cal.add messages = poll.messages(sqs, queue_url, message_count).to_blocking() messages_observer = rx.Observer(on_next_message, on_error, on_poll_completed) messages.subscribe(messages_observer) messages.for_each(block) logger.info("Update API count: {}".format(update.update_count)) logger.info("Delete API count: {}".format(delete.delete_count)) logger.info("Delete Message count: {}".format(delete.message_count))
def __init__(self, loop): self.fetch_count = 0 self.message_count = 0 self.loop = loop self.cal = Sum()