class Poll(object): def __init__(self, loop): self.fetch_count = 0 self.message_count = 0 self.loop = loop self.cal = Sum() @asyncio.coroutine def _one_request(self, queue): self.fetch_count += 1 result = yield From(self.loop.run_in_executor(None, one_request, queue)) self.message_count += len(result) for msg in result: self.cal.add(msg) def messages(self, sqs, queue_url, message_count): queue = sqs.Queue(queue_url) num_of_calls = message_count // batch_count tasks = [] for i in range(num_of_calls): tasks.append(self._one_request(queue)) self.loop.run_until_complete(asyncio.wait(tasks))
class Poll(object): def __init__(self, loop): self.fetch_count = 0 self.message_count = 0 self.loop = loop self.cal = Sum() @asyncio.coroutine def _one_request(self, queue): self.fetch_count += 1 result = yield From(self.loop.run_in_executor( None, one_request, queue)) self.message_count += len(result) for msg in result: self.cal.add(msg) def messages(self, sqs, queue_url, message_count): queue = sqs.Queue(queue_url) num_of_calls = message_count // batch_count tasks = [] for i in range(num_of_calls): tasks.append(self._one_request(queue)) self.loop.run_until_complete(asyncio.wait(tasks))
def handler(event, contest): logger.info("Start!") executor = Pool(100) cal = Sum() queue_url = event['queueUrl'] message_count = event['messageCount'] queue = sqs.Queue(queue_url) num_of_calls = message_count // batch_count queues = [] for i in range(num_of_calls): queues.append(queue) message_count = 0 responses = executor.map(one_request, queues) for response in responses: message_count += len(response) for msg in response: cal.add(msg) logger.info("Receive API count: {}".format(num_of_calls)) logger.info("Fetched messages: {}".format(message_count)) executor.close()
def handler(event, contest): logger.info("Start!") executor = ThreadPoolExecutor(max_workers=100) cal = Sum() queue_url = event['queueUrl'] message_count = event['messageCount'] queue = sqs.Queue(queue_url) num_of_calls = message_count // batch_count queues = [] for i in range(num_of_calls): queues.append(queue) message_count = 0 responses = executor.map(one_request, queues) for response in responses: message_count += len(response) for msg in response: cal.add(msg) logger.info("Receive API count: {}".format(num_of_calls)) logger.info("Fetched messages: {}".format(message_count)) executor.shutdown()
class Poll(object): def __init__(self, loop): self.fetch_count = 0 self.message_count = 0 self.loop = loop self.cal = Sum() @gen.coroutine def _one_request(self, sqs_receive_message, queue_url): self.fetch_count += 1 resp = yield gen.Task(sqs_receive_message.call, QueueUrl=queue_url, MaxNumberOfMessages=batch_count, WaitTimeSeconds=wait_time) messages = resp['Messages'] self.message_count += len(messages) for msg in messages: self.cal.add(msg) def messages(self, sqs_receive_message, queue_url, message_count): num_of_calls = message_count // batch_count @gen.coroutine def poll(): r = [] for i in range(num_of_calls): r.append(self._one_request(sqs_receive_message, queue_url)) yield r self.loop.run_sync(poll)