class Poll(object):
def __init__(self, loop):
self.fetch_count = 0
self.message_count = 0
self.loop = loop
self.cal = Sum()
@asyncio.coroutine
def _one_request(self, queue):
self.fetch_count += 1
result = yield From(self.loop.run_in_executor(None, one_request,
queue))
self.message_count += len(result)
for msg in result:
self.cal.add(msg)
def messages(self, sqs, queue_url, message_count):
queue = sqs.Queue(queue_url)
num_of_calls = message_count // batch_count
tasks = []
for i in range(num_of_calls):
tasks.append(self._one_request(queue))
self.loop.run_until_complete(asyncio.wait(tasks))
class Poll(object):
def __init__(self, loop):
self.fetch_count = 0
self.message_count = 0
self.loop = loop
self.cal = Sum()
@asyncio.coroutine
def _one_request(self, queue):
self.fetch_count += 1
result = yield From(self.loop.run_in_executor(
None, one_request, queue))
self.message_count += len(result)
for msg in result:
self.cal.add(msg)
def messages(self, sqs, queue_url, message_count):
queue = sqs.Queue(queue_url)
num_of_calls = message_count // batch_count
tasks = []
for i in range(num_of_calls):
tasks.append(self._one_request(queue))
self.loop.run_until_complete(asyncio.wait(tasks))
def handler(event, contest):
logger.info("Start!")
executor = Pool(100)
cal = Sum()
queue_url = event['queueUrl']
message_count = event['messageCount']
queue = sqs.Queue(queue_url)
num_of_calls = message_count // batch_count
queues = []
for i in range(num_of_calls):
queues.append(queue)
message_count = 0
responses = executor.map(one_request, queues)
for response in responses:
message_count += len(response)
for msg in response:
cal.add(msg)
logger.info("Receive API count: {}".format(num_of_calls))
logger.info("Fetched messages: {}".format(message_count))
executor.close()
def handler(event, contest):
logger.info("Start!")
executor = ThreadPoolExecutor(max_workers=100)
cal = Sum()
queue_url = event['queueUrl']
message_count = event['messageCount']
queue = sqs.Queue(queue_url)
num_of_calls = message_count // batch_count
queues = []
for i in range(num_of_calls):
queues.append(queue)
message_count = 0
responses = executor.map(one_request, queues)
for response in responses:
message_count += len(response)
for msg in response:
cal.add(msg)
logger.info("Receive API count: {}".format(num_of_calls))
logger.info("Fetched messages: {}".format(message_count))
executor.shutdown()
class Poll(object):
def __init__(self, loop):
self.fetch_count = 0
self.message_count = 0
self.loop = loop
self.cal = Sum()
@gen.coroutine
def _one_request(self, sqs_receive_message, queue_url):
self.fetch_count += 1
resp = yield gen.Task(sqs_receive_message.call,
QueueUrl=queue_url,
MaxNumberOfMessages=batch_count,
WaitTimeSeconds=wait_time)
messages = resp['Messages']
self.message_count += len(messages)
for msg in messages:
self.cal.add(msg)
def messages(self, sqs_receive_message, queue_url, message_count):
num_of_calls = message_count // batch_count
@gen.coroutine
def poll():
r = []
for i in range(num_of_calls):
r.append(self._one_request(sqs_receive_message,
queue_url))
yield r
self.loop.run_sync(poll)
class Poll(object):
def __init__(self, loop):
self.fetch_count = 0
self.message_count = 0
self.loop = loop
self.cal = Sum()
@gen.coroutine
def _one_request(self, sqs_receive_message, queue_url):
self.fetch_count += 1
resp = yield gen.Task(sqs_receive_message.call,
QueueUrl=queue_url,
MaxNumberOfMessages=batch_count,
WaitTimeSeconds=wait_time)
messages = resp['Messages']
self.message_count += len(messages)
for msg in messages:
self.cal.add(msg)
def messages(self, sqs_receive_message, queue_url, message_count):
num_of_calls = message_count // batch_count
@gen.coroutine
def poll():
r = []
for i in range(num_of_calls):
r.append(self._one_request(sqs_receive_message, queue_url))
yield r
self.loop.run_sync(poll)
