class Poll(object):
def __init__(self, loop):
self.fetch_count = 0
self.message_count = 0
self.loop = loop
self.cal = Sum()
@asyncio.coroutine
def _one_request(self, queue):
self.fetch_count += 1
result = yield From(self.loop.run_in_executor(
None, one_request, queue))
self.message_count += len(result)
for msg in result:
self.cal.add(msg)
def messages(self, sqs, queue_url, message_count):
queue = sqs.Queue(queue_url)
num_of_calls = message_count // batch_count
tasks = []
for i in range(num_of_calls):
tasks.append(self._one_request(queue))
self.loop.run_until_complete(asyncio.wait(tasks))
class Poll(object):
def __init__(self, loop):
self.fetch_count = 0
self.message_count = 0
self.loop = loop
self.cal = Sum()
@asyncio.coroutine
def _one_request(self, queue):
self.fetch_count += 1
result = yield From(self.loop.run_in_executor(None, one_request,
queue))
self.message_count += len(result)
for msg in result:
self.cal.add(msg)
def messages(self, sqs, queue_url, message_count):
queue = sqs.Queue(queue_url)
num_of_calls = message_count // batch_count
tasks = []
for i in range(num_of_calls):
tasks.append(self._one_request(queue))
self.loop.run_until_complete(asyncio.wait(tasks))
def handler(event, contest):
logger.info("Start!")
executor = Pool(100)
cal = Sum()
queue_url = event['queueUrl']
message_count = event['messageCount']
queue = sqs.Queue(queue_url)
num_of_calls = message_count // batch_count
queues = []
for i in range(num_of_calls):
queues.append(queue)
message_count = 0
responses = executor.map(one_request, queues)
for response in responses:
message_count += len(response)
for msg in response:
cal.add(msg)
logger.info("Receive API count: {}".format(num_of_calls))
logger.info("Fetched messages: {}".format(message_count))
executor.close()
def handler(event, contest):
logger.info("Start!")
executor = ThreadPoolExecutor(max_workers=100)
cal = Sum()
queue_url = event['queueUrl']
message_count = event['messageCount']
queue = sqs.Queue(queue_url)
num_of_calls = message_count // batch_count
queues = []
for i in range(num_of_calls):
queues.append(queue)
message_count = 0
responses = executor.map(one_request, queues)
for response in responses:
message_count += len(response)
for msg in response:
cal.add(msg)
logger.info("Receive API count: {}".format(num_of_calls))
logger.info("Fetched messages: {}".format(message_count))
executor.shutdown()
class Poll(object):
def __init__(self, loop):
self.fetch_count = 0
self.message_count = 0
self.loop = loop
self.cal = Sum()
@gen.coroutine
def _one_request(self, sqs_receive_message, queue_url):
self.fetch_count += 1
resp = yield gen.Task(sqs_receive_message.call,
QueueUrl=queue_url,
MaxNumberOfMessages=batch_count,
WaitTimeSeconds=wait_time)
messages = resp['Messages']
self.message_count += len(messages)
for msg in messages:
self.cal.add(msg)
def messages(self, sqs_receive_message, queue_url, message_count):
num_of_calls = message_count // batch_count
@gen.coroutine
def poll():
r = []
for i in range(num_of_calls):
r.append(self._one_request(sqs_receive_message,
queue_url))
yield r
self.loop.run_sync(poll)
class Poll(object):
def __init__(self, loop):
self.fetch_count = 0
self.message_count = 0
self.loop = loop
self.cal = Sum()
@gen.coroutine
def _one_request(self, sqs_receive_message, queue_url):
self.fetch_count += 1
resp = yield gen.Task(sqs_receive_message.call,
QueueUrl=queue_url,
MaxNumberOfMessages=batch_count,
WaitTimeSeconds=wait_time)
messages = resp['Messages']
self.message_count += len(messages)
for msg in messages:
self.cal.add(msg)
def messages(self, sqs_receive_message, queue_url, message_count):
num_of_calls = message_count // batch_count
@gen.coroutine
def poll():
r = []
for i in range(num_of_calls):
r.append(self._one_request(sqs_receive_message, queue_url))
yield r
self.loop.run_sync(poll)
def handler(event, context):
logger.info("Start!")
poll = Poll()
cal = Sum()
update = Update()
delete = Delete()
table = event['table']
queue_url = event['queueUrl']
message_count = event['messageCount']
def on_error(e):
raise e
def on_poll_completed():
logger.info("Receive API count: {}".format(poll.fetch_count))
logger.info("Fetched messages: {}".format(poll.message_count))
update_and_delete()
def update_and_delete_one(key):
updated_message_ids = update.execute(db, table, cal.stats[key])
return delete.execute(sqs_client, queue_url, updated_message_ids)
def update_and_delete():
delete_results = []
async_one = rx.Observable.to_async(update_and_delete_one)
for key in cal.stats:
delete_results.append(async_one(key))
rx.Observable.merge(delete_results).to_blocking().for_each(block)
on_next_message = cal.add
messages = poll.messages(sqs, queue_url, message_count).to_blocking()
messages_observer = rx.Observer(on_next_message, on_error,
on_poll_completed)
messages.subscribe(messages_observer)
messages.for_each(block)
logger.info("Update API count: {}".format(update.update_count))
logger.info("Delete API count: {}".format(delete.delete_count))
logger.info("Delete Message count: {}".format(delete.message_count))
def __init__(self, loop):
self.fetch_count = 0
self.message_count = 0
self.loop = loop
self.cal = Sum()
def __init__(self, loop):
self.fetch_count = 0
self.message_count = 0
self.loop = loop
self.cal = Sum()
