Exemple #1
0
def handler(event, contest):
    logger.info("Start!")

    executor = Pool(100)

    cal = Sum()

    queue_url = event['queueUrl']
    message_count = event['messageCount']

    queue = sqs.Queue(queue_url)
    num_of_calls = message_count // batch_count

    queues = []
    for i in range(num_of_calls):
        queues.append(queue)

    message_count = 0

    responses = executor.map(one_request, queues)
    for response in responses:
        message_count += len(response)
        for msg in response:
            cal.add(msg)

    logger.info("Receive API count: {}".format(num_of_calls))
    logger.info("Fetched messages: {}".format(message_count))

    executor.close()
def handler(event, context):
    logger.info("Start!")

    poll = Poll()
    cal = Sum()
    update = Update()
    delete = Delete()

    table = event['table']
    queue_url = event['queueUrl']
    message_count = event['messageCount']

    def on_error(e):
        raise e

    def on_poll_completed():
        logger.info("Receive API count: {}".format(poll.fetch_count))
        logger.info("Fetched messages: {}".format(poll.message_count))

        update_and_delete()

    def update_and_delete_one(key):
        updated_message_ids = update.execute(db, table, cal.stats[key])
        return delete.execute(sqs_client, queue_url, updated_message_ids)

    def update_and_delete():
        delete_results = []
        async_one = rx.Observable.to_async(update_and_delete_one)
        for key in cal.stats:
            delete_results.append(async_one(key))

        rx.Observable.merge(delete_results).to_blocking().for_each(block)

    on_next_message = cal.add

    messages = poll.messages(sqs, queue_url, message_count).to_blocking()
    messages_observer = rx.Observer(on_next_message, on_error,
                                    on_poll_completed)
    messages.subscribe(messages_observer)
    messages.for_each(block)

    logger.info("Update API count: {}".format(update.update_count))
    logger.info("Delete API count: {}".format(delete.delete_count))
    logger.info("Delete Message count: {}".format(delete.message_count))
Exemple #3
0
    def __init__(self, loop):
        self.fetch_count = 0
        self.message_count = 0

        self.loop = loop
        self.cal = Sum()