示例#1
0
    def test_executor_submit(self):
        with SQSExecutor(session_factory, self.map_queue, self.reduce_queue) as w:
            # Submit work
            futures = []
            for i in range(10):
                futures.append(w.submit(int_processor, i))

            # Manually process and send results
            messages = MessageIterator(self.client, self.map_queue, limit=10)
            for m in messages:
                d = utils.loads(m['Body'])
                self.assertEqual(
                    m['MessageAttributes']['op']['StringValue'],
                    'test_sqs:int_processor')
                self.client.send_message(
                    QueueUrl=self.reduce_queue,
                    MessageBody=utils.dumps([
                        d['args'], int_processor(*d['args'])]),
                    MessageAttributes=m['MessageAttributes'])
            w.gather()
            results = [json.loads(r.result()['Body'])
                       for r in list(as_completed(futures))]
            self.assertEqual(
                list(sorted(results))[-1],
                [[9], 18])
示例#2
0
 def wait_for_config(self, session, queue_url, resource_id):
     # lazy import to avoid circular
     from c7n.sqsexec import MessageIterator
     client = session.client('sqs')
     messages = MessageIterator(client, queue_url, timeout=20)
     results = []
     while True:
         for m in messages:
             msg = json.loads(m['Body'])
             change = json.loads(msg['Message'])
             messages.ack(m)
             if change['configurationItem']['resourceId'] != resource_id:
                 continue
             results.append(change['configurationItem'])
             break
         if results:
             break
     return results
示例#3
0
    def wait_for_config(self, session, queue_url, resource_id=None):
        # lazy import to avoid circular
        from c7n.sqsexec import MessageIterator

        client = session.client("sqs")
        messages = MessageIterator(client, queue_url, timeout=20)
        results = []
        while True:
            for m in messages:
                msg = json.loads(m["Body"])
                change = json.loads(msg["Message"])
                messages.ack(m)
                if resource_id and change["configurationItem"][
                        "resourceId"] != resource_id:
                    continue
                results.append(change["configurationItem"])
                break
            if results:
                break
        return results
示例#4
0
def worker_config(queue, s3_key, period, verbose):
    """daemon queue worker for config notifications"""
    logging.basicConfig(level=(verbose and logging.DEBUG or logging.INFO))
    logging.getLogger('botocore').setLevel(logging.WARNING)
    logging.getLogger('s3transfer').setLevel(logging.WARNING)

    queue, region = get_queue(queue)
    factory = SessionFactory(region)
    session = factory()
    client = session.client('sqs')
    messages = MessageIterator(client, queue, timeout=20)

    for m in messages:
        msg = unwrap(m)
        if 'configurationItemSummary' in msg:
            rtype = msg['configurationItemSummary']['resourceType']
        else:
            rtype = msg['configurationItem']['resourceType']
        if rtype not in RESOURCE_MAPPING.values():
            log.info("skipping %s" % rtype)
            messages.ack(m)
        log.info("message received %s", m)
示例#5
0
def worker_config(queue, s3_key, period, verbose):
    """daemon queue worker for config notifications"""
    logging.basicConfig(level=(verbose and logging.DEBUG or logging.INFO))
    logging.getLogger('botocore').setLevel(logging.WARNING)
    logging.getLogger('s3transfer').setLevel(logging.WARNING)

    queue, region = get_queue(queue)
    factory = SessionFactory(region)
    session = factory()
    client = session.client('sqs')
    messages = MessageIterator(client, queue, timeout=20)

    for m in messages:
        msg = unwrap(m)
        if 'configurationItemSummary' in msg:
            rtype = msg['configurationItemSummary']['resourceType']
        else:
            rtype = msg['configurationItem']['resourceType']
        if rtype not in RESOURCE_MAPPING.values():
            log.info("skipping %s" % rtype)
            messages.ack(m)
        log.info("message received %s", m)
    def test_sqsexec(self):
        session_factory = self.replay_flight_data("test_sqs_exec")
        client = session_factory().client("sqs")
        map_queue = client.create_queue(
            QueueName="%s-map-%s"
            % (TEST_SQS_PREFIX, "".join(random.sample(string.ascii_letters, 3)))
        )[
            "QueueUrl"
        ]
        self.addCleanup(client.delete_queue, QueueUrl=map_queue)
        reduce_queue = client.create_queue(
            QueueName="%s-map-%s"
            % (TEST_SQS_PREFIX, "".join(random.sample(string.ascii_letters, 3)))
        )[
            "QueueUrl"
        ]
        self.addCleanup(client.delete_queue, QueueUrl=reduce_queue)

        with SQSExecutor(session_factory, map_queue, reduce_queue) as w:
            w.op_sequence_start = 699723
            w.op_sequence = 699723
            # Submit work
            futures = []
            for i in range(10):
                futures.append(w.submit(int_processor, i))

            # Manually process and send results
            messages = MessageIterator(client, map_queue, limit=10)
            for m in messages:
                d = utils.loads(m["Body"])
                self.assertEqual(
                    m["MessageAttributes"]["op"]["StringValue"],
                    "tests.test_sqsexec:int_processor",
                )
                client.send_message(
                    QueueUrl=reduce_queue,
                    MessageBody=utils.dumps([d["args"], int_processor(*d["args"])]),
                    MessageAttributes=m["MessageAttributes"],
                )
            w.gather()
            results = [
                json.loads(r.result()["Body"]) for r in list(as_completed(futures))
            ]
            self.assertEqual(list(sorted(results))[-1], [[9], 18])