def test_executor_submit(self):
with SQSExecutor(session_factory, self.map_queue, self.reduce_queue) as w:
# Submit work
futures = []
for i in range(10):
futures.append(w.submit(int_processor, i))
# Manually process and send results
messages = MessageIterator(self.client, self.map_queue, limit=10)
for m in messages:
d = utils.loads(m['Body'])
self.assertEqual(
m['MessageAttributes']['op']['StringValue'],
'test_sqs:int_processor')
self.client.send_message(
QueueUrl=self.reduce_queue,
MessageBody=utils.dumps([
d['args'], int_processor(*d['args'])]),
MessageAttributes=m['MessageAttributes'])
w.gather()
results = [json.loads(r.result()['Body'])
for r in list(as_completed(futures))]
self.assertEqual(
list(sorted(results))[-1],
[[9], 18])
def wait_for_config(self, session, queue_url, resource_id):
# lazy import to avoid circular
from c7n.sqsexec import MessageIterator
client = session.client('sqs')
messages = MessageIterator(client, queue_url, timeout=20)
results = []
while True:
for m in messages:
msg = json.loads(m['Body'])
change = json.loads(msg['Message'])
messages.ack(m)
if change['configurationItem']['resourceId'] != resource_id:
continue
results.append(change['configurationItem'])
break
if results:
break
return results
def wait_for_config(self, session, queue_url, resource_id=None):
# lazy import to avoid circular
from c7n.sqsexec import MessageIterator
client = session.client("sqs")
messages = MessageIterator(client, queue_url, timeout=20)
results = []
while True:
for m in messages:
msg = json.loads(m["Body"])
change = json.loads(msg["Message"])
messages.ack(m)
if resource_id and change["configurationItem"][
"resourceId"] != resource_id:
continue
results.append(change["configurationItem"])
break
if results:
break
return results
def worker_config(queue, s3_key, period, verbose):
"""daemon queue worker for config notifications"""
logging.basicConfig(level=(verbose and logging.DEBUG or logging.INFO))
logging.getLogger('botocore').setLevel(logging.WARNING)
logging.getLogger('s3transfer').setLevel(logging.WARNING)
queue, region = get_queue(queue)
factory = SessionFactory(region)
session = factory()
client = session.client('sqs')
messages = MessageIterator(client, queue, timeout=20)
for m in messages:
msg = unwrap(m)
if 'configurationItemSummary' in msg:
rtype = msg['configurationItemSummary']['resourceType']
else:
rtype = msg['configurationItem']['resourceType']
if rtype not in RESOURCE_MAPPING.values():
log.info("skipping %s" % rtype)
messages.ack(m)
log.info("message received %s", m)
def worker_config(queue, s3_key, period, verbose):
"""daemon queue worker for config notifications"""
logging.basicConfig(level=(verbose and logging.DEBUG or logging.INFO))
logging.getLogger('botocore').setLevel(logging.WARNING)
logging.getLogger('s3transfer').setLevel(logging.WARNING)
queue, region = get_queue(queue)
factory = SessionFactory(region)
session = factory()
client = session.client('sqs')
messages = MessageIterator(client, queue, timeout=20)
for m in messages:
msg = unwrap(m)
if 'configurationItemSummary' in msg:
rtype = msg['configurationItemSummary']['resourceType']
else:
rtype = msg['configurationItem']['resourceType']
if rtype not in RESOURCE_MAPPING.values():
log.info("skipping %s" % rtype)
messages.ack(m)
log.info("message received %s", m)
def test_sqsexec(self):
session_factory = self.replay_flight_data("test_sqs_exec")
client = session_factory().client("sqs")
map_queue = client.create_queue(
QueueName="%s-map-%s"
% (TEST_SQS_PREFIX, "".join(random.sample(string.ascii_letters, 3)))
)[
"QueueUrl"
]
self.addCleanup(client.delete_queue, QueueUrl=map_queue)
reduce_queue = client.create_queue(
QueueName="%s-map-%s"
% (TEST_SQS_PREFIX, "".join(random.sample(string.ascii_letters, 3)))
)[
"QueueUrl"
]
self.addCleanup(client.delete_queue, QueueUrl=reduce_queue)
with SQSExecutor(session_factory, map_queue, reduce_queue) as w:
w.op_sequence_start = 699723
w.op_sequence = 699723
# Submit work
futures = []
for i in range(10):
futures.append(w.submit(int_processor, i))
# Manually process and send results
messages = MessageIterator(client, map_queue, limit=10)
for m in messages:
d = utils.loads(m["Body"])
self.assertEqual(
m["MessageAttributes"]["op"]["StringValue"],
"tests.test_sqsexec:int_processor",
)
client.send_message(
QueueUrl=reduce_queue,
MessageBody=utils.dumps([d["args"], int_processor(*d["args"])]),
MessageAttributes=m["MessageAttributes"],
)
w.gather()
results = [
json.loads(r.result()["Body"]) for r in list(as_completed(futures))
]
self.assertEqual(list(sorted(results))[-1], [[9], 18])