def load(options, path, format='yaml', validate=True):
# should we do os.path.expanduser here?
if not os.path.exists(path):
raise IOError("Invalid path for config %r" % path)
load_resources()
with open(path) as fh:
if format == 'yaml':
data = utils.yaml_load(fh.read())
elif format == 'json':
data = utils.loads(fh.read())
validate = False
# Test for empty policy file
if not data or data.get('policies') is None:
return None
if validate:
from c7n.schema import validate
errors = validate(data)
if errors:
raise Exception("Failed to validate on policy %s \n %s" %
(errors[1], errors[0]))
collection = PolicyCollection.from_data(data, options)
return collection
def test_executor_submit(self):
with SQSExecutor(session_factory, self.map_queue, self.reduce_queue) as w:
# Submit work
futures = []
for i in range(10):
futures.append(w.submit(int_processor, i))
# Manually process and send results
messages = MessageIterator(self.client, self.map_queue, limit=10)
for m in messages:
d = utils.loads(m['Body'])
self.assertEqual(
m['MessageAttributes']['op']['StringValue'],
'test_sqs:int_processor')
self.client.send_message(
QueueUrl=self.reduce_queue,
MessageBody=utils.dumps([
d['args'], int_processor(*d['args'])]),
MessageAttributes=m['MessageAttributes'])
w.gather()
results = [json.loads(r.result()['Body'])
for r in list(as_completed(futures))]
self.assertEqual(
list(sorted(results))[-1],
[[9], 18])
def process_message(self, m):
msg = utils.loads(m['Body'])
op_name = m['MessageAttributes']['op']['StringValue']
func = resolve(op_name)
try:
func(*msg['args'], **msg['kwargs'])
except Exception, e:
log.exception("Error invoking %s %s" % (op_name, e))
return
def process_message(self, m):
msg = utils.loads(m['Body'])
op_name = m['MessageAttributes']['op']['StringValue']
func = resolve(op_name)
try:
func(*msg['args'], **msg['kwargs'])
except Exception as e:
log.exception(
"Error invoking %s %s" % (
op_name, e))
return
def load(options, path, format='yaml', validate=True):
if not os.path.exists(path):
raise ValueError("Invalid path for config %r" % path)
with open(path) as fh:
if format == 'yaml':
data = utils.yaml_load(fh.read())
elif format == 'json':
data = utils.loads(fh.read())
validate = False
if validate:
from c7n.schema import validate
errors = validate(data)
if errors:
raise errors[0]
return PolicyCollection(data, options)
def test_sqsexec(self):
session_factory = self.replay_flight_data("test_sqs_exec")
client = session_factory().client("sqs")
map_queue = client.create_queue(
QueueName="%s-map-%s"
% (TEST_SQS_PREFIX, "".join(random.sample(string.ascii_letters, 3)))
)[
"QueueUrl"
]
self.addCleanup(client.delete_queue, QueueUrl=map_queue)
reduce_queue = client.create_queue(
QueueName="%s-map-%s"
% (TEST_SQS_PREFIX, "".join(random.sample(string.ascii_letters, 3)))
)[
"QueueUrl"
]
self.addCleanup(client.delete_queue, QueueUrl=reduce_queue)
with SQSExecutor(session_factory, map_queue, reduce_queue) as w:
w.op_sequence_start = 699723
w.op_sequence = 699723
# Submit work
futures = []
for i in range(10):
futures.append(w.submit(int_processor, i))
# Manually process and send results
messages = MessageIterator(client, map_queue, limit=10)
for m in messages:
d = utils.loads(m["Body"])
self.assertEqual(
m["MessageAttributes"]["op"]["StringValue"],
"tests.test_sqsexec:int_processor",
)
client.send_message(
QueueUrl=reduce_queue,
MessageBody=utils.dumps([d["args"], int_processor(*d["args"])]),
MessageAttributes=m["MessageAttributes"],
)
w.gather()
results = [
json.loads(r.result()["Body"]) for r in list(as_completed(futures))
]
self.assertEqual(list(sorted(results))[-1], [[9], 18])
def test_sqsexec(self):
session_factory = self.replay_flight_data("test_sqs_exec")
client = session_factory().client("sqs")
map_queue = client.create_queue(
QueueName="%s-map-%s"
% (TEST_SQS_PREFIX, "".join(random.sample(string.ascii_letters, 3)))
)[
"QueueUrl"
]
self.addCleanup(client.delete_queue, QueueUrl=map_queue)
reduce_queue = client.create_queue(
QueueName="%s-map-%s"
% (TEST_SQS_PREFIX, "".join(random.sample(string.ascii_letters, 3)))
)[
"QueueUrl"
]
self.addCleanup(client.delete_queue, QueueUrl=reduce_queue)
with SQSExecutor(session_factory, map_queue, reduce_queue) as w:
w.op_sequence_start = 699723
w.op_sequence = 699723
# Submit work
futures = []
for i in range(10):
futures.append(w.submit(int_processor, i))
# Manually process and send results
messages = MessageIterator(client, map_queue, limit=10)
for m in messages:
d = utils.loads(m["Body"])
self.assertEqual(
m["MessageAttributes"]["op"]["StringValue"],
"tests.test_sqsexec:int_processor",
)
client.send_message(
QueueUrl=reduce_queue,
MessageBody=utils.dumps([d["args"], int_processor(*d["args"])]),
MessageAttributes=m["MessageAttributes"],
)
w.gather()
results = [
json.loads(r.result()["Body"]) for r in list(as_completed(futures))
]
self.assertEqual(list(sorted(results))[-1], [[9], 18])
def load(options, path, format='yaml', validate=True):
# should we do os.path.expanduser here?
if not os.path.exists(path):
raise IOError("Invalid path for config %r" % path)
load_resources()
with open(path) as fh:
if format == 'yaml':
data = utils.yaml_load(fh.read())
elif format == 'json':
data = utils.loads(fh.read())
validate = False
# Test for empty policy file
if not data or data.get('policies') is None:
return None
if validate:
from c7n.schema import validate
errors = validate(data)
if errors:
raise Exception("Failed to validate on policy %s \n %s" % (errors[1], errors[0]))
return PolicyCollection(data, options)
