def provision(self):
# Avoiding runtime lambda dep, premature optimization?
from c7n.mu import PolicyLambda, LambdaManager
with self.policy.ctx:
self.policy.log.info("Provisioning policy lambda %s",
self.policy.name)
variables = {
'account_id': self.policy.options.account_id,
'region': self.policy.options.region
}
self.policy.data = self.expand_variables(variables)
try:
manager = LambdaManager(self.policy.session_factory)
except ClientError:
# For cli usage by normal users, don't assume the role just use
# it for the lambda
manager = LambdaManager(
lambda assume=False: self.policy.session_factory(assume))
return manager.publish(PolicyLambda(self.policy),
'current',
role=self.policy.options.assume_role)
def process(self, buckets):
from c7n.mu import LambdaManager
from c7n.ufuncs.s3crypt import get_function
session = local_session(self.manager.session_factory)
account_id = get_account_id(session)
func = get_function(
None, self.data.get('role', self.manager.config.assume_role),
account_id=account_id)
regions = set([
b.get('Location', {
'LocationConstraint': 'us-east-1'})['LocationConstraint']
for b in buckets])
# session managers by region
region_sessions = {}
for r in regions:
region_sessions[r] = functools.partial(
self.manager.session_factory, region=r)
# Publish function to all of our buckets regions
region_funcs = {}
for r in regions:
lambda_mgr = LambdaManager(region_sessions[r])
lambda_mgr.publish(func)
region_funcs[r] = func
with self.executor_factory(max_workers=3) as w:
results = []
futures = []
for b in buckets:
region = b.get('Location', {
'LocationConstraint': 'us-east-1'}).get(
'LocationConstraint')
futures.append(
w.submit(
self.process_bucket,
region_funcs[region],
b,
account_id,
region_sessions[region]
))
for f in as_completed(futures):
if f.exception():
log.exception(
"Error attaching lambda-encrypt %s" % (f.exception()))
results.append(f.result())
return filter(None, results)
def test_cwe_trail(self):
session_factory = self.replay_flight_data("test_cwe_trail", zdata=True)
p = self.load_policy(
{
"resource":
"s3",
"name":
"s3-bucket-policy",
"mode": {
"type": "cloudtrail",
"events": ["CreateBucket"]
},
"filters": [{
"type": "missing-policy-statement",
"statement_ids": ["RequireEncryptedPutObject"],
}],
"actions": ["no-op"]
},
session_factory=session_factory)
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
self.addCleanup(mgr.remove, pl)
result = mgr.publish(pl, "Dev", role=ROLE)
events = pl.get_events(session_factory)
self.assertEqual(len(events), 1)
event = events.pop()
self.assertEqual(
json.loads(event.render_event_pattern()),
{
u"detail": {
u"eventName": [u"CreateBucket"],
u"eventSource": [u"s3.amazonaws.com"],
},
u"detail-type": ["AWS API Call via CloudTrail"],
},
)
self.assert_items(
result,
{
"Description": "cloud-custodian lambda policy",
"FunctionName": "custodian-s3-bucket-policy",
"Handler": "custodian_policy.run",
"MemorySize": 512,
"Runtime": "python2.7",
"Timeout": 60,
},
)
def test_cwe_instance(self):
session_factory = self.replay_flight_data("test_cwe_instance",
zdata=True)
p = self.load_policy(
{
"resource": "s3",
"name": "ec2-encrypted-vol",
"mode": {
"type": "ec2-instance-state",
"events": ["pending"]
}
},
session_factory=session_factory)
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
self.addCleanup(mgr.remove, pl)
result = mgr.publish(pl, "Dev", role=ROLE)
self.assert_items(
result,
{
"Description": "cloud-custodian lambda policy",
"FunctionName": "custodian-ec2-encrypted-vol",
"Handler": "custodian_policy.run",
"MemorySize": 512,
"Runtime": "python2.7",
"Timeout": 60,
},
)
events = session_factory().client("events")
result = events.list_rules(NamePrefix="custodian-ec2-encrypted-vol")
self.assert_items(
result["Rules"][0],
{
"State": "ENABLED",
"Name": "custodian-ec2-encrypted-vol"
},
)
self.assertEqual(
json.loads(result["Rules"][0]["EventPattern"]),
{
"source": ["aws.ec2"],
"detail": {
"state": ["pending"]
},
"detail-type": ["EC2 Instance State-change Notification"],
},
)
def get_metrics(self, start, end, period):
# Avoiding runtime lambda dep, premature optimization?
from c7n.mu import PolicyLambda, LambdaManager
values = {}
# Pickup lambda specific metrics (errors, invocations, durations)
if self.is_lambda:
manager = LambdaManager(self.session_factory)
values = manager.metrics([PolicyLambda(self)], start, end,
period)[0]
metrics = ['ResourceCount']
else:
metrics = ['ResourceCount', 'ResourceTime', 'ActionTime']
default_dimensions = {
'Policy': self.name,
'ResType': self.resource_type,
'Scope': 'Policy'
}
# Support action, and filter custom metrics
for el in itertools.chain(self.resource_manager.actions,
self.resource_manager.filters):
if el.metrics:
metrics.extend(el.metrics)
session = utils.local_session(self.session_factory)
client = session.client('cloudwatch')
for m in metrics:
if isinstance(m, basestring):
dimensions = default_dimensions
else:
m, m_dimensions = m
dimensions = dict(default_dimensions)
dimensions.update(m_dimensions)
results = client.get_metric_statistics(
Namespace=DEFAULT_NAMESPACE,
Dimensions=[{
'Name': k,
'Value': v
} for k, v in dimensions.items()],
Statistics=['Sum', 'Average'],
StartTime=start,
EndTime=end,
Period=period,
MetricName=m)
values[m] = results['Datapoints']
return values
def test_attach_encrypt(self):
self.patch(s3, 'S3_AUGMENT_TABLE', [])
session_factory = self.replay_flight_data('test_s3_attach_encrypt')
bname = "custodian-attach-encrypt-test"
role = 'arn:aws:iam::619193117841:role/lambda_s3_exec_role'
self.maxDiff = None
session = session_factory()
client = session.client('s3')
client.create_bucket(Bucket=bname)
self.addCleanup(destroyBucket, client, bname)
p = self.load_policy(
{
'name': 'attach-encrypt',
'resource': 's3',
'filters': [{
'Name': bname
}],
'actions': [{
'type': 'attach-encrypt',
'role': role
}]
},
session_factory=session_factory)
self.addCleanup(
LambdaManager(session_factory).remove,
s3crypt.get_function(None, role))
resources = p.run()
notifications = client.get_bucket_notification_configuration(
Bucket=bname)
notifications.pop('ResponseMetadata')
self.assertEqual(
notifications, {
'LambdaFunctionConfigurations': [{
'Events': ['s3:ObjectCreated:*'],
'Id':
'custodian-s3-encrypt',
'LambdaFunctionArn':
'arn:aws:lambda:us-east-1:619193117841:function:custodian-s3-encrypt'
}]
})
client.put_object(Bucket=bname,
Key='hello-world.txt',
Body='hello world',
ContentType='text/plain')
info = client.head_object(Bucket=bname, Key='hello-world.txt')
self.assertTrue('ServerSideEncryption' in info)
def test_config_rule_provision(self):
session_factory = self.replay_flight_data("test_config_rule")
p = Policy(
{
"resource": "security-group",
"name": "sg-modified",
"mode": {"type": "config-rule"},
},
Config.empty(),
)
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
result = mgr.publish(pl, "Dev", role=ROLE)
self.assertEqual(result["FunctionName"], "custodian-sg-modified")
self.addCleanup(mgr.remove, pl)
def test_config_rule_provision(self):
session_factory = self.replay_flight_data('test_config_rule')
p = Policy(
{
'resource': 'security-group',
'name': 'sg-modified',
'mode': {
'type': 'config-rule'
},
}, Config.empty())
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
result = mgr.publish(pl, 'Dev', role=ROLE)
self.assertEqual(result['FunctionName'], 'custodian-sg-modified')
self.addCleanup(mgr.remove, pl)
def test_lambda_cross_account(self):
self.patch(CrossAccountAccessFilter, "executor_factory",
MainThreadExecutor)
session_factory = self.replay_flight_data("test_cross_account_lambda")
client = session_factory().client("lambda")
name = "c7n-cross-check"
tmp_dir = tempfile.mkdtemp()
self.addCleanup(os.rmdir, tmp_dir)
archive = PythonPackageArchive()
archive.add_contents("handler.py", LAMBDA_SRC)
archive.close()
func = LambdaFunction(
{
"runtime": "python2.7",
"name": name,
"description": "",
"handler": "handler.handler",
"memory_size": 128,
"timeout": 5,
"role": self.role,
},
archive,
)
manager = LambdaManager(session_factory)
manager.publish(func)
self.addCleanup(manager.remove, func)
client.add_permission(
FunctionName=name,
StatementId="oops",
Principal="*",
Action="lambda:InvokeFunction",
)
p = self.load_policy(
{
"name": "lambda-cross",
"resource": "lambda",
"filters": ["cross-account"],
},
session_factory=session_factory,
)
resources = p.run()
self.assertEqual(len(resources), 1)
self.assertEqual(resources[0]["FunctionName"], name)
def create_a_lambda(self, flight, **extra):
session_factory = self.replay_flight_data(flight, zdata=True)
mode = {
'type': 'config-rule',
'role':'arn:aws:iam::644160558196:role/custodian-mu'}
mode.update(extra)
p = Policy({
'resource': 's3',
'name': 'hello-world',
'actions': ['no-op'],
'mode': mode,
}, Config.empty())
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
self.addCleanup(mgr.remove, pl)
return mgr, mgr.publish(pl)
def test_attach_encrypt(self):
self.patch(s3, 'S3_AUGMENT_TABLE',
[('get_bucket_location', 'Location', None, None)])
self.patch(s3.S3, 'executor_factory', MainThreadExecutor)
session_factory = self.replay_flight_data('test_s3_attach_encrypt')
bname = "custodian-attach-encrypt-test"
role = "arn:aws:iam::644160558196:role/custodian-mu"
self.maxDiff = None
session = session_factory(region='us-west-2')
client = session.client('s3')
client.create_bucket(
Bucket=bname,
CreateBucketConfiguration={
'LocationConstraint': 'us-west-2'})
self.addCleanup(destroyBucket, client, bname)
p = self.load_policy({
'name': 'attach-encrypt',
'resource': 's3',
'filters': [{'Name': bname}],
'actions': [{
'type': 'attach-encrypt',
'role': role}]
}, session_factory=session_factory)
self.addCleanup(
LambdaManager(functools.partial(session_factory, region='us-west-2')).remove,
s3crypt.get_function(None, role))
resources = p.run()
self.assertEqual(len(resources), 1)
#time.sleep(10)
notifications = client.get_bucket_notification_configuration(
Bucket=bname)
notifications.pop('ResponseMetadata')
self.assertEqual(
notifications,
{'LambdaFunctionConfigurations': [{
'Events': ['s3:ObjectCreated:*'],
'Id': 'c7n-s3-encrypt',
'LambdaFunctionArn': 'arn:aws:lambda:us-west-2:644160558196:function:c7n-s3-encrypt'}]})
client.put_object(
Bucket=bname, Key='hello-world.txt',
Body='hello world', ContentType='text/plain')
#time.sleep(30)
info = client.head_object(Bucket=bname, Key='hello-world.txt')
self.assertTrue('ServerSideEncryption' in info)
def test_cwe_asg_instance(self):
session_factory = self.replay_flight_data("test_cwe_asg", zdata=True)
p = self.load_policy(
{
"resource": "asg",
"name": "asg-spin-detector",
"mode": {
"type": "asg-instance-state",
"events": ["launch-failure"]
},
},
session_factory=session_factory)
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
self.addCleanup(mgr.remove, pl)
result = mgr.publish(pl, "Dev", role=ROLE)
self.assert_items(
result,
{
"FunctionName": "custodian-asg-spin-detector",
"Handler": "custodian_policy.run",
"MemorySize": 512,
"Runtime": "python2.7",
"Timeout": 60,
},
)
events = session_factory().client("events")
result = events.list_rules(NamePrefix="custodian-asg-spin-detector")
self.assert_items(
result["Rules"][0],
{
"State": "ENABLED",
"Name": "custodian-asg-spin-detector"
},
)
self.assertEqual(
json.loads(result["Rules"][0]["EventPattern"]),
{
"source": ["aws.autoscaling"],
"detail-type": ["EC2 Instance Launch Unsuccessful"],
},
)
def test_cwe_trail(self):
session_factory = self.replay_flight_data('test_cwe_trail', zdata=True)
p = Policy(
{
'resource':
's3',
'name':
's3-bucket-policy',
'mode': {
'type': 'cloudtrail',
'events': ["CreateBucket"],
},
'filters': [{
'type': 'missing-policy-statement',
'statement_ids': ['RequireEncryptedPutObject']
}],
'actions': ['no-op']
}, Config.empty())
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
result = mgr.publish(pl, 'Dev', role=self.role)
events = pl.get_events(session_factory)
self.assertEqual(len(events), 1)
event = events.pop()
self.assertEqual(
json.loads(event.render_event_pattern()), {
u'detail': {
u'eventName': [u'CreateBucket'],
u'eventSource': [u's3.amazonaws.com']
},
u'detail-type': ['AWS API Call via CloudTrail']
})
self.assert_items(
result, {
'Description': 'cloud-custodian lambda policy',
'FunctionName': 'custodian-s3-bucket-policy',
'Handler': 'custodian_policy.run',
'MemorySize': 512,
'Runtime': 'python2.7',
'Timeout': 60
})
mgr.remove(pl)
def test_sqs_subscriber(self):
session_factory = self.replay_flight_data('test_mu_sqs_subscriber')
func_name = 'c7n-hello-sqs'
queue_name = "my-dev-test-3"
# Setup Queues
session = session_factory()
client = session.client('sqs')
queue_url = client.create_queue(QueueName=queue_name).get('QueueUrl')
queue_arn = client.get_queue_attributes(
QueueUrl=queue_url,
AttributeNames=['QueueArn'])['Attributes']['QueueArn']
self.addCleanup(client.delete_queue, QueueUrl=queue_url)
# Setup Function
params = dict(
session_factory=session_factory,
name=func_name,
role="arn:aws:iam::644160558196:role/custodian-mu",
events=[SQSSubscription(session_factory, [queue_arn])])
func = helloworld.get_function(**params)
manager = LambdaManager(session_factory)
manager.publish(func)
self.addCleanup(manager.remove, func)
# Send and Receive Check
client.send_message(
QueueUrl=queue_url, MessageBody=json.dumps({'jurassic': 'block'}))
if self.recording:
time.sleep(60)
log_events = list(manager.logs(func, "1970-1-1 UTC", "9170-1-1"))
messages = [
e["message"] for e in log_events if e["message"].startswith('{"Records')
]
self.addCleanup(
session.client("logs").delete_log_group,
logGroupName="/aws/lambda/%s" % func_name)
self.assertIn(
'jurassic',
json.loads(messages[0])["Records"][0]["body"])
def test_lambda_cross_account(self):
self.patch(CrossAccountAccessFilter, 'executor_factory',
MainThreadExecutor)
session_factory = self.replay_flight_data('test_cross_account_lambda')
client = session_factory().client('lambda')
name = 'c7n-cross-check'
tmp_dir = tempfile.mkdtemp()
self.addCleanup(os.rmdir, tmp_dir)
archive = PythonPackageArchive(tmp_dir, tmp_dir)
archive.create()
archive.add_contents('handler.py', LAMBDA_SRC)
archive.close()
func = LambdaFunction(
{
'runtime': 'python2.7',
'name': name,
'description': '',
'handler': 'handler.handler',
'memory_size': 128,
'timeout': 5,
'role': self.role
}, archive)
manager = LambdaManager(session_factory)
info = manager.publish(func)
self.addCleanup(manager.remove, func)
client.add_permission(FunctionName=name,
StatementId='oops',
Principal='*',
Action='lambda:InvokeFunction')
p = self.load_policy(
{
'name': 'lambda-cross',
'resource': 'lambda',
'filters': ['cross-account']
},
session_factory=session_factory)
resources = p.run()
self.assertEqual(len(resources), 1)
self.assertEqual(resources[0]['FunctionName'], name)
def test_mu_metrics(self):
session_factory = self.replay_flight_data('test_mu_metrics')
p = Policy({
'resources': 's3',
'name': 's3-bucket-policy',
'resource': 's3',
'mode': {
'type': 'cloudtrail',
'events': ['CreateBucket'],
},
'actions': ['no-op']}, Config.empty())
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
end = datetime.utcnow()
start = end - timedelta(1)
results = mgr.metrics([pl], start, end, 3600)
self.assertEqual(
results, [{'Durations': [], 'Errors': [],
'Throttles': [], 'Invocations': []}])
def test_mu_metrics(self):
session_factory = self.replay_flight_data("test_mu_metrics")
p = self.load_policy(
{
"name": "s3-bucket-policy",
"resource": "s3",
"mode": {"type": "cloudtrail", "events": ["CreateBucket"]},
"actions": ["no-op"],
}, session_factory=session_factory)
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
end = datetime.utcnow()
start = end - timedelta(1)
results = mgr.metrics([pl], start, end, 3600)
self.assertEqual(
results,
[{"Durations": [], "Errors": [], "Throttles": [], "Invocations": []}],
)
def test_sns_subscriber_and_ipaddress(self):
self.patch(SNSSubscription, "iam_delay", 0.01)
session_factory = self.replay_flight_data(
"test_sns_subscriber_and_ipaddress")
session = session_factory()
client = session.client("sns")
# create an sns topic
tname = "custodian-test-sns-sub"
topic_arn = client.create_topic(Name=tname)["TopicArn"]
self.addCleanup(client.delete_topic, TopicArn=topic_arn)
# provision a lambda via mu
params = dict(
session_factory=session_factory,
name="c7n-hello-world",
role="arn:aws:iam::644160558196:role/custodian-mu",
events=[SNSSubscription(session_factory, [topic_arn])],
)
func = helloworld.get_function(**params)
manager = LambdaManager(session_factory)
manager.publish(func)
self.addCleanup(manager.remove, func)
# now publish to the topic and look for lambda log output
client.publish(TopicArn=topic_arn, Message="Greetings, program!")
if self.recording:
time.sleep(30)
log_events = manager.logs(func, "1970-1-1 UTC", "2037-1-1")
messages = [
e["message"] for e in log_events
if e["message"].startswith('{"Records')
]
self.addCleanup(
session.client("logs").delete_log_group,
logGroupName="/aws/lambda/c7n-hello-world",
)
self.assertEqual(
json.loads(messages[0])["Records"][0]["Sns"]["Message"],
"Greetings, program!",
)
def test_cwe_instance(self):
session_factory = self.replay_flight_data('test_cwe_instance',
zdata=True)
p = Policy(
{
'resource': 's3',
'name': 'ec2-encrypted-vol',
'mode': {
'type': 'ec2-instance-state',
'events': ['pending']
}
}, Config.empty())
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
result = mgr.publish(pl, 'Dev', role=self.role)
self.assert_items(
result, {
'Description': 'cloud-maid lambda policy',
'FunctionName': 'maid-ec2-encrypted-vol',
'Handler': 'maid_policy.run',
'MemorySize': 512,
'Runtime': 'python2.7',
'Timeout': 60
})
events = session_factory().client('events')
result = events.list_rules(NamePrefix="maid-ec2-encrypted-vol")
self.assert_items(result['Rules'][0], {
"State": "ENABLED",
"Name": "maid-ec2-encrypted-vol"
})
self.assertEqual(
json.loads(result['Rules'][0]['EventPattern']), {
"source": ["aws.ec2"],
"detail": {
"state": ["pending"]
},
"detail-type": ["EC2 Instance State-change Notification"]
})
mgr.remove(pl)
def test_cwe_schedule(self):
session_factory = self.replay_flight_data("test_cwe_schedule",
zdata=True)
p = Policy(
{
"resource": "ec2",
"name": "periodic-ec2-checker",
"mode": {
"type": "periodic",
"schedule": "rate(1 day)"
},
},
Config.empty(),
)
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
self.addCleanup(mgr.remove, pl)
result = mgr.publish(pl, "Dev", role=ROLE)
self.assert_items(
result,
{
"FunctionName": "custodian-periodic-ec2-checker",
"Handler": "custodian_policy.run",
"MemorySize": 512,
"Runtime": "python2.7",
"Timeout": 60,
},
)
events = session_factory().client("events")
result = events.list_rules(NamePrefix="custodian-periodic-ec2-checker")
self.assert_items(
result["Rules"][0],
{
"State": "ENABLED",
"ScheduleExpression": "rate(1 day)",
"Name": "custodian-periodic-ec2-checker",
},
)
def deploy_one(region_name, account, policy, sentry_dsn):
from c7n.mu import LambdaManager
def session_factory():
return boto3.Session(region_name=region_name)
log_group_name = '/aws/lambda/custodian-{}'.format(policy['name'])
arn = 'arn:aws:logs:{}:{}:log-group:{}:*'.format(
region_name, account['account_id'], log_group_name)
function = get_function(
session_factory=session_factory,
name='cloud-custodian-sentry',
handler='handler.process_log_event',
role=account['role'],
log_groups=[{'logGroupName': log_group_name, 'arn': arn}],
project=None,
account_name=account['name'],
account_id=account['account_id'],
sentry_dsn=sentry_dsn,
)
log.info("Deploying lambda for {} in {}".format(
log_group_name, region_name))
LambdaManager(session_factory).publish(function)
def test_publish_a_lambda_with_layer_and_concurrency(self):
factory = self.replay_flight_data('test_lambda_layer_concurrent_publish')
mgr = LambdaManager(factory)
layers = ['arn:aws:lambda:us-east-1:644160558196:layer:CustodianLayer:2']
func = self.make_func(
concurrency=5,
layers=layers)
self.addCleanup(mgr.remove, func)
result = mgr.publish(func)
self.assertEqual(result['Layers'][0]['Arn'], layers[0])
state = mgr.get(func.name)
self.assertEqual(state['Concurrency']['ReservedConcurrentExecutions'], 5)
func = self.make_func(layers=layers)
output = self.capture_logging("custodian.serverless", level=logging.DEBUG)
result = mgr.publish(func)
self.assertEqual(result['Layers'][0]['Arn'], layers[0])
lines = output.getvalue().strip().split("\n")
self.assertFalse('Updating function: test-foo-bar config Layers' in lines)
self.assertTrue('Removing function: test-foo-bar concurrency' in lines)
def create_a_lambda(self, flight, **extra):
session_factory = self.replay_flight_data(flight, zdata=True)
mode = {
"type": "config-rule", "role": "arn:aws:iam::644160558196:role/custodian-mu"
}
mode.update(extra)
p = self.load_policy({
"resource": "s3",
"name": "hello-world",
"actions": ["no-op"],
"mode": mode},
session_factory=session_factory)
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
def cleanup():
mgr.remove(pl)
if self.recording:
time.sleep(60)
self.addCleanup(cleanup)
return mgr, mgr.publish(pl)
def metrics(function, api, start, period):
"""lambda/api/db metrics"""
from c7n.mu import LambdaManager
manager = LambdaManager(boto3.Session)
start = parse_date(start)
period = int(abs(parse_timedelta(period).total_seconds()))
print("Lambda Metrics")
metrics = manager.metrics([{
'FunctionName': function
}],
start=start,
end=datetime.utcnow(),
period=period)
for k in ('Invocations', 'Throttles', 'Errors'):
values = [n['Sum'] for n in metrics[0][k]]
render_metrics(k, values)
if not api:
return
print("Api Metrics")
metrics = gateway_metrics(boto3.Session, api, "latest", start,
datetime.utcnow(), period)
for k, data in metrics.items():
if "Count" in k:
values = [n['Sum'] for n in data]
else:
values = [n['Average'] for n in data]
render_metrics(k, values)
print("Db Metrics")
metrics = db_metrics(boto3.Session, "Sphere11.Dev.ResourceLocks", start,
datetime.utcnow(), period)
for k, data in metrics.items():
values = [n['Average'] for n in data]
render_metrics(k, values)
def test_sns_subscriber_and_ipaddress(self):
self.patch(SNSSubscription, 'iam_delay', 0.01)
session_factory = self.replay_flight_data(
'test_sns_subscriber_and_ipaddress')
session = session_factory()
client = session.client('sns')
# create an sns topic
tname = "custodian-test-sns-sub"
topic_arn = client.create_topic(Name=tname)['TopicArn']
self.addCleanup(client.delete_topic, TopicArn=topic_arn)
# provision a lambda via mu
params = dict(session_factory=session_factory,
name='c7n-hello-world',
role='arn:aws:iam::644160558196:role/custodian-mu',
events=[SNSSubscription(session_factory, [topic_arn])])
func = helloworld.get_function(**params)
manager = LambdaManager(session_factory)
manager.publish(func)
self.addCleanup(manager.remove, func)
# now publish to the topic and look for lambda log output
client.publish(TopicArn=topic_arn, Message='Greetings, program!')
if self.recording:
time.sleep(30)
log_events = manager.logs(func, '1970-1-1 UTC', '9170-1-1')
messages = [
e['message'] for e in log_events
if e['message'].startswith('{"Records')
]
self.addCleanup(session.client('logs').delete_log_group,
logGroupName='/aws/lambda/c7n-hello-world')
self.assertEqual(
json.loads(messages[0])['Records'][0]['Sns']['Message'],
'Greetings, program!')
def provision(config, session_factory):
func_config = dict(
name=config.get('lambda_name', 'cloud-custodian-mailer'),
description=config.get('lambda_description', 'Cloud Custodian Mailer'),
tags=config.get('lambda_tags', {}),
handler='periodic.dispatch',
runtime=config['runtime'],
memory_size=config['memory'],
timeout=config['timeout'],
role=config['role'],
subnets=config['subnets'],
security_groups=config['security_groups'],
dead_letter_config=config.get('dead_letter_config', {}),
events=[
CloudWatchEventSource(
{'type': 'periodic',
'schedule': config.get('lambda_schedule', 'rate(5 minutes)')},
session_factory)
])
archive = get_archive(config)
func = LambdaFunction(func_config, archive)
manager = LambdaManager(session_factory)
manager.publish(func)
def provision(config, session_factory):
func_config = dict(name='cloud-custodian-mailer',
description='Cloud Custodian/Maid Mailer',
handler='periodic.dispatch',
runtime='python2.7',
memory_size=config['memory'],
timeout=config['timeout'],
role=config['role'],
subnets=config['subnets'],
security_groups=config['security_groups'],
events=[
CloudWatchEventSource(
{
'type': 'periodic',
'schedule': 'rate(5 minutes)'
},
session_factory,
prefix="")
])
archive = get_archive(config)
func = LambdaFunction(func_config, archive)
manager = LambdaManager(session_factory)
manager.publish(func)
def test_cwe_asg_instance(self):
session_factory = self.replay_flight_data('test_cwe_asg', zdata=True)
p = Policy(
{
'resource': 'asg',
'name': 'asg-spin-detector',
'mode': {
'type': 'asg-instance-state',
'events': ['launch-failure']
}
}, Config.empty())
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
result = mgr.publish(pl, 'Dev', role=self.role)
self.assert_items(
result, {
'FunctionName': 'maid-asg-spin-detector',
'Handler': 'maid_policy.run',
'MemorySize': 512,
'Runtime': 'python2.7',
'Timeout': 60
})
events = session_factory().client('events')
result = events.list_rules(NamePrefix="maid-asg-spin-detector")
self.assert_items(result['Rules'][0], {
"State": "ENABLED",
"Name": "maid-asg-spin-detector"
})
self.assertEqual(
json.loads(result['Rules'][0]['EventPattern']), {
"source": ["aws.autoscaling"],
"detail-type": ["EC2 Instance Launch Unsuccessful"]
})
mgr.remove(pl)
def main():
parser = setup_parser()
options = parser.parse_args()
logging.basicConfig(level=logging.DEBUG)
logging.getLogger('botocore').setLevel(logging.ERROR)
if not options.group and not options.prefix:
print("Error: Either group or prefix must be specified")
sys.exit(1)
session_factory = SessionFactory(options.region, options.profile,
options.assume)
groups = get_groups(session_factory, options)
func = get_function(session_factory, options, groups)
manager = LambdaManager(session_factory)
try:
manager.publish(func)
except Exception:
import traceback, pdb, sys
traceback.print_exc()
pdb.post_mortem(sys.exc_info()[-1])
def test_cwe_update_config_and_code(self):
# Originally this was testing the no update case.. but
# That is tricky to record, any updates to the code end up
# causing issues due to checksum mismatches which imply updating
# the function code / which invalidate the recorded data and
# the focus of the test.
session_factory = self.replay_flight_data("test_cwe_update",
zdata=True)
p = self.load_policy({
"resource":
"s3",
"name":
"s3-bucket-policy",
"mode": {
"type": "cloudtrail",
"events": ["CreateBucket"],
'runtime': 'python2.7'
},
"filters": [
{
"type": "missing-policy-statement",
"statement_ids": ["RequireEncryptedPutObject"]
},
],
"actions": ["no-op"],
})
pl = PolicyLambda(p)
mgr = LambdaManager(session_factory)
result = mgr.publish(pl, "Dev", role=ROLE)
self.addCleanup(mgr.remove, pl)
p = self.load_policy(
{
"resource":
"s3",
"name":
"s3-bucket-policy",
"mode": {
"type":
"cloudtrail",
"memory":
256,
'runtime':
'python2.7',
"events": [
"CreateBucket",
{
"event": "PutBucketPolicy",
"ids": "requestParameters.bucketName",
"source": "s3.amazonaws.com",
},
],
},
"filters": [{
"type": "missing-policy-statement",
"statement_ids": ["RequireEncryptedPutObject"],
}],
"actions": ["no-op"],
}, )
output = self.capture_logging("custodian.serverless",
level=logging.DEBUG)
result2 = mgr.publish(PolicyLambda(p), "Dev", role=ROLE)
lines = output.getvalue().strip().split("\n")
self.assertTrue(
"Updating function custodian-s3-bucket-policy code" in lines)
self.assertTrue(
"Updating function: custodian-s3-bucket-policy config MemorySize"
in lines)
self.assertEqual(result["FunctionName"], result2["FunctionName"])
# drive by coverage
functions = [
i for i in mgr.list_functions()
if i["FunctionName"] == "custodian-s3-bucket-policy"
]
self.assertTrue(len(functions), 1)
start = 0
end = time.time() * 1000
self.assertEqual(list(mgr.logs(pl, start, end)), [])
