def process(self, buckets):
from c7n.mu import LambdaManager
from c7n.ufuncs.s3crypt import get_function
func = get_function(
None, self.data.get('role', self.manager.config.assume_role))
# Publish function to all of our buckets regions
region_funcs = {}
regions = set([
b.get('LocationConstraint', 'us-east-1') for b in buckets])
for r in regions:
lambda_mgr = LambdaManager(
functools.partial(self.manager.session_factory, region=r))
region_funcs[r] = lambda_mgr.publish(func)
with self.executor_factory(max_workers=3) as w:
results = []
futures = []
for b in buckets:
futures.append(
w.submit(
self.process_bucket,
region_funcs[b.get('LocationConstraint', 'us-east-1')],
b))
for f in as_completed(futures):
if f.exception():
log.exception(
"Error attaching lambda-encrypt %s" % (f.exception()))
results.append(f.result())
return filter(None, results)
def process(self, buckets):
from c7n.mu import LambdaManager
from c7n.ufuncs.s3crypt import get_function
func = get_function(
None, self.data.get('role', self.manager.config.assume_role))
# Publish function to all of our buckets regions
region_funcs = {}
regions = set(
[b.get('LocationConstraint', 'us-east-1') for b in buckets])
for r in regions:
lambda_mgr = LambdaManager(
functools.partial(self.manager.session_factory, region=r))
region_funcs[r] = lambda_mgr.publish(func)
with self.executor_factory(max_workers=3) as w:
results = []
futures = []
for b in buckets:
futures.append(
w.submit(
self.process_bucket,
region_funcs[b.get('LocationConstraint',
'us-east-1')], b))
for f in as_completed(futures):
if f.exception():
log.exception("Error attaching lambda-encrypt %s" %
(f.exception()))
results.append(f.result())
return filter(None, results)
def test_attach_encrypt(self):
self.patch(s3, 'S3_AUGMENT_TABLE',
[('get_bucket_location', 'Location', None, None)])
self.patch(s3.S3, 'executor_factory', MainThreadExecutor)
session_factory = self.replay_flight_data('test_s3_attach_encrypt')
bname = "custodian-attach-encrypt-test"
role = "arn:aws:iam::644160558196:role/custodian-mu"
self.maxDiff = None
session = session_factory(region='us-west-2')
client = session.client('s3')
client.create_bucket(
Bucket=bname,
CreateBucketConfiguration={'LocationConstraint': 'us-west-2'})
self.addCleanup(destroyBucket, client, bname)
p = self.load_policy(
{
'name': 'attach-encrypt',
'resource': 's3',
'filters': [{
'Name': bname
}],
'actions': [{
'type': 'attach-encrypt',
'role': role
}]
},
session_factory=session_factory)
self.addCleanup(
LambdaManager(
functools.partial(session_factory, region='us-west-2')).remove,
s3crypt.get_function(None, role))
resources = p.run()
self.assertEqual(len(resources), 1)
#time.sleep(10)
notifications = client.get_bucket_notification_configuration(
Bucket=bname)
notifications.pop('ResponseMetadata')
self.assertEqual(
notifications, {
'LambdaFunctionConfigurations': [{
'Events': ['s3:ObjectCreated:*'],
'Id':
'c7n-s3-encrypt',
'LambdaFunctionArn':
'arn:aws:lambda:us-west-2:644160558196:function:c7n-s3-encrypt'
}]
})
client.put_object(Bucket=bname,
Key='hello-world.txt',
Body='hello world',
ContentType='text/plain')
#time.sleep(30)
info = client.head_object(Bucket=bname, Key='hello-world.txt')
self.assertTrue('ServerSideEncryption' in info)
def process(self, buckets):
from c7n.mu import LambdaManager
from c7n.ufuncs.s3crypt import get_function
session = local_session(self.manager.session_factory)
account_id = get_account_id(session)
func = get_function(
None, self.data.get('role', self.manager.config.assume_role),
account_id=account_id)
regions = set([
b.get('Location', {
'LocationConstraint': 'us-east-1'})['LocationConstraint']
for b in buckets])
# session managers by region
region_sessions = {}
for r in regions:
region_sessions[r] = functools.partial(
self.manager.session_factory, region=r)
# Publish function to all of our buckets regions
region_funcs = {}
for r in regions:
lambda_mgr = LambdaManager(region_sessions[r])
lambda_mgr.publish(func)
region_funcs[r] = func
with self.executor_factory(max_workers=3) as w:
results = []
futures = []
for b in buckets:
region = b.get('Location', {
'LocationConstraint': 'us-east-1'}).get(
'LocationConstraint')
futures.append(
w.submit(
self.process_bucket,
region_funcs[region],
b,
account_id,
region_sessions[region]
))
for f in as_completed(futures):
if f.exception():
log.exception(
"Error attaching lambda-encrypt %s" % (f.exception()))
results.append(f.result())
return filter(None, results)
def process(self, buckets):
from c7n.mu import LambdaManager
from c7n.ufuncs.s3crypt import get_function
session = local_session(self.manager.session_factory)
account_id = get_account_id(session)
func = get_function(
None, self.data.get('role', self.manager.config.assume_role),
account_id=account_id)
regions = set([
b.get('Location', {
'LocationConstraint': 'us-east-1'})['LocationConstraint']
for b in buckets])
# session managers by region
region_sessions = {}
for r in regions:
region_sessions[r] = functools.partial(
self.manager.session_factory, region=r)
# Publish function to all of our buckets regions
region_funcs = {}
for r in regions:
lambda_mgr = LambdaManager(region_sessions[r])
lambda_mgr.publish(func)
region_funcs[r] = func
with self.executor_factory(max_workers=3) as w:
results = []
futures = []
for b in buckets:
region = b.get('Location', {
'LocationConstraint': 'us-east-1'}).get(
'LocationConstraint')
futures.append(
w.submit(
self.process_bucket,
region_funcs[region],
b,
account_id,
region_sessions[region]
))
for f in as_completed(futures):
if f.exception():
log.exception(
"Error attaching lambda-encrypt %s" % (f.exception()))
results.append(f.result())
return filter(None, results)
def test_attach_encrypt(self):
self.patch(s3, 'S3_AUGMENT_TABLE', [])
session_factory = self.replay_flight_data('test_s3_attach_encrypt')
bname = "custodian-attach-encrypt-test"
role = 'arn:aws:iam::619193117841:role/lambda_s3_exec_role'
self.maxDiff = None
session = session_factory()
client = session.client('s3')
client.create_bucket(Bucket=bname)
self.addCleanup(destroyBucket, client, bname)
p = self.load_policy(
{
'name': 'attach-encrypt',
'resource': 's3',
'filters': [{
'Name': bname
}],
'actions': [{
'type': 'attach-encrypt',
'role': role
}]
},
session_factory=session_factory)
self.addCleanup(
LambdaManager(session_factory).remove,
s3crypt.get_function(None, role))
resources = p.run()
notifications = client.get_bucket_notification_configuration(
Bucket=bname)
notifications.pop('ResponseMetadata')
self.assertEqual(
notifications, {
'LambdaFunctionConfigurations': [{
'Events': ['s3:ObjectCreated:*'],
'Id':
'custodian-s3-encrypt',
'LambdaFunctionArn':
'arn:aws:lambda:us-east-1:619193117841:function:custodian-s3-encrypt'
}]
})
client.put_object(Bucket=bname,
Key='hello-world.txt',
Body='hello world',
ContentType='text/plain')
info = client.head_object(Bucket=bname, Key='hello-world.txt')
self.assertTrue('ServerSideEncryption' in info)
def test_attach_encrypt(self):
self.patch(s3, 'S3_AUGMENT_TABLE',
[('get_bucket_location', 'Location', None, None)])
self.patch(s3.S3, 'executor_factory', MainThreadExecutor)
session_factory = self.replay_flight_data('test_s3_attach_encrypt')
bname = "custodian-attach-encrypt-test"
role = "arn:aws:iam::644160558196:role/custodian-mu"
self.maxDiff = None
session = session_factory(region='us-west-2')
client = session.client('s3')
client.create_bucket(
Bucket=bname,
CreateBucketConfiguration={
'LocationConstraint': 'us-west-2'})
self.addCleanup(destroyBucket, client, bname)
p = self.load_policy({
'name': 'attach-encrypt',
'resource': 's3',
'filters': [{'Name': bname}],
'actions': [{
'type': 'attach-encrypt',
'role': role}]
}, session_factory=session_factory)
self.addCleanup(
LambdaManager(functools.partial(session_factory, region='us-west-2')).remove,
s3crypt.get_function(None, role))
resources = p.run()
self.assertEqual(len(resources), 1)
#time.sleep(10)
notifications = client.get_bucket_notification_configuration(
Bucket=bname)
notifications.pop('ResponseMetadata')
self.assertEqual(
notifications,
{'LambdaFunctionConfigurations': [{
'Events': ['s3:ObjectCreated:*'],
'Id': 'c7n-s3-encrypt',
'LambdaFunctionArn': 'arn:aws:lambda:us-west-2:644160558196:function:c7n-s3-encrypt'}]})
client.put_object(
Bucket=bname, Key='hello-world.txt',
Body='hello world', ContentType='text/plain')
#time.sleep(30)
info = client.head_object(Bucket=bname, Key='hello-world.txt')
self.assertTrue('ServerSideEncryption' in info)
def test_attach_encrypt(self):
self.patch(s3, 'S3_AUGMENT_TABLE', [])
session_factory = self.replay_flight_data('test_s3_attach_encrypt')
bname = "custodian-attach-encrypt-test"
role = 'arn:aws:iam::619193117841:role/lambda_s3_exec_role'
self.maxDiff = None
session = session_factory()
client = session.client('s3')
client.create_bucket(Bucket=bname)
self.addCleanup(destroyBucket, client, bname)
p = self.load_policy({
'name': 'attach-encrypt',
'resource': 's3',
'filters': [{'Name': bname}],
'actions': [{
'type': 'attach-encrypt',
'role': role}]
}, session_factory=session_factory)
self.addCleanup(
LambdaManager(session_factory).remove,
s3crypt.get_function(None, role))
resources = p.run()
notifications = client.get_bucket_notification_configuration(
Bucket=bname)
notifications.pop('ResponseMetadata')
self.assertEqual(
notifications,
{'LambdaFunctionConfigurations': [{
'Events': ['s3:ObjectCreated:*'],
'Id': 'custodian-s3-encrypt',
'LambdaFunctionArn': 'arn:aws:lambda:us-east-1:619193117841:function:custodian-s3-encrypt'}]})
client.put_object(
Bucket=bname, Key='hello-world.txt',
Body='hello world', ContentType='text/plain')
# For recording
#import time
#time.sleep(7)
info = client.head_object(Bucket=bname, Key='hello-world.txt')
self.assertTrue('ServerSideEncryption' in info)
