def process(self, buckets): from c7n.mu import LambdaManager from c7n.ufuncs.s3crypt import get_function func = get_function( None, self.data.get('role', self.manager.config.assume_role)) # Publish function to all of our buckets regions region_funcs = {} regions = set([ b.get('LocationConstraint', 'us-east-1') for b in buckets]) for r in regions: lambda_mgr = LambdaManager( functools.partial(self.manager.session_factory, region=r)) region_funcs[r] = lambda_mgr.publish(func) with self.executor_factory(max_workers=3) as w: results = [] futures = [] for b in buckets: futures.append( w.submit( self.process_bucket, region_funcs[b.get('LocationConstraint', 'us-east-1')], b)) for f in as_completed(futures): if f.exception(): log.exception( "Error attaching lambda-encrypt %s" % (f.exception())) results.append(f.result()) return filter(None, results)
def process(self, buckets): from c7n.mu import LambdaManager from c7n.ufuncs.s3crypt import get_function func = get_function( None, self.data.get('role', self.manager.config.assume_role)) # Publish function to all of our buckets regions region_funcs = {} regions = set( [b.get('LocationConstraint', 'us-east-1') for b in buckets]) for r in regions: lambda_mgr = LambdaManager( functools.partial(self.manager.session_factory, region=r)) region_funcs[r] = lambda_mgr.publish(func) with self.executor_factory(max_workers=3) as w: results = [] futures = [] for b in buckets: futures.append( w.submit( self.process_bucket, region_funcs[b.get('LocationConstraint', 'us-east-1')], b)) for f in as_completed(futures): if f.exception(): log.exception("Error attaching lambda-encrypt %s" % (f.exception())) results.append(f.result()) return filter(None, results)
def test_attach_encrypt(self): self.patch(s3, 'S3_AUGMENT_TABLE', [('get_bucket_location', 'Location', None, None)]) self.patch(s3.S3, 'executor_factory', MainThreadExecutor) session_factory = self.replay_flight_data('test_s3_attach_encrypt') bname = "custodian-attach-encrypt-test" role = "arn:aws:iam::644160558196:role/custodian-mu" self.maxDiff = None session = session_factory(region='us-west-2') client = session.client('s3') client.create_bucket( Bucket=bname, CreateBucketConfiguration={'LocationConstraint': 'us-west-2'}) self.addCleanup(destroyBucket, client, bname) p = self.load_policy( { 'name': 'attach-encrypt', 'resource': 's3', 'filters': [{ 'Name': bname }], 'actions': [{ 'type': 'attach-encrypt', 'role': role }] }, session_factory=session_factory) self.addCleanup( LambdaManager( functools.partial(session_factory, region='us-west-2')).remove, s3crypt.get_function(None, role)) resources = p.run() self.assertEqual(len(resources), 1) #time.sleep(10) notifications = client.get_bucket_notification_configuration( Bucket=bname) notifications.pop('ResponseMetadata') self.assertEqual( notifications, { 'LambdaFunctionConfigurations': [{ 'Events': ['s3:ObjectCreated:*'], 'Id': 'c7n-s3-encrypt', 'LambdaFunctionArn': 'arn:aws:lambda:us-west-2:644160558196:function:c7n-s3-encrypt' }] }) client.put_object(Bucket=bname, Key='hello-world.txt', Body='hello world', ContentType='text/plain') #time.sleep(30) info = client.head_object(Bucket=bname, Key='hello-world.txt') self.assertTrue('ServerSideEncryption' in info)
def process(self, buckets): from c7n.mu import LambdaManager from c7n.ufuncs.s3crypt import get_function session = local_session(self.manager.session_factory) account_id = get_account_id(session) func = get_function( None, self.data.get('role', self.manager.config.assume_role), account_id=account_id) regions = set([ b.get('Location', { 'LocationConstraint': 'us-east-1'})['LocationConstraint'] for b in buckets]) # session managers by region region_sessions = {} for r in regions: region_sessions[r] = functools.partial( self.manager.session_factory, region=r) # Publish function to all of our buckets regions region_funcs = {} for r in regions: lambda_mgr = LambdaManager(region_sessions[r]) lambda_mgr.publish(func) region_funcs[r] = func with self.executor_factory(max_workers=3) as w: results = [] futures = [] for b in buckets: region = b.get('Location', { 'LocationConstraint': 'us-east-1'}).get( 'LocationConstraint') futures.append( w.submit( self.process_bucket, region_funcs[region], b, account_id, region_sessions[region] )) for f in as_completed(futures): if f.exception(): log.exception( "Error attaching lambda-encrypt %s" % (f.exception())) results.append(f.result()) return filter(None, results)
def test_attach_encrypt(self): self.patch(s3, 'S3_AUGMENT_TABLE', []) session_factory = self.replay_flight_data('test_s3_attach_encrypt') bname = "custodian-attach-encrypt-test" role = 'arn:aws:iam::619193117841:role/lambda_s3_exec_role' self.maxDiff = None session = session_factory() client = session.client('s3') client.create_bucket(Bucket=bname) self.addCleanup(destroyBucket, client, bname) p = self.load_policy( { 'name': 'attach-encrypt', 'resource': 's3', 'filters': [{ 'Name': bname }], 'actions': [{ 'type': 'attach-encrypt', 'role': role }] }, session_factory=session_factory) self.addCleanup( LambdaManager(session_factory).remove, s3crypt.get_function(None, role)) resources = p.run() notifications = client.get_bucket_notification_configuration( Bucket=bname) notifications.pop('ResponseMetadata') self.assertEqual( notifications, { 'LambdaFunctionConfigurations': [{ 'Events': ['s3:ObjectCreated:*'], 'Id': 'custodian-s3-encrypt', 'LambdaFunctionArn': 'arn:aws:lambda:us-east-1:619193117841:function:custodian-s3-encrypt' }] }) client.put_object(Bucket=bname, Key='hello-world.txt', Body='hello world', ContentType='text/plain') info = client.head_object(Bucket=bname, Key='hello-world.txt') self.assertTrue('ServerSideEncryption' in info)
def test_attach_encrypt(self): self.patch(s3, 'S3_AUGMENT_TABLE', [('get_bucket_location', 'Location', None, None)]) self.patch(s3.S3, 'executor_factory', MainThreadExecutor) session_factory = self.replay_flight_data('test_s3_attach_encrypt') bname = "custodian-attach-encrypt-test" role = "arn:aws:iam::644160558196:role/custodian-mu" self.maxDiff = None session = session_factory(region='us-west-2') client = session.client('s3') client.create_bucket( Bucket=bname, CreateBucketConfiguration={ 'LocationConstraint': 'us-west-2'}) self.addCleanup(destroyBucket, client, bname) p = self.load_policy({ 'name': 'attach-encrypt', 'resource': 's3', 'filters': [{'Name': bname}], 'actions': [{ 'type': 'attach-encrypt', 'role': role}] }, session_factory=session_factory) self.addCleanup( LambdaManager(functools.partial(session_factory, region='us-west-2')).remove, s3crypt.get_function(None, role)) resources = p.run() self.assertEqual(len(resources), 1) #time.sleep(10) notifications = client.get_bucket_notification_configuration( Bucket=bname) notifications.pop('ResponseMetadata') self.assertEqual( notifications, {'LambdaFunctionConfigurations': [{ 'Events': ['s3:ObjectCreated:*'], 'Id': 'c7n-s3-encrypt', 'LambdaFunctionArn': 'arn:aws:lambda:us-west-2:644160558196:function:c7n-s3-encrypt'}]}) client.put_object( Bucket=bname, Key='hello-world.txt', Body='hello world', ContentType='text/plain') #time.sleep(30) info = client.head_object(Bucket=bname, Key='hello-world.txt') self.assertTrue('ServerSideEncryption' in info)
def test_attach_encrypt(self): self.patch(s3, 'S3_AUGMENT_TABLE', []) session_factory = self.replay_flight_data('test_s3_attach_encrypt') bname = "custodian-attach-encrypt-test" role = 'arn:aws:iam::619193117841:role/lambda_s3_exec_role' self.maxDiff = None session = session_factory() client = session.client('s3') client.create_bucket(Bucket=bname) self.addCleanup(destroyBucket, client, bname) p = self.load_policy({ 'name': 'attach-encrypt', 'resource': 's3', 'filters': [{'Name': bname}], 'actions': [{ 'type': 'attach-encrypt', 'role': role}] }, session_factory=session_factory) self.addCleanup( LambdaManager(session_factory).remove, s3crypt.get_function(None, role)) resources = p.run() notifications = client.get_bucket_notification_configuration( Bucket=bname) notifications.pop('ResponseMetadata') self.assertEqual( notifications, {'LambdaFunctionConfigurations': [{ 'Events': ['s3:ObjectCreated:*'], 'Id': 'custodian-s3-encrypt', 'LambdaFunctionArn': 'arn:aws:lambda:us-east-1:619193117841:function:custodian-s3-encrypt'}]}) client.put_object( Bucket=bname, Key='hello-world.txt', Body='hello world', ContentType='text/plain') # For recording #import time #time.sleep(7) info = client.head_object(Bucket=bname, Key='hello-world.txt') self.assertTrue('ServerSideEncryption' in info)