def remove_notification(self, bucket_name): try: conf = self.client.get_bucket_notification_configuration( Bucket=bucket_name) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'NoSuchBucket': raise S3Exception('Bucket not found : {0}'.format(bucket_name), 's3-bucket-not-found') elif e.response['Error']['Code'] == 'AccessDenied': raise S3Exception('Access denied: {0}'.format(bucket_name), 's3-access-denied', 'get-bucket-notification') raise e if conf: new_conf = self._update_conf_to_remove_lambda_notification( conf, bucket_name) try: self.client.put_bucket_notification_configuration( Bucket=bucket_name, NotificationConfiguration=new_conf) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'AccessDenied': raise S3Exception('Access denied: {0}'.format(bucket_name), 's3-access-denied', 'put-bucket-notification') raise e
def add_notification(self, bucket_name): conf = { 'LambdaFunctionConfigurations': [ { 'Id': self._notification_id_for_bucket(bucket_name), 'LambdaFunctionArn': settings.S3_LAMBDA_ARN, 'Events': ['s3:ObjectCreated:*'] }, ] } try: self.client.put_bucket_notification_configuration( Bucket=bucket_name, NotificationConfiguration=conf) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'NoSuchBucket': raise S3Exception('Bucket not found: {0}'.format(bucket_name), 's3-bucket-not-found') elif e.response['Error']['Code'] == 'AccessDenied': raise S3Exception('Access denied: {0}'.format(bucket_name), 's3-access-denied', 'put-bucket-notification') elif e.response['Error']['Code'] == 'InvalidArgument': raise S3Exception( 'Bucket does not have permission on lambda function : {0}' ''.format(bucket_name), 's3-no-perms-on-lambda') raise e
def remove_policy_for_lambda(self, bucket_name): statement_id = self._statement_id_for_bucket(bucket_name) existing_policy = self.get_bucket_policy(bucket_name) if not existing_policy: return None exists = [ s for s in existing_policy['Statement'] if s['Sid'] == statement_id ] if not exists: # Policy not present in the bucket return try: if len(existing_policy['Statement']) == 1: self.client.delete_bucket_policy(Bucket=bucket_name) else: new_policy = self._update_policy_to_remove_statement( existing_policy, bucket_name) self.client.put_bucket_policy(Bucket=bucket_name, Policy=json.dumps(new_policy)) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'NoSuchBucket': raise S3Exception('Bucket not found : {0}'.format(bucket_name), 's3-bucket-not-found') elif e.response['Error']['Code'] == 'AccessDenied': raise S3Exception('Access denied: {0}'.format(bucket_name), 's3-access-denied', 'put-bucket-policy') raise e
def get_bucket_policy(self, bucket_name): try: policy = self.client.get_bucket_policy(Bucket=bucket_name) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'NoSuchBucketPolicy': # Bucket has no policy return None elif e.response['Error']['Code'] == 'NoSuchBucket': raise S3Exception('Bucket not found : {0}'.format(bucket_name), 's3-bucket-not-found') elif e.response['Error']['Code'] == 'AccessDenied': raise S3Exception('Access denied: {0}'.format(bucket_name), 's3-access-denied', 'get-bucket-policy') raise e return json.loads(policy['Policy'])
def test_set_up_bucket_on_aws_lambda_add_permission_fails_revert_also_fails(): args = ('mock_access_key_id', 'mock_secret_access_key', 'test_bucket') with mock.patch('goodtablesio.integrations.s3.utils.bucket._check_connection'), \ mock.patch('goodtablesio.integrations.s3.utils.bucket._add_policy'), \ mock.patch.object(LambdaClient, 'add_permission_to_bucket') as a, \ mock.patch.object(S3Client, 'remove_policy_for_lambda') as b: a.side_effect = S3Exception('Access denied', 's3-access-denied', 'add-permission') b.side_effect = S3Exception('Access denied', 's3-access-denied', 'remove-policy') assert set_up_bucket_on_aws( *args) == (False, 'Access denied (add-permission)')
def test_set_up_bucket_on_aws_lambda_connection_error(mock_s3_client): args = ('mock_access_key_id', 'mock_secret_access_key', 'test_bucket') with mock.patch.object(LambdaClient, 'check_connection') as mock_call: mock_call.side_effect = S3Exception( 'Could not connect to the Lambda endpoint', 's3-connection-error') assert set_up_bucket_on_aws( *args) == (False, 'Could not connect to the Lambda endpoint')
def add_policy_for_lambda(self, bucket_name): existing_policy = self.get_bucket_policy(bucket_name) new_policy = self._update_policy_to_add_statement( existing_policy, bucket_name) if not new_policy: return None try: self.client.put_bucket_policy(Bucket=bucket_name, Policy=json.dumps(new_policy)) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'NoSuchBucket': raise S3Exception('Bucket not found : {0}'.format(bucket_name), 's3-bucket-not-found') elif e.response['Error']['Code'] == 'AccessDenied': raise S3Exception('Access denied: {0}'.format(bucket_name), 's3-access-denied', 'put-bucket-policy') raise e
def check_connection(self, bucket_name): try: return self.client.head_bucket(Bucket=bucket_name) except botocore.exceptions.ParamValidationError: raise S3Exception('Invalid bucket name: {}'.format(bucket_name), 's3-invalid-bucket-name') except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'EndpointConnectionError': raise S3Exception( 'Could not connect to the S3 endpoint: {}'.format(e), 's3-connection-error') elif e.response['Error']['Code'] == 'NoSuchBucket': raise S3Exception('Bucket not found: {}'.format(bucket_name), 's3-bucket-not-found') elif e.response['Error']['Code'] == 'InvalidAccessKeyId': raise S3Exception('Invalid Access Key', 's3-invalid-access-key') elif e.response['Error']['Code'] == 'SignatureDoesNotMatch': raise S3Exception( 'Invalid signature, please check your secret key', 's3-invalid-signature') elif e.response['Error']['Code'] == 'AccessDeniedException': raise S3Exception('Access denied', 's3-access-denied', 'get-bucket-policy') raise e
def test_set_up_bucket_on_aws_lambda_permission_already_exists_passes(): args = ('mock_access_key_id', 'mock_secret_access_key', 'test_bucket') with mock.patch('goodtablesio.integrations.s3.utils.bucket._check_connection'), \ mock.patch('goodtablesio.integrations.s3.utils.bucket._add_policy'), \ mock.patch.object(LambdaClient, 'add_permission_to_bucket') as a, \ mock.patch('goodtablesio.integrations.s3.utils.bucket._add_notification'): a.side_effect = S3Exception('Permission already exists', 's3-bucket-has-already-perm-on-lambda') assert set_up_bucket_on_aws(*args) == (True, '')
def test_set_up_bucket_on_aws_s3_add_policy_access_denied(): args = ('mock_access_key_id', 'mock_secret_access_key', 'test_bucket') with mock.patch('goodtablesio.integrations.s3.utils.bucket._check_connection'), \ mock.patch.object(S3Client, 'add_policy_for_lambda') as mock_call: mock_call.side_effect = S3Exception('Access denied', 's3-access-denied', 'get-bucket-policy') assert set_up_bucket_on_aws( *args) == (False, 'Access denied (get-bucket-policy)')
def test_disable_bucket_on_aws_lambda_permission_not_exists_passes(): args = ('mock_access_key_id', 'mock_secret_access_key', 'test_bucket') with mock.patch('goodtablesio.integrations.s3.utils.bucket._check_connection'), \ mock.patch('goodtablesio.integrations.s3.utils.bucket._remove_policy'), \ mock.patch.object(LambdaClient, 'remove_permission_to_bucket') as a, \ mock.patch('goodtablesio.integrations.s3.utils.bucket._remove_notification'): a.side_effect = S3Exception('Permission does not exist', 's3-lambda-perm-not-found') assert disable_bucket_on_aws(*args) == (True, '')
def test_disable_bucket_on_aws_s3_remove_notification_fails(): args = ('mock_access_key_id', 'mock_secret_access_key', 'test_bucket') with mock.patch('goodtablesio.integrations.s3.utils.bucket._check_connection'), \ mock.patch('goodtablesio.integrations.s3.utils.bucket._remove_policy'), \ mock.patch('goodtablesio.integrations.s3.utils.bucket._remove_lambda_permission'), \ mock.patch.object(S3Client, 'remove_notification') as a: a.side_effect = S3Exception('Access denied', 's3-access-denied', 'remove-notification') assert disable_bucket_on_aws( *args) == (False, 'Access denied (remove-notification)')
def check_connection(self): try: return self.client.get_function(FunctionName=self.lambda_arn) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'EndpointConnectionError': raise S3Exception( 'Could not connect to the Lambda endpoint: {}'.format(e), 's3-connection-error') elif e.response['Error']['Code'] == 'InvalidAccessKeyId': raise S3Exception('Invalid Access Key', 's3-invalid-access-key') elif e.response['Error']['Code'] == 'SignatureDoesNotMatch': raise S3Exception( 'Invalid signature, please check your secret key', 's3-invalid-signature') elif e.response['Error']['Code'] == 'ResourceNotFound': raise S3Exception( 'AWS Lambda function not found: {}'.format(e), 's3-lambda-not-found') elif e.response['Error']['Code'] == 'AccessDeniedException': raise S3Exception('Access denied', 's3-access-denied', 'get-function') raise e
def remove_permission_to_bucket(self, bucket_name): statement_id = self._statement_id_for_bucket(bucket_name) try: self.client.remove_permission(FunctionName=self.lambda_arn, StatementId=statement_id) log.debug('Removed permission for bucket {0} on lambda'.format( bucket_name)) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'ResourceNotFoundException': raise S3Exception( 'Permission not found : {0}'.format(statement_id), 's3-lambda-perm-not-found') raise e
def add_permission_to_bucket(self, bucket_name): statement_id = self._statement_id_for_bucket(bucket_name) try: self.client.add_permission( FunctionName=self.lambda_arn, StatementId=statement_id, Action='lambda:InvokeFunction', Principal='s3.amazonaws.com', SourceArn=self._arn_for_bucket(bucket_name)) log.debug( 'Added permission to bucket {0} on lambda'.format(bucket_name)) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'ResourceConflictException': raise S3Exception( 'Bucket already has permission on function: {}'.format( bucket_name), 's3-bucket-has-already-perm-on-lambda') raise e