def remove_notification(self, bucket_name):
try:
conf = self.client.get_bucket_notification_configuration(
Bucket=bucket_name)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'NoSuchBucket':
raise S3Exception('Bucket not found : {0}'.format(bucket_name),
's3-bucket-not-found')
elif e.response['Error']['Code'] == 'AccessDenied':
raise S3Exception('Access denied: {0}'.format(bucket_name),
's3-access-denied',
'get-bucket-notification')
raise e
if conf:
new_conf = self._update_conf_to_remove_lambda_notification(
conf, bucket_name)
try:
self.client.put_bucket_notification_configuration(
Bucket=bucket_name, NotificationConfiguration=new_conf)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'AccessDenied':
raise S3Exception('Access denied: {0}'.format(bucket_name),
's3-access-denied',
'put-bucket-notification')
raise e
def add_notification(self, bucket_name):
conf = {
'LambdaFunctionConfigurations': [
{
'Id': self._notification_id_for_bucket(bucket_name),
'LambdaFunctionArn': settings.S3_LAMBDA_ARN,
'Events': ['s3:ObjectCreated:*']
},
]
}
try:
self.client.put_bucket_notification_configuration(
Bucket=bucket_name, NotificationConfiguration=conf)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'NoSuchBucket':
raise S3Exception('Bucket not found: {0}'.format(bucket_name),
's3-bucket-not-found')
elif e.response['Error']['Code'] == 'AccessDenied':
raise S3Exception('Access denied: {0}'.format(bucket_name),
's3-access-denied',
'put-bucket-notification')
elif e.response['Error']['Code'] == 'InvalidArgument':
raise S3Exception(
'Bucket does not have permission on lambda function : {0}'
''.format(bucket_name), 's3-no-perms-on-lambda')
raise e
def remove_policy_for_lambda(self, bucket_name):
statement_id = self._statement_id_for_bucket(bucket_name)
existing_policy = self.get_bucket_policy(bucket_name)
if not existing_policy:
return None
exists = [
s for s in existing_policy['Statement'] if s['Sid'] == statement_id
]
if not exists:
# Policy not present in the bucket
return
try:
if len(existing_policy['Statement']) == 1:
self.client.delete_bucket_policy(Bucket=bucket_name)
else:
new_policy = self._update_policy_to_remove_statement(
existing_policy, bucket_name)
self.client.put_bucket_policy(Bucket=bucket_name,
Policy=json.dumps(new_policy))
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'NoSuchBucket':
raise S3Exception('Bucket not found : {0}'.format(bucket_name),
's3-bucket-not-found')
elif e.response['Error']['Code'] == 'AccessDenied':
raise S3Exception('Access denied: {0}'.format(bucket_name),
's3-access-denied', 'put-bucket-policy')
raise e
def get_bucket_policy(self, bucket_name):
try:
policy = self.client.get_bucket_policy(Bucket=bucket_name)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'NoSuchBucketPolicy':
# Bucket has no policy
return None
elif e.response['Error']['Code'] == 'NoSuchBucket':
raise S3Exception('Bucket not found : {0}'.format(bucket_name),
's3-bucket-not-found')
elif e.response['Error']['Code'] == 'AccessDenied':
raise S3Exception('Access denied: {0}'.format(bucket_name),
's3-access-denied', 'get-bucket-policy')
raise e
return json.loads(policy['Policy'])
def test_set_up_bucket_on_aws_lambda_add_permission_fails_revert_also_fails():
args = ('mock_access_key_id', 'mock_secret_access_key', 'test_bucket')
with mock.patch('goodtablesio.integrations.s3.utils.bucket._check_connection'), \
mock.patch('goodtablesio.integrations.s3.utils.bucket._add_policy'), \
mock.patch.object(LambdaClient, 'add_permission_to_bucket') as a, \
mock.patch.object(S3Client, 'remove_policy_for_lambda') as b:
a.side_effect = S3Exception('Access denied', 's3-access-denied',
'add-permission')
b.side_effect = S3Exception('Access denied', 's3-access-denied',
'remove-policy')
assert set_up_bucket_on_aws(
*args) == (False, 'Access denied (add-permission)')
def test_set_up_bucket_on_aws_lambda_connection_error(mock_s3_client):
args = ('mock_access_key_id', 'mock_secret_access_key', 'test_bucket')
with mock.patch.object(LambdaClient, 'check_connection') as mock_call:
mock_call.side_effect = S3Exception(
'Could not connect to the Lambda endpoint', 's3-connection-error')
assert set_up_bucket_on_aws(
*args) == (False, 'Could not connect to the Lambda endpoint')
def add_policy_for_lambda(self, bucket_name):
existing_policy = self.get_bucket_policy(bucket_name)
new_policy = self._update_policy_to_add_statement(
existing_policy, bucket_name)
if not new_policy:
return None
try:
self.client.put_bucket_policy(Bucket=bucket_name,
Policy=json.dumps(new_policy))
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'NoSuchBucket':
raise S3Exception('Bucket not found : {0}'.format(bucket_name),
's3-bucket-not-found')
elif e.response['Error']['Code'] == 'AccessDenied':
raise S3Exception('Access denied: {0}'.format(bucket_name),
's3-access-denied', 'put-bucket-policy')
raise e
def check_connection(self, bucket_name):
try:
return self.client.head_bucket(Bucket=bucket_name)
except botocore.exceptions.ParamValidationError:
raise S3Exception('Invalid bucket name: {}'.format(bucket_name),
's3-invalid-bucket-name')
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'EndpointConnectionError':
raise S3Exception(
'Could not connect to the S3 endpoint: {}'.format(e),
's3-connection-error')
elif e.response['Error']['Code'] == 'NoSuchBucket':
raise S3Exception('Bucket not found: {}'.format(bucket_name),
's3-bucket-not-found')
elif e.response['Error']['Code'] == 'InvalidAccessKeyId':
raise S3Exception('Invalid Access Key',
's3-invalid-access-key')
elif e.response['Error']['Code'] == 'SignatureDoesNotMatch':
raise S3Exception(
'Invalid signature, please check your secret key',
's3-invalid-signature')
elif e.response['Error']['Code'] == 'AccessDeniedException':
raise S3Exception('Access denied', 's3-access-denied',
'get-bucket-policy')
raise e
def test_set_up_bucket_on_aws_lambda_permission_already_exists_passes():
args = ('mock_access_key_id', 'mock_secret_access_key', 'test_bucket')
with mock.patch('goodtablesio.integrations.s3.utils.bucket._check_connection'), \
mock.patch('goodtablesio.integrations.s3.utils.bucket._add_policy'), \
mock.patch.object(LambdaClient, 'add_permission_to_bucket') as a, \
mock.patch('goodtablesio.integrations.s3.utils.bucket._add_notification'):
a.side_effect = S3Exception('Permission already exists',
's3-bucket-has-already-perm-on-lambda')
assert set_up_bucket_on_aws(*args) == (True, '')
def test_set_up_bucket_on_aws_s3_add_policy_access_denied():
args = ('mock_access_key_id', 'mock_secret_access_key', 'test_bucket')
with mock.patch('goodtablesio.integrations.s3.utils.bucket._check_connection'), \
mock.patch.object(S3Client, 'add_policy_for_lambda') as mock_call:
mock_call.side_effect = S3Exception('Access denied',
's3-access-denied',
'get-bucket-policy')
assert set_up_bucket_on_aws(
*args) == (False, 'Access denied (get-bucket-policy)')
def test_disable_bucket_on_aws_lambda_permission_not_exists_passes():
args = ('mock_access_key_id', 'mock_secret_access_key', 'test_bucket')
with mock.patch('goodtablesio.integrations.s3.utils.bucket._check_connection'), \
mock.patch('goodtablesio.integrations.s3.utils.bucket._remove_policy'), \
mock.patch.object(LambdaClient, 'remove_permission_to_bucket') as a, \
mock.patch('goodtablesio.integrations.s3.utils.bucket._remove_notification'):
a.side_effect = S3Exception('Permission does not exist',
's3-lambda-perm-not-found')
assert disable_bucket_on_aws(*args) == (True, '')
def test_disable_bucket_on_aws_s3_remove_notification_fails():
args = ('mock_access_key_id', 'mock_secret_access_key', 'test_bucket')
with mock.patch('goodtablesio.integrations.s3.utils.bucket._check_connection'), \
mock.patch('goodtablesio.integrations.s3.utils.bucket._remove_policy'), \
mock.patch('goodtablesio.integrations.s3.utils.bucket._remove_lambda_permission'), \
mock.patch.object(S3Client, 'remove_notification') as a:
a.side_effect = S3Exception('Access denied', 's3-access-denied',
'remove-notification')
assert disable_bucket_on_aws(
*args) == (False, 'Access denied (remove-notification)')
def check_connection(self):
try:
return self.client.get_function(FunctionName=self.lambda_arn)
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'EndpointConnectionError':
raise S3Exception(
'Could not connect to the Lambda endpoint: {}'.format(e),
's3-connection-error')
elif e.response['Error']['Code'] == 'InvalidAccessKeyId':
raise S3Exception('Invalid Access Key',
's3-invalid-access-key')
elif e.response['Error']['Code'] == 'SignatureDoesNotMatch':
raise S3Exception(
'Invalid signature, please check your secret key',
's3-invalid-signature')
elif e.response['Error']['Code'] == 'ResourceNotFound':
raise S3Exception(
'AWS Lambda function not found: {}'.format(e),
's3-lambda-not-found')
elif e.response['Error']['Code'] == 'AccessDeniedException':
raise S3Exception('Access denied', 's3-access-denied',
'get-function')
raise e
def remove_permission_to_bucket(self, bucket_name):
statement_id = self._statement_id_for_bucket(bucket_name)
try:
self.client.remove_permission(FunctionName=self.lambda_arn,
StatementId=statement_id)
log.debug('Removed permission for bucket {0} on lambda'.format(
bucket_name))
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceNotFoundException':
raise S3Exception(
'Permission not found : {0}'.format(statement_id),
's3-lambda-perm-not-found')
raise e
def add_permission_to_bucket(self, bucket_name):
statement_id = self._statement_id_for_bucket(bucket_name)
try:
self.client.add_permission(
FunctionName=self.lambda_arn,
StatementId=statement_id,
Action='lambda:InvokeFunction',
Principal='s3.amazonaws.com',
SourceArn=self._arn_for_bucket(bucket_name))
log.debug(
'Added permission to bucket {0} on lambda'.format(bucket_name))
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == 'ResourceConflictException':
raise S3Exception(
'Bucket already has permission on function: {}'.format(
bucket_name), 's3-bucket-has-already-perm-on-lambda')
raise e
