def create_bucket(bucket_name, region):
"""
Creates a S3 bucket in a specific region
Parameters
----------
bucket_name : string
Bucket name
region : string
Region name
Raises
------
NoRegionError
If region is not specified
"""
if region is None:
raise NoRegionError()
if region == "us-east-1":
s3_client = boto3.client("s3")
s3_client.create_bucket(Bucket=bucket_name)
else:
s3_client = boto3.client("s3", region_name=region)
location = {"LocationConstraint": region}
s3_client.create_bucket(Bucket=bucket_name,
CreateBucketConfiguration=location)
def construct_endpoint(self, service_name, region_name, **kwargs):
# We take **kwargs so that custom rules can be added that have
# additional constraint keys that we don't know about.
# We also need to fold the names used in the spec into **kwargs
# so we can format the URI later.
kwargs['service'] = service_name
kwargs['region'] = region_name
if 'scheme' not in kwargs:
kwargs['scheme'] = self.DEFAULT_SCHEME
service_rules = self._rules.get(service_name, [])
endpoint = self._match_rules(service_rules, region_name, **kwargs)
if endpoint is None:
# If we didn't find any in the service section, try again
# with the default section.
endpoint = self._match_rules(self._rules.get('_default', []),
region_name, **kwargs)
if endpoint is None:
if region_name is None:
# Raise a more specific error message that will give
# better guidance to the user what needs to happen.
raise NoRegionError()
else:
raise UnknownEndpointError(service_name=service_name,
region_name=region_name)
return endpoint
def _endpoint_for_partition(self,
partition,
service_name,
region_name,
force_partition=False):
# Get the service from the partition, or an empty template.
service_data = partition['services'].get(service_name,
DEFAULT_SERVICE_DATA)
# Use the partition endpoint if no region is supplied.
if region_name is None:
if 'partitionEndpoint' in service_data:
region_name = service_data['partitionEndpoint']
else:
raise NoRegionError()
# Attempt to resolve the exact region for this partition.
if region_name in service_data['endpoints']:
return self._resolve(partition, service_name, service_data,
region_name)
# Check to see if the endpoint provided is valid for the partition.
if self._region_match(partition, region_name) or force_partition:
# Use the partition endpoint if set and not regionalized.
partition_endpoint = service_data.get('partitionEndpoint')
is_regionalized = service_data.get('isRegionalized', True)
if partition_endpoint and not is_regionalized:
LOG.debug('Using partition endpoint for %s, %s: %s',
service_name, region_name, partition_endpoint)
return self._resolve(partition, service_name, service_data,
partition_endpoint)
LOG.debug('Creating a regex based endpoint for %s, %s',
service_name, region_name)
return self._resolve(partition, service_name, service_data,
region_name)
def _get_signature_version_and_region(self, endpoint, service_model):
# An endpoint-aware signature version and region check
scoped_config = self.session.get_scoped_config()
resolver = self.session.get_component('endpoint_resolver')
scheme = endpoint.host.split(':')[0]
if endpoint.region_name is None:
raise NoRegionError()
endpoint_config = resolver.construct_endpoint(
service_model.endpoint_prefix, endpoint.region_name, scheme=scheme)
# Signature version override from endpoint
signature_version = self.service.signature_version
if 'signatureVersion' in endpoint_config.get('properties', {}):
signature_version = endpoint_config['properties']\
['signatureVersion']
# Signature overrides from a configuration file
if scoped_config is not None:
service_config = scoped_config.get(service_model.endpoint_prefix)
if service_config is not None and isinstance(service_config, dict):
override = service_config.get('signature_version')
if override:
logger.debug(
"Switching signature version for service %s "
"to version %s based on config file override.",
service_model.endpoint_prefix, override)
signature_version = override
return signature_version, endpoint.region_name
def test_boto3_raises_exceptions_fail(self, boto3, config):
boto3.session.Session.side_effect = PartialCredentialsError(
provider=Mock(), cred_var=Mock())
assert CWDataSource(config).test() == False
boto3.session.Session.side_effect = ClientError(MagicMock(), Mock())
assert CWDataSource(config).test() == False
boto3.session.Session.side_effect = NoRegionError()
assert CWDataSource(config).test() == False
def test_no_completions_when_cant_create_client(describer_creator):
client_creator = mock.Mock(spec=index.CachedClientCreator)
# This is raised when you don't have a region configured via config file
# env var or manually via a session.
client_creator.create_client.side_effect = NoRegionError()
completer = index.ServerSideCompleter(client_creator=client_creator,
describer_creator=describer_creator)
assert completer.retrieve_candidate_values('ec2', 'foo', 'Bar') == []
def test_raises_boto3clienterror(self, boto3, config):
boto3.session.Session.side_effect = PartialCredentialsError(
provider=Mock(), cred_var=Mock())
with pytest.raises(Boto3ClientError):
CWDataSource(config)._cw_client()
boto3.session.Session.side_effect = ClientError(MagicMock(), Mock())
with pytest.raises(Boto3ClientError):
CWDataSource(config)._cw_client()
boto3.session.Session.side_effect = NoRegionError()
with pytest.raises(Boto3ClientError):
CWDataSource(config)._cw_client()
def manage_stack(
region: str,
stack_name: str,
template_body: str,
profile: Optional[str] = None,
parameter_overrides: Optional[Dict[str, Union[str, List[str]]]] = None,
) -> List[Dict[str, str]]:
"""
get or create a CloudFormation stack
Parameters
----------
region: str
AWS region for the CloudFormation stack
stack_name: str
CloudFormation stack name
template_body: str
CloudFormation template's content
profile: Optional[str]
AWS named profile for the AWS account
parameter_overrides: Optional[Dict[str, Union[str, List[str]]]]
Values of template parameters, if any.
Returns: Stack output section(list of OutputKey, OutputValue pairs)
"""
try:
if profile:
session = boto3.Session(
profile_name=profile,
region_name=region if region else None) # type: ignore
cloudformation_client = session.client("cloudformation")
else:
cloudformation_client = boto3.client(
"cloudformation",
config=Config(region_name=region if region else None))
except ProfileNotFound as ex:
raise ProfileNotFound(
f"Error Setting Up Managed Stack Client: the provided AWS name profile '{profile}' is not found. "
"please check the documentation for setting up a named profile: "
"https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html"
) from ex
except NoCredentialsError as ex:
raise NoCredentialsError(
"Error Setting Up Managed Stack Client: Unable to resolve credentials for the AWS SDK for Python client. "
"Please see their documentation for options to pass in credentials: "
"https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html"
) from ex
except NoRegionError as ex:
raise NoRegionError(
"Error Setting Up Managed Stack Client: Unable to resolve a region. "
"Please provide a region via the --region parameter or by the AWS_REGION environment variable."
) from ex
return _create_or_get_stack(cloudformation_client, stack_name,
template_body, parameter_overrides)
def _endpoint_for_partition(self,
partition,
service_name,
region_name,
use_dualstack_endpoint,
use_fips_endpoint,
force_partition=False):
partition_name = partition["partition"]
if (use_dualstack_endpoint
and partition_name in self._UNSUPPORTED_DUALSTACK_PARTITIONS):
error_msg = ("Dualstack endpoints are currently not supported"
" for %s partition" % partition_name)
raise EndpointVariantError(tags=['dualstack'], error_msg=error_msg)
# Get the service from the partition, or an empty template.
service_data = partition['services'].get(service_name,
DEFAULT_SERVICE_DATA)
# Use the partition endpoint if no region is supplied.
if region_name is None:
if 'partitionEndpoint' in service_data:
region_name = service_data['partitionEndpoint']
else:
raise NoRegionError()
resolve_kwargs = {
'partition': partition,
'service_name': service_name,
'service_data': service_data,
'endpoint_name': region_name,
'use_dualstack_endpoint': use_dualstack_endpoint,
'use_fips_endpoint': use_fips_endpoint,
}
# Attempt to resolve the exact region for this partition.
if region_name in service_data['endpoints']:
return self._resolve(**resolve_kwargs)
# Check to see if the endpoint provided is valid for the partition.
if self._region_match(partition, region_name) or force_partition:
# Use the partition endpoint if set and not regionalized.
partition_endpoint = service_data.get('partitionEndpoint')
is_regionalized = service_data.get('isRegionalized', True)
if partition_endpoint and not is_regionalized:
LOG.debug('Using partition endpoint for %s, %s: %s',
service_name, region_name, partition_endpoint)
resolve_kwargs['endpoint_name'] = partition_endpoint
return self._resolve(**resolve_kwargs)
LOG.debug('Creating a regex based endpoint for %s, %s',
service_name, region_name)
return self._resolve(**resolve_kwargs)
def test_check_config_unauthorized_error(
aws: AWS, boto: Mock, caplog: LogCaptureFixture
) -> None:
"""
Given: An incorrectly configured AWS adapter object.
When: Configuration is checked.
Then: The user is informed of the incorrect state and an exception is raised.
"""
boto.client.side_effect = NoRegionError()
with pytest.raises(AWSConfigurationError):
aws.check_configuration()
assert ("drode.adapters.aws", logging.ERROR, "AWS: KO") in caplog.record_tuples
def _no_region_error():
raise NoRegionError()
def test_client_missing_region(self, boto_mock):
boto_mock.side_effect = NoRegionError()
with self.assertRaises(RegionError):
manage_stack("testprofile", "fake-region")
class SQSSensorTestCase(BaseSensorTestCase):
sensor_cls = AWSSQSSensor
class MockResource(object):
def __init__(self, msgs=[]):
self.msgs = msgs
def get_queue_by_name(self, **kwargs):
return SQSSensorTestCase.MockQueue(self.msgs)
def Queue(self, queue):
return SQSSensorTestCase.MockQueue(self.msgs)
class MockResourceNonExistentQueue(object):
def __init__(self, msgs=[]):
self.msgs = msgs
def get_queue_by_name(self, **kwargs):
raise ClientError(
{'Error': {
'Code': 'AWS.SimpleQueueService.NonExistentQueue'
}}, 'sqs_test')
def Queue(self, queue):
raise ClientError(
{'Error': {
'Code': 'AWS.SimpleQueueService.NonExistentQueue'
}}, 'sqs_test')
def create_queue(self, **kwargs):
return SQSSensorTestCase.MockQueue(self.msgs)
class MockResourceRaiseClientError(object):
def __init__(self, error_code=''):
self.error_code = error_code
def get_queue_by_name(self, **kwargs):
raise ClientError({'Error': {'Code': self.error_code}}, 'sqs_test')
def Queue(self, queue):
raise ClientError({'Error': {'Code': self.error_code}}, 'sqs_test')
class MockResourceRaiseNoCredentialsError(object):
def get_queue_by_name(self, **kwargs):
raise NoCredentialsError()
def Queue(self, queue):
raise NoCredentialsError()
class MockResourceRaiseEndpointConnectionError(object):
def get_queue_by_name(self, **kwargs):
raise EndpointConnectionError(endpoint_url='')
def Queue(self, queue):
raise EndpointConnectionError(endpoint_url='')
class MockStsClient(object):
def __init__(self):
self.meta = mock.Mock(service_model={})
def get_caller_identity(self):
ci = mock.Mock()
ci.get = lambda attribute: '111222333444' if attribute == 'Account' else None
return ci
def assume_role(self, RoleArn, RoleSessionName):
return {
'Credentials': {
'AccessKeyId': 'access_key_id_example',
'SecretAccessKey': 'secret_access_key_example',
'SessionToken': 'session_token_example'
}
}
class MockStsClientRaiseClientError(MockStsClient):
def assume_role(self, RoleArn, RoleSessionName):
raise ClientError({'Error': {'Code': 'AccessDenied'}}, 'sqs_test')
class MockQueue(object):
def __init__(self, msgs=[]):
self.dummy_messages = [
SQSSensorTestCase.MockMessage(x) for x in msgs
]
def receive_messages(self, **kwargs):
return self.dummy_messages
class MockMessage(object):
def __init__(self, body=None):
self.body = body
def delete(self):
return mock.MagicMock(return_value=None)
def setUp(self):
super(SQSSensorTestCase, self).setUp()
self.full_config = self.load_yaml('full.yaml')
self.blank_config = self.load_yaml('blank.yaml')
self.multiaccount_config = self.load_yaml('multiaccount.yaml')
self.mixed_config = self.load_yaml('mixed.yaml')
def load_yaml(self, filename):
return yaml.safe_load(self.get_fixture_content(filename))
@mock.patch.object(Session, 'client',
mock.Mock(return_value=MockStsClient()))
def test_poll_with_blank_config(self):
sensor = self.get_sensor_instance(config=self.blank_config)
sensor.setup()
sensor.poll()
self.assertEqual(self.get_dispatched_triggers(), [])
@mock.patch.object(Session, 'client',
mock.Mock(return_value=MockStsClient()))
@mock.patch.object(Session, 'resource',
mock.Mock(return_value=MockResource()))
def _poll_without_message(self, config):
sensor = self.get_sensor_instance(config=config)
sensor.setup()
sensor.poll()
self.assertEqual(self.get_dispatched_triggers(), [])
def test_poll_without_message_full_config(self):
self._poll_without_message(self.full_config)
def test_poll_without_message_multiaccount_config(self):
self._poll_without_message(self.multiaccount_config)
def test_poll_without_message_mixed_config(self):
self._poll_without_message(self.mixed_config)
@mock.patch.object(Session, 'client',
mock.Mock(return_value=MockStsClient()))
@mock.patch.object(Session, 'resource',
mock.Mock(return_value=MockResource(['{"foo":"bar"}'])))
def _poll_with_message(self, config):
sensor = self.get_sensor_instance(config=config)
sensor.setup()
sensor.poll()
self.assertTriggerDispatched(trigger='aws.sqs_new_message')
self.assertNotEqual(self.get_dispatched_triggers(), [])
def test_poll_with_message_full_config(self):
self._poll_with_message(self.full_config)
def test_poll_with_message_multiaccount_config(self):
self._poll_with_message(self.multiaccount_config)
@mock.patch.object(Session, 'client',
mock.Mock(return_value=MockStsClient()))
@mock.patch.object(
Session, 'resource',
mock.Mock(return_value=MockResourceNonExistentQueue(['{"foo":"bar"}']))
)
def _poll_with_non_existent_queue(self, config):
sensor = self.get_sensor_instance(config=config)
sensor.setup()
sensor.poll()
contexts = self.get_dispatched_triggers()
self.assertNotEqual(contexts, [])
self.assertTriggerDispatched(trigger='aws.sqs_new_message')
def test_poll_with_non_existent_queue_full_config(self):
self._poll_with_non_existent_queue(self.full_config)
def test_poll_with_non_existent_queue_multiaccount_config(self):
self._poll_with_non_existent_queue(self.multiaccount_config)
@mock.patch.object(Session, 'client',
mock.Mock(return_value=MockStsClient()))
@mock.patch.object(Session, 'resource',
mock.Mock(return_value=MockResource(['{"foo":"bar"}'])))
def test_set_input_queues_config_dynamically(self):
sensor = self.get_sensor_instance(config=self.blank_config)
sensor._sensor_service.set_value(
'aws.roles', ['arn:aws:iam::123456789098:role/rolename1'],
local=False)
sensor.setup()
# set credential mock to prevent sending request to AWS
mock_credentials = mock.Mock()
mock_credentials.access_key = sensor._get_config_entry(
'aws_access_key_id')
mock_credentials.secret_key = sensor._get_config_entry(
'aws_secret_access_key')
Session.get_credentials = mock_credentials
# set test value to datastore
sensor._sensor_service.set_value('aws.input_queues',
'hoge',
local=False)
sensor.poll()
# update input_queues to check this is reflected
sensor._sensor_service.set_value('aws.input_queues',
'fuga,puyo',
local=False)
sensor.poll()
# update input_queues to check this is reflected
sensor._sensor_service.set_value(
'aws.input_queues',
'https://sqs.us-west-2.amazonaws.com/123456789098/queue_name_3',
local=False)
sensor.poll()
contexts = self.get_dispatched_triggers()
self.assertNotEqual(contexts, [])
self.assertTriggerDispatched(trigger='aws.sqs_new_message')
# get message from queue 'hoge', 'fuga' then 'puyo'
self.assertEqual([x['payload']['queue'] for x in contexts], [
six.moves.urllib.parse.urlparse(queue) for queue in [
'hoge', 'fuga', 'puyo',
'https://sqs.us-west-2.amazonaws.com/123456789098/queue_name_3'
]
])
@mock.patch.object(Session, 'client',
mock.Mock(return_value=MockStsClient()))
@mock.patch.object(Session, 'resource',
mock.Mock(return_value=MockResource(['{"foo":"bar"}'])))
def test_set_input_queues_config_with_list(self):
# set 'input_queues' config with list type
config = self.full_config
config['sqs_sensor']['input_queues'] = [
'foo', 'bar',
'https://sqs.us-west-2.amazonaws.com/123456789098/queue_name_3'
]
config['sqs_sensor']['roles'] = [
'arn:aws:iam::123456789098:role/rolename1'
]
sensor = self.get_sensor_instance(config=config)
sensor.setup()
sensor.poll()
contexts = self.get_dispatched_triggers()
self.assertNotEqual(contexts, [])
self.assertTriggerDispatched(trigger='aws.sqs_new_message')
self.assertEqual([x['payload']['queue'] for x in contexts], [
six.moves.urllib.parse.urlparse(queue) for queue in [
'foo', 'bar',
'https://sqs.us-west-2.amazonaws.com/123456789098/queue_name_3'
]
])
@mock.patch.object(Session, 'client',
mock.Mock(return_value=MockStsClient()))
@mock.patch.object(
Session, 'resource',
mock.Mock(
return_value=MockResourceRaiseClientError('InvalidClientTokenId')))
def _fails_with_invalid_token(self, config):
sensor = self.get_sensor_instance(config=config)
sensor.setup()
sensor.poll()
self.assertEqual(self.get_dispatched_triggers(), [])
def test_fails_with_invalid_token_full_config(self):
self._fails_with_invalid_token(self.full_config)
def test_fails_with_invalid_token_multiaccount_config(self):
self._fails_with_invalid_token(self.multiaccount_config)
@mock.patch.object(Session, 'client',
mock.Mock(return_value=MockStsClient()))
@mock.patch.object(
Session, 'resource',
mock.Mock(return_value=MockResourceRaiseNoCredentialsError()))
def _fails_without_credentials(self, config):
sensor = self.get_sensor_instance(config=config)
sensor.setup()
sensor.poll()
self.assertEqual(self.get_dispatched_triggers(), [])
def test_fails_without_credentials_full_config(self):
self._fails_without_credentials(self.full_config)
def test_fails_without_credentials_multiaccount_config(self):
self._fails_without_credentials(self.multiaccount_config)
@mock.patch.object(Session, 'client',
mock.Mock(return_value=MockStsClient()))
@mock.patch.object(
Session, 'resource',
mock.Mock(return_value=MockResourceRaiseEndpointConnectionError()))
def _fails_with_invalid_region(self, config):
sensor = self.get_sensor_instance(config=config)
sensor.setup()
sensor.poll()
self.assertEqual(self.get_dispatched_triggers(), [])
def test_fails_with_invalid_region_full_config(self):
self._fails_with_invalid_region(self.full_config)
def test_fails_with_invalid_region_multiaccount_config(self):
self._fails_with_invalid_region(self.multiaccount_config)
@mock.patch.object(Session, 'client',
mock.Mock(return_value=MockStsClientRaiseClientError()))
@mock.patch.object(Session, 'resource',
mock.Mock(return_value=MockResource(['{"foo":"bar"}'])))
def _fails_assuming_role(self, config):
sensor = self.get_sensor_instance(config=config)
sensor.setup()
sensor.poll()
def test_fails_assuming_role_full_config(self):
self._fails_assuming_role(self.full_config)
self.assertTriggerDispatched(trigger='aws.sqs_new_message')
self.assertNotEqual(self.get_dispatched_triggers(), [])
def test_fails_assuming_role_multiaccount_config(self):
self._fails_assuming_role(self.multiaccount_config)
self.assertEqual(self.get_dispatched_triggers(), [])
@mock.patch.object(Session, 'client',
mock.Mock(return_value=MockStsClient()))
@mock.patch.object(Session, 'resource',
mock.Mock(side_effect=NoRegionError(
service_name='sqs', region_name='us-east-1')))
def test_fails_creating_sqs_resource(self):
sensor = self.get_sensor_instance(config=self.mixed_config)
sensor.setup()
sensor.poll()
self.assertEqual(self.get_dispatched_triggers(), [])
@mock.patch.object(Session, 'client',
mock.Mock(return_value=MockStsClient()))
@mock.patch.object(Session, 'resource',
mock.Mock(return_value=MockResource(['{"foo":"bar"}'])))
def _poll_with_missing_arn(self, config):
config['sqs_sensor']['roles'] = []
sensor = self.get_sensor_instance(config=config)
sensor.setup()
sensor.poll()
def test_poll_with_missing_arn_full_config(self):
self._poll_with_missing_arn(self.full_config)
self.assertNotEqual(self.get_dispatched_triggers(), [])
self.assertTriggerDispatched(trigger='aws.sqs_new_message')
def test_poll_with_missing_arn_multiaccount_config(self):
self._poll_with_missing_arn(self.multiaccount_config)
self.assertEqual(self.get_dispatched_triggers(), [])
def test_poll_with_missing_arn_mixed_config(self):
self._poll_with_missing_arn(self.mixed_config)
self.assertNotEqual(self.get_dispatched_triggers(), [])
self.assertTriggerDispatched(trigger='aws.sqs_new_message')
def test_client_missing_region(self, boto_mock):
session_mock = Mock()
session_mock.client.side_effect = NoRegionError()
boto_mock.return_value = session_mock
with self.assertRaises(RegionError):
manage_stack("testprofile", "fake-region")
def test_client_missing_region(self, boto_mock):
boto_mock.side_effect = NoRegionError()
with self.assertRaises(RegionError):
manage_stack(None, "fake-region", SAM_CLI_STACK_NAME,
_get_stack_template())
