def create_bucket(bucket_name, region): """ Creates a S3 bucket in a specific region Parameters ---------- bucket_name : string Bucket name region : string Region name Raises ------ NoRegionError If region is not specified """ if region is None: raise NoRegionError() if region == "us-east-1": s3_client = boto3.client("s3") s3_client.create_bucket(Bucket=bucket_name) else: s3_client = boto3.client("s3", region_name=region) location = {"LocationConstraint": region} s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration=location)
def construct_endpoint(self, service_name, region_name, **kwargs): # We take **kwargs so that custom rules can be added that have # additional constraint keys that we don't know about. # We also need to fold the names used in the spec into **kwargs # so we can format the URI later. kwargs['service'] = service_name kwargs['region'] = region_name if 'scheme' not in kwargs: kwargs['scheme'] = self.DEFAULT_SCHEME service_rules = self._rules.get(service_name, []) endpoint = self._match_rules(service_rules, region_name, **kwargs) if endpoint is None: # If we didn't find any in the service section, try again # with the default section. endpoint = self._match_rules(self._rules.get('_default', []), region_name, **kwargs) if endpoint is None: if region_name is None: # Raise a more specific error message that will give # better guidance to the user what needs to happen. raise NoRegionError() else: raise UnknownEndpointError(service_name=service_name, region_name=region_name) return endpoint
def _endpoint_for_partition(self, partition, service_name, region_name, force_partition=False): # Get the service from the partition, or an empty template. service_data = partition['services'].get(service_name, DEFAULT_SERVICE_DATA) # Use the partition endpoint if no region is supplied. if region_name is None: if 'partitionEndpoint' in service_data: region_name = service_data['partitionEndpoint'] else: raise NoRegionError() # Attempt to resolve the exact region for this partition. if region_name in service_data['endpoints']: return self._resolve(partition, service_name, service_data, region_name) # Check to see if the endpoint provided is valid for the partition. if self._region_match(partition, region_name) or force_partition: # Use the partition endpoint if set and not regionalized. partition_endpoint = service_data.get('partitionEndpoint') is_regionalized = service_data.get('isRegionalized', True) if partition_endpoint and not is_regionalized: LOG.debug('Using partition endpoint for %s, %s: %s', service_name, region_name, partition_endpoint) return self._resolve(partition, service_name, service_data, partition_endpoint) LOG.debug('Creating a regex based endpoint for %s, %s', service_name, region_name) return self._resolve(partition, service_name, service_data, region_name)
def _get_signature_version_and_region(self, endpoint, service_model): # An endpoint-aware signature version and region check scoped_config = self.session.get_scoped_config() resolver = self.session.get_component('endpoint_resolver') scheme = endpoint.host.split(':')[0] if endpoint.region_name is None: raise NoRegionError() endpoint_config = resolver.construct_endpoint( service_model.endpoint_prefix, endpoint.region_name, scheme=scheme) # Signature version override from endpoint signature_version = self.service.signature_version if 'signatureVersion' in endpoint_config.get('properties', {}): signature_version = endpoint_config['properties']\ ['signatureVersion'] # Signature overrides from a configuration file if scoped_config is not None: service_config = scoped_config.get(service_model.endpoint_prefix) if service_config is not None and isinstance(service_config, dict): override = service_config.get('signature_version') if override: logger.debug( "Switching signature version for service %s " "to version %s based on config file override.", service_model.endpoint_prefix, override) signature_version = override return signature_version, endpoint.region_name
def test_boto3_raises_exceptions_fail(self, boto3, config): boto3.session.Session.side_effect = PartialCredentialsError( provider=Mock(), cred_var=Mock()) assert CWDataSource(config).test() == False boto3.session.Session.side_effect = ClientError(MagicMock(), Mock()) assert CWDataSource(config).test() == False boto3.session.Session.side_effect = NoRegionError() assert CWDataSource(config).test() == False
def test_no_completions_when_cant_create_client(describer_creator): client_creator = mock.Mock(spec=index.CachedClientCreator) # This is raised when you don't have a region configured via config file # env var or manually via a session. client_creator.create_client.side_effect = NoRegionError() completer = index.ServerSideCompleter(client_creator=client_creator, describer_creator=describer_creator) assert completer.retrieve_candidate_values('ec2', 'foo', 'Bar') == []
def test_raises_boto3clienterror(self, boto3, config): boto3.session.Session.side_effect = PartialCredentialsError( provider=Mock(), cred_var=Mock()) with pytest.raises(Boto3ClientError): CWDataSource(config)._cw_client() boto3.session.Session.side_effect = ClientError(MagicMock(), Mock()) with pytest.raises(Boto3ClientError): CWDataSource(config)._cw_client() boto3.session.Session.side_effect = NoRegionError() with pytest.raises(Boto3ClientError): CWDataSource(config)._cw_client()
def manage_stack( region: str, stack_name: str, template_body: str, profile: Optional[str] = None, parameter_overrides: Optional[Dict[str, Union[str, List[str]]]] = None, ) -> List[Dict[str, str]]: """ get or create a CloudFormation stack Parameters ---------- region: str AWS region for the CloudFormation stack stack_name: str CloudFormation stack name template_body: str CloudFormation template's content profile: Optional[str] AWS named profile for the AWS account parameter_overrides: Optional[Dict[str, Union[str, List[str]]]] Values of template parameters, if any. Returns: Stack output section(list of OutputKey, OutputValue pairs) """ try: if profile: session = boto3.Session( profile_name=profile, region_name=region if region else None) # type: ignore cloudformation_client = session.client("cloudformation") else: cloudformation_client = boto3.client( "cloudformation", config=Config(region_name=region if region else None)) except ProfileNotFound as ex: raise ProfileNotFound( f"Error Setting Up Managed Stack Client: the provided AWS name profile '{profile}' is not found. " "please check the documentation for setting up a named profile: " "https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html" ) from ex except NoCredentialsError as ex: raise NoCredentialsError( "Error Setting Up Managed Stack Client: Unable to resolve credentials for the AWS SDK for Python client. " "Please see their documentation for options to pass in credentials: " "https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html" ) from ex except NoRegionError as ex: raise NoRegionError( "Error Setting Up Managed Stack Client: Unable to resolve a region. " "Please provide a region via the --region parameter or by the AWS_REGION environment variable." ) from ex return _create_or_get_stack(cloudformation_client, stack_name, template_body, parameter_overrides)
def _endpoint_for_partition(self, partition, service_name, region_name, use_dualstack_endpoint, use_fips_endpoint, force_partition=False): partition_name = partition["partition"] if (use_dualstack_endpoint and partition_name in self._UNSUPPORTED_DUALSTACK_PARTITIONS): error_msg = ("Dualstack endpoints are currently not supported" " for %s partition" % partition_name) raise EndpointVariantError(tags=['dualstack'], error_msg=error_msg) # Get the service from the partition, or an empty template. service_data = partition['services'].get(service_name, DEFAULT_SERVICE_DATA) # Use the partition endpoint if no region is supplied. if region_name is None: if 'partitionEndpoint' in service_data: region_name = service_data['partitionEndpoint'] else: raise NoRegionError() resolve_kwargs = { 'partition': partition, 'service_name': service_name, 'service_data': service_data, 'endpoint_name': region_name, 'use_dualstack_endpoint': use_dualstack_endpoint, 'use_fips_endpoint': use_fips_endpoint, } # Attempt to resolve the exact region for this partition. if region_name in service_data['endpoints']: return self._resolve(**resolve_kwargs) # Check to see if the endpoint provided is valid for the partition. if self._region_match(partition, region_name) or force_partition: # Use the partition endpoint if set and not regionalized. partition_endpoint = service_data.get('partitionEndpoint') is_regionalized = service_data.get('isRegionalized', True) if partition_endpoint and not is_regionalized: LOG.debug('Using partition endpoint for %s, %s: %s', service_name, region_name, partition_endpoint) resolve_kwargs['endpoint_name'] = partition_endpoint return self._resolve(**resolve_kwargs) LOG.debug('Creating a regex based endpoint for %s, %s', service_name, region_name) return self._resolve(**resolve_kwargs)
def test_check_config_unauthorized_error( aws: AWS, boto: Mock, caplog: LogCaptureFixture ) -> None: """ Given: An incorrectly configured AWS adapter object. When: Configuration is checked. Then: The user is informed of the incorrect state and an exception is raised. """ boto.client.side_effect = NoRegionError() with pytest.raises(AWSConfigurationError): aws.check_configuration() assert ("drode.adapters.aws", logging.ERROR, "AWS: KO") in caplog.record_tuples
def _no_region_error(): raise NoRegionError()
def test_client_missing_region(self, boto_mock): boto_mock.side_effect = NoRegionError() with self.assertRaises(RegionError): manage_stack("testprofile", "fake-region")
class SQSSensorTestCase(BaseSensorTestCase): sensor_cls = AWSSQSSensor class MockResource(object): def __init__(self, msgs=[]): self.msgs = msgs def get_queue_by_name(self, **kwargs): return SQSSensorTestCase.MockQueue(self.msgs) def Queue(self, queue): return SQSSensorTestCase.MockQueue(self.msgs) class MockResourceNonExistentQueue(object): def __init__(self, msgs=[]): self.msgs = msgs def get_queue_by_name(self, **kwargs): raise ClientError( {'Error': { 'Code': 'AWS.SimpleQueueService.NonExistentQueue' }}, 'sqs_test') def Queue(self, queue): raise ClientError( {'Error': { 'Code': 'AWS.SimpleQueueService.NonExistentQueue' }}, 'sqs_test') def create_queue(self, **kwargs): return SQSSensorTestCase.MockQueue(self.msgs) class MockResourceRaiseClientError(object): def __init__(self, error_code=''): self.error_code = error_code def get_queue_by_name(self, **kwargs): raise ClientError({'Error': {'Code': self.error_code}}, 'sqs_test') def Queue(self, queue): raise ClientError({'Error': {'Code': self.error_code}}, 'sqs_test') class MockResourceRaiseNoCredentialsError(object): def get_queue_by_name(self, **kwargs): raise NoCredentialsError() def Queue(self, queue): raise NoCredentialsError() class MockResourceRaiseEndpointConnectionError(object): def get_queue_by_name(self, **kwargs): raise EndpointConnectionError(endpoint_url='') def Queue(self, queue): raise EndpointConnectionError(endpoint_url='') class MockStsClient(object): def __init__(self): self.meta = mock.Mock(service_model={}) def get_caller_identity(self): ci = mock.Mock() ci.get = lambda attribute: '111222333444' if attribute == 'Account' else None return ci def assume_role(self, RoleArn, RoleSessionName): return { 'Credentials': { 'AccessKeyId': 'access_key_id_example', 'SecretAccessKey': 'secret_access_key_example', 'SessionToken': 'session_token_example' } } class MockStsClientRaiseClientError(MockStsClient): def assume_role(self, RoleArn, RoleSessionName): raise ClientError({'Error': {'Code': 'AccessDenied'}}, 'sqs_test') class MockQueue(object): def __init__(self, msgs=[]): self.dummy_messages = [ SQSSensorTestCase.MockMessage(x) for x in msgs ] def receive_messages(self, **kwargs): return self.dummy_messages class MockMessage(object): def __init__(self, body=None): self.body = body def delete(self): return mock.MagicMock(return_value=None) def setUp(self): super(SQSSensorTestCase, self).setUp() self.full_config = self.load_yaml('full.yaml') self.blank_config = self.load_yaml('blank.yaml') self.multiaccount_config = self.load_yaml('multiaccount.yaml') self.mixed_config = self.load_yaml('mixed.yaml') def load_yaml(self, filename): return yaml.safe_load(self.get_fixture_content(filename)) @mock.patch.object(Session, 'client', mock.Mock(return_value=MockStsClient())) def test_poll_with_blank_config(self): sensor = self.get_sensor_instance(config=self.blank_config) sensor.setup() sensor.poll() self.assertEqual(self.get_dispatched_triggers(), []) @mock.patch.object(Session, 'client', mock.Mock(return_value=MockStsClient())) @mock.patch.object(Session, 'resource', mock.Mock(return_value=MockResource())) def _poll_without_message(self, config): sensor = self.get_sensor_instance(config=config) sensor.setup() sensor.poll() self.assertEqual(self.get_dispatched_triggers(), []) def test_poll_without_message_full_config(self): self._poll_without_message(self.full_config) def test_poll_without_message_multiaccount_config(self): self._poll_without_message(self.multiaccount_config) def test_poll_without_message_mixed_config(self): self._poll_without_message(self.mixed_config) @mock.patch.object(Session, 'client', mock.Mock(return_value=MockStsClient())) @mock.patch.object(Session, 'resource', mock.Mock(return_value=MockResource(['{"foo":"bar"}']))) def _poll_with_message(self, config): sensor = self.get_sensor_instance(config=config) sensor.setup() sensor.poll() self.assertTriggerDispatched(trigger='aws.sqs_new_message') self.assertNotEqual(self.get_dispatched_triggers(), []) def test_poll_with_message_full_config(self): self._poll_with_message(self.full_config) def test_poll_with_message_multiaccount_config(self): self._poll_with_message(self.multiaccount_config) @mock.patch.object(Session, 'client', mock.Mock(return_value=MockStsClient())) @mock.patch.object( Session, 'resource', mock.Mock(return_value=MockResourceNonExistentQueue(['{"foo":"bar"}'])) ) def _poll_with_non_existent_queue(self, config): sensor = self.get_sensor_instance(config=config) sensor.setup() sensor.poll() contexts = self.get_dispatched_triggers() self.assertNotEqual(contexts, []) self.assertTriggerDispatched(trigger='aws.sqs_new_message') def test_poll_with_non_existent_queue_full_config(self): self._poll_with_non_existent_queue(self.full_config) def test_poll_with_non_existent_queue_multiaccount_config(self): self._poll_with_non_existent_queue(self.multiaccount_config) @mock.patch.object(Session, 'client', mock.Mock(return_value=MockStsClient())) @mock.patch.object(Session, 'resource', mock.Mock(return_value=MockResource(['{"foo":"bar"}']))) def test_set_input_queues_config_dynamically(self): sensor = self.get_sensor_instance(config=self.blank_config) sensor._sensor_service.set_value( 'aws.roles', ['arn:aws:iam::123456789098:role/rolename1'], local=False) sensor.setup() # set credential mock to prevent sending request to AWS mock_credentials = mock.Mock() mock_credentials.access_key = sensor._get_config_entry( 'aws_access_key_id') mock_credentials.secret_key = sensor._get_config_entry( 'aws_secret_access_key') Session.get_credentials = mock_credentials # set test value to datastore sensor._sensor_service.set_value('aws.input_queues', 'hoge', local=False) sensor.poll() # update input_queues to check this is reflected sensor._sensor_service.set_value('aws.input_queues', 'fuga,puyo', local=False) sensor.poll() # update input_queues to check this is reflected sensor._sensor_service.set_value( 'aws.input_queues', 'https://sqs.us-west-2.amazonaws.com/123456789098/queue_name_3', local=False) sensor.poll() contexts = self.get_dispatched_triggers() self.assertNotEqual(contexts, []) self.assertTriggerDispatched(trigger='aws.sqs_new_message') # get message from queue 'hoge', 'fuga' then 'puyo' self.assertEqual([x['payload']['queue'] for x in contexts], [ six.moves.urllib.parse.urlparse(queue) for queue in [ 'hoge', 'fuga', 'puyo', 'https://sqs.us-west-2.amazonaws.com/123456789098/queue_name_3' ] ]) @mock.patch.object(Session, 'client', mock.Mock(return_value=MockStsClient())) @mock.patch.object(Session, 'resource', mock.Mock(return_value=MockResource(['{"foo":"bar"}']))) def test_set_input_queues_config_with_list(self): # set 'input_queues' config with list type config = self.full_config config['sqs_sensor']['input_queues'] = [ 'foo', 'bar', 'https://sqs.us-west-2.amazonaws.com/123456789098/queue_name_3' ] config['sqs_sensor']['roles'] = [ 'arn:aws:iam::123456789098:role/rolename1' ] sensor = self.get_sensor_instance(config=config) sensor.setup() sensor.poll() contexts = self.get_dispatched_triggers() self.assertNotEqual(contexts, []) self.assertTriggerDispatched(trigger='aws.sqs_new_message') self.assertEqual([x['payload']['queue'] for x in contexts], [ six.moves.urllib.parse.urlparse(queue) for queue in [ 'foo', 'bar', 'https://sqs.us-west-2.amazonaws.com/123456789098/queue_name_3' ] ]) @mock.patch.object(Session, 'client', mock.Mock(return_value=MockStsClient())) @mock.patch.object( Session, 'resource', mock.Mock( return_value=MockResourceRaiseClientError('InvalidClientTokenId'))) def _fails_with_invalid_token(self, config): sensor = self.get_sensor_instance(config=config) sensor.setup() sensor.poll() self.assertEqual(self.get_dispatched_triggers(), []) def test_fails_with_invalid_token_full_config(self): self._fails_with_invalid_token(self.full_config) def test_fails_with_invalid_token_multiaccount_config(self): self._fails_with_invalid_token(self.multiaccount_config) @mock.patch.object(Session, 'client', mock.Mock(return_value=MockStsClient())) @mock.patch.object( Session, 'resource', mock.Mock(return_value=MockResourceRaiseNoCredentialsError())) def _fails_without_credentials(self, config): sensor = self.get_sensor_instance(config=config) sensor.setup() sensor.poll() self.assertEqual(self.get_dispatched_triggers(), []) def test_fails_without_credentials_full_config(self): self._fails_without_credentials(self.full_config) def test_fails_without_credentials_multiaccount_config(self): self._fails_without_credentials(self.multiaccount_config) @mock.patch.object(Session, 'client', mock.Mock(return_value=MockStsClient())) @mock.patch.object( Session, 'resource', mock.Mock(return_value=MockResourceRaiseEndpointConnectionError())) def _fails_with_invalid_region(self, config): sensor = self.get_sensor_instance(config=config) sensor.setup() sensor.poll() self.assertEqual(self.get_dispatched_triggers(), []) def test_fails_with_invalid_region_full_config(self): self._fails_with_invalid_region(self.full_config) def test_fails_with_invalid_region_multiaccount_config(self): self._fails_with_invalid_region(self.multiaccount_config) @mock.patch.object(Session, 'client', mock.Mock(return_value=MockStsClientRaiseClientError())) @mock.patch.object(Session, 'resource', mock.Mock(return_value=MockResource(['{"foo":"bar"}']))) def _fails_assuming_role(self, config): sensor = self.get_sensor_instance(config=config) sensor.setup() sensor.poll() def test_fails_assuming_role_full_config(self): self._fails_assuming_role(self.full_config) self.assertTriggerDispatched(trigger='aws.sqs_new_message') self.assertNotEqual(self.get_dispatched_triggers(), []) def test_fails_assuming_role_multiaccount_config(self): self._fails_assuming_role(self.multiaccount_config) self.assertEqual(self.get_dispatched_triggers(), []) @mock.patch.object(Session, 'client', mock.Mock(return_value=MockStsClient())) @mock.patch.object(Session, 'resource', mock.Mock(side_effect=NoRegionError( service_name='sqs', region_name='us-east-1'))) def test_fails_creating_sqs_resource(self): sensor = self.get_sensor_instance(config=self.mixed_config) sensor.setup() sensor.poll() self.assertEqual(self.get_dispatched_triggers(), []) @mock.patch.object(Session, 'client', mock.Mock(return_value=MockStsClient())) @mock.patch.object(Session, 'resource', mock.Mock(return_value=MockResource(['{"foo":"bar"}']))) def _poll_with_missing_arn(self, config): config['sqs_sensor']['roles'] = [] sensor = self.get_sensor_instance(config=config) sensor.setup() sensor.poll() def test_poll_with_missing_arn_full_config(self): self._poll_with_missing_arn(self.full_config) self.assertNotEqual(self.get_dispatched_triggers(), []) self.assertTriggerDispatched(trigger='aws.sqs_new_message') def test_poll_with_missing_arn_multiaccount_config(self): self._poll_with_missing_arn(self.multiaccount_config) self.assertEqual(self.get_dispatched_triggers(), []) def test_poll_with_missing_arn_mixed_config(self): self._poll_with_missing_arn(self.mixed_config) self.assertNotEqual(self.get_dispatched_triggers(), []) self.assertTriggerDispatched(trigger='aws.sqs_new_message')
def test_client_missing_region(self, boto_mock): session_mock = Mock() session_mock.client.side_effect = NoRegionError() boto_mock.return_value = session_mock with self.assertRaises(RegionError): manage_stack("testprofile", "fake-region")
def test_client_missing_region(self, boto_mock): boto_mock.side_effect = NoRegionError() with self.assertRaises(RegionError): manage_stack(None, "fake-region", SAM_CLI_STACK_NAME, _get_stack_template())