def _get_config_variable_with_custom_methods(self, logical_name, methods):
# If a custom list of methods was supplied we need to perserve the
# behavior with the new system. To do so a new chain that is a copy of
# the old one will be constructed, but only with the supplied methods
# being added to the chain. This chain will be consulted for a value
# and then thrown out. This is not efficient, nor is the methods arg
# used in botocore, this is just for backwards compatibility.
chain_builder = SubsetChainConfigFactory(session=self, methods=methods)
mapping = create_botocore_default_config_mapping(self)
for name, config_options in self.session_var_map.items():
config_name, env_vars, default, typecast = config_options
build_chain_config_args = {
'conversion_func': typecast,
'default': default,
}
if 'instance' in methods:
build_chain_config_args['instance_name'] = name
if 'env' in methods:
build_chain_config_args['env_var_names'] = env_vars
if 'config' in methods:
build_chain_config_args['config_property_name'] = config_name
mapping[name] = chain_builder.create_config_chain(
**build_chain_config_args
)
config_store_component = ConfigValueStore(
mapping=mapping
)
value = config_store_component.get_config_variable(logical_name)
return value
def test_config_store_deepcopy(self):
config_store = ConfigValueStore()
config_store.set_config_provider('foo', ConstantProvider('bar'))
config_store_copy = copy.deepcopy(config_store)
config_store_copy.set_config_provider('fizz', ConstantProvider('buzz'))
assert config_store.get_config_variable('fizz') is None
assert config_store_copy.get_config_variable('foo') == 'bar'
def _get_config_variable_with_custom_methods(self, logical_name, methods):
# If a custom list of methods was supplied we need to perserve the
# behavior with the new system. To do so a new chain that is a copy of
# the old one will be constructed, but only with the supplied methods
# being added to the chain. This chain will be consulted for a value
# and then thrown out. This is not efficient, nor is the methods arg
# used in botocore, this is just for backwards compatibility.
chain_builder = SubsetChainConfigFactory(session=self, methods=methods)
mapping = create_botocore_default_config_mapping(
chain_builder
)
for name, config_options in self.session_var_map.items():
config_name, env_vars, default, typecast = config_options
build_chain_config_args = {
'conversion_func': typecast,
'default': default,
}
if 'instance' in methods:
build_chain_config_args['instance_name'] = name
if 'env' in methods:
build_chain_config_args['env_var_names'] = env_vars
if 'config' in methods:
build_chain_config_args['config_property_name'] = config_name
mapping[name] = chain_builder.create_config_chain(
**build_chain_config_args
)
config_store_component = ConfigValueStore(
mapping=mapping
)
value = config_store_component.get_config_variable(logical_name)
return value
def test_does_provide_value_if_variable_exists(self):
mock_value_provider = mock.Mock(spec=BaseProvider)
mock_value_provider.provide.return_value = 'foo'
provider = ConfigValueStore(mapping={
'fake_variable': mock_value_provider,
})
value = provider.get_config_variable('fake_variable')
self.assertEqual(value, 'foo')
def test_does_provide_value_if_variable_exists(self):
mock_value_provider = mock.Mock(spec=BaseProvider)
mock_value_provider.provide.return_value = 'foo'
provider = ConfigValueStore(mapping={
'fake_variable': mock_value_provider,
})
value = provider.get_config_variable('fake_variable')
self.assertEqual(value, 'foo')
def test_can_get_config_provider_non_chain_provider(self):
constant_provider = ConstantProvider(value='bar')
config_value_store = ConfigValueStore(
mapping={
'fake_variable': constant_provider,
})
provider = config_value_store.get_config_provider('fake_variable')
value = config_value_store.get_config_variable('fake_variable')
self.assertIsInstance(provider, ConstantProvider)
self.assertEqual(value, 'bar')
def setUp(self):
self.event_emitter = mock.Mock(HierarchicalEmitter)
self.config_store = ConfigValueStore()
self.args_create = args.ClientArgsCreator(self.event_emitter, None,
None, None, None,
self.config_store)
self.service_name = 'ec2'
self.region = 'us-west-2'
self.endpoint_url = 'https://ec2/'
self.service_model = self._get_service_model()
self.bridge = mock.Mock(ClientEndpointBridge)
self._set_endpoint_bridge_resolve()
self.default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY,
1)]
def _register_config_store(self):
chain_builder = ConfigChainFactory(session=self)
config_store_component = ConfigValueStore(
mapping=create_botocore_default_config_mapping(chain_builder)
)
self._components.register_component('config_store',
config_store_component)
def test_can_set_config_provider(self):
foo_value_provider = mock.Mock(spec=BaseProvider)
foo_value_provider.provide.return_value = 'foo'
provider = ConfigValueStore(mapping={
'fake_variable': foo_value_provider,
})
value = provider.get_config_variable('fake_variable')
self.assertEqual(value, 'foo')
bar_value_provider = mock.Mock(spec=BaseProvider)
bar_value_provider.provide.return_value = 'bar'
provider.set_config_provider('fake_variable', bar_value_provider)
value = provider.get_config_variable('fake_variable')
self.assertEqual(value, 'bar')
def setup_client(self):
with open(self.example_model_file, 'w') as f:
json.dump(self.example_json_model, f)
with open(self.waiter_model_file, 'w') as f:
json.dump(self.waiter_json_model, f)
with open(self.paginator_model_file, 'w') as f:
json.dump(self.paginator_json_model, f)
with open(self.model_file, 'w') as f:
json.dump(self.json_model, f)
self.loader = Loader(extra_search_paths=[self.root_dir])
endpoint_resolver = mock.Mock()
endpoint_resolver.construct_endpoint.return_value = {
'hostname': 'foo.us-east-1',
'partition': 'aws',
'endpointName': 'us-east-1',
'signatureVersions': ['v4']
}
self.creator = ClientCreator(
loader=self.loader, endpoint_resolver=endpoint_resolver,
user_agent='user-agent', event_emitter=self.events,
retry_handler_factory=mock.Mock(),
retry_config_translator=mock.Mock(),
exceptions_factory=mock.Mock(),
config_store=ConfigValueStore()
)
self.client = self.creator.create_client('myservice', 'us-east-1')
def _f(config_loader: Optional[ConfigValueStore] = None) -> AioSession:
if not config_loader:
config_loader = ConfigValueStore()
fake_instance_variables = {
'credentials_file': 'a',
'legacy_config_file': 'b',
'config_file': 'c',
'metadata_service_timeout': 1,
'metadata_service_num_attempts': 1,
}
def fake_get_component(self, key):
if key == 'config_provider':
return config_loader
return None
def fake_set_config_variable(self, logical_name, value):
fake_instance_variables[logical_name] = value
session = mock.Mock(spec=AioSession)
session.get_component = fake_get_component
session.full_config = {}
for name, value in fake_instance_variables.items():
config_loader.set_config_variable(name, value)
session.get_config_variable = config_loader.get_config_variable
session.set_config_variable = fake_set_config_variable
return session
def _create_config_value_store(self, s3_mapping={}, **override_kwargs):
provider_foo = ConstantProvider(value='foo')
environment_provider_foo = EnvironmentProvider(
name='AWS_RETRY_MODE', env={'AWS_RETRY_MODE': None})
fake_session = mock.Mock(spec=session.Session)
fake_session.get_scoped_config.return_value = {}
# Testing with three different providers to validate
# SmartDefaultsConfigStoreFactory._get_new_chain_provider
mapping = {
'sts_regional_endpoints': ChainProvider(providers=[provider_foo]),
'retry_mode': ChainProvider(providers=[environment_provider_foo]),
's3': SectionConfigProvider('s3', fake_session, s3_mapping)
}
mapping.update(**override_kwargs)
config_store = ConfigValueStore(mapping=mapping)
return config_store
def test_can_set_config_provider(self):
foo_value_provider = mock.Mock(spec=BaseProvider)
foo_value_provider.provide.return_value = 'foo'
provider = ConfigValueStore(mapping={
'fake_variable': foo_value_provider,
})
value = provider.get_config_variable('fake_variable')
self.assertEqual(value, 'foo')
bar_value_provider = mock.Mock(spec=BaseProvider)
bar_value_provider.provide.return_value = 'bar'
provider.set_config_provider('fake_variable', bar_value_provider)
value = provider.get_config_variable('fake_variable')
self.assertEqual(value, 'bar')
def test_can_set_variable(self):
provider = ConfigValueStore()
provider.set_config_variable('fake_variable', 'foo')
value = provider.get_config_variable('fake_variable')
self.assertEquals(value, 'foo')
def _register_config_store(self):
config_store_component = ConfigValueStore(
mapping=create_botocore_default_config_mapping(self)
)
self._components.register_component('config_store',
config_store_component)
def test_does_provide_none_if_no_variable_exists(self):
provider = ConfigValueStore()
value = provider.get_config_variable('fake_variable')
self.assertIsNone(value)
def test_does_provide_none_if_no_variable_exists(self):
provider = ConfigValueStore()
value = provider.get_config_variable('fake_variable')
self.assertIsNone(value)
def test_can_set_variable(self):
provider = ConfigValueStore()
provider.set_config_variable('fake_variable', 'foo')
value = provider.get_config_variable('fake_variable')
self.assertEquals(value, 'foo')
def _register_config_store(self):
chain_builder = ConfigChainFactory(session=self)
config_store = ConfigValueStore(
mapping=create_botocore_default_config_mapping(chain_builder))
return config_store
class TestCreateClientArgs(unittest.TestCase):
def setUp(self):
self.event_emitter = mock.Mock(HierarchicalEmitter)
self.config_store = ConfigValueStore()
self.args_create = args.ClientArgsCreator(self.event_emitter, None,
None, None, None,
self.config_store)
self.service_name = 'ec2'
self.region = 'us-west-2'
self.endpoint_url = 'https://ec2/'
self.service_model = self._get_service_model()
self.bridge = mock.Mock(ClientEndpointBridge)
self._set_endpoint_bridge_resolve()
self.default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY,
1)]
def _get_service_model(self, service_name=None):
if service_name is None:
service_name = self.service_name
service_model = mock.Mock(ServiceModel)
service_model.service_name = service_name
service_model.endpoint_prefix = service_name
service_model.metadata = {
'serviceFullName': 'MyService',
'protocol': 'query'
}
service_model.operation_names = []
return service_model
def _set_endpoint_bridge_resolve(self, **override_kwargs):
ret_val = {
'region_name': self.region,
'signature_version': 'v4',
'endpoint_url': self.endpoint_url,
'signing_name': self.service_name,
'signing_region': self.region,
'metadata': {}
}
ret_val.update(**override_kwargs)
self.bridge.resolve.return_value = ret_val
def call_get_client_args(self, **override_kwargs):
call_kwargs = {
'service_model': self.service_model,
'region_name': self.region,
'is_secure': True,
'endpoint_url': self.endpoint_url,
'verify': True,
'credentials': None,
'scoped_config': {},
'client_config': None,
'endpoint_bridge': self.bridge
}
call_kwargs.update(**override_kwargs)
return self.args_create.get_client_args(**call_kwargs)
def assert_create_endpoint_call(self, mock_endpoint, **override_kwargs):
call_kwargs = {
'endpoint_url': self.endpoint_url,
'region_name': self.region,
'response_parser_factory': None,
'timeout': (60, 60),
'verify': True,
'max_pool_connections': 10,
'proxies': None,
'socket_options': self.default_socket_options,
'client_cert': None,
}
call_kwargs.update(**override_kwargs)
mock_endpoint.return_value.create_endpoint.assert_called_with(
self.service_model, **call_kwargs)
def test_compute_s3_configuration(self):
self.assertIsNone(self.args_create.compute_s3_config(None))
def test_compute_s3_config_only_config_store(self):
self.config_store.set_config_variable(
's3', {'use_accelerate_endpoint': True})
self.assertEqual(self.args_create.compute_s3_config(None),
{'use_accelerate_endpoint': True})
def test_client_s3_accelerate_from_client_config(self):
self.assertEqual(
self.args_create.compute_s3_config(client_config=Config(
s3={'use_accelerate_endpoint': True})),
{'use_accelerate_endpoint': True})
def test_client_s3_accelerate_client_config_overrides_config_store(self):
self.config_store.set_config_variable(
's3', {'use_accelerate_endpoint': False})
self.assertEqual(
self.args_create.compute_s3_config(client_config=Config(
s3={'use_accelerate_endpoint': True})),
# client_config beats scoped_config
{'use_accelerate_endpoint': True})
def test_max_pool_from_client_config_forwarded_to_endpoint_creator(self):
config = botocore.config.Config(max_pool_connections=20)
with mock.patch('botocore.args.EndpointCreator') as m:
self.call_get_client_args(client_config=config)
self.assert_create_endpoint_call(m, max_pool_connections=20)
def test_proxies_from_client_config_forwarded_to_endpoint_creator(self):
proxies = {
'http': 'http://foo.bar:1234',
'https': 'https://foo.bar:4321'
}
config = botocore.config.Config(proxies=proxies)
with mock.patch('botocore.args.EndpointCreator') as m:
self.call_get_client_args(client_config=config)
self.assert_create_endpoint_call(m, proxies=proxies)
def test_s3_with_endpoint_url_still_resolves_region(self):
self.service_model.endpoint_prefix = 's3'
self.service_model.metadata = {'protocol': 'rest-xml'}
self.bridge.resolve.side_effect = [{
'region_name': None,
'signature_version': 's3v4',
'endpoint_url': 'http://other.com/',
'signing_name': 's3',
'signing_region': None,
'metadata': {}
}, {
'region_name': 'us-west-2',
'signature_version': 's3v4',
'enpoint_url': 'https://s3-us-west-2.amazonaws.com',
'signing_name': 's3',
'signing_region': 'us-west-2',
'metadata': {}
}]
client_args = self.call_get_client_args(
endpoint_url='http://other.com/')
self.assertEqual(client_args['client_config'].region_name, 'us-west-2')
def test_region_does_not_resolve_if_not_s3_and_endpoint_url_provided(self):
self.service_model.endpoint_prefix = 'ec2'
self.service_model.metadata = {'protocol': 'query'}
self.bridge.resolve.side_effect = [{
'region_name': None,
'signature_version': 'v4',
'endpoint_url': 'http://other.com/',
'signing_name': 'ec2',
'signing_region': None,
'metadata': {}
}]
client_args = self.call_get_client_args(
endpoint_url='http://other.com/')
self.assertEqual(client_args['client_config'].region_name, None)
def test_tcp_keepalive_enabled(self):
scoped_config = {'tcp_keepalive': 'true'}
with mock.patch('botocore.args.EndpointCreator') as m:
self.call_get_client_args(scoped_config=scoped_config)
self.assert_create_endpoint_call(
m,
socket_options=self.default_socket_options +
[(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)])
def test_tcp_keepalive_not_specified(self):
scoped_config = {}
with mock.patch('botocore.args.EndpointCreator') as m:
self.call_get_client_args(scoped_config=scoped_config)
self.assert_create_endpoint_call(
m, socket_options=self.default_socket_options)
def test_tcp_keepalive_explicitly_disabled(self):
scoped_config = {'tcp_keepalive': 'false'}
with mock.patch('botocore.args.EndpointCreator') as m:
self.call_get_client_args(scoped_config=scoped_config)
self.assert_create_endpoint_call(
m, socket_options=self.default_socket_options)
def test_tcp_keepalive_enabled_case_insensitive(self):
scoped_config = {'tcp_keepalive': 'True'}
with mock.patch('botocore.args.EndpointCreator') as m:
self.call_get_client_args(scoped_config=scoped_config)
self.assert_create_endpoint_call(
m,
socket_options=self.default_socket_options +
[(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)])
def test_sts_override_resolved_endpoint_for_legacy_region(self):
self.config_store.set_config_variable('sts_regional_endpoints',
'legacy')
client_args = self.call_get_client_args(
service_model=self._get_service_model('sts'),
region_name='us-west-2',
endpoint_url=None)
self.assertEqual(client_args['endpoint'].host,
'https://sts.amazonaws.com')
self.assertEqual(client_args['request_signer'].region_name,
'us-east-1')
def test_sts_use_resolved_endpoint_for_nonlegacy_region(self):
resolved_endpoint = 'https://resolved-endpoint'
resolved_region = 'resolved-region'
self._set_endpoint_bridge_resolve(endpoint_url=resolved_endpoint,
signing_region=resolved_region)
self.config_store.set_config_variable('sts_regional_endpoints',
'legacy')
client_args = self.call_get_client_args(
service_model=self._get_service_model('sts'),
region_name='ap-east-1',
endpoint_url=None)
self.assertEqual(client_args['endpoint'].host, resolved_endpoint)
self.assertEqual(client_args['request_signer'].region_name,
resolved_region)
def test_sts_use_resolved_endpoint_for_regional_configuration(self):
resolved_endpoint = 'https://resolved-endpoint'
resolved_region = 'resolved-region'
self._set_endpoint_bridge_resolve(endpoint_url=resolved_endpoint,
signing_region=resolved_region)
self.config_store.set_config_variable('sts_regional_endpoints',
'regional')
client_args = self.call_get_client_args(
service_model=self._get_service_model('sts'),
region_name='us-west-2',
endpoint_url=None)
self.assertEqual(client_args['endpoint'].host, resolved_endpoint)
self.assertEqual(client_args['request_signer'].region_name,
resolved_region)
def test_sts_with_endpoint_override_and_legacy_configured(self):
override_endpoint = 'https://override-endpoint'
self._set_endpoint_bridge_resolve(endpoint_url=override_endpoint)
self.config_store.set_config_variable('sts_regional_endpoints',
'legacy')
client_args = self.call_get_client_args(
service_model=self._get_service_model('sts'),
region_name='us-west-2',
endpoint_url=override_endpoint)
self.assertEqual(client_args['endpoint'].host, override_endpoint)
def test_sts_http_scheme_for_override_endpoint(self):
self.config_store.set_config_variable('sts_regional_endpoints',
'legacy')
client_args = self.call_get_client_args(
service_model=self._get_service_model('sts'),
region_name='us-west-2',
endpoint_url=None,
is_secure=False,
)
self.assertEqual(client_args['endpoint'].host,
'http://sts.amazonaws.com')
def test_sts_regional_endpoints_defaults_to_legacy_if_not_set(self):
self.config_store.set_config_variable('sts_regional_endpoints', None)
client_args = self.call_get_client_args(
service_model=self._get_service_model('sts'),
region_name='us-west-2',
endpoint_url=None)
self.assertEqual(client_args['endpoint'].host,
'https://sts.amazonaws.com')
self.assertEqual(client_args['request_signer'].region_name,
'us-east-1')
def test_invalid_sts_regional_endpoints(self):
self.config_store.set_config_variable('sts_regional_endpoints',
'invalid')
with self.assertRaises(
exceptions.InvalidSTSRegionalEndpointsConfigError):
self.call_get_client_args(
service_model=self._get_service_model('sts'),
region_name='us-west-2',
endpoint_url=None)
def test_provides_total_max_attempts(self):
config = botocore.config.Config(retries={'total_max_attempts': 10})
client_args = self.call_get_client_args(client_config=config)
self.assertEqual(
client_args['client_config'].retries['total_max_attempts'], 10)
def test_provides_total_max_attempts_has_precedence(self):
config = botocore.config.Config(retries={
'total_max_attempts': 10,
'max_attempts': 5
})
client_args = self.call_get_client_args(client_config=config)
self.assertEqual(
client_args['client_config'].retries['total_max_attempts'], 10)
self.assertNotIn('max_attempts', client_args['client_config'].retries)
def test_provide_retry_config_maps_total_max_attempts(self):
config = botocore.config.Config(retries={'max_attempts': 10})
client_args = self.call_get_client_args(client_config=config)
self.assertEqual(
client_args['client_config'].retries['total_max_attempts'], 11)
self.assertNotIn('max_attempts', client_args['client_config'].retries)
def test_can_merge_max_attempts(self):
self.config_store.set_config_variable('max_attempts', 4)
config = self.call_get_client_args()['client_config']
self.assertEqual(config.retries['total_max_attempts'], 4)
def test_uses_config_value_if_present_for_max_attempts(self):
config = self.call_get_client_args(client_config=Config(
retries={'max_attempts': 2}))['client_config']
self.assertEqual(config.retries['total_max_attempts'], 3)
def test_uses_client_config_over_config_store_max_attempts(self):
self.config_store.set_config_variable('max_attempts', 4)
config = self.call_get_client_args(client_config=Config(
retries={'max_attempts': 2}))['client_config']
self.assertEqual(config.retries['total_max_attempts'], 3)
def test_uses_client_config_total_over_config_store_max_attempts(self):
self.config_store.set_config_variable('max_attempts', 4)
config = self.call_get_client_args(client_config=Config(
retries={'total_max_attempts': 2}))['client_config']
self.assertEqual(config.retries['total_max_attempts'], 2)
def test_max_attempts_unset_if_retries_is_none(self):
config = self.call_get_client_args(client_config=Config(
retries=None))['client_config']
self.assertEqual(config.retries, {'mode': 'legacy'})
def test_retry_mode_set_on_config_store(self):
self.config_store.set_config_variable('retry_mode', 'standard')
config = self.call_get_client_args()['client_config']
self.assertEqual(config.retries['mode'], 'standard')
def test_retry_mode_set_on_client_config(self):
config = self.call_get_client_args(client_config=Config(
retries={'mode': 'standard'}))['client_config']
self.assertEqual(config.retries['mode'], 'standard')
def test_client_config_beats_config_store(self):
self.config_store.set_config_variable('retry_mode', 'adaptive')
config = self.call_get_client_args(client_config=Config(
retries={'mode': 'standard'}))['client_config']
self.assertEqual(config.retries['mode'], 'standard')
