def _get_config_variable_with_custom_methods(self, logical_name, methods): # If a custom list of methods was supplied we need to perserve the # behavior with the new system. To do so a new chain that is a copy of # the old one will be constructed, but only with the supplied methods # being added to the chain. This chain will be consulted for a value # and then thrown out. This is not efficient, nor is the methods arg # used in botocore, this is just for backwards compatibility. chain_builder = SubsetChainConfigFactory(session=self, methods=methods) mapping = create_botocore_default_config_mapping(self) for name, config_options in self.session_var_map.items(): config_name, env_vars, default, typecast = config_options build_chain_config_args = { 'conversion_func': typecast, 'default': default, } if 'instance' in methods: build_chain_config_args['instance_name'] = name if 'env' in methods: build_chain_config_args['env_var_names'] = env_vars if 'config' in methods: build_chain_config_args['config_property_name'] = config_name mapping[name] = chain_builder.create_config_chain( **build_chain_config_args ) config_store_component = ConfigValueStore( mapping=mapping ) value = config_store_component.get_config_variable(logical_name) return value
def test_config_store_deepcopy(self): config_store = ConfigValueStore() config_store.set_config_provider('foo', ConstantProvider('bar')) config_store_copy = copy.deepcopy(config_store) config_store_copy.set_config_provider('fizz', ConstantProvider('buzz')) assert config_store.get_config_variable('fizz') is None assert config_store_copy.get_config_variable('foo') == 'bar'
def _get_config_variable_with_custom_methods(self, logical_name, methods): # If a custom list of methods was supplied we need to perserve the # behavior with the new system. To do so a new chain that is a copy of # the old one will be constructed, but only with the supplied methods # being added to the chain. This chain will be consulted for a value # and then thrown out. This is not efficient, nor is the methods arg # used in botocore, this is just for backwards compatibility. chain_builder = SubsetChainConfigFactory(session=self, methods=methods) mapping = create_botocore_default_config_mapping( chain_builder ) for name, config_options in self.session_var_map.items(): config_name, env_vars, default, typecast = config_options build_chain_config_args = { 'conversion_func': typecast, 'default': default, } if 'instance' in methods: build_chain_config_args['instance_name'] = name if 'env' in methods: build_chain_config_args['env_var_names'] = env_vars if 'config' in methods: build_chain_config_args['config_property_name'] = config_name mapping[name] = chain_builder.create_config_chain( **build_chain_config_args ) config_store_component = ConfigValueStore( mapping=mapping ) value = config_store_component.get_config_variable(logical_name) return value
def test_does_provide_value_if_variable_exists(self): mock_value_provider = mock.Mock(spec=BaseProvider) mock_value_provider.provide.return_value = 'foo' provider = ConfigValueStore(mapping={ 'fake_variable': mock_value_provider, }) value = provider.get_config_variable('fake_variable') self.assertEqual(value, 'foo')
def test_can_get_config_provider_non_chain_provider(self): constant_provider = ConstantProvider(value='bar') config_value_store = ConfigValueStore( mapping={ 'fake_variable': constant_provider, }) provider = config_value_store.get_config_provider('fake_variable') value = config_value_store.get_config_variable('fake_variable') self.assertIsInstance(provider, ConstantProvider) self.assertEqual(value, 'bar')
def setUp(self): self.event_emitter = mock.Mock(HierarchicalEmitter) self.config_store = ConfigValueStore() self.args_create = args.ClientArgsCreator(self.event_emitter, None, None, None, None, self.config_store) self.service_name = 'ec2' self.region = 'us-west-2' self.endpoint_url = 'https://ec2/' self.service_model = self._get_service_model() self.bridge = mock.Mock(ClientEndpointBridge) self._set_endpoint_bridge_resolve() self.default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
def _register_config_store(self): chain_builder = ConfigChainFactory(session=self) config_store_component = ConfigValueStore( mapping=create_botocore_default_config_mapping(chain_builder) ) self._components.register_component('config_store', config_store_component)
def test_can_set_config_provider(self): foo_value_provider = mock.Mock(spec=BaseProvider) foo_value_provider.provide.return_value = 'foo' provider = ConfigValueStore(mapping={ 'fake_variable': foo_value_provider, }) value = provider.get_config_variable('fake_variable') self.assertEqual(value, 'foo') bar_value_provider = mock.Mock(spec=BaseProvider) bar_value_provider.provide.return_value = 'bar' provider.set_config_provider('fake_variable', bar_value_provider) value = provider.get_config_variable('fake_variable') self.assertEqual(value, 'bar')
def setup_client(self): with open(self.example_model_file, 'w') as f: json.dump(self.example_json_model, f) with open(self.waiter_model_file, 'w') as f: json.dump(self.waiter_json_model, f) with open(self.paginator_model_file, 'w') as f: json.dump(self.paginator_json_model, f) with open(self.model_file, 'w') as f: json.dump(self.json_model, f) self.loader = Loader(extra_search_paths=[self.root_dir]) endpoint_resolver = mock.Mock() endpoint_resolver.construct_endpoint.return_value = { 'hostname': 'foo.us-east-1', 'partition': 'aws', 'endpointName': 'us-east-1', 'signatureVersions': ['v4'] } self.creator = ClientCreator( loader=self.loader, endpoint_resolver=endpoint_resolver, user_agent='user-agent', event_emitter=self.events, retry_handler_factory=mock.Mock(), retry_config_translator=mock.Mock(), exceptions_factory=mock.Mock(), config_store=ConfigValueStore() ) self.client = self.creator.create_client('myservice', 'us-east-1')
def _f(config_loader: Optional[ConfigValueStore] = None) -> AioSession: if not config_loader: config_loader = ConfigValueStore() fake_instance_variables = { 'credentials_file': 'a', 'legacy_config_file': 'b', 'config_file': 'c', 'metadata_service_timeout': 1, 'metadata_service_num_attempts': 1, } def fake_get_component(self, key): if key == 'config_provider': return config_loader return None def fake_set_config_variable(self, logical_name, value): fake_instance_variables[logical_name] = value session = mock.Mock(spec=AioSession) session.get_component = fake_get_component session.full_config = {} for name, value in fake_instance_variables.items(): config_loader.set_config_variable(name, value) session.get_config_variable = config_loader.get_config_variable session.set_config_variable = fake_set_config_variable return session
def _create_config_value_store(self, s3_mapping={}, **override_kwargs): provider_foo = ConstantProvider(value='foo') environment_provider_foo = EnvironmentProvider( name='AWS_RETRY_MODE', env={'AWS_RETRY_MODE': None}) fake_session = mock.Mock(spec=session.Session) fake_session.get_scoped_config.return_value = {} # Testing with three different providers to validate # SmartDefaultsConfigStoreFactory._get_new_chain_provider mapping = { 'sts_regional_endpoints': ChainProvider(providers=[provider_foo]), 'retry_mode': ChainProvider(providers=[environment_provider_foo]), 's3': SectionConfigProvider('s3', fake_session, s3_mapping) } mapping.update(**override_kwargs) config_store = ConfigValueStore(mapping=mapping) return config_store
def test_can_set_variable(self): provider = ConfigValueStore() provider.set_config_variable('fake_variable', 'foo') value = provider.get_config_variable('fake_variable') self.assertEquals(value, 'foo')
def _register_config_store(self): config_store_component = ConfigValueStore( mapping=create_botocore_default_config_mapping(self) ) self._components.register_component('config_store', config_store_component)
def test_does_provide_none_if_no_variable_exists(self): provider = ConfigValueStore() value = provider.get_config_variable('fake_variable') self.assertIsNone(value)
def _register_config_store(self): chain_builder = ConfigChainFactory(session=self) config_store = ConfigValueStore( mapping=create_botocore_default_config_mapping(chain_builder)) return config_store
class TestCreateClientArgs(unittest.TestCase): def setUp(self): self.event_emitter = mock.Mock(HierarchicalEmitter) self.config_store = ConfigValueStore() self.args_create = args.ClientArgsCreator(self.event_emitter, None, None, None, None, self.config_store) self.service_name = 'ec2' self.region = 'us-west-2' self.endpoint_url = 'https://ec2/' self.service_model = self._get_service_model() self.bridge = mock.Mock(ClientEndpointBridge) self._set_endpoint_bridge_resolve() self.default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] def _get_service_model(self, service_name=None): if service_name is None: service_name = self.service_name service_model = mock.Mock(ServiceModel) service_model.service_name = service_name service_model.endpoint_prefix = service_name service_model.metadata = { 'serviceFullName': 'MyService', 'protocol': 'query' } service_model.operation_names = [] return service_model def _set_endpoint_bridge_resolve(self, **override_kwargs): ret_val = { 'region_name': self.region, 'signature_version': 'v4', 'endpoint_url': self.endpoint_url, 'signing_name': self.service_name, 'signing_region': self.region, 'metadata': {} } ret_val.update(**override_kwargs) self.bridge.resolve.return_value = ret_val def call_get_client_args(self, **override_kwargs): call_kwargs = { 'service_model': self.service_model, 'region_name': self.region, 'is_secure': True, 'endpoint_url': self.endpoint_url, 'verify': True, 'credentials': None, 'scoped_config': {}, 'client_config': None, 'endpoint_bridge': self.bridge } call_kwargs.update(**override_kwargs) return self.args_create.get_client_args(**call_kwargs) def assert_create_endpoint_call(self, mock_endpoint, **override_kwargs): call_kwargs = { 'endpoint_url': self.endpoint_url, 'region_name': self.region, 'response_parser_factory': None, 'timeout': (60, 60), 'verify': True, 'max_pool_connections': 10, 'proxies': None, 'socket_options': self.default_socket_options, 'client_cert': None, } call_kwargs.update(**override_kwargs) mock_endpoint.return_value.create_endpoint.assert_called_with( self.service_model, **call_kwargs) def test_compute_s3_configuration(self): self.assertIsNone(self.args_create.compute_s3_config(None)) def test_compute_s3_config_only_config_store(self): self.config_store.set_config_variable( 's3', {'use_accelerate_endpoint': True}) self.assertEqual(self.args_create.compute_s3_config(None), {'use_accelerate_endpoint': True}) def test_client_s3_accelerate_from_client_config(self): self.assertEqual( self.args_create.compute_s3_config(client_config=Config( s3={'use_accelerate_endpoint': True})), {'use_accelerate_endpoint': True}) def test_client_s3_accelerate_client_config_overrides_config_store(self): self.config_store.set_config_variable( 's3', {'use_accelerate_endpoint': False}) self.assertEqual( self.args_create.compute_s3_config(client_config=Config( s3={'use_accelerate_endpoint': True})), # client_config beats scoped_config {'use_accelerate_endpoint': True}) def test_max_pool_from_client_config_forwarded_to_endpoint_creator(self): config = botocore.config.Config(max_pool_connections=20) with mock.patch('botocore.args.EndpointCreator') as m: self.call_get_client_args(client_config=config) self.assert_create_endpoint_call(m, max_pool_connections=20) def test_proxies_from_client_config_forwarded_to_endpoint_creator(self): proxies = { 'http': 'http://foo.bar:1234', 'https': 'https://foo.bar:4321' } config = botocore.config.Config(proxies=proxies) with mock.patch('botocore.args.EndpointCreator') as m: self.call_get_client_args(client_config=config) self.assert_create_endpoint_call(m, proxies=proxies) def test_s3_with_endpoint_url_still_resolves_region(self): self.service_model.endpoint_prefix = 's3' self.service_model.metadata = {'protocol': 'rest-xml'} self.bridge.resolve.side_effect = [{ 'region_name': None, 'signature_version': 's3v4', 'endpoint_url': 'http://other.com/', 'signing_name': 's3', 'signing_region': None, 'metadata': {} }, { 'region_name': 'us-west-2', 'signature_version': 's3v4', 'enpoint_url': 'https://s3-us-west-2.amazonaws.com', 'signing_name': 's3', 'signing_region': 'us-west-2', 'metadata': {} }] client_args = self.call_get_client_args( endpoint_url='http://other.com/') self.assertEqual(client_args['client_config'].region_name, 'us-west-2') def test_region_does_not_resolve_if_not_s3_and_endpoint_url_provided(self): self.service_model.endpoint_prefix = 'ec2' self.service_model.metadata = {'protocol': 'query'} self.bridge.resolve.side_effect = [{ 'region_name': None, 'signature_version': 'v4', 'endpoint_url': 'http://other.com/', 'signing_name': 'ec2', 'signing_region': None, 'metadata': {} }] client_args = self.call_get_client_args( endpoint_url='http://other.com/') self.assertEqual(client_args['client_config'].region_name, None) def test_tcp_keepalive_enabled(self): scoped_config = {'tcp_keepalive': 'true'} with mock.patch('botocore.args.EndpointCreator') as m: self.call_get_client_args(scoped_config=scoped_config) self.assert_create_endpoint_call( m, socket_options=self.default_socket_options + [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]) def test_tcp_keepalive_not_specified(self): scoped_config = {} with mock.patch('botocore.args.EndpointCreator') as m: self.call_get_client_args(scoped_config=scoped_config) self.assert_create_endpoint_call( m, socket_options=self.default_socket_options) def test_tcp_keepalive_explicitly_disabled(self): scoped_config = {'tcp_keepalive': 'false'} with mock.patch('botocore.args.EndpointCreator') as m: self.call_get_client_args(scoped_config=scoped_config) self.assert_create_endpoint_call( m, socket_options=self.default_socket_options) def test_tcp_keepalive_enabled_case_insensitive(self): scoped_config = {'tcp_keepalive': 'True'} with mock.patch('botocore.args.EndpointCreator') as m: self.call_get_client_args(scoped_config=scoped_config) self.assert_create_endpoint_call( m, socket_options=self.default_socket_options + [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]) def test_sts_override_resolved_endpoint_for_legacy_region(self): self.config_store.set_config_variable('sts_regional_endpoints', 'legacy') client_args = self.call_get_client_args( service_model=self._get_service_model('sts'), region_name='us-west-2', endpoint_url=None) self.assertEqual(client_args['endpoint'].host, 'https://sts.amazonaws.com') self.assertEqual(client_args['request_signer'].region_name, 'us-east-1') def test_sts_use_resolved_endpoint_for_nonlegacy_region(self): resolved_endpoint = 'https://resolved-endpoint' resolved_region = 'resolved-region' self._set_endpoint_bridge_resolve(endpoint_url=resolved_endpoint, signing_region=resolved_region) self.config_store.set_config_variable('sts_regional_endpoints', 'legacy') client_args = self.call_get_client_args( service_model=self._get_service_model('sts'), region_name='ap-east-1', endpoint_url=None) self.assertEqual(client_args['endpoint'].host, resolved_endpoint) self.assertEqual(client_args['request_signer'].region_name, resolved_region) def test_sts_use_resolved_endpoint_for_regional_configuration(self): resolved_endpoint = 'https://resolved-endpoint' resolved_region = 'resolved-region' self._set_endpoint_bridge_resolve(endpoint_url=resolved_endpoint, signing_region=resolved_region) self.config_store.set_config_variable('sts_regional_endpoints', 'regional') client_args = self.call_get_client_args( service_model=self._get_service_model('sts'), region_name='us-west-2', endpoint_url=None) self.assertEqual(client_args['endpoint'].host, resolved_endpoint) self.assertEqual(client_args['request_signer'].region_name, resolved_region) def test_sts_with_endpoint_override_and_legacy_configured(self): override_endpoint = 'https://override-endpoint' self._set_endpoint_bridge_resolve(endpoint_url=override_endpoint) self.config_store.set_config_variable('sts_regional_endpoints', 'legacy') client_args = self.call_get_client_args( service_model=self._get_service_model('sts'), region_name='us-west-2', endpoint_url=override_endpoint) self.assertEqual(client_args['endpoint'].host, override_endpoint) def test_sts_http_scheme_for_override_endpoint(self): self.config_store.set_config_variable('sts_regional_endpoints', 'legacy') client_args = self.call_get_client_args( service_model=self._get_service_model('sts'), region_name='us-west-2', endpoint_url=None, is_secure=False, ) self.assertEqual(client_args['endpoint'].host, 'http://sts.amazonaws.com') def test_sts_regional_endpoints_defaults_to_legacy_if_not_set(self): self.config_store.set_config_variable('sts_regional_endpoints', None) client_args = self.call_get_client_args( service_model=self._get_service_model('sts'), region_name='us-west-2', endpoint_url=None) self.assertEqual(client_args['endpoint'].host, 'https://sts.amazonaws.com') self.assertEqual(client_args['request_signer'].region_name, 'us-east-1') def test_invalid_sts_regional_endpoints(self): self.config_store.set_config_variable('sts_regional_endpoints', 'invalid') with self.assertRaises( exceptions.InvalidSTSRegionalEndpointsConfigError): self.call_get_client_args( service_model=self._get_service_model('sts'), region_name='us-west-2', endpoint_url=None) def test_provides_total_max_attempts(self): config = botocore.config.Config(retries={'total_max_attempts': 10}) client_args = self.call_get_client_args(client_config=config) self.assertEqual( client_args['client_config'].retries['total_max_attempts'], 10) def test_provides_total_max_attempts_has_precedence(self): config = botocore.config.Config(retries={ 'total_max_attempts': 10, 'max_attempts': 5 }) client_args = self.call_get_client_args(client_config=config) self.assertEqual( client_args['client_config'].retries['total_max_attempts'], 10) self.assertNotIn('max_attempts', client_args['client_config'].retries) def test_provide_retry_config_maps_total_max_attempts(self): config = botocore.config.Config(retries={'max_attempts': 10}) client_args = self.call_get_client_args(client_config=config) self.assertEqual( client_args['client_config'].retries['total_max_attempts'], 11) self.assertNotIn('max_attempts', client_args['client_config'].retries) def test_can_merge_max_attempts(self): self.config_store.set_config_variable('max_attempts', 4) config = self.call_get_client_args()['client_config'] self.assertEqual(config.retries['total_max_attempts'], 4) def test_uses_config_value_if_present_for_max_attempts(self): config = self.call_get_client_args(client_config=Config( retries={'max_attempts': 2}))['client_config'] self.assertEqual(config.retries['total_max_attempts'], 3) def test_uses_client_config_over_config_store_max_attempts(self): self.config_store.set_config_variable('max_attempts', 4) config = self.call_get_client_args(client_config=Config( retries={'max_attempts': 2}))['client_config'] self.assertEqual(config.retries['total_max_attempts'], 3) def test_uses_client_config_total_over_config_store_max_attempts(self): self.config_store.set_config_variable('max_attempts', 4) config = self.call_get_client_args(client_config=Config( retries={'total_max_attempts': 2}))['client_config'] self.assertEqual(config.retries['total_max_attempts'], 2) def test_max_attempts_unset_if_retries_is_none(self): config = self.call_get_client_args(client_config=Config( retries=None))['client_config'] self.assertEqual(config.retries, {'mode': 'legacy'}) def test_retry_mode_set_on_config_store(self): self.config_store.set_config_variable('retry_mode', 'standard') config = self.call_get_client_args()['client_config'] self.assertEqual(config.retries['mode'], 'standard') def test_retry_mode_set_on_client_config(self): config = self.call_get_client_args(client_config=Config( retries={'mode': 'standard'}))['client_config'] self.assertEqual(config.retries['mode'], 'standard') def test_client_config_beats_config_store(self): self.config_store.set_config_variable('retry_mode', 'adaptive') config = self.call_get_client_args(client_config=Config( retries={'mode': 'standard'}))['client_config'] self.assertEqual(config.retries['mode'], 'standard')