def _register_endpoint_discovery(self, client, endpoint_url, config):
if endpoint_url is not None:
# Don't register any handlers in the case of a custom endpoint url
return
# Only attach handlers if the service supports discovery
if client.meta.service_model.endpoint_discovery_operation is None:
return
events = client.meta.events
service_id = client.meta.service_model.service_id.hyphenize()
enabled = False
if config and config.endpoint_discovery_enabled is not None:
enabled = config.endpoint_discovery_enabled
elif self._config_store:
enabled = self._config_store.get_config_variable(
'endpoint_discovery_enabled')
enabled = self._normalize_endpoint_discovery_config(enabled)
if enabled and self._requires_endpoint_discovery(client, enabled):
discover = enabled is True
manager = EndpointDiscoveryManager(client, always_discover=discover)
handler = EndpointDiscoveryHandler(manager)
handler.register(events, service_id)
else:
events.register('before-parameter-build',
block_endpoint_discovery_required_operations)
def construct_manager(self, cache=None, time=None, side_effect=None):
self.service_model = ServiceModel(self.service_description)
self.meta = mock.Mock(spec=ClientMeta)
self.meta.service_model = self.service_model
self.client = mock.Mock()
if side_effect is None:
side_effect = [{
'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]
}]
self.client.describe_endpoints.side_effect = side_effect
self.client.meta = self.meta
self.manager = EndpointDiscoveryManager(self.client,
cache=cache,
current_time=time)
def construct_manager(self, cache=None, time=None, side_effect=None):
self.service_model = ServiceModel(self.service_description)
self.meta = Mock(spec=ClientMeta)
self.meta.service_model = self.service_model
self.client = Mock()
if side_effect is None:
side_effect = [{
'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]
}]
self.client.describe_endpoints.side_effect = side_effect
self.client.meta = self.meta
self.manager = EndpointDiscoveryManager(
self.client, cache=cache, current_time=time
)
class TestEndpointDiscoveryManager(BaseEndpointDiscoveryTest):
def setUp(self):
super(TestEndpointDiscoveryManager, self).setUp()
self.construct_manager()
def construct_manager(self, cache=None, time=None, side_effect=None):
self.service_model = ServiceModel(self.service_description)
self.meta = mock.Mock(spec=ClientMeta)
self.meta.service_model = self.service_model
self.client = mock.Mock()
if side_effect is None:
side_effect = [{
'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]
}]
self.client.describe_endpoints.side_effect = side_effect
self.client.meta = self.meta
self.manager = EndpointDiscoveryManager(self.client,
cache=cache,
current_time=time)
def test_injects_api_version_if_endpoint_operation(self):
model = self.service_model.operation_model('DescribeEndpoints')
params = {'headers': {}}
inject_api_version_header_if_needed(model, params)
self.assertEqual(params['headers'].get('x-amz-api-version'),
'2018-08-31')
def test_no_inject_api_version_if_not_endpoint_operation(self):
model = self.service_model.operation_model('TestDiscoveryRequired')
params = {'headers': {}}
inject_api_version_header_if_needed(model, params)
self.assertNotIn('x-amz-api-version', params['headers'])
def test_gather_identifiers(self):
params = {'Foo': 'value1', 'Nested': {'Bar': 'value2'}}
operation = self.service_model.operation_model('TestDiscoveryRequired')
ids = self.manager.gather_identifiers(operation, params)
self.assertEqual(ids, {'Foo': 'value1', 'Bar': 'value2'})
def test_gather_identifiers_none(self):
operation = self.service_model.operation_model('TestDiscovery')
ids = self.manager.gather_identifiers(operation, {})
self.assertEqual(ids, {})
def test_describe_endpoint(self):
kwargs = {
'Operation': 'FooBar',
'Identifiers': {
'Foo': 'value1',
'Bar': 'value2'
},
}
self.manager.describe_endpoint(**kwargs)
self.client.describe_endpoints.assert_called_with(**kwargs)
def test_describe_endpoint_no_input(self):
describe = self.service_description['operations']['DescribeEndpoints']
del describe['input']
self.construct_manager()
self.manager.describe_endpoint(Operation='FooBar', Identifiers={})
self.client.describe_endpoints.assert_called_with()
def test_describe_endpoint_empty_input(self):
describe = self.service_description['operations']['DescribeEndpoints']
describe['input'] = {'shape': 'EmptyStruct'}
self.construct_manager()
self.manager.describe_endpoint(Operation='FooBar', Identifiers={})
self.client.describe_endpoints.assert_called_with()
def test_describe_endpoint_ids_and_operation(self):
cache = {}
self.construct_manager(cache=cache)
ids = {'Foo': 'value1', 'Bar': 'value2'}
kwargs = {
'Operation': 'TestDiscoveryRequired',
'Identifiers': ids,
}
self.manager.describe_endpoint(**kwargs)
self.client.describe_endpoints.assert_called_with(**kwargs)
key = ((('Bar', 'value2'), ('Foo', 'value1')), 'TestDiscoveryRequired')
self.assertIn(key, cache)
self.assertEqual(cache[key][0]['Address'], 'new.com')
self.manager.describe_endpoint(**kwargs)
call_count = self.client.describe_endpoints.call_count
self.assertEqual(call_count, 1)
def test_describe_endpoint_no_ids_or_operation(self):
cache = {}
describe = self.service_description['operations']['DescribeEndpoints']
describe['input'] = {'shape': 'EmptyStruct'}
self.construct_manager(cache=cache)
self.manager.describe_endpoint(Operation='TestDiscoveryRequired',
Identifiers={})
self.client.describe_endpoints.assert_called_with()
key = ()
self.assertIn(key, cache)
self.assertEqual(cache[key][0]['Address'], 'new.com')
self.manager.describe_endpoint(Operation='TestDiscoveryRequired',
Identifiers={})
call_count = self.client.describe_endpoints.call_count
self.assertEqual(call_count, 1)
def test_describe_endpoint_expired_entry(self):
current_time = time.time()
key = ()
cache = {
key: [{
'Address': 'old.com',
'Expiration': current_time - 10
}]
}
self.construct_manager(cache=cache)
kwargs = {
'Identifiers': {},
'Operation': 'TestDiscoveryRequired',
}
self.manager.describe_endpoint(**kwargs)
self.client.describe_endpoints.assert_called_with()
self.assertIn(key, cache)
self.assertEqual(cache[key][0]['Address'], 'new.com')
self.manager.describe_endpoint(**kwargs)
call_count = self.client.describe_endpoints.call_count
self.assertEqual(call_count, 1)
def test_describe_endpoint_cache_expiration(self):
def _time():
return float(0)
cache = {}
self.construct_manager(cache=cache, time=_time)
self.manager.describe_endpoint(Operation='TestDiscoveryRequired',
Identifiers={})
key = ()
self.assertIn(key, cache)
self.assertEqual(cache[key][0]['Expiration'], float(120))
def test_delete_endpoints_present(self):
key = ()
cache = {key: [{'Address': 'old.com', 'Expiration': 0}]}
self.construct_manager(cache=cache)
kwargs = {
'Identifiers': {},
'Operation': 'TestDiscoveryRequired',
}
self.manager.delete_endpoints(**kwargs)
self.assertEqual(cache, {})
def test_delete_endpoints_absent(self):
cache = {}
self.construct_manager(cache=cache)
kwargs = {
'Identifiers': {},
'Operation': 'TestDiscoveryRequired',
}
self.manager.delete_endpoints(**kwargs)
self.assertEqual(cache, {})
def test_describe_endpoint_optional_fails_no_cache(self):
side_effect = [ConnectionError(error=None)]
self.construct_manager(side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryOptional'}
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertIsNone(endpoint)
# This second call should be blocked as we just failed
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertIsNone(endpoint)
self.client.describe_endpoints.call_args_list == [mock.call()]
def test_describe_endpoint_optional_fails_stale_cache(self):
key = ()
cache = {key: [{'Address': 'old.com', 'Expiration': 0}]}
side_effect = [ConnectionError(error=None)] * 2
self.construct_manager(cache=cache, side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryOptional'}
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'old.com')
# This second call shouldn't go through as we just failed
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'old.com')
self.client.describe_endpoints.call_args_list == [mock.call()]
def test_describe_endpoint_required_fails_no_cache(self):
side_effect = [ConnectionError(error=None)] * 2
self.construct_manager(side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryRequired'}
with self.assertRaises(EndpointDiscoveryRefreshFailed):
self.manager.describe_endpoint(**kwargs)
# This second call should go through, as we have no cache
with self.assertRaises(EndpointDiscoveryRefreshFailed):
self.manager.describe_endpoint(**kwargs)
describe_count = self.client.describe_endpoints.call_count
self.assertEqual(describe_count, 2)
def test_describe_endpoint_required_fails_stale_cache(self):
key = ()
cache = {key: [{'Address': 'old.com', 'Expiration': 0}]}
side_effect = [ConnectionError(error=None)] * 2
self.construct_manager(cache=cache, side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryRequired'}
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'old.com')
# We have a stale endpoint, so this shouldn't fail or force a refresh
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'old.com')
self.client.describe_endpoints.call_args_list == [mock.call()]
def test_describe_endpoint_required_force_refresh_success(self):
side_effect = [
ConnectionError(error=None),
{
'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]
},
]
self.construct_manager(side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryRequired'}
# First call will fail
with self.assertRaises(EndpointDiscoveryRefreshFailed):
self.manager.describe_endpoint(**kwargs)
self.client.describe_endpoints.call_args_list == [mock.call()]
# Force a refresh if the cache is empty but discovery is required
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'new.com')
def test_describe_endpoint_retries_after_failing(self):
fake_time = mock.Mock()
fake_time.side_effect = [0, 100, 200]
side_effect = [
ConnectionError(error=None),
{
'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]
},
]
self.construct_manager(side_effect=side_effect, time=fake_time)
kwargs = {'Operation': 'TestDiscoveryOptional'}
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertIsNone(endpoint)
self.client.describe_endpoints.call_args_list == [mock.call()]
# Second time should try again as enough time has elapsed
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'new.com')
class TestEndpointDiscoveryManager(BaseEndpointDiscoveryTest):
def setUp(self):
super(TestEndpointDiscoveryManager, self).setUp()
self.construct_manager()
def construct_manager(self, cache=None, time=None, side_effect=None):
self.service_model = ServiceModel(self.service_description)
self.meta = Mock(spec=ClientMeta)
self.meta.service_model = self.service_model
self.client = Mock()
if side_effect is None:
side_effect = [{
'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]
}]
self.client.describe_endpoints.side_effect = side_effect
self.client.meta = self.meta
self.manager = EndpointDiscoveryManager(
self.client, cache=cache, current_time=time
)
def test_injects_api_version_if_endpoint_operation(self):
model = self.service_model.operation_model('DescribeEndpoints')
params = {'headers': {}}
inject_api_version_header_if_needed(model, params)
self.assertEqual(params['headers'].get('x-amz-api-version'),
'2018-08-31')
def test_no_inject_api_version_if_not_endpoint_operation(self):
model = self.service_model.operation_model('TestDiscoveryRequired')
params = {'headers': {}}
inject_api_version_header_if_needed(model, params)
self.assertNotIn('x-amz-api-version', params['headers'])
def test_gather_identifiers(self):
params = {
'Foo': 'value1',
'Nested': {'Bar': 'value2'}
}
operation = self.service_model.operation_model('TestDiscoveryRequired')
ids = self.manager.gather_identifiers(operation, params)
self.assertEqual(ids, {'Foo': 'value1', 'Bar': 'value2'})
def test_gather_identifiers_none(self):
operation = self.service_model.operation_model('TestDiscovery')
ids = self.manager.gather_identifiers(operation, {})
self.assertEqual(ids, {})
def test_describe_endpoint(self):
kwargs = {
'Operation': 'FooBar',
'Identifiers': {'Foo': 'value1', 'Bar': 'value2'},
}
self.manager.describe_endpoint(**kwargs)
self.client.describe_endpoints.assert_called_with(**kwargs)
def test_describe_endpoint_no_input(self):
describe = self.service_description['operations']['DescribeEndpoints']
del describe['input']
self.construct_manager()
self.manager.describe_endpoint(Operation='FooBar', Identifiers={})
self.client.describe_endpoints.assert_called_with()
def test_describe_endpoint_empty_input(self):
describe = self.service_description['operations']['DescribeEndpoints']
describe['input'] = {'shape': 'EmptyStruct'}
self.construct_manager()
self.manager.describe_endpoint(Operation='FooBar', Identifiers={})
self.client.describe_endpoints.assert_called_with()
def test_describe_endpoint_ids_and_operation(self):
cache = {}
self.construct_manager(cache=cache)
ids = {'Foo': 'value1', 'Bar': 'value2'}
kwargs = {
'Operation': 'TestDiscoveryRequired',
'Identifiers': ids,
}
self.manager.describe_endpoint(**kwargs)
self.client.describe_endpoints.assert_called_with(**kwargs)
key = ((('Bar', 'value2'), ('Foo', 'value1')), 'TestDiscoveryRequired')
self.assertIn(key, cache)
self.assertEqual(cache[key][0]['Address'], 'new.com')
self.manager.describe_endpoint(**kwargs)
call_count = self.client.describe_endpoints.call_count
self.assertEqual(call_count, 1)
def test_describe_endpoint_no_ids_or_operation(self):
cache = {}
describe = self.service_description['operations']['DescribeEndpoints']
describe['input'] = {'shape': 'EmptyStruct'}
self.construct_manager(cache=cache)
self.manager.describe_endpoint(
Operation='TestDiscoveryRequired', Identifiers={}
)
self.client.describe_endpoints.assert_called_with()
key = ()
self.assertIn(key, cache)
self.assertEqual(cache[key][0]['Address'], 'new.com')
self.manager.describe_endpoint(
Operation='TestDiscoveryRequired', Identifiers={}
)
call_count = self.client.describe_endpoints.call_count
self.assertEqual(call_count, 1)
def test_describe_endpoint_expired_entry(self):
current_time = time.time()
key = ()
cache = {
key: [{'Address': 'old.com', 'Expiration': current_time - 10}]
}
self.construct_manager(cache=cache)
kwargs = {
'Identifiers': {},
'Operation': 'TestDiscoveryRequired',
}
self.manager.describe_endpoint(**kwargs)
self.client.describe_endpoints.assert_called_with()
self.assertIn(key, cache)
self.assertEqual(cache[key][0]['Address'], 'new.com')
self.manager.describe_endpoint(**kwargs)
call_count = self.client.describe_endpoints.call_count
self.assertEqual(call_count, 1)
def test_describe_endpoint_cache_expiration(self):
def _time():
return float(0)
cache = {}
self.construct_manager(cache=cache, time=_time)
self.manager.describe_endpoint(
Operation='TestDiscoveryRequired', Identifiers={}
)
key = ()
self.assertIn(key, cache)
self.assertEqual(cache[key][0]['Expiration'], float(120))
def test_delete_endpoints_present(self):
key = ()
cache = {
key: [{'Address': 'old.com', 'Expiration': 0}]
}
self.construct_manager(cache=cache)
kwargs = {
'Identifiers': {},
'Operation': 'TestDiscoveryRequired',
}
self.manager.delete_endpoints(**kwargs)
self.assertEqual(cache, {})
def test_delete_endpoints_absent(self):
cache = {}
self.construct_manager(cache=cache)
kwargs = {
'Identifiers': {},
'Operation': 'TestDiscoveryRequired',
}
self.manager.delete_endpoints(**kwargs)
self.assertEqual(cache, {})
def test_describe_endpoint_optional_fails_no_cache(self):
side_effect = [ConnectionError(error=None)]
self.construct_manager(side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryOptional'}
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertIsNone(endpoint)
# This second call should be blocked as we just failed
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertIsNone(endpoint)
self.client.describe_endpoints.call_args_list == [call()]
def test_describe_endpoint_optional_fails_stale_cache(self):
key = ()
cache = {
key: [{'Address': 'old.com', 'Expiration': 0}]
}
side_effect = [ConnectionError(error=None)] * 2
self.construct_manager(cache=cache, side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryOptional'}
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'old.com')
# This second call shouldn't go through as we just failed
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'old.com')
self.client.describe_endpoints.call_args_list == [call()]
def test_describe_endpoint_required_fails_no_cache(self):
side_effect = [ConnectionError(error=None)] * 2
self.construct_manager(side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryRequired'}
with self.assertRaises(EndpointDiscoveryRefreshFailed):
self.manager.describe_endpoint(**kwargs)
# This second call should go through, as we have no cache
with self.assertRaises(EndpointDiscoveryRefreshFailed):
self.manager.describe_endpoint(**kwargs)
describe_count = self.client.describe_endpoints.call_count
self.assertEqual(describe_count, 2)
def test_describe_endpoint_required_fails_stale_cache(self):
key = ()
cache = {
key: [{'Address': 'old.com', 'Expiration': 0}]
}
side_effect = [ConnectionError(error=None)] * 2
self.construct_manager(cache=cache, side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryRequired'}
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'old.com')
# We have a stale endpoint, so this shouldn't fail or force a refresh
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'old.com')
self.client.describe_endpoints.call_args_list == [call()]
def test_describe_endpoint_required_force_refresh_success(self):
side_effect = [
ConnectionError(error=None),
{'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]},
]
self.construct_manager(side_effect=side_effect)
kwargs = {'Operation': 'TestDiscoveryRequired'}
# First call will fail
with self.assertRaises(EndpointDiscoveryRefreshFailed):
self.manager.describe_endpoint(**kwargs)
self.client.describe_endpoints.call_args_list == [call()]
# Force a refresh if the cache is empty but discovery is required
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'new.com')
def test_describe_endpoint_retries_after_failing(self):
fake_time = Mock()
fake_time.side_effect = [0, 100, 200]
side_effect = [
ConnectionError(error=None),
{'Endpoints': [{
'Address': 'new.com',
'CachePeriodInMinutes': 2,
}]},
]
self.construct_manager(side_effect=side_effect, time=fake_time)
kwargs = {'Operation': 'TestDiscoveryOptional'}
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertIsNone(endpoint)
self.client.describe_endpoints.call_args_list == [call()]
# Second time should try again as enough time has elapsed
endpoint = self.manager.describe_endpoint(**kwargs)
self.assertEqual(endpoint, 'new.com')
