def _register_endpoint_discovery(self, client, endpoint_url, config): if endpoint_url is not None: # Don't register any handlers in the case of a custom endpoint url return # Only attach handlers if the service supports discovery if client.meta.service_model.endpoint_discovery_operation is None: return events = client.meta.events service_id = client.meta.service_model.service_id.hyphenize() enabled = False if config and config.endpoint_discovery_enabled is not None: enabled = config.endpoint_discovery_enabled elif self._config_store: enabled = self._config_store.get_config_variable( 'endpoint_discovery_enabled') enabled = self._normalize_endpoint_discovery_config(enabled) if enabled and self._requires_endpoint_discovery(client, enabled): discover = enabled is True manager = EndpointDiscoveryManager(client, always_discover=discover) handler = EndpointDiscoveryHandler(manager) handler.register(events, service_id) else: events.register('before-parameter-build', block_endpoint_discovery_required_operations)
def construct_manager(self, cache=None, time=None, side_effect=None): self.service_model = ServiceModel(self.service_description) self.meta = mock.Mock(spec=ClientMeta) self.meta.service_model = self.service_model self.client = mock.Mock() if side_effect is None: side_effect = [{ 'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }] }] self.client.describe_endpoints.side_effect = side_effect self.client.meta = self.meta self.manager = EndpointDiscoveryManager(self.client, cache=cache, current_time=time)
def construct_manager(self, cache=None, time=None, side_effect=None): self.service_model = ServiceModel(self.service_description) self.meta = Mock(spec=ClientMeta) self.meta.service_model = self.service_model self.client = Mock() if side_effect is None: side_effect = [{ 'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }] }] self.client.describe_endpoints.side_effect = side_effect self.client.meta = self.meta self.manager = EndpointDiscoveryManager( self.client, cache=cache, current_time=time )
class TestEndpointDiscoveryManager(BaseEndpointDiscoveryTest): def setUp(self): super(TestEndpointDiscoveryManager, self).setUp() self.construct_manager() def construct_manager(self, cache=None, time=None, side_effect=None): self.service_model = ServiceModel(self.service_description) self.meta = mock.Mock(spec=ClientMeta) self.meta.service_model = self.service_model self.client = mock.Mock() if side_effect is None: side_effect = [{ 'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }] }] self.client.describe_endpoints.side_effect = side_effect self.client.meta = self.meta self.manager = EndpointDiscoveryManager(self.client, cache=cache, current_time=time) def test_injects_api_version_if_endpoint_operation(self): model = self.service_model.operation_model('DescribeEndpoints') params = {'headers': {}} inject_api_version_header_if_needed(model, params) self.assertEqual(params['headers'].get('x-amz-api-version'), '2018-08-31') def test_no_inject_api_version_if_not_endpoint_operation(self): model = self.service_model.operation_model('TestDiscoveryRequired') params = {'headers': {}} inject_api_version_header_if_needed(model, params) self.assertNotIn('x-amz-api-version', params['headers']) def test_gather_identifiers(self): params = {'Foo': 'value1', 'Nested': {'Bar': 'value2'}} operation = self.service_model.operation_model('TestDiscoveryRequired') ids = self.manager.gather_identifiers(operation, params) self.assertEqual(ids, {'Foo': 'value1', 'Bar': 'value2'}) def test_gather_identifiers_none(self): operation = self.service_model.operation_model('TestDiscovery') ids = self.manager.gather_identifiers(operation, {}) self.assertEqual(ids, {}) def test_describe_endpoint(self): kwargs = { 'Operation': 'FooBar', 'Identifiers': { 'Foo': 'value1', 'Bar': 'value2' }, } self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.assert_called_with(**kwargs) def test_describe_endpoint_no_input(self): describe = self.service_description['operations']['DescribeEndpoints'] del describe['input'] self.construct_manager() self.manager.describe_endpoint(Operation='FooBar', Identifiers={}) self.client.describe_endpoints.assert_called_with() def test_describe_endpoint_empty_input(self): describe = self.service_description['operations']['DescribeEndpoints'] describe['input'] = {'shape': 'EmptyStruct'} self.construct_manager() self.manager.describe_endpoint(Operation='FooBar', Identifiers={}) self.client.describe_endpoints.assert_called_with() def test_describe_endpoint_ids_and_operation(self): cache = {} self.construct_manager(cache=cache) ids = {'Foo': 'value1', 'Bar': 'value2'} kwargs = { 'Operation': 'TestDiscoveryRequired', 'Identifiers': ids, } self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.assert_called_with(**kwargs) key = ((('Bar', 'value2'), ('Foo', 'value1')), 'TestDiscoveryRequired') self.assertIn(key, cache) self.assertEqual(cache[key][0]['Address'], 'new.com') self.manager.describe_endpoint(**kwargs) call_count = self.client.describe_endpoints.call_count self.assertEqual(call_count, 1) def test_describe_endpoint_no_ids_or_operation(self): cache = {} describe = self.service_description['operations']['DescribeEndpoints'] describe['input'] = {'shape': 'EmptyStruct'} self.construct_manager(cache=cache) self.manager.describe_endpoint(Operation='TestDiscoveryRequired', Identifiers={}) self.client.describe_endpoints.assert_called_with() key = () self.assertIn(key, cache) self.assertEqual(cache[key][0]['Address'], 'new.com') self.manager.describe_endpoint(Operation='TestDiscoveryRequired', Identifiers={}) call_count = self.client.describe_endpoints.call_count self.assertEqual(call_count, 1) def test_describe_endpoint_expired_entry(self): current_time = time.time() key = () cache = { key: [{ 'Address': 'old.com', 'Expiration': current_time - 10 }] } self.construct_manager(cache=cache) kwargs = { 'Identifiers': {}, 'Operation': 'TestDiscoveryRequired', } self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.assert_called_with() self.assertIn(key, cache) self.assertEqual(cache[key][0]['Address'], 'new.com') self.manager.describe_endpoint(**kwargs) call_count = self.client.describe_endpoints.call_count self.assertEqual(call_count, 1) def test_describe_endpoint_cache_expiration(self): def _time(): return float(0) cache = {} self.construct_manager(cache=cache, time=_time) self.manager.describe_endpoint(Operation='TestDiscoveryRequired', Identifiers={}) key = () self.assertIn(key, cache) self.assertEqual(cache[key][0]['Expiration'], float(120)) def test_delete_endpoints_present(self): key = () cache = {key: [{'Address': 'old.com', 'Expiration': 0}]} self.construct_manager(cache=cache) kwargs = { 'Identifiers': {}, 'Operation': 'TestDiscoveryRequired', } self.manager.delete_endpoints(**kwargs) self.assertEqual(cache, {}) def test_delete_endpoints_absent(self): cache = {} self.construct_manager(cache=cache) kwargs = { 'Identifiers': {}, 'Operation': 'TestDiscoveryRequired', } self.manager.delete_endpoints(**kwargs) self.assertEqual(cache, {}) def test_describe_endpoint_optional_fails_no_cache(self): side_effect = [ConnectionError(error=None)] self.construct_manager(side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryOptional'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertIsNone(endpoint) # This second call should be blocked as we just failed endpoint = self.manager.describe_endpoint(**kwargs) self.assertIsNone(endpoint) self.client.describe_endpoints.call_args_list == [mock.call()] def test_describe_endpoint_optional_fails_stale_cache(self): key = () cache = {key: [{'Address': 'old.com', 'Expiration': 0}]} side_effect = [ConnectionError(error=None)] * 2 self.construct_manager(cache=cache, side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryOptional'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') # This second call shouldn't go through as we just failed endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') self.client.describe_endpoints.call_args_list == [mock.call()] def test_describe_endpoint_required_fails_no_cache(self): side_effect = [ConnectionError(error=None)] * 2 self.construct_manager(side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryRequired'} with self.assertRaises(EndpointDiscoveryRefreshFailed): self.manager.describe_endpoint(**kwargs) # This second call should go through, as we have no cache with self.assertRaises(EndpointDiscoveryRefreshFailed): self.manager.describe_endpoint(**kwargs) describe_count = self.client.describe_endpoints.call_count self.assertEqual(describe_count, 2) def test_describe_endpoint_required_fails_stale_cache(self): key = () cache = {key: [{'Address': 'old.com', 'Expiration': 0}]} side_effect = [ConnectionError(error=None)] * 2 self.construct_manager(cache=cache, side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryRequired'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') # We have a stale endpoint, so this shouldn't fail or force a refresh endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') self.client.describe_endpoints.call_args_list == [mock.call()] def test_describe_endpoint_required_force_refresh_success(self): side_effect = [ ConnectionError(error=None), { 'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }] }, ] self.construct_manager(side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryRequired'} # First call will fail with self.assertRaises(EndpointDiscoveryRefreshFailed): self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.call_args_list == [mock.call()] # Force a refresh if the cache is empty but discovery is required endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'new.com') def test_describe_endpoint_retries_after_failing(self): fake_time = mock.Mock() fake_time.side_effect = [0, 100, 200] side_effect = [ ConnectionError(error=None), { 'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }] }, ] self.construct_manager(side_effect=side_effect, time=fake_time) kwargs = {'Operation': 'TestDiscoveryOptional'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertIsNone(endpoint) self.client.describe_endpoints.call_args_list == [mock.call()] # Second time should try again as enough time has elapsed endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'new.com')
class TestEndpointDiscoveryManager(BaseEndpointDiscoveryTest): def setUp(self): super(TestEndpointDiscoveryManager, self).setUp() self.construct_manager() def construct_manager(self, cache=None, time=None, side_effect=None): self.service_model = ServiceModel(self.service_description) self.meta = Mock(spec=ClientMeta) self.meta.service_model = self.service_model self.client = Mock() if side_effect is None: side_effect = [{ 'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }] }] self.client.describe_endpoints.side_effect = side_effect self.client.meta = self.meta self.manager = EndpointDiscoveryManager( self.client, cache=cache, current_time=time ) def test_injects_api_version_if_endpoint_operation(self): model = self.service_model.operation_model('DescribeEndpoints') params = {'headers': {}} inject_api_version_header_if_needed(model, params) self.assertEqual(params['headers'].get('x-amz-api-version'), '2018-08-31') def test_no_inject_api_version_if_not_endpoint_operation(self): model = self.service_model.operation_model('TestDiscoveryRequired') params = {'headers': {}} inject_api_version_header_if_needed(model, params) self.assertNotIn('x-amz-api-version', params['headers']) def test_gather_identifiers(self): params = { 'Foo': 'value1', 'Nested': {'Bar': 'value2'} } operation = self.service_model.operation_model('TestDiscoveryRequired') ids = self.manager.gather_identifiers(operation, params) self.assertEqual(ids, {'Foo': 'value1', 'Bar': 'value2'}) def test_gather_identifiers_none(self): operation = self.service_model.operation_model('TestDiscovery') ids = self.manager.gather_identifiers(operation, {}) self.assertEqual(ids, {}) def test_describe_endpoint(self): kwargs = { 'Operation': 'FooBar', 'Identifiers': {'Foo': 'value1', 'Bar': 'value2'}, } self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.assert_called_with(**kwargs) def test_describe_endpoint_no_input(self): describe = self.service_description['operations']['DescribeEndpoints'] del describe['input'] self.construct_manager() self.manager.describe_endpoint(Operation='FooBar', Identifiers={}) self.client.describe_endpoints.assert_called_with() def test_describe_endpoint_empty_input(self): describe = self.service_description['operations']['DescribeEndpoints'] describe['input'] = {'shape': 'EmptyStruct'} self.construct_manager() self.manager.describe_endpoint(Operation='FooBar', Identifiers={}) self.client.describe_endpoints.assert_called_with() def test_describe_endpoint_ids_and_operation(self): cache = {} self.construct_manager(cache=cache) ids = {'Foo': 'value1', 'Bar': 'value2'} kwargs = { 'Operation': 'TestDiscoveryRequired', 'Identifiers': ids, } self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.assert_called_with(**kwargs) key = ((('Bar', 'value2'), ('Foo', 'value1')), 'TestDiscoveryRequired') self.assertIn(key, cache) self.assertEqual(cache[key][0]['Address'], 'new.com') self.manager.describe_endpoint(**kwargs) call_count = self.client.describe_endpoints.call_count self.assertEqual(call_count, 1) def test_describe_endpoint_no_ids_or_operation(self): cache = {} describe = self.service_description['operations']['DescribeEndpoints'] describe['input'] = {'shape': 'EmptyStruct'} self.construct_manager(cache=cache) self.manager.describe_endpoint( Operation='TestDiscoveryRequired', Identifiers={} ) self.client.describe_endpoints.assert_called_with() key = () self.assertIn(key, cache) self.assertEqual(cache[key][0]['Address'], 'new.com') self.manager.describe_endpoint( Operation='TestDiscoveryRequired', Identifiers={} ) call_count = self.client.describe_endpoints.call_count self.assertEqual(call_count, 1) def test_describe_endpoint_expired_entry(self): current_time = time.time() key = () cache = { key: [{'Address': 'old.com', 'Expiration': current_time - 10}] } self.construct_manager(cache=cache) kwargs = { 'Identifiers': {}, 'Operation': 'TestDiscoveryRequired', } self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.assert_called_with() self.assertIn(key, cache) self.assertEqual(cache[key][0]['Address'], 'new.com') self.manager.describe_endpoint(**kwargs) call_count = self.client.describe_endpoints.call_count self.assertEqual(call_count, 1) def test_describe_endpoint_cache_expiration(self): def _time(): return float(0) cache = {} self.construct_manager(cache=cache, time=_time) self.manager.describe_endpoint( Operation='TestDiscoveryRequired', Identifiers={} ) key = () self.assertIn(key, cache) self.assertEqual(cache[key][0]['Expiration'], float(120)) def test_delete_endpoints_present(self): key = () cache = { key: [{'Address': 'old.com', 'Expiration': 0}] } self.construct_manager(cache=cache) kwargs = { 'Identifiers': {}, 'Operation': 'TestDiscoveryRequired', } self.manager.delete_endpoints(**kwargs) self.assertEqual(cache, {}) def test_delete_endpoints_absent(self): cache = {} self.construct_manager(cache=cache) kwargs = { 'Identifiers': {}, 'Operation': 'TestDiscoveryRequired', } self.manager.delete_endpoints(**kwargs) self.assertEqual(cache, {}) def test_describe_endpoint_optional_fails_no_cache(self): side_effect = [ConnectionError(error=None)] self.construct_manager(side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryOptional'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertIsNone(endpoint) # This second call should be blocked as we just failed endpoint = self.manager.describe_endpoint(**kwargs) self.assertIsNone(endpoint) self.client.describe_endpoints.call_args_list == [call()] def test_describe_endpoint_optional_fails_stale_cache(self): key = () cache = { key: [{'Address': 'old.com', 'Expiration': 0}] } side_effect = [ConnectionError(error=None)] * 2 self.construct_manager(cache=cache, side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryOptional'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') # This second call shouldn't go through as we just failed endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') self.client.describe_endpoints.call_args_list == [call()] def test_describe_endpoint_required_fails_no_cache(self): side_effect = [ConnectionError(error=None)] * 2 self.construct_manager(side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryRequired'} with self.assertRaises(EndpointDiscoveryRefreshFailed): self.manager.describe_endpoint(**kwargs) # This second call should go through, as we have no cache with self.assertRaises(EndpointDiscoveryRefreshFailed): self.manager.describe_endpoint(**kwargs) describe_count = self.client.describe_endpoints.call_count self.assertEqual(describe_count, 2) def test_describe_endpoint_required_fails_stale_cache(self): key = () cache = { key: [{'Address': 'old.com', 'Expiration': 0}] } side_effect = [ConnectionError(error=None)] * 2 self.construct_manager(cache=cache, side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryRequired'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') # We have a stale endpoint, so this shouldn't fail or force a refresh endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'old.com') self.client.describe_endpoints.call_args_list == [call()] def test_describe_endpoint_required_force_refresh_success(self): side_effect = [ ConnectionError(error=None), {'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }]}, ] self.construct_manager(side_effect=side_effect) kwargs = {'Operation': 'TestDiscoveryRequired'} # First call will fail with self.assertRaises(EndpointDiscoveryRefreshFailed): self.manager.describe_endpoint(**kwargs) self.client.describe_endpoints.call_args_list == [call()] # Force a refresh if the cache is empty but discovery is required endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'new.com') def test_describe_endpoint_retries_after_failing(self): fake_time = Mock() fake_time.side_effect = [0, 100, 200] side_effect = [ ConnectionError(error=None), {'Endpoints': [{ 'Address': 'new.com', 'CachePeriodInMinutes': 2, }]}, ] self.construct_manager(side_effect=side_effect, time=fake_time) kwargs = {'Operation': 'TestDiscoveryOptional'} endpoint = self.manager.describe_endpoint(**kwargs) self.assertIsNone(endpoint) self.client.describe_endpoints.call_args_list == [call()] # Second time should try again as enough time has elapsed endpoint = self.manager.describe_endpoint(**kwargs) self.assertEqual(endpoint, 'new.com')