def retry_when_empty_result_test(self, mock_http_response_list, max_tries, retry_when_empty_result, expected_http_status_code, expected_json_response, expected_requests_number): self.adapter.register_uri( method='GET', url= 'mock://metadata.pikselpalette.com/data/contents?include=assets,categories&owner=test&withRef=test:c0007', response_list=mock_http_response_list) http_executor = http.HttpExecutor(auth.AuthFactory.create( auth_type=auth.AuthType.BYO_TOKEN, byo_token='tkn'), session=self.session_mock, user_agent='backoff_test', backoff_strategy={ 'max_tries': max_tries, 'interval': 0 }) actual_response = http_executor.request( method="GET", url= "mock://metadata.pikselpalette.com/data/contents?include=assets,categories&owner=test&withRef=test:c0007", resource_name='contents', retry_when_empty_result=retry_when_empty_result) assert_that(actual_response.status, is_(expected_http_status_code)) assert_that(actual_response.data, equal_to(expected_json_response)) assert_that(self.adapter.call_count, is_(expected_requests_number)) for i in range(expected_requests_number): assert_that(self.adapter.request_history[i].method, is_('GET')) assert_that( self.adapter.request_history[i].url, is_('mock://metadata.pikselpalette.com/data/contents?include=assets,categories&owner=test&withRef=test:c0007' ))
def test_request_given_server_returns_an_authorisation_error_then_fetch_token_does_not_count_as_retry( self, mock_fetch_token): mock_fetch_token.return_value = "validToken" json_response = 'Error getting resource' self.adapter.register_uri('GET', 'mock://test.com', [{ 'text': 'resp1', 'status_code': 401 }, { 'text': json_response, 'status_code': 500 }, { 'text': json_response, 'status_code': 200 }]) http_executor = http.HttpExecutor(auth.AuthFactory.create( grant_client_id="client_id", grant_client_secret="client_secret", token_url='mock://token-url.com'), session=self.session_mock, backoff_strategy={ 'interval': 0, 'max_tries': 1 }) with pytest.raises(error.HttpError) as e: http_executor.request("GET", "mock://test.com") assert_that( e.value.args[0], is_('An unexpected error occurred. HTTP Status code: 500. Error message: ' 'Expecting value: line 1 column 1 (char 0). '))
def test_request_given_byo_type_and_server_returns_an_authorisation_error_then_error_is_propagated( self): json_response = '{"resp2": "resp2"}' self.adapter.register_uri('GET', 'mock://test.com', [{ 'text': 'resp1', 'status_code': 401 }, { 'text': json_response, 'status_code': 200 }]) http_executor = http.HttpExecutor(auth.AuthFactory.create( auth_type=auth.AuthType.BYO_TOKEN, byo_token='asdf'), session=self.session_mock, backoff_strategy={ 'interval': 0, 'max_tries': 1 }) with pytest.raises(error.HttpError) as e: http_executor.request("GET", "mock://test.com") assert_that( e.value.args[0], is_('An unexpected error occurred. HTTP Status code: 401.' ' Error message: Expecting value: line 1 column 1 (char 0). '))
def test_request_given_server_returns_an_authorisation_error_then_the_request_should_be_retried( self, mock_fetch_token): mock_fetch_token.return_value = "validToken" json_response = '{"resp2": "resp2"}' self.adapter.register_uri('GET', 'mock://test.com', [{ 'text': 'resp1', 'status_code': 401 }, { 'text': json_response, 'status_code': 200 }]) http_executor = http.HttpExecutor(auth.AuthFactory.create( grant_client_id="client_id", grant_client_secret="client_secret", token_url='mock://token-url.com'), session=self.session_mock, backoff_strategy={ 'interval': 0, 'max_tries': 1 }) response = http_executor.request("GET", "mock://test.com") assert_that(response.data, equal_to(json.loads(json_response)))
def test_request_given_server_returns_an_authorisation_error_fetching_the_token_then_error_is_not_retried( self): mock_auth = Mock() mock_session = Mock() mock_session.request.side_effect = error.AuthorisationError( 'Auth error') http_executor = http.HttpExecutor(mock_auth, session=mock_session, backoff_strategy={ 'interval': 0, 'max_tries': 3 }) with pytest.raises(error.AuthorisationError): http_executor.request("GET", "mock://test.com") mock_session.request.assert_called_once_with( 'GET', 'mock://test.com', allow_redirects=False, data=None, headers={ 'User-Agent': mock.ANY, 'Content-Type': 'application/vnd.piksel+json', 'Accept': 'application/vnd.piksel+json', 'X-Correlation-ID': None }, params=None, timeout=240)
def test_given_none_content_type_property_then_header_should_contain_none_content_type( self): http_executor = http.HttpExecutor(None, content_type=None) assert_that(http_executor.common_headers['Content-Type'], is_('application/vnd.piksel+json')) assert_that(http_executor.common_headers['Accept'], is_('application/vnd.piksel+json'))
def test_request_given_server_returns_an_authorisation_error_fetching_the_token_then_error_is_not_retried( self, mock_fetch_token): mock_fetch_token.side_effect = OAuth2Error() json_response = '{"resp2": "resp2"}' self.adapter.register_uri('GET', 'mock://test.com', [{ 'text': 'resp1', 'status_code': 401 }, { 'text': json_response, 'status_code': 200 }]) http_executor = http.HttpExecutor(auth.AuthFactory.create( grant_client_id="client_id", grant_client_secret="client_secret", token_url='mock://token-url.com'), session=self.session_mock, backoff_strategy={ 'interval': 0, 'max_tries': 1 }) with pytest.raises(error.AuthorisationError): http_executor.request("GET", "mock://test.com")
def test_request_given_a_list_of_parameters_then_they_are_added_to_the_request( self, session_mock): # There is an issue where parameters won't be added to the request if the prefix does not start # with http https://bugs.launchpad.net/requests-mock/+bug/1518497. So request-mock can't be used here # to check parameters session_mock.request.return_value.url = 'mock://some_url' session_mock.request.return_value.status_code = 200 session_mock.request.return_value.is_redirect = False http_executor = http.HttpExecutor(auth.AuthFactory.create( grant_client_id="client_id", grant_client_secret="client_secret"), session=session_mock, correlation_id="my_correlation_id") http_executor.request("POST", "mock://some_url", data='some data', headers={'New-Header': 'SomeValue'}, params={'key1': 'value1'}) expected_headers = { 'User-Agent': http_executor.user_agent, "Content-Type": "application/vnd.piksel+json", "Accept": "application/vnd.piksel+json", "X-Correlation-ID": "my_correlation_id", "New-Header": 'SomeValue' } session_mock.request.assert_called_with('POST', 'mock://some_url', allow_redirects=False, data='some data', headers=expected_headers, params={'key1': 'value1'}, timeout=240)
def __init__(self, registry_url, proxies=None, user_agent=None, backoff_strategy=None, adapters=None, request_timeout=None, model_resolution=None, correlation_id=None, user_id=None, application_id=None, content_type=None, **auth_kwargs): logging.debug('Client initialising with registry_url=%s ', registry_url) self._registry_url = registry_url self._request_timeout = request_timeout or env.DEFAULT_REQUEST_TIMEOUT_SECONDS self._proxies = proxies self._user_agent = user_agent self._correlation_id = correlation_id.strip() if correlation_id else None self.user_id = user_id.strip() if user_id else None self.application_id = application_id.strip() if application_id else None self._model_resolution = model_resolution self._registry = self._initialize_registry(adapters, backoff_strategy, content_type, **auth_kwargs) self._auth = AuthFactory.create(token_url=self._get_token_url(auth_kwargs.get("auth_type", None)), request_timeout=self._request_timeout, **auth_kwargs) self._auth.register_adapters(adapters) self._auth.init_session() self._http = http.HttpExecutor(self._auth, proxies=self._proxies, user_agent=self._user_agent, session=self._auth.session, request_timeout=self._request_timeout, correlation_id=self._correlation_id, user_id=self.user_id, application_id=self.application_id, backoff_strategy=backoff_strategy, content_type=content_type)
def backoff_test(self, mock_http_response_list, max_tries, retry_http_codes, expected_http_status_code, expected_json_response, expected_requests_number): self.adapter.register_uri(method='GET', url='mock://test.com', response_list=mock_http_response_list) http_executor = http.HttpExecutor(auth.AuthFactory.create( auth_type=auth.AuthType.BYO_TOKEN, byo_token='tkn'), session=self.session_mock, user_agent='backoff_test', backoff_strategy={ 'max_tries': max_tries, 'max_time': 300, 'interval': 0, 'retry_http_status_codes': retry_http_codes }) actual_response = http_executor.request("GET", "mock://test.com") assert_that(actual_response.status, is_(expected_http_status_code)) assert_that(actual_response.data, equal_to(expected_json_response)) assert_that(self.adapter.call_count, is_(expected_requests_number)) for i in range(expected_requests_number): assert_that(self.adapter.request_history[i].method, is_('GET')) and \ assert_that(self.adapter.request_history[i].url, is_('mock://test.com'))
def test_retries_for_http_status_code_specified_in_backoff_strategy_reach_default_limit( self): def _patch_max_time_to_run_unit_test_faster(): import sequoia sequoia.http.HttpExecutor.MAX_TIME_SECONDS = 0.5 json_response_404 = { 'statusCode': 404, 'error': 'Not Found', 'message': 'Not Found' } mock_http_response_list = [{ 'json': json_response_404, 'status_code': 404 }] _patch_max_time_to_run_unit_test_faster() self.adapter.register_uri(method='GET', url='mock://test.com', response_list=mock_http_response_list) http_executor = http.HttpExecutor(auth.AuthFactory.create( auth_type=auth.AuthType.BYO_TOKEN, byo_token='tkn'), session=self.session_mock, backoff_strategy={ 'max_tries': None, 'max_time': None, 'retry_http_status_codes': 404 }) with pytest.raises(error.HttpError) as e: http_executor.request("GET", "mock://test.com") assert_that(e.value.status_code, is_(404)) assert_that(self.adapter.called, is_(True)) assert_that(self.adapter.called_once, is_(False))
def _initialize_registry(self, adapters, backoff_strategy): auth = AuthFactory.create(auth_type=AuthType.NO_AUTH) auth.register_adapters(adapters) http_executor = http.HttpExecutor( auth, proxies=self._proxies, user_agent=self._user_agent, session=auth.session, request_timeout=self._request_timeout, correlation_id=self._correlation_id, backoff_strategy=backoff_strategy) return registry.Registry(self._registry_url, http_executor)
def test_request_given_server_returns_an_token_expired_error_ever_then_the_request_should_fail( self): """ Testing the max. number of retries when requesting new token and getting 401. """ json_response = '{"resp2": "resp2"}' mock_response_401 = Mock() mock_response_401.is_redirect = False mock_response_401.status_code = 401 mock_response_401.json.return_value = { "statusCode": 401, "error": "Unauthorized", "message": "Invalid token", "attributes": { "error": "Invalid token" } } mock_response_401.return_value.text = '{"statusCode":401,"error":"Unauthorized","message":"Invalid token","attributes":{"error":"Invalid token"}}' mock_auth = Mock() mock_session = Mock() mock_session.request.return_value = mock_response_401 http_executor = http.HttpExecutor(mock_auth, session=mock_session, backoff_strategy={ 'interval': 0, 'max_tries': 2 }) with pytest.raises(error.HttpError): http_executor.request("GET", "mock://test.com") single_call = call('GET', 'mock://test.com', allow_redirects=False, data=None, headers={ 'User-Agent': mock.ANY, 'Content-Type': 'application/vnd.piksel+json', 'Accept': 'application/vnd.piksel+json', 'X-Correlation-ID': None }, params=None, timeout=240) call_list = [single_call, single_call, single_call, single_call] mock_session.request.assert_has_calls(call_list)
def test_request_given_a_resource_name_for_a_request_then_it_should_be_returned_with_the_request_result( self): self.adapter.register_uri('GET', 'mock://test.com', status_code=200) http_executor = http.HttpExecutor(auth.AuthFactory.create( grant_client_id="client_id", grant_client_secret="client_secret"), session=self.session_mock) resource_name_expected = 'resource_name_test' response = http_executor.request("GET", "mock://test.com", resource_name=resource_name_expected) assert_that(response.resource_name, equal_to(resource_name_expected))
def test_request_given_server_returns_an_error_then_the_request_should_be_retried_configured_times_by_default( self): json_response = '{"resp2": "resp2"}' self.adapter.register_uri('GET', 'mock://test.com', [{ 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': json_response, 'status_code': 200 }]) http_executor = http.HttpExecutor(auth.AuthFactory.create( grant_client_id="client_id", grant_client_secret="client_secret"), session=self.session_mock, backoff_strategy={ 'interval': 0, 'max_tries': 11 }) response = http_executor.request("GET", "mock://test.com") assert_that(response.data, equal_to(json.loads(json_response)))
def test_request_given_get_method_and_server_returns_an_error_code_then_that_error_should_be_populated( self): self.adapter.register_uri('GET', 'mock://test.com', text='some json value', status_code=403) http_executor = http.HttpExecutor(auth.AuthFactory.create( grant_client_id="client_id", grant_client_secret="client_secret"), session=self.session_mock) with pytest.raises(error.HttpError) as sequoia_error: http_executor.request("GET", "mock://test.com") assert_that(sequoia_error.value.status_code, 403) assert_that(sequoia_error.value.message, 'some json value') assert_that(sequoia_error.value.cause, none())
def test_request_given_server_returns_an_error_then_the_request_should_be_retried_10_times_by_default( self): json_response = '{"resp2": "resp2"}' self.adapter.register_uri('GET', 'mock://test.com', [{ 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': 'resp1', 'status_code': 500 }, { 'text': json_response, 'status_code': 200 }]) http_executor = http.HttpExecutor(auth.AuthFactory.create( grant_client_id="client_id", grant_client_secret="client_secret"), session=self.session_mock) with pytest.raises(error.HttpError) as sequoia_error: http_executor.request("GET", "mock://test.com") assert_that(sequoia_error.value.status_code, is_(500))
def test_request_given_server_returns_an_error_then_the_request_should_be_retried( self): json_response = {"resp2": "resp2"} self.adapter.register_uri('GET', 'mock://test.com', [{ 'text': 'resp1', 'status_code': 500 }, { 'json': json_response, 'status_code': 200 }]) http_executor = http.HttpExecutor(auth.AuthFactory.create( grant_client_id="client_id", grant_client_secret="client_secret"), session=self.session_mock) response = http_executor.request("GET", "mock://test.com") assert_that(response.data, equal_to(json_response))
def _initialize_registry(self, adapters, backoff_strategy, content_type, **auth_kwargs): auth = AuthFactory.create(**auth_kwargs) if auth_kwargs.get("auth_type", None) == AuthType.MUTUAL else AuthFactory.create( auth_type=AuthType.NO_AUTH) auth.register_adapters(adapters) http_executor = http.HttpExecutor(auth, proxies=self._proxies, user_agent=self._user_agent, session=auth.session, request_timeout=self._request_timeout, correlation_id=self._correlation_id, user_id=self.user_id, application_id=self.application_id, backoff_strategy=backoff_strategy, content_type=content_type) return registry.Registry(self._registry_url, http_executor)
def test_request_given_server_returns_too_many_redirects_then_error_should_be_raised( self): self.adapter.register_uri( 'GET', 'mock://some_url', exc=requests.exceptions.TooManyRedirects('some error desc')) http_executor = http.HttpExecutor(auth.AuthFactory.create( grant_client_id="client_id", grant_client_secret="client_secret"), session=self.session_mock) with pytest.raises(error.TooManyRedirects) as sequoia_error: http_executor.request("GET", "mock://some_url") assert_that('some error desc', is_in(sequoia_error.value.args)) assert_that(sequoia_error.value.cause, instance_of(requests.exceptions.TooManyRedirects))
def test_request_given_get_method_and_an_unreachable_url_then_a_connectivity_error_should_be_raised( self): self.adapter.register_uri( 'GET', 'mock://some_url', exc=requests.exceptions.ConnectionError('some error desc')) http_executor = http.HttpExecutor(auth.AuthFactory.create( grant_client_id="client_id", grant_client_secret="client_secret"), session=self.session_mock) with pytest.raises(error.ConnectionError) as sequoia_error: http_executor.request("GET", "mock://some_url") assert_that('some error desc', is_in(sequoia_error.value.args)) assert_that(sequoia_error.value.cause, instance_of(requests.exceptions.ConnectionError))
def test_request_given_additional_headers_and_data_then_they_are_added_to_the_request( self): self.adapter.register_uri( 'POST', 'mock://some_url', text='{"key_1": "value_1"}', request_headers={'New-Header': 'SomeValue'}, additional_matcher=HttpExecutorTest.match_request_text) http_executor = http.HttpExecutor(auth.AuthFactory.create( grant_client_id="client_id", grant_client_secret="client_secret"), session=self.session_mock) response = http_executor.request("POST", "mock://some_url", headers={'New-Header': 'SomeValue'}, data='some data') assert_that(response.data, equal_to({"key_1": "value_1"}))
def test_request_given_server_returns_an_authorisation_error_then_fetch_token_does_not_count_as_retry( self): json_response = 'Error getting resource' mock_auth = Mock() mock_session = Mock() mock_session.request.side_effect = [ error.AuthorisationError('Auth error'), { 'text': json_response, 'status_code': 500 }, { 'text': json_response, 'status_code': 200 } ] http_executor = http.HttpExecutor(mock_auth, session=mock_session, backoff_strategy={ 'interval': 0, 'max_tries': 1 }) with pytest.raises(error.AuthorisationError) as e: http_executor.request("GET", "mock://test.com") assert_that(e.value.args[0], is_('Auth error')) mock_session.request.assert_called_once_with( 'GET', 'mock://test.com', allow_redirects=False, data=None, headers={ 'User-Agent': mock.ANY, 'Content-Type': 'application/vnd.piksel+json', 'Accept': 'application/vnd.piksel+json', 'X-Correlation-ID': None }, params=None, timeout=240)
def test_request_given_server_returns_an_token_expired_error_then_the_request_should_be_retried( self): """ Testing the use of an invalid token and how the client-sdk should automatically get a new token. There are two type of errors when a new token is automatically retrieved: getting the TokenExpiredError exception and getting a valid response from the service with a 401 and using the auth method of providing credentials. This unit test checks both types: the exception and the 401 status error. """ json_response = '{"resp2": "resp2"}' mock_response_401 = Mock() mock_response_401.is_redirect = False mock_response_401.status_code = 401 mock_response_401.json.return_value = { "statusCode": 401, "error": "Unauthorized", "message": "Invalid token", "attributes": { "error": "Invalid token" } } mock_response_401.return_value.text = '{"statusCode":401,"error":"Unauthorized","message":"Invalid token","attributes":{"error":"Invalid token"}}' mock_response_200 = Mock() mock_response_200.is_redirect = False mock_response_200.status_code = 200 mock_response_200.return_value.text = json_response mock_response_200.json.return_value = {"resp2": "resp2"} mock_auth = Mock() mock_session = Mock() mock_session.request.side_effect = [ error.TokenExpiredError('Token Expired'), mock_response_401, mock_response_200 ] http_executor = http.HttpExecutor(mock_auth, session=mock_session, backoff_strategy={ 'interval': 0, 'max_tries': 2 }) response = http_executor.request("GET", "mock://test.com") assert_that(response.data, equal_to(json.loads(json_response))) call_list = [ call('GET', 'mock://test.com', allow_redirects=False, data=None, headers={ 'User-Agent': mock.ANY, 'Content-Type': 'application/vnd.piksel+json', 'Accept': 'application/vnd.piksel+json', 'X-Correlation-ID': None }, params=None, timeout=240), call('GET', 'mock://test.com', allow_redirects=False, data=None, headers={ 'User-Agent': mock.ANY, 'Content-Type': 'application/vnd.piksel+json', 'Accept': 'application/vnd.piksel+json', 'X-Correlation-ID': None }, params=None, timeout=240), call('GET', 'mock://test.com', allow_redirects=False, data=None, headers={ 'User-Agent': mock.ANY, 'Content-Type': 'application/vnd.piksel+json', 'Accept': 'application/vnd.piksel+json', 'X-Correlation-ID': None }, params=None, timeout=240) ] assert_that(mock_session.request.call_count, is_(3)) mock_session.request.assert_has_calls(call_list) assert_that(mock_session.auth.update_token.call_count, is_(2))
def test_request_given_server_returns_an_error_then_the_request_should_be_retried( self): json_response = '{"resp2": "resp2"}' mock_response_500 = Mock() mock_response_500.is_redirect = False mock_response_500.status_code = 500 mock_response_500.json.return_value = {} mock_response_500.return_value.text = json_response mock_response_200 = Mock() mock_response_200.is_redirect = False mock_response_200.status_code = 200 mock_response_200.return_value.text = json_response mock_response_200.json.return_value = {"resp2": "resp2"} mock_auth = Mock() mock_session = Mock() mock_session.request.side_effect = [ error.TokenExpiredError('Token Expired'), mock_response_500, mock_response_200 ] http_executor = http.HttpExecutor(mock_auth, session=mock_session, backoff_strategy={ 'interval': 0, 'max_tries': 2 }) response = http_executor.request("GET", "mock://test.com") assert_that(response.data, equal_to(json.loads(json_response))) call_list = [ call('GET', 'mock://test.com', allow_redirects=False, data=None, headers={ 'User-Agent': mock.ANY, 'Content-Type': 'application/vnd.piksel+json', 'Accept': 'application/vnd.piksel+json', 'X-Correlation-ID': None }, params=None, timeout=240), call('GET', 'mock://test.com', allow_redirects=False, data=None, headers={ 'User-Agent': mock.ANY, 'Content-Type': 'application/vnd.piksel+json', 'Accept': 'application/vnd.piksel+json', 'X-Correlation-ID': None }, params=None, timeout=240), call('GET', 'mock://test.com', allow_redirects=False, data=None, headers={ 'User-Agent': mock.ANY, 'Content-Type': 'application/vnd.piksel+json', 'Accept': 'application/vnd.piksel+json', 'X-Correlation-ID': None }, params=None, timeout=240) ] assert_that(mock_session.request.call_count, is_(3)) mock_session.request.assert_has_calls(call_list) assert_that(mock_session.auth.update_token.call_count, is_(1))
def test_given_a_content_type_property_then_header_should_contain_that_content_type( self): http_executor = http.HttpExecutor(None, content_type='abc') assert_that(http_executor.common_headers['Content-Type'], is_('abc')) assert_that(http_executor.common_headers['Accept'], is_('abc'))