def retry_when_empty_result_test(self, mock_http_response_list, max_tries,
retry_when_empty_result,
expected_http_status_code,
expected_json_response,
expected_requests_number):
self.adapter.register_uri(
method='GET',
url=
'mock://metadata.pikselpalette.com/data/contents?include=assets,categories&owner=test&withRef=test:c0007',
response_list=mock_http_response_list)
http_executor = http.HttpExecutor(auth.AuthFactory.create(
auth_type=auth.AuthType.BYO_TOKEN, byo_token='tkn'),
session=self.session_mock,
user_agent='backoff_test',
backoff_strategy={
'max_tries': max_tries,
'interval': 0
})
actual_response = http_executor.request(
method="GET",
url=
"mock://metadata.pikselpalette.com/data/contents?include=assets,categories&owner=test&withRef=test:c0007",
resource_name='contents',
retry_when_empty_result=retry_when_empty_result)
assert_that(actual_response.status, is_(expected_http_status_code))
assert_that(actual_response.data, equal_to(expected_json_response))
assert_that(self.adapter.call_count, is_(expected_requests_number))
for i in range(expected_requests_number):
assert_that(self.adapter.request_history[i].method, is_('GET'))
assert_that(
self.adapter.request_history[i].url,
is_('mock://metadata.pikselpalette.com/data/contents?include=assets,categories&owner=test&withRef=test:c0007'
))
def test_request_given_server_returns_an_authorisation_error_then_fetch_token_does_not_count_as_retry(
self, mock_fetch_token):
mock_fetch_token.return_value = "validToken"
json_response = 'Error getting resource'
self.adapter.register_uri('GET', 'mock://test.com', [{
'text': 'resp1',
'status_code': 401
}, {
'text': json_response,
'status_code': 500
}, {
'text': json_response,
'status_code': 200
}])
http_executor = http.HttpExecutor(auth.AuthFactory.create(
grant_client_id="client_id",
grant_client_secret="client_secret",
token_url='mock://token-url.com'),
session=self.session_mock,
backoff_strategy={
'interval': 0,
'max_tries': 1
})
with pytest.raises(error.HttpError) as e:
http_executor.request("GET", "mock://test.com")
assert_that(
e.value.args[0],
is_('An unexpected error occurred. HTTP Status code: 500. Error message: '
'Expecting value: line 1 column 1 (char 0). '))
def test_request_given_byo_type_and_server_returns_an_authorisation_error_then_error_is_propagated(
self):
json_response = '{"resp2": "resp2"}'
self.adapter.register_uri('GET', 'mock://test.com', [{
'text': 'resp1',
'status_code': 401
}, {
'text': json_response,
'status_code': 200
}])
http_executor = http.HttpExecutor(auth.AuthFactory.create(
auth_type=auth.AuthType.BYO_TOKEN, byo_token='asdf'),
session=self.session_mock,
backoff_strategy={
'interval': 0,
'max_tries': 1
})
with pytest.raises(error.HttpError) as e:
http_executor.request("GET", "mock://test.com")
assert_that(
e.value.args[0],
is_('An unexpected error occurred. HTTP Status code: 401.'
' Error message: Expecting value: line 1 column 1 (char 0). '))
def test_request_given_server_returns_an_authorisation_error_then_the_request_should_be_retried(
self, mock_fetch_token):
mock_fetch_token.return_value = "validToken"
json_response = '{"resp2": "resp2"}'
self.adapter.register_uri('GET', 'mock://test.com', [{
'text': 'resp1',
'status_code': 401
}, {
'text': json_response,
'status_code': 200
}])
http_executor = http.HttpExecutor(auth.AuthFactory.create(
grant_client_id="client_id",
grant_client_secret="client_secret",
token_url='mock://token-url.com'),
session=self.session_mock,
backoff_strategy={
'interval': 0,
'max_tries': 1
})
response = http_executor.request("GET", "mock://test.com")
assert_that(response.data, equal_to(json.loads(json_response)))
def test_request_given_server_returns_an_authorisation_error_fetching_the_token_then_error_is_not_retried(
self):
mock_auth = Mock()
mock_session = Mock()
mock_session.request.side_effect = error.AuthorisationError(
'Auth error')
http_executor = http.HttpExecutor(mock_auth,
session=mock_session,
backoff_strategy={
'interval': 0,
'max_tries': 3
})
with pytest.raises(error.AuthorisationError):
http_executor.request("GET", "mock://test.com")
mock_session.request.assert_called_once_with(
'GET',
'mock://test.com',
allow_redirects=False,
data=None,
headers={
'User-Agent': mock.ANY,
'Content-Type': 'application/vnd.piksel+json',
'Accept': 'application/vnd.piksel+json',
'X-Correlation-ID': None
},
params=None,
timeout=240)
def test_given_none_content_type_property_then_header_should_contain_none_content_type(
self):
http_executor = http.HttpExecutor(None, content_type=None)
assert_that(http_executor.common_headers['Content-Type'],
is_('application/vnd.piksel+json'))
assert_that(http_executor.common_headers['Accept'],
is_('application/vnd.piksel+json'))
def test_request_given_server_returns_an_authorisation_error_fetching_the_token_then_error_is_not_retried(
self, mock_fetch_token):
mock_fetch_token.side_effect = OAuth2Error()
json_response = '{"resp2": "resp2"}'
self.adapter.register_uri('GET', 'mock://test.com', [{
'text': 'resp1',
'status_code': 401
}, {
'text': json_response,
'status_code': 200
}])
http_executor = http.HttpExecutor(auth.AuthFactory.create(
grant_client_id="client_id",
grant_client_secret="client_secret",
token_url='mock://token-url.com'),
session=self.session_mock,
backoff_strategy={
'interval': 0,
'max_tries': 1
})
with pytest.raises(error.AuthorisationError):
http_executor.request("GET", "mock://test.com")
def test_request_given_a_list_of_parameters_then_they_are_added_to_the_request(
self, session_mock):
# There is an issue where parameters won't be added to the request if the prefix does not start
# with http https://bugs.launchpad.net/requests-mock/+bug/1518497. So request-mock can't be used here
# to check parameters
session_mock.request.return_value.url = 'mock://some_url'
session_mock.request.return_value.status_code = 200
session_mock.request.return_value.is_redirect = False
http_executor = http.HttpExecutor(auth.AuthFactory.create(
grant_client_id="client_id", grant_client_secret="client_secret"),
session=session_mock,
correlation_id="my_correlation_id")
http_executor.request("POST",
"mock://some_url",
data='some data',
headers={'New-Header': 'SomeValue'},
params={'key1': 'value1'})
expected_headers = {
'User-Agent': http_executor.user_agent,
"Content-Type": "application/vnd.piksel+json",
"Accept": "application/vnd.piksel+json",
"X-Correlation-ID": "my_correlation_id",
"New-Header": 'SomeValue'
}
session_mock.request.assert_called_with('POST',
'mock://some_url',
allow_redirects=False,
data='some data',
headers=expected_headers,
params={'key1': 'value1'},
timeout=240)
def __init__(self, registry_url, proxies=None, user_agent=None, backoff_strategy=None, adapters=None,
request_timeout=None, model_resolution=None, correlation_id=None, user_id=None, application_id=None,
content_type=None, **auth_kwargs):
logging.debug('Client initialising with registry_url=%s ', registry_url)
self._registry_url = registry_url
self._request_timeout = request_timeout or env.DEFAULT_REQUEST_TIMEOUT_SECONDS
self._proxies = proxies
self._user_agent = user_agent
self._correlation_id = correlation_id.strip() if correlation_id else None
self.user_id = user_id.strip() if user_id else None
self.application_id = application_id.strip() if application_id else None
self._model_resolution = model_resolution
self._registry = self._initialize_registry(adapters, backoff_strategy, content_type, **auth_kwargs)
self._auth = AuthFactory.create(token_url=self._get_token_url(auth_kwargs.get("auth_type", None)),
request_timeout=self._request_timeout,
**auth_kwargs)
self._auth.register_adapters(adapters)
self._auth.init_session()
self._http = http.HttpExecutor(self._auth,
proxies=self._proxies,
user_agent=self._user_agent,
session=self._auth.session,
request_timeout=self._request_timeout,
correlation_id=self._correlation_id,
user_id=self.user_id,
application_id=self.application_id,
backoff_strategy=backoff_strategy,
content_type=content_type)
def backoff_test(self, mock_http_response_list, max_tries,
retry_http_codes, expected_http_status_code,
expected_json_response, expected_requests_number):
self.adapter.register_uri(method='GET',
url='mock://test.com',
response_list=mock_http_response_list)
http_executor = http.HttpExecutor(auth.AuthFactory.create(
auth_type=auth.AuthType.BYO_TOKEN, byo_token='tkn'),
session=self.session_mock,
user_agent='backoff_test',
backoff_strategy={
'max_tries':
max_tries,
'max_time':
300,
'interval':
0,
'retry_http_status_codes':
retry_http_codes
})
actual_response = http_executor.request("GET", "mock://test.com")
assert_that(actual_response.status, is_(expected_http_status_code))
assert_that(actual_response.data, equal_to(expected_json_response))
assert_that(self.adapter.call_count, is_(expected_requests_number))
for i in range(expected_requests_number): assert_that(self.adapter.request_history[i].method, is_('GET')) and \
assert_that(self.adapter.request_history[i].url, is_('mock://test.com'))
def test_retries_for_http_status_code_specified_in_backoff_strategy_reach_default_limit(
self):
def _patch_max_time_to_run_unit_test_faster():
import sequoia
sequoia.http.HttpExecutor.MAX_TIME_SECONDS = 0.5
json_response_404 = {
'statusCode': 404,
'error': 'Not Found',
'message': 'Not Found'
}
mock_http_response_list = [{
'json': json_response_404,
'status_code': 404
}]
_patch_max_time_to_run_unit_test_faster()
self.adapter.register_uri(method='GET',
url='mock://test.com',
response_list=mock_http_response_list)
http_executor = http.HttpExecutor(auth.AuthFactory.create(
auth_type=auth.AuthType.BYO_TOKEN, byo_token='tkn'),
session=self.session_mock,
backoff_strategy={
'max_tries': None,
'max_time': None,
'retry_http_status_codes': 404
})
with pytest.raises(error.HttpError) as e:
http_executor.request("GET", "mock://test.com")
assert_that(e.value.status_code, is_(404))
assert_that(self.adapter.called, is_(True))
assert_that(self.adapter.called_once, is_(False))
def _initialize_registry(self, adapters, backoff_strategy):
auth = AuthFactory.create(auth_type=AuthType.NO_AUTH)
auth.register_adapters(adapters)
http_executor = http.HttpExecutor(
auth,
proxies=self._proxies,
user_agent=self._user_agent,
session=auth.session,
request_timeout=self._request_timeout,
correlation_id=self._correlation_id,
backoff_strategy=backoff_strategy)
return registry.Registry(self._registry_url, http_executor)
def test_request_given_server_returns_an_token_expired_error_ever_then_the_request_should_fail(
self):
"""
Testing the max. number of retries when requesting new token and getting 401.
"""
json_response = '{"resp2": "resp2"}'
mock_response_401 = Mock()
mock_response_401.is_redirect = False
mock_response_401.status_code = 401
mock_response_401.json.return_value = {
"statusCode": 401,
"error": "Unauthorized",
"message": "Invalid token",
"attributes": {
"error": "Invalid token"
}
}
mock_response_401.return_value.text = '{"statusCode":401,"error":"Unauthorized","message":"Invalid token","attributes":{"error":"Invalid token"}}'
mock_auth = Mock()
mock_session = Mock()
mock_session.request.return_value = mock_response_401
http_executor = http.HttpExecutor(mock_auth,
session=mock_session,
backoff_strategy={
'interval': 0,
'max_tries': 2
})
with pytest.raises(error.HttpError):
http_executor.request("GET", "mock://test.com")
single_call = call('GET',
'mock://test.com',
allow_redirects=False,
data=None,
headers={
'User-Agent': mock.ANY,
'Content-Type': 'application/vnd.piksel+json',
'Accept': 'application/vnd.piksel+json',
'X-Correlation-ID': None
},
params=None,
timeout=240)
call_list = [single_call, single_call, single_call, single_call]
mock_session.request.assert_has_calls(call_list)
def test_request_given_a_resource_name_for_a_request_then_it_should_be_returned_with_the_request_result(
self):
self.adapter.register_uri('GET', 'mock://test.com', status_code=200)
http_executor = http.HttpExecutor(auth.AuthFactory.create(
grant_client_id="client_id", grant_client_secret="client_secret"),
session=self.session_mock)
resource_name_expected = 'resource_name_test'
response = http_executor.request("GET",
"mock://test.com",
resource_name=resource_name_expected)
assert_that(response.resource_name, equal_to(resource_name_expected))
def test_request_given_server_returns_an_error_then_the_request_should_be_retried_configured_times_by_default(
self):
json_response = '{"resp2": "resp2"}'
self.adapter.register_uri('GET', 'mock://test.com', [{
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': json_response,
'status_code': 200
}])
http_executor = http.HttpExecutor(auth.AuthFactory.create(
grant_client_id="client_id", grant_client_secret="client_secret"),
session=self.session_mock,
backoff_strategy={
'interval': 0,
'max_tries': 11
})
response = http_executor.request("GET", "mock://test.com")
assert_that(response.data, equal_to(json.loads(json_response)))
def test_request_given_get_method_and_server_returns_an_error_code_then_that_error_should_be_populated(
self):
self.adapter.register_uri('GET',
'mock://test.com',
text='some json value',
status_code=403)
http_executor = http.HttpExecutor(auth.AuthFactory.create(
grant_client_id="client_id", grant_client_secret="client_secret"),
session=self.session_mock)
with pytest.raises(error.HttpError) as sequoia_error:
http_executor.request("GET", "mock://test.com")
assert_that(sequoia_error.value.status_code, 403)
assert_that(sequoia_error.value.message, 'some json value')
assert_that(sequoia_error.value.cause, none())
def test_request_given_server_returns_an_error_then_the_request_should_be_retried_10_times_by_default(
self):
json_response = '{"resp2": "resp2"}'
self.adapter.register_uri('GET', 'mock://test.com', [{
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': 'resp1',
'status_code': 500
}, {
'text': json_response,
'status_code': 200
}])
http_executor = http.HttpExecutor(auth.AuthFactory.create(
grant_client_id="client_id", grant_client_secret="client_secret"),
session=self.session_mock)
with pytest.raises(error.HttpError) as sequoia_error:
http_executor.request("GET", "mock://test.com")
assert_that(sequoia_error.value.status_code, is_(500))
def test_request_given_server_returns_an_error_then_the_request_should_be_retried(
self):
json_response = {"resp2": "resp2"}
self.adapter.register_uri('GET', 'mock://test.com', [{
'text': 'resp1',
'status_code': 500
}, {
'json': json_response,
'status_code': 200
}])
http_executor = http.HttpExecutor(auth.AuthFactory.create(
grant_client_id="client_id", grant_client_secret="client_secret"),
session=self.session_mock)
response = http_executor.request("GET", "mock://test.com")
assert_that(response.data, equal_to(json_response))
def _initialize_registry(self, adapters, backoff_strategy, content_type, **auth_kwargs):
auth = AuthFactory.create(**auth_kwargs) if auth_kwargs.get("auth_type",
None) == AuthType.MUTUAL else AuthFactory.create(
auth_type=AuthType.NO_AUTH)
auth.register_adapters(adapters)
http_executor = http.HttpExecutor(auth,
proxies=self._proxies,
user_agent=self._user_agent,
session=auth.session,
request_timeout=self._request_timeout,
correlation_id=self._correlation_id,
user_id=self.user_id,
application_id=self.application_id,
backoff_strategy=backoff_strategy,
content_type=content_type)
return registry.Registry(self._registry_url, http_executor)
def test_request_given_server_returns_too_many_redirects_then_error_should_be_raised(
self):
self.adapter.register_uri(
'GET',
'mock://some_url',
exc=requests.exceptions.TooManyRedirects('some error desc'))
http_executor = http.HttpExecutor(auth.AuthFactory.create(
grant_client_id="client_id", grant_client_secret="client_secret"),
session=self.session_mock)
with pytest.raises(error.TooManyRedirects) as sequoia_error:
http_executor.request("GET", "mock://some_url")
assert_that('some error desc', is_in(sequoia_error.value.args))
assert_that(sequoia_error.value.cause,
instance_of(requests.exceptions.TooManyRedirects))
def test_request_given_get_method_and_an_unreachable_url_then_a_connectivity_error_should_be_raised(
self):
self.adapter.register_uri(
'GET',
'mock://some_url',
exc=requests.exceptions.ConnectionError('some error desc'))
http_executor = http.HttpExecutor(auth.AuthFactory.create(
grant_client_id="client_id", grant_client_secret="client_secret"),
session=self.session_mock)
with pytest.raises(error.ConnectionError) as sequoia_error:
http_executor.request("GET", "mock://some_url")
assert_that('some error desc', is_in(sequoia_error.value.args))
assert_that(sequoia_error.value.cause,
instance_of(requests.exceptions.ConnectionError))
def test_request_given_additional_headers_and_data_then_they_are_added_to_the_request(
self):
self.adapter.register_uri(
'POST',
'mock://some_url',
text='{"key_1": "value_1"}',
request_headers={'New-Header': 'SomeValue'},
additional_matcher=HttpExecutorTest.match_request_text)
http_executor = http.HttpExecutor(auth.AuthFactory.create(
grant_client_id="client_id", grant_client_secret="client_secret"),
session=self.session_mock)
response = http_executor.request("POST",
"mock://some_url",
headers={'New-Header': 'SomeValue'},
data='some data')
assert_that(response.data, equal_to({"key_1": "value_1"}))
def test_request_given_server_returns_an_authorisation_error_then_fetch_token_does_not_count_as_retry(
self):
json_response = 'Error getting resource'
mock_auth = Mock()
mock_session = Mock()
mock_session.request.side_effect = [
error.AuthorisationError('Auth error'), {
'text': json_response,
'status_code': 500
}, {
'text': json_response,
'status_code': 200
}
]
http_executor = http.HttpExecutor(mock_auth,
session=mock_session,
backoff_strategy={
'interval': 0,
'max_tries': 1
})
with pytest.raises(error.AuthorisationError) as e:
http_executor.request("GET", "mock://test.com")
assert_that(e.value.args[0], is_('Auth error'))
mock_session.request.assert_called_once_with(
'GET',
'mock://test.com',
allow_redirects=False,
data=None,
headers={
'User-Agent': mock.ANY,
'Content-Type': 'application/vnd.piksel+json',
'Accept': 'application/vnd.piksel+json',
'X-Correlation-ID': None
},
params=None,
timeout=240)
def test_request_given_server_returns_an_token_expired_error_then_the_request_should_be_retried(
self):
"""
Testing the use of an invalid token and how the client-sdk should automatically get a new token.
There are two type of errors when a new token is automatically retrieved: getting the TokenExpiredError exception
and getting a valid response from the service with a 401 and using the auth method of providing credentials.
This unit test checks both types: the exception and the 401 status error.
"""
json_response = '{"resp2": "resp2"}'
mock_response_401 = Mock()
mock_response_401.is_redirect = False
mock_response_401.status_code = 401
mock_response_401.json.return_value = {
"statusCode": 401,
"error": "Unauthorized",
"message": "Invalid token",
"attributes": {
"error": "Invalid token"
}
}
mock_response_401.return_value.text = '{"statusCode":401,"error":"Unauthorized","message":"Invalid token","attributes":{"error":"Invalid token"}}'
mock_response_200 = Mock()
mock_response_200.is_redirect = False
mock_response_200.status_code = 200
mock_response_200.return_value.text = json_response
mock_response_200.json.return_value = {"resp2": "resp2"}
mock_auth = Mock()
mock_session = Mock()
mock_session.request.side_effect = [
error.TokenExpiredError('Token Expired'), mock_response_401,
mock_response_200
]
http_executor = http.HttpExecutor(mock_auth,
session=mock_session,
backoff_strategy={
'interval': 0,
'max_tries': 2
})
response = http_executor.request("GET", "mock://test.com")
assert_that(response.data, equal_to(json.loads(json_response)))
call_list = [
call('GET',
'mock://test.com',
allow_redirects=False,
data=None,
headers={
'User-Agent': mock.ANY,
'Content-Type': 'application/vnd.piksel+json',
'Accept': 'application/vnd.piksel+json',
'X-Correlation-ID': None
},
params=None,
timeout=240),
call('GET',
'mock://test.com',
allow_redirects=False,
data=None,
headers={
'User-Agent': mock.ANY,
'Content-Type': 'application/vnd.piksel+json',
'Accept': 'application/vnd.piksel+json',
'X-Correlation-ID': None
},
params=None,
timeout=240),
call('GET',
'mock://test.com',
allow_redirects=False,
data=None,
headers={
'User-Agent': mock.ANY,
'Content-Type': 'application/vnd.piksel+json',
'Accept': 'application/vnd.piksel+json',
'X-Correlation-ID': None
},
params=None,
timeout=240)
]
assert_that(mock_session.request.call_count, is_(3))
mock_session.request.assert_has_calls(call_list)
assert_that(mock_session.auth.update_token.call_count, is_(2))
def test_request_given_server_returns_an_error_then_the_request_should_be_retried(
self):
json_response = '{"resp2": "resp2"}'
mock_response_500 = Mock()
mock_response_500.is_redirect = False
mock_response_500.status_code = 500
mock_response_500.json.return_value = {}
mock_response_500.return_value.text = json_response
mock_response_200 = Mock()
mock_response_200.is_redirect = False
mock_response_200.status_code = 200
mock_response_200.return_value.text = json_response
mock_response_200.json.return_value = {"resp2": "resp2"}
mock_auth = Mock()
mock_session = Mock()
mock_session.request.side_effect = [
error.TokenExpiredError('Token Expired'), mock_response_500,
mock_response_200
]
http_executor = http.HttpExecutor(mock_auth,
session=mock_session,
backoff_strategy={
'interval': 0,
'max_tries': 2
})
response = http_executor.request("GET", "mock://test.com")
assert_that(response.data, equal_to(json.loads(json_response)))
call_list = [
call('GET',
'mock://test.com',
allow_redirects=False,
data=None,
headers={
'User-Agent': mock.ANY,
'Content-Type': 'application/vnd.piksel+json',
'Accept': 'application/vnd.piksel+json',
'X-Correlation-ID': None
},
params=None,
timeout=240),
call('GET',
'mock://test.com',
allow_redirects=False,
data=None,
headers={
'User-Agent': mock.ANY,
'Content-Type': 'application/vnd.piksel+json',
'Accept': 'application/vnd.piksel+json',
'X-Correlation-ID': None
},
params=None,
timeout=240),
call('GET',
'mock://test.com',
allow_redirects=False,
data=None,
headers={
'User-Agent': mock.ANY,
'Content-Type': 'application/vnd.piksel+json',
'Accept': 'application/vnd.piksel+json',
'X-Correlation-ID': None
},
params=None,
timeout=240)
]
assert_that(mock_session.request.call_count, is_(3))
mock_session.request.assert_has_calls(call_list)
assert_that(mock_session.auth.update_token.call_count, is_(1))
def test_given_a_content_type_property_then_header_should_contain_that_content_type(
self):
http_executor = http.HttpExecutor(None, content_type='abc')
assert_that(http_executor.common_headers['Content-Type'], is_('abc'))
assert_that(http_executor.common_headers['Accept'], is_('abc'))
