def test_retries_on_429_status_code_initial_access_token(self):
"""
Ensures that in the event of a 429 HTTP Status Code being returned when communicating with an identity
provider, the request is retried.
"""
refreshing_token = RefreshingToken(api_configuration=self.config)
with patch("requests.post") as identity_mock:
identity_mock.side_effect = [
# Return a 429 on the first attempt
MockApiResponse(json_data={
"error": "rate_limit",
"error_description": "API rate limit exceeded."
},
status_code=429),
# Return a 200 on the second attempt
MockApiResponse(json_data={
"access_token": "mock_access_token",
"refresh_token": "mock_refresh_token",
"expires_in": 60
},
status_code=200),
]
# Ensure that we were able to get the token, if not retrying this would be impossible
self.assertEqual(f"{refreshing_token}", "mock_access_token")
self.assertEqual(identity_mock.call_count, 2)
def test_token_when_refresh_token_expired_still_refreshes(self):
refreshed_token = RefreshingToken(api_configuration=self.config, expiry_offset=3599)
self.assertIsNotNone(refreshed_token)
# force de-referencing the token value
first_value = f"{refreshed_token}"
sleep(1)
with patch("requests.post", side_effect=[
MockApiResponse(
json_data={
"error": "invalid_grant",
"error_description": "The refresh token is invalid or expired."
},
status_code=400
),
MockApiResponse(
json_data={
"access_token": "mock_access_token",
"refresh_token": "mock_refresh_token",
"expires_in": 60
},
status_code=200
),
]):
self.assertNotEqual(first_value, refreshed_token)
def test_can_use_id_provider_handler_to_provide_retry_after_header_from_custom_header(
self):
"""
Ensures that the "Retry-After" header can be used after being created from a custom header using the
id_provider_response_handler.
"""
time_to_wait = 5
def header_creator(id_provider_response):
rate_limit_reset = id_provider_response.headers.get(
"X-Rate-Limit-Reset", None)
if rate_limit_reset is not None:
id_provider_response.headers["Retry-After"] = max(
int(rate_limit_reset - datetime.utcnow().timestamp()), 0)
refreshing_token = RefreshingToken(
api_configuration=self.config,
id_provider_response_handler=header_creator)
with patch(
"requests.post",
side_effect=[
# Return a 429 on the first attempt
MockApiResponse(json_data={
"error":
"rate_limit",
"error_description":
"API rate limit exceeded."
},
status_code=429,
headers={
"X-Rate-Limit-Reset":
datetime.utcnow().timestamp() +
time_to_wait
}),
# Return a 200 on the second attempt
MockApiResponse(json_data={
"access_token": "mock_access_token",
"refresh_token": "mock_refresh_token",
"expires_in": 60
},
status_code=200),
]):
start = time()
# Ensure that we were able to get the token, if not retrying this would be impossible
self.assertEqual(f"{refreshing_token}", "mock_access_token")
elapsed = time() - start
# Ensure that the wait was for an appropriate amount of time, because the seconds to wait are calculated
# here instead of being provided directly the delay could be a second less
self.assertGreaterEqual(int(elapsed), time_to_wait - 1)
self.assertLessEqual(int(elapsed), time_to_wait)
def test_use_refresh_token_multiple_threads(self):
def force_refresh(refresh_token):
return f"{refresh_token}"
refreshed_token = RefreshingToken(api_configuration=self.config)
thread1 = Thread(target=force_refresh, args=[refreshed_token])
thread2 = Thread(target=force_refresh, args=[refreshed_token])
thread3 = Thread(target=force_refresh, args=[refreshed_token])
with patch("requests.post") as identity_mock:
identity_mock.side_effect = lambda *args, **kwargs: MockApiResponse(
json_data={
"access_token": "mock_access_token",
"refresh_token": "mock_refresh_token",
"expires_in": 3600
},
status_code=200
)
thread1.start()
thread2.start()
thread3.start()
thread1.join()
thread2.join()
thread3.join()
# Ensure that we only got an access token once
self.assertEqual(1, identity_mock.call_count)
def test_retries_on_429s_up_till_retry_limit(self):
"""
Ensures that the refreshing token only retries up until the retry limit to prevent
an infinite retry loop
"""
refreshing_token = RefreshingToken(api_configuration=self.config)
refreshing_token.retry_limit = 2 # Override default to ensure test runs in reasonable amount of time
expected_requests = 1 + refreshing_token.retry_limit # Initial request plus expected number retries
with patch("requests.post") as identity_mock:
identity_mock.side_effect = [
MockApiResponse(json_data={
"error": "rate_limit",
"error_description": "API rate limit exceeded."
},
status_code=429,
headers={})
] * expected_requests # Return a 429 every time up until expected number of attempts
# Ensure that a an error is raised once reaching the retry limit
with self.assertRaises(ValueError) as retry_limit_error:
self.force_refresh(refreshing_token)
self.assertIn("Max retry limit", str(retry_limit_error.exception))
# Ensure that we only tried as many times as expected
self.assertEqual(expected_requests, identity_mock.call_count)
def test_does_not_retry_on_4xx_status_code_other_than_429(self):
"""
Ensures that we do not retry on other common 4xx status codes such as 400 - Bad Request
"""
refreshing_token = RefreshingToken(api_configuration=self.config)
with patch("requests.post") as identity_mock:
identity_mock.side_effect = [
# Return a 400
MockApiResponse(json_data={
"error":
"invalid_grant",
"error_description":
"The refresh token is invalid or expired."
},
status_code=400),
]
# Ensure that a 400 is raised as an error and not retried
with self.assertRaises(ValueError) as bad_request_exception:
self.force_refresh(refreshing_token)
self.assertEqual(identity_mock.call_count, 1) # No retrying
self.assertIn("invalid_grant",
str(bad_request_exception.exception))
def test_retries_on_429s_uses_retry_after_header_with_http_date_in_future_if_exists(
self):
"""
Ensures that if the HTTP Date returned on the "Retry-After" header is x seconds in the future
it takes approximately x seconds to retry and get the token.
"""
time_to_wait = 5
refreshing_token = RefreshingToken(api_configuration=self.config)
with patch(
"requests.post",
side_effect=[
# Return a 429 on the first attempt
MockApiResponse(json_data={
"error":
"rate_limit",
"error_description":
"API rate limit exceeded."
},
status_code=429,
headers={
"Retry-After":
self.convert_to_http_date(
datetime.utcnow() +
timedelta(seconds=time_to_wait))
}),
# Return a 200 on the second attempt
MockApiResponse(json_data={
"access_token": "mock_access_token",
"refresh_token": "mock_refresh_token",
"expires_in": 60
},
status_code=200),
]):
start = time()
# Ensure that we were able to get the token, if not retrying this would be impossible
self.assertEqual(f"{refreshing_token}", "mock_access_token")
elapsed = time() - start
# Ensure that the wait was for an appropriate amount of time, because the seconds to wait are calculated
# here instead of being provided directly the delay could be a second less
self.assertGreaterEqual(int(elapsed), time_to_wait - 1)
self.assertLessEqual(int(elapsed), time_to_wait)
def test_retries_on_429_status_code_using_refresh_token(self):
"""
Ensures that in the event of a 429 HTTP Status Code being returned when communicating with an identity
provider, the request is retried.
"""
refreshing_token = RefreshingToken(api_configuration=self.config)
with patch("requests.post") as identity_mock:
identity_mock.side_effect = [
# Get initial access token
MockApiResponse(
json_data={
"access_token": "mock_access_token",
"refresh_token": "mock_refresh_token",
"expires_in": 1 # Expires almost immediately
},
status_code=200),
# Return a 429 on the second attempt
MockApiResponse(json_data={
"error": "rate_limit",
"error_description": "API rate limit exceeded."
},
status_code=429),
# Return a 200 on the third attempt
MockApiResponse(json_data={
"access_token": "mock_access_token_2",
"refresh_token": "mock_refresh_token",
"expires_in": 60
},
status_code=200),
]
# Ensure that we were able to get the first access token
self.assertEqual(f"{refreshing_token}", "mock_access_token")
sleep(1) # Wait for initial token to expire
# Try and get access token again forcing refresh, if we can get it then retry was called
self.assertEqual(f"{refreshing_token}", "mock_access_token_2")
self.assertEqual(identity_mock.call_count, 3)
def test_retries_on_429s_uses_retry_after_header_with_http_date_in_past_if_exists(
self):
"""
Ensures that if the HTTP Date returned on the "Retry-After" header is x seconds in the past
an retry attempt to get the token is made immediately
"""
refreshing_token = RefreshingToken(api_configuration=self.config)
with patch(
"requests.post",
side_effect=[
# Return a 429 on the first attempt
MockApiResponse(
json_data={
"error": "rate_limit",
"error_description": "API rate limit exceeded."
},
status_code=429,
headers={
"Retry-After":
self.convert_to_http_date(datetime.utcnow() -
timedelta(seconds=5))
}),
# Return a 200 on the second attempt
MockApiResponse(json_data={
"access_token": "mock_access_token",
"refresh_token": "mock_refresh_token",
"expires_in": 60
},
status_code=200),
]):
start = time()
# Ensure that we were able to get the token, if not retrying this would be impossible
self.assertEqual(f"{refreshing_token}", "mock_access_token")
elapsed = time() - start
# Ensure that the wait was essentially no wait before retrying
self.assertLess(elapsed, 1)
def test_retries_on_429s_uses_exponential_back_off_if_no_retry_after_header(
self, _, number_attempts_till_success, expected_delay):
"""
Ensures that if no "Retry-After" header is provided then a simple exponential back-off strategy is used. This
is confirmed by checking that the time taken to successfully retrieve a token scales exponentially as the number
of retries increases.
"""
refreshing_token = RefreshingToken(api_configuration=self.config)
refreshing_token.backoff_base = 2 # Use a 2 second base for calculating back-off
with patch(
"requests.post",
side_effect=[
# Return a 429 on the first attempts
MockApiResponse(
json_data={
"error": "rate_limit",
"error_description": "API rate limit exceeded."
},
status_code=429,
)
] * number_attempts_till_success +
# Return a 200 on the last attempt
[
MockApiResponse(json_data={
"access_token": "mock_access_token",
"refresh_token": "mock_refresh_token",
"expires_in": 60
},
status_code=200)
]):
start = time()
# Ensure that we were able to get the token, if not retrying this would be impossible
self.assertEqual(f"{refreshing_token}", "mock_access_token")
elapsed = time() - start
# Ensure that the elapsed time is as expected
self.assertEqual(int(elapsed), expected_delay)
def test_retries_on_429s_uses_retry_after_header_with_seconds_delay_if_exists(
self, _, seconds_delay):
"""
Ensures that if a seconds delay is contained in the "Retry-After" header then a retry is attempted after
the appropriate amount of time.
:param _: The name of the tests
:param seconds_delay: The number of seconds to wait before retrying
"""
refreshing_token = RefreshingToken(api_configuration=self.config)
with patch(
"requests.post",
side_effect=[
# Return a 429 on the first attempt
MockApiResponse(
json_data={
"error": "rate_limit",
"error_description": "API rate limit exceeded."
},
status_code=429,
headers={"Retry-After": str(seconds_delay)}),
# Return a 200 on the second attempt
MockApiResponse(json_data={
"access_token": "mock_access_token",
"refresh_token": "mock_refresh_token",
"expires_in": 60
},
status_code=200),
]):
start = time()
# Ensure that we were able to get the token, if not retrying this would be impossible
self.assertEqual(f"{refreshing_token}", "mock_access_token")
elapsed = time() - start
# Ensure that the wait was for an appropriate amount of time
self.assertEqual(int(elapsed), seconds_delay)
def test_use_apifactory_multiple_threads(self):
with patch.dict('os.environ', self.get_env_vars_without_pat(), clear=True):
access_token = str(ApiClientFactory(
api_secrets_filename=CredentialsSource.secrets_path()
).api_client.configuration.access_token)
api_factory = ApiClientFactory(
api_secrets_filename=CredentialsSource.secrets_path()
)
def get_identifier_types(factory):
return factory.build(InstrumentsApi).get_instrument_identifier_types()
thread1 = Thread(target=get_identifier_types, args=[api_factory])
thread2 = Thread(target=get_identifier_types, args=[api_factory])
thread3 = Thread(target=get_identifier_types, args=[api_factory])
with patch("requests.post") as identity_mock:
identity_mock.side_effect = lambda *args, **kwargs: MockApiResponse(
json_data={
"access_token": f"{access_token}",
"refresh_token": "mock_refresh_token",
"expires_in": 3600
},
status_code=200
)
thread1.start()
thread2.start()
thread3.start()
thread1.join()
thread2.join()
thread3.join()
# Ensure that we only got an access token once
self.assertEqual(1, identity_mock.call_count)