def test_service_is_marked_as_unavailable(self):
"""
Verify that a function that is failing `failure_threshold` times is
marked as broken and skipped
"""
# create a list of service where the first function yields an exception
services = RemoteServiceList(expected_exception=CustomException)
func = generate_failing_func(exception=CustomException)
services.append(func)
services.append(lambda: 42)
# initially all services should be marked as functional
self.assertEqual(services[0].state, State.FUNCTIONAL)
self.assertEqual(services[1].state, State.FUNCTIONAL)
# after calling for `failure_threshold` times the failing service
# should be marked as UNAVAILABLE
for _ in range(0, services.failure_threshold):
self.assertEqual(services.call_first_available(), 42)
self.assertGreater(func.call_count, 0)
self.assertEqual(func.call_count, services.failure_threshold)
self.assertEqual(services[0].state, State.UNAVAILABLE)
# the second function must still be FUNCTIONAL
self.assertEqual(services[1].state, State.FUNCTIONAL)
# and return the expected value
self.assertEqual(services.call_first_available(), 42)
def test_passes_arguments(self):
"""
A service should get arguments passed on
"""
services = RemoteServiceList()
services.append(lambda arg: arg)
self.assertEqual(services.call_first_available(42), 42)
def test_failing_service_list_should_throw(self):
"""
A list of failing services should throw an AllServicesUnavailable exception
"""
services = RemoteServiceList()
services.append(generate_failing_func())
services.append(generate_failing_func())
with self.assertRaises(AllServicesUnavailable):
services.call_first_available()
def test_service_failover(self):
"""
Services that fail should cause a failover to another service on first failure
"""
services = RemoteServiceList()
func = generate_failing_func()
services.append(func)
services.append(lambda: 42)
self.assertEqual(services.call_first_available(), 42)
self.assertEqual(func.call_count, 1)
def test_custom_exception_handling(self):
"""
A custom exception should be caught successfully
"""
services = RemoteServiceList(expected_exception=CustomException)
func = generate_failing_func(exception=CustomException)
services.append(func)
services.append(lambda: 23)
self.assertEqual(services.call_first_available(), 23)
self.assertEqual(func.call_count, 1)
def get_or_create_remote_services(cls, urls, garbage_collect_timeout=120):
"""
Create a session per unique set of connections.
On each access to the list we purge entries that haven't been accesses within
`garbage_collect_timeout`
:parm urls: urls within the service, used to retrieve and construct
a new RemoteServiceList
:param garbage_collect_timeout: timeout in seconds after which to
remove entries from the cache
"""
# start by hashing all urls so we have a way to refer to a configuration
m = hashlib.md5()
for u in urls:
m.update(u)
h = m.hexdigest()
# get the current time once
now = time.time()
# cleanup the dict of existing cache entries based on the given timeout
for key in cls._remote_services.keys():
ts, _ = cls._remote_services[key]
if ts + garbage_collect_timeout < now:
del cls._remote_services[key]
# search for the entry we are looking for
item = cls._remote_services.get(h)
if not item:
# if no entry was found create a new session and store it within the cache
log.debug('Cache miss. Creating new entry for hash %s', h)
session = requests.Session()
rs = RemoteServiceList(
failure_threshold=2, # after two failures try next service
recovery_timeout=30, # after 30 seconds retry a failed service
# mark services a failed if a requests.ConnectionError occured
expected_exception=requests.ConnectionError,
)
for url in urls:
rs.append(partial(session.post, url))
item = (now, rs)
cls._remote_services[h] = item
else:
# update the access time to the current time if the entry already existed
log.debug('Cache hit. Updating timestamp of %s', h)
_, rs = item
cls._remote_services[h] = (now, rs)
# return only the actual RemoteServiceList
_, rs = item
return rs
def test_passes_kwargs(self):
"""
A function should get keyword-arguments passed on
"""
services = RemoteServiceList()
services.append(generate_passthru_func())
# the arguments we pass into the service should be returned for investigation
args, kwargs = services.call_first_available(1, 2, 3, one=1, two=2, three=3)
self.assertEqual(args, (1,2,3))
self.assertEqual(kwargs, dict(one=1, two=2, three=3))
def test_recovery(self, dt_now):
"""
Ensure that a failing function recovers after the configured timeout.
Before the timeout expires other good functions should be used.
"""
# build a service list with a variadic function & a lambda that tells
# us that we are in failover mode
services = RemoteServiceList(expected_exception=CustomException)
func = generate_variadic_func(exception=CustomException)
services.append(func)
services.append(lambda: "failover")
# record the current time for our mocking
start_time = datetime.now()
# initially return the current time
dt_now.return_value = start_time
# calling the function (before it is marked as failing) returns the
# input parameters (as expected)
assert services.call_first_available(1) == ((1, ), {})
# mark the first function in the list as failing & move into the future
func.fail = True
dt_now.return_value += timedelta(seconds=1)
# call n times until the function is marked as failed
for _ in range(0, services.failure_threshold):
assert services.call_first_available() == "failover"
assert services[0].state == State.UNAVAILABLE
# Every second until recovery call the first available service
# The return value should always be 'failover'
while dt_now.return_value <= start_time + timedelta(
seconds=services.recovery_timeout):
assert services.call_first_available() == "failover"
dt_now.return_value += timedelta(seconds=1)
# the state of the primary function should still be UNAVAILABLE
assert services[0].state == State.UNAVAILABLE
assert services[1].state == State.FUNCTIONAL
# tell function to return again
func.fail = False
# after the recovery timeout the first service should start returing
# again
dt_now.return_value += timedelta(seconds=1)
assert services.call_first_available(1) == ((1, ), {})
assert services[0].state == State.FUNCTIONAL
def test_recovery(self, dt_now):
"""
Ensure that a failing function recovers after the configured timeout.
Before the timeout expires other good functions should be used.
"""
# build a service list with a variadic function & a lambda that tells
# us that we are in failover mode
services = RemoteServiceList(expected_exception=CustomException)
func = generate_variadic_func(exception=CustomException)
services.append(func)
services.append(lambda: 'failover')
# record the current time for our mocking
start_time = datetime.now()
# initially return the current time
dt_now.return_value = start_time
# calling the function (before it is marked as failing) returns the
# input parameters (as expected)
self.assertEqual(services.call_first_available(1), ((1,), {}))
# mark the first function in the list as failing & move into the future
func.fail = True
dt_now.return_value += timedelta(seconds=1)
# call n times until the function is marked as failed
for _ in range(0, services.failure_threshold):
self.assertEqual(services.call_first_available(), 'failover')
self.assertEqual(services[0].state, State.UNAVAILABLE)
# Every second until recovery call the first available service
# The return value should always be 'failover'
while dt_now.return_value <= start_time + timedelta(seconds=services.recovery_timeout):
self.assertEqual(services.call_first_available(), 'failover')
dt_now.return_value += timedelta(seconds=1)
# the state of the primary function should still be UNAVAILABLE
self.assertEqual(services[0].state, State.UNAVAILABLE)
self.assertEqual(services[1].state, State.FUNCTIONAL)
# tell function to return again
func.fail = False
# after the recovery timeout the first service should start returing again
dt_now.return_value += timedelta(seconds=1)
self.assertEqual(services.call_first_available(1), ((1,), {}))
self.assertEqual(services[0].state, State.FUNCTIONAL)