def setUp(self):
self.__fake_cache = FakeCache()
self.__warmer_test_instance = ControlledCacheWarmer(
max_failure_count=5, blacklist_time_secs=300)
self.assertFalse(self.__warmer_test_instance.is_running())
# noinspection PyTypeChecker
self.__warmer_test_instance.set_k8s_cache(self.__fake_cache)
self.__warmer_test_instance.start()
self.assertTrue(self.__warmer_test_instance.is_running())
self.timeout = 5
class ControlledCacheWarmerTest(ScalyrTestCase):
CONTAINER_1 = "container_1"
NAMESPACE_1 = "namespace_1"
POD_1 = "pod_1"
CONTAINER_2 = "container_2"
NAMESPACE_2 = "namespace_2"
POD_2 = "pod_2"
def setUp(self):
super(ControlledCacheWarmerTest, self).setUp()
self.__fake_cache = FakeCache()
self.__warmer_test_instance = ControlledCacheWarmer(
max_failure_count=5, blacklist_time_secs=300)
self.assertFalse(self.__warmer_test_instance.is_running())
# noinspection PyTypeChecker
self.__warmer_test_instance.set_k8s_cache(self.__fake_cache)
self.__warmer_test_instance.start()
self.assertTrue(self.__warmer_test_instance.is_running())
self.timeout = 5
def tearDown(self):
self.__warmer_test_instance.stop(wait_on_join=False)
self.__fake_cache.stop()
self.assertFalse(self.__warmer_test_instance.is_running())
def test_basic_case(self):
warmer = self.__warmer_test_instance
fake_cache = self.__fake_cache
warmer.begin_marking()
warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1)
warmer.end_marking()
fake_cache.wait_until_request_pending()
self.assertEqual(warmer.active_containers(), [self.CONTAINER_1])
self.assertEqual(warmer.warming_containers(), [self.CONTAINER_1])
self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1))
warmer.begin_marking()
warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1)
warmer.end_marking()
self.assertEqual(warmer.warming_containers(), [self.CONTAINER_1])
fake_cache.set_response(self.NAMESPACE_1, self.POD_1, success=True)
warmer.block_until_idle(self.timeout)
self.assertEqual(warmer.warming_containers(), [])
self.assertTrue(warmer.is_warm(self.NAMESPACE_1, self.POD_1))
def test_already_warm(self):
# Test case where a pod was put into the warming list, but when it is returned by `pick_next_pod`, it is
# already warmed in the cache.
# This is a difficult one to test, so we do a little trickery. We first have the thread that is invoking
# `pick_next_pod` get tied up blocking for a request for another pod. That gives us time to manipulate the
# the caching state for another pod (i.e., get it added to the warming list but then put it into cache before
# it is returned by `pick_next_pod`)
warmer = self.__warmer_test_instance
fake_cache = self.__fake_cache
# Only add in one container so that the ControlledCacheWarmer thread is blocked on getting it.
warmer.begin_marking()
warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1)
warmer.end_marking()
self.assertEqual(warmer.active_containers(), [self.CONTAINER_1])
self.assertEqual(warmer.warming_containers(), [self.CONTAINER_1])
# Guarantee that the thread is blocked.
fake_cache.wait_until_request_pending()
# Now introduce the other pod.
warmer.begin_marking()
warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1)
warmer.mark_to_warm(self.CONTAINER_2, self.NAMESPACE_2, self.POD_2)
warmer.end_marking()
self.assertEqual(warmer.active_containers(),
[self.CONTAINER_1, self.CONTAINER_2])
self.assertEqual(warmer.warming_containers(),
[self.CONTAINER_1, self.CONTAINER_2])
# Simulate adding the second one to the cache. This will trigger it to be `already_warm`
fake_cache.simulate_add_pod_to_cache(self.NAMESPACE_2, self.POD_2)
# Ok, now let the ControlledCacheWarmer thread go and try to process everything.
fake_cache.set_response(self.NAMESPACE_1, self.POD_1, success=True)
warmer.block_until_idle(self.timeout)
stats = warmer.get_report_stats()
success_count = stats.get("success", (0, 0))
already_warm_count = stats.get("already_warm", (0, 0))
self.assertEqual(success_count[0], 1)
self.assertEqual(already_warm_count[0], 1)
self.assertEqual(warmer.active_containers(),
[self.CONTAINER_1, self.CONTAINER_2])
self.assertEqual(warmer.warming_containers(), [])
self.assertEqual(warmer.blacklisted_containers(), [])
self.assertTrue(warmer.is_warm(self.NAMESPACE_1, self.POD_1))
self.assertTrue(warmer.is_warm(self.NAMESPACE_2, self.POD_2))
def test_already_warm_two(self):
# Second case type of already_warmed. This occurs when the pod is found in cache the second time you
# try to mark it as active.
warmer = self.__warmer_test_instance
fake_cache = self.__fake_cache
# Stop the warmer thread since we don't need it for the test, and to avoid a race condition that sometimes
# results in finding too many "already_warm" results
warmer.stop()
warmer.begin_marking()
warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1)
warmer.end_marking()
warmer.begin_marking()
fake_cache.simulate_add_pod_to_cache(self.NAMESPACE_1, self.POD_1)
warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1)
warmer.end_marking()
stats = warmer.get_report_stats()
success_count = stats.get("success", (1, 1))
already_warm_count = stats.get("already_warm", (0, 0))
self.assertEqual(success_count[0], 0)
self.assertEqual(already_warm_count[0], 1)
def test_remove_inactive(self):
warmer = self.__warmer_test_instance
warmer.begin_marking()
warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1)
warmer.end_marking()
self.assertEqual(warmer.active_containers(), [self.CONTAINER_1])
warmer.begin_marking()
warmer.end_marking()
self.assertEqual(warmer.active_containers(), [])
def test_multiple_pods(self):
warmer = self.__warmer_test_instance
fake_cache = self.__fake_cache
warmer.begin_marking()
warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1)
warmer.mark_to_warm(self.CONTAINER_2, self.NAMESPACE_2, self.POD_2)
warmer.end_marking()
pod_namespace, _ = fake_cache.wait_until_request_pending()
if pod_namespace == self.NAMESPACE_1:
# We have two containers and we don't know which will be warmed first by the abstraction.. so we
# just handle the two different cases, switching off of the actual request that is pending in the
# abstraction. request_a is the first one that has been requested and request_b is the other container.
request_a_pod_namespace = self.NAMESPACE_1
request_a_pod_name = self.POD_1
request_b_pod_namespace = self.NAMESPACE_2
request_b_pod_name = self.POD_2
request_b_container = self.CONTAINER_2
else:
request_a_pod_namespace = self.NAMESPACE_2
request_a_pod_name = self.POD_2
request_b_pod_namespace = self.NAMESPACE_1
request_b_pod_name = self.POD_1
request_b_container = self.CONTAINER_1
self.assertEqual(warmer.active_containers(),
[self.CONTAINER_1, self.CONTAINER_2])
self.assertEqual(warmer.warming_containers(),
[self.CONTAINER_1, self.CONTAINER_2])
self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1))
self.assertFalse(warmer.is_warm(self.NAMESPACE_2, self.POD_2))
fake_cache.set_response(request_a_pod_namespace,
request_a_pod_name,
success=True)
fake_cache.wait_until_request_pending(
namespace=request_b_pod_namespace, name=request_b_pod_name)
pod_namespace, _ = fake_cache.wait_until_request_pending()
self.assertEqual(pod_namespace, request_b_pod_namespace)
self.assertEqual(warmer.active_containers(),
[self.CONTAINER_1, self.CONTAINER_2])
self.assertEqual(warmer.warming_containers(), [request_b_container])
self.assertTrue(
warmer.is_warm(request_a_pod_namespace, request_a_pod_name))
self.assertFalse(
warmer.is_warm(request_b_pod_namespace, request_b_pod_name))
fake_cache.set_response(request_b_pod_namespace,
request_b_pod_name,
success=True)
warmer.block_until_idle(self.timeout)
self.assertEqual(warmer.active_containers(),
[self.CONTAINER_1, self.CONTAINER_2])
self.assertEqual(warmer.warming_containers(), [])
self.assertTrue(
warmer.is_warm(request_a_pod_namespace, request_a_pod_name))
self.assertTrue(
warmer.is_warm(request_b_pod_namespace, request_b_pod_name))
def test_permanent_error(self):
warmer = self.__warmer_test_instance
fake_cache = self.__fake_cache
warmer.begin_marking()
warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1)
warmer.end_marking()
fake_cache.wait_until_request_pending()
fake_cache.set_response(self.NAMESPACE_1,
self.POD_1,
permanent_error=True)
warmer.block_until_idle(self.timeout)
stats = warmer.get_report_stats()
success_count = stats.get("success", (1, 1))
perm_error_count = stats.get("perm_error", (0, 0))
self.assertEqual(success_count[0], 0)
self.assertEqual(perm_error_count[0], 1)
self.assertEqual(warmer.active_containers(), [self.CONTAINER_1])
self.assertEqual(warmer.warming_containers(), [])
self.assertEqual(warmer.blacklisted_containers(), [self.CONTAINER_1])
self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1))
def test_temporary_error(self):
warmer = self.__warmer_test_instance
fake_cache = self.__fake_cache
warmer.begin_marking()
warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1)
warmer.end_marking()
for i in range(0, 4):
fake_cache.set_response(self.NAMESPACE_1,
self.POD_1,
temporary_error=True)
fake_cache.wait_until_request_finished(self.NAMESPACE_1,
self.POD_1)
fake_cache.wait_until_request_pending()
stats = warmer.get_report_stats()
success_count = stats.get("success", (1, 1))
temp_error_count = stats.get("temp_error", (0, 0))
self.assertEqual(success_count[0], 0)
self.assertEqual(temp_error_count[0], 4)
self.assertEqual(warmer.active_containers(), [self.CONTAINER_1])
self.assertEqual(warmer.warming_containers(), [self.CONTAINER_1])
self.assertEqual(warmer.blacklisted_containers(), [])
self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1))
fake_cache.set_response(self.NAMESPACE_1,
self.POD_1,
temporary_error=True)
warmer.block_until_idle(self.timeout)
self.assertEqual(warmer.active_containers(), [self.CONTAINER_1])
self.assertEqual(warmer.warming_containers(), [])
self.assertEqual(warmer.blacklisted_containers(), [self.CONTAINER_1])
self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1))
stats = warmer.get_report_stats()
temp_error_count = stats.get("temp_error", (0, 0))
self.assertEqual(temp_error_count[0], 5)
def test_retry_from_blacklist(self):
fake_time = 5
def fake_get_current_time(_):
return fake_time
warmer = self.__warmer_test_instance
fake_cache = self.__fake_cache
with mock.patch.object(ControlledCacheWarmer, "_get_current_time",
fake_get_current_time):
warmer.begin_marking()
warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1)
warmer.end_marking()
# Have to have 5 temporary errors before it is blacklisted.
for i in range(0, 4):
fake_cache.set_response(self.NAMESPACE_1,
self.POD_1,
temporary_error=True)
fake_cache.wait_until_request_finished(self.NAMESPACE_1,
self.POD_1)
fake_cache.wait_until_request_pending()
fake_cache.set_response(self.NAMESPACE_1,
self.POD_1,
temporary_error=True)
warmer.block_until_idle(self.timeout)
self.assertEqual(warmer.active_containers(), [self.CONTAINER_1])
self.assertEqual(warmer.warming_containers(), [])
self.assertEqual(warmer.blacklisted_containers(),
[self.CONTAINER_1])
self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1))
# Now wait at lest 300 seconds to see it come off from the blacklist.
fake_time += 400
# Items are only removed from the blacklist during calls to `end_marking`, so we need to do that here.
warmer.begin_marking()
warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1)
warmer.end_marking()
self.assertEqual(warmer.active_containers(), [self.CONTAINER_1])
self.assertEqual(warmer.warming_containers(), [self.CONTAINER_1])
self.assertEqual(warmer.blacklisted_containers(), [])
self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1))
# To test that the entry was properly reset to error count = 0, see if it gets blacklisted after
# 5 new temporary errors.
for i in range(0, 4):
fake_cache.set_response(self.NAMESPACE_1,
self.POD_1,
temporary_error=True)
fake_cache.wait_until_request_finished(self.NAMESPACE_1,
self.POD_1)
fake_cache.wait_until_request_pending()
fake_cache.set_response(self.NAMESPACE_1,
self.POD_1,
temporary_error=True)
warmer.block_until_idle(self.timeout)
self.assertEqual(warmer.active_containers(), [self.CONTAINER_1])
self.assertEqual(warmer.warming_containers(), [])
self.assertEqual(warmer.blacklisted_containers(),
[self.CONTAINER_1])
self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1))