def setUp(self): self.__fake_cache = FakeCache() self.__warmer_test_instance = ControlledCacheWarmer( max_failure_count=5, blacklist_time_secs=300) self.assertFalse(self.__warmer_test_instance.is_running()) # noinspection PyTypeChecker self.__warmer_test_instance.set_k8s_cache(self.__fake_cache) self.__warmer_test_instance.start() self.assertTrue(self.__warmer_test_instance.is_running()) self.timeout = 5
class ControlledCacheWarmerTest(ScalyrTestCase): CONTAINER_1 = "container_1" NAMESPACE_1 = "namespace_1" POD_1 = "pod_1" CONTAINER_2 = "container_2" NAMESPACE_2 = "namespace_2" POD_2 = "pod_2" def setUp(self): super(ControlledCacheWarmerTest, self).setUp() self.__fake_cache = FakeCache() self.__warmer_test_instance = ControlledCacheWarmer( max_failure_count=5, blacklist_time_secs=300) self.assertFalse(self.__warmer_test_instance.is_running()) # noinspection PyTypeChecker self.__warmer_test_instance.set_k8s_cache(self.__fake_cache) self.__warmer_test_instance.start() self.assertTrue(self.__warmer_test_instance.is_running()) self.timeout = 5 def tearDown(self): self.__warmer_test_instance.stop(wait_on_join=False) self.__fake_cache.stop() self.assertFalse(self.__warmer_test_instance.is_running()) def test_basic_case(self): warmer = self.__warmer_test_instance fake_cache = self.__fake_cache warmer.begin_marking() warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1) warmer.end_marking() fake_cache.wait_until_request_pending() self.assertEqual(warmer.active_containers(), [self.CONTAINER_1]) self.assertEqual(warmer.warming_containers(), [self.CONTAINER_1]) self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1)) warmer.begin_marking() warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1) warmer.end_marking() self.assertEqual(warmer.warming_containers(), [self.CONTAINER_1]) fake_cache.set_response(self.NAMESPACE_1, self.POD_1, success=True) warmer.block_until_idle(self.timeout) self.assertEqual(warmer.warming_containers(), []) self.assertTrue(warmer.is_warm(self.NAMESPACE_1, self.POD_1)) def test_already_warm(self): # Test case where a pod was put into the warming list, but when it is returned by `pick_next_pod`, it is # already warmed in the cache. # This is a difficult one to test, so we do a little trickery. We first have the thread that is invoking # `pick_next_pod` get tied up blocking for a request for another pod. That gives us time to manipulate the # the caching state for another pod (i.e., get it added to the warming list but then put it into cache before # it is returned by `pick_next_pod`) warmer = self.__warmer_test_instance fake_cache = self.__fake_cache # Only add in one container so that the ControlledCacheWarmer thread is blocked on getting it. warmer.begin_marking() warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1) warmer.end_marking() self.assertEqual(warmer.active_containers(), [self.CONTAINER_1]) self.assertEqual(warmer.warming_containers(), [self.CONTAINER_1]) # Guarantee that the thread is blocked. fake_cache.wait_until_request_pending() # Now introduce the other pod. warmer.begin_marking() warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1) warmer.mark_to_warm(self.CONTAINER_2, self.NAMESPACE_2, self.POD_2) warmer.end_marking() self.assertEqual(warmer.active_containers(), [self.CONTAINER_1, self.CONTAINER_2]) self.assertEqual(warmer.warming_containers(), [self.CONTAINER_1, self.CONTAINER_2]) # Simulate adding the second one to the cache. This will trigger it to be `already_warm` fake_cache.simulate_add_pod_to_cache(self.NAMESPACE_2, self.POD_2) # Ok, now let the ControlledCacheWarmer thread go and try to process everything. fake_cache.set_response(self.NAMESPACE_1, self.POD_1, success=True) warmer.block_until_idle(self.timeout) stats = warmer.get_report_stats() success_count = stats.get("success", (0, 0)) already_warm_count = stats.get("already_warm", (0, 0)) self.assertEqual(success_count[0], 1) self.assertEqual(already_warm_count[0], 1) self.assertEqual(warmer.active_containers(), [self.CONTAINER_1, self.CONTAINER_2]) self.assertEqual(warmer.warming_containers(), []) self.assertEqual(warmer.blacklisted_containers(), []) self.assertTrue(warmer.is_warm(self.NAMESPACE_1, self.POD_1)) self.assertTrue(warmer.is_warm(self.NAMESPACE_2, self.POD_2)) def test_already_warm_two(self): # Second case type of already_warmed. This occurs when the pod is found in cache the second time you # try to mark it as active. warmer = self.__warmer_test_instance fake_cache = self.__fake_cache # Stop the warmer thread since we don't need it for the test, and to avoid a race condition that sometimes # results in finding too many "already_warm" results warmer.stop() warmer.begin_marking() warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1) warmer.end_marking() warmer.begin_marking() fake_cache.simulate_add_pod_to_cache(self.NAMESPACE_1, self.POD_1) warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1) warmer.end_marking() stats = warmer.get_report_stats() success_count = stats.get("success", (1, 1)) already_warm_count = stats.get("already_warm", (0, 0)) self.assertEqual(success_count[0], 0) self.assertEqual(already_warm_count[0], 1) def test_remove_inactive(self): warmer = self.__warmer_test_instance warmer.begin_marking() warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1) warmer.end_marking() self.assertEqual(warmer.active_containers(), [self.CONTAINER_1]) warmer.begin_marking() warmer.end_marking() self.assertEqual(warmer.active_containers(), []) def test_multiple_pods(self): warmer = self.__warmer_test_instance fake_cache = self.__fake_cache warmer.begin_marking() warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1) warmer.mark_to_warm(self.CONTAINER_2, self.NAMESPACE_2, self.POD_2) warmer.end_marking() pod_namespace, _ = fake_cache.wait_until_request_pending() if pod_namespace == self.NAMESPACE_1: # We have two containers and we don't know which will be warmed first by the abstraction.. so we # just handle the two different cases, switching off of the actual request that is pending in the # abstraction. request_a is the first one that has been requested and request_b is the other container. request_a_pod_namespace = self.NAMESPACE_1 request_a_pod_name = self.POD_1 request_b_pod_namespace = self.NAMESPACE_2 request_b_pod_name = self.POD_2 request_b_container = self.CONTAINER_2 else: request_a_pod_namespace = self.NAMESPACE_2 request_a_pod_name = self.POD_2 request_b_pod_namespace = self.NAMESPACE_1 request_b_pod_name = self.POD_1 request_b_container = self.CONTAINER_1 self.assertEqual(warmer.active_containers(), [self.CONTAINER_1, self.CONTAINER_2]) self.assertEqual(warmer.warming_containers(), [self.CONTAINER_1, self.CONTAINER_2]) self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1)) self.assertFalse(warmer.is_warm(self.NAMESPACE_2, self.POD_2)) fake_cache.set_response(request_a_pod_namespace, request_a_pod_name, success=True) fake_cache.wait_until_request_pending( namespace=request_b_pod_namespace, name=request_b_pod_name) pod_namespace, _ = fake_cache.wait_until_request_pending() self.assertEqual(pod_namespace, request_b_pod_namespace) self.assertEqual(warmer.active_containers(), [self.CONTAINER_1, self.CONTAINER_2]) self.assertEqual(warmer.warming_containers(), [request_b_container]) self.assertTrue( warmer.is_warm(request_a_pod_namespace, request_a_pod_name)) self.assertFalse( warmer.is_warm(request_b_pod_namespace, request_b_pod_name)) fake_cache.set_response(request_b_pod_namespace, request_b_pod_name, success=True) warmer.block_until_idle(self.timeout) self.assertEqual(warmer.active_containers(), [self.CONTAINER_1, self.CONTAINER_2]) self.assertEqual(warmer.warming_containers(), []) self.assertTrue( warmer.is_warm(request_a_pod_namespace, request_a_pod_name)) self.assertTrue( warmer.is_warm(request_b_pod_namespace, request_b_pod_name)) def test_permanent_error(self): warmer = self.__warmer_test_instance fake_cache = self.__fake_cache warmer.begin_marking() warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1) warmer.end_marking() fake_cache.wait_until_request_pending() fake_cache.set_response(self.NAMESPACE_1, self.POD_1, permanent_error=True) warmer.block_until_idle(self.timeout) stats = warmer.get_report_stats() success_count = stats.get("success", (1, 1)) perm_error_count = stats.get("perm_error", (0, 0)) self.assertEqual(success_count[0], 0) self.assertEqual(perm_error_count[0], 1) self.assertEqual(warmer.active_containers(), [self.CONTAINER_1]) self.assertEqual(warmer.warming_containers(), []) self.assertEqual(warmer.blacklisted_containers(), [self.CONTAINER_1]) self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1)) def test_temporary_error(self): warmer = self.__warmer_test_instance fake_cache = self.__fake_cache warmer.begin_marking() warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1) warmer.end_marking() for i in range(0, 4): fake_cache.set_response(self.NAMESPACE_1, self.POD_1, temporary_error=True) fake_cache.wait_until_request_finished(self.NAMESPACE_1, self.POD_1) fake_cache.wait_until_request_pending() stats = warmer.get_report_stats() success_count = stats.get("success", (1, 1)) temp_error_count = stats.get("temp_error", (0, 0)) self.assertEqual(success_count[0], 0) self.assertEqual(temp_error_count[0], 4) self.assertEqual(warmer.active_containers(), [self.CONTAINER_1]) self.assertEqual(warmer.warming_containers(), [self.CONTAINER_1]) self.assertEqual(warmer.blacklisted_containers(), []) self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1)) fake_cache.set_response(self.NAMESPACE_1, self.POD_1, temporary_error=True) warmer.block_until_idle(self.timeout) self.assertEqual(warmer.active_containers(), [self.CONTAINER_1]) self.assertEqual(warmer.warming_containers(), []) self.assertEqual(warmer.blacklisted_containers(), [self.CONTAINER_1]) self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1)) stats = warmer.get_report_stats() temp_error_count = stats.get("temp_error", (0, 0)) self.assertEqual(temp_error_count[0], 5) def test_retry_from_blacklist(self): fake_time = 5 def fake_get_current_time(_): return fake_time warmer = self.__warmer_test_instance fake_cache = self.__fake_cache with mock.patch.object(ControlledCacheWarmer, "_get_current_time", fake_get_current_time): warmer.begin_marking() warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1) warmer.end_marking() # Have to have 5 temporary errors before it is blacklisted. for i in range(0, 4): fake_cache.set_response(self.NAMESPACE_1, self.POD_1, temporary_error=True) fake_cache.wait_until_request_finished(self.NAMESPACE_1, self.POD_1) fake_cache.wait_until_request_pending() fake_cache.set_response(self.NAMESPACE_1, self.POD_1, temporary_error=True) warmer.block_until_idle(self.timeout) self.assertEqual(warmer.active_containers(), [self.CONTAINER_1]) self.assertEqual(warmer.warming_containers(), []) self.assertEqual(warmer.blacklisted_containers(), [self.CONTAINER_1]) self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1)) # Now wait at lest 300 seconds to see it come off from the blacklist. fake_time += 400 # Items are only removed from the blacklist during calls to `end_marking`, so we need to do that here. warmer.begin_marking() warmer.mark_to_warm(self.CONTAINER_1, self.NAMESPACE_1, self.POD_1) warmer.end_marking() self.assertEqual(warmer.active_containers(), [self.CONTAINER_1]) self.assertEqual(warmer.warming_containers(), [self.CONTAINER_1]) self.assertEqual(warmer.blacklisted_containers(), []) self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1)) # To test that the entry was properly reset to error count = 0, see if it gets blacklisted after # 5 new temporary errors. for i in range(0, 4): fake_cache.set_response(self.NAMESPACE_1, self.POD_1, temporary_error=True) fake_cache.wait_until_request_finished(self.NAMESPACE_1, self.POD_1) fake_cache.wait_until_request_pending() fake_cache.set_response(self.NAMESPACE_1, self.POD_1, temporary_error=True) warmer.block_until_idle(self.timeout) self.assertEqual(warmer.active_containers(), [self.CONTAINER_1]) self.assertEqual(warmer.warming_containers(), []) self.assertEqual(warmer.blacklisted_containers(), [self.CONTAINER_1]) self.assertFalse(warmer.is_warm(self.NAMESPACE_1, self.POD_1))