class BlockingRateLimiterTest(ScalyrTestCase):
    def setUp(self):
        super(BlockingRateLimiterTest, self).setUp()
        self._fake_clock = FakeClock()
        self._test_state = {
            'count': 0,
            'times': [],
        }
        self._test_state_lock = threading.Lock()
        self._outcome_generator_lock = threading.Lock()

    def test_init_exceptions(self):
        BRL = BlockingRateLimiter

        # strategy must be restricted to proper values
        self.assertRaises(
            ValueError, lambda: BRL(
                num_agents=1,
                initial_cluster_rate=100,
                max_cluster_rate=1000,
                min_cluster_rate=1,
                consecutive_success_threshold=5,
                strategy='bad strategy',
            ))

        # max_concurrency must be 1 or more
        self.assertRaises(
            ValueError, lambda: BRL(
                num_agents=1,
                initial_cluster_rate=100,
                max_cluster_rate=1000,
                min_cluster_rate=1,
                consecutive_success_threshold=5,
                max_concurrency=0,
            ))

        # consecutive_success_threshold must be a positive int
        self.assertRaises(
            ValueError, lambda: BRL(
                num_agents=1,
                initial_cluster_rate=100,
                max_cluster_rate=1000,
                min_cluster_rate=1,
                consecutive_success_threshold=-5.0,
            ))
        self.assertRaises(
            ValueError, lambda: BRL(
                num_agents=1,
                initial_cluster_rate=100,
                max_cluster_rate=1000,
                min_cluster_rate=1,
                consecutive_success_threshold=99.99,
            ))

        # user-supplied init must be between user-supplied min/max
        self.assertRaises(
            ValueError, lambda: BRL(
                num_agents=1,
                initial_cluster_rate=0.7,
                max_cluster_rate=1000,
                min_cluster_rate=1,
                consecutive_success_threshold=5,
            ))
        self.assertRaises(
            ValueError, lambda: BRL(
                num_agents=1,
                initial_cluster_rate=7777,
                max_cluster_rate=1000,
                min_cluster_rate=1,
                consecutive_success_threshold=5,
            ))

    def test_lazy_adjust_min_max_init_rates(self):
        """Tests the one-time lazy adjustments"""

        # Make sure proper adjustments are made if out of bounds
        BRL = BlockingRateLimiter
        rl = BRL(
            num_agents=1,
            initial_cluster_rate=BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE + 1000,
            max_cluster_rate=BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE + 2000000,
            min_cluster_rate=float(BRL.HARD_LIMIT_MIN_CLUSTER_RATE) / 10,
            consecutive_success_threshold=5,
        )
        rl._lazy_adjust_min_max_rates()
        self.assertEqual(rl._initial_cluster_rate,
                         BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE)
        self.assertEqual(rl._max_cluster_rate,
                         BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE + 2000000)
        self.assertEqual(rl._min_cluster_rate, BRL.HARD_LIMIT_MIN_CLUSTER_RATE)

        # Make sure adjustments are NOT made if within bounds
        rl = BRL(
            num_agents=1,
            initial_cluster_rate=BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE - 1,
            max_cluster_rate=BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE - 1,
            min_cluster_rate=float(BRL.HARD_LIMIT_MIN_CLUSTER_RATE) + 1,
            consecutive_success_threshold=5,
        )
        rl._lazy_adjust_min_max_rates()
        self.assertEqual(rl._initial_cluster_rate,
                         BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE - 1)
        self.assertEqual(rl._max_cluster_rate,
                         BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE + -1)
        self.assertEqual(rl._min_cluster_rate,
                         BRL.HARD_LIMIT_MIN_CLUSTER_RATE + 1)

        # Init rate cannot be lower than min rate
        rl = BlockingRateLimiter(
            num_agents=1,
            initial_cluster_rate=0.1,
            max_cluster_rate=1000,
            min_cluster_rate=0,
            consecutive_success_threshold=5,
        )
        rl._lazy_adjust_min_max_rates()
        self.assertEqual(rl._initial_cluster_rate, 1)

    def __create_consumer_threads(self, num_threads, rate_limiter,
                                  experiment_end_time,
                                  reported_outcome_generator):
        """
        Create a list of client thread that will consume from the rate limiter.
        @param num_threads: Number of client threads
        @param rate_limiter: The rate limiter to test
        @param experiment_end_time: consumer threads terminate their loops when the fake clock has exceeded this.
        @param reported_outcome_generator: Generator to get reported outcome boolean value

        @type rate_limiter: BlockingRateLimiter
        @type threads: iterable of Thread objects
        @type experiment_end_time: int (utc seconds)
        @type reported_outcome_generator: generator

        @returns: list of consumer threads
        @rtype: list(Thread)
        """
        def consume_token_blocking():
            """consumer threads will keep acquiring tokens until experiment end time"""
            while self._fake_clock.time() < experiment_end_time:
                # A simple loop that acquires token, updates a counter, then releases token with an outcome
                # provided by reported_outcome_generator()
                t1 = self._fake_clock.time()
                token = rate_limiter.acquire_token()
                # update test state
                self._test_state_lock.acquire()
                try:
                    self._test_state['count'] += 1
                    self._test_state['times'].append(int(t1))
                finally:
                    self._test_state_lock.release()

                self._outcome_generator_lock.acquire()
                try:
                    outcome = reported_outcome_generator.next()
                    rate_limiter.release_token(token, outcome)
                finally:
                    self._outcome_generator_lock.release()

        return [
            threading.Thread(target=consume_token_blocking)
            for _ in range(num_threads)
        ]

    def __create_fake_clock_advancer_thread(self, rate_limiter,
                                            consumer_threads):
        """
        Run a separate thread that advances time in order to allow rate_limiter to release tokens
        @param rate_limiter: The rate limiter to test
        @param consumer_threads: Other threads that are consumers of the Rate Limiter.  The advancer thread will run
            for as long as the consumer threads continue to run.

        @type rate_limiter: BlockingRateLimiter
        @type consumer_threads: iterable of Thread objects

        @returns: Thread object
        @rtype: Thread
        """
        class FakeClockAdvancerThread(StoppableThread):
            def __init__(self2):
                StoppableThread.__init__(self2,
                                         name='FakeClockAdvancerThread',
                                         is_daemon=True)

            def run_and_propagate(self2):
                at_least_one_client_thread_incomplete = True
                while at_least_one_client_thread_incomplete:
                    # All client threads are likely blocking on the token queue at this point
                    # The experiment will never start until at least one token is granted (and waiting threads notified)
                    # To jump start this, simply advance fake clock to the ripe_time.
                    delta = rate_limiter._ripe_time - self._fake_clock.time()
                    self._fake_clock.advance_time(increment_by=delta)
                    # Wake up after 1 second just in case and trigger another advance
                    [t.join(1) for t in consumer_threads]
                    at_least_one_client_thread_incomplete = False
                    for t in consumer_threads:
                        if t.isAlive():
                            at_least_one_client_thread_incomplete = True
                            break

        return FakeClockAdvancerThread()

    def test_rate_adjustment_logic(self):
        """Make sure rate adjustments work according to spec

        The cases we test are:
        1. All successes. Actual rate is very high. New target rate is always increase_factor * current target rate.
        2. All successes. Actual rate is very low. New target rate is always increase_factor * actual rate,
            (but no lower than current rate).
        3. All failures. Actual rate is very low. New target rate is always backoff_factor * current target rate.
        4. All failures. Actual rate is very high. New target rate is always backoff_factor * actual rate.
            (but no higher than current rate).
        """
        required_successes = 5
        max_cluster_rate = 10000
        min_cluster_rate = 1
        initial_cluster_rate = 100
        increase_factor = 2.0
        backoff_factor = 0.5

        def _create_rate_limiter():
            return BlockingRateLimiter(
                num_agents=1,
                initial_cluster_rate=initial_cluster_rate,
                max_cluster_rate=max_cluster_rate,
                min_cluster_rate=min_cluster_rate,
                consecutive_success_threshold=required_successes,
                strategy=BlockingRateLimiter.STRATEGY_MULTIPLY,
                increase_factor=increase_factor,
                backoff_factor=backoff_factor,
                max_concurrency=1,
                fake_clock=self._fake_clock)

        def _mock_get_next_ripe_time_actual_rate_leads(rate_limiter):
            # Make the rate limiter grant tokens faster than the target rate
            def _func(*args, **kwargs):
                delta = 1.0 / rate_limiter._current_cluster_rate
                return rl._ripe_time + 0.9 * delta

            return _func

        def _mock_get_next_ripe_time_actual_rate_lags(rate_limiter):
            # Make the rate limiter grant tokens slower than the target rate
            def _func(*args, **kwargs):
                delta = 1.0 / rate_limiter._current_cluster_rate
                return rl._ripe_time + 1.1 * delta

            return _func

        rl = _create_rate_limiter()

        @patch.object(rl, '_get_next_ripe_time')
        def _case1_test_successes_actual_rate_leads_target_rate(
                mock_get_next_ripe_time):
            """Examine rate-increasing behavior in the context of very high actual rates"""
            mock_get_next_ripe_time.side_effect = _mock_get_next_ripe_time_actual_rate_leads(
                rl)

            advancer = self.__create_fake_clock_advancer_thread(
                rl, [threading.currentThread()])
            advancer.start()

            while True:
                token = rl.acquire_token()
                for x in range(required_successes - 1):
                    rl.release_token(token, True)
                    rl.acquire_token()

                # Actual rate always leads target rate
                old_target_rate = rl._current_cluster_rate
                old_actual_rate = rl._get_actual_cluster_rate()
                self.assertGreater(old_actual_rate, old_target_rate)

                # Token grant causes new rate to be calculated
                rl.release_token(token, True)

                # assert that the new rate is calculated based on the (lower/more conservative) target rate
                if increase_factor * old_target_rate < max_cluster_rate:
                    self.assertEqual(rl._current_cluster_rate,
                                     increase_factor * old_target_rate)
                else:
                    # assert that new rate never exceeds max rate
                    self.assertEqual(rl._current_cluster_rate,
                                     max_cluster_rate)
                    break
            advancer.stop(wait_on_join=False)

        _case1_test_successes_actual_rate_leads_target_rate()

        rl = _create_rate_limiter()

        @patch.object(rl, '_get_next_ripe_time')
        def _case2_test_successes_actual_rate_lags_target_rate(
                mock_get_next_ripe_time):
            """Examine rate-increasing behavior in the context of very high actual rates"""
            mock_get_next_ripe_time.side_effect = _mock_get_next_ripe_time_actual_rate_lags(
                rl)

            advancer = self.__create_fake_clock_advancer_thread(
                rl, [threading.currentThread()])
            advancer.start()

            while True:
                token = rl.acquire_token()
                for x in range(required_successes - 1):
                    rl.release_token(token, True)
                    rl.acquire_token()

                # Actual rate always lags target rate
                old_target_rate = rl._current_cluster_rate
                old_actual_rate = rl._get_actual_cluster_rate()
                self.assertLess(old_actual_rate, old_target_rate)

                # Token grant causes new rate to be calculated
                rl.release_token(token, True)

                # assert that the new rate is calculated based on the (lower/more conservative) actual rate
                if increase_factor * old_actual_rate < max_cluster_rate:
                    if increase_factor * old_actual_rate < old_target_rate:
                        self.assertEqual(round(rl._current_cluster_rate, 2),
                                         round(old_target_rate, 2))
                    else:
                        self.assertEqual(
                            round(rl._current_cluster_rate, 2),
                            round(increase_factor * old_actual_rate, 2))
                else:
                    # assert that new rate never exceeds max rate
                    self.assertEqual(rl._current_cluster_rate,
                                     max_cluster_rate)
                    break
            advancer.stop(wait_on_join=False)

        _case2_test_successes_actual_rate_lags_target_rate()

        rl = _create_rate_limiter()

        @patch.object(rl, '_get_next_ripe_time')
        def _case3_test_failures_actual_rate_lags_target_rate(
                mock_get_next_ripe_time):
            """Examine rate-decreasing behavior in the context of very high actual rates"""
            mock_get_next_ripe_time.side_effect = _mock_get_next_ripe_time_actual_rate_lags(
                rl)

            advancer = self.__create_fake_clock_advancer_thread(
                rl, [threading.currentThread()])
            advancer.start()

            counter = 0
            while True:
                token = rl.acquire_token()

                # Actual rate always lags target rate
                old_target_rate = rl._current_cluster_rate
                old_actual_rate = rl._get_actual_cluster_rate()
                if old_actual_rate is not None:
                    self.assertLess(old_actual_rate, old_target_rate)

                # Token grant a 100 initial successes followed by all failures
                counter += 1
                if counter <= required_successes:
                    rl.release_token(token, True)
                    continue
                else:
                    rl.release_token(token, False)

                # assert that the new rate is calculated based on the (higher/more conservative) target rate
                if backoff_factor * old_target_rate > min_cluster_rate:
                    self.assertEqual(
                        round(rl._current_cluster_rate, 2),
                        round(backoff_factor * old_target_rate, 2))
                else:
                    # assert that new rate never goes lower than min rate
                    self.assertEqual(round(rl._current_cluster_rate, 2),
                                     round(min_cluster_rate, 2))
                    break
            advancer.stop(wait_on_join=False)

        _case3_test_failures_actual_rate_lags_target_rate()

        rl = _create_rate_limiter()

        @patch.object(rl, '_get_next_ripe_time')
        def _case4_test_failures_actual_rate_leads_target_rate(
                mock_get_next_ripe_time):
            """Examine rate-decreasing behavior in the context of very high actual rates"""
            mock_get_next_ripe_time.side_effect = _mock_get_next_ripe_time_actual_rate_leads(
                rl)

            advancer = self.__create_fake_clock_advancer_thread(
                rl, [threading.currentThread()])
            advancer.start()

            counter = 0
            while True:
                token = rl.acquire_token()

                # Actual rate is always None because
                old_target_rate = rl._current_cluster_rate
                old_actual_rate = rl._get_actual_cluster_rate()
                if old_actual_rate is not None:
                    self.assertGreater(old_actual_rate, old_target_rate)

                # Token grant a 100 initial successes followed by all failures
                counter += 1
                if counter <= required_successes:
                    rl.release_token(token, True)
                    continue
                else:
                    rl.release_token(token, False)

                # assert that the new rate is calculated based on the (higher/more conservative) actual rate
                if backoff_factor * old_target_rate > min_cluster_rate:
                    self.assertEqual(
                        round(rl._current_cluster_rate, 2),
                        round(backoff_factor * old_target_rate, 2))
                else:
                    # assert that new rate never goes lower than min rate
                    self.assertEqual(rl._current_cluster_rate,
                                     min_cluster_rate)
                    break
            advancer.stop(wait_on_join=False)

        _case4_test_failures_actual_rate_leads_target_rate()

    def test_fixed_rate_single_concurrency(self):
        """Longer experiment"""
        # 1 rps x 1000 simulated seconds => 1000 increments
        self._test_fixed_rate(desired_agent_rate=1.0,
                              experiment_duration=10000,
                              concurrency=1,
                              expected_requests=10000,
                              allowed_variance=(0.95, 1.05))

    def test_fixed_rate_multiple_concurrency(self):
        """Multiple clients should not affect overall rate"""
        # 1 rps x 100 simulated seconds => 100 increments
        # Higher concurrency has more variance
        self._test_fixed_rate(desired_agent_rate=1.0,
                              experiment_duration=10000,
                              concurrency=3,
                              expected_requests=10000,
                              allowed_variance=(0.8, 1.2))

    def _test_fixed_rate(self, desired_agent_rate, experiment_duration,
                         concurrency, expected_requests, allowed_variance):
        """A utility pass-through that fixes the initial, upper and lower rates"""
        num_agents = 1000
        cluster_rate = num_agents * desired_agent_rate
        self._test_rate_limiter(num_agents=num_agents,
                                initial_cluster_rate=cluster_rate,
                                max_cluster_rate=cluster_rate,
                                min_cluster_rate=cluster_rate,
                                consecutive_success_threshold=5,
                                experiment_duration=experiment_duration,
                                max_concurrency=concurrency,
                                expected_requests=expected_requests,
                                allowed_variance=allowed_variance)

    def test_variable_rate_single_concurrency_all_successes(self):
        """Rate should quickly max out at max_cluster_rate"""

        # Test variables
        initial_agent_rate = 1
        experiment_duration = 1000
        concurrency = 1
        max_rate_multiplier = 10

        # Expected behavior (saturates at upper rate given all successes)
        expected_requests = max_rate_multiplier * experiment_duration
        allowed_variance = (0.8, 1.2)

        # Derived values
        num_agents = 10
        initial_cluster_rate = num_agents * initial_agent_rate
        max_cluster_rate = max_rate_multiplier * initial_cluster_rate

        self._test_rate_limiter(
            num_agents=10,
            initial_cluster_rate=initial_cluster_rate,
            max_cluster_rate=max_cluster_rate,
            min_cluster_rate=0,
            experiment_duration=experiment_duration,
            max_concurrency=concurrency,
            consecutive_success_threshold=5,
            increase_strategy=BlockingRateLimiter.STRATEGY_RESET_THEN_MULTIPLY,
            expected_requests=expected_requests,
            allowed_variance=allowed_variance,
        )

    def test_variable_rate_single_concurrency_all_failures(self):
        """Rate should quickly decrease to min_cluster_rate"""

        # temporarily disable hard lower limit as it is irrelevant to this test and we want to let it go down to really
        # low numbers to gather enough data.
        orig_hard_limit = BlockingRateLimiter.HARD_LIMIT_MIN_CLUSTER_RATE
        BlockingRateLimiter.HARD_LIMIT_MIN_CLUSTER_RATE = -float('inf')

        # Test variables
        initial_agent_rate = 1
        experiment_duration = 1000000
        concurrency = 1
        min_rate_multiplier = 0.01  # min rate should be at least an order of magnitude lower than initial rate.

        # Expected behavior
        expected_requests = min_rate_multiplier * experiment_duration
        allowed_variance = (0.8, 1.2)

        # Derived values
        num_agents = 10
        initial_cluster_rate = num_agents * initial_agent_rate
        max_cluster_rate = initial_cluster_rate
        min_cluster_rate = min_rate_multiplier * initial_cluster_rate

        self._test_rate_limiter(
            num_agents=num_agents,
            initial_cluster_rate=initial_cluster_rate,
            max_cluster_rate=max_cluster_rate,
            min_cluster_rate=min_cluster_rate,
            experiment_duration=experiment_duration,
            max_concurrency=concurrency,
            consecutive_success_threshold=5,
            increase_strategy=BlockingRateLimiter.STRATEGY_RESET_THEN_MULTIPLY,
            expected_requests=expected_requests,
            allowed_variance=allowed_variance,
            reported_outcome_generator=always_false(),
        )

        # Restore the min hard limit
        BlockingRateLimiter.HARD_LIMIT_MIN_CLUSTER_RATE = orig_hard_limit

    def test_variable_rate_single_concurrency_push_pull(self):
        """Rate should fluctuate closely around initial rate because of equal successes/failures"""

        # Test variables
        initial_agent_rate = 1
        experiment_duration = 10000
        concurrency = 1
        min_rate_multiplier = 0.1
        max_rate_multiplier = 10
        consecutive_success_threshold = 5
        backoff_factor = 0.5
        increase_factor = 2.0

        # Expected behavior
        # 1 * 1 rps * 100s
        expected_requests = 1 * experiment_duration
        allowed_variance = (
            0.8, 1.2
        )  # variance increases as difference between backoffs increase

        # Derived values
        num_agents = 100
        initial_cluster_rate = num_agents * initial_agent_rate

        self._test_rate_limiter(
            num_agents=num_agents,
            initial_cluster_rate=initial_cluster_rate,
            max_cluster_rate=max_rate_multiplier * initial_cluster_rate,
            min_cluster_rate=min_rate_multiplier * initial_cluster_rate,
            experiment_duration=experiment_duration,
            max_concurrency=concurrency,
            consecutive_success_threshold=consecutive_success_threshold,
            increase_strategy=BlockingRateLimiter.STRATEGY_RESET_THEN_MULTIPLY,
            expected_requests=expected_requests,
            allowed_variance=allowed_variance,
            reported_outcome_generator=rate_maintainer(
                consecutive_success_threshold, backoff_factor,
                increase_factor),
            backoff_factor=backoff_factor,
            increase_factor=increase_factor,
        )

    def _test_rate_limiter(
        self,
        num_agents,
        consecutive_success_threshold,
        initial_cluster_rate,
        max_cluster_rate,
        min_cluster_rate,
        experiment_duration,
        max_concurrency,
        expected_requests,
        allowed_variance,
        reported_outcome_generator=always_true(),
        increase_strategy=BlockingRateLimiter.STRATEGY_MULTIPLY,
        backoff_factor=0.5,
        increase_factor=2.0,
    ):
        """Main test logic that runs max_concurrency client threads for a defined experiment duration.

        The experiment is driven off a fake_clock (so it can complete in seconds, not minutes or hours).

        Each time a successful acquire() completes, a counter is incremented.  At experiment end, this counter
        should be close enough to a calculated expected value, based on the specified rate.

        Concurrency should not affect the overall rate of allowed acquisitions.

        The reported outcome by acquiring clients is determined by invoking the callable `reported_outcome_callable`.


        @param num_agents: Num agents in cluster (to derive agent rate from cluster rate)
        @param consecutive_success_threshold:
        @param initial_cluster_rate: Initial cluster rate
        @param max_cluster_rate:  Upper bound on cluster rate
        @param min_cluster_rate: Lower bound on cluster rate
        @param increase_strategy: Strategy for increasing rate
        @param experiment_duration: Experiment duration in seconds
        @param max_concurrency: Number of tokens to create
        @param expected_requests: Expected number of requests at the end of experiment
        @param allowed_variance: Allowed variance between expected and actual number of requests. (e.g. 0.1 = 10%)
        @param reported_outcome_generator: Generator to get reported outcome boolean value
        @param fake_clock_increment: Fake clock increment by (seconds)
        """
        rate_limiter = BlockingRateLimiter(
            num_agents=num_agents,
            initial_cluster_rate=initial_cluster_rate,
            max_cluster_rate=max_cluster_rate,
            min_cluster_rate=min_cluster_rate,
            consecutive_success_threshold=consecutive_success_threshold,
            strategy=increase_strategy,
            increase_factor=increase_factor,
            backoff_factor=backoff_factor,
            max_concurrency=max_concurrency,
            fake_clock=self._fake_clock,
        )

        # Create and start a list of consumer threads
        experiment_end_time = self._fake_clock.time() + experiment_duration
        threads = self.__create_consumer_threads(max_concurrency, rate_limiter,
                                                 experiment_end_time,
                                                 reported_outcome_generator)
        [t.setDaemon(True) for t in threads]
        [t.start() for t in threads]

        # Create and join and advancer thread (which in turn lasts until all client threads die
        advancer = self.__create_fake_clock_advancer_thread(
            rate_limiter, threads)
        advancer.start()
        advancer.join()

        requests = self._test_state['count']
        # Assert that count is close enough to the expected count
        observed_ratio = float(requests) / expected_requests
        self.assertGreater(observed_ratio, allowed_variance[0])
        self.assertLess(observed_ratio, allowed_variance[1])
Ejemplo n.º 2
0
class Test_K8sCache(ScalyrTestCase):
    """ Tests the _K8sCache
    """

    NAMESPACE_1 = "namespace_1"
    POD_1 = "pod_1"

    class DummyObject(object):
        def __init__(self, access_time):
            self.access_time = access_time

    def setUp(self):
        super(Test_K8sCache, self).setUp()
        self.k8s = FakeK8s()
        self.clock = FakeClock()
        self.processor = FakeProcessor()
        self.cache = _K8sCache(self.processor, "foo")

    def tearDown(self):
        self.k8s.stop()

    def test_purge_expired(self):

        processor = Mock()
        cache = _K8sCache(processor, "foo")

        current_time = time.time()
        obj1 = self.DummyObject(current_time - 10)
        obj2 = self.DummyObject(current_time + 15)
        obj3 = self.DummyObject(current_time - 20)

        objects = {"default": {"obj1": obj1, "obj2": obj2, "obj3": obj3}}

        # we should probably look at using actual values returned from k8s here
        # and loading them via 'cache.update'
        cache._objects = objects

        cache.purge_unused(current_time)

        objects = cache._objects.get("default", {})
        self.assertEquals(1, len(objects))
        self.assertTrue("obj2" in objects)
        self.assertTrue(objects["obj2"] is obj2)

    def test_lookup_not_in_cache(self):

        self.k8s.set_response(self.NAMESPACE_1, self.POD_1, success=True)

        self.assertFalse(
            self.cache.is_cached(self.NAMESPACE_1,
                                 self.POD_1,
                                 allow_expired=True))

        obj = self.cache.lookup(self.k8s, self.clock.time(), self.NAMESPACE_1,
                                self.POD_1)

        self.assertTrue(
            self.cache.is_cached(self.NAMESPACE_1,
                                 self.POD_1,
                                 allow_expired=True))
        self.assertEqual(obj.name, self.POD_1)
        self.assertEqual(obj.namespace, self.NAMESPACE_1)

    def test_lookup_already_in_cache(self):
        query_options = ApiQueryOptions()

        self.k8s.set_response(self.NAMESPACE_1, self.POD_1, success=True)
        obj = self.cache.lookup(
            self.k8s,
            self.clock.time(),
            self.NAMESPACE_1,
            self.POD_1,
            query_options=query_options,
        )
        self.assertTrue(
            self.cache.is_cached(self.NAMESPACE_1,
                                 self.POD_1,
                                 allow_expired=True))

        self.k8s.set_response(self.NAMESPACE_1,
                              self.POD_1,
                              permanent_error=True)
        obj = self.cache.lookup(
            self.k8s,
            self.clock.time(),
            self.NAMESPACE_1,
            self.POD_1,
            query_options=query_options,
        )

        self.assertTrue(
            self.cache.is_cached(self.NAMESPACE_1,
                                 self.POD_1,
                                 allow_expired=True))
        self.assertEqual(obj.name, self.POD_1)
        self.assertEqual(obj.namespace, self.NAMESPACE_1)

    def test_raise_exception_on_query_error(self):
        query_options = ApiQueryOptions()

        self.k8s.set_response(self.NAMESPACE_1,
                              self.POD_1,
                              permanent_error=True)
        self.assertRaises(
            K8sApiPermanentError,
            lambda: self.cache.lookup(
                self.k8s,
                self.clock.time(),
                self.NAMESPACE_1,
                self.POD_1,
                query_options=query_options,
            ),
        )

    def test_return_none_on_query_error_without_options(self):

        self.k8s.set_response(self.NAMESPACE_1,
                              self.POD_1,
                              permanent_error=True)
        obj = self.cache.lookup(
            self.k8s,
            self.clock.time(),
            self.NAMESPACE_1,
            self.POD_1,
            ignore_k8s_api_exception=True,
        )
        self.assertIsNone(obj)