class BlockingRateLimiterTest(ScalyrTestCase):
def setUp(self):
super(BlockingRateLimiterTest, self).setUp()
self._fake_clock = FakeClock()
self._test_state = {
'count': 0,
'times': [],
}
self._test_state_lock = threading.Lock()
self._outcome_generator_lock = threading.Lock()
def test_init_exceptions(self):
BRL = BlockingRateLimiter
# strategy must be restricted to proper values
self.assertRaises(
ValueError, lambda: BRL(
num_agents=1,
initial_cluster_rate=100,
max_cluster_rate=1000,
min_cluster_rate=1,
consecutive_success_threshold=5,
strategy='bad strategy',
))
# max_concurrency must be 1 or more
self.assertRaises(
ValueError, lambda: BRL(
num_agents=1,
initial_cluster_rate=100,
max_cluster_rate=1000,
min_cluster_rate=1,
consecutive_success_threshold=5,
max_concurrency=0,
))
# consecutive_success_threshold must be a positive int
self.assertRaises(
ValueError, lambda: BRL(
num_agents=1,
initial_cluster_rate=100,
max_cluster_rate=1000,
min_cluster_rate=1,
consecutive_success_threshold=-5.0,
))
self.assertRaises(
ValueError, lambda: BRL(
num_agents=1,
initial_cluster_rate=100,
max_cluster_rate=1000,
min_cluster_rate=1,
consecutive_success_threshold=99.99,
))
# user-supplied init must be between user-supplied min/max
self.assertRaises(
ValueError, lambda: BRL(
num_agents=1,
initial_cluster_rate=0.7,
max_cluster_rate=1000,
min_cluster_rate=1,
consecutive_success_threshold=5,
))
self.assertRaises(
ValueError, lambda: BRL(
num_agents=1,
initial_cluster_rate=7777,
max_cluster_rate=1000,
min_cluster_rate=1,
consecutive_success_threshold=5,
))
def test_lazy_adjust_min_max_init_rates(self):
"""Tests the one-time lazy adjustments"""
# Make sure proper adjustments are made if out of bounds
BRL = BlockingRateLimiter
rl = BRL(
num_agents=1,
initial_cluster_rate=BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE + 1000,
max_cluster_rate=BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE + 2000000,
min_cluster_rate=float(BRL.HARD_LIMIT_MIN_CLUSTER_RATE) / 10,
consecutive_success_threshold=5,
)
rl._lazy_adjust_min_max_rates()
self.assertEqual(rl._initial_cluster_rate,
BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE)
self.assertEqual(rl._max_cluster_rate,
BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE + 2000000)
self.assertEqual(rl._min_cluster_rate, BRL.HARD_LIMIT_MIN_CLUSTER_RATE)
# Make sure adjustments are NOT made if within bounds
rl = BRL(
num_agents=1,
initial_cluster_rate=BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE - 1,
max_cluster_rate=BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE - 1,
min_cluster_rate=float(BRL.HARD_LIMIT_MIN_CLUSTER_RATE) + 1,
consecutive_success_threshold=5,
)
rl._lazy_adjust_min_max_rates()
self.assertEqual(rl._initial_cluster_rate,
BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE - 1)
self.assertEqual(rl._max_cluster_rate,
BRL.HARD_LIMIT_INITIAL_CLUSTER_RATE + -1)
self.assertEqual(rl._min_cluster_rate,
BRL.HARD_LIMIT_MIN_CLUSTER_RATE + 1)
# Init rate cannot be lower than min rate
rl = BlockingRateLimiter(
num_agents=1,
initial_cluster_rate=0.1,
max_cluster_rate=1000,
min_cluster_rate=0,
consecutive_success_threshold=5,
)
rl._lazy_adjust_min_max_rates()
self.assertEqual(rl._initial_cluster_rate, 1)
def __create_consumer_threads(self, num_threads, rate_limiter,
experiment_end_time,
reported_outcome_generator):
"""
Create a list of client thread that will consume from the rate limiter.
@param num_threads: Number of client threads
@param rate_limiter: The rate limiter to test
@param experiment_end_time: consumer threads terminate their loops when the fake clock has exceeded this.
@param reported_outcome_generator: Generator to get reported outcome boolean value
@type rate_limiter: BlockingRateLimiter
@type threads: iterable of Thread objects
@type experiment_end_time: int (utc seconds)
@type reported_outcome_generator: generator
@returns: list of consumer threads
@rtype: list(Thread)
"""
def consume_token_blocking():
"""consumer threads will keep acquiring tokens until experiment end time"""
while self._fake_clock.time() < experiment_end_time:
# A simple loop that acquires token, updates a counter, then releases token with an outcome
# provided by reported_outcome_generator()
t1 = self._fake_clock.time()
token = rate_limiter.acquire_token()
# update test state
self._test_state_lock.acquire()
try:
self._test_state['count'] += 1
self._test_state['times'].append(int(t1))
finally:
self._test_state_lock.release()
self._outcome_generator_lock.acquire()
try:
outcome = reported_outcome_generator.next()
rate_limiter.release_token(token, outcome)
finally:
self._outcome_generator_lock.release()
return [
threading.Thread(target=consume_token_blocking)
for _ in range(num_threads)
]
def __create_fake_clock_advancer_thread(self, rate_limiter,
consumer_threads):
"""
Run a separate thread that advances time in order to allow rate_limiter to release tokens
@param rate_limiter: The rate limiter to test
@param consumer_threads: Other threads that are consumers of the Rate Limiter. The advancer thread will run
for as long as the consumer threads continue to run.
@type rate_limiter: BlockingRateLimiter
@type consumer_threads: iterable of Thread objects
@returns: Thread object
@rtype: Thread
"""
class FakeClockAdvancerThread(StoppableThread):
def __init__(self2):
StoppableThread.__init__(self2,
name='FakeClockAdvancerThread',
is_daemon=True)
def run_and_propagate(self2):
at_least_one_client_thread_incomplete = True
while at_least_one_client_thread_incomplete:
# All client threads are likely blocking on the token queue at this point
# The experiment will never start until at least one token is granted (and waiting threads notified)
# To jump start this, simply advance fake clock to the ripe_time.
delta = rate_limiter._ripe_time - self._fake_clock.time()
self._fake_clock.advance_time(increment_by=delta)
# Wake up after 1 second just in case and trigger another advance
[t.join(1) for t in consumer_threads]
at_least_one_client_thread_incomplete = False
for t in consumer_threads:
if t.isAlive():
at_least_one_client_thread_incomplete = True
break
return FakeClockAdvancerThread()
def test_rate_adjustment_logic(self):
"""Make sure rate adjustments work according to spec
The cases we test are:
1. All successes. Actual rate is very high. New target rate is always increase_factor * current target rate.
2. All successes. Actual rate is very low. New target rate is always increase_factor * actual rate,
(but no lower than current rate).
3. All failures. Actual rate is very low. New target rate is always backoff_factor * current target rate.
4. All failures. Actual rate is very high. New target rate is always backoff_factor * actual rate.
(but no higher than current rate).
"""
required_successes = 5
max_cluster_rate = 10000
min_cluster_rate = 1
initial_cluster_rate = 100
increase_factor = 2.0
backoff_factor = 0.5
def _create_rate_limiter():
return BlockingRateLimiter(
num_agents=1,
initial_cluster_rate=initial_cluster_rate,
max_cluster_rate=max_cluster_rate,
min_cluster_rate=min_cluster_rate,
consecutive_success_threshold=required_successes,
strategy=BlockingRateLimiter.STRATEGY_MULTIPLY,
increase_factor=increase_factor,
backoff_factor=backoff_factor,
max_concurrency=1,
fake_clock=self._fake_clock)
def _mock_get_next_ripe_time_actual_rate_leads(rate_limiter):
# Make the rate limiter grant tokens faster than the target rate
def _func(*args, **kwargs):
delta = 1.0 / rate_limiter._current_cluster_rate
return rl._ripe_time + 0.9 * delta
return _func
def _mock_get_next_ripe_time_actual_rate_lags(rate_limiter):
# Make the rate limiter grant tokens slower than the target rate
def _func(*args, **kwargs):
delta = 1.0 / rate_limiter._current_cluster_rate
return rl._ripe_time + 1.1 * delta
return _func
rl = _create_rate_limiter()
@patch.object(rl, '_get_next_ripe_time')
def _case1_test_successes_actual_rate_leads_target_rate(
mock_get_next_ripe_time):
"""Examine rate-increasing behavior in the context of very high actual rates"""
mock_get_next_ripe_time.side_effect = _mock_get_next_ripe_time_actual_rate_leads(
rl)
advancer = self.__create_fake_clock_advancer_thread(
rl, [threading.currentThread()])
advancer.start()
while True:
token = rl.acquire_token()
for x in range(required_successes - 1):
rl.release_token(token, True)
rl.acquire_token()
# Actual rate always leads target rate
old_target_rate = rl._current_cluster_rate
old_actual_rate = rl._get_actual_cluster_rate()
self.assertGreater(old_actual_rate, old_target_rate)
# Token grant causes new rate to be calculated
rl.release_token(token, True)
# assert that the new rate is calculated based on the (lower/more conservative) target rate
if increase_factor * old_target_rate < max_cluster_rate:
self.assertEqual(rl._current_cluster_rate,
increase_factor * old_target_rate)
else:
# assert that new rate never exceeds max rate
self.assertEqual(rl._current_cluster_rate,
max_cluster_rate)
break
advancer.stop(wait_on_join=False)
_case1_test_successes_actual_rate_leads_target_rate()
rl = _create_rate_limiter()
@patch.object(rl, '_get_next_ripe_time')
def _case2_test_successes_actual_rate_lags_target_rate(
mock_get_next_ripe_time):
"""Examine rate-increasing behavior in the context of very high actual rates"""
mock_get_next_ripe_time.side_effect = _mock_get_next_ripe_time_actual_rate_lags(
rl)
advancer = self.__create_fake_clock_advancer_thread(
rl, [threading.currentThread()])
advancer.start()
while True:
token = rl.acquire_token()
for x in range(required_successes - 1):
rl.release_token(token, True)
rl.acquire_token()
# Actual rate always lags target rate
old_target_rate = rl._current_cluster_rate
old_actual_rate = rl._get_actual_cluster_rate()
self.assertLess(old_actual_rate, old_target_rate)
# Token grant causes new rate to be calculated
rl.release_token(token, True)
# assert that the new rate is calculated based on the (lower/more conservative) actual rate
if increase_factor * old_actual_rate < max_cluster_rate:
if increase_factor * old_actual_rate < old_target_rate:
self.assertEqual(round(rl._current_cluster_rate, 2),
round(old_target_rate, 2))
else:
self.assertEqual(
round(rl._current_cluster_rate, 2),
round(increase_factor * old_actual_rate, 2))
else:
# assert that new rate never exceeds max rate
self.assertEqual(rl._current_cluster_rate,
max_cluster_rate)
break
advancer.stop(wait_on_join=False)
_case2_test_successes_actual_rate_lags_target_rate()
rl = _create_rate_limiter()
@patch.object(rl, '_get_next_ripe_time')
def _case3_test_failures_actual_rate_lags_target_rate(
mock_get_next_ripe_time):
"""Examine rate-decreasing behavior in the context of very high actual rates"""
mock_get_next_ripe_time.side_effect = _mock_get_next_ripe_time_actual_rate_lags(
rl)
advancer = self.__create_fake_clock_advancer_thread(
rl, [threading.currentThread()])
advancer.start()
counter = 0
while True:
token = rl.acquire_token()
# Actual rate always lags target rate
old_target_rate = rl._current_cluster_rate
old_actual_rate = rl._get_actual_cluster_rate()
if old_actual_rate is not None:
self.assertLess(old_actual_rate, old_target_rate)
# Token grant a 100 initial successes followed by all failures
counter += 1
if counter <= required_successes:
rl.release_token(token, True)
continue
else:
rl.release_token(token, False)
# assert that the new rate is calculated based on the (higher/more conservative) target rate
if backoff_factor * old_target_rate > min_cluster_rate:
self.assertEqual(
round(rl._current_cluster_rate, 2),
round(backoff_factor * old_target_rate, 2))
else:
# assert that new rate never goes lower than min rate
self.assertEqual(round(rl._current_cluster_rate, 2),
round(min_cluster_rate, 2))
break
advancer.stop(wait_on_join=False)
_case3_test_failures_actual_rate_lags_target_rate()
rl = _create_rate_limiter()
@patch.object(rl, '_get_next_ripe_time')
def _case4_test_failures_actual_rate_leads_target_rate(
mock_get_next_ripe_time):
"""Examine rate-decreasing behavior in the context of very high actual rates"""
mock_get_next_ripe_time.side_effect = _mock_get_next_ripe_time_actual_rate_leads(
rl)
advancer = self.__create_fake_clock_advancer_thread(
rl, [threading.currentThread()])
advancer.start()
counter = 0
while True:
token = rl.acquire_token()
# Actual rate is always None because
old_target_rate = rl._current_cluster_rate
old_actual_rate = rl._get_actual_cluster_rate()
if old_actual_rate is not None:
self.assertGreater(old_actual_rate, old_target_rate)
# Token grant a 100 initial successes followed by all failures
counter += 1
if counter <= required_successes:
rl.release_token(token, True)
continue
else:
rl.release_token(token, False)
# assert that the new rate is calculated based on the (higher/more conservative) actual rate
if backoff_factor * old_target_rate > min_cluster_rate:
self.assertEqual(
round(rl._current_cluster_rate, 2),
round(backoff_factor * old_target_rate, 2))
else:
# assert that new rate never goes lower than min rate
self.assertEqual(rl._current_cluster_rate,
min_cluster_rate)
break
advancer.stop(wait_on_join=False)
_case4_test_failures_actual_rate_leads_target_rate()
def test_fixed_rate_single_concurrency(self):
"""Longer experiment"""
# 1 rps x 1000 simulated seconds => 1000 increments
self._test_fixed_rate(desired_agent_rate=1.0,
experiment_duration=10000,
concurrency=1,
expected_requests=10000,
allowed_variance=(0.95, 1.05))
def test_fixed_rate_multiple_concurrency(self):
"""Multiple clients should not affect overall rate"""
# 1 rps x 100 simulated seconds => 100 increments
# Higher concurrency has more variance
self._test_fixed_rate(desired_agent_rate=1.0,
experiment_duration=10000,
concurrency=3,
expected_requests=10000,
allowed_variance=(0.8, 1.2))
def _test_fixed_rate(self, desired_agent_rate, experiment_duration,
concurrency, expected_requests, allowed_variance):
"""A utility pass-through that fixes the initial, upper and lower rates"""
num_agents = 1000
cluster_rate = num_agents * desired_agent_rate
self._test_rate_limiter(num_agents=num_agents,
initial_cluster_rate=cluster_rate,
max_cluster_rate=cluster_rate,
min_cluster_rate=cluster_rate,
consecutive_success_threshold=5,
experiment_duration=experiment_duration,
max_concurrency=concurrency,
expected_requests=expected_requests,
allowed_variance=allowed_variance)
def test_variable_rate_single_concurrency_all_successes(self):
"""Rate should quickly max out at max_cluster_rate"""
# Test variables
initial_agent_rate = 1
experiment_duration = 1000
concurrency = 1
max_rate_multiplier = 10
# Expected behavior (saturates at upper rate given all successes)
expected_requests = max_rate_multiplier * experiment_duration
allowed_variance = (0.8, 1.2)
# Derived values
num_agents = 10
initial_cluster_rate = num_agents * initial_agent_rate
max_cluster_rate = max_rate_multiplier * initial_cluster_rate
self._test_rate_limiter(
num_agents=10,
initial_cluster_rate=initial_cluster_rate,
max_cluster_rate=max_cluster_rate,
min_cluster_rate=0,
experiment_duration=experiment_duration,
max_concurrency=concurrency,
consecutive_success_threshold=5,
increase_strategy=BlockingRateLimiter.STRATEGY_RESET_THEN_MULTIPLY,
expected_requests=expected_requests,
allowed_variance=allowed_variance,
)
def test_variable_rate_single_concurrency_all_failures(self):
"""Rate should quickly decrease to min_cluster_rate"""
# temporarily disable hard lower limit as it is irrelevant to this test and we want to let it go down to really
# low numbers to gather enough data.
orig_hard_limit = BlockingRateLimiter.HARD_LIMIT_MIN_CLUSTER_RATE
BlockingRateLimiter.HARD_LIMIT_MIN_CLUSTER_RATE = -float('inf')
# Test variables
initial_agent_rate = 1
experiment_duration = 1000000
concurrency = 1
min_rate_multiplier = 0.01 # min rate should be at least an order of magnitude lower than initial rate.
# Expected behavior
expected_requests = min_rate_multiplier * experiment_duration
allowed_variance = (0.8, 1.2)
# Derived values
num_agents = 10
initial_cluster_rate = num_agents * initial_agent_rate
max_cluster_rate = initial_cluster_rate
min_cluster_rate = min_rate_multiplier * initial_cluster_rate
self._test_rate_limiter(
num_agents=num_agents,
initial_cluster_rate=initial_cluster_rate,
max_cluster_rate=max_cluster_rate,
min_cluster_rate=min_cluster_rate,
experiment_duration=experiment_duration,
max_concurrency=concurrency,
consecutive_success_threshold=5,
increase_strategy=BlockingRateLimiter.STRATEGY_RESET_THEN_MULTIPLY,
expected_requests=expected_requests,
allowed_variance=allowed_variance,
reported_outcome_generator=always_false(),
)
# Restore the min hard limit
BlockingRateLimiter.HARD_LIMIT_MIN_CLUSTER_RATE = orig_hard_limit
def test_variable_rate_single_concurrency_push_pull(self):
"""Rate should fluctuate closely around initial rate because of equal successes/failures"""
# Test variables
initial_agent_rate = 1
experiment_duration = 10000
concurrency = 1
min_rate_multiplier = 0.1
max_rate_multiplier = 10
consecutive_success_threshold = 5
backoff_factor = 0.5
increase_factor = 2.0
# Expected behavior
# 1 * 1 rps * 100s
expected_requests = 1 * experiment_duration
allowed_variance = (
0.8, 1.2
) # variance increases as difference between backoffs increase
# Derived values
num_agents = 100
initial_cluster_rate = num_agents * initial_agent_rate
self._test_rate_limiter(
num_agents=num_agents,
initial_cluster_rate=initial_cluster_rate,
max_cluster_rate=max_rate_multiplier * initial_cluster_rate,
min_cluster_rate=min_rate_multiplier * initial_cluster_rate,
experiment_duration=experiment_duration,
max_concurrency=concurrency,
consecutive_success_threshold=consecutive_success_threshold,
increase_strategy=BlockingRateLimiter.STRATEGY_RESET_THEN_MULTIPLY,
expected_requests=expected_requests,
allowed_variance=allowed_variance,
reported_outcome_generator=rate_maintainer(
consecutive_success_threshold, backoff_factor,
increase_factor),
backoff_factor=backoff_factor,
increase_factor=increase_factor,
)
def _test_rate_limiter(
self,
num_agents,
consecutive_success_threshold,
initial_cluster_rate,
max_cluster_rate,
min_cluster_rate,
experiment_duration,
max_concurrency,
expected_requests,
allowed_variance,
reported_outcome_generator=always_true(),
increase_strategy=BlockingRateLimiter.STRATEGY_MULTIPLY,
backoff_factor=0.5,
increase_factor=2.0,
):
"""Main test logic that runs max_concurrency client threads for a defined experiment duration.
The experiment is driven off a fake_clock (so it can complete in seconds, not minutes or hours).
Each time a successful acquire() completes, a counter is incremented. At experiment end, this counter
should be close enough to a calculated expected value, based on the specified rate.
Concurrency should not affect the overall rate of allowed acquisitions.
The reported outcome by acquiring clients is determined by invoking the callable `reported_outcome_callable`.
@param num_agents: Num agents in cluster (to derive agent rate from cluster rate)
@param consecutive_success_threshold:
@param initial_cluster_rate: Initial cluster rate
@param max_cluster_rate: Upper bound on cluster rate
@param min_cluster_rate: Lower bound on cluster rate
@param increase_strategy: Strategy for increasing rate
@param experiment_duration: Experiment duration in seconds
@param max_concurrency: Number of tokens to create
@param expected_requests: Expected number of requests at the end of experiment
@param allowed_variance: Allowed variance between expected and actual number of requests. (e.g. 0.1 = 10%)
@param reported_outcome_generator: Generator to get reported outcome boolean value
@param fake_clock_increment: Fake clock increment by (seconds)
"""
rate_limiter = BlockingRateLimiter(
num_agents=num_agents,
initial_cluster_rate=initial_cluster_rate,
max_cluster_rate=max_cluster_rate,
min_cluster_rate=min_cluster_rate,
consecutive_success_threshold=consecutive_success_threshold,
strategy=increase_strategy,
increase_factor=increase_factor,
backoff_factor=backoff_factor,
max_concurrency=max_concurrency,
fake_clock=self._fake_clock,
)
# Create and start a list of consumer threads
experiment_end_time = self._fake_clock.time() + experiment_duration
threads = self.__create_consumer_threads(max_concurrency, rate_limiter,
experiment_end_time,
reported_outcome_generator)
[t.setDaemon(True) for t in threads]
[t.start() for t in threads]
# Create and join and advancer thread (which in turn lasts until all client threads die
advancer = self.__create_fake_clock_advancer_thread(
rate_limiter, threads)
advancer.start()
advancer.join()
requests = self._test_state['count']
# Assert that count is close enough to the expected count
observed_ratio = float(requests) / expected_requests
self.assertGreater(observed_ratio, allowed_variance[0])
self.assertLess(observed_ratio, allowed_variance[1])
class Test_K8sCache(ScalyrTestCase):
""" Tests the _K8sCache
"""
NAMESPACE_1 = "namespace_1"
POD_1 = "pod_1"
class DummyObject(object):
def __init__(self, access_time):
self.access_time = access_time
def setUp(self):
super(Test_K8sCache, self).setUp()
self.k8s = FakeK8s()
self.clock = FakeClock()
self.processor = FakeProcessor()
self.cache = _K8sCache(self.processor, "foo")
def tearDown(self):
self.k8s.stop()
def test_purge_expired(self):
processor = Mock()
cache = _K8sCache(processor, "foo")
current_time = time.time()
obj1 = self.DummyObject(current_time - 10)
obj2 = self.DummyObject(current_time + 15)
obj3 = self.DummyObject(current_time - 20)
objects = {"default": {"obj1": obj1, "obj2": obj2, "obj3": obj3}}
# we should probably look at using actual values returned from k8s here
# and loading them via 'cache.update'
cache._objects = objects
cache.purge_unused(current_time)
objects = cache._objects.get("default", {})
self.assertEquals(1, len(objects))
self.assertTrue("obj2" in objects)
self.assertTrue(objects["obj2"] is obj2)
def test_lookup_not_in_cache(self):
self.k8s.set_response(self.NAMESPACE_1, self.POD_1, success=True)
self.assertFalse(
self.cache.is_cached(self.NAMESPACE_1,
self.POD_1,
allow_expired=True))
obj = self.cache.lookup(self.k8s, self.clock.time(), self.NAMESPACE_1,
self.POD_1)
self.assertTrue(
self.cache.is_cached(self.NAMESPACE_1,
self.POD_1,
allow_expired=True))
self.assertEqual(obj.name, self.POD_1)
self.assertEqual(obj.namespace, self.NAMESPACE_1)
def test_lookup_already_in_cache(self):
query_options = ApiQueryOptions()
self.k8s.set_response(self.NAMESPACE_1, self.POD_1, success=True)
obj = self.cache.lookup(
self.k8s,
self.clock.time(),
self.NAMESPACE_1,
self.POD_1,
query_options=query_options,
)
self.assertTrue(
self.cache.is_cached(self.NAMESPACE_1,
self.POD_1,
allow_expired=True))
self.k8s.set_response(self.NAMESPACE_1,
self.POD_1,
permanent_error=True)
obj = self.cache.lookup(
self.k8s,
self.clock.time(),
self.NAMESPACE_1,
self.POD_1,
query_options=query_options,
)
self.assertTrue(
self.cache.is_cached(self.NAMESPACE_1,
self.POD_1,
allow_expired=True))
self.assertEqual(obj.name, self.POD_1)
self.assertEqual(obj.namespace, self.NAMESPACE_1)
def test_raise_exception_on_query_error(self):
query_options = ApiQueryOptions()
self.k8s.set_response(self.NAMESPACE_1,
self.POD_1,
permanent_error=True)
self.assertRaises(
K8sApiPermanentError,
lambda: self.cache.lookup(
self.k8s,
self.clock.time(),
self.NAMESPACE_1,
self.POD_1,
query_options=query_options,
),
)
def test_return_none_on_query_error_without_options(self):
self.k8s.set_response(self.NAMESPACE_1,
self.POD_1,
permanent_error=True)
obj = self.cache.lookup(
self.k8s,
self.clock.time(),
self.NAMESPACE_1,
self.POD_1,
ignore_k8s_api_exception=True,
)
self.assertIsNone(obj)