def test_condition_init_ok():
"""Test initializing a Condition with no errors."""
c = Condition('test', lambda x: x == 'foo', 'foo')
assert c.fn is not None
assert c.args == ('foo', )
assert c.kwargs == {}
def wait_until_created(
obj: objects.ApiObject,
timeout: int = None,
interval: Union[int, float] = 1,
) -> None:
"""Wait until the specified object has been created.
Here, creation is judged on whether or not refreshing the object (e.g.
getting it) returns an object (created) or an error (not yet created).
Args:
obj: The ApiObject to wait on.
timeout: The maximum time to wait, in seconds.
interval: The time, in seconds, to sleep before re-checking the
created state of the object.
"""
def check_ready(api_obj):
try:
api_obj.refresh()
except: # noqa
return False
return True
wait_condition = Condition(
f"wait for {type(obj).__name__}:{obj.name} to be created",
check_ready,
obj,
)
utils.wait_for_condition(
condition=wait_condition, timeout=timeout, interval=interval
)
def wait_for_registered(
self, timeout: int = None, interval: Union[int, float] = 1
) -> None:
"""Wait for all of the pre-registered objects to be ready on the cluster.
An object is pre-registered with the test client if it is specified
to the test via the ``applymanifests`` pytest marker. The marker will load
the manifest and add the object to the cluster, and register it with
the test client. This method waits until all such loaded manifest objects
are in the ready state simultaneously.
Args:
timeout: The maximum time to wait, in seconds.
interval: The time, in seconds, to sleep before re-checking the ready
state for pre-registered objects.
"""
def check_registered():
for obj in self.pre_registered:
if not obj.is_ready():
return False
return True
wait_condition = Condition(
"wait for pre-registered objects to be ready",
check_registered,
)
utils.wait_for_condition(
condition=wait_condition,
timeout=timeout,
interval=interval,
)
def wait_for_ready_nodes(
self,
count: int,
timeout: int = None,
interval: Union[int, float] = 1,
) -> None:
"""Wait until there are at least ``count`` number of nodes available
in the cluster.
Notes:
This should only be used for clusters that auto-scale the
nodes. This will not create/delete nodes on its own.
Args:
count: The number of nodes to wait for.
timeout: The maximum time to wait, in seconds.
interval: The time, in seconds, to sleep before re-checking the
number of nodes.
"""
def node_count_match(node_count):
nodes = self.get_nodes()
return [n.is_ready() for n in nodes.values()].count(True) >= node_count
wait_condition = Condition(
f"wait for {count} nodes",
node_count_match,
count,
)
utils.wait_for_condition(
condition=wait_condition,
timeout=timeout,
interval=interval,
)
def wait_for_condition(
condition: Condition,
timeout: int = None,
interval: Union[int, float] = 1,
fail_on_api_error: bool = True,
) -> None:
"""Wait for a condition to be met.
Args:
condition: The Condition to wait for.
timeout: The maximum time to wait, in seconds, for the condition to be met.
If unspecified, this function will wait indefinitely. If specified and
the timeout is met or exceeded, a TimeoutError will be raised.
interval: The time, in seconds, to wait before re-checking the condition.
fail_on_api_error: Fail the condition checks if a Kubernetes API error is
incurred. An API error can be raised for a number of reasons, including
a Pod being restarted and temporarily unavailable. Disabling this will
cause those errors to be ignored, allowing the check to continue until
timeout or resolution. (default: True).
Raises:
TimeoutError: The specified timeout was exceeded.
"""
log.info(f"waiting for condition: {condition}")
# define the maximum time to wait. once this is met, we should
# stop waiting.
max_time = None
if timeout is not None:
max_time = time.time() + timeout
# start the wait block
start = time.time()
while True:
if max_time and time.time() >= max_time:
raise TimeoutError(
f"timed out ({timeout}s) while waiting for condition {condition}"
)
# check if the condition is met and break out if it is
try:
if condition.check():
break
except ApiException as e:
log.warning(f"got api exception while waiting: {e}")
if fail_on_api_error:
raise
# if the condition is not met, sleep for the interval
# to re-check later
time.sleep(interval)
end = time.time()
log.info(f"wait completed (total={end-start}s) {condition}")
def wait_for_conditions(
*args: Condition,
timeout: int = None,
interval: Union[float, int] = 1,
policy: Policy = Policy.ONCE,
fail_on_api_error: bool = True,
) -> None:
"""Wait for all of the provided Conditions to be met.
All Conditions must be met for this to unblock. If no Conditions are
provided, this method will do nothing.
Args:
*args: Conditions to check.
timeout: The maximum time to wait, in seconds, for the provided
Conditions to be met. If all of the Conditions are not met within
the given timeout, this will raise a TimeoutError. By default,
there is no timeout so this will wait indefinitely.
interval: The time, in seconds, to sleep before re-evaluating the
conditions. Default: 1s
policy: The condition checking policy that defines the checking
behavior. Default: ONCE
fail_on_api_error: Fail the condition checks if a Kubernetes API error
is incurred. An API error can be raised for a number of reasons,
including a Pod being restarted and temporarily unavailable.
Disabling this will cause those errors to be ignored, allowing
the check to continue until timeout or resolution. (default: True).
Raises:
TimeoutError: The Conditions were not met within the specified
timeout period.
ValueError: Not all arguments are a Condition.
"""
# If no Conditions were given, there is nothing to do.
if not args:
return
# If something was given, make sure they are all Conditions
if not all(map(lambda c: isinstance(c, Condition), args)):
raise ValueError('All arguments must be a Condition')
# make a copy of the conditions
to_check = list(args)
def condition_checker(conditions):
# check that the conditions were met according to the
# condition checking policy
met, unmet = check_and_sort(*conditions)
if policy == Policy.ONCE:
log.info(f'check met: {met}')
conditions[:] = unmet
return len(unmet) == 0
elif policy == Policy.SIMULTANEOUS:
return len(unmet) == 0 and len(met) == len(args)
else:
raise ValueError(
f'Invalid condition policy specified: {policy}',
)
wait_condition = Condition(
'wait for conditions',
condition_checker,
to_check,
)
try:
utils.wait_for_condition(
condition=wait_condition,
timeout=timeout,
interval=interval,
fail_on_api_error=fail_on_api_error,
)
except TimeoutError:
# If we time out here, we want to show all the conditions
# that we weren't able to resolve in the error message, not
# the 'wait for conditions' wrapper.
raise TimeoutError(
f'timed out wile waiting for conditions to be met: {to_check}',
)
def test_condition_check_false(fn, args, kwargs):
"""Test checking when the function returns Falsey values."""
c = Condition("test", fn, *args, **kwargs)
assert not c.check()
def test_condition_check_true(fn, args, kwargs):
"""Test checking when the function returns Truthy values."""
c = Condition("test", fn, *args, **kwargs)
assert c.check()
def test_condition_init_err():
"""Test initializing a Condition where the provided fn is not callable."""
with pytest.raises(ValueError):
Condition("test", "not-callable")
"x": {}
}),
],
)
def test_condition_check_false(fn, args, kwargs):
"""Test checking when the function returns Falsey values."""
c = Condition("test", fn, *args, **kwargs)
assert not c.check()
@pytest.mark.parametrize(
"conditions,expected",
[
([], True),
([Condition("test", lambda: True)], True),
([Condition("test", lambda: False)], False),
([Condition("test", lambda: True),
Condition("test", lambda: True)], True),
([Condition("test", lambda: True),
Condition("test", lambda: False)], False),
([Condition("test", lambda: False),
Condition("test", lambda: False)], False),
],
)
def test_check_all(conditions, expected):
"""Test checking all conditions."""
actual = check_all(*conditions)
assert expected == actual
