def wait_for_ready_replica_count_to_reach_expected_value(
machine_set, expected_value, timeout=180):
"""
Wait for the ready replica count to reach an expected value
Args:
machine_set (str): Name of the machine set
expected_value (int): The expected value to reach
timeout (int): Time to wait for the ready replica count to reach the expected value
Return:
bool: True, in case of the ready replica count reached the expected value. False otherwise
"""
ready_replica_count = get_ready_replica_count(machine_set)
log.info(f"ready replica count = {ready_replica_count}")
log.info(
f"Wait {timeout} seconds for the ready replica count to reach the "
f"expected value {expected_value}")
sample = TimeoutSampler(
timeout=timeout,
sleep=10,
func=get_ready_replica_count,
machine_set=machine_set,
)
try:
sample.wait_for_func_value(value=expected_value)
res = True
except TimeoutExpiredError:
log.info(
f"Ready replica count failed to reach the expected value {expected_value}"
)
res = False
return res
def wait_for_replication_resources_deletion(namespace, timeout, check_state=True):
"""
Wait for replication resources to be deleted
Args:
namespace (str): the namespace of the resources'
timeout (int): time in seconds to wait for resources to reach expected
state or deleted
check_state (bool): True for checking resources state before deletion, False otherwise
Raises:
TimeoutExpiredError: In case replication resources not deleted
"""
if check_state:
logger.info("Waiting for all VRs to reach secondary state")
sample = TimeoutSampler(
timeout=timeout,
sleep=5,
func=check_vr_state,
state="secondary",
namespace=namespace,
)
if not sample.wait_for_func_status(result=True):
error_msg = "One or more VR haven't reached expected state secondary within the time limit."
logger.error(error_msg)
raise TimeoutExpiredError(error_msg)
logger.info("Waiting for VRG to reach secondary state")
sample = TimeoutSampler(
timeout=timeout,
sleep=5,
func=check_vrg_state,
state="secondary",
namespace=namespace,
)
if not sample.wait_for_func_status(result=True):
error_msg = (
"VRG hasn't reached expected state secondary within the time limit."
)
logger.info(error_msg)
raise TimeoutExpiredError(error_msg)
logger.info("Waiting for VRG to be deleted")
sample = TimeoutSampler(
timeout=timeout, sleep=5, func=check_vrg_existence, namespace=namespace
)
if not sample.wait_for_func_status(result=False):
error_msg = "VRG resource not deleted"
logger.info(error_msg)
raise TimeoutExpiredError(error_msg)
logger.info("Waiting for all VRs to be deleted")
sample = TimeoutSampler(
timeout=timeout,
sleep=5,
func=get_vr_count,
namespace=namespace,
)
sample.wait_for_func_value(0)
def wait_for_replication_resources_creation(vr_count, namespace, timeout):
"""
Wait for replication resources to be created
Args:
vr_count (int): Expected number of VR resources
namespace (str): the namespace of the VR resources
timeout (int): time in seconds to wait for VR resources to be created
or reach expected state
Raises:
TimeoutExpiredError: In case replication resources not created
"""
logger.info("Waiting for VRG to be created")
sample = TimeoutSampler(
timeout=timeout, sleep=5, func=check_vrg_existence, namespace=namespace
)
if not sample.wait_for_func_status(result=True):
error_msg = "VRG resource is not created"
logger.error(error_msg)
raise TimeoutExpiredError(error_msg)
logger.info(f"Waiting for {vr_count} VRs to be created")
sample = TimeoutSampler(
timeout=timeout,
sleep=5,
func=get_vr_count,
namespace=namespace,
)
sample.wait_for_func_value(vr_count)
logger.info(f"Waiting for {vr_count} VRs to reach primary state")
sample = TimeoutSampler(
timeout=timeout,
sleep=5,
func=check_vr_state,
state="primary",
namespace=namespace,
)
if not sample.wait_for_func_status(result=True):
error_msg = "One or more VR haven't reached expected state primary within the time limit."
logger.error(error_msg)
raise TimeoutExpiredError(error_msg)
logger.info("Waiting for VRG to reach primary state")
sample = TimeoutSampler(
timeout=timeout,
sleep=5,
func=check_vrg_state,
state="primary",
namespace=namespace,
)
if not sample.wait_for_func_status(result=True):
error_msg = "VRG hasn't reached expected state primary within the time limit."
logger.error(error_msg)
raise TimeoutExpiredError(error_msg)
def test_ts_wait_for_value_positive():
"""
Check that wait_for_value() function waits for func to return given value
as expected.
"""
timeout = 10
sleep_time = 1
func_state = []
def func():
func_state.append(0)
return len(func_state)
ts = TimeoutSampler(timeout, sleep_time, func)
start = time.time()
ts.wait_for_func_value(3)
end = time.time()
assert 2 < (end - start) < 3
def test_ts_wait_for_value_negative(caplog):
"""
Check that when wait_for_value() fails to see expected return value of
given func within given timeout, exception is raised and the problem
logged.
"""
timeout = 3
sleep_time = 2
func = lambda: 1 # noqa: E731
ts = TimeoutSampler(timeout, sleep_time, func)
caplog.set_level(logging.ERROR)
with pytest.raises(TimeoutExpiredError):
ts.wait_for_func_value(2)
# check that the problem was logged properly
assert len(caplog.records) == 1
for rec in caplog.records:
log_msg = rec.getMessage()
assert "function <lambda> failed" in log_msg
assert "failed to return expected value 2" in log_msg
assert "during 3 second timeout" in log_msg
def wait_for_vr_count(count, namespace, timeout=300):
"""
Wait for all VR resources to reach expected count in the given namespace
Args:
count (int): Expected number of VR resources
namespace (str): the namespace of the VR resources
timeout (int): time in seconds to wait for VR resources to be created
or reach expected state
Returns:
bool: True if all VR are in expected state
"""
sample = TimeoutSampler(
timeout=timeout,
sleep=5,
func=get_vr_count,
namespace=namespace,
)
sample.wait_for_func_value(count)
return True
