def check_stability(logger, repeat_loop=10, repeat_restart=5, chaos_mode=True, max_time=None,
output_results=True, **kwargs):
kwargs_extras = [{}]
if chaos_mode and kwargs["product"] == "firefox":
kwargs_extras.append({"chaos_mode_flags": "0xfb"})
steps = get_steps(logger, repeat_loop, repeat_restart, kwargs_extras)
start_time = datetime.now()
step_results = []
github_checks_outputter = get_gh_checks_outputter(kwargs["github_checks_text_file"])
for desc, step_func, expected_iterations in steps:
if max_time and datetime.now() - start_time > max_time:
logger.info("::: Test verification is taking too long: Giving up!")
logger.info("::: So far, all checks passed, but not all checks were run.")
write_summary(logger, step_results, "TIMEOUT")
return 2
logger.info(':::')
logger.info('::: Running test verification step "%s"...' % desc)
logger.info(':::')
results, inconsistent, slow, iterations = step_func(**kwargs)
if iterations <= 1 and expected_iterations > 1:
step_results.append((desc, "FAIL"))
logger.info("::: Reached iteration timeout before finishing 2 or more repeat runs.")
logger.info("::: At least 2 successful repeat runs are required to validate stability.")
write_summary(logger, step_results, "TIMEOUT")
return 1
if output_results:
write_results(logger.info, results, iterations)
if inconsistent:
step_results.append((desc, "FAIL"))
if github_checks_outputter:
write_github_checks_summary_inconsistent(github_checks_outputter.output, inconsistent, iterations)
write_inconsistent(logger.info, inconsistent, iterations)
write_summary(logger, step_results, "FAIL")
return 1
if slow:
step_results.append((desc, "FAIL"))
if github_checks_outputter:
write_github_checks_summary_slow_tests(github_checks_outputter.output, slow)
write_slow_tests(logger.info, slow)
write_summary(logger, step_results, "FAIL")
return 1
# If the tests passed but the number of iterations didn't match the number expected to run,
# it is likely that the runs were stopped early to avoid a timeout.
if iterations != expected_iterations:
result = f"PASS * {iterations}/{expected_iterations} repeats completed"
step_results.append((desc, result))
else:
step_results.append((desc, "PASS"))
write_summary(logger, step_results, "PASS")
def check_stability(logger,
repeat_loop=10,
repeat_restart=5,
chaos_mode=True,
max_time=None,
output_results=True,
**kwargs):
kwargs_extras = [{}]
if chaos_mode and kwargs["product"] == "firefox":
kwargs_extras.append({"chaos_mode_flags": "0xfb"})
steps = get_steps(logger, repeat_loop, repeat_restart, kwargs_extras)
start_time = datetime.now()
step_results = []
github_checks_outputter = get_gh_checks_outputter(
kwargs["github_checks_text_file"])
for desc, step_func in steps:
if max_time and datetime.now() - start_time > max_time:
logger.info("::: Test verification is taking too long: Giving up!")
logger.info(
"::: So far, all checks passed, but not all checks were run.")
write_summary(logger, step_results, "TIMEOUT")
return 2
logger.info(':::')
logger.info('::: Running test verification step "%s"...' % desc)
logger.info(':::')
results, inconsistent, slow, iterations = step_func(**kwargs)
if output_results:
write_results(logger.info, results, iterations)
if inconsistent:
step_results.append((desc, "FAIL"))
if github_checks_outputter:
write_github_checks_summary_inconsistent(
github_checks_outputter.output, inconsistent, iterations)
write_inconsistent(logger.info, inconsistent, iterations)
write_summary(logger, step_results, "FAIL")
return 1
if slow:
step_results.append((desc, "FAIL"))
if github_checks_outputter:
write_github_checks_summary_slow_tests(
github_checks_outputter.output, slow)
write_slow_tests(logger.info, slow)
write_summary(logger, step_results, "FAIL")
return 1
step_results.append((desc, "PASS"))
write_summary(logger, step_results, "PASS")