import os
from testlib import clean_binary_tests, build_tests, test_binaries, run_test
from concurrent.futures import as_completed, ThreadPoolExecutor
TIMES = 10
if __name__ == "__main__":
clean_binary_tests()
build_tests()
binaries = test_binaries(exclude=[r'test_regression-.*'])
print(f'========= collected {len(binaries)} test binaries:')
print('\n'.join(binaries))
print(f"Run all tests sequentially for {TIMES} times: ")
test_with_fails = {}
for binary in binaries:
fails = []
for i in range(TIMES):
exitcode, stdout, stderr = run_test(binary, isolate=False)
print(f'Run {binary} {i+1} of {TIMES}, exit code {exitcode}')
if exitcode != 0:
fails.append(exitcode)
if fails:
test_with_fails[binary] = fails
if test_with_fails:
print("Some tests failed: ")
for t, f in test_with_fails.items():
print(f'{t} failed {len(f)} times')
if result[0] != 0:
fails.append((binary_full_name, result))
else:
show_test_result(binary, result)
print(f"========= finished run {completed} test binaries")
if fails:
if len(fails) <= RERUN_THRESHOLD:
# if not fail a lot, retry run test sequentially to avoid potential timeout
new_fails = []
for f in fails:
binary_full_name = f[0]
result = f[1]
binary = os.path.basename(binary_full_name)
print(f'========= test binary {binary} run in parallel failed, exit code {result[0]}, retry run equentially ...')
result = run_test(binary_full_name, isolate=False)
if result[0] != 0:
new_fails.append((binary_full_name, result))
else:
show_test_result(binary, result)
if new_fails:
new_fail_summary = []
for f in new_fails:
binary_full_name = f[0]
result = f[1]
binary = os.path.basename(binary_full_name)
show_test_result(binary, result)
new_fail_summary.append(f'========= test binary {binary} run sequentially failed, exit code {result[0]}')
for s in new_fail_summary:
print(s)
exit(1)