def bucket_metric_and_test_suites(self):
all_tests = self.tests()
self.test_suites = []
if len(all_tests):
for suite_name, tests in groupby(
sorted(all_tests.values(), key=lambda t: t.name),
lambda t: parse_test_name(t.name)[0]):
test_suite = TestRun.TestSuite()
test_suite.name = suite_name
test_suite.tests = [t for t in tests]
self.test_suites.append(test_suite)
all_metrics = self.metrics()
if len(all_metrics):
self.metric_suites = []
for suite_name, metrics in groupby(
sorted(all_metrics.values(), key=lambda m: m.name),
lambda m: parse_metric_name(m.name)[0]):
metric_suite = TestRun.MetricSuite()
metric_suite.name = suite_name
metric_suite.metrics = [m for m in metrics]
self.metric_suites.append(metric_suite)
def bucket_metric_and_test_suites(self):
all_tests = self.tests()
self.test_suites = []
if len(all_tests):
for suite_name, tests in groupby(
sorted(all_tests.values(), key=lambda t: t.name),
lambda t: parse_test_name(t.name)[0]):
test_suite = Suite()
test_suite.name = suite_name
self.test_suites.append(test_suite)
for test in tests:
test_suite.add_test(test)
all_metrics = self.metrics()
if len(all_metrics):
self.metric_suites = []
for suite_name, metrics in groupby(
sorted(all_metrics.values(), key=lambda m: m.name),
lambda m: parse_metric_name(m.name)[0]):
metric_suite = MetricSuite()
metric_suite.name = suite_name
[metric_suite.add_metric(m) for m in metrics]
self.metric_suites.append(metric_suite)
def find_stable_tests(tests, envs={}, suites={}):
"""
Print a list of stable tests
- if environments are given
| envA | envb |
suiteA | 2/2 | 1/2 |
- testA | Y | 90% |
- testB | Y | 80% |
suiteB/testA | 50% | Y |
"""
if len(tests) == 0:
print('*** No tests available ***')
return
envs_slugs = sorted([env.slug for env in envs.values()])
tests_names = sorted(set([test.name for test in tests]))
tests_dict = {}
# Longest test name
longest_test_name = max(
[len(parse_test_name(name)[1]) for name in tests_names]) + len('- ')
# Saves a summary by suite
suites_stableness = defaultdict(list)
if len(envs):
# Longest env slug
longest_env_slug = max(10, max([len(slug) for slug in envs_slugs]))
envs_dict = {env.slug: [] for env in envs.values()}
tests_dict = defaultdict(lambda: envs_dict.copy())
for test in tests:
env = envs[getid(test.environment)]
tests_dict[test.name][env.slug].append(test.status)
print(' ' * (3 + longest_test_name), end='')
envs_header = '|'.join(
[slug.center(longest_env_slug) for slug in envs_slugs])
print(f'|{envs_header}|')
prev_suite_slug = None
for test_name in tests_names:
suite_slug, test = parse_test_name(test_name)
if suite_slug != prev_suite_slug:
prev_suite_slug = suite_slug
print(f'\n\033[1m{prev_suite_slug}\033[0m')
print(f'- {test.ljust(longest_test_name)} ', end='')
for env_slug in envs_slugs:
n, out = stableness(tests_dict[test_name][env_slug],
pad=longest_env_slug)
suites_stableness[suite_slug].append(n)
print(f'|{out}', end='')
print('|')
else:
tests_dict = defaultdict(list)
for test in tests:
tests_dict[test.name].append(test.status)
prev_suite_slug = None
for test_name in tests_names:
suite_slug, test = parse_test_name(test_name)
if suite_slug != prev_suite_slug:
prev_suite_slug = suite_slug
print(f'\n\033[1m{prev_suite_slug}\033[0m')
n, out = stableness(tests_dict[test_name])
suites_stableness[suite_slug].append(n)
print(f'- {test.ljust(longest_test_name)} {out}')
longest_suite_slug = max([len(slug) for slug in suites_stableness.keys()])
print()
print("*** Suite summary ***")
for suite_slug in suites_stableness.keys():
n, out = stableness(suites_stableness[suite_slug], target=1)
print(f'\033[1m{suite_slug.ljust(longest_suite_slug)}\033[0m: {out}')