def group_summaries(self):
"""All group summaries combining retriggers.
Returns:
dict: A dictionary of the form {<group>: [<GroupSummary>]}.
"""
groups = defaultdict(list)
for task in self.tasks:
if not isinstance(task, TestTask):
continue
for group in task.groups:
groups[group].append(task)
group_summaries = {}
for group, tasks in groups.items():
# Because of https://bugzilla.mozilla.org/show_bug.cgi?id=1640758, we can't trust
# test-verify when there are no reported failures.
# TODO: Drop the clean call and the length check a few months after the bug mentioned above is fixed.
tasks = self.clean_test_verify_tasks(group, tasks)
if len(tasks) == 0:
continue
group_summaries[group] = GroupSummary(group, tasks)
return group_summaries
def group_summaries(self):
"""All group summaries combining retriggers.
Returns:
dict: A dictionary of the form {<group>: [<GroupSummary>]}.
"""
groups = defaultdict(list)
for task in self.tasks:
if not isinstance(task, TestTask):
continue
for group in task.groups:
groups[group].append(task)
group_summaries = {}
for group, tasks in groups.items():
group_summary = GroupSummary(group, tasks)
# TODO: We need this check because GroupSummary might ignore some test-verify
# tasks. A few months after https://bugzilla.mozilla.org/show_bug.cgi?id=1640758
# is fixed, we can remove it.
if len(group_summary.tasks) > 0:
group_summaries[group] = group_summary
return group_summaries
def __make_group_summary_objects(pushes, branch):
"""Generates a list of GroupSummary objects from a list of pushes.
Args:
pushes (list): List of Push objects.
Returns:
list: List of GroupSummary objects.
"""
import adr
# Obtain all the task.id values contained in the pushes. It will be used in
# the `where` query against ActiveData.
results = []
revs = [p.rev for p in pushes]
# if we have too many revisions, ActiveData returns an error
for i in range(0, len(revs), 30):
for revs_chunk in revs[i:i + 30]:
try:
results += adr.query.run_query(
"group_durations",
Namespace(push_ids=revs_chunk, branch=branch))["data"]
except adr.MissingDataError:
pass
# Sort by the result.group attribute.
results = sorted(results, key=lambda x: x[1])
tasks_by_config = {}
task_id_to_task = {t.id: t for push in pushes for t in push.tasks}
for task_id, result_group, result_duration in results:
# TODO: remove this when https://github.com/mozilla/mozci/issues/297 is fixed
if task_id not in task_id_to_task:
continue
# tasks that had exception or failed by timeout have no duration for a group
if result_duration is None:
continue
task = task_id_to_task[task_id]
if task.configuration not in tasks_by_config:
# Dictionary to hold the mapping keyed by result.group mapped to list of
# task.id and list of and result.duration.
tasks_by_config[task.configuration] = defaultdict(
lambda: defaultdict(list))
# Build the mapping of group to the group durations and TestTask objects.
tasks_by_config[task.configuration][result_group]["tasks"].append(task)
tasks_by_config[task.configuration][result_group]["durations"].append(
result_duration)
return [
GroupSummary(key, value["tasks"], value["durations"])
for config in tasks_by_config.keys()
for key, value in tasks_by_config[config].items()
]
def group_summaries(self):
"""All group summaries combining retriggers.
Returns:
dict: A dictionary of the form {<group>: [<GroupSummary>]}.
"""
groups = defaultdict(list)
for task in self.tasks:
if not isinstance(task, TestTask):
continue
for group in task.groups:
groups[group].append(task)
return {
group: GroupSummary(group, tasks)
for group, tasks in groups.items()
}
def config_group_summaries(self):
"""All group summaries, on given configurations, combining retriggers.
Returns:
dict: A dictionary of the form {(<configuration>, <group>): [<GroupSummary>]}.
"""
config_groups = defaultdict(list)
for task in self.tasks:
if not isinstance(task, TestTask):
continue
for group in task.groups:
config_groups[(task.configuration, group)].append(task)
return {
config_group: GroupSummary(config_group[1], tasks)
for config_group, tasks in config_groups.items()
}
def group_summaries(self):
"""All group summaries combining retriggers.
Returns:
dict: A dictionary of the form {<group>: [<GroupSummary>]}.
"""
groups = defaultdict(list)
future_to_task = {
Push.THREAD_POOL_EXECUTOR.submit(lambda task: task.groups, task): task
for task in self.tasks
if isinstance(task, TestTask)
}
for future in concurrent.futures.as_completed(future_to_task):
task = future_to_task[future]
for group in future.result():
groups[group].append(task)
groups = {group: GroupSummary(group, tasks) for group, tasks in groups.items()}
return groups
def __make_group_summary_objects(pushes):
"""Generates a list of GroupSummary objects from a list of pushes.
Args:
pushes (list): List of Push objects.
Returns:
list: List of GroupSummary objects.
"""
# Obtain all the task.id values contained in the pushes. It will be used in
# the `where` query against ActiveData.
task_ids = [task.id for push in pushes for task in push.tasks]
# Extract the results of the above query.
results = run_query("group_durations", Namespace(task_id=task_ids))["data"]
# Sort by the result.group attribute.
results = sorted(results, key=lambda x: x[1])
# Dictionary to hold the mapping keyed by result.group mapped to list of
# task.id and list of and result.duration.
mapping = defaultdict(lambda: defaultdict(list))
for task_id, result_group, result_duration in results:
# Obtain TestTask object that matches the task_id.
task = [t for push in pushes for t in push.tasks
if t.id == task_id].pop()
# Build the mapping of group to the group durations and TestTask objects.
mapping[result_group]["tasks"].append(task)
mapping[result_group]["durations"].append(result_duration)
return [
GroupSummary(key, value["tasks"], value["durations"])
for key, value in mapping.items()
]
def test_GroupSummary_classifications():
task1 = Task.create(
id=1,
label="test-task1",
result="failed",
classification="fixed by commit",
classification_note="xxx",
)
task1._results = [GroupResult("group1", False, duration=42)]
assert GroupSummary("group1", [task1]).classifications == [
("fixed by commit", "xxx")
]
with pytest.raises(AssertionError):
GroupSummary("group2", [task1])
task1 = Task.create(
id=1,
label="test-task1",
result="failed",
classification="fixed by commit",
classification_note="xxx",
)
task1._results = [
GroupResult("group1", False, duration=42),
GroupResult("group2", False, duration=42),
]
assert GroupSummary("group1", [task1]).classifications == [
("fixed by commit", "xxx")
]
assert GroupSummary("group2", [task1]).classifications == [
("fixed by commit", "xxx")
]
task1 = Task.create(
id=1, label="test-task1", result="failed", classification="intermittent"
)
task1._results = [
GroupResult("group1", False, duration=42),
GroupResult("group2", False, duration=42),
]
assert GroupSummary("group1", [task1]).classifications == [("intermittent", None)]
assert GroupSummary("group2", [task1]).classifications == [("intermittent", None)]
task1 = Task.create(
id=1,
label="test-task1",
result="failed",
classification="fixed by commit",
classification_note="xxx",
)
task1._results = [
GroupResult("group1", True, duration=42),
GroupResult("group2", False, duration=42),
]
assert GroupSummary("group1", [task1]).classifications == []
assert GroupSummary("group2", [task1]).classifications == [
("fixed by commit", "xxx")
]
task1 = Task.create(
id=1,
label="test-task1",
result="failed",
classification="fixed by commit",
classification_note="xxx",
)
task1._results = [
GroupResult("group1", True, duration=42),
GroupResult("group2", False, duration=42),
]
task2 = Task.create(
id=1, label="test-task1", result="failed", classification="intermittent"
)
task2._results = [
GroupResult("group1", False, duration=42),
GroupResult("group2", False, duration=42),
]
assert GroupSummary("group1", [task1, task2]).classifications == [
("intermittent", None)
]
assert GroupSummary("group2", [task1, task2]).classifications == [
("fixed by commit", "xxx"),
("intermittent", None),
]
assert task.results == [
GroupResult(group="layout/base/tests/browser.ini", ok=True, duration=12430),
]
@pytest.mark.parametrize(
"group_summary, expected_result",
[
(
GroupSummary(
"group1",
[
Task.create(
id=1,
label="test-task1",
_results=[
GroupResult(group="group1", ok=False, duration=42),
GR_2,
GR_3,
],
)
],
),
None,
), # Only one task run and failed
(
GroupSummary(
"group1",
[
Task.create(
id=1,
label="test-linux1804-64/opt-xpcshell-e10s-1",
}
NUMBER_OF_DEFAULT_GROUPS = 5
NUMBER_OF_INTERMITTENT_GROUPS_IN_DEFAULT = 2
GROUP_SUMMARIES_DEFAULT = {
group.name: group
for group in [
GroupSummary(
f"group{i}",
[
Task.create(
id=j,
label=f"test-task{j}",
result="failed",
_results=[GroupResult(group=f"group{i}", ok=False)],
) for j in range(1, 4)
] + ([
Task.create(
id=4,
label="test-task1",
result="passed",
_results=[GroupResult(group=f"group{i}", ok=True)],
)
] if i <= NUMBER_OF_INTERMITTENT_GROUPS_IN_DEFAULT else []),
) for i in range(1, NUMBER_OF_DEFAULT_GROUPS + 1)
]
}
def test_group_summaries_default_status():
assert {
**{
task.retrieve_results(push)
assert task.results == [
GroupResult(group="layout/base/tests/browser.ini", ok=True),
]
@pytest.mark.parametrize(
"group_summary, expected_result",
[
(
GroupSummary(
"group1",
[
Task.create(
id=i,
label=f"test-task{i}",
_results=[GroupResult(group="group1", ok=False), GR_2, GR_3],
)
for i in range(1, 11)
],
),
False,
), # All related tasks failed
(
GroupSummary(
"group1",
[
Task.create(
id=i,
label=f"test-task{i}",
_results=[
"reduced_tasks_higher": {},
"known_tasks": [
"test-windows10-64-2004-qr/debug-web-platform-tests-swr-e10s-9",
"test-windows10-64-2004-qr/debug-mochitest-devtools-chrome-fis-e10s-1",
],
}
GROUP_SUMMARIES_DEFAULT = {
group.name: group
for group in [
GroupSummary(
f"group{i}",
[
Task.create(
id=j,
label=f"test-task{j}",
result="failed",
_results=[GroupResult(group=f"group{i}", ok=False)],
) for j in range(1, 4)
],
) for i in range(1, 6)
]
}
def make_tasks(group_id):
return [
TestTask(
id=j,
label=f"test-task{j}",
result="failed",
