def test_far_intermittent_without_classification_and_not_backedout(
monkeypatch, create_pushes
):
"""
Tests the scenario where a task succeeded in a parent push, didn't run in the
in the push of interest, was intermittent in a following push, which was not
backed-out and didn't have a classification.
"""
monkeypatch.setattr(HGMO, "is_backout", property(lambda cls: True))
p = create_pushes(4)
i = 1 # the index of the push we are mainly interested in
p[i - 1].tasks = [Task.create(id="1", label="test-intermittent", result="success")]
p[i + 1].tasks = [
Task.create(id="1", label="test-intermittent", result="success"),
Task.create(
id="2",
label="test-intermittent",
result="testfailed",
classification="not classified",
),
]
assert p[i].get_regressions("label") == {"test-intermittent": 4}
assert p[i + 1].get_regressions("label") == {"test-intermittent": 4}
def test_fixed_by_commit_push_wasnt_backedout(monkeypatch, create_pushes):
"""
Tests the scenario where a task succeeded in a parent push, didn't run in the
push of interest and failed in a following push, with 'fixed by commit' information
pointing to a back-out of another push.
"""
monkeypatch.setattr(HGMO, "is_backout", property(lambda cls: True))
p = create_pushes(4)
i = 1 # the index of the push we are mainly interested in
p[i - 1].tasks = [
Task.create(id="1", label="test-failure-current", result="success")
]
p[i + 1].tasks = [
Task.create(
id="1",
label="test-failure-current",
result="testfailed",
classification="fixed by commit",
classification_note="xxx",
)
]
p[i + 1].backedoutby = "012c3f1626b3e9bcd803d19aaf9584a81c5c95de"
assert p[i].get_regressions("label") == {}
assert p[i + 1].get_regressions("label") == {}
def test_intermittent_fixed_by_commit(monkeypatch, create_pushes):
"""
Tests the scenario where a task succeeded in a parent push, didn't run in the
in the push of interest, was intermittent in a following push, which was
backed-out and had a 'fixed by commit' classification.
"""
monkeypatch.setattr(HGMO, "is_backout", property(lambda cls: True))
p = create_pushes(5)
i = 2 # the index of the push we are mainly interested in
p[i - 2].tasks = [Task.create(id="1", label="test-intermittent", result="success")]
p[i - 2].backedoutby = None
p[i].backedoutby = "d25e5c66de225e2d1b989af61a0420874707dd14"
p[i + 1].tasks = [
Task.create(id="1", label="test-intermittent", result="success"),
Task.create(
id="2",
label="test-intermittent",
result="testfailed",
classification="fixed by commit",
classification_note="d25e5c66de225e2d1b989af61a0420874707dd14",
),
]
p[i + 1].backedoutby = "012c3f1626b3e9bcd803d19aaf9584a81c5c95de"
assert p[i].get_regressions("label") == {"test-intermittent": 0}
assert p[i + 1].get_regressions("label") == {}
def test_intermittent_classification(monkeypatch, create_pushes):
"""
Tests the scenario where a task succeeded in a parent push, didn't run in the
in the push of interest, failed in a following push, which was
backed-out and had a 'intermittent' classification.
"""
monkeypatch.setattr(HGMO, "is_backout", property(lambda cls: True))
p = create_pushes(5)
i = 2 # the index of the push we are mainly interested in
p[i - 1].tasks = [Task.create(id="1", label="test-intermittent", result="success")]
p[i].backedoutby = "xxx"
p[i + 1].tasks = [
Task.create(
id="1",
label="test-intermittent",
result="testfailed",
classification="intermittent",
)
]
p[i + 1].backedoutby = "yyy"
assert p[i].get_regressions("label") == {}
assert p[i + 1].get_regressions("label") == {}
def test_succeeded_in_parent_didnt_run_in_current_passed_in_child_failed_in_grandchild(
create_pushes,
):
"""
Tests the scenario where a task succeeded in a parent push, didn't run in the
push of interest, succeeded in a following push, and failed in a second
following push.
"""
p = create_pushes(7)
i = 3 # the index of the push we are mainly interested in
p[i - 1].tasks = [Task.create(id="1", label="test-prova", result="success")]
p[i + 1].tasks = [Task.create(id="1", label="test-prova", result="success")]
p[i + 2].tasks = [
Task.create(
id="1",
label="test-prova",
result="testfailed",
classification="not classified",
)
]
p[i + 2].backedoutby = "xxx"
assert p[i - 2].get_regressions("label") == {}
assert p[i - 1].get_regressions("label") == {}
assert p[i].get_regressions("label") == {}
assert p[i + 1].get_regressions("label") == {}
assert p[i + 2].get_regressions("label") == {"test-prova": 0}
def test_push_tasks_with_cached_completed_tasks(monkeypatch, responses):
rev = "abcdef"
branch = "autoland"
cached_tasks = [
Task.create(id=1,
label="test-task",
result="passed",
state="completed")
]
monkeypatch.setattr(config.cache, "get", lambda x: cached_tasks)
responses.add(
responses.GET,
f"https://hg.mozilla.org/integration/autoland/json-automationrelevance/{rev}",
json={"changesets": [{
"node": rev,
"pushdate": [1638349140]
}]},
status=200,
)
responses.add(
responses.GET,
"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.abcdef.taskgraph.decision",
json={"taskId": 1},
status=200,
)
push = Push(rev, branch)
tasks = push.tasks
assert len(tasks) == 1
def test_fixed_by_commit_task_didnt_run_in_parents(monkeypatch, create_pushes):
"""
Tests the scenario where a task didn't run in a parent push, didn't run in the
push of interest and failed in a following push, with 'fixed by commit' information
pointing to the back-outs.
"""
monkeypatch.setattr(HGMO, "is_backout", property(lambda cls: True))
p = create_pushes(4)
i = 1 # the index of the push we are mainly interested in
p[i].backedoutby = "d25e5c66de225e2d1b989af61a0420874707dd14"
p[i + 1].tasks = [
Task.create(
id="1",
label="test-failure-current",
result="testfailed",
classification="fixed by commit",
classification_note="d25e5c66de225e2d1b989af61a0420874707dd14",
)
]
p[i + 1].backedoutby = "012c3f1626b3e9bcd803d19aaf9584a81c5c95de"
assert p[i].get_regressions("label") == {"test-failure-current": 0}
assert p[i + 1].get_regressions("label") == {}
def test_fixed_by_commit_no_backout(monkeypatch, create_pushes):
"""
Tests the scenario where two tasks succeeded in a parent push, didn't run in the
push of interest and failed in a following push, with 'fixed by commit' information
pointing to a bustage fix.
"""
def mock_is_backout(cls):
if cls.context["rev"] == "xxx":
return False
return True
monkeypatch.setattr(HGMO, "is_backout", property(mock_is_backout))
p = create_pushes(4)
i = 1 # the index of the push we are mainly interested in
p[i - 1].tasks = [
Task.create(id="1", label="test-failure-current", result="success"),
Task.create(id="1", label="test-failure-next", result="success"),
]
p[i].backedoutby = "d25e5c66de225e2d1b989af61a0420874707dd14"
p[i + 1].tasks = [
Task.create(
id="1",
label="test-failure-current",
result="testfailed",
classification="fixed by commit",
classification_note="xxx",
),
Task.create(
id="1",
label="test-failure-next",
result="testfailed",
classification="fixed by commit",
classification_note="012c3f1626b3",
),
]
p[i + 1].backedoutby = "012c3f1626b3e9bcd803d19aaf9584a81c5c95de"
assert p[i].get_regressions("label") == {"test-failure-current": 1}
assert p[i + 1].get_regressions("label") == {
"test-failure-current": 1,
"test-failure-next": 0,
}
def decision_task(self):
"""A representation of the decision task.
Returns:
Task: A `Task` instance representing the decision task.
"""
index = self.index + ".taskgraph.decision"
return Task.create(index=index)
def test_failed_and_not_backedout(create_pushes):
"""
Tests the scenario where a task failed in a push which was not backed-out.
"""
p = create_pushes(3)
i = 1 # the index of the push we are mainly interested in
p[i - 1].tasks = [Task.create(id="1", label="test-prova", result="success")]
p[i].tasks = [
Task.create(
id="1",
label="test-prova",
result="testfailed",
classification="not classified",
)
]
assert p[i].get_regressions("label") == {"test-prova": 0}
def test_far_child_failed_and_backedout(create_pushes):
"""
Tests the scenario where a task didn't run in the push of interest, which was not
backed-out, and failed in a (far away) following push.
"""
p = create_pushes(3 + (MAX_DEPTH // 2 + 1))
i = 1 # the index of the push we are mainly interested in
p[i - 1].tasks = [Task.create(id="1", label="test-prova", result="success")]
p[len(p) - 2].tasks = [
Task.create(
id="1",
label="test-prova",
result="testfailed",
classification="not classified",
)
]
assert p[i].get_regressions("label") == {}
def test_succeeded_and_backedout(create_pushes):
"""
Tests the scenario where a task succeeded in a push which was backed-out.
"""
p = create_pushes(3)
i = 1 # the index of the push we are mainly interested in
p[i].tasks = [Task.create(id="1", label="test-prova", result="success")]
p[i].backedoutby = "xxx"
assert p[i].get_regressions("label") == {}
def test_configuration():
assert (Task.create(
id=1, label="test-windows7-32/debug-reftest-gpu-e10s-1").configuration
== "test-windows7-32/debug-*-gpu-e10s")
assert (Task.create(
id=1,
label="test-linux1804-64/debug-mochitest-plain-gpu-e10s").configuration
== "test-linux1804-64/debug-*-e10s")
assert (Task.create(
id=1,
label=
"test-macosx1014-64-shippable/opt-web-platform-tests-wdspec-headless-e10s-1",
).configuration == "test-macosx1014-64-shippable/opt-*-headless-e10s")
assert (Task.create(
id=1, label="test-linux1804-64-asan/opt-web-platform-tests-e10s-3").
configuration == "test-linux1804-64-asan/opt-*-e10s")
assert (Task.create(
id=1,
label="test-linux1804-64-qr/debug-web-platform-tests-wdspec-fis-e10s-1",
).configuration == "test-linux1804-64-qr/debug-*-fis-e10s")
assert (Task.create(
id=1,
label=
"test-windows7-32-shippable/opt-firefox-ui-functional-remote-e10s",
).configuration == "test-windows7-32-shippable/opt-*-e10s")
def test_results_for_incomplete_task(responses):
push = FakePush("autoland", "rev")
for state in ["running", "pending", "unscheduled", "exception"]:
task = Task.create(
id=1,
label="test-task",
state="running",
)
task.retrieve_results(push)
assert task.results == []
responses.add(
responses.GET,
"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/1/artifacts",
json={
"artifacts": [{"name": "errorsummary.log"}],
},
status=200,
)
responses.add(
responses.GET,
"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/1/artifacts/errorsummary.log",
body=r"""
{"action": "test_groups", "line": 3, "groups": ["layout/base/tests/browser.ini"]}
{"status": "OK", "duration": 12430, "line": 4465, "group": "layout/base/tests/browser.ini", "action": "group_result"}
""".strip(),
status=200,
)
task = Task.create(
id=1,
label="test-task",
state="completed",
)
task.retrieve_results(push)
assert task.results == [
GroupResult(group="layout/base/tests/browser.ini", ok=True, duration=12430),
]
def test_to_json():
kwargs = {
"id": 1,
"label": "foobar",
"result": "pass",
"duration": 100,
}
task = Task.create(**kwargs)
result = task.to_json()
json.dumps(result) # assert doesn't raise
for k, v in kwargs.items():
assert k in result
assert result[k] == v
def get_shadow_scheduler_tasks(self, name: str) -> Set[str]:
"""Returns all tasks the given shadow scheduler would have scheduled,
or None if the given scheduler didn't run.
Args:
name (str): The name of the shadow scheduler to query.
Returns:
set: All task labels that would have been scheduled.
"""
index = self.index + ".source.shadow-scheduler-{}".format(name)
task = Task.create(index=index)
labels = task.get_artifact(
"public/shadow-scheduler/optimized_tasks.list")
return set(labels.splitlines())
def get_shadow_scheduler_tasks(self, name: str) -> List[dict]:
"""Returns all tasks the given shadow scheduler would have scheduled,
or None if the given scheduler didn't run.
Args:
name (str): The name of the shadow scheduler to query.
Returns:
set: All task labels that would have been scheduled.
"""
index = self.index + ".source.shadow-scheduler-{}".format(name)
task = Task.create(index=index)
optimized = task.get_artifact(
"public/shadow-scheduler/optimized-tasks.json")
return list(optimized.values())
def _normalized_tasks(tasks):
# If we are missing one of these keys, discard the task.
required_keys = (
"id",
"label",
"result",
)
# Normalize and validate.
normalized_tasks = []
for task in tasks.values():
missing = [k for k in required_keys if k not in task]
taskstr = task.get("label", task["id"])
if missing:
logger.trace(
f"Skipping task '{taskstr}' because it is missing "
f"the following attributes: {', '.join(missing)}")
continue
if task.get("tags"):
task["tags"] = {t["name"]: t["value"] for t in task["tags"]}
if task.get("classification_note"):
if isinstance(task["classification_note"], list):
task["classification_note"] = task["classification_note"][
-1]
groups = task.pop("_result_group", None)
oks = task.pop("_result_ok", None)
if groups is not None:
if oks:
task["_results"] = [
GroupResult(group=group, ok=ok)
for group, ok in zip(groups, oks)
]
normalized_tasks.append(task)
return [Task.create(**task) for task in normalized_tasks]
def get_shadow_scheduler_tasks(self, name: str) -> Set[str]:
"""Returns all tasks the given shadow scheduler would have scheduled,
or None if the given scheduler didn't run.
Args:
name (str): The name of the shadow scheduler to query.
Returns:
set: All task labels that would have been scheduled.
"""
index = self.index + ".source.shadow-scheduler-{}".format(name)
task = Task.create(index=index)
try:
optimized = task.get_artifact(
"public/shadow-scheduler/optimized-tasks.json")
return set(t["label"] for t in optimized.values())
except ArtifactNotFound:
# TODO Legacy artifact format, remove after Jan 1st 2021.
labels = task.get_artifact(
"public/shadow-scheduler/optimized_tasks.list")
return set(labels.splitlines())
def test_finalized_push_tasks_with_cache(monkeypatch, responses):
rev = "abcdef"
branch = "autoland"
cached_tasks = [Task.create(id=1, label="test-task", result="passed")]
monkeypatch.setattr(config.cache, "get", lambda x: cached_tasks)
monkeypatch.setattr(Push, "is_finalized", True)
responses.add(
responses.GET,
f"https://hg.mozilla.org/integration/autoland/json-automationrelevance/{rev}",
json={"changesets": [{
"node": rev,
"pushdate": [1638349140]
}]},
status=200,
)
push = Push(rev, branch)
tasks = push.tasks
assert len(tasks) == 1
assert tasks == cached_tasks
def test_create(responses):
# Creating a task with just a label doesn't work.
with pytest.raises(TypeError):
Task.create(label="foobar")
# Specifying an id works with or without label.
assert Task.create(id=1, label="foobar").label == "foobar"
assert Task.create(id=1).label is None
# Can also specify an index.
index = "index.path"
responses.add(
responses.GET,
get_index_url(index),
json={"taskId": 1},
status=200,
)
assert Task.create(index=index, label="foobar").label == "foobar"
assert Task.create(index=index).label is None
# Specifying non-existent task index raises.
responses.replace(responses.GET, get_index_url(index), status=404)
with pytest.raises(TaskNotFound):
Task.create(index=index)
def test_GroupSummary_classifications():
task1 = Task.create(
id=1,
label="test-task1",
result="failed",
classification="fixed by commit",
classification_note="xxx",
)
task1._results = [GroupResult("group1", False, duration=42)]
assert GroupSummary("group1", [task1]).classifications == [
("fixed by commit", "xxx")
]
with pytest.raises(AssertionError):
GroupSummary("group2", [task1])
task1 = Task.create(
id=1,
label="test-task1",
result="failed",
classification="fixed by commit",
classification_note="xxx",
)
task1._results = [
GroupResult("group1", False, duration=42),
GroupResult("group2", False, duration=42),
]
assert GroupSummary("group1", [task1]).classifications == [
("fixed by commit", "xxx")
]
assert GroupSummary("group2", [task1]).classifications == [
("fixed by commit", "xxx")
]
task1 = Task.create(
id=1, label="test-task1", result="failed", classification="intermittent"
)
task1._results = [
GroupResult("group1", False, duration=42),
GroupResult("group2", False, duration=42),
]
assert GroupSummary("group1", [task1]).classifications == [("intermittent", None)]
assert GroupSummary("group2", [task1]).classifications == [("intermittent", None)]
task1 = Task.create(
id=1,
label="test-task1",
result="failed",
classification="fixed by commit",
classification_note="xxx",
)
task1._results = [
GroupResult("group1", True, duration=42),
GroupResult("group2", False, duration=42),
]
assert GroupSummary("group1", [task1]).classifications == []
assert GroupSummary("group2", [task1]).classifications == [
("fixed by commit", "xxx")
]
task1 = Task.create(
id=1,
label="test-task1",
result="failed",
classification="fixed by commit",
classification_note="xxx",
)
task1._results = [
GroupResult("group1", True, duration=42),
GroupResult("group2", False, duration=42),
]
task2 = Task.create(
id=1, label="test-task1", result="failed", classification="intermittent"
)
task2._results = [
GroupResult("group1", False, duration=42),
GroupResult("group2", False, duration=42),
]
assert GroupSummary("group1", [task1, task2]).classifications == [
("intermittent", None)
]
assert GroupSummary("group2", [task1, task2]).classifications == [
("fixed by commit", "xxx"),
("intermittent", None),
]
def handle(self) -> None:
branch = self.argument("branch")
self.line("<comment>Loading pushes...</comment>")
self.pushes = classify_commands_pushes(
branch,
self.option("from-date"),
self.option("to-date"),
self.option("rev"),
)
if self.option("recalculate"):
try:
medium_conf = (float(self.option("medium-confidence"))
if self.option("medium-confidence") else 0.8)
except ValueError:
self.line(
"<error>Provided --medium-confidence should be a float.</error>"
)
exit(1)
try:
high_conf = (float(self.option("high-confidence"))
if self.option("high-confidence") else 0.9)
except ValueError:
self.line(
"<error>Provided --high-confidence should be a float.</error>"
)
exit(1)
elif self.option("medium-confidence") or self.option(
"high-confidence"):
self.line(
"<error>--recalculate isn't set, you shouldn't provide either --medium-confidence nor --high-confidence attributes.</error>"
)
return
# Progress bar will display time stats & messages
progress = self.progress_bar(len(self.pushes))
progress.set_format(
" %current%/%max% [%bar%] %percent:3s%% %elapsed:6s% %message%")
# Setup specific route prefix for existing tasks, according to environment
environment = self.option("environment")
route_prefix = ("project.mozci.classification"
if environment == "production" else
f"project.mozci.{environment}.classification")
self.errors = {}
self.classifications = {}
self.failures = {}
for push in self.pushes:
if self.option("recalculate"):
progress.set_message(f"Calc. {branch} {push.id}")
all_pushes = set([push] + [
parent
for parent in push._iterate_parents(max_depth=MAX_DEPTH)
] + [
child
for child in push._iterate_children(max_depth=MAX_DEPTH)
])
removed_tasks: Dict[str, List[Task]] = {}
old_classifications: Dict[str, Dict[str, Dict[str, str]]] = {}
for p in all_pushes:
# Ignore retriggers and backfills on current push/its parents/its children.
removed_tasks[p.id] = [
task for task in p.tasks
if task.is_backfill or task.is_retrigger
]
p.tasks = [
task for task in p.tasks
if task not in removed_tasks[p.id]
]
# Pretend no tasks were classified to run the model without any outside help.
old_classifications[p.id] = {}
for task in p.tasks:
old_classifications[p.id][task.id] = {
"classification": task.classification,
"note": task.classification_note,
}
task.classification = "not classified"
task.classification_note = None
try:
self.classifications[push], regressions = push.classify(
intermittent_confidence_threshold=medium_conf,
real_confidence_threshold=high_conf,
)
self.failures[push] = {
"real": regressions.real,
"intermittent": regressions.intermittent,
"unknown": regressions.unknown,
}
except Exception as e:
self.line(
f"<error>Classification failed on {branch} {push.rev}: {e}</error>"
)
self.errors[push] = e
for p in all_pushes:
# Once the Mozci algorithm has run, restore Sheriffs classifications to be able to properly compare failures classifications.
for task in p.tasks:
task.classification = old_classifications[p.id][
task.id]["classification"]
task.classification_note = old_classifications[p.id][
task.id]["note"]
# And also restore tasks marked as a backfill or a retrigger.
p.tasks = p.tasks + removed_tasks[p.id]
else:
progress.set_message(f"Fetch {branch} {push.id}")
try:
index = f"{route_prefix}.{branch}.revision.{push.rev}"
task = Task.create(index=index,
root_url=COMMUNITY_TASKCLUSTER_ROOT_URL)
artifact = task.get_artifact(
"public/classification.json",
root_url=COMMUNITY_TASKCLUSTER_ROOT_URL,
)
self.classifications[push] = PushStatus[artifact["push"]
["classification"]]
self.failures[push] = artifact["failures"]
except TaskNotFound as e:
self.line(
f"<comment>Taskcluster task missing for {branch} {push.rev}</comment>"
)
self.errors[push] = e
except Exception as e:
self.line(
f"<error>Fetch failed on {branch} {push.rev}: {e}</error>"
)
self.errors[push] = e
# Advance the overall progress bar
progress.advance()
# Conclude the progress bar
progress.finish()
print("\n")
error_line = ""
if self.errors:
if self.option("recalculate"):
error_line = "Failed to recalculate classification"
else:
error_line = "Failed to fetch classification"
error_line += f" for {len(self.errors)} out of {len(self.pushes)} pushes."
if not self.option("recalculate") and not self.option(
"send-email"):
error_line += " Use the '--recalculate' option if you want to generate them yourself."
self.line(f"<error>{error_line}</error>")
stats = [
self.log_pushes(PushStatus.BAD, False),
self.log_pushes(PushStatus.BAD, True),
self.log_pushes(PushStatus.GOOD, False),
self.log_pushes(PushStatus.GOOD, True),
self.log_pushes(PushStatus.UNKNOWN, False),
self.log_pushes(PushStatus.UNKNOWN, True),
]
if self.option("detailed-classifications"):
self.line("\n")
real_stats = intermittent_stats = {
"total": 0,
"correct": 0,
"wrong": 0,
"pending": 0,
"conflicting": 0,
"missed": 0,
}
for push in self.pushes:
self.line(
f"<comment>Printing detailed classifications comparison for push {push.branch}/{push.rev}</comment>"
)
# Compare real failures that were predicted by mozci with the ones classified by Sheriffs
try:
sheriff_reals = set()
# Get likely regressions of this push
likely_regressions = push.get_likely_regressions(
"group", True)
# Only consider groups that were classified as "fixed by commit" to exclude likely regressions mozci found via heuristics.
for other in push._iterate_children():
for name, group in other.group_summaries.items():
classifications = set(
[c for c, _ in group.classifications])
if (classifications == {"fixed by commit"}
and name in likely_regressions):
sheriff_reals.add(name)
except Exception:
self.line(
"<error>Failed to retrieve Sheriff classifications for the real failures of this push.</error>"
)
try:
push_real_stats = self.log_details(
push,
"real",
sheriff_reals,
{"fixed by commit"},
)
real_stats = {
key: value + push_real_stats[key]
for key, value in real_stats.items()
}
except Exception:
self.line(
"<error>Failed to compare true and predicted real failures of this push.</error>"
)
# Compare intermittent failures that were predicted by mozci with the ones classified by Sheriffs
try:
sheriff_intermittents = set()
for name, group in push.group_summaries.items():
classifications = set(
[c for c, _ in group.classifications])
if classifications == {"intermittent"}:
sheriff_intermittents.add(name)
except Exception:
self.line(
"<error>Failed to retrieve Sheriff classifications for the intermittent failures of this push.</error>"
)
try:
push_intermittent_stats = self.log_details(
push,
"intermittent",
sheriff_intermittents,
{"intermittent"},
)
intermittent_stats = {
key: value + push_intermittent_stats[key]
for key, value in intermittent_stats.items()
}
except Exception:
self.line(
"<error>Failed to compare true and predicted intermittent failures of this push.</error>"
)
self.line(
f"\n<comment>Printing overall detailed classifications comparison for {len(self.pushes)} pushes</comment>"
)
detailed_stats = [
f"{real_stats['correct']} out of {real_stats['total']} real failures were correctly classified ('fixed by commit' by Sheriffs).",
f"{real_stats['wrong']} out of {real_stats['total']} real failures were wrongly classified ('intermittent' by Sheriffs).",
f"{real_stats['pending']} out of {real_stats['total']} real failures are waiting to be classified by Sheriffs.",
f"{real_stats['conflicting']} out of {real_stats['total']} real failures have conflicting classifications applied by Sheriffs.",
f"{real_stats['missed']} real failures were missed or classified as unknown by Mozci.",
f"{intermittent_stats['correct']} out of {intermittent_stats['total']} intermittent failures were correctly classified ('intermittent' by Sheriffs).",
f"{intermittent_stats['wrong']} out of {intermittent_stats['total']} intermittent failures were wrongly classified ('fixed by commit' by Sheriffs).",
f"{intermittent_stats['pending']} out of {intermittent_stats['total']} intermittent failures are waiting to be classified by Sheriffs.",
f"{intermittent_stats['conflicting']} out of {intermittent_stats['total']} intermittent failures have conflicting classifications applied by Sheriffs.",
f"{intermittent_stats['missed']} intermittent failures were missed or classified as unknown by Mozci.",
]
for line in detailed_stats:
self.line(line)
stats += detailed_stats
if self.option("send-email"):
self.send_emails(len(self.pushes), stats, error_line)
output = self.option("output")
if output:
# Build stats for CSV
with open(output, "w") as csvfile:
writer = csv.DictWriter(
csvfile,
fieldnames=[
"revision",
"date",
"classification",
"backedout",
"error_type",
"error_message",
],
)
writer.writeheader()
writer.writerows(
[self.build_stats(push) for push in self.pushes])
self.line(
f"<info>Written stats for {len(self.pushes)} pushes in {output}</info>"
)
task.retrieve_results(push)
assert task.results == [
GroupResult(group="layout/base/tests/browser.ini", ok=True),
]
@pytest.mark.parametrize(
"group_summary, expected_result",
[
(
GroupSummary(
"group1",
[
Task.create(
id=i,
label=f"test-task{i}",
_results=[GroupResult(group="group1", ok=False), GR_2, GR_3],
)
for i in range(1, 11)
],
),
False,
), # All related tasks failed
(
GroupSummary(
"group1",
[
Task.create(
id=i,
label=f"test-task{i}",
_results=[
def tasks(self):
"""All tasks that ran on the push, including retriggers and backfills.
Returns:
list: A list of `Task` objects.
"""
# Gather information from the treeherder table.
tasks = []
try:
tasks = data.handler.get("push_tasks",
branch=self.branch,
rev=self.rev)
except MissingDataError:
pass
# Gather task tags from the task table.
try:
tags_by_task = data.handler.get("push_tasks_tags",
branch=self.branch,
rev=self.rev)
for task in tasks:
tags = tags_by_task.get(task["id"])
if tags:
if "tags" not in task:
task["tags"] = {}
task["tags"].update(tags)
except MissingDataError:
pass
# Let's gather error/results from cache or AD/Taskcluster
test_tasks_results = config.cache.get(self.push_uuid, {})
was_cached = len(test_tasks_results.keys()) != 0
groups = None
if not was_cached:
# Gather information from the unittest table. We allow missing data for this table because
# ActiveData only holds very recent data in it, but we have fallbacks on Taskcluster
# artifacts.
try:
groups = data.handler.get("push_test_groups",
branch=self.branch,
rev=self.rev)
for task in tasks:
results = groups.get(task["id"])
if results is not None:
task["_results"] = [
GroupResult(group=group, ok=ok)
for group, ok in results.items()
]
except MissingDataError:
pass
tasks = [Task.create(**task) for task in tasks]
# Add any data available in the cache
if was_cached:
# Let's add cached error summaries to TestTasks
for t in tasks:
if isinstance(t, TestTask):
error_summary = test_tasks_results.get(t.id)
# Only Test tasks have stored error summary information in the cache
if error_summary:
t._errors = error_summary["errors"]
t._results = error_summary["results"]
logger.debug(
"Fetched tasks errors/results for {} from the cache".format(
self.push_uuid))
# Gather group data which could have been missing in ActiveData.
concurrent.futures.wait(
[
Push.THREAD_POOL_EXECUTOR.submit(lambda task: task.groups,
task)
for task in tasks if isinstance(task, TestTask)
],
return_when=concurrent.futures.FIRST_EXCEPTION,
)
# Now we can cache the results.
self._cache_test_tasks(tasks)
return tasks
"test-windows10-64-2004-qr/debug-web-platform-tests-swr-e10s-9",
"test-windows10-64-2004-qr/debug-mochitest-devtools-chrome-fis-e10s-1",
],
}
NUMBER_OF_DEFAULT_GROUPS = 5
NUMBER_OF_INTERMITTENT_GROUPS_IN_DEFAULT = 2
GROUP_SUMMARIES_DEFAULT = {
group.name: group
for group in [
GroupSummary(
f"group{i}",
[
Task.create(
id=j,
label=f"test-task{j}",
result="failed",
_results=[GroupResult(group=f"group{i}", ok=False)],
) for j in range(1, 4)
] + ([
Task.create(
id=4,
label="test-task1",
result="passed",
_results=[GroupResult(group=f"group{i}", ok=True)],
)
] if i <= NUMBER_OF_INTERMITTENT_GROUPS_IN_DEFAULT else []),
) for i in range(1, NUMBER_OF_DEFAULT_GROUPS + 1)
]
}
def test_push_tasks_with_cached_uncompleted_tasks(monkeypatch, responses):
rev = "abcdef"
branch = "autoland"
cached_tasks = [Task.create(id=1, label="test-task", state="running")]
monkeypatch.setattr(config.cache, "get", lambda x: cached_tasks)
responses.add(
responses.GET,
f"https://hg.mozilla.org/integration/autoland/json-automationrelevance/{rev}",
json={"changesets": [{
"node": rev,
"pushdate": [1638349140]
}]},
status=200,
)
responses.add(
responses.GET,
"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.abcdef.taskgraph.decision",
json={"taskId": 1},
status=200,
)
responses.add(
responses.GET,
"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/1",
json={"taskGroupId": "xyz789"},
status=200,
)
responses.add(
responses.GET,
"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task-group/xyz789/list",
json={
"tasks": [
{
"task": {
"extra": {
"treeherder": {
"tier": 3
},
},
"metadata": {
"name": "task-A",
},
"tags": {
"name": "tag-A"
},
},
"status": {
"taskId": "abc13",
"state": "unscheduled",
},
},
{
"task": {
"extra": {
"treeherder": {
"tier": 1
},
},
"metadata": {
"name": "task-B",
},
"tags": {
"name": "tag-A"
},
},
"status": {
"taskId": "abc123",
"state": "unscheduled",
},
},
]
},
status=200,
)
responses.add(
responses.GET,
"https://treeherder.mozilla.org/api/project/autoland/note/push_notes/?revision=abcdef&format=json",
json={},
status=200,
)
push = Push(rev, branch)
tasks = push.tasks
assert len(tasks) == 1
GroupResult(group="layout/base/tests/browser.ini", ok=True, duration=12430),
]
@pytest.mark.parametrize(
"group_summary, expected_result",
[
(
GroupSummary(
"group1",
[
Task.create(
id=1,
label="test-task1",
_results=[
GroupResult(group="group1", ok=False, duration=42),
GR_2,
GR_3,
],
)
],
),
None,
), # Only one task run and failed
(
GroupSummary(
"group1",
[
Task.create(
id=1,
label="test-linux1804-64/opt-xpcshell-e10s-1",
def tasks(self):
"""All tasks that ran on the push, including retriggers and backfills.
Returns:
list: A list of `Task` objects.
"""
args = Namespace(rev=self.rev, branch=self.branch)
tasks = defaultdict(dict)
retries = defaultdict(int)
list_keys = (
"_result_ok",
"_result_group",
)
def add(result):
if "header" in result:
result["data"] = [
{
field: value
for field, value in zip(result["header"], entry)
if value is not None
}
for entry in result["data"]
]
for task in result["data"]:
if "id" not in task:
logger.trace(f"Skipping {task} because of missing id.")
continue
task_id = task["id"]
# If a task is re-run, use the data from the last run.
if "retry_id" in task:
if task["retry_id"] < retries[task_id]:
logger.trace(
f"Skipping {task} because there is a newer run of it."
)
continue
retries[task_id] = task["retry_id"]
# We don't need to store the retry ID.
del task["retry_id"]
cur_task = tasks[task_id]
for key, val in task.items():
if key in list_keys:
if key not in cur_task:
cur_task[key] = []
cur_task[key].append(val)
else:
cur_task[key] = val
# Gather information from the treeherder table.
try:
add(run_query("push_tasks_from_treeherder", args))
except MissingDataError:
pass
# Gather information from the unittest table. We allow missing data for this table because
# ActiveData only holds very recent data in it, but we have fallbacks on Taskcluster
# artifacts.
# TODO: We have fallbacks for groups and results, but not for kind.
try:
add(run_query("push_tasks_results_from_unittest", args))
except MissingDataError:
pass
try:
add(run_query("push_tasks_groups_from_unittest", args))
except MissingDataError:
pass
# If we are missing one of these keys, discard the task.
required_keys = (
"id",
"label",
)
# Normalize and validate.
normalized_tasks = []
for task in tasks.values():
missing = [k for k in required_keys if k not in task]
taskstr = task.get("label", task["id"])
if missing:
logger.trace(
f"Skipping task '{taskstr}' because it is missing "
f"the following attributes: {', '.join(missing)}"
)
continue
if task.get("tags"):
task["tags"] = {t["name"]: t["value"] for t in task["tags"]}
if task.get("classification_note"):
if isinstance(task["classification_note"], list):
task["classification_note"] = task["classification_note"][-1]
if task.get("_groups"):
if isinstance(task["_groups"], str):
task["_groups"] = [task["_groups"]]
if task.get("_result_ok"):
oks = task.pop("_result_ok")
if task.get("_result_group"):
groups = task.pop("_result_group")
task["_results"] = [
GroupResult(group=group, ok=ok)
for group, ok in zip(groups, oks)
]
normalized_tasks.append(task)
return [Task.create(**task) for task in normalized_tasks]
def inner(**kwargs):
nonlocal id
task = Task.create(id=id, **kwargs)
id += 1
return task
