def test_far_intermittent_without_classification_and_not_backedout( monkeypatch, create_pushes ): """ Tests the scenario where a task succeeded in a parent push, didn't run in the in the push of interest, was intermittent in a following push, which was not backed-out and didn't have a classification. """ monkeypatch.setattr(HGMO, "is_backout", property(lambda cls: True)) p = create_pushes(4) i = 1 # the index of the push we are mainly interested in p[i - 1].tasks = [Task.create(id="1", label="test-intermittent", result="success")] p[i + 1].tasks = [ Task.create(id="1", label="test-intermittent", result="success"), Task.create( id="2", label="test-intermittent", result="testfailed", classification="not classified", ), ] assert p[i].get_regressions("label") == {"test-intermittent": 4} assert p[i + 1].get_regressions("label") == {"test-intermittent": 4}
def test_fixed_by_commit_push_wasnt_backedout(monkeypatch, create_pushes): """ Tests the scenario where a task succeeded in a parent push, didn't run in the push of interest and failed in a following push, with 'fixed by commit' information pointing to a back-out of another push. """ monkeypatch.setattr(HGMO, "is_backout", property(lambda cls: True)) p = create_pushes(4) i = 1 # the index of the push we are mainly interested in p[i - 1].tasks = [ Task.create(id="1", label="test-failure-current", result="success") ] p[i + 1].tasks = [ Task.create( id="1", label="test-failure-current", result="testfailed", classification="fixed by commit", classification_note="xxx", ) ] p[i + 1].backedoutby = "012c3f1626b3e9bcd803d19aaf9584a81c5c95de" assert p[i].get_regressions("label") == {} assert p[i + 1].get_regressions("label") == {}
def test_intermittent_fixed_by_commit(monkeypatch, create_pushes): """ Tests the scenario where a task succeeded in a parent push, didn't run in the in the push of interest, was intermittent in a following push, which was backed-out and had a 'fixed by commit' classification. """ monkeypatch.setattr(HGMO, "is_backout", property(lambda cls: True)) p = create_pushes(5) i = 2 # the index of the push we are mainly interested in p[i - 2].tasks = [Task.create(id="1", label="test-intermittent", result="success")] p[i - 2].backedoutby = None p[i].backedoutby = "d25e5c66de225e2d1b989af61a0420874707dd14" p[i + 1].tasks = [ Task.create(id="1", label="test-intermittent", result="success"), Task.create( id="2", label="test-intermittent", result="testfailed", classification="fixed by commit", classification_note="d25e5c66de225e2d1b989af61a0420874707dd14", ), ] p[i + 1].backedoutby = "012c3f1626b3e9bcd803d19aaf9584a81c5c95de" assert p[i].get_regressions("label") == {"test-intermittent": 0} assert p[i + 1].get_regressions("label") == {}
def test_intermittent_classification(monkeypatch, create_pushes): """ Tests the scenario where a task succeeded in a parent push, didn't run in the in the push of interest, failed in a following push, which was backed-out and had a 'intermittent' classification. """ monkeypatch.setattr(HGMO, "is_backout", property(lambda cls: True)) p = create_pushes(5) i = 2 # the index of the push we are mainly interested in p[i - 1].tasks = [Task.create(id="1", label="test-intermittent", result="success")] p[i].backedoutby = "xxx" p[i + 1].tasks = [ Task.create( id="1", label="test-intermittent", result="testfailed", classification="intermittent", ) ] p[i + 1].backedoutby = "yyy" assert p[i].get_regressions("label") == {} assert p[i + 1].get_regressions("label") == {}
def test_succeeded_in_parent_didnt_run_in_current_passed_in_child_failed_in_grandchild( create_pushes, ): """ Tests the scenario where a task succeeded in a parent push, didn't run in the push of interest, succeeded in a following push, and failed in a second following push. """ p = create_pushes(7) i = 3 # the index of the push we are mainly interested in p[i - 1].tasks = [Task.create(id="1", label="test-prova", result="success")] p[i + 1].tasks = [Task.create(id="1", label="test-prova", result="success")] p[i + 2].tasks = [ Task.create( id="1", label="test-prova", result="testfailed", classification="not classified", ) ] p[i + 2].backedoutby = "xxx" assert p[i - 2].get_regressions("label") == {} assert p[i - 1].get_regressions("label") == {} assert p[i].get_regressions("label") == {} assert p[i + 1].get_regressions("label") == {} assert p[i + 2].get_regressions("label") == {"test-prova": 0}
def test_push_tasks_with_cached_completed_tasks(monkeypatch, responses): rev = "abcdef" branch = "autoland" cached_tasks = [ Task.create(id=1, label="test-task", result="passed", state="completed") ] monkeypatch.setattr(config.cache, "get", lambda x: cached_tasks) responses.add( responses.GET, f"https://hg.mozilla.org/integration/autoland/json-automationrelevance/{rev}", json={"changesets": [{ "node": rev, "pushdate": [1638349140] }]}, status=200, ) responses.add( responses.GET, "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.abcdef.taskgraph.decision", json={"taskId": 1}, status=200, ) push = Push(rev, branch) tasks = push.tasks assert len(tasks) == 1
def test_fixed_by_commit_task_didnt_run_in_parents(monkeypatch, create_pushes): """ Tests the scenario where a task didn't run in a parent push, didn't run in the push of interest and failed in a following push, with 'fixed by commit' information pointing to the back-outs. """ monkeypatch.setattr(HGMO, "is_backout", property(lambda cls: True)) p = create_pushes(4) i = 1 # the index of the push we are mainly interested in p[i].backedoutby = "d25e5c66de225e2d1b989af61a0420874707dd14" p[i + 1].tasks = [ Task.create( id="1", label="test-failure-current", result="testfailed", classification="fixed by commit", classification_note="d25e5c66de225e2d1b989af61a0420874707dd14", ) ] p[i + 1].backedoutby = "012c3f1626b3e9bcd803d19aaf9584a81c5c95de" assert p[i].get_regressions("label") == {"test-failure-current": 0} assert p[i + 1].get_regressions("label") == {}
def test_fixed_by_commit_no_backout(monkeypatch, create_pushes): """ Tests the scenario where two tasks succeeded in a parent push, didn't run in the push of interest and failed in a following push, with 'fixed by commit' information pointing to a bustage fix. """ def mock_is_backout(cls): if cls.context["rev"] == "xxx": return False return True monkeypatch.setattr(HGMO, "is_backout", property(mock_is_backout)) p = create_pushes(4) i = 1 # the index of the push we are mainly interested in p[i - 1].tasks = [ Task.create(id="1", label="test-failure-current", result="success"), Task.create(id="1", label="test-failure-next", result="success"), ] p[i].backedoutby = "d25e5c66de225e2d1b989af61a0420874707dd14" p[i + 1].tasks = [ Task.create( id="1", label="test-failure-current", result="testfailed", classification="fixed by commit", classification_note="xxx", ), Task.create( id="1", label="test-failure-next", result="testfailed", classification="fixed by commit", classification_note="012c3f1626b3", ), ] p[i + 1].backedoutby = "012c3f1626b3e9bcd803d19aaf9584a81c5c95de" assert p[i].get_regressions("label") == {"test-failure-current": 1} assert p[i + 1].get_regressions("label") == { "test-failure-current": 1, "test-failure-next": 0, }
def decision_task(self): """A representation of the decision task. Returns: Task: A `Task` instance representing the decision task. """ index = self.index + ".taskgraph.decision" return Task.create(index=index)
def test_failed_and_not_backedout(create_pushes): """ Tests the scenario where a task failed in a push which was not backed-out. """ p = create_pushes(3) i = 1 # the index of the push we are mainly interested in p[i - 1].tasks = [Task.create(id="1", label="test-prova", result="success")] p[i].tasks = [ Task.create( id="1", label="test-prova", result="testfailed", classification="not classified", ) ] assert p[i].get_regressions("label") == {"test-prova": 0}
def test_far_child_failed_and_backedout(create_pushes): """ Tests the scenario where a task didn't run in the push of interest, which was not backed-out, and failed in a (far away) following push. """ p = create_pushes(3 + (MAX_DEPTH // 2 + 1)) i = 1 # the index of the push we are mainly interested in p[i - 1].tasks = [Task.create(id="1", label="test-prova", result="success")] p[len(p) - 2].tasks = [ Task.create( id="1", label="test-prova", result="testfailed", classification="not classified", ) ] assert p[i].get_regressions("label") == {}
def test_succeeded_and_backedout(create_pushes): """ Tests the scenario where a task succeeded in a push which was backed-out. """ p = create_pushes(3) i = 1 # the index of the push we are mainly interested in p[i].tasks = [Task.create(id="1", label="test-prova", result="success")] p[i].backedoutby = "xxx" assert p[i].get_regressions("label") == {}
def test_configuration(): assert (Task.create( id=1, label="test-windows7-32/debug-reftest-gpu-e10s-1").configuration == "test-windows7-32/debug-*-gpu-e10s") assert (Task.create( id=1, label="test-linux1804-64/debug-mochitest-plain-gpu-e10s").configuration == "test-linux1804-64/debug-*-e10s") assert (Task.create( id=1, label= "test-macosx1014-64-shippable/opt-web-platform-tests-wdspec-headless-e10s-1", ).configuration == "test-macosx1014-64-shippable/opt-*-headless-e10s") assert (Task.create( id=1, label="test-linux1804-64-asan/opt-web-platform-tests-e10s-3"). configuration == "test-linux1804-64-asan/opt-*-e10s") assert (Task.create( id=1, label="test-linux1804-64-qr/debug-web-platform-tests-wdspec-fis-e10s-1", ).configuration == "test-linux1804-64-qr/debug-*-fis-e10s") assert (Task.create( id=1, label= "test-windows7-32-shippable/opt-firefox-ui-functional-remote-e10s", ).configuration == "test-windows7-32-shippable/opt-*-e10s")
def test_results_for_incomplete_task(responses): push = FakePush("autoland", "rev") for state in ["running", "pending", "unscheduled", "exception"]: task = Task.create( id=1, label="test-task", state="running", ) task.retrieve_results(push) assert task.results == [] responses.add( responses.GET, "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/1/artifacts", json={ "artifacts": [{"name": "errorsummary.log"}], }, status=200, ) responses.add( responses.GET, "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/1/artifacts/errorsummary.log", body=r""" {"action": "test_groups", "line": 3, "groups": ["layout/base/tests/browser.ini"]} {"status": "OK", "duration": 12430, "line": 4465, "group": "layout/base/tests/browser.ini", "action": "group_result"} """.strip(), status=200, ) task = Task.create( id=1, label="test-task", state="completed", ) task.retrieve_results(push) assert task.results == [ GroupResult(group="layout/base/tests/browser.ini", ok=True, duration=12430), ]
def test_to_json(): kwargs = { "id": 1, "label": "foobar", "result": "pass", "duration": 100, } task = Task.create(**kwargs) result = task.to_json() json.dumps(result) # assert doesn't raise for k, v in kwargs.items(): assert k in result assert result[k] == v
def get_shadow_scheduler_tasks(self, name: str) -> Set[str]: """Returns all tasks the given shadow scheduler would have scheduled, or None if the given scheduler didn't run. Args: name (str): The name of the shadow scheduler to query. Returns: set: All task labels that would have been scheduled. """ index = self.index + ".source.shadow-scheduler-{}".format(name) task = Task.create(index=index) labels = task.get_artifact( "public/shadow-scheduler/optimized_tasks.list") return set(labels.splitlines())
def get_shadow_scheduler_tasks(self, name: str) -> List[dict]: """Returns all tasks the given shadow scheduler would have scheduled, or None if the given scheduler didn't run. Args: name (str): The name of the shadow scheduler to query. Returns: set: All task labels that would have been scheduled. """ index = self.index + ".source.shadow-scheduler-{}".format(name) task = Task.create(index=index) optimized = task.get_artifact( "public/shadow-scheduler/optimized-tasks.json") return list(optimized.values())
def _normalized_tasks(tasks): # If we are missing one of these keys, discard the task. required_keys = ( "id", "label", "result", ) # Normalize and validate. normalized_tasks = [] for task in tasks.values(): missing = [k for k in required_keys if k not in task] taskstr = task.get("label", task["id"]) if missing: logger.trace( f"Skipping task '{taskstr}' because it is missing " f"the following attributes: {', '.join(missing)}") continue if task.get("tags"): task["tags"] = {t["name"]: t["value"] for t in task["tags"]} if task.get("classification_note"): if isinstance(task["classification_note"], list): task["classification_note"] = task["classification_note"][ -1] groups = task.pop("_result_group", None) oks = task.pop("_result_ok", None) if groups is not None: if oks: task["_results"] = [ GroupResult(group=group, ok=ok) for group, ok in zip(groups, oks) ] normalized_tasks.append(task) return [Task.create(**task) for task in normalized_tasks]
def get_shadow_scheduler_tasks(self, name: str) -> Set[str]: """Returns all tasks the given shadow scheduler would have scheduled, or None if the given scheduler didn't run. Args: name (str): The name of the shadow scheduler to query. Returns: set: All task labels that would have been scheduled. """ index = self.index + ".source.shadow-scheduler-{}".format(name) task = Task.create(index=index) try: optimized = task.get_artifact( "public/shadow-scheduler/optimized-tasks.json") return set(t["label"] for t in optimized.values()) except ArtifactNotFound: # TODO Legacy artifact format, remove after Jan 1st 2021. labels = task.get_artifact( "public/shadow-scheduler/optimized_tasks.list") return set(labels.splitlines())
def test_finalized_push_tasks_with_cache(monkeypatch, responses): rev = "abcdef" branch = "autoland" cached_tasks = [Task.create(id=1, label="test-task", result="passed")] monkeypatch.setattr(config.cache, "get", lambda x: cached_tasks) monkeypatch.setattr(Push, "is_finalized", True) responses.add( responses.GET, f"https://hg.mozilla.org/integration/autoland/json-automationrelevance/{rev}", json={"changesets": [{ "node": rev, "pushdate": [1638349140] }]}, status=200, ) push = Push(rev, branch) tasks = push.tasks assert len(tasks) == 1 assert tasks == cached_tasks
def test_create(responses): # Creating a task with just a label doesn't work. with pytest.raises(TypeError): Task.create(label="foobar") # Specifying an id works with or without label. assert Task.create(id=1, label="foobar").label == "foobar" assert Task.create(id=1).label is None # Can also specify an index. index = "index.path" responses.add( responses.GET, get_index_url(index), json={"taskId": 1}, status=200, ) assert Task.create(index=index, label="foobar").label == "foobar" assert Task.create(index=index).label is None # Specifying non-existent task index raises. responses.replace(responses.GET, get_index_url(index), status=404) with pytest.raises(TaskNotFound): Task.create(index=index)
def test_GroupSummary_classifications(): task1 = Task.create( id=1, label="test-task1", result="failed", classification="fixed by commit", classification_note="xxx", ) task1._results = [GroupResult("group1", False, duration=42)] assert GroupSummary("group1", [task1]).classifications == [ ("fixed by commit", "xxx") ] with pytest.raises(AssertionError): GroupSummary("group2", [task1]) task1 = Task.create( id=1, label="test-task1", result="failed", classification="fixed by commit", classification_note="xxx", ) task1._results = [ GroupResult("group1", False, duration=42), GroupResult("group2", False, duration=42), ] assert GroupSummary("group1", [task1]).classifications == [ ("fixed by commit", "xxx") ] assert GroupSummary("group2", [task1]).classifications == [ ("fixed by commit", "xxx") ] task1 = Task.create( id=1, label="test-task1", result="failed", classification="intermittent" ) task1._results = [ GroupResult("group1", False, duration=42), GroupResult("group2", False, duration=42), ] assert GroupSummary("group1", [task1]).classifications == [("intermittent", None)] assert GroupSummary("group2", [task1]).classifications == [("intermittent", None)] task1 = Task.create( id=1, label="test-task1", result="failed", classification="fixed by commit", classification_note="xxx", ) task1._results = [ GroupResult("group1", True, duration=42), GroupResult("group2", False, duration=42), ] assert GroupSummary("group1", [task1]).classifications == [] assert GroupSummary("group2", [task1]).classifications == [ ("fixed by commit", "xxx") ] task1 = Task.create( id=1, label="test-task1", result="failed", classification="fixed by commit", classification_note="xxx", ) task1._results = [ GroupResult("group1", True, duration=42), GroupResult("group2", False, duration=42), ] task2 = Task.create( id=1, label="test-task1", result="failed", classification="intermittent" ) task2._results = [ GroupResult("group1", False, duration=42), GroupResult("group2", False, duration=42), ] assert GroupSummary("group1", [task1, task2]).classifications == [ ("intermittent", None) ] assert GroupSummary("group2", [task1, task2]).classifications == [ ("fixed by commit", "xxx"), ("intermittent", None), ]
def handle(self) -> None: branch = self.argument("branch") self.line("<comment>Loading pushes...</comment>") self.pushes = classify_commands_pushes( branch, self.option("from-date"), self.option("to-date"), self.option("rev"), ) if self.option("recalculate"): try: medium_conf = (float(self.option("medium-confidence")) if self.option("medium-confidence") else 0.8) except ValueError: self.line( "<error>Provided --medium-confidence should be a float.</error>" ) exit(1) try: high_conf = (float(self.option("high-confidence")) if self.option("high-confidence") else 0.9) except ValueError: self.line( "<error>Provided --high-confidence should be a float.</error>" ) exit(1) elif self.option("medium-confidence") or self.option( "high-confidence"): self.line( "<error>--recalculate isn't set, you shouldn't provide either --medium-confidence nor --high-confidence attributes.</error>" ) return # Progress bar will display time stats & messages progress = self.progress_bar(len(self.pushes)) progress.set_format( " %current%/%max% [%bar%] %percent:3s%% %elapsed:6s% %message%") # Setup specific route prefix for existing tasks, according to environment environment = self.option("environment") route_prefix = ("project.mozci.classification" if environment == "production" else f"project.mozci.{environment}.classification") self.errors = {} self.classifications = {} self.failures = {} for push in self.pushes: if self.option("recalculate"): progress.set_message(f"Calc. {branch} {push.id}") all_pushes = set([push] + [ parent for parent in push._iterate_parents(max_depth=MAX_DEPTH) ] + [ child for child in push._iterate_children(max_depth=MAX_DEPTH) ]) removed_tasks: Dict[str, List[Task]] = {} old_classifications: Dict[str, Dict[str, Dict[str, str]]] = {} for p in all_pushes: # Ignore retriggers and backfills on current push/its parents/its children. removed_tasks[p.id] = [ task for task in p.tasks if task.is_backfill or task.is_retrigger ] p.tasks = [ task for task in p.tasks if task not in removed_tasks[p.id] ] # Pretend no tasks were classified to run the model without any outside help. old_classifications[p.id] = {} for task in p.tasks: old_classifications[p.id][task.id] = { "classification": task.classification, "note": task.classification_note, } task.classification = "not classified" task.classification_note = None try: self.classifications[push], regressions = push.classify( intermittent_confidence_threshold=medium_conf, real_confidence_threshold=high_conf, ) self.failures[push] = { "real": regressions.real, "intermittent": regressions.intermittent, "unknown": regressions.unknown, } except Exception as e: self.line( f"<error>Classification failed on {branch} {push.rev}: {e}</error>" ) self.errors[push] = e for p in all_pushes: # Once the Mozci algorithm has run, restore Sheriffs classifications to be able to properly compare failures classifications. for task in p.tasks: task.classification = old_classifications[p.id][ task.id]["classification"] task.classification_note = old_classifications[p.id][ task.id]["note"] # And also restore tasks marked as a backfill or a retrigger. p.tasks = p.tasks + removed_tasks[p.id] else: progress.set_message(f"Fetch {branch} {push.id}") try: index = f"{route_prefix}.{branch}.revision.{push.rev}" task = Task.create(index=index, root_url=COMMUNITY_TASKCLUSTER_ROOT_URL) artifact = task.get_artifact( "public/classification.json", root_url=COMMUNITY_TASKCLUSTER_ROOT_URL, ) self.classifications[push] = PushStatus[artifact["push"] ["classification"]] self.failures[push] = artifact["failures"] except TaskNotFound as e: self.line( f"<comment>Taskcluster task missing for {branch} {push.rev}</comment>" ) self.errors[push] = e except Exception as e: self.line( f"<error>Fetch failed on {branch} {push.rev}: {e}</error>" ) self.errors[push] = e # Advance the overall progress bar progress.advance() # Conclude the progress bar progress.finish() print("\n") error_line = "" if self.errors: if self.option("recalculate"): error_line = "Failed to recalculate classification" else: error_line = "Failed to fetch classification" error_line += f" for {len(self.errors)} out of {len(self.pushes)} pushes." if not self.option("recalculate") and not self.option( "send-email"): error_line += " Use the '--recalculate' option if you want to generate them yourself." self.line(f"<error>{error_line}</error>") stats = [ self.log_pushes(PushStatus.BAD, False), self.log_pushes(PushStatus.BAD, True), self.log_pushes(PushStatus.GOOD, False), self.log_pushes(PushStatus.GOOD, True), self.log_pushes(PushStatus.UNKNOWN, False), self.log_pushes(PushStatus.UNKNOWN, True), ] if self.option("detailed-classifications"): self.line("\n") real_stats = intermittent_stats = { "total": 0, "correct": 0, "wrong": 0, "pending": 0, "conflicting": 0, "missed": 0, } for push in self.pushes: self.line( f"<comment>Printing detailed classifications comparison for push {push.branch}/{push.rev}</comment>" ) # Compare real failures that were predicted by mozci with the ones classified by Sheriffs try: sheriff_reals = set() # Get likely regressions of this push likely_regressions = push.get_likely_regressions( "group", True) # Only consider groups that were classified as "fixed by commit" to exclude likely regressions mozci found via heuristics. for other in push._iterate_children(): for name, group in other.group_summaries.items(): classifications = set( [c for c, _ in group.classifications]) if (classifications == {"fixed by commit"} and name in likely_regressions): sheriff_reals.add(name) except Exception: self.line( "<error>Failed to retrieve Sheriff classifications for the real failures of this push.</error>" ) try: push_real_stats = self.log_details( push, "real", sheriff_reals, {"fixed by commit"}, ) real_stats = { key: value + push_real_stats[key] for key, value in real_stats.items() } except Exception: self.line( "<error>Failed to compare true and predicted real failures of this push.</error>" ) # Compare intermittent failures that were predicted by mozci with the ones classified by Sheriffs try: sheriff_intermittents = set() for name, group in push.group_summaries.items(): classifications = set( [c for c, _ in group.classifications]) if classifications == {"intermittent"}: sheriff_intermittents.add(name) except Exception: self.line( "<error>Failed to retrieve Sheriff classifications for the intermittent failures of this push.</error>" ) try: push_intermittent_stats = self.log_details( push, "intermittent", sheriff_intermittents, {"intermittent"}, ) intermittent_stats = { key: value + push_intermittent_stats[key] for key, value in intermittent_stats.items() } except Exception: self.line( "<error>Failed to compare true and predicted intermittent failures of this push.</error>" ) self.line( f"\n<comment>Printing overall detailed classifications comparison for {len(self.pushes)} pushes</comment>" ) detailed_stats = [ f"{real_stats['correct']} out of {real_stats['total']} real failures were correctly classified ('fixed by commit' by Sheriffs).", f"{real_stats['wrong']} out of {real_stats['total']} real failures were wrongly classified ('intermittent' by Sheriffs).", f"{real_stats['pending']} out of {real_stats['total']} real failures are waiting to be classified by Sheriffs.", f"{real_stats['conflicting']} out of {real_stats['total']} real failures have conflicting classifications applied by Sheriffs.", f"{real_stats['missed']} real failures were missed or classified as unknown by Mozci.", f"{intermittent_stats['correct']} out of {intermittent_stats['total']} intermittent failures were correctly classified ('intermittent' by Sheriffs).", f"{intermittent_stats['wrong']} out of {intermittent_stats['total']} intermittent failures were wrongly classified ('fixed by commit' by Sheriffs).", f"{intermittent_stats['pending']} out of {intermittent_stats['total']} intermittent failures are waiting to be classified by Sheriffs.", f"{intermittent_stats['conflicting']} out of {intermittent_stats['total']} intermittent failures have conflicting classifications applied by Sheriffs.", f"{intermittent_stats['missed']} intermittent failures were missed or classified as unknown by Mozci.", ] for line in detailed_stats: self.line(line) stats += detailed_stats if self.option("send-email"): self.send_emails(len(self.pushes), stats, error_line) output = self.option("output") if output: # Build stats for CSV with open(output, "w") as csvfile: writer = csv.DictWriter( csvfile, fieldnames=[ "revision", "date", "classification", "backedout", "error_type", "error_message", ], ) writer.writeheader() writer.writerows( [self.build_stats(push) for push in self.pushes]) self.line( f"<info>Written stats for {len(self.pushes)} pushes in {output}</info>" )
task.retrieve_results(push) assert task.results == [ GroupResult(group="layout/base/tests/browser.ini", ok=True), ] @pytest.mark.parametrize( "group_summary, expected_result", [ ( GroupSummary( "group1", [ Task.create( id=i, label=f"test-task{i}", _results=[GroupResult(group="group1", ok=False), GR_2, GR_3], ) for i in range(1, 11) ], ), False, ), # All related tasks failed ( GroupSummary( "group1", [ Task.create( id=i, label=f"test-task{i}", _results=[
def tasks(self): """All tasks that ran on the push, including retriggers and backfills. Returns: list: A list of `Task` objects. """ # Gather information from the treeherder table. tasks = [] try: tasks = data.handler.get("push_tasks", branch=self.branch, rev=self.rev) except MissingDataError: pass # Gather task tags from the task table. try: tags_by_task = data.handler.get("push_tasks_tags", branch=self.branch, rev=self.rev) for task in tasks: tags = tags_by_task.get(task["id"]) if tags: if "tags" not in task: task["tags"] = {} task["tags"].update(tags) except MissingDataError: pass # Let's gather error/results from cache or AD/Taskcluster test_tasks_results = config.cache.get(self.push_uuid, {}) was_cached = len(test_tasks_results.keys()) != 0 groups = None if not was_cached: # Gather information from the unittest table. We allow missing data for this table because # ActiveData only holds very recent data in it, but we have fallbacks on Taskcluster # artifacts. try: groups = data.handler.get("push_test_groups", branch=self.branch, rev=self.rev) for task in tasks: results = groups.get(task["id"]) if results is not None: task["_results"] = [ GroupResult(group=group, ok=ok) for group, ok in results.items() ] except MissingDataError: pass tasks = [Task.create(**task) for task in tasks] # Add any data available in the cache if was_cached: # Let's add cached error summaries to TestTasks for t in tasks: if isinstance(t, TestTask): error_summary = test_tasks_results.get(t.id) # Only Test tasks have stored error summary information in the cache if error_summary: t._errors = error_summary["errors"] t._results = error_summary["results"] logger.debug( "Fetched tasks errors/results for {} from the cache".format( self.push_uuid)) # Gather group data which could have been missing in ActiveData. concurrent.futures.wait( [ Push.THREAD_POOL_EXECUTOR.submit(lambda task: task.groups, task) for task in tasks if isinstance(task, TestTask) ], return_when=concurrent.futures.FIRST_EXCEPTION, ) # Now we can cache the results. self._cache_test_tasks(tasks) return tasks
"test-windows10-64-2004-qr/debug-web-platform-tests-swr-e10s-9", "test-windows10-64-2004-qr/debug-mochitest-devtools-chrome-fis-e10s-1", ], } NUMBER_OF_DEFAULT_GROUPS = 5 NUMBER_OF_INTERMITTENT_GROUPS_IN_DEFAULT = 2 GROUP_SUMMARIES_DEFAULT = { group.name: group for group in [ GroupSummary( f"group{i}", [ Task.create( id=j, label=f"test-task{j}", result="failed", _results=[GroupResult(group=f"group{i}", ok=False)], ) for j in range(1, 4) ] + ([ Task.create( id=4, label="test-task1", result="passed", _results=[GroupResult(group=f"group{i}", ok=True)], ) ] if i <= NUMBER_OF_INTERMITTENT_GROUPS_IN_DEFAULT else []), ) for i in range(1, NUMBER_OF_DEFAULT_GROUPS + 1) ] }
def test_push_tasks_with_cached_uncompleted_tasks(monkeypatch, responses): rev = "abcdef" branch = "autoland" cached_tasks = [Task.create(id=1, label="test-task", state="running")] monkeypatch.setattr(config.cache, "get", lambda x: cached_tasks) responses.add( responses.GET, f"https://hg.mozilla.org/integration/autoland/json-automationrelevance/{rev}", json={"changesets": [{ "node": rev, "pushdate": [1638349140] }]}, status=200, ) responses.add( responses.GET, "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.abcdef.taskgraph.decision", json={"taskId": 1}, status=200, ) responses.add( responses.GET, "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/1", json={"taskGroupId": "xyz789"}, status=200, ) responses.add( responses.GET, "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task-group/xyz789/list", json={ "tasks": [ { "task": { "extra": { "treeherder": { "tier": 3 }, }, "metadata": { "name": "task-A", }, "tags": { "name": "tag-A" }, }, "status": { "taskId": "abc13", "state": "unscheduled", }, }, { "task": { "extra": { "treeherder": { "tier": 1 }, }, "metadata": { "name": "task-B", }, "tags": { "name": "tag-A" }, }, "status": { "taskId": "abc123", "state": "unscheduled", }, }, ] }, status=200, ) responses.add( responses.GET, "https://treeherder.mozilla.org/api/project/autoland/note/push_notes/?revision=abcdef&format=json", json={}, status=200, ) push = Push(rev, branch) tasks = push.tasks assert len(tasks) == 1
GroupResult(group="layout/base/tests/browser.ini", ok=True, duration=12430), ] @pytest.mark.parametrize( "group_summary, expected_result", [ ( GroupSummary( "group1", [ Task.create( id=1, label="test-task1", _results=[ GroupResult(group="group1", ok=False, duration=42), GR_2, GR_3, ], ) ], ), None, ), # Only one task run and failed ( GroupSummary( "group1", [ Task.create( id=1, label="test-linux1804-64/opt-xpcshell-e10s-1",
def tasks(self): """All tasks that ran on the push, including retriggers and backfills. Returns: list: A list of `Task` objects. """ args = Namespace(rev=self.rev, branch=self.branch) tasks = defaultdict(dict) retries = defaultdict(int) list_keys = ( "_result_ok", "_result_group", ) def add(result): if "header" in result: result["data"] = [ { field: value for field, value in zip(result["header"], entry) if value is not None } for entry in result["data"] ] for task in result["data"]: if "id" not in task: logger.trace(f"Skipping {task} because of missing id.") continue task_id = task["id"] # If a task is re-run, use the data from the last run. if "retry_id" in task: if task["retry_id"] < retries[task_id]: logger.trace( f"Skipping {task} because there is a newer run of it." ) continue retries[task_id] = task["retry_id"] # We don't need to store the retry ID. del task["retry_id"] cur_task = tasks[task_id] for key, val in task.items(): if key in list_keys: if key not in cur_task: cur_task[key] = [] cur_task[key].append(val) else: cur_task[key] = val # Gather information from the treeherder table. try: add(run_query("push_tasks_from_treeherder", args)) except MissingDataError: pass # Gather information from the unittest table. We allow missing data for this table because # ActiveData only holds very recent data in it, but we have fallbacks on Taskcluster # artifacts. # TODO: We have fallbacks for groups and results, but not for kind. try: add(run_query("push_tasks_results_from_unittest", args)) except MissingDataError: pass try: add(run_query("push_tasks_groups_from_unittest", args)) except MissingDataError: pass # If we are missing one of these keys, discard the task. required_keys = ( "id", "label", ) # Normalize and validate. normalized_tasks = [] for task in tasks.values(): missing = [k for k in required_keys if k not in task] taskstr = task.get("label", task["id"]) if missing: logger.trace( f"Skipping task '{taskstr}' because it is missing " f"the following attributes: {', '.join(missing)}" ) continue if task.get("tags"): task["tags"] = {t["name"]: t["value"] for t in task["tags"]} if task.get("classification_note"): if isinstance(task["classification_note"], list): task["classification_note"] = task["classification_note"][-1] if task.get("_groups"): if isinstance(task["_groups"], str): task["_groups"] = [task["_groups"]] if task.get("_result_ok"): oks = task.pop("_result_ok") if task.get("_result_group"): groups = task.pop("_result_group") task["_results"] = [ GroupResult(group=group, ok=ok) for group, ok in zip(groups, oks) ] normalized_tasks.append(task) return [Task.create(**task) for task in normalized_tasks]
def inner(**kwargs): nonlocal id task = Task.create(id=id, **kwargs) id += 1 return task