def run_push_test_selection_data(self, branch, rev):
task_name = f"gecko.v2.{branch}.revision.{rev}.taskgraph.decision"
try:
decision_task_id = taskcluster.find_task_id(task_name)
except requests.exceptions.HTTPError as e:
# If the decision task was not indexed, it means it was broken. So we can
# assume we didn't run any task for this push.
if e.response.status_code == 404:
logger.warning(f"Decision task broken in {rev} on {branch}")
raise ContractNotFilled(
self.name,
"push_test_selection_data",
f"could not retrieve decision task '{task_name}'",
)
try:
results = taskcluster.get_artifact(
decision_task_id, "public/bugbug-push-schedules.json")
except requests.exceptions.HTTPError:
raise ContractNotFilled(
self.name,
"push_test_selection_data",
"could not retrieve schedules from cache",
)
return results
def decision_task(self):
"""A representation of the decision task.
Returns:
Task: A `Task` instance representing the decision task.
"""
index = self.index + ".taskgraph.decision"
task_id = find_task_id(index)
return Task(id=task_id)
def get_shadow_scheduler_tasks(self, name):
"""Returns all tasks the given shadow scheduler would have scheduled,
or None if the given scheduler didn't run.
Args:
name (str): The name of the shadow scheduler to query.
Returns:
list: All task labels that would have been scheduled.
"""
index = self.index + ".source.shadow-scheduler-{}".format(name)
task = Task(id=find_task_id(index))
labels = task.get_artifact("public/shadow-scheduler/optimized_tasks.list")
return set(labels.splitlines())
def create(index=None, **kwargs):
"""Factory method to create a new Task instance.
One of ``index`` or ``id`` must be specified.
Args:
index (str): Taskcluster index path used to find the task id (optional).
kwargs (dict): Arguments to forward to the :class:`~mozci.task.Task` constructor.
Raises:
:class:`~mozci.errors.TaskNotFound`: when the task identified by
specified index or task id could not be found.
"""
if index and "id" not in kwargs:
try:
kwargs["id"] = find_task_id(index)
except requests.exceptions.HTTPError as e:
label = kwargs.get("label", "unknown label")
raise TaskNotFound(id=index, label=label) from e
if kwargs.get("label", "").startswith("test-"):
return TestTask(**kwargs)
return Task(**kwargs)
def handle(self) -> None:
branch = self.option("branch")
environment = self.option("environment")
matrix_room = config.get("matrix-room-id")
current_task_id = os.environ.get("TASK_ID")
try:
nb_pushes = int(self.option("nb-pushes"))
except ValueError:
self.line("<error>Provided --nb-pushes should be an int.</error>")
exit(1)
self.line("<comment>Loading pushes...</comment>")
self.pushes = make_push_objects(nb=nb_pushes, branch=branch)
nb_pushes = len(self.pushes)
to_notify: Dict[str, Dict[str, Any]] = {}
for index, push in enumerate(self.pushes, start=1):
self.line(
f"<comment>Processing push {index}/{nb_pushes}: {push.push_uuid}</comment>"
)
backfill_tasks = []
try:
indexed_tasks = list_indexed_tasks(
f"gecko.v2.{push.branch}.revision.{push.rev}.taskgraph.actions"
)
except requests.exceptions.HTTPError as e:
self.line(
f"<error>Couldn't fetch indexed tasks on push {push.push_uuid}: {e}</error>"
)
continue
for indexed_task in indexed_tasks:
task_id = indexed_task["taskId"]
try:
children_tasks = list_dependent_tasks(task_id)
except requests.exceptions.HTTPError as e:
self.line(
f"<error>Couldn't fetch dependent tasks of indexed task {task_id} on push {push.push_uuid}: {e}</error>"
)
continue
for child_task in children_tasks:
task_section = child_task.get("task", {})
task_action = task_section.get("tags",
{}).get("action", "")
# We are looking for the Treeherder symbol because Sheriffs are
# only interested in backfill-tasks holding the '-bk' suffix in TH
th_symbol = (task_section.get("extra", {}).get(
"treeherder", {}).get("symbol", ""))
status = child_task.get("status", {})
if task_action == "backfill-task" and th_symbol.endswith(
"-bk"):
assert status.get(
"taskId"
), "Missing taskId attribute in backfill task status"
label = task_section.get(
"tags", {}).get("label") or task_section.get(
"metadata", {}).get("name")
assert (
label
), "Missing label attribute in backfill task tags or name attribute in backfill task metadata"
assert status.get(
"state"
), "Missing state attribute in backfill task status"
backfill_tasks.append(
BackfillTask(status["taskId"], label, th_symbol,
status["state"]))
else:
logger.debug(
f"Skipping non-backfill task {status.get('taskId')}"
)
def group_key(task):
return task.th_symbol
# Sorting backfill tasks by their Treeherder symbol
backfill_tasks = sorted(backfill_tasks, key=group_key)
# Grouping ordered backfill tasks by their associated Treeherder symbol
for th_symbol, tasks_iter in groupby(backfill_tasks, group_key):
if th_symbol not in to_notify:
to_notify[th_symbol] = {
"newest_push": None,
"backfill_tasks": set(),
}
# make_push_objects returns the latest pushes in chronological order from oldest to newest
# We only need to store the newest Push that appeared for this Treeherder symbol
to_notify[th_symbol]["newest_push"] = push
# Storing all backfill tasks for this symbol across multiple pushes
to_notify[th_symbol]["backfill_tasks"].update(tasks_iter)
for th_symbol, data in to_notify.items():
all_backfill_tasks = data["backfill_tasks"]
# Checking that all backfill tasks for this symbol are in a "final" state
if not all(task.state in TASK_FINAL_STATES
for task in all_backfill_tasks):
logger.debug(
f"Not all backfill tasks for the Treeherder symbol {th_symbol} are in a final state, not notifying now."
)
continue
newest_push = data["newest_push"]
index_path = f"project.mozci.check-backfill.{environment}.{newest_push.branch}.{newest_push.rev}.{th_symbol}"
try:
find_task_id(index_path,
root_url=COMMUNITY_TASKCLUSTER_ROOT_URL)
except requests.exceptions.HTTPError:
pass
else:
logger.debug(
f"A notification was already sent for the backfill tasks associated to the Treeherder symbol {th_symbol}."
)
continue
try:
parents = [
parent
for parent in newest_push._iterate_parents(max_depth=20)
]
except Exception as e:
logger.debug(
f"Failed to load the last twenty parent pushes for push {newest_push.push_uuid}, because: {e}."
)
parents = None
cleaned_label = re.sub(r"(-e10s|-1proc)?(-\d+)?$", "",
all_backfill_tasks.pop().label)
notification = NOTIFICATION_BACKFILL_GROUP_COMPLETED.format(
th_symbol=th_symbol,
push=newest_push,
tochange=f"&tochange={newest_push.child.rev}",
fromchange=f"&fromchange={parents[-1].rev}" if parents else "",
searchstr=f"&searchStr={cleaned_label}",
)
if not matrix_room:
self.line(
f"<comment>A notification should be sent for the backfill tasks associated to the Treeherder symbol {th_symbol} but no matrix room was provided in the secret.</comment>"
)
logger.debug(f"The notification: {notification}")
continue
# Sending a notification to the Matrix channel defined in secret
notify_matrix(
room=matrix_room,
body=notification,
)
if not current_task_id:
self.line(
f"<comment>The current task should be indexed in {index_path} but TASK_ID environment variable isn't set.</comment>"
)
continue
# Populating the index with the current task to prevent sending the notification once again
index_current_task(
index_path,
root_url=COMMUNITY_TASKCLUSTER_ROOT_URL,
)
def run_push_tasks(self, branch, rev):
try:
decision_task_id = taskcluster.find_task_id(
f"gecko.v2.{branch}.revision.{rev}.taskgraph.decision")
except requests.exceptions.HTTPError as e:
# If the decision task was not indexed, it means it was broken. So we can
# assume we didn't run any task for this push.
if e.response.status_code == 404:
logger.warning(f"Decision task broken in {rev} on {branch}")
return []
raise
task_data = taskcluster.get_task(decision_task_id)
results = taskcluster.get_tasks_in_group(task_data["taskGroupId"])
tasks = []
for result in results:
# Skip the decision task.
if result["status"]["taskId"] == decision_task_id:
continue
# Skip "Action" tasks.
if result["task"]["metadata"]["name"].startswith("Action"):
continue
task = {
"id": result["status"]["taskId"],
"label": result["task"]["metadata"]["name"],
"tags": result["task"]["tags"],
"state": result["status"]["state"],
}
tier = result["task"]["extra"].get("treeherder", {}).get("tier")
if tier:
task["tier"] = tier
# Use the latest run (earlier ones likely had exceptions that
# caused an automatic retry).
if task["state"] != "unscheduled":
run = result["status"]["runs"][-1]
# Normalize the state to match treeherder's values.
if task["state"] == "failed":
task["state"] = "completed"
# Derive a result from the reasonResolved.
reason = run.get("reasonResolved")
if reason == "completed":
task["result"] = "passed"
elif reason in ("canceled", "superseded", "failed"):
task["result"] = reason
elif reason:
task["result"] = "exception"
else:
# Task is not finished, so there is no result yet.
assert task["state"] in ("pending", "running", "exception")
# Compute duration.
if "started" in run and "resolved" in run:
task["duration"] = self.to_ms(
run["resolved"]) - self.to_ms(run["started"])
tasks.append(task)
return tasks