def get_artifact(self, path):
"""Downloads and returns the content of an artifact.
Args:
path (str): The path component of the artifact URL. This is usually
something like `public/file.txt`. The values listed by
the `Task.artifacts` property can be passed into this
function.
Returns:
Contents of the artifact.
Raises:
:class:`~mozci.errors.ArtifactNotFound`: When the requested
artifact does not exist.
"""
try:
data = get_artifact(self.id, path)
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
raise ArtifactNotFound(path, self.id, self.label) from e
raise
if not isinstance(data, HTTPResponse):
return data
output = data.read()
if isinstance(output, bytes):
output = output.decode("utf-8")
return output
def run_push_test_selection_data(self, branch, rev):
task_name = f"gecko.v2.{branch}.revision.{rev}.taskgraph.decision"
try:
decision_task_id = taskcluster.find_task_id(task_name)
except requests.exceptions.HTTPError as e:
# If the decision task was not indexed, it means it was broken. So we can
# assume we didn't run any task for this push.
if e.response.status_code == 404:
logger.warning(f"Decision task broken in {rev} on {branch}")
raise ContractNotFilled(
self.name,
"push_test_selection_data",
f"could not retrieve decision task '{task_name}'",
)
try:
results = taskcluster.get_artifact(
decision_task_id, "public/bugbug-push-schedules.json")
except requests.exceptions.HTTPError:
raise ContractNotFilled(
self.name,
"push_test_selection_data",
"could not retrieve schedules from cache",
)
return results
def _load_errorsummary(self, task_id) -> None:
"""Load the task's errorsummary.log.
We gather all data we can and store it in the TASK_* caches so we don't
have to load it again for a different contract.
"""
try:
artifacts = [a["name"] for a in list_artifacts(task_id)]
paths = [a for a in artifacts if a.endswith("errorsummary.log")]
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
return
raise
except IndexError:
return
groups = set()
group_results = {}
lines = (
json.loads(line)
for path in paths
for line in get_artifact(task_id, path).iter_lines(decode_unicode=True)
if line
)
for line in lines:
if line["action"] == "test_groups":
groups |= set(line["groups"]) - {"default"}
elif line["action"] == "group_result":
group = line["group"]
if group not in group_results or line["status"] != "OK":
group_results[group] = (line["status"], line["duration"])
elif line["action"] == "log":
if task_id not in self.TASK_ERRORS:
self.TASK_ERRORS[task_id] = []
self.TASK_ERRORS[task_id].append(line["message"])
missing_groups = groups - set(group_results)
if len(missing_groups) > 0:
log_level = "DEBUG" if self.is_try else "WARNING"
logger.log(
log_level,
f"Some groups in {task_id} are missing results: {missing_groups}",
)
self.TASK_GROUPS[task_id] = {
group: (result == "OK", duration)
for group, (result, duration) in group_results.items()
if result != "SKIP"
}
def get_artifact(self, path):
"""Downloads and returns the content of an artifact.
Args:
path (str): The path component of the artifact URL. This is usually
something like `public/file.txt`. The values listed by
the `Task.artifacts` property can be passed into this
function.
Returns:
Contents of the artifact.
"""
data = get_artifact(self.id, path)
if not isinstance(data, HTTPResponse):
return data
output = data.read()
if isinstance(output, bytes):
output = output.decode("utf-8")
return output
def get_artifact(self, path, root_url=PRODUCTION_TASKCLUSTER_ROOT_URL):
"""Downloads and returns the content of an artifact.
Args:
path (str): The path component of the artifact URL. This is usually
something like `public/file.txt`. The values listed by
the `Task.artifacts` property can be passed into this
function.
Returns:
Contents of the artifact.
Raises:
:class:`~mozci.errors.ArtifactNotFound`: When the requested
artifact does not exist.
"""
try:
data = get_artifact(self.id, path, root_url=root_url)
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
raise ArtifactNotFound(path, self.id, self.label) from e
raise
return data
def _load_errorsummary(self, task_id) -> None:
"""Load the task's errorsummary.log.
We gather all data we can and store it in the TASK_* caches so we don't
have to load it again for a different contract.
"""
try:
artifacts = [a["name"] for a in list_artifacts(task_id)]
paths = [a for a in artifacts if a.endswith("errorsummary.log")]
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
return
raise
except IndexError:
return
groups = set()
group_results = {}
test_results: Dict[GroupName, List[Tuple[TestName, FailureType]]] = {}
lines = (json.loads(line) for path in paths
for line in get_artifact(task_id, path).iter_lines(
decode_unicode=True) if line)
for line in lines:
if line["action"] == "test_groups":
groups |= set(line["groups"]) - {"default"}
elif line["action"] == "group_result":
group = line["group"]
if group not in group_results or line["status"] != "OK":
group_results[group] = (line["status"], line["duration"])
elif line["action"] == "log":
if task_id not in self.TASK_ERRORS:
self.TASK_ERRORS[task_id] = []
self.TASK_ERRORS[task_id].append(line["message"])
if line.get("test") and line.get("group"):
if not test_results.get(line["group"]):
test_results[line["group"]] = []
if (line.get("status") != line.get("expected")
and line.get("status") == "TIMEOUT"):
failure_type = FailureType.TIMEOUT
elif (line.get("signature") is not None
and line.get("action") == "crash"):
failure_type = FailureType.CRASH
else:
failure_type = FailureType.GENERIC
test_results[line["group"]].append(
(line["test"], failure_type))
missing_groups = groups - set(group_results)
if len(missing_groups) > 0:
log_level = "DEBUG" if self.is_try else "WARNING"
logger.log(
log_level,
f"Some groups in {task_id} are missing results: {missing_groups}",
)
self.TASK_GROUPS[task_id] = {
group: (result == "OK", duration)
for group, (result, duration) in group_results.items()
if result != "SKIP"
}
self.TASK_FAILURE_TYPES[task_id] = test_results