def get_artifact(self, path): """Downloads and returns the content of an artifact. Args: path (str): The path component of the artifact URL. This is usually something like `public/file.txt`. The values listed by the `Task.artifacts` property can be passed into this function. Returns: Contents of the artifact. Raises: :class:`~mozci.errors.ArtifactNotFound`: When the requested artifact does not exist. """ try: data = get_artifact(self.id, path) except requests.exceptions.HTTPError as e: if e.response.status_code == 404: raise ArtifactNotFound(path, self.id, self.label) from e raise if not isinstance(data, HTTPResponse): return data output = data.read() if isinstance(output, bytes): output = output.decode("utf-8") return output
def run_push_test_selection_data(self, branch, rev): task_name = f"gecko.v2.{branch}.revision.{rev}.taskgraph.decision" try: decision_task_id = taskcluster.find_task_id(task_name) except requests.exceptions.HTTPError as e: # If the decision task was not indexed, it means it was broken. So we can # assume we didn't run any task for this push. if e.response.status_code == 404: logger.warning(f"Decision task broken in {rev} on {branch}") raise ContractNotFilled( self.name, "push_test_selection_data", f"could not retrieve decision task '{task_name}'", ) try: results = taskcluster.get_artifact( decision_task_id, "public/bugbug-push-schedules.json") except requests.exceptions.HTTPError: raise ContractNotFilled( self.name, "push_test_selection_data", "could not retrieve schedules from cache", ) return results
def _load_errorsummary(self, task_id) -> None: """Load the task's errorsummary.log. We gather all data we can and store it in the TASK_* caches so we don't have to load it again for a different contract. """ try: artifacts = [a["name"] for a in list_artifacts(task_id)] paths = [a for a in artifacts if a.endswith("errorsummary.log")] except requests.exceptions.HTTPError as e: if e.response.status_code == 404: return raise except IndexError: return groups = set() group_results = {} lines = ( json.loads(line) for path in paths for line in get_artifact(task_id, path).iter_lines(decode_unicode=True) if line ) for line in lines: if line["action"] == "test_groups": groups |= set(line["groups"]) - {"default"} elif line["action"] == "group_result": group = line["group"] if group not in group_results or line["status"] != "OK": group_results[group] = (line["status"], line["duration"]) elif line["action"] == "log": if task_id not in self.TASK_ERRORS: self.TASK_ERRORS[task_id] = [] self.TASK_ERRORS[task_id].append(line["message"]) missing_groups = groups - set(group_results) if len(missing_groups) > 0: log_level = "DEBUG" if self.is_try else "WARNING" logger.log( log_level, f"Some groups in {task_id} are missing results: {missing_groups}", ) self.TASK_GROUPS[task_id] = { group: (result == "OK", duration) for group, (result, duration) in group_results.items() if result != "SKIP" }
def get_artifact(self, path): """Downloads and returns the content of an artifact. Args: path (str): The path component of the artifact URL. This is usually something like `public/file.txt`. The values listed by the `Task.artifacts` property can be passed into this function. Returns: Contents of the artifact. """ data = get_artifact(self.id, path) if not isinstance(data, HTTPResponse): return data output = data.read() if isinstance(output, bytes): output = output.decode("utf-8") return output
def get_artifact(self, path, root_url=PRODUCTION_TASKCLUSTER_ROOT_URL): """Downloads and returns the content of an artifact. Args: path (str): The path component of the artifact URL. This is usually something like `public/file.txt`. The values listed by the `Task.artifacts` property can be passed into this function. Returns: Contents of the artifact. Raises: :class:`~mozci.errors.ArtifactNotFound`: When the requested artifact does not exist. """ try: data = get_artifact(self.id, path, root_url=root_url) except requests.exceptions.HTTPError as e: if e.response.status_code == 404: raise ArtifactNotFound(path, self.id, self.label) from e raise return data
def _load_errorsummary(self, task_id) -> None: """Load the task's errorsummary.log. We gather all data we can and store it in the TASK_* caches so we don't have to load it again for a different contract. """ try: artifacts = [a["name"] for a in list_artifacts(task_id)] paths = [a for a in artifacts if a.endswith("errorsummary.log")] except requests.exceptions.HTTPError as e: if e.response.status_code == 404: return raise except IndexError: return groups = set() group_results = {} test_results: Dict[GroupName, List[Tuple[TestName, FailureType]]] = {} lines = (json.loads(line) for path in paths for line in get_artifact(task_id, path).iter_lines( decode_unicode=True) if line) for line in lines: if line["action"] == "test_groups": groups |= set(line["groups"]) - {"default"} elif line["action"] == "group_result": group = line["group"] if group not in group_results or line["status"] != "OK": group_results[group] = (line["status"], line["duration"]) elif line["action"] == "log": if task_id not in self.TASK_ERRORS: self.TASK_ERRORS[task_id] = [] self.TASK_ERRORS[task_id].append(line["message"]) if line.get("test") and line.get("group"): if not test_results.get(line["group"]): test_results[line["group"]] = [] if (line.get("status") != line.get("expected") and line.get("status") == "TIMEOUT"): failure_type = FailureType.TIMEOUT elif (line.get("signature") is not None and line.get("action") == "crash"): failure_type = FailureType.CRASH else: failure_type = FailureType.GENERIC test_results[line["group"]].append( (line["test"], failure_type)) missing_groups = groups - set(group_results) if len(missing_groups) > 0: log_level = "DEBUG" if self.is_try else "WARNING" logger.log( log_level, f"Some groups in {task_id} are missing results: {missing_groups}", ) self.TASK_GROUPS[task_id] = { group: (result == "OK", duration) for group, (result, duration) in group_results.items() if result != "SKIP" } self.TASK_FAILURE_TYPES[task_id] = test_results