def get_job_log(repo_name, job_id): '''For a given job id return the URL to the log associated to it.''' th_client = TreeherderClient() query_params = {'job_id': job_id, 'name': 'text_log_summary'} try: return str(th_client.get_artifacts(repo_name, **query_params)[0]['blob']['logurl']) except IndexError: print 'No artifacts for {}'.format(job_id) except requests.exceptions.ConnectionError as e: print 'Connection failed for {}'.format(job_id) traceback.print_exc()
class TreeherderApi(QueryApi): def __init__(self): self.treeherder_client = TreeherderClient() def _get_all_jobs(self, repo_name, revision, **params): """ Return all jobs for a given revision. If we can't query about this revision in treeherder api, we return an empty list. """ # We query treeherder for its internal revision_id, and then get the jobs from them. # We cannot get jobs directly from revision and repo_name in TH api. # See: https://bugzilla.mozilla.org/show_bug.cgi?id=1165401 results = self.treeherder_client.get_resultsets(repo_name, revision=revision, **params) all_jobs = [] if results: revision_id = results[0]["id"] all_jobs = self.treeherder_client.get_jobs( repo_name, count=2000, result_set_id=revision_id, **params) return all_jobs def get_buildapi_request_id(self, repo_name, job): """ Method to return buildapi's request_id. """ job_id = job["id"] query_params = {'job_id': job_id, 'name': 'buildapi'} LOG.debug("We are fetching request_id from treeherder artifacts api") artifact_content = self.treeherder_client.get_artifacts( repo_name, **query_params) return artifact_content[0]["blob"]["request_id"] def get_hidden_jobs(self, repo_name, revision): """ Return all hidden jobs on Treeherder """ return self._get_all_jobs(repo_name, revision=revision, visibility='excluded') def get_matching_jobs(self, repo_name, revision, buildername): """ Return all jobs that matched the criteria. """ LOG.debug("Find jobs matching '%s'" % buildername) all_jobs = self._get_all_jobs(repo_name, revision) matching_jobs = [] for j in all_jobs: if j["ref_data_name"] == buildername: matching_jobs.append(j) LOG.debug("We have found %d job(s) of '%s'." % (len(matching_jobs), buildername)) return matching_jobs def get_job_status(self, job): """ Helper to determine the scheduling status of a job from treeherder. Raises a TreeherderError if the job doesn't complete. """ if job["job_coalesced_to_guid"] is not None: return COALESCED if job["result"] == "unknown": if job["state"] == "pending": return PENDING elif job["state"] == "running": return RUNNING else: return UNKNOWN # If the job 'state' is completed, we can have the following possible statuses: # https://github.com/mozilla/treeherder/blob/master/treeherder/etl/buildbot.py#L7 status_dict = { "success": SUCCESS, "busted": FAILURE, "testfailed": FAILURE, "skipped": SKIPPED, "exception": EXCEPTION, "retry": RETRY, "usercancel": CANCELLED } if job["state"] == "completed": return status_dict[job["result"]] LOG.debug(job) raise TreeherderError("Unexpected status")
class TreeherderApi(QueryApi): def __init__(self): self.treeherder_client = TreeherderClient() def get_all_jobs(self, repo_name, revision, **params): """ Return all jobs for a given revision. If we can't query about this revision in treeherder api, we return an empty list. """ # We query treeherder for its internal revision_id, and then get the jobs from them. # We cannot get jobs directly from revision and repo_name in TH api. # See: https://bugzilla.mozilla.org/show_bug.cgi?id=1165401 results = self.treeherder_client.get_resultsets(repo_name, revision=revision, **params) all_jobs = [] if results: revision_id = results[0]["id"] all_jobs = self.treeherder_client.get_jobs(repo_name, count=2000, result_set_id=revision_id, **params) return all_jobs def get_buildapi_request_id(self, repo_name, job): """ Method to return buildapi's request_id. """ job_id = job["id"] query_params = {'job_id': job_id, 'name': 'buildapi'} LOG.debug("We are fetching request_id from treeherder artifacts api") artifact_content = self.treeherder_client.get_artifacts(repo_name, **query_params) return artifact_content[0]["blob"]["request_id"] def get_hidden_jobs(self, repo_name, revision): """ Return all hidden jobs on Treeherder """ return self.get_all_jobs(repo_name, revision=revision, visibility='excluded') def get_matching_jobs(self, repo_name, revision, buildername): """ Return all jobs that matched the criteria. """ LOG.debug("Find jobs matching '%s'" % buildername) all_jobs = self.get_all_jobs(repo_name, revision) matching_jobs = [] for j in all_jobs: if j["ref_data_name"] == buildername: matching_jobs.append(j) LOG.debug("We have found %d job(s) of '%s'." % (len(matching_jobs), buildername)) return matching_jobs def get_job_status(self, job): """ Helper to determine the scheduling status of a job from treeherder. Raises a TreeherderError if the job doesn't complete. """ if job["job_coalesced_to_guid"] is not None: return COALESCED if job["result"] == "unknown": if job["state"] == "pending": return PENDING elif job["state"] == "running": return RUNNING else: return UNKNOWN # If the job 'state' is completed, we can have the following possible statuses: # https://github.com/mozilla/treeherder/blob/master/treeherder/etl/buildbot.py#L7 status_dict = { "success": SUCCESS, "busted": FAILURE, "testfailed": FAILURE, "skipped": SKIPPED, "exception": EXCEPTION, "retry": RETRY, "usercancel": CANCELLED } if job["state"] == "completed": return status_dict[job["result"]] LOG.debug(job) raise TreeherderError("Unexpected status") def find_all_jobs_by_status(self, repo_name, revision, status): builder_names = [] jobs = self.get_all_jobs(repo_name, revision) # filer out those jobs without builder name jobs = [job for job in jobs if job['machine_name'] != 'unknown'] for job in jobs: try: job_status = self.get_job_status(job) except TreeherderError: continue if job_status == status: if job['build_system_type'] == 'taskcluster': job_name = job['job_type_name'] else: job_name = job['ref_data_name'] builder_names.append(job_name) return builder_names
class TreeherderApi(QueryApi): def __init__(self, server_url='https://treeherder.mozilla.org', treeherder_host=None): if treeherder_host: LOG.warning( "The `TreeherderApi()` parameter `treeherder_host` is deprecated. " "Use `server_url` instead, or omit entirely to use the default of " "production Treeherder.") server_url = 'https://%s' % treeherder_host self.treeherder_client = TreeherderClient(server_url=server_url) def get_all_jobs(self, repo_name, revision, **params): """ Return all jobs for a given revision. If we can't query about this revision in treeherder api, we return an empty list. """ # We query treeherder for its internal revision_id, and then get the jobs from them. # We cannot get jobs directly from revision and repo_name in TH api. # See: https://bugzilla.mozilla.org/show_bug.cgi?id=1165401 results = self.treeherder_client.get_resultsets(repo_name, revision=revision, **params) all_jobs = [] if results: revision_id = results[0]["id"] all_jobs = self.treeherder_client.get_jobs( repo_name, count=2000, result_set_id=revision_id, **params) return all_jobs def get_buildapi_request_id(self, repo_name, job): """ Method to return buildapi's request_id. """ job_id = job["id"] query_params = {'job_id': job_id, 'name': 'buildapi'} LOG.debug("We are fetching request_id from treeherder artifacts api") artifact_content = self.treeherder_client.get_artifacts( repo_name, **query_params) # This can raise IndexError return artifact_content[0]["blob"]["request_id"] def get_hidden_jobs(self, repo_name, revision): """ Return all hidden jobs on Treeherder """ return self.get_all_jobs(repo_name, revision=revision, visibility='excluded') def get_matching_jobs(self, repo_name, revision, buildername): """ Return all jobs that matched the criteria. """ LOG.debug("Find jobs matching '%s'" % buildername) all_jobs = self.get_all_jobs(repo_name, revision) matching_jobs = [] for j in all_jobs: if j["ref_data_name"] == buildername: matching_jobs.append(j) LOG.debug("We have found %d job(s) of '%s'." % (len(matching_jobs), buildername)) return matching_jobs def get_job_status(self, job): """ Helper to determine the scheduling status of a job from treeherder. Raises a TreeherderError if the job doesn't complete. """ if job["job_coalesced_to_guid"] is not None: return COALESCED if job["result"] == "unknown": if job["state"] == "pending": return PENDING elif job["state"] == "running": return RUNNING else: return UNKNOWN # If the job 'state' is completed, we can have the following possible statuses: # https://github.com/mozilla/treeherder/blob/master/treeherder/etl/buildbot.py#L7 status_dict = { "success": SUCCESS, "busted": FAILURE, "testfailed": FAILURE, "skipped": SKIPPED, "exception": EXCEPTION, "retry": RETRY, "usercancel": CANCELLED } if job["state"] == "completed": return status_dict[job["result"]] LOG.debug(job) raise TreeherderError("Unexpected status") def find_all_jobs_by_status(self, repo_name, revision, status): builder_names = [] jobs = self.get_all_jobs(repo_name, revision) # filer out those jobs without builder name jobs = [job for job in jobs if job['machine_name'] != 'unknown'] for job in jobs: try: job_status = self.get_job_status(job) except TreeherderError: continue if job_status == status: if job['build_system_type'] == 'taskcluster': job_name = job['job_type_name'] else: job_name = job['ref_data_name'] builder_names.append(job_name) return builder_names def query_revision_for_job(self, repo_name, job_id): '''Return revision for a known Treeherder job id.''' job_info = self.treeherder_client.get_jobs(repo_name, id=job_id)[0] result_sets = self.treeherder_client.get_resultsets( repo_name, id=job_info["result_set_id"]) revision = result_sets[0]["revision"] return revision def query_revision_for_resultset(self, repo_name, resultset_id): '''Return revision for a known Treeherder resultset id.''' return self.treeherder_client.get_resultsets( repo_name, id=resultset_id)[0]["revision"]