def get_job_log(repo_name, job_id):
'''For a given job id return the URL to the log associated to it.'''
th_client = TreeherderClient()
query_params = {'job_id': job_id, 'name': 'text_log_summary'}
try:
return str(th_client.get_artifacts(repo_name, **query_params)[0]['blob']['logurl'])
except IndexError:
print 'No artifacts for {}'.format(job_id)
except requests.exceptions.ConnectionError as e:
print 'Connection failed for {}'.format(job_id)
traceback.print_exc()
class TreeherderApi(QueryApi):
def __init__(self):
self.treeherder_client = TreeherderClient()
def _get_all_jobs(self, repo_name, revision, **params):
"""
Return all jobs for a given revision.
If we can't query about this revision in treeherder api, we return an empty list.
"""
# We query treeherder for its internal revision_id, and then get the jobs from them.
# We cannot get jobs directly from revision and repo_name in TH api.
# See: https://bugzilla.mozilla.org/show_bug.cgi?id=1165401
results = self.treeherder_client.get_resultsets(repo_name,
revision=revision,
**params)
all_jobs = []
if results:
revision_id = results[0]["id"]
all_jobs = self.treeherder_client.get_jobs(
repo_name, count=2000, result_set_id=revision_id, **params)
return all_jobs
def get_buildapi_request_id(self, repo_name, job):
""" Method to return buildapi's request_id. """
job_id = job["id"]
query_params = {'job_id': job_id, 'name': 'buildapi'}
LOG.debug("We are fetching request_id from treeherder artifacts api")
artifact_content = self.treeherder_client.get_artifacts(
repo_name, **query_params)
return artifact_content[0]["blob"]["request_id"]
def get_hidden_jobs(self, repo_name, revision):
""" Return all hidden jobs on Treeherder """
return self._get_all_jobs(repo_name,
revision=revision,
visibility='excluded')
def get_matching_jobs(self, repo_name, revision, buildername):
"""
Return all jobs that matched the criteria.
"""
LOG.debug("Find jobs matching '%s'" % buildername)
all_jobs = self._get_all_jobs(repo_name, revision)
matching_jobs = []
for j in all_jobs:
if j["ref_data_name"] == buildername:
matching_jobs.append(j)
LOG.debug("We have found %d job(s) of '%s'." %
(len(matching_jobs), buildername))
return matching_jobs
def get_job_status(self, job):
"""
Helper to determine the scheduling status of a job from treeherder.
Raises a TreeherderError if the job doesn't complete.
"""
if job["job_coalesced_to_guid"] is not None:
return COALESCED
if job["result"] == "unknown":
if job["state"] == "pending":
return PENDING
elif job["state"] == "running":
return RUNNING
else:
return UNKNOWN
# If the job 'state' is completed, we can have the following possible statuses:
# https://github.com/mozilla/treeherder/blob/master/treeherder/etl/buildbot.py#L7
status_dict = {
"success": SUCCESS,
"busted": FAILURE,
"testfailed": FAILURE,
"skipped": SKIPPED,
"exception": EXCEPTION,
"retry": RETRY,
"usercancel": CANCELLED
}
if job["state"] == "completed":
return status_dict[job["result"]]
LOG.debug(job)
raise TreeherderError("Unexpected status")
class TreeherderApi(QueryApi):
def __init__(self):
self.treeherder_client = TreeherderClient()
def get_all_jobs(self, repo_name, revision, **params):
"""
Return all jobs for a given revision.
If we can't query about this revision in treeherder api, we return an empty list.
"""
# We query treeherder for its internal revision_id, and then get the jobs from them.
# We cannot get jobs directly from revision and repo_name in TH api.
# See: https://bugzilla.mozilla.org/show_bug.cgi?id=1165401
results = self.treeherder_client.get_resultsets(repo_name, revision=revision, **params)
all_jobs = []
if results:
revision_id = results[0]["id"]
all_jobs = self.treeherder_client.get_jobs(repo_name, count=2000,
result_set_id=revision_id, **params)
return all_jobs
def get_buildapi_request_id(self, repo_name, job):
""" Method to return buildapi's request_id. """
job_id = job["id"]
query_params = {'job_id': job_id,
'name': 'buildapi'}
LOG.debug("We are fetching request_id from treeherder artifacts api")
artifact_content = self.treeherder_client.get_artifacts(repo_name,
**query_params)
return artifact_content[0]["blob"]["request_id"]
def get_hidden_jobs(self, repo_name, revision):
""" Return all hidden jobs on Treeherder """
return self.get_all_jobs(repo_name, revision=revision, visibility='excluded')
def get_matching_jobs(self, repo_name, revision, buildername):
"""
Return all jobs that matched the criteria.
"""
LOG.debug("Find jobs matching '%s'" % buildername)
all_jobs = self.get_all_jobs(repo_name, revision)
matching_jobs = []
for j in all_jobs:
if j["ref_data_name"] == buildername:
matching_jobs.append(j)
LOG.debug("We have found %d job(s) of '%s'." %
(len(matching_jobs), buildername))
return matching_jobs
def get_job_status(self, job):
"""
Helper to determine the scheduling status of a job from treeherder.
Raises a TreeherderError if the job doesn't complete.
"""
if job["job_coalesced_to_guid"] is not None:
return COALESCED
if job["result"] == "unknown":
if job["state"] == "pending":
return PENDING
elif job["state"] == "running":
return RUNNING
else:
return UNKNOWN
# If the job 'state' is completed, we can have the following possible statuses:
# https://github.com/mozilla/treeherder/blob/master/treeherder/etl/buildbot.py#L7
status_dict = {
"success": SUCCESS,
"busted": FAILURE,
"testfailed": FAILURE,
"skipped": SKIPPED,
"exception": EXCEPTION,
"retry": RETRY,
"usercancel": CANCELLED
}
if job["state"] == "completed":
return status_dict[job["result"]]
LOG.debug(job)
raise TreeherderError("Unexpected status")
def find_all_jobs_by_status(self, repo_name, revision, status):
builder_names = []
jobs = self.get_all_jobs(repo_name, revision)
# filer out those jobs without builder name
jobs = [job for job in jobs if job['machine_name'] != 'unknown']
for job in jobs:
try:
job_status = self.get_job_status(job)
except TreeherderError:
continue
if job_status == status:
if job['build_system_type'] == 'taskcluster':
job_name = job['job_type_name']
else:
job_name = job['ref_data_name']
builder_names.append(job_name)
return builder_names
class TreeherderApi(QueryApi):
def __init__(self,
server_url='https://treeherder.mozilla.org',
treeherder_host=None):
if treeherder_host:
LOG.warning(
"The `TreeherderApi()` parameter `treeherder_host` is deprecated. "
"Use `server_url` instead, or omit entirely to use the default of "
"production Treeherder.")
server_url = 'https://%s' % treeherder_host
self.treeherder_client = TreeherderClient(server_url=server_url)
def get_all_jobs(self, repo_name, revision, **params):
"""
Return all jobs for a given revision.
If we can't query about this revision in treeherder api, we return an empty list.
"""
# We query treeherder for its internal revision_id, and then get the jobs from them.
# We cannot get jobs directly from revision and repo_name in TH api.
# See: https://bugzilla.mozilla.org/show_bug.cgi?id=1165401
results = self.treeherder_client.get_resultsets(repo_name,
revision=revision,
**params)
all_jobs = []
if results:
revision_id = results[0]["id"]
all_jobs = self.treeherder_client.get_jobs(
repo_name, count=2000, result_set_id=revision_id, **params)
return all_jobs
def get_buildapi_request_id(self, repo_name, job):
""" Method to return buildapi's request_id. """
job_id = job["id"]
query_params = {'job_id': job_id, 'name': 'buildapi'}
LOG.debug("We are fetching request_id from treeherder artifacts api")
artifact_content = self.treeherder_client.get_artifacts(
repo_name, **query_params)
# This can raise IndexError
return artifact_content[0]["blob"]["request_id"]
def get_hidden_jobs(self, repo_name, revision):
""" Return all hidden jobs on Treeherder """
return self.get_all_jobs(repo_name,
revision=revision,
visibility='excluded')
def get_matching_jobs(self, repo_name, revision, buildername):
"""
Return all jobs that matched the criteria.
"""
LOG.debug("Find jobs matching '%s'" % buildername)
all_jobs = self.get_all_jobs(repo_name, revision)
matching_jobs = []
for j in all_jobs:
if j["ref_data_name"] == buildername:
matching_jobs.append(j)
LOG.debug("We have found %d job(s) of '%s'." %
(len(matching_jobs), buildername))
return matching_jobs
def get_job_status(self, job):
"""
Helper to determine the scheduling status of a job from treeherder.
Raises a TreeherderError if the job doesn't complete.
"""
if job["job_coalesced_to_guid"] is not None:
return COALESCED
if job["result"] == "unknown":
if job["state"] == "pending":
return PENDING
elif job["state"] == "running":
return RUNNING
else:
return UNKNOWN
# If the job 'state' is completed, we can have the following possible statuses:
# https://github.com/mozilla/treeherder/blob/master/treeherder/etl/buildbot.py#L7
status_dict = {
"success": SUCCESS,
"busted": FAILURE,
"testfailed": FAILURE,
"skipped": SKIPPED,
"exception": EXCEPTION,
"retry": RETRY,
"usercancel": CANCELLED
}
if job["state"] == "completed":
return status_dict[job["result"]]
LOG.debug(job)
raise TreeherderError("Unexpected status")
def find_all_jobs_by_status(self, repo_name, revision, status):
builder_names = []
jobs = self.get_all_jobs(repo_name, revision)
# filer out those jobs without builder name
jobs = [job for job in jobs if job['machine_name'] != 'unknown']
for job in jobs:
try:
job_status = self.get_job_status(job)
except TreeherderError:
continue
if job_status == status:
if job['build_system_type'] == 'taskcluster':
job_name = job['job_type_name']
else:
job_name = job['ref_data_name']
builder_names.append(job_name)
return builder_names
def query_revision_for_job(self, repo_name, job_id):
'''Return revision for a known Treeherder job id.'''
job_info = self.treeherder_client.get_jobs(repo_name, id=job_id)[0]
result_sets = self.treeherder_client.get_resultsets(
repo_name, id=job_info["result_set_id"])
revision = result_sets[0]["revision"]
return revision
def query_revision_for_resultset(self, repo_name, resultset_id):
'''Return revision for a known Treeherder resultset id.'''
return self.treeherder_client.get_resultsets(
repo_name, id=resultset_id)[0]["revision"]