def get_job_url(task_id, run_id, **params): """Build a Treeherder job url for a given Taskcluster task""" treeherder_client = TreeherderClient() uuid = slugid.decode(task_id) # Fetch specific job id from treeherder job_details = treeherder_client.get_job_details( job_guid=f"{uuid}/{run_id}") if len(job_details) > 0: params["selectedJob"] = job_details[0]["job_id"] return f"https://treeherder.mozilla.org/#/jobs?{urlencode(params)}"
def get_job_url(repository, revision, task_id=None, run_id=None, **params): """Build a Treeherder job url for a given Taskcluster task""" assert isinstance(repository, str) and repository, "Missing repository" assert isinstance(revision, str) and revision, "Missing revision" assert "repo" not in params, "repo cannot be set in params" assert "revision" not in params, "revision cannot be set in params" params.update({"repo": repository, "revision": revision}) if task_id is not None and run_id is not None: treeherder_client = TreeherderClient() uuid = slugid.decode(task_id) # Fetch specific job id from treeherder job_details = treeherder_client.get_job_details( job_guid=f"{uuid}/{run_id}") if len(job_details) > 0: params["selectedJob"] = job_details[0]["job_id"] return f"{JOBS_URL}?{urlencode(params)}"
class TreeherderApi(QueryApi): def __init__(self, server_url='https://treeherder.mozilla.org', treeherder_host=None): if treeherder_host: LOG.warning( "The `TreeherderApi()` parameter `treeherder_host` is deprecated. " "Use `server_url` instead, or omit entirely to use the default of " "production Treeherder.") server_url = 'https://%s' % treeherder_host self.treeherder_client = TreeherderClient(server_url=server_url) def get_all_jobs(self, repo_name, revision, **params): """ Return all jobs for a given revision. If we can't query about this revision in treeherder api, we return an empty list. """ # We query treeherder for its internal revision_id, and then get the jobs from them. # We cannot get jobs directly from revision and repo_name in TH api. # See: https://bugzilla.mozilla.org/show_bug.cgi?id=1165401 results = self.treeherder_client.get_resultsets(repo_name, revision=revision, **params) all_jobs = [] if results: revision_id = results[0]["id"] all_jobs = self.treeherder_client.get_jobs( repo_name, count=2000, result_set_id=revision_id, **params) return all_jobs def get_buildapi_request_id(self, repo_name, job): """ Method to return buildapi's request_id. """ job_details = self.treeherder_client.get_job_details( job_id=job["id"], title='buildbot_request_id', repository=repo_name) if not job_details: raise ValueError( "No buildbot request id for job ({}, {}, {})".format( job["id"], 'buildbot_request_id', repo_name)) return int(job_details[0]["value"]) def get_hidden_jobs(self, repo_name, revision): """ Return all hidden jobs on Treeherder """ return self.get_all_jobs(repo_name, revision=revision, visibility='excluded') def get_matching_jobs(self, repo_name, revision, buildername): """ Return all jobs that matched the criteria. """ LOG.debug("Find jobs matching '%s'" % buildername) all_jobs = self.get_all_jobs(repo_name, revision) matching_jobs = [] for j in all_jobs: if j["ref_data_name"] == buildername: matching_jobs.append(j) LOG.debug("We have found %d job(s) of '%s'." % (len(matching_jobs), buildername)) return matching_jobs def get_job_status(self, job): """ Helper to determine the scheduling status of a job from treeherder. Raises a TreeherderError if the job doesn't complete. """ if job["job_coalesced_to_guid"] is not None: return COALESCED if job["result"] == "unknown": if job["state"] == "pending": return PENDING elif job["state"] == "running": return RUNNING else: return UNKNOWN # If the job 'state' is completed, we can have the following possible statuses: # https://github.com/mozilla/treeherder/blob/master/treeherder/etl/buildbot.py#L7 status_dict = { "success": SUCCESS, "busted": FAILURE, "testfailed": FAILURE, "skipped": SKIPPED, "exception": EXCEPTION, "retry": RETRY, "usercancel": CANCELLED } if job["state"] == "completed": return status_dict[job["result"]] LOG.debug(job) raise TreeherderError("Unexpected status") def find_all_jobs_by_status(self, repo_name, revision, status): builder_names = [] jobs = self.get_all_jobs(repo_name, revision) # filer out those jobs without builder name jobs = [job for job in jobs if job['machine_name'] != 'unknown'] for job in jobs: try: job_status = self.get_job_status(job) except TreeherderError: continue if job_status == status: if job['build_system_type'] == 'taskcluster': job_name = job['job_type_name'] else: job_name = job['ref_data_name'] builder_names.append(job_name) return builder_names def query_revision_for_job(self, repo_name, job_id): '''Return revision for a known Treeherder job id.''' job_info = self.treeherder_client.get_jobs(repo_name, id=job_id)[0] result_sets = self.treeherder_client.get_resultsets( repo_name, id=job_info["result_set_id"]) revision = result_sets[0]["revision"] return revision def query_revision_for_resultset(self, repo_name, resultset_id): '''Return revision for a known Treeherder resultset id.''' return self.treeherder_client.get_resultsets( repo_name, id=resultset_id)[0]["revision"]
class TreeherderApi(QueryApi): def __init__(self, server_url='https://treeherder.mozilla.org', treeherder_host=None): if treeherder_host: LOG.warning("The `TreeherderApi()` parameter `treeherder_host` is deprecated. " "Use `server_url` instead, or omit entirely to use the default of " "production Treeherder.") server_url = 'https://%s' % treeherder_host self.treeherder_client = TreeherderClient(server_url=server_url) def get_all_jobs(self, repo_name, revision, **params): """ Return all jobs for a given revision. If we can't query about this revision in treeherder api, we return an empty list. """ # We query treeherder for its internal revision_id, and then get the jobs from them. # We cannot get jobs directly from revision and repo_name in TH api. # See: https://bugzilla.mozilla.org/show_bug.cgi?id=1165401 results = self.treeherder_client.get_resultsets(repo_name, revision=revision, **params) all_jobs = [] if results: revision_id = results[0]["id"] all_jobs = self.treeherder_client.get_jobs(repo_name, count=2000, result_set_id=revision_id, **params) return all_jobs def get_buildapi_request_id(self, repo_name, job): """ Method to return buildapi's request_id. """ job_details = self.treeherder_client.get_job_details( job_id=job["id"], title='buildbot_request_id', repository=repo_name) if not job_details: raise ValueError("No buildbot request id for job ({}, {}, {})".format( job["id"], 'buildbot_request_id', repo_name )) return int(job_details[0]["value"]) def get_hidden_jobs(self, repo_name, revision): """ Return all hidden jobs on Treeherder """ return self.get_all_jobs(repo_name, revision=revision, visibility='excluded') def get_matching_jobs(self, repo_name, revision, buildername): """ Return all jobs that matched the criteria. """ LOG.debug("Find jobs matching '%s'" % buildername) all_jobs = self.get_all_jobs(repo_name, revision) matching_jobs = [] for j in all_jobs: if j["ref_data_name"] == buildername: matching_jobs.append(j) LOG.debug("We have found %d job(s) of '%s'." % (len(matching_jobs), buildername)) return matching_jobs def get_job_status(self, job): """ Helper to determine the scheduling status of a job from treeherder. Raises a TreeherderError if the job doesn't complete. """ if job["job_coalesced_to_guid"] is not None: return COALESCED if job["result"] == "unknown": if job["state"] == "pending": return PENDING elif job["state"] == "running": return RUNNING else: return UNKNOWN # If the job 'state' is completed, we can have the following possible statuses: # https://github.com/mozilla/treeherder/blob/master/treeherder/etl/buildbot.py#L7 status_dict = { "success": SUCCESS, "busted": FAILURE, "testfailed": FAILURE, "skipped": SKIPPED, "exception": EXCEPTION, "retry": RETRY, "usercancel": CANCELLED } if job["state"] == "completed": return status_dict[job["result"]] LOG.debug(job) raise TreeherderError("Unexpected status") def find_all_jobs_by_status(self, repo_name, revision, status): builder_names = [] jobs = self.get_all_jobs(repo_name, revision) # filer out those jobs without builder name jobs = [job for job in jobs if job['machine_name'] != 'unknown'] for job in jobs: try: job_status = self.get_job_status(job) except TreeherderError: continue if job_status == status: if job['build_system_type'] == 'taskcluster': job_name = job['job_type_name'] else: job_name = job['ref_data_name'] builder_names.append(job_name) return builder_names def query_revision_for_job(self, repo_name, job_id): '''Return revision for a known Treeherder job id.''' job_info = self.treeherder_client.get_jobs(repo_name, id=job_id)[0] result_sets = self.treeherder_client.get_resultsets(repo_name, id=job_info["result_set_id"]) revision = result_sets[0]["revision"] return revision def query_revision_for_resultset(self, repo_name, resultset_id): '''Return revision for a known Treeherder resultset id.''' return self.treeherder_client.get_resultsets(repo_name, id=resultset_id)[0]["revision"]
def on_event(data, message, dry_run, treeherder_server_url, **kwargs): """Act upon Treeherder job events. Return if the outcome was successful or not """ exit_code = 0 # SUCCESS if ignored(data): return exit_code # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} treeherder_client = TreeherderClient(server_url=treeherder_server_url) action = data['action'].capitalize() job_id = data['job_id'] repo_name = data['project'] status = None # We want to know the status of the job we're processing try: job_info = treeherder_client.get_jobs(repo_name, id=job_id)[0] except IndexError: LOG.info("We could not find any job_info for repo_name: %s and " "job_id: %s" % (repo_name, job_id)) return exit_code # We want to know the revision associated for this job result_set = treeherder_client.get_resultsets( repo_name, id=job_info["result_set_id"])[0] revision = result_set["revision"] link_to_job = '{}/#/jobs?repo={}&revision={}&selectedJob={}'.format( treeherder_server_url, repo_name, revision, job_id) # There are various actions that can be taken on a job, however, we currently # only process the backfill one if action == "Backfill": if job_info["build_system_type"] == "taskcluster": jobs = [] jobs_per_call = 250 offset = 0 while True: results = treeherder_client.get_jobs( repo_name, push_id=job_info["result_set_id"], count=jobs_per_call, offset=offset) jobs += results if (len(results) < jobs_per_call): break offset += jobs_per_call decision = [ t for t in jobs if t["job_type_name"] == "Gecko Decision Task" ][0] details = treeherder_client.get_job_details( job_guid=decision["job_guid"]) inspect = [ detail["url"] for detail in details if detail["value"] == "Inspect Task" ][0] # Pull out the taskId from the URL e.g. # oN1NErz_Rf2DZJ1hi7YVfA from <tc_tools_site>/task-inspector/#oN1NErz_Rf2DZJ1hi7YVfA/ decision_id = inspect.partition("#")[-1].rpartition("/")[0] mgr = TaskClusterManager(dry_run=dry_run) mgr.schedule_action_task(decision_id=decision_id, action="backfill", action_args={ "project": repo_name, "job": job_info["id"] }) else: buildername = job_info["ref_data_name"] LOG.info("{} action requested by {} for '{}'".format( action, data['requester'], buildername, )) LOG.info('Request for {}'.format(link_to_job)) buildername = filter_invalid_builders(buildername) if buildername is None: LOG.info('Treeherder can send us invalid builder names.') LOG.info( 'See https://bugzilla.mozilla.org/show_bug.cgi?id=1242038.' ) LOG.warning('Requested job name "%s" is invalid.' % job_info['ref_data_name']) exit_code = -1 # FAILURE else: exit_code = manual_backfill( revision=revision, buildername=buildername, dry_run=dry_run, ) if not dry_run: status = 'Backfill request sent' else: status = 'Dry-run mode, nothing was backfilled.' LOG.debug(status) else: LOG.error('We were not aware of the "{}" action. Please file an issue'. format(action)) exit_code = -1 # FAILURE return exit_code
for job in jobs: logging.debug(job['job_type_name']) if ( job['state'] == 'completed' and job['job_type_name'] == mapping_builds[tb_version][platform] ): logging.info("%d\t%s\t%s\t%s\t%s\t%s" % ( job['start_timestamp'], job['build_platform'], job['job_type_name'], job['platform'], job['platform_option'], job['state']) ) found_test = False found_app = False for detail in client.get_job_details(job_guid=job['job_guid']): if detail['title'] == 'artifact uploaded': logging.debug('\t\t' + detail['url']) if detail['url'].find(platform_data['testzip']) >= 0: found_test = True test_urls[platform] = '/'.join( detail['url'].split('/')[:-1] ) data[tb_version][platform]['testzip'] = ( detail['url'].split('/')[-1] ) elif detail['url'].find(platform_data['appzip']) >= 0: found_app = True app_urls[platform] = '/'.join( detail['url'].split('/')[:-1]) if found_app and found_test: