def get_task_url(task_id, use_proxy=False): if use_proxy: # Until bug 1460015 is finished, use the old baseUrl style of proxy URL task_tmpl = os.environ['TASKCLUSTER_PROXY_URL'] + '/queue/v1/task/{}' else: task_tmpl = liburls.api(get_root_url(), 'queue', 'v1', 'task/{}') return task_tmpl.format(task_id)
def get_index_url(index_path, use_proxy=False, multiple=False): if use_proxy: # Until bug 1460015 is finished, use the old baseUrl style of proxy URL index_tmpl = os.environ['TASKCLUSTER_PROXY_URL'] + '/index/v1/task{}/{}' else: index_tmpl = liburls.api(get_root_url(), 'index', 'v1', 'task{}/{}') return index_tmpl.format('s' if multiple else '', index_path)
def find_task_id(index_path, root_url): index_url = liburls.api(root_url, 'index', 'v1', 'task/{}'.format(index_path)) response = requests.get(index_url) if response.status_code == 404: raise Exception("Index URL {} not found".format(index_url)) return response.json()['taskId']
async def addArtifactUploadedLinks(root_url, taskId, runId, job): artifacts = [] try: artifacts = await fetchArtifacts(root_url, taskId, runId) except Exception: logger.warning("Artifacts could not be found for task: %s run: %s", taskId, runId) return job seen = {} links = [] for artifact in artifacts: name = os.path.basename(artifact["name"]) if not seen.get(name): seen[name] = [artifact["name"]] else: seen[name].append(artifact["name"]) name = "{name} ({length})".format(name=name, length=len(seen[name])-1) links.append({ "label": "artifact uploaded", "linkText": name, "url": taskcluster_urls.api( root_url, "queue", "v1", "task/{taskId}/runs/{runId}/artifacts/{artifact_name}".format( taskId=taskId, runId=runId, artifact_name=artifact["name"] )), }) job["jobInfo"]["links"] = links return job
def get_indexed_tasks_url(namespace, root_url=PRODUCTION_TASKCLUSTER_ROOT_URL): return liburls.api( root_url, "index", "v1", f"tasks/{namespace}", )
class TestBuildUrl(ClientTest): apiPath = liburls.api(ClientTest.test_root_url, 'fake', 'v1', 'two_args_no_input/arg0/arg1') def test_build_url_positional(self): actual = self.client.buildUrl('two_args_no_input', 'arg0', 'arg1') self.assertEqual(self.apiPath, actual) def test_build_url_keyword(self): actual = self.client.buildUrl('two_args_no_input', arg0='arg0', arg1='arg1') self.assertEqual(self.apiPath, actual) def test_build_url_query_string(self): actual = self.client.buildUrl('two_args_no_input', params={ 'arg0': 'arg0', 'arg1': 'arg1' }, query={'qs0': 1}) self.assertEqual(self.apiPath + '?qs0=1', actual) def test_fails_to_build_url_for_missing_method(self): with self.assertRaises(exc.TaskclusterFailure): self.client.buildUrl('non-existing') def test_fails_to_build_not_enough_args(self): with self.assertRaises(exc.TaskclusterFailure): self.client.buildUrl('two_args_no_input', 'not-enough-args')
def get_artifact_url(task_id, path, root_url=PRODUCTION_TASKCLUSTER_ROOT_URL): return liburls.api( root_url, "queue", "v1", f"task/{task_id}/artifacts/{path}", )
def get_index_url(index_path, root_url=PRODUCTION_TASKCLUSTER_ROOT_URL): return liburls.api( root_url, "index", "v1", f"task/{index_path}", )
def _constructUrl(self, route): """Construct a URL for the given route on this service, based on the rootUrl""" return liburls.api( self.options['rootUrl'], self.serviceName, self.apiVersion, route.rstrip('/'))
def get_purge_cache_url(provisioner_id, worker_type, use_proxy=False): if use_proxy: # Until bug 1460015 is finished, use the old baseUrl style of proxy URL url_tmpl = os.environ[ 'TASKCLUSTER_PROXY_URL'] + '/purge-cache/v1/purge-cache/{}/{}' else: url_tmpl = liburls.api(get_root_url(), 'purge-cache', 'v1', 'purge-cache/{}/{}') return url_tmpl.format(provisioner_id, worker_type)
def createLogReference(root_url, taskId, runId): logUrl = taskcluster_urls.api( root_url, "queue", "v1", "task/{taskId}/runs/{runId}/artifacts/public/logs/live_backing.log" ).format(taskId=taskId, runId=runId) return { "name": "live_backing_log", "url": logUrl, }
def get_artifact_url(task_id, path, old_deployment=False): if not old_deployment: return liburls.api( PRODUCTION_TASKCLUSTER_ROOT_URL, "queue", "v1", f"task/{task_id}/artifacts/{path}", ) else: return f"https://queue.taskcluster.net/v1/task/{task_id}/artifacts/{path}"
def createLogReference(taskId, runId): logUrl = taskcluster_urls.api( root_url, "queue", "v1", "task/{taskId}/runs/{runId}/artifacts/public/logs/live_backing.log" ).format(taskId=taskId, runId=runId) return { # XXX: This is a magical name see 1147958 which enables the log viewer. "name": "builds-4h", "url": logUrl, }
def send_email(address, subject, content, link, use_proxy=False): """Sends an email using the notify service""" logger.info('Sending email to {}.'.format(address)) url = liburls.api(get_root_url(use_proxy), 'notify', 'v1', 'email') _do_request(url, json={ 'address': address, 'subject': subject, 'content': content, 'link': link, })
def buildUrl(self, methodName, *args, **kwargs): entry = self.funcinfo.get(methodName) if not entry: raise exceptions.TaskclusterFailure( 'Requested method "%s" not found in API Reference' % methodName) routeParams, _, query, _, _ = self._processArgs(entry, *args, **kwargs) route = self._subArgsInRoute(entry, routeParams) if query: route += '?' + urllib.parse.urlencode(query) return liburls.api(self.options['rootUrl'], self.serviceName, self.apiVersion, route)
def _root_url(self, artifactdir=None, objdir=None): """Generate a publicly-accessible URL for the tasks's artifacts, or an objdir path""" if 'TASK_ID' in os.environ and 'RUN_ID' in os.environ: import taskcluster_urls from taskgraph.util.taskcluster import get_root_url return taskcluster_urls.api( get_root_url(False), 'queue', 'v1', 'task/{}/runs/{}/artifacts/{}'.format(os.environ['TASK_ID'], os.environ['RUN_ID'], artifactdir)) else: return os.path.join(self.topobjdir, objdir)
def send_email(address, subject, content, link, use_proxy=False): """Sends an email using the notify service""" logger.info(f"Sending email to {address}.") url = liburls.api(get_root_url(use_proxy), "notify", "v1", "email") _do_request( url, json={ "address": address, "subject": subject, "content": content, "link": link, }, )
def list_task_group_tasks(task_group_id): """Generate the tasks in a task group""" params = {} while True: url = liburls.api(get_root_url(False), 'queue', 'v1', 'task-group/{}/list'.format(task_group_id)) resp = _do_request(url, method="get", params=params).json() for task in resp['tasks']: yield task if resp.get('continuationToken'): params = {'continuationToken': resp.get('continuationToken')} else: break
def list_task_group_incomplete_tasks(task_group_id): """Generate the incomplete tasks in a task group""" params = {} while True: url = liburls.api(get_root_url(), 'queue', 'v1', 'task-group/{}/list'.format(task_group_id)) resp = _do_request(url, force_get=True, params=params).json() for task in [t['status'] for t in resp['tasks']]: if task['state'] in ['running', 'pending', 'unscheduled']: yield task['taskId'] if resp.get('continuationToken'): params = {'continuationToken': resp.get('continuationToken')} else: break
def get_artifact_url(task_id, path, use_proxy=False): artifact_tmpl = liburls.api(get_root_url(), 'queue', 'v1', 'task/{}/artifacts/{}') data = artifact_tmpl.format(task_id, path) if use_proxy: # Until Bug 1405889 is deployed, we can't download directly # from the taskcluster-proxy. Work around by using the /bewit # endpoint instead. # The bewit URL is the body of a 303 redirect, which we don't # want to follow (which fetches a potentially large resource). response = _do_request(os.environ['TASKCLUSTER_PROXY_URL'] + '/bewit', data=data, allow_redirects=False) return response.text return data
def send_email(address, subject, content, link, use_proxy=False): """Sends an email using the notify service""" logger.info('Sending email to {}.'.format(address)) if use_proxy: # Until bug 1460015 is finished, use the old baseUrl style of proxy URL url = os.environ['TASKCLUSTER_PROXY_URL'] + '/notify/v1/email' else: url = liburls.api(get_root_url(), 'notify', 'v1', 'email') _do_request(url, json={ 'address': address, 'subject': subject, 'content': content, 'link': link, })
def list_task_group_tasks(task_group_id): """Generate the tasks in a task group""" params = {} while True: url = liburls.api( get_root_url(False), "queue", "v1", f"task-group/{task_group_id}/list", ) resp = _do_request(url, method="get", params=params).json() yield from resp["tasks"] if resp.get("continuationToken"): params = {"continuationToken": resp.get("continuationToken")} else: break
def download_artifact(root_url, task_id, path): """ Downloads a Taskcluster artifact. Supports specific file formats like json and yaml. Returns either the parsed json, the parsed yaml or the plain response. """ artifact_url = taskcluster_urls.api( root_url, 'queue', 'v1', 'task/{}/artifacts/{}'.format(task_id, path)) if path.endswith(".json"): return fetch_json(artifact_url) if path.endswith(".yml"): return yaml.safe_load(fetch_text(artifact_url)) return make_request(artifact_url)
class TestBuildSignedUrl(ClientTest): apiPath = liburls.api(ClientTest.test_root_url, 'fake', 'v1', 'two_args_no_input/arg0/arg1') def test_builds_surl_positional(self): actual = self.client.buildSignedUrl('two_args_no_input', 'arg0', 'arg1') actual = re.sub('bewit=[^&]*', 'bewit=X', actual) self.assertEqual(self.apiPath + '?bewit=X', actual) def test_builds_surl_keyword(self): actual = self.client.buildSignedUrl('two_args_no_input', arg0='arg0', arg1='arg1') actual = re.sub('bewit=[^&]*', 'bewit=X', actual) self.assertEqual(self.apiPath + '?bewit=X', actual)
def get_artifact_url(task_id, path, use_proxy=False): artifact_tmpl = liburls.api( get_root_url(False), "queue", "v1", "task/{}/artifacts/{}" ) data = six.ensure_text(artifact_tmpl.format(task_id, path)) if use_proxy: # Until Bug 1405889 is deployed, we can't download directly # from the taskcluster-proxy. Work around by using the /bewit # endpoint instead. # The bewit URL is the body of a 303 redirect, which we don't # want to follow (which fetches a potentially large resource). response = _do_request( os.environ["TASKCLUSTER_PROXY_URL"] + "/bewit", data=data, allow_redirects=False, ) return six.ensure_text(response.text) return data
def list_task_group_incomplete_tasks(task_group_id): """Generate the incomplete tasks in a task group""" params = {} while True: url = liburls.api( get_root_url(False), "queue", "v1", f"task-group/{task_group_id}/list", ) resp = _do_request(url, force_get=True, params=params).json() for task in [t["status"] for t in resp["tasks"]]: if task["state"] in ["running", "pending", "unscheduled"]: yield task["taskId"] if resp.get("continuationToken"): params = {"continuationToken": resp.get("continuationToken")} else: break
async def addArtifactUploadedLinks(root_url, taskId, runId, job, session): artifacts = [] try: artifacts = await fetchArtifacts(root_url, taskId, runId, session) except Exception: logger.debug("Artifacts could not be found for task: %s run: %s", taskId, runId) return job seen = {} links = [] for artifact in artifacts: name = os.path.basename(artifact["name"]) # Bug 1595902 - It seems that directories are showing up as artifacts; skip them if not name: continue if not seen.get(name): seen[name] = [artifact["name"]] else: seen[name].append(artifact["name"]) name = "{name} ({length})".format(name=name, length=len(seen[name]) - 1) links.append({ "label": "artifact uploaded", "linkText": name, "url": taskcluster_urls.api( root_url, "queue", "v1", "task/{taskId}/runs/{runId}/artifacts/{artifact_name}".format( taskId=taskId, runId=runId, artifact_name=artifact["name"]), ), }) job["jobInfo"]["links"] = links return job
def get_purge_cache_url(provisioner_id, worker_type, use_proxy=False): url_tmpl = liburls.api(get_root_url(use_proxy), "purge-cache", "v1", "purge-cache/{}/{}") return url_tmpl.format(provisioner_id, worker_type)
def get_dependent_tasks_url(task_id, root_url=PRODUCTION_TASKCLUSTER_ROOT_URL): return liburls.api(root_url, "queue", "v1", f"task/{task_id}/dependents")
def get_task_url(task_id): return liburls.api(PRODUCTION_TASKCLUSTER_ROOT_URL, "queue", "v1", f"task/{task_id}")
def get_current_scopes(): """Get the current scopes. This only makes sense in a task with the Taskcluster proxy enabled, where it returns the actual scopes accorded to the task.""" auth_url = liburls.api(get_root_url(True), "auth", "v1", "scopes/current") resp = _do_request(auth_url) return resp.json().get("scopes", [])