async def invoke(self): """Claims and processes Taskcluster work.""" while not self.is_cancelled and not self.is_stopped: num_tasks_to_claim = min( self.config["max_concurrent_tasks"] - len(self.running_tasks), MAX_CLAIM_WORK_TASKS) if num_tasks_to_claim > 0: async with aiohttp.ClientSession() as session: queue = Queue( options={ "credentials": { "accessToken": self.config["taskcluster_access_token"], "clientId": self.config["taskcluster_client_id"] }, "rootUrl": self.config["taskcluster_root_url"], }, session=session, ) new_tasks = await self._run_cancellable( claim_work(self.config, queue, num_tasks=num_tasks_to_claim)) or {} self.last_claim_work = arrow.utcnow() for claim_task in new_tasks.get("tasks", []): new_task = Task(self.config, claim_task) new_task.start() self.running_tasks.append(new_task) await self.prune_running_tasks() sleep_time = self.last_claim_work.timestamp + self.config[ "claim_work_interval"] - arrow.utcnow().timestamp sleep_time > 0 and await self._run_cancellable(sleep(sleep_time)) self.running_tasks and await asyncio.wait( [task.main_fut for task in self.running_tasks if task.main_fut])
async def complete_task(self): """Submit task status to Taskcluster.""" reversed_statuses = get_reversed_statuses() args = [self.task_id, self.run_id] try: async with aiohttp.ClientSession() as session: temp_queue = Queue(options={ "credentials": self.task_credentials, "rootUrl": self.config["taskcluster_root_url"] }, session=session) if self.status == STATUSES["success"]: log.info("Reporting task complete...") response = await temp_queue.reportCompleted(*args) elif self.status != 1 and self.status in reversed_statuses: reason = reversed_statuses[self.status] log.info("Reporting task exception {}...".format(reason)) payload = {"reason": reason} response = await temp_queue.reportException(*args, payload) else: log.info("Reporting task failed...") response = await temp_queue.reportFailed(*args) log.debug("Task status response:\n{}".format( pprint.pformat(response))) except taskcluster.exceptions.TaskclusterRestFailure as exc: if exc.status_code == 409: log.info( "complete_task: 409: not reporting complete/failed for %s %s", self.task_id, self.run_id) else: log.exception("complete_task: unknown exception for %s %s", self.task_id, self.run_id)
async def reclaim_task(self): """Try to reclaim a task from the queue. This is a keepalive. Without it the task will expire and be re-queued. A 409 status means the task has been resolved. This generally means the task has expired, reached its deadline, or has been cancelled. Raises: TaskclusterRestFailure: on non-409 status_code from `taskcluster.aio.Queue.reclaimTask` """ while True: log.debug("waiting %s seconds before reclaiming..." % self.config["reclaim_interval"]) await asyncio.sleep(self.config["reclaim_interval"]) log.debug("Reclaiming task %s %s", self.task_id, self.run_id) try: async with aiohttp.ClientSession() as session: temp_queue = Queue(options={"credentials": self.task_credentials, "rootUrl": self.config["taskcluster_root_url"]}, session=session) self._reclaim_task = await temp_queue.reclaimTask(self.task_id, self.run_id) except taskcluster.exceptions.TaskclusterRestFailure as exc: if exc.status_code == 409: log.warning("Stopping task after receiving 409 response from reclaim_task: %s %s", self.task_id, self.run_id) self.status = STATUSES["superseded"] else: log.exception("reclaim_task unexpected exception: %s %s", self.task_id, self.run_id) self.status = STATUSES["internal-error"] self.task_fut and self.task_fut.cancel() break
async def test_verify_production_cot(branch_context): index = Index(options={"rootUrl": branch_context["taskcluster_root_url"]}) queue = Queue(options={"rootUrl": branch_context["taskcluster_root_url"]}) async def get_task_id_from_index(index_path): res = await index.findTask(index_path) return res["taskId"] async def get_completed_task_info_from_labels(decision_task_id, label_to_task_type): label_to_taskid = await queue.getLatestArtifact( decision_task_id, "public/label-to-taskid.json") task_info = {} for re_label, task_type in label_to_task_type.items(): r = re.compile(re_label) for label, task_id in label_to_taskid.items(): if r.match(label): status = await queue.status(task_id) # only run verify_cot against tasks with completed deps. if status["status"]["state"] in ("completed", "running", "pending", "failed"): task_info[task_id] = task_type break else: log.warning( "Not running verify_cot against {} {} because there are no elegible completed tasks" .format(decision_task_id, task_type)) return task_info async def verify_cot(name, task_id, task_type, check_task=True): log.info("Verifying {} {} {}...".format(name, task_id, task_type)) context.task = await queue.task(task_id) cot = ChainOfTrust(context, task_type, task_id=task_id) await verify_chain_of_trust(cot, check_task=check_task) async with get_context({ "cot_product": branch_context["cot_product"], "verify_cot_signature": True }) as context: context.queue = queue task_id = await get_task_id_from_index(branch_context["index"]) assert task_id, "{}: Can't get task_id from index {}!".format( branch_context["name"], branch_context["index"]) if branch_context.get("task_label_to_task_type"): task_info = await get_completed_task_info_from_labels( task_id, branch_context["task_label_to_task_type"]) assert "check_task" not in branch_context, "{}: Can't disable check_task.".format( branch_context["name"], ) for task_id, task_type in task_info.items(): name = "{} {}".format(branch_context["name"], task_type) await verify_cot(name, task_id, task_type) else: await verify_cot( branch_context["name"], task_id, branch_context["task_type"], branch_context.get("check_task", True), )
async def get_action_task_details(session, taskid): async with async_timeout.timeout(100): queue = Queue(session=session) task = await queue.task(taskid) return dict(taskid=taskid, name=task['extra']['action']['name'], buildnum=task['extra']['action']['context']['input'] ['build_number'], flavor=task['extra']['action']['context']['input'] ['release_promotion_flavor'], ci=task['taskGroupId'])
def create_queue(self, credentials): """Create a taskcluster queue. Args: credentials (dict): taskcluster credentials. """ if credentials: session = self.session or aiohttp.ClientSession( loop=self.event_loop) return Queue({ 'credentials': credentials, }, session=session)
async def __init__(self, json=None, task_id=None, queue=None): """Init.""" # taskId is not provided in the definition if task_id: self.task_id = task_id if json: self.def_json = json.get('task', json) return if not task_id: raise ValueError('No task definition or taskId provided') self.queue = queue if not self.queue: self.queue = Queue(tc_options()) await self._fetch_definition()
async def test_verify_production_cot(branch_context): index = Index(options={'rootUrl': DEFAULT_CONFIG['taskcluster_root_url']}) queue = Queue(options={'rootUrl': DEFAULT_CONFIG['taskcluster_root_url']}) async def get_task_id_from_index(index_path): res = await index.findTask(index_path) return res['taskId'] async def get_completed_task_info_from_labels(decision_task_id, label_to_task_type): label_to_taskid = await queue.getLatestArtifact( decision_task_id, "public/label-to-taskid.json") task_info = {} for re_label, task_type in label_to_task_type.items(): r = re.compile(re_label) for label, task_id in label_to_taskid.items(): if r.match(label): status = await queue.status(task_id) # only run verify_cot against tasks with completed deps. if status['status']['state'] in ('completed', 'running', 'pending', 'failed'): task_info[task_id] = task_type break else: log.warning( "Not running verify_cot against {} {} because there are no elegible completed tasks" .format(decision_task_id, task_type)) return task_info async def verify_cot(name, task_id, task_type): log.info("Verifying {} {} {}...".format(name, task_id, task_type)) context.task = await queue.task(task_id) cot = ChainOfTrust(context, task_type, task_id=task_id) await verify_chain_of_trust(cot) async with get_context({'cot_product': branch_context['cot_product']}) as context: context.queue = queue task_id = await get_task_id_from_index(branch_context['index']) assert task_id, "{}: Can't get task_id from index {}!".format( branch_context['name'], branch_context['index']) if branch_context.get('task_label_to_task_type'): task_info = await get_completed_task_info_from_labels( task_id, branch_context['task_label_to_task_type']) for task_id, task_type in task_info.items(): name = "{} {}".format(branch_context['name'], task_type) await verify_cot(name, task_id, task_type) else: await verify_cot(branch_context['name'], task_id, branch_context['task_type'])
def create_queue(self, credentials): """Create a taskcluster queue. Args: credentials (dict): taskcluster credentials. """ if credentials: session = self.session or aiohttp.ClientSession( loop=self.event_loop) return Queue(options={ 'credentials': credentials, 'rootUrl': self.config['taskcluster_root_url'], }, session=session)
async def __init__(self, json=None, task_id=None, queue=None): """Init.""" if task_id: self.task_id = task_id if json: # We might be passed {'status': ... } or just the contents self.status_json = json.get('status', json) self.task_id = self.status_json['taskId'] return if not task_id: raise ValueError('No task definition or taskId provided') self.queue = queue if not self.queue: self.queue = Queue(tc_options()) await self._fetch_status()
async def _upload_log(self): payload = {"storageType": "s3", "expires": arrow.get(self.claim_task["task"]["expires"]).isoformat(), "contentType": "text/plain"} args = [self.task_id, self.run_id, "public/logs/live_backing.log", payload] async with aiohttp.ClientSession() as session: temp_queue = Queue(options={"credentials": self.task_credentials, "rootUrl": self.config["taskcluster_root_url"]}, session=session) tc_response = await temp_queue.createArtifact(*args) headers = {aiohttp.hdrs.CONTENT_TYPE: "text/plain", aiohttp.hdrs.CONTENT_ENCODING: "gzip"} skip_auto_headers = [aiohttp.hdrs.CONTENT_TYPE] with open(self.log_path, "rb") as fh: async with async_timeout.timeout(self.config["artifact_upload_timeout"]): async with session.put(tc_response["putUrl"], data=fh, headers=headers, skip_auto_headers=skip_auto_headers, compress=False) as resp: log.info("create_artifact public/logs/live_backing.log: {}".format(resp.status)) response_text = await resp.text() log.info(response_text) if resp.status not in (200, 204): raise RetryError("Bad status {}".format(resp.status))
def create_queue(self, credentials: Optional[Dict[str, Any]]) -> Optional[Queue]: """Create a taskcluster queue. Args: credentials (dict): taskcluster credentials. """ assert self.config if credentials: session = self.session or aiohttp.ClientSession( loop=self.event_loop) return Queue(options={ "credentials": credentials, "rootUrl": self.config["taskcluster_root_url"] }, session=session) return None
async def __init__(self, json=None, task_id=None, queue=None): """init.""" if json: self.def_json = json.get('task') self.status_json = json.get('status') self.task_id = self.status_json['taskId'] return if task_id: self.task_id = task_id else: raise ValueError('No task definition or taskId provided') self.queue = queue if not self.queue: self.queue = Queue(tc_options()) if self.task_id: await self._fetch_definition() await self._fetch_status()
async def get_tc_run_artifacts(taskid, runid): log.info('Fetching TC artifact info for %s/%s', taskid, runid) artifacts = [] query = {} async with aiohttp.ClientSession() as session: queue = Queue(options=tc_options(), session=session) while True: resp = await queue.listArtifacts(taskid, runid, query=query) # Ammend the artifact information with the task and run ids # to make it easy to find the corresponding S3 object for a in resp['artifacts']: a['_name'] = f'{taskid}/{runid}/{a["name"]}' artifacts.append(a) if 'continuationToken' in resp: query.update({'continuationToken': resp['continuationToken']}) else: break return artifacts
async def fetch_tasks(self, limit=None): """Return tasks with the associated group ID. Handles continuationToken without the user being aware of it. Enforces the limit parameter as a limit of the total number of tasks to be returned. """ if self.cache_file: if self._read_file_cache(): return query = {} if limit: # Default taskcluster-client api asks for 1000 tasks. query['limit'] = min(limit, 1000) def under_limit(length): """Indicate if we've returned enough tasks.""" if not limit or length < limit: return True return False async with aiohttp.ClientSession() as session: queue = Queue(session=session) outcome = await queue.listTaskGroup(self.groupid, query=query) tasks = outcome.get('tasks', []) while under_limit(len(tasks)) and outcome.get('continuationToken'): query.update({ 'continuationToken': outcome.get('continuationToken') }) outcome = await queue.listTaskGroup(self.groupid, query=query) tasks.extend(outcome.get('tasks', [])) if limit: tasks = tasks[:limit] self.tasklist = [Task(json=data) for data in tasks] if self.cache_file: self._write_file_cache()
async def load_nightly_graph(dt=None, platform='linux-opt'): """Given a date, load the relevant nightly task graph.""" async with aiohttp.ClientSession() as session: index = Index(options=tc_options(), session=session) queue = Queue(options=tc_options(), session=session) if not dt: dt = datetime.now() datestr = dt.strftime("%Y.%m.%d") basestr = "gecko.v2.mozilla-central.nightly.{date}.latest.firefox.{platform}" found = await index.findTask(basestr.format(date=datestr, platform=platform)) taskid = found.get('taskId') taskdef = await queue.task(taskid) # except taskcluster.exceptions.TaskclusterRestFailure: taskgroup = taskdef.get('taskGroupId') log.debug("Looking at {} for {}".format(taskgroup, datestr)) if taskgroup: return {'date': datestr, 'graph': await TaskGraph(taskgroup)} return None