async def invoke(self):
"""Claims and processes Taskcluster work."""
while not self.is_cancelled and not self.is_stopped:
num_tasks_to_claim = min(
self.config["max_concurrent_tasks"] - len(self.running_tasks),
MAX_CLAIM_WORK_TASKS)
if num_tasks_to_claim > 0:
async with aiohttp.ClientSession() as session:
queue = Queue(
options={
"credentials": {
"accessToken":
self.config["taskcluster_access_token"],
"clientId":
self.config["taskcluster_client_id"]
},
"rootUrl": self.config["taskcluster_root_url"],
},
session=session,
)
new_tasks = await self._run_cancellable(
claim_work(self.config,
queue,
num_tasks=num_tasks_to_claim)) or {}
self.last_claim_work = arrow.utcnow()
for claim_task in new_tasks.get("tasks", []):
new_task = Task(self.config, claim_task)
new_task.start()
self.running_tasks.append(new_task)
await self.prune_running_tasks()
sleep_time = self.last_claim_work.timestamp + self.config[
"claim_work_interval"] - arrow.utcnow().timestamp
sleep_time > 0 and await self._run_cancellable(sleep(sleep_time))
self.running_tasks and await asyncio.wait(
[task.main_fut for task in self.running_tasks if task.main_fut])
async def complete_task(self):
"""Submit task status to Taskcluster."""
reversed_statuses = get_reversed_statuses()
args = [self.task_id, self.run_id]
try:
async with aiohttp.ClientSession() as session:
temp_queue = Queue(options={
"credentials":
self.task_credentials,
"rootUrl":
self.config["taskcluster_root_url"]
},
session=session)
if self.status == STATUSES["success"]:
log.info("Reporting task complete...")
response = await temp_queue.reportCompleted(*args)
elif self.status != 1 and self.status in reversed_statuses:
reason = reversed_statuses[self.status]
log.info("Reporting task exception {}...".format(reason))
payload = {"reason": reason}
response = await temp_queue.reportException(*args, payload)
else:
log.info("Reporting task failed...")
response = await temp_queue.reportFailed(*args)
log.debug("Task status response:\n{}".format(
pprint.pformat(response)))
except taskcluster.exceptions.TaskclusterRestFailure as exc:
if exc.status_code == 409:
log.info(
"complete_task: 409: not reporting complete/failed for %s %s",
self.task_id, self.run_id)
else:
log.exception("complete_task: unknown exception for %s %s",
self.task_id, self.run_id)
async def reclaim_task(self):
"""Try to reclaim a task from the queue.
This is a keepalive. Without it the task will expire and be re-queued.
A 409 status means the task has been resolved. This generally means the
task has expired, reached its deadline, or has been cancelled.
Raises:
TaskclusterRestFailure: on non-409 status_code from
`taskcluster.aio.Queue.reclaimTask`
"""
while True:
log.debug("waiting %s seconds before reclaiming..." % self.config["reclaim_interval"])
await asyncio.sleep(self.config["reclaim_interval"])
log.debug("Reclaiming task %s %s", self.task_id, self.run_id)
try:
async with aiohttp.ClientSession() as session:
temp_queue = Queue(options={"credentials": self.task_credentials, "rootUrl": self.config["taskcluster_root_url"]}, session=session)
self._reclaim_task = await temp_queue.reclaimTask(self.task_id, self.run_id)
except taskcluster.exceptions.TaskclusterRestFailure as exc:
if exc.status_code == 409:
log.warning("Stopping task after receiving 409 response from reclaim_task: %s %s", self.task_id, self.run_id)
self.status = STATUSES["superseded"]
else:
log.exception("reclaim_task unexpected exception: %s %s", self.task_id, self.run_id)
self.status = STATUSES["internal-error"]
self.task_fut and self.task_fut.cancel()
break
async def test_verify_production_cot(branch_context):
index = Index(options={"rootUrl": branch_context["taskcluster_root_url"]})
queue = Queue(options={"rootUrl": branch_context["taskcluster_root_url"]})
async def get_task_id_from_index(index_path):
res = await index.findTask(index_path)
return res["taskId"]
async def get_completed_task_info_from_labels(decision_task_id,
label_to_task_type):
label_to_taskid = await queue.getLatestArtifact(
decision_task_id, "public/label-to-taskid.json")
task_info = {}
for re_label, task_type in label_to_task_type.items():
r = re.compile(re_label)
for label, task_id in label_to_taskid.items():
if r.match(label):
status = await queue.status(task_id)
# only run verify_cot against tasks with completed deps.
if status["status"]["state"] in ("completed", "running",
"pending", "failed"):
task_info[task_id] = task_type
break
else:
log.warning(
"Not running verify_cot against {} {} because there are no elegible completed tasks"
.format(decision_task_id, task_type))
return task_info
async def verify_cot(name, task_id, task_type, check_task=True):
log.info("Verifying {} {} {}...".format(name, task_id, task_type))
context.task = await queue.task(task_id)
cot = ChainOfTrust(context, task_type, task_id=task_id)
await verify_chain_of_trust(cot, check_task=check_task)
async with get_context({
"cot_product": branch_context["cot_product"],
"verify_cot_signature": True
}) as context:
context.queue = queue
task_id = await get_task_id_from_index(branch_context["index"])
assert task_id, "{}: Can't get task_id from index {}!".format(
branch_context["name"], branch_context["index"])
if branch_context.get("task_label_to_task_type"):
task_info = await get_completed_task_info_from_labels(
task_id, branch_context["task_label_to_task_type"])
assert "check_task" not in branch_context, "{}: Can't disable check_task.".format(
branch_context["name"], )
for task_id, task_type in task_info.items():
name = "{} {}".format(branch_context["name"], task_type)
await verify_cot(name, task_id, task_type)
else:
await verify_cot(
branch_context["name"],
task_id,
branch_context["task_type"],
branch_context.get("check_task", True),
)
async def get_action_task_details(session, taskid):
async with async_timeout.timeout(100):
queue = Queue(session=session)
task = await queue.task(taskid)
return dict(taskid=taskid,
name=task['extra']['action']['name'],
buildnum=task['extra']['action']['context']['input']
['build_number'],
flavor=task['extra']['action']['context']['input']
['release_promotion_flavor'],
ci=task['taskGroupId'])
def create_queue(self, credentials):
"""Create a taskcluster queue.
Args:
credentials (dict): taskcluster credentials.
"""
if credentials:
session = self.session or aiohttp.ClientSession(
loop=self.event_loop)
return Queue({
'credentials': credentials,
}, session=session)
async def __init__(self, json=None, task_id=None, queue=None):
"""Init."""
# taskId is not provided in the definition
if task_id:
self.task_id = task_id
if json:
self.def_json = json.get('task', json)
return
if not task_id:
raise ValueError('No task definition or taskId provided')
self.queue = queue
if not self.queue:
self.queue = Queue(tc_options())
await self._fetch_definition()
async def test_verify_production_cot(branch_context):
index = Index(options={'rootUrl': DEFAULT_CONFIG['taskcluster_root_url']})
queue = Queue(options={'rootUrl': DEFAULT_CONFIG['taskcluster_root_url']})
async def get_task_id_from_index(index_path):
res = await index.findTask(index_path)
return res['taskId']
async def get_completed_task_info_from_labels(decision_task_id,
label_to_task_type):
label_to_taskid = await queue.getLatestArtifact(
decision_task_id, "public/label-to-taskid.json")
task_info = {}
for re_label, task_type in label_to_task_type.items():
r = re.compile(re_label)
for label, task_id in label_to_taskid.items():
if r.match(label):
status = await queue.status(task_id)
# only run verify_cot against tasks with completed deps.
if status['status']['state'] in ('completed', 'running',
'pending', 'failed'):
task_info[task_id] = task_type
break
else:
log.warning(
"Not running verify_cot against {} {} because there are no elegible completed tasks"
.format(decision_task_id, task_type))
return task_info
async def verify_cot(name, task_id, task_type):
log.info("Verifying {} {} {}...".format(name, task_id, task_type))
context.task = await queue.task(task_id)
cot = ChainOfTrust(context, task_type, task_id=task_id)
await verify_chain_of_trust(cot)
async with get_context({'cot_product':
branch_context['cot_product']}) as context:
context.queue = queue
task_id = await get_task_id_from_index(branch_context['index'])
assert task_id, "{}: Can't get task_id from index {}!".format(
branch_context['name'], branch_context['index'])
if branch_context.get('task_label_to_task_type'):
task_info = await get_completed_task_info_from_labels(
task_id, branch_context['task_label_to_task_type'])
for task_id, task_type in task_info.items():
name = "{} {}".format(branch_context['name'], task_type)
await verify_cot(name, task_id, task_type)
else:
await verify_cot(branch_context['name'], task_id,
branch_context['task_type'])
def create_queue(self, credentials):
"""Create a taskcluster queue.
Args:
credentials (dict): taskcluster credentials.
"""
if credentials:
session = self.session or aiohttp.ClientSession(
loop=self.event_loop)
return Queue(options={
'credentials': credentials,
'rootUrl': self.config['taskcluster_root_url'],
},
session=session)
async def __init__(self, json=None, task_id=None, queue=None):
"""Init."""
if task_id:
self.task_id = task_id
if json:
# We might be passed {'status': ... } or just the contents
self.status_json = json.get('status', json)
self.task_id = self.status_json['taskId']
return
if not task_id:
raise ValueError('No task definition or taskId provided')
self.queue = queue
if not self.queue:
self.queue = Queue(tc_options())
await self._fetch_status()
async def _upload_log(self):
payload = {"storageType": "s3", "expires": arrow.get(self.claim_task["task"]["expires"]).isoformat(), "contentType": "text/plain"}
args = [self.task_id, self.run_id, "public/logs/live_backing.log", payload]
async with aiohttp.ClientSession() as session:
temp_queue = Queue(options={"credentials": self.task_credentials, "rootUrl": self.config["taskcluster_root_url"]}, session=session)
tc_response = await temp_queue.createArtifact(*args)
headers = {aiohttp.hdrs.CONTENT_TYPE: "text/plain", aiohttp.hdrs.CONTENT_ENCODING: "gzip"}
skip_auto_headers = [aiohttp.hdrs.CONTENT_TYPE]
with open(self.log_path, "rb") as fh:
async with async_timeout.timeout(self.config["artifact_upload_timeout"]):
async with session.put(tc_response["putUrl"], data=fh, headers=headers, skip_auto_headers=skip_auto_headers, compress=False) as resp:
log.info("create_artifact public/logs/live_backing.log: {}".format(resp.status))
response_text = await resp.text()
log.info(response_text)
if resp.status not in (200, 204):
raise RetryError("Bad status {}".format(resp.status))
def create_queue(self,
credentials: Optional[Dict[str, Any]]) -> Optional[Queue]:
"""Create a taskcluster queue.
Args:
credentials (dict): taskcluster credentials.
"""
assert self.config
if credentials:
session = self.session or aiohttp.ClientSession(
loop=self.event_loop)
return Queue(options={
"credentials": credentials,
"rootUrl": self.config["taskcluster_root_url"]
},
session=session)
return None
async def __init__(self, json=None, task_id=None, queue=None):
"""init."""
if json:
self.def_json = json.get('task')
self.status_json = json.get('status')
self.task_id = self.status_json['taskId']
return
if task_id:
self.task_id = task_id
else:
raise ValueError('No task definition or taskId provided')
self.queue = queue
if not self.queue:
self.queue = Queue(tc_options())
if self.task_id:
await self._fetch_definition()
await self._fetch_status()
async def get_tc_run_artifacts(taskid, runid):
log.info('Fetching TC artifact info for %s/%s', taskid, runid)
artifacts = []
query = {}
async with aiohttp.ClientSession() as session:
queue = Queue(options=tc_options(), session=session)
while True:
resp = await queue.listArtifacts(taskid, runid, query=query)
# Ammend the artifact information with the task and run ids
# to make it easy to find the corresponding S3 object
for a in resp['artifacts']:
a['_name'] = f'{taskid}/{runid}/{a["name"]}'
artifacts.append(a)
if 'continuationToken' in resp:
query.update({'continuationToken': resp['continuationToken']})
else:
break
return artifacts
async def fetch_tasks(self, limit=None):
"""Return tasks with the associated group ID.
Handles continuationToken without the user being aware of it.
Enforces the limit parameter as a limit of the total number of tasks
to be returned.
"""
if self.cache_file:
if self._read_file_cache():
return
query = {}
if limit:
# Default taskcluster-client api asks for 1000 tasks.
query['limit'] = min(limit, 1000)
def under_limit(length):
"""Indicate if we've returned enough tasks."""
if not limit or length < limit:
return True
return False
async with aiohttp.ClientSession() as session:
queue = Queue(session=session)
outcome = await queue.listTaskGroup(self.groupid, query=query)
tasks = outcome.get('tasks', [])
while under_limit(len(tasks)) and outcome.get('continuationToken'):
query.update({
'continuationToken': outcome.get('continuationToken')
})
outcome = await queue.listTaskGroup(self.groupid, query=query)
tasks.extend(outcome.get('tasks', []))
if limit:
tasks = tasks[:limit]
self.tasklist = [Task(json=data) for data in tasks]
if self.cache_file:
self._write_file_cache()
async def load_nightly_graph(dt=None, platform='linux-opt'):
"""Given a date, load the relevant nightly task graph."""
async with aiohttp.ClientSession() as session:
index = Index(options=tc_options(), session=session)
queue = Queue(options=tc_options(), session=session)
if not dt:
dt = datetime.now()
datestr = dt.strftime("%Y.%m.%d")
basestr = "gecko.v2.mozilla-central.nightly.{date}.latest.firefox.{platform}"
found = await index.findTask(basestr.format(date=datestr, platform=platform))
taskid = found.get('taskId')
taskdef = await queue.task(taskid)
# except taskcluster.exceptions.TaskclusterRestFailure:
taskgroup = taskdef.get('taskGroupId')
log.debug("Looking at {} for {}".format(taskgroup, datestr))
if taskgroup:
return {'date': datestr, 'graph': await TaskGraph(taskgroup)}
return None
