async def write_data(session, taskid, csvwriter, semaphore):
queue = taskcluster.Queue(session=session)
decision_task_status = await queue.status(taskid)
decision_task = await queue.task(taskid)
queue = None
created = decision_task['created']
scheduled = decision_task_status['status']['runs'][-1]['scheduled']
async with semaphore:
log.info("Downloading task json object for %s", taskid)
taskjson = await get_taskgraph_json(session, taskid)
if not taskjson:
log.info("task json was empty")
return
log.info("task json for %s had %d entries", taskid, len(taskjson))
useful_tasks = find_relevant_tasks(taskjson)
taskjson = None
if not useful_tasks:
return
log.info("Processing rows for %s", taskid)
for task in tuple(useful_tasks.keys()):
attributes = useful_tasks.pop(task)
rows = await get_task_data_rows(session, task, attributes,
created,
scheduled)
for row in rows:
csvwriter.writerow(row)
log.info("Wrote data for %s", taskid)
async def get_taskgraph_json(session, taskid):
queue = taskcluster.Queue(session=session)
artifact = {}
try:
artifact = await queue.getLatestArtifact(taskid, 'public/task-graph.json')
except taskcluster.exceptions.TaskclusterRestFailure as e:
pass
return artifact
async def get_nightly_taskgraphids(session):
idx = taskcluster.Index(session=session)
queue = taskcluster.Queue(session=session)
revision_indexes = []
_ret = await idx.listNamespaces('gecko.v2.mozilla-central.nightly')
namespaces = sorted(
[n['namespace'] for n in _ret['namespaces'] if n['name'].startswith('2')],
reverse=True,
)
# Recurse down through the namespaces until we have reached 'revision'
for _ in ['years', 'months', 'days', 'revision']:
tasks = [asyncio.ensure_future(idx.listNamespaces(n)) for n in namespaces]
ret = await asyncio.gather(*tasks)
namespaces = sorted(n['namespace'] for m in ret for n in m['namespaces'])
# Remove 'latest'
revision_indexes = [n for n in namespaces if 'revision' in n]
semaphore = asyncio.Semaphore(50)
aiotasks = []
for rev_ns in revision_indexes:
aiotasks.append(
asyncio.ensure_future(
_semaphore_wrapper(
idx.findTask,
'{rev_ns}.firefox.linux64-opt'.format(rev_ns=rev_ns),
semaphore=semaphore,
),
),
)
aiotasks.append(
asyncio.ensure_future(
_semaphore_wrapper(
idx.findTask,
'{rev_ns}.mobile.android-api-16-opt'.format(rev_ns=rev_ns),
semaphore=semaphore,
),
),
)
_ret = await asyncio.gather(*aiotasks, return_exceptions=True)
taskids = [n['taskId'] for n in _ret if isinstance(n, dict)]
aiotasks = []
for task in taskids:
aiotasks.append(
asyncio.ensure_future(
_semaphore_wrapper(
queue.task,
task,
semaphore=semaphore,
),
),
)
_ret = await asyncio.gather(*aiotasks, return_exceptions=True)
return [r['taskGroupId'] for r in _ret if isinstance(r, dict)]
async def get_task_data_rows(session, taskid, attributes, context, scheduled):
queue = taskcluster.Queue(session=session)
try:
status = await queue.status(taskid)
except taskcluster.exceptions.TaskclusterRestFailure:
return []
except Exception:
print("CALLEK-SOMETHING WENT WRONG")
raise
rows = []
print('.')
for r in status['status'].get('runs', []):
row = {'kind': attributes.get('kind')}
row['provisioner'] = status['provisionerId']
row['workertype'] = status['workerType']
row['taskid'] = taskid
row['run'] = r['runId']
row['state'] = r['state']
row['started'] = r.get('started')
row['scheduled'] = r['scheduled']
row['resolved'] = r['resolved']
row['product'] = context['product']
row['version'] = context['version']
row['display_version'] = context['display_version']
row['decision_scheduled'] = scheduled
row['build_platform'] = attributes.get('build_platform')
if 'locale' in attributes:
row['locale'] = attributes['locale']
rows.append(row)
elif 'chunk_locales' in attributes:
for l in attributes['chunk_locales']:
row_ = copy.deepcopy(row)
row_['locale'] = l
rows.append(row_)
elif 'all_locales' in attributes:
for l in attributes['all_locales']:
row_ = copy.deepcopy(row)
row_['locale'] = l
rows.append(row_)
else:
row['locale'] = None
rows.append(row)
return rows
async def write_data(session, taskid, context, csvwriter, csvf):
queue = taskcluster.Queue(session=session)
taskjson = await get_taskgraph_json(session, taskid, context)
try:
decision_task_status = await queue.status(taskid)
except taskcluster.exceptions.TaskclusterRestFailure:
return
queue = None
scheduled = decision_task_status['status']['runs'][-1]['scheduled']
if not taskjson:
return
useful_tasks = find_relevant_tasks(taskjson, all_tasks=True)
if not useful_tasks:
return
print('-')
for task, attributes in useful_tasks.items():
print('=')
rows = await get_task_data_rows(session, task, attributes, context,
scheduled)
csvwriter.writerows(rows)
print('%')
async def find_taskgroup_by_revision(revision,
project,
product,
nightly=False):
if nightly:
nightly_index = "nightly."
else:
nightly_index = ""
index = "gecko.v2.{project}.{nightly}revision.{revision}.{product}.linux64-opt".format(
project=project,
nightly=nightly_index,
revision=revision,
product=product,
)
print(index)
idx = taskcluster.Index()
queue = taskcluster.Queue()
build_task = await idx.findTask(index)
task_def = await queue.task(build_task['taskId'])
return task_def['taskGroupId']