def __init__(self, credentials=None, web_auth=False, dry_run=False):
''' Initialize the authentication method.'''
self.dry_run = dry_run
if dry_run:
self.queue = taskcluster_client.Queue()
elif credentials_available():
self.queue = taskcluster_client.Queue()
elif credentials:
self.queue = taskcluster_client.Queue({'credentials': credentials})
elif web_auth:
# Your browser will open a new tab asking you to authenticate
# through TaskCluster and then grant access to this
self.queue = taskcluster_client.Queue(
{'credentials': authenticate()})
else:
raise TaskClusterError(
""
"Since you're not running in dry run mode, you need to provide "
"an authentication method:\n"
" 1) call authenticate() to get credentials and pass it as credentials.\n"
" 2) set TASKCLUSTER_{CLIENT_ID,ACCESS_TOKEN} as env variables.\n"
" 3) use web_auth=True to authenticate through your web browser.."
)
def get_docker_task_id(self, properties):
"""Retrieve docker image task Id used by tests on TC.
To get the task id of the 'desktop-test' docker image as used by desktop tests
for a given build the Index has to be queried first. When the task for the build
has been found, check all its dependent tasks for the first appearance of the
'desktop-test' worker type. Within its payload the task id of the docker task
can be extracted.
Bug 1284236 - Not all Taskcluster builds report correctly to the Index.
To ensure we get a TC build force to Linux64 debug for now.
:param properties: Properties of the build and necessary resources.
"""
build_index = 'gecko.v2.{branch}.revision.{rev}.firefox.{platform}-debug'.format(
branch=properties['branch'],
rev=properties['revision'],
platform=properties['platform'],
)
try:
logger.debug(
'Querying Taskcluster for "desktop-test" docker image for "{}"...'
.format(properties['branch']))
build_task_id = taskcluster.Index().findTask(build_index)['taskId']
except taskcluster.exceptions.TaskclusterFailure:
raise errors.NotFoundException(
'Required build not found for TC index', build_index)
task_id = None
continuation_token = None
while not task_id:
options = {'limit': 5}
if continuation_token:
options.update({'continuationToken': continuation_token})
resp = taskcluster.Queue().listDependentTasks(build_task_id,
options=options)
for task in resp['tasks']:
if task['task'].get('extra',
{}).get('suite',
{}).get('name') == 'firefox-ui':
task_id = task['status']['taskId']
break
continuation_token = resp.get('continuationToken')
if not continuation_token:
raise errors.NotFoundException(
'No tests found which use docker image', image_name)
task_definition = taskcluster.Queue().task(task_id)
return task_definition['payload']['image']['taskId']
def get_queue_service(self):
"""
Configured Queue Service
"""
return taskcluster.Queue(
self.build_options('queue/v1')
)
def __init__(self):
self.now = datetime.datetime.utcnow()
self.found_or_created_indexed_tasks = {}
options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}
self.queue_service = taskcluster.Queue(options)
self.index_service = taskcluster.Index(options)
def __init__(self,
branch,
rank,
client_id,
access_token,
log_obj,
task_id=None):
self.rank = rank
self.log_obj = log_obj
# Try builds use a different set of credentials which have access to the
# buildbot-try scope.
if branch == 'try':
self.buildbot = 'buildbot-try'
else:
self.buildbot = 'buildbot'
# We can't import taskcluster at the top of the script because it is
# part of the virtualenv, so import it now. The virtualenv needs to be
# activated before this point by the mozharness script, or else we won't
# be able to find this module.
import taskcluster
taskcluster.config['credentials']['clientId'] = client_id
taskcluster.config['credentials']['accessToken'] = access_token
self.taskcluster_queue = taskcluster.Queue()
self.task_id = task_id or taskcluster.slugId()
self.put_file = taskcluster.utils.putFile
def download_artifact(artifact_path, task_id, artifact_name):
if os.path.exists(artifact_path):
return artifact_path
# Build artifact public url
# Use un-authenticated Taskcluster client to avoid taskcluster-proxy rewrite issue
# https://github.com/taskcluster/taskcluster-proxy/issues/44
queue = taskcluster.Queue(
{"rootUrl": "https://firefox-ci-tc.services.mozilla.com"})
url = queue.buildUrl("getLatestArtifact", task_id, artifact_name)
logger.debug("Downloading artifact", url=url)
@tenacity.retry(
reraise=True,
wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
stop=tenacity.stop_after_attempt(5),
)
def perform_download():
r = requests.get(url, stream=True)
r.raise_for_status()
with open(artifact_path, "wb") as f:
r.raw.decode_content = True
shutil.copyfileobj(r.raw, f)
if artifact_path.endswith(".zip") and not is_zipfile(artifact_path):
raise BadZipFile("File is not a zip file")
perform_download()
def update_task(pulse_data):
import taskcluster
from .models import Task
status = pulse_data["status"]
run_id = pulse_data["runId"]
run_obj = next(run for run in status["runs"] if run["runId"] == run_id)
pool = _get_or_create_pool(status["workerType"])
defaults = {
"decision_id": status["taskGroupId"],
"expires": status["expires"],
"pool": pool,
"resolved": run_obj.get("resolved"),
"started": run_obj.get("started"),
"state": run_obj["state"],
}
task_obj, created = Task.objects.update_or_create(
task_id=status["taskId"],
run_id=run_id,
defaults=defaults,
)
if created:
# `created` field isn't available via pulse, so get it from Taskcluster
queue_svc = taskcluster.Queue({"rootUrl": settings.TC_ROOT_URL})
task = queue_svc.task(status["taskId"])
Task.objects.filter(id=task_obj.id).update(created=task["created"])
def nightly(apks, commit, date_string, is_staging):
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
date = arrow.get(date_string)
task_graph = {}
build_task_id, build_task = generate_build_task(apks)
lib.tasks.schedule_task(queue, build_task_id, build_task)
task_graph[build_task_id] = {}
task_graph[build_task_id]['task'] = queue.task(build_task_id)
sign_task_id, sign_task = generate_signing_task(build_task_id, apks, date,
is_staging)
lib.tasks.schedule_task(queue, sign_task_id, sign_task)
task_graph[sign_task_id] = {}
task_graph[sign_task_id]['task'] = queue.task(sign_task_id)
push_task_id, push_task = generate_push_task(sign_task_id, apks, commit,
is_staging)
lib.tasks.schedule_task(queue, push_task_id, push_task)
task_graph[push_task_id] = {}
task_graph[push_task_id]['task'] = queue.task(push_task_id)
print(json.dumps(task_graph, indent=4, separators=(',', ': ')))
with open('task-graph.json', 'w') as f:
json.dump(task_graph, f)
populate_chain_of_trust_required_but_unused_files()
def main():
parser = argparse.ArgumentParser(
description=
'Spawn tasks to fetch missing symbols from Microsoft symbol server')
args = parser.parse_args()
decision_task_id = os.environ.get('TASK_ID')
if decision_task_id:
task_group_id = decision_task_id
options = {'baseUrl': 'http://taskcluster/queue/v1/'}
else:
task_group_id = taskcluster.utils.slugId()
options = {'credentials': read_tc_auth()}
now = datetime.datetime.utcnow()
keys = {
'task_group_id': task_group_id,
'task_created': format_timedelta(now),
'task_deadline': format_timedelta(now, hours=8),
'artifacts_expires': format_timedelta(now, days=1),
'date_index': now.strftime('%Y%m%d%H%M%S'),
}
try:
queue = taskcluster.Queue(options)
fetch_task_id = spawn_task(queue, keys, decision_task_id,
"fetch-task.json")
keys['fetch_task_id'] = fetch_task_id
spawn_task(queue, keys, decision_task_id, "upload-task.json")
print('https://tools.taskcluster.net/task-group-inspector/#/' +
task_group_id)
except taskcluster.exceptions.TaskclusterAuthFailure as e:
print('TaskclusterAuthFailure: {}'.format(e.body), file=sys.stderr)
raise
def raptor(is_staging):
build_tasks = {}
signing_tasks = {}
other_tasks = {}
mozharness_task_id = fetch_mozharness_task_id()
gecko_revision = taskcluster.Queue().task(
mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
variant = get_variant('raptor')
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(
assemble_task_id, variant, is_staging)
all_raptor_craft_functions = [
BUILDER.craft_raptor_tp6m_task(for_suite=i) for i in range(1, 11)
] + [
BUILDER.craft_raptor_speedometer_task,
BUILDER.craft_raptor_speedometer_power_task,
]
for craft_function in all_raptor_craft_functions:
args = (signing_task_id, mozharness_task_id, variant, gecko_revision)
other_tasks[taskcluster.slugId()] = craft_function(
'armeabi-v7a', *args)
other_tasks[taskcluster.slugId()] = craft_function('arm64-v8a', *args)
other_tasks[taskcluster.slugId()] = craft_function(
'armeabi-v7a', *args, force_run_on_64_bit_device=True)
return (build_tasks, signing_tasks, other_tasks)
def release(version):
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
task_graph = {}
build_task_id, build_task = generate_build_task(version)
lib.tasks.schedule_task(queue, build_task_id, build_task)
task_graph[build_task_id] = {}
task_graph[build_task_id]["task"] = queue.task(build_task_id)
artifacts_info = load_artifacts_manifest()
for artifact, info in artifacts_info.items():
beetmover_task_id, beetmover_task = generate_beetmover_task(
build_task_id, version, artifact, info)
lib.tasks.schedule_task(queue, beetmover_task_id, beetmover_task)
task_graph[beetmover_task_id] = {}
task_graph[beetmover_task_id]["task"] = queue.task(beetmover_task_id)
print(json.dumps(task_graph, indent=4, separators=(',', ': ')))
task_graph_path = "task-graph.json"
with open(task_graph_path, 'w') as f:
json.dump(task_graph, f)
populate_chain_of_trust_required_but_unused_files()
def raptor(builder, is_staging):
mozharness_task_id = fetch_mozharness_task_id()
gecko_revision = taskcluster.Queue({
'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'),
}).task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
variant = get_variant('forPerformanceTest', 'geckoNightly')
build_task = builder.craft_assemble_raptor_task(variant)
signing_task = builder.craft_raptor_signing_task(build_task['label'], variant, is_staging)
tasks = [build_task, signing_task]
for abi in ('armeabi-v7a', 'arm64-v8a'):
variant_apk = variant.get_apk(abi)
all_raptor_craft_functions = [
builder.craft_raptor_tp6m_cold_task(for_suite=i)
for i in range(1, 28)
] + [
builder.craft_raptor_youtube_playback_task,
]
for craft_function in all_raptor_craft_functions:
raptor_task = craft_function(
signing_task['label'], mozharness_task_id, variant_apk, gecko_revision, is_staging
)
tasks.append(raptor_task)
return tasks
def raptor(is_staging):
build_tasks = {}
signing_tasks = {}
other_tasks = {}
geckoview_nightly_version = get_geckoview_versions()
mozharness_task_id = gecko_revision_for_version(geckoview_nightly_version)
gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
for variant in [Variant.from_values(abi, False, 'raptor') for abi in ('arm', 'aarch64')]:
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(
assemble_task_id, variant, is_staging)
all_raptor_craft_functions = [
BUILDER.craft_raptor_tp6m_task(for_suite=i)
for i in range(1, 11)
] + [
BUILDER.craft_raptor_speedometer_task,
BUILDER.craft_raptor_speedometer_power_task,
]
for craft_function in all_raptor_craft_functions:
args = (signing_task_id, mozharness_task_id, variant, gecko_revision)
other_tasks[taskcluster.slugId()] = craft_function(*args)
# we also want the arm APK to be tested on 64-bit-devices
if variant.abi == 'arm':
other_tasks[taskcluster.slugId()] = craft_function(*args, force_run_on_64_bit_device=True)
return (build_tasks, signing_tasks, other_tasks)
def release(version):
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
task_graph = {}
artifacts_info = [
info for info in lib.module_definitions.from_gradle()
if info['shouldPublish']
]
if len(artifacts_info) == 0:
print("Could not get module names from gradle")
sys.exit(2)
build_task_id, build_task = generate_build_task(version, artifacts_info)
lib.tasks.schedule_task(queue, build_task_id, build_task)
task_graph[build_task_id] = {}
task_graph[build_task_id]["task"] = queue.task(build_task_id)
for info in artifacts_info:
beetmover_task_id, beetmover_task = generate_beetmover_task(
build_task_id, version, info['artifact'], info['name'])
lib.tasks.schedule_task(queue, beetmover_task_id, beetmover_task)
task_graph[beetmover_task_id] = {}
task_graph[beetmover_task_id]["task"] = queue.task(beetmover_task_id)
print(json.dumps(task_graph, indent=4, separators=(',', ': ')))
task_graph_path = "task-graph.json"
with open(task_graph_path, 'w') as f:
json.dump(task_graph, f)
lib.util.populate_chain_of_trust_required_but_unused_files()
def main(*task_group_ids):
for task_group_id in task_group_ids:
print("https://community-tc.services.mozilla.com/tasks/groups/" +
task_group_id)
timings = {}
def handler(result):
for task in result["tasks"]:
name = task["task"]["metadata"]["name"]
for run in task["status"]["runs"]:
resolved = run.get("resolved")
if not resolved:
print("Not resolved yet:", name)
continue
key = task["task"]["workerType"]
if "WPT" in name:
key += " WPT"
# fromisoformat doesn’t like the "Z" timezone, [:-1] to remove it
timings.setdefault(key, []).append(
datetime.datetime.fromisoformat(resolved[:-1]) -
datetime.datetime.fromisoformat(run["started"][:-1]))
queue = taskcluster.Queue(taskcluster.optionsFromEnvironment())
queue.listTaskGroup(task_group_id, paginationHandler=handler)
r = lambda d: datetime.timedelta(seconds=round(d.total_seconds()))
for worker_type, t in sorted(timings.items()):
print("count {}, total {}, max: {}\t{}\t{}".format(
len(t),
r(sum(t[1:], start=t[0])),
r(max(t)),
worker_type,
' '.join(str(r(s)) for s in t),
))
def push_artifacts(target, args):
'''
Push all artifacts from dependant tasks
'''
assert args.task_id is not None, 'Missing task id'
# Load config from file/secret
config = Configuration(args)
assert config.has_docker_auth(), 'Missing Docker authentication'
# Setup skopeo
skopeo = Skopeo(
config.docker['registry'],
config.docker['username'],
config.docker['password'],
)
# Load queue service
queue = taskcluster.Queue(config.get_taskcluster_options())
# Load current task description to list its dependencies
logger.info('Loading task status {}'.format(args.task_id))
task = queue.task(args.task_id)
nb_deps = len(task['dependencies'])
assert nb_deps > 0, 'No task dependencies'
# Load dependencies artifacts
artifacts = load_artifacts(task, queue, args.artifact_filter,
args.exclude_filter)
for task_id, artifact_name in artifacts:
push_artifact(queue, skopeo, task_id, artifact_name)
logger.info('All found artifacts were pushed.')
def schedule_phase(name, phase):
session = flask.g.db.session
try:
phase = session.query(Phase) \
.filter(Release.id == Phase.release_id) \
.filter(Release.name == name) \
.filter(Phase.name == phase).one()
except NoResultFound:
flask.abort(404)
if phase.submitted:
flask.abort(409, 'Already submitted!')
queue = taskcluster.Queue({
'credentials': {
'clientId': os.environ.get('TASKCLUSTER_CLIENT_ID'),
'accessToken': os.environ.get('TASKCLUSTER_ACCESS_TOKEN')
},
'maxRetries': 12
})
queue.createTask(phase.task_id, phase.rendered)
phase.submitted = True
phase.completed_by = flask.g.userinfo['email']
phase.completed = datetime.datetime.utcnow()
if all([ph.submitted for ph in phase.release.phases]):
phase.release.status = 'shipped'
session.commit()
return phase.json
def __init__(self, options):
cert = options["credentials"].get("certificate")
if cert and not isinstance(cert, basestring):
options["credentials"]["certificate"] = json.dumps(cert)
self.queue = taskcluster.Queue(options)
self.scheduler = taskcluster.Scheduler(options)
log.debug("Dict of options: %s", options)
def check_task_statuses(task_ids, github_checks_outputter):
"""Verifies whether a set of Taskcluster tasks completed successfully or not.
Returns 0 if all tasks passed completed successfully, 1 otherwise."""
queue = taskcluster.Queue({'rootUrl': os.environ['TASKCLUSTER_ROOT_URL']})
failed_tasks = []
for task in task_ids:
status = queue.status(task)
state = status['status']['state']
if state == 'failed' or state == 'exception':
logger.error('Task {0} failed with state "{1}"'.format(
task, state))
failed_tasks.append(status)
elif state != 'completed':
logger.error('Task {0} had unexpected state "{1}"'.format(
task, state))
failed_tasks.append(status)
if failed_tasks and github_checks_outputter:
github_checks_outputter.output('Failed tasks:')
for task in failed_tasks:
# We need to make an additional call to get the task name.
task_id = task['status']['taskId']
task_name = queue.task(task_id)['metadata']['name']
github_checks_outputter.output(
'* `{}` failed with status `{}`'.format(
task_name, task['status']['state']))
else:
logger.info('All tasks completed successfully')
if github_checks_outputter:
github_checks_outputter.output('All tasks completed successfully')
return 1 if failed_tasks else 0
def _preprocess_message(self, body, message=None):
"""Download the update manifest by processing the received funsize message."""
# Filter out messages which do not apply to our expected routing key regex
if message:
self.logger.debug('CC routing keys: %s' % message.headers['CC'])
if not any([
self.cc_key_regex.search(key)
for key in message.headers['CC']
]):
raise ValueError(
'Routing keys do not match. Skipping message.')
# In case of --push-update-message we already have the wanted manifest
if 'workerId' not in body:
return body
# Retrieve build properties to be used as the manifest
queue = taskcluster.Queue()
task_definition = queue.task(body['status']['taskId'])
manifest = task_definition.get('extra', {}).get('build_props')
# Fake specific properties so we are backward compatible
manifest['tree'] = 'release-%s' % manifest['branch']
manifest['product'] = 'firefox'
try:
d = datetime.strptime(body['status']['runs'][-1]['scheduled'],
'%Y-%m-%dT%H:%M:%S.%fZ')
manifest['buildid'] = d.strftime('%Y%m%d%H%M')
except:
pass
return manifest
def _recreate_task(task_id):
one_year = 365
queue = taskcluster_client.Queue()
task = queue.task(task_id)
LOG.debug("Original task: (Limit 1024 char)")
LOG.debug(str(json.dumps(task))[:1024])
# Start updating the task
task['taskId'] = taskcluster_client.slugId()
artifacts = task['payload'].get('artifacts', {})
for artifact, definition in artifacts.iteritems():
definition['expires'] = taskcluster_client.fromNow('%s days' %
one_year)
# https://bugzilla.mozilla.org/show_bug.cgi?id=1190660
# TC workers create public logs which are 365 days; if the task expiration
# date is the same or less than that we won't have logs for the task
task['expires'] = taskcluster_client.fromNow('%s days' % (one_year + 1))
now = datetime.datetime.utcnow()
tomorrow = now + datetime.timedelta(hours=24)
task['created'] = taskcluster_client.stringDate(now)
task['deadline'] = taskcluster_client.stringDate(tomorrow)
LOG.debug("Contents of new task: (Limit 1024 char)")
LOG.debug(str(task)[:1024])
return task
def release(apks, channel, commit, tag):
queue = taskcluster.Queue({'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']})
task_graph = {}
build_task_id, build_task = generate_build_task(apks, tag)
lib.tasks.schedule_task(queue, build_task_id, build_task)
task_graph[build_task_id] = {}
task_graph[build_task_id]["task"] = queue.task(build_task_id)
sign_task_id, sign_task = generate_signing_task(build_task_id, apks, tag)
lib.tasks.schedule_task(queue, sign_task_id, sign_task)
task_graph[sign_task_id] = {}
task_graph[sign_task_id]["task"] = queue.task(sign_task_id)
push_task_id, push_task = generate_push_task(sign_task_id, apks, channel, commit)
lib.tasks.schedule_task(queue, push_task_id, push_task)
task_graph[push_task_id] = {}
task_graph[push_task_id]["task"] = queue.task(push_task_id)
print json.dumps(task_graph, indent=4, separators=(',', ': '))
task_graph_path = "task-graph.json"
with open(task_graph_path, 'w') as f:
json.dump(task_graph, f)
populate_chain_of_trust_required_but_unused_files()
def raptor(is_staging):
build_tasks = {}
signing_tasks = {}
other_tasks = {}
geckoview_nightly_version = get_geckoview_versions()['nightly']
mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version)
gecko_revision = taskcluster.Queue().task(
mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
for variant in [
Variant.from_values(abi, False, 'forPerformanceTest')
for abi in ('aarch64', 'arm')
]:
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_raptor_task(
variant)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(
assemble_task_id, variant, is_staging)
all_raptor_craft_functions = [
BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
for i in range(1, 15)
]
for craft_function in all_raptor_craft_functions:
args = (signing_task_id, mozharness_task_id, variant,
gecko_revision)
other_tasks[taskcluster.slugId()] = craft_function(*args)
return (build_tasks, signing_tasks, other_tasks)
def fetch_actions_json(task_id):
queue = taskcluster.Queue()
actions_url = queue.buildUrl("getLatestArtifact", task_id,
'public/actions.json')
q = requests.get(actions_url)
q.raise_for_status()
return q.json()
def main():
parser = argparse.ArgumentParser(description='Build and upload binaries')
parser.add_argument('taskcluster_auth',
help='Path to a file containing Taskcluster client '
'ID and authentication token as a JSON file in '
'the form {"clientId": "...", "accessToken": "..."}')
parser.add_argument('--tooltool-auth',
help='Path to a file containing a tooltool '
'authentication token valid for uploading files')
parser.add_argument(
'--local-gecko-clone',
help='Path to a local Gecko clone whose tooltool '
'manifests will be updated with the newly-built binaries')
parser.add_argument('--rust-branch',
default='stable',
help='Revision of the rust repository to use')
parser.add_argument('--task', help='Use an existing task')
args = parser.parse_args()
tc_auth = read_tc_auth(args.taskcluster_auth)
queue = taskcluster.Queue({'credentials': tc_auth})
if args.task:
task_id, initial_wait = args.task, 0
else:
task_id, initial_wait = spawn_task(queue, args), 25
run_id = wait_for_task(queue, task_id, initial_wait)
for artifact in fetch_artifacts(queue, task_id, run_id):
if args.tooltool_auth:
manifest = upload_to_tooltool(args.tooltool_auth, task_id,
artifact)
if args.local_gecko_clone:
update_manifest(artifact, manifest, args.local_gecko_clone)
def release(track, commit, tag):
queue = taskcluster.Queue({ 'baseUrl': 'http://taskcluster/queue/v1' })
task_graph = {}
build_task_id, build_task = generate_build_task(tag)
lib.tasks.schedule_task(queue, build_task_id, build_task)
task_graph[build_task_id] = {}
task_graph[build_task_id]["task"] = queue.task(build_task_id)
sign_task_id, sign_task = generate_signing_task(build_task_id)
lib.tasks.schedule_task(queue, sign_task_id, sign_task)
task_graph[sign_task_id] = {}
task_graph[sign_task_id]["task"] = queue.task(sign_task_id)
push_task_id, push_task = generate_push_task(sign_task_id, track, commit)
lib.tasks.schedule_task(queue, push_task_id, push_task)
task_graph[push_task_id] = {}
task_graph[push_task_id]["task"] = queue.task(push_task_id)
print json.dumps(task_graph, indent=4, separators=(',', ': '))
task_graph_path = "task-graph.json"
with open(task_graph_path, 'w') as token_file:
token_file.write(json.dumps(task_graph))
def raptor(is_staging):
build_tasks = {}
signing_tasks = {}
other_tasks = {}
mozharness_task_id = fetch_mozharness_task_id()
gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
variant = get_variant('forPerformanceTest', 'geckoNightly')
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_raptor_task(variant)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
for abi in ('armeabi-v7a', 'arm64-v8a'):
variant_apk = variant.get_apk(abi)
all_raptor_craft_functions = [
BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
for i in range(1, 28)
] + [
BUILDER.craft_raptor_youtube_playback_task,
]
for craft_function in all_raptor_craft_functions:
args = (signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging)
other_tasks[taskcluster.slugId()] = craft_function(*args)
return (build_tasks, signing_tasks, other_tasks)
def tc_branches():
decision_namespace = 'gecko.v2.%s.latest.firefox.decision'
index = taskcluster.Index()
queue = taskcluster.Queue()
result = index.listNamespaces('gecko.v2', dict(limit=1000))
branches = {
i['name']: dict(name=i['name'], workerTypes=dict())
for i in result.get('namespaces', [])
}
for branchName, branch in branches.items():
# decision task might not exist
try:
decision_task = index.findTask(decision_namespace % branchName)
decision_graph = queue.getLatestArtifact(
decision_task['taskId'], 'public/graph.json')
except taskcluster.exceptions.TaskclusterRestFailure:
continue
for task in decision_graph.get('tasks', []):
task = task['task']
task_cache = task.get('payload', dict()).get('cache', dict())
provisionerId = task.get('provisionerId')
if provisionerId:
branch['provisionerId'] = provisionerId
workerType = task.get('workerType')
if workerType:
branch['workerTypes'].setdefault(
workerType, dict(name=workerType, caches=[]))
if len(task_cache) > 0:
branch['workerTypes'][workerType]['caches'] = list(set(
branch['workerTypes'][workerType]['caches'] +
task_cache.keys()
))
caches_to_skip = current_app.config.get('TASKCLUSTER_CACHES_TO_SKIP', [])
return [
rest.TCBranch(
name=branchName,
provisionerId=branch.get('provisionerId'),
workerTypes={
workerType: rest.TCWorkerType(
name=workerType,
caches=[
cache
for cache in branch['workerTypes'][workerType]['caches']
if cache not in caches_to_skip
],
)
for workerType in branch['workerTypes']
})
for branchName, branch in branches.items()]
def release(apks, track, commit, tag):
queue = taskcluster.Queue({ 'baseUrl': 'http://taskcluster/queue/v1' })
task_graph = {}
build_task_id, build_task = generate_build_task(apks, tag)
lib.tasks.schedule_task(queue, build_task_id, build_task)
task_graph[build_task_id] = {}
task_graph[build_task_id]["task"] = queue.task(build_task_id)
sign_task_id, sign_task = generate_signing_task(build_task_id, apks, tag)
lib.tasks.schedule_task(queue, sign_task_id, sign_task)
task_graph[sign_task_id] = {}
task_graph[sign_task_id]["task"] = queue.task(sign_task_id)
push_task_id, push_task = generate_push_task(sign_task_id, apks, track, commit)
lib.tasks.schedule_task(queue, push_task_id, push_task)
task_graph[push_task_id] = {}
task_graph[push_task_id]["task"] = queue.task(push_task_id)
print json.dumps(task_graph, indent=4, separators=(',', ': '))
task_graph_path = "task-graph.json"
with open(task_graph_path, 'w') as f:
json.dump(task_graph, f)
populate_chain_of_trust_required_but_unused_files()
def __init__(self,
root_url,
client_id=None,
access_token=None,
unit_testing_this=False):
# TODO: remove when backfill tool' soft launch is complete ->
if not unit_testing_this:
raise RuntimeError(
f"Must not instantiate real {self.__class__.__name__} instance "
f"before backfill tool' soft launch is complete")
# <- up to here
options = {'rootUrl': root_url}
credentials = {}
if client_id:
credentials['clientId'] = client_id
if access_token:
credentials['accessToken'] = access_token
# Taskcluster APIs
self.hooks = taskcluster.Hooks({**options, 'credentials': credentials})
# Following least-privilege principle, as services
# bellow don't really need authorization credentials.
self.queue = taskcluster.Queue(options)
self.auth = taskcluster.Auth(options)
