def schedule_task(self, task, update_timestamps=True, dry_run=False):
""" It schedules a TaskCluster task
For more details visit:
http://docs.taskcluster.net/queue/api-docs/#createTask
:param task: It is a TaskCluster task.
:type task: json
:param update_timestamps: It will not update the timestamps if False.
:type update_timestamps: bool
:param dry_run: It allows overwriting the dry_run mode at creation of the manager.
:type dry_run: bool
:returns: Task Status Structure (see link to createTask documentation)
:rtype: dict
"""
LOG.debug("We want to schedule a TC task")
if update_timestamps:
task = refresh_timestamps(task)
# http://schemas.taskcluster.net/queue/v1/create-task-request.json#
if not (dry_run or self.dry_run):
# https://github.com/taskcluster/taskcluster-client.py#create-new-task
task_id = taskcluster_client.slugId()
result = self.queue.createTask(taskId=task_id, payload=task)
LOG.info("Inspect the task in {}".format(get_task_inspector_url(task_id)))
return result
else:
LOG.info("We did not schedule anything because we're running on dry run mode.")
def schedule_graph(task_graph, task_graph_id=None, dry_run=False, *args, **kwargs):
""" It schedules a TaskCluster graph and returns its id.
:param task_graph: It is a TaskCluster graph as defined in here:
http://docs.taskcluster.net/scheduler/api-docs/#createTaskGraph
:type task_graph: json
:param task_graph_id: TC graph id to which this task belongs to
:type task_graph_id: str
:param dry_run: It does not schedule the graph
:type dry_run: bool
:returns: task graph id.
:rtype: int
"""
if not task_graph_id:
task_graph_id = taskcluster_client.slugId()
scheduler = taskcluster_client.Scheduler()
LOG.info("Outputting the graph (graph id: %s):" % task_graph_id)
# We print to stdout instead of using the standard logging with dates and info levels
# XXX: Use a different formatter for other tools to work better with this code
print(json.dumps(task_graph, indent=4))
if dry_run:
LOG.info("DRY-RUN: We have not scheduled the graph.")
else:
if not credentials_available():
return None
try:
# https://github.com/taskcluster/taskcluster-client.py#create-new-task-graph
result = scheduler.createTaskGraph(task_graph_id, task_graph)
LOG.info("See the graph in %s%s" % (TC_TASK_GRAPH_INSPECTOR, task_graph_id))
return result
except Exception as e:
handle_exception(e)
def _recreate_task(task_id):
one_year = 365
queue = taskcluster_client.Queue()
task = queue.task(task_id)
LOG.debug("Original task: (Limit 1024 char)")
LOG.debug(str(json.dumps(task))[:1024])
# Start updating the task
task["taskId"] = taskcluster_client.slugId()
artifacts = task["payload"].get("artifacts", {})
for artifact, definition in artifacts.iteritems():
definition["expires"] = taskcluster_client.fromNow("%s days" % one_year)
# https://bugzilla.mozilla.org/show_bug.cgi?id=1190660
# TC workers create public logs which are 365 days; if the task expiration
# date is the same or less than that we won't have logs for the task
task["expires"] = taskcluster_client.fromNow("%s days" % (one_year + 1))
now = datetime.datetime.utcnow()
tomorrow = now + datetime.timedelta(hours=24)
task["created"] = taskcluster_client.stringDate(now)
task["deadline"] = taskcluster_client.stringDate(tomorrow)
LOG.debug("Contents of new task: (Limit 1024 char)")
LOG.debug(str(task)[:1024])
return task
def nightly_to_production_app(is_staging, version_name):
# Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the
# "nightly" track. We're moving towards having each channel be published to its own app, but we need to
# keep updating this "backwards-compatible" nightly for a while yet
build_type = 'nightlyLegacy'
variants = get_variants_for_build_type(build_type)
architectures = [variant.abi for variant in variants]
apk_paths = [
"public/build/{}/target.apk".format(arch) for arch in architectures
]
build_tasks = {}
signing_tasks = {}
push_tasks = {}
other_tasks = {}
build_task_id = taskcluster.slugId()
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(
architectures, build_type, is_staging, version_name)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
build_task_id,
apk_paths=apk_paths,
channel=
'production', # Since we're publishing to the "production" app, we need to sign for production
index_channel='nightly',
is_staging=is_staging,
)
push_task_id = taskcluster.slugId()
push_tasks[push_task_id] = BUILDER.craft_push_task(
signing_task_id,
apks=apk_paths,
channel=
'production', # We're publishing to the "production" app on the "nightly" track
override_google_play_track='nightly',
is_staging=is_staging,
)
nimbledroid_task_id = taskcluster.slugId()
other_tasks[
nimbledroid_task_id] = BUILDER.craft_upload_apk_nimbledroid_task(
build_task_id)
return (build_tasks, signing_tasks, push_tasks, other_tasks)
def generate_code_quality_task(buildTaskId):
return taskcluster.slugId(), generate_task(
name="(Focus for Android) Code quality",
description=
"Run code quality tools on Focus/Klar for Android code base.",
command=
'echo "--" > .adjust_token && ./gradlew --no-daemon clean detektCheck ktlint lint pmd checkstyle spotbugs',
dependencies=[buildTaskId])
def generate_raw_task(name, description, command_to_run):
checkout = ("export TERM=dumb && git fetch {} {} && "
"git config advice.detachedHead false && "
"git checkout {} && ".format(REPO_URL, BRANCH, HEAD_REV))
return taskcluster.slugId(), BUILDER.raw_task(name=name,
description=description,
command=(checkout +
command_to_run))
def craft_new_task_id():
slug_id = taskcluster.slugId()
# slugId() returns bytes in Python 3. This is not expected by the rest of the API.
# That's why we have to convert it to str.
if not isinstance(slug_id, str):
return slug_id.decode('utf-8')
return slug_id
def upload_apk_nimbledroid_task(dependencies):
return taskcluster.slugId(), generate_task(
name = "(Focus for Android) Upload Debug APK to Nimbledroid",
description = "Upload APKs to Nimbledroid for performance measurement and tracking.",
command = ('echo "--" > .adjust_token'
' && ./gradlew --no-daemon clean assembleKlarArmNightly'
' && python tools/taskcluster/upload_apk_nimbledroid.py'),
dependencies = dependencies,
scopes = [ 'secrets:get:project/focus/nimbledroid' ],
)
def nightly_to_production_app(is_staging, version_name):
# Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the
# "nightly" track. We're moving towards having each channel be published to its own app, but we need to
# keep updating this "backwards-compatible" nightly for a while yet
variant = get_variant('fenixNightlyLegacy', 'geckoNightly')
taskcluster_apk_paths = variant.upstream_artifacts()
build_tasks = {}
signing_tasks = {}
push_tasks = {}
other_tasks = {}
build_task_id = taskcluster.slugId()
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(
variant, is_staging, version_name)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
build_task_id,
taskcluster_apk_paths,
channel=
'production', # Since we're publishing to the "production" app, we need to sign for production
index_channel='nightly',
is_staging=is_staging,
)
push_task_id = taskcluster.slugId()
push_tasks[push_task_id] = BUILDER.craft_push_task(
signing_task_id,
taskcluster_apk_paths,
channel=
'production', # We're publishing to the "production" app on the "nightly" track
override_google_play_track='nightly',
is_staging=is_staging,
)
if not is_staging:
nimbledroid_task_id = taskcluster.slugId()
other_tasks[
nimbledroid_task_id] = BUILDER.craft_upload_apk_nimbledroid_task(
build_task_id)
return (build_tasks, signing_tasks, push_tasks, other_tasks)
def create(self):
"""
Call the Queue’s `createTask` API to schedule a new task, and return its ID.
<https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/references/api#createTask>
"""
worker_payload = self.build_worker_payload()
assert not self.treeherder_required, \
"make sure to call with_treeherder() for this task: %s" % self.name
assert CONFIG.decision_task_id
assert CONFIG.task_owner
assert CONFIG.task_source
def dedup(xs):
seen = set()
return [x for x in xs if not (x in seen or seen.add(x))]
queue_payload = {
"taskGroupId": CONFIG.decision_task_id,
"dependencies": dedup([CONFIG.decision_task_id] + self.dependencies),
"schedulerId": self.scheduler_id,
"provisionerId": self.provisioner_id,
"workerType": self.worker_type,
"created": SHARED.from_now_json(""),
"deadline": SHARED.from_now_json(self.deadline_in),
"expires": SHARED.from_now_json(self.expires_in),
"metadata": {
"name": CONFIG.task_name_template % self.name,
"description": self.description,
"owner": CONFIG.task_owner,
"source": CONFIG.task_source,
},
"payload": worker_payload,
}
scopes = self.scopes + CONFIG.scopes_for_all_subtasks
routes = self.routes + CONFIG.routes_for_all_subtasks
if any(r.startswith("index.") for r in routes):
self.extra.setdefault("index", {})["expires"] = \
SHARED.from_now_json(self.index_and_artifacts_expire_in)
dict_update_if_truthy(
queue_payload,
scopes=scopes,
routes=routes,
extra=self.extra,
priority=self.priority,
)
task_id = taskcluster.slugId()
SHARED.queue_service.createTask(task_id, queue_payload)
print("Scheduled %s: %s" % (task_id, self.name))
return task_id
def pr(artifacts_info):
if SKIP_TASKS_TRIGGER in PR_TITLE:
print("Pull request title contains", SKIP_TASKS_TRIGGER)
print("Exit")
exit(0)
modules = [_get_gradle_module_name(artifact_info) for artifact_info in artifacts_info]
build_tasks = {}
other_tasks = {}
for module in modules:
tasks = create_module_tasks(module)
for task in tasks:
build_tasks[taskcluster.slugId()] = task
for craft_function in (BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_compare_locales_task):
other_tasks[taskcluster.slugId()] = craft_function()
return (build_tasks, other_tasks)
def release_as_fennec(is_staging, version_name):
variant = get_variant('fennecProduction', 'geckoBeta')
channel = 'fennec-production'
build_tasks = {}
signing_tasks = {}
build_task_id = taskcluster.slugId()
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(
variant, channel, is_staging, version_name)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
build_task_id,
variant.upstream_artifacts(),
channel,
is_staging,
)
return (build_tasks, signing_tasks)
def pr_or_push(is_push=False):
if SKIP_TASKS_TRIGGER in PR_TITLE:
print("Pull request title contains", SKIP_TASKS_TRIGGER)
print("Exit")
return {}
variants = get_build_variants()
geckoview_nightly_version = get_geckoview_versions()
mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version)
build_tasks = {}
signing_tasks = {}
other_tasks = {}
for variant in variants:
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
architecture, build_type, _ = get_architecture_and_build_type_and_product_from_variant(
variant)
if (is_push and architecture in ('aarch64', 'arm')
and build_type == 'releaseRaptor'
and SHORT_HEAD_BRANCH == 'master'):
signing_task_id = taskcluster.slugId()
signing_tasks[
signing_task_id] = BUILDER.craft_master_commit_signing_task(
assemble_task_id, variant)
other_tasks[taskcluster.slugId()] = BUILDER.craft_raptor_task(
signing_task_id, mozharness_task_id, variant)
build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant)
for craft_function in (
BUILDER.craft_detekt_task,
BUILDER.craft_ktlint_task,
BUILDER.craft_lint_task,
BUILDER.craft_compare_locales_task,
):
other_tasks[taskcluster.slugId()] = craft_function()
return (build_tasks, signing_tasks, other_tasks)
def submit_task_graph(self, branch, revision, platform, update_number,
locale_desc, extra, mar_signing_format):
graph_id = slugId()
log.info("Submitting a new graph %s", graph_id)
task_graph = self.from_template(
extra=extra, update_number=update_number, platform=platform,
locale_desc=locale_desc, revision=revision,
branch=branch, mar_signing_format=mar_signing_format)
log.debug("Graph definition: %s", task_graph)
res = self.scheduler.createTaskGraph(graph_id, task_graph)
log.info("Result was: %s", res)
return graph_id
def try_ami(ami_id, tc_options):
pool = tc.parse_yaml("worker-pools.yml")["win2016"]
pool.pop("kind")
pool = tc.aws_windows(**pool)
pool = dict(description="", emailOnError=False, owner="*****@*****.**", **pool)
now = datetime.datetime.now().replace(microsecond=0).isoformat()
worker_type = "tmp-" + re.sub("[-:T]", "", now)
pool_id = "proj-servo/" + worker_type
task = {h["hookId"]: h for h in tc.parse_yaml("hooks.yml")}["daily"]["task"]
task["metadata"]["name"] = "Trying new Windows image " + ami_id
task["metadata"]["source"] = \
"https://github.com/servo/taskcluster-config/blob/master/commands/try-ami.py"
task["payload"]["env"]["SOURCE"] = task["metadata"]["source"]
task["payload"]["env"]["TASK_FOR"] = "try-windows-ami"
task["payload"]["env"]["GIT_REF"] = "refs/heads/try-windows-ami"
task["payload"]["env"]["NEW_AMI_WORKER_TYPE"] = worker_type
task["created"] = {"$eval": "now"}
task = jsone.render(task, {})
task_id = taskcluster.slugId()
wm = taskcluster.WorkerManager(tc_options)
queue = taskcluster.Queue(tc_options)
wm.createWorkerPool(pool_id, pool)
try:
queue.createTask(task_id, task)
task_view = "https://community-tc.services.mozilla.com/tasks/"
log("Created " + task_view + task_id)
while 1:
time.sleep(2)
result = queue.status(task_id)
state = result["status"]["state"]
if state not in ["unscheduled", "pending", "running"]:
log("Decision task:", state)
break
# The decision task has finished, so any other task should be scheduled now
while 1:
tasks = []
def handler(result):
for task in result["tasks"]:
if task["status"]["taskId"] != task_id:
tasks.append((task["status"]["taskId"], task["status"]["state"]))
queue.listTaskGroup(result["status"]["taskGroupId"], paginationHandler=handler)
if all(state not in ["unscheduled", "pending"] for _, state in tasks):
for task, _ in tasks:
log("Running " + task_view + task)
break
time.sleep(2)
finally:
wm.deleteWorkerPool(pool_id)
def submit_task_graph(self, branch, revision, platform, update_number,
chunk_name, subchunk, extra):
graph_id = slugId()
log.info("Submitting a new graph %s", graph_id)
task_graph = self.from_template(
extra=extra, update_number=update_number, platform=platform,
chunk_name=chunk_name, subchunk=subchunk, revision=revision,
branch=branch)
log.debug("Graph definition: %s", task_graph)
res = self.scheduler.createTaskGraph(graph_id, task_graph)
log.info("Result was: %s", res)
return graph_id
def nightly(is_staging):
build_tasks = {}
signing_tasks = {}
push_tasks = {}
other_tasks = {}
formatted_date = datetime.datetime.now().strftime('%y%V')
version_name = '1.0.{}'.format(formatted_date)
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_nightly_task(version_name, is_staging)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_nightly_signing_task(
assemble_task_id, is_staging=is_staging
)
push_task_id = taskcluster.slugId()
push_tasks[push_task_id] = BUILDER.craft_push_task(signing_task_id, is_staging)
other_tasks[taskcluster.slugId()] = BUILDER.craft_upload_apk_nimbledroid_task(assemble_task_id)
return (build_tasks, signing_tasks, push_tasks, other_tasks)
def nightly(is_staging):
build_tasks = {}
signing_tasks = {}
push_tasks = {}
other_tasks = {}
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_release_task(
is_staging)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_nightly_signing_task(
assemble_task_id, is_staging=is_staging)
push_task_id = taskcluster.slugId()
push_tasks[push_task_id] = BUILDER.craft_push_task(signing_task_id,
is_staging)
other_tasks[taskcluster.slugId(
)] = BUILDER.craft_upload_apk_nimbledroid_task(assemble_task_id)
return (build_tasks, signing_tasks, push_tasks, other_tasks)
def pr_or_push(is_push):
if not is_push and SKIP_TASKS_TRIGGER in PR_TITLE:
print("Pull request title contains", SKIP_TASKS_TRIGGER)
print("Exit")
return {}
debug_variants = [
variant for variant in get_build_variants()
if variant.endswith('Debug')
]
geckoview_nightly_version = get_geckoview_versions()['nightly']
mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version)
gecko_revision = taskcluster.Queue().task(
mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
build_tasks = {}
signing_tasks = {}
other_tasks = {}
for variant in debug_variants:
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant)
if is_push and SHORT_HEAD_BRANCH == 'master':
other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task()
for variant in ('armRaptor', 'aarch64Raptor'):
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(
variant)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(
assemble_task_id, variant)
ALL_RAPTOR_CRAFT_FUNCTIONS = [
BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
for i in range(1, 15)
]
for craft_function in ALL_RAPTOR_CRAFT_FUNCTIONS:
args = (signing_task_id, mozharness_task_id, variant,
gecko_revision)
other_tasks[taskcluster.slugId()] = craft_function(*args)
for craft_function in (
BUILDER.craft_detekt_task,
BUILDER.craft_ktlint_task,
BUILDER.craft_lint_task,
BUILDER.craft_compare_locales_task,
):
other_tasks[taskcluster.slugId()] = craft_function()
return (build_tasks, signing_tasks, other_tasks)
def generate_signing_task(build_task_id):
return taskcluster.slugId(), BUILDER.build_signing_task(
build_task_id,
name="(Focus for Android) Signing task",
description="Sign release builds of Focus/Klar",
apks=[
"public/focus.apk",
"public/klar.apk"
],
scopes = [
"project:mobile:focus:releng:signing:cert:release-signing",
"project:mobile:focus:releng:signing:format:focus-jar"
]
)
def raptor(is_staging):
build_tasks = {}
signing_tasks = {}
other_tasks = {}
mozharness_task_id = fetch_mozharness_task_id()
gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
for variant in [Variant.from_values(abi, False, 'forPerformanceTest') for abi in ('aarch64', 'arm')]:
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_raptor_task(variant)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
all_raptor_craft_functions = [
BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
for i in range(1, 27)
]
for craft_function in all_raptor_craft_functions:
args = (signing_task_id, mozharness_task_id, variant, gecko_revision)
other_tasks[taskcluster.slugId()] = craft_function(*args)
return (build_tasks, signing_tasks, other_tasks)
def generate_push_task(signing_task_id, apks, commit, is_staging):
artifacts = ["public/{}".format(os.path.basename(apk)) for apk in apks]
return taskcluster.slugId(), BUILDER.craft_push_task(
signing_task_id,
name="(Reference Browser) Push task",
description="Upload signed release builds of Reference Browser to Google Play",
apks=artifacts,
scopes=[
"project:mobile:reference-browser:releng:googleplay:product:reference-browser{}".format(':dep' if is_staging else '')
],
commit=commit,
is_staging=is_staging
)
def submit_task_graph(self, branch, revision, platform, update_number,
locale_desc, extra, mar_signing_format):
task_group_id = slugId()
atomic_task_id = slugId()
log.info("Submitting a new graph %s", task_group_id)
task_graph = self.from_template(extra=extra,
update_number=update_number,
platform=platform,
locale_desc=locale_desc,
revision=revision,
branch=branch,
mar_signing_format=mar_signing_format,
task_group_id=task_group_id,
atomic_task_id=atomic_task_id)
log.debug("Graph definition: %s", task_graph)
for t in task_graph["tasks"]:
log.info("Submitting %s", t["taskId"])
self.tc_queue.createTask(t["taskId"], t["task"])
# The "atomic submission" task is resolved only on successful
# submission of all tasks and unblocks the rest of the tasks.
log.info("Resolving atomic task %s", atomic_task_id)
self.resolve_task(atomic_task_id)
return task_group_id
def raptor(is_staging):
build_tasks = {}
signing_tasks = {}
other_tasks = {}
geckoview_nightly_version = get_geckoview_versions()
mozharness_task_id = gecko_revision_for_version(geckoview_nightly_version)
gecko_revision = taskcluster.Queue().task(
mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
for variant in [
Variant.from_values(abi, False, 'raptor')
for abi in ('arm', 'aarch64')
]:
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(
assemble_task_id, variant, is_staging)
all_raptor_craft_functions = [
BUILDER.craft_raptor_tp6m_task(for_suite=i) for i in range(1, 11)
] + [
BUILDER.craft_raptor_speedometer_task,
BUILDER.craft_raptor_speedometer_power_task,
]
for craft_function in all_raptor_craft_functions:
args = (signing_task_id, mozharness_task_id, variant,
gecko_revision)
other_tasks[taskcluster.slugId()] = craft_function(*args)
# we also want the arm APK to be tested on 64-bit-devices
if variant.abi == 'arm':
other_tasks[taskcluster.slugId()] = craft_function(
*args, force_run_on_64_bit_device=True)
return (build_tasks, signing_tasks, other_tasks)
def release(artifacts_info, version, is_snapshot, is_staging):
version = components_version() if version is None else version
build_tasks = {}
wait_on_builds_tasks = {}
beetmover_tasks = {}
other_tasks = {}
wait_on_builds_task_id = taskcluster.slugId()
for artifact_info in artifacts_info:
build_task_id = taskcluster.slugId()
module_name = _get_gradle_module_name(artifact_info)
build_tasks[build_task_id] = BUILDER.craft_build_task(
module_name=module_name,
gradle_tasks=_get_release_gradle_tasks(module_name, is_snapshot),
subtitle='({}{})'.format(version,
'-SNAPSHOT' if is_snapshot else ''),
run_coverage=False,
is_snapshot=is_snapshot,
artifact_info=artifact_info)
beetmover_tasks[taskcluster.slugId()] = BUILDER.craft_beetmover_task(
build_task_id, wait_on_builds_task_id, version,
artifact_info['artifact'], artifact_info['name'], is_snapshot,
is_staging)
wait_on_builds_tasks[
wait_on_builds_task_id] = BUILDER.craft_wait_on_builds_task(
build_tasks.keys())
if is_snapshot: # XXX These jobs perma-fail on release
for craft_function in (BUILDER.craft_detekt_task,
BUILDER.craft_ktlint_task,
BUILDER.craft_compare_locales_task):
other_tasks[taskcluster.slugId()] = craft_function()
return (build_tasks, wait_on_builds_tasks, beetmover_tasks, other_tasks)
def pr_or_push(is_push=False):
if SKIP_TASKS_TRIGGER in PR_TITLE:
print("Pull request title contains", SKIP_TASKS_TRIGGER)
print("Exit")
return {}
debug_variants = [variant for variant in get_build_variants() if variant.endswith('Debug')]
geckoview_nightly_version = get_geckoview_versions()
mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version)
gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
build_tasks = {}
signing_tasks = {}
other_tasks = {}
for variant in debug_variants:
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant)
if is_push and SHORT_HEAD_BRANCH == 'master':
for variant in ('armRaptor', 'aarch64Raptor'):
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_master_commit_signing_task(assemble_task_id, variant)
ALL_RAPTOR_CRAFT_FUNCTIONS = [
BUILDER.craft_raptor_tp6m_task(for_suite=i)
for i in range(1, 11)
] + [
BUILDER.craft_raptor_speedometer_task,
BUILDER.craft_raptor_speedometer_power_task,
]
for craft_function in ALL_RAPTOR_CRAFT_FUNCTIONS:
args = (signing_task_id, mozharness_task_id, variant, gecko_revision)
other_tasks[taskcluster.slugId()] = craft_function(*args)
architecture, _ = get_architecture_and_build_type_from_variant(variant)
# we also want the arm APK to be tested on 64-bit-devices
if architecture == 'arm':
other_tasks[taskcluster.slugId()] = craft_function(*args, force_run_on_64_bit_device=True)
for craft_function in (
BUILDER.craft_detekt_task,
BUILDER.craft_ktlint_task,
BUILDER.craft_lint_task,
BUILDER.craft_compare_locales_task,
):
other_tasks[taskcluster.slugId()] = craft_function()
return (build_tasks, signing_tasks, other_tasks)
def generate_compare_locales_task():
return taskcluster.slugId(), generate_task(
name="(Focus for Android) String validation",
description="Check Focus/Klar for Android for errors in en-US and l10n.",
command=('pip install "compare-locales>=5.0.2,<6.0"'
' && mkdir -p /opt/focus-android/test_artifacts'
' && compare-locales --validate l10n.toml .'
' && compare-locales --json=/opt/focus-android/test_artifacts/data.json l10n.toml .'),
artifacts={
"public": {
"type": "directory",
"path": "/opt/focus-android/test_artifacts",
"expires": taskcluster.stringDate(taskcluster.fromNow('1 week'))
}
})
def release_snapshot(components, is_snapshot, is_staging):
version = components_version()
build_tasks = {}
wait_on_builds_tasks = {}
beetmover_tasks = {}
other_tasks = {}
wait_on_builds_task_id = taskcluster.slugId()
for component in components:
build_task_id = taskcluster.slugId()
module_name = _get_gradle_module_name(component)
build_tasks[build_task_id] = BUILDER.craft_build_task(
module_name=module_name,
gradle_tasks=_get_release_gradle_tasks(module_name, is_snapshot),
subtitle='({}{})'.format(version,
'-SNAPSHOT' if is_snapshot else ''),
run_coverage=False,
is_snapshot=is_snapshot,
component=component)
beetmover_tasks[
taskcluster.slugId()] = BUILDER.craft_snapshot_beetmover_task(
build_task_id, wait_on_builds_task_id, version,
component['artifact'], component['name'], is_snapshot,
is_staging)
wait_on_builds_tasks[wait_on_builds_task_id] = BUILDER.craft_barrier_task(
build_tasks.keys())
for craft_function in (BUILDER.craft_detekt_task,
BUILDER.craft_ktlint_task,
BUILDER.craft_compare_locales_task):
other_tasks[taskcluster.slugId()] = craft_function()
return (build_tasks, wait_on_builds_tasks, beetmover_tasks, other_tasks)
def generate_push_task(signing_task_id, track, commit):
return taskcluster.slugId(), BUILDER.build_push_task(
signing_task_id,
name="(Focus for Android) Push task",
description="Upload signed release builds of Focus/Klar to Google Play",
apks=[
"public/focus.apk",
"public/klar.apk"
],
scopes=[
"project:mobile:focus:releng:googleplay:product:focus"
],
track = track,
commit = commit
)
def generate_push_task(signing_task_id, apks, channel, commit):
artifacts = []
for apk in apks:
artifacts.append("public/" + os.path.basename(apk))
print artifacts
return taskcluster.slugId(), BUILDER.build_push_task(
signing_task_id,
name="(Focus for Android) Push task",
description="Upload signed release builds of Focus/Klar to Google Play",
apks=artifacts,
scopes=["project:mobile:focus:releng:googleplay:product:focus"],
channel=channel,
commit=commit)
def pr():
if SKIP_TASKS_TRIGGER in PR_TITLE:
print("Pull request title contains", SKIP_TASKS_TRIGGER)
print("Exit")
return {}
build_tasks = {}
signing_tasks = {}
other_tasks = {}
variant = get_variant('debug', 'geckoNightly')
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_pr_task(variant)
build_tasks[taskcluster.slugId()] = BUILDER.craft_test_pr_task(variant)
for craft_function in (
BUILDER.craft_detekt_task,
BUILDER.craft_ktlint_task,
BUILDER.craft_lint_task,
BUILDER.craft_compare_locales_task,
):
other_tasks[taskcluster.slugId()] = craft_function()
return (build_tasks, signing_tasks, other_tasks)
def create(self):
"""
Call the Queue’s `createTask` API to schedule a new task, and return its ID.
<https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/references/api#createTask>
"""
worker_payload = self.build_worker_payload()
assert not self.treeherder_required, \
"make sure to call with_treeherder() for this task: %s" % self.name
assert CONFIG.decision_task_id
assert CONFIG.task_owner
assert CONFIG.task_source
queue_payload = {
"taskGroupId": CONFIG.decision_task_id,
"dependencies": [CONFIG.decision_task_id] + self.dependencies,
"schedulerId": self.scheduler_id,
"provisionerId": self.provisioner_id,
"workerType": self.worker_type,
"created": SHARED.from_now_json(""),
"deadline": SHARED.from_now_json(self.deadline_in),
"expires": SHARED.from_now_json(self.expires_in),
"metadata": {
"name": CONFIG.task_name_template % self.name,
"description": self.description,
"owner": CONFIG.task_owner,
"source": CONFIG.task_source,
},
"payload": worker_payload,
}
scopes = self.scopes + CONFIG.scopes_for_all_subtasks
routes = self.routes + CONFIG.routes_for_all_subtasks
if any(r.startswith("index.") for r in routes):
self.extra.setdefault("index", {})["expires"] = \
SHARED.from_now_json(self.index_and_artifacts_expire_in)
dict_update_if_truthy(
queue_payload,
scopes=scopes,
routes=routes,
extra=self.extra,
)
task_id = taskcluster.slugId().decode("utf8")
SHARED.queue_service.createTask(task_id, queue_payload)
print("Scheduled %s: %s" % (task_id, self.name))
return task_id
def build_task_graph(event, all_tasks, tasks):
task_id_map = OrderedDict()
taskgroup_id = os.environ.get("TASK_ID", taskcluster.slugId())
def add_task(task_name, task):
depends_on_ids = []
if "depends-on" in task:
for depends_name in task["depends-on"]:
if depends_name not in task_id_map:
add_task(depends_name, all_tasks[depends_name])
depends_on_ids.append(task_id_map[depends_name][0])
env_extra = {}
if "download-artifacts" in task:
env_extra["TASK_ARTIFACTS"] = json.dumps([
get_artifact_data(artifact, task_id_map)
for artifact in task["download-artifacts"]
])
task_id, task_data = create_tc_task(event,
task,
taskgroup_id,
depends_on_ids,
env_extra=env_extra)
task_id_map[task_name] = (task_id, task_data)
for task_name, task in tasks.items():
if task_name == "sink-task":
# sink-task will be created below at the end of the ordered dict,
# so that it can depend on all other tasks.
continue
add_task(task_name, task)
# GitHub branch protection for pull requests needs us to name explicit
# required tasks - which doesn't suffice when using a dynamic task graph.
# To work around this we declare a sink task that depends on all the other
# tasks completing, and checks if they have succeeded. We can then
# make the sink task the sole required task for pull requests.
sink_task = tasks.get("sink-task")
if sink_task:
logger.info("Scheduling sink-task")
depends_on_ids = [x[0] for x in task_id_map.values()]
sink_task["command"] += " {}".format(" ".join(depends_on_ids))
task_id_map["sink-task"] = create_tc_task(event, sink_task,
taskgroup_id, depends_on_ids)
else:
logger.info("sink-task is not scheduled")
return task_id_map
def retrigger(self):
"""This function implements ability to perform retriggers on tasks"""
new_task_id = taskcluster.slugId()
task = get_task(self.id)
task["payload"]["id"] = new_task_id
task["created"] = taskcluster.stringDate(datetime.utcnow())
task["deadline"] = taskcluster.stringDate(taskcluster.fromNow("90 minutes"))
if self._should_retrigger(task) == "false":
logger.info(
"Not retriggering task '{}', task should not be retriggered "
"and force not specified.".format(task["tags"]["label"])
)
return
logger.info("Retriggering task '{}'".format(task["tags"]["label"]))
create_task(new_task_id, task)
def nightly():
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
task_graph = {}
build_task_id = taskcluster.slugId()
build_task = generate_build_task()
lib.tasks.schedule_task(queue, build_task_id, build_task)
task_graph[build_task_id] = {}
task_graph[build_task_id]["task"] = queue.task(build_task_id)
print(json.dumps(task_graph, indent=4, separators=(',', ': ')))
task_graph_path = "task-graph.json"
with open(task_graph_path, 'w') as f:
json.dump(task_graph, f)
def generate_ui_test_task(dependencies):
return taskcluster.slugId(), generate_task(
name = "(Focus for Android) UI tests",
description = "Run UI tests for Focus/Klar for Android.",
command = ('echo "--" > .adjust_token'
' && ./gradlew --no-daemon clean assembleFocusWebviewUniversalDebug assembleFocusWebviewUniversalDebugAndroidTest'
' && tools/taskcluster/execute-firebase-test.sh'),
dependencies = dependencies,
scopes = [ 'secrets:get:project/focus/firebase' ],
artifacts = {
"public": {
"type": "directory",
"path": "/opt/focus-android/test_artifacts",
"expires": taskcluster.stringDate(taskcluster.fromNow('1 week'))
}
})
def retrigger_task(task_id, dry_run=False):
""" Given a task id (our uuid) we query it and build
a new task based on the old one which we schedule on TaskCluster.
We don't call the rerun API since we can't rerun a task past
its deadline, instead we create a new task with a new taskGroupId,
expiration, creation and deadline values.
task_id (int) - ID that identifies a task on Taskcluster
dry_run (bool) - Default to False. If True, it won't trigger
a task.
returns - 0 for dry_run case, -1 for any failure or the task id (int)
of a successful retriggered task.
http://docs.taskcluster.net/queue/api-docs/#createTask
"""
if not credentials_available():
return None
results = None
# XXX: evaluate this code for when we can still extend the graph
# insted of scheduling a new one
try:
# Copy the task with fresh timestamps
original_task = get_task(task_id)
new_task = refresh_timestamps(original_task)
new_task['taskId'] = taskcluster_client.slugId()
# Create the graph
task_graph = generate_task_graph(
scopes=[],
tasks=[new_task],
metadata=new_task['metadata']
)
if not dry_run:
LOG.info("Attempting to schedule new graph")
results = schedule_graph(task_graph)
else:
LOG.info("Dry-run mode: Nothing was retriggered.")
except Exception as e:
handle_exception(e)
return results
def __init__(self, branch, rank, client_id, access_token, log_obj):
self.rank = rank
self.log_obj = log_obj
# Try builds use a different set of credentials which have access to the
# buildbot-try scope.
if branch == 'try':
self.buildbot = 'buildbot-try'
else:
self.buildbot = 'buildbot'
# We can't import taskcluster at the top of the script because it is
# part of the virtualenv, so import it now. The virtualenv needs to be
# activated before this point by the mozharness script, or else we won't
# be able to find this module.
import taskcluster
taskcluster.config['credentials']['clientId'] = client_id
taskcluster.config['credentials']['accessToken'] = access_token
self.taskcluster_queue = taskcluster.Queue()
self.task_id = taskcluster.slugId()
self.put_file = taskcluster.utils.putFile
def retrigger_task(task_id, dry_run=False):
""" Given a task id (our uuid) we query it and build
a new task based on the old one which we schedule on TaskCluster.
We don't call the rerun API since we can't rerun a task past
its deadline, instead we create a new task with a new taskGroupId,
expiration, creation and deadline values.
task_id (int) - ID that identifies a task on Taskcluster
dry_run (bool) - Default to False. If True, it won't trigger
a task.
returns - 0 for dry_run case, -1 for any failure or the task id (int)
of a succesful retriggered task.
http://docs.taskcluster.net/queue/api-docs/#createTask
"""
one_year = 365
new_task_id = 0
try:
queue = taskcluster_client.Queue()
task = queue.task(task_id)
LOG.debug("Original task: (Limit 1024 char)")
LOG.debug(str(json.dumps(task))[:1024])
new_task_id = taskcluster_client.slugId()
artifacts = task['payload'].get('artifacts', {})
for artifact, definition in artifacts.iteritems():
definition['expires'] = taskcluster_client.fromNow('%s days' % one_year)
# The task group will be identified by the ID of the only
# task in the group
task['taskGroupId'] = new_task_id
# https://bugzilla.mozilla.org/show_bug.cgi?id=1190660
# TC workers create public logs which are 365 days; if the task expiration
# date is the same or less than that we won't have logs for the task
task['expires'] = taskcluster_client.fromNow('%s days' % (one_year + 1))
now = datetime.datetime.utcnow()
tomorrow = now + datetime.timedelta(hours=24)
task['created'] = taskcluster_client.stringDate(now)
task['deadline'] = taskcluster_client.stringDate(tomorrow)
LOG.debug("Contents of new task: (Limit 1024 char)")
LOG.debug(str(task)[:1024])
if not dry_run:
LOG.info("Attempting to schedule new task with task_id: {}".format(new_task_id))
result = queue.createTask(new_task_id, task)
LOG.debug(json.dumps(result))
LOG.info("{}/task-inspector/#{}".format(TASKCLUSTER_TOOLS_HOST, new_task_id))
else:
LOG.info("Dry-run mode: Nothing was retriggered.")
except taskcluster_client.exceptions.TaskclusterRestFailure as e:
traceback.print_exc()
new_task_id = -1
except taskcluster_client.exceptions.TaskclusterAuthFailure as e:
# Hack until we fix it in the issue
if str(e) == "Authorization Failed":
LOG.error("The taskclaster client that you specified is lacking "
"the right set of scopes.")
LOG.error("Run this same command with --debug and you will see "
"the missing scopes (the output comes from the "
"taskcluster python client)")
elif str(e) == "Authentication Error":
LOG.error("Make sure that you create permanent credentials and you "
"set these environment variables: TASKCLUSTER_CLIENT_ID & "
"TASKCLUSTER_ACCESS_TOKEN")
new_task_id = -1
return new_task_id
# def create_detekt_task():
# return create_task(
# name='Android Components - detekt',
# description='Running detekt over all modules',
# command='detektCheck')
# def create_ktlint_task():
# return create_task(
# name='Android Components - ktlint',
# description='Running ktlint over all modules',
# command='ktlint')
if __name__ == "__main__":
queue = taskcluster.Queue({ 'baseUrl': 'http://taskcluster/queue/v1' })
modules = fetch_module_names()
if len(modules) == 0:
print "Could not get module names from gradle"
sys.exit(2)
for module in modules:
task = create_module_task(module)
task_id = taskcluster.slugId()
schedule_task(queue, task_id, task)
# schedule_task(queue, taskcluster.slugId(), create_detekt_task())
# schedule_task(queue, taskcluster.slugId(), create_ktlint_task())
config["credentials"]["certificate"] = json.dumps(config["credentials"]["certificate"])
scheduler = taskcluster.Scheduler(config)
# funsize scheduler key
pvt_key = open("id_rsa").read()
template_vars = {
"stableSlugId": stableSlugId(),
"now": stringDate(datetime.datetime.utcnow()),
"now_ms": time.time() * 1000,
"fromNow": fromNow,
"sign_task": partial(sign_task, pvt_key=pvt_key),
### TODO: change these
"url": "http://people.mozilla.org/~raliiev/bouncer.apk",
"filename": "bouncer.apk",
"signign_format": "jar",
"hash": "8f4210c62cf533322b09237a3741bc3e9bb52582b8d0b88c52a0d78a6eabe08e29b636d5c9668e8db721c9dead36736db643c53231e966c86dbc28d86b9eb699",
}
template_file = os.path.join(os.path.dirname(__file__), "graph.yml")
with open(template_file) as f:
template = Template(f.read(), undefined=StrictUndefined)
rendered = template.render(**template_vars)
graph = yaml.safe_load(rendered)
import pprint
pprint.pprint(graph)
graph_id = slugId()
print "submitting", graph_id
print scheduler.createTaskGraph(graph_id, graph)
print "https://tools.taskcluster.net/task-graph-inspector/#{}/".format(graph_id)
import os
import taskcluster
queue = taskcluster.Queue({"credentials": {
"clientId": os.environ["TASKCLUSTER_CLIENT_ID"],
"accessToken": os.environ["TASKCLUSTER_ACCESS_TOKEN"],
}})
taskId, taskGroupId = taskcluster.slugId(), taskcluster.slugId()
task = queue.createTask({
"created": queue.Time(time.Now()),
"deadline": queue.Time(time.Now().AddDate(0, 0, 1)),
"expires": queue.Time(time.Now().AddDate(0, 1, 0)),
"payload": {
"command": [ ... ],
"maxRunTime": 7200,
},
"retries": 1,
...
}, taskId=taskId)
