def _recreate_task(task_id):
one_year = 365
queue = taskcluster_client.Queue()
task = queue.task(task_id)
LOG.debug("Original task: (Limit 1024 char)")
LOG.debug(str(json.dumps(task))[:1024])
# Start updating the task
task["taskId"] = taskcluster_client.slugId()
artifacts = task["payload"].get("artifacts", {})
for artifact, definition in artifacts.iteritems():
definition["expires"] = taskcluster_client.fromNow("%s days" % one_year)
# https://bugzilla.mozilla.org/show_bug.cgi?id=1190660
# TC workers create public logs which are 365 days; if the task expiration
# date is the same or less than that we won't have logs for the task
task["expires"] = taskcluster_client.fromNow("%s days" % (one_year + 1))
now = datetime.datetime.utcnow()
tomorrow = now + datetime.timedelta(hours=24)
task["created"] = taskcluster_client.stringDate(now)
task["deadline"] = taskcluster_client.stringDate(tomorrow)
LOG.debug("Contents of new task: (Limit 1024 char)")
LOG.debug(str(task)[:1024])
return task
def refresh_timestamps(task):
''' It refreshes the timestamps of the task. '''
# XXX split this function
LOG.debug("Updating timestamps of task.")
LOG.debug("Original task: (Limit 1024 char)")
LOG.debug(str(json.dumps(task))[:1024])
artifacts = task['payload'].get('artifacts', {})
for artifact, definition in artifacts.iteritems():
definition['expires'] = taskcluster_client.fromNow('%s days' % 365)
# https://bugzilla.mozilla.org/show_bug.cgi?id=1190660
# TC workers create public logs which are 365 days; if the task expiration
# date is the same or less than that we won't have logs for the task
task['expires'] = taskcluster_client.fromNow('%s days' % (365 + 1))
now = datetime.datetime.utcnow()
tomorrow = now + datetime.timedelta(hours=24)
task['created'] = taskcluster_client.stringDate(now)
task['deadline'] = taskcluster_client.stringDate(tomorrow)
LOG.debug("Contents of new task: (Limit 1024 char)")
LOG.debug(str(task)[:1024])
return task
def generate_task_payload(self, flavor, properties):
"""Generate the task payload data for the given type of test and properties.
:param flavor: Type of test to run (functional or update).
:param properties: Task properties for template rendering
"""
template_file = os.path.join(os.path.dirname(__file__),
'tasks', '{}.yml'.format(flavor))
if not os.path.isfile(template_file):
raise errors.NotSupportedException('Test type "{}" not supported.'.format(flavor))
with open(template_file) as f:
template = jinja2.Template(f.read(), undefined=jinja2.StrictUndefined)
template_vars = copy.deepcopy(properties)
template_vars.update({
'stableSlugId': taskcluster.stableSlugId(),
'now': taskcluster.stringDate(datetime.datetime.utcnow()),
'fromNow': taskcluster.fromNow,
'docker_task_id': self.get_docker_task_id(properties),
})
rendered = template.render(**template_vars)
return yaml.safe_load(rendered)
def build_task(self,
name,
description,
command,
artifacts={},
scopes=[],
features={}):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType": 'gecko-focus',
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [self.task_id],
"routes": [],
"scopes": scopes,
"requires": "all-completed",
"payload": {
"features": features,
"maxRunTime": 7200,
"image": "mozillamobile/android-components:1.15",
"command": ["/bin/bash", "--login", "-cx", command],
"artifacts": artifacts,
"deadline": taskcluster.stringDate(deadline)
},
"provisionerId": "aws-provisioner-v1",
"metadata": {
"name": name,
"description": description,
"owner": self.owner,
"source": self.source
}
}
def _craft_artifacts_from_variant(variant):
return {
apk.taskcluster_path: {
'type':
'file',
'path':
apk.absolute_path(variant.build_type),
'expires':
taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
}
for apk in variant.apks
}
def _craft_artifacts_from_variant(variant):
return {
'public/target.apk': {
'type':
'file',
'path':
_craft_apk_full_path_from_variant(variant),
'expires':
taskcluster.stringDate(
taskcluster.fromNow(lib.tasks.DEFAULT_EXPIRES_IN)),
}
}
def create_task(name, description, command):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType": 'github-worker',
"taskGroupId": TASK_ID,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"schedulerId": "taskcluster-github",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [ TASK_ID ],
"routes": [],
"scopes": [],
"requires": "all-completed",
"payload": {
"features": {},
"maxRunTime": 7200,
"image": "mozillamobile/android-components:1.4",
"command": [
"/bin/bash",
"--login",
"-cx",
"export TERM=dumb && git fetch %s %s && git config advice.detachedHead false && git checkout %s && ./gradlew --no-daemon clean %s" % (REPO_URL, BRANCH, COMMIT, command)
],
"artifacts": {},
"deadline": taskcluster.stringDate(deadline)
},
"provisionerId": "aws-provisioner-v1",
"metadata": {
"name": name,
"description": description,
"owner": "*****@*****.**",
"source": "https://github.com/mozilla-mobile/android-components"
}
}
def create_task(name, description, command):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType": 'github-worker',
"taskGroupId": TASK_ID,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"schedulerId": "taskcluster-github",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [ TASK_ID ],
"routes": [],
"scopes": [],
"requires": "all-completed",
"payload": {
"features": {},
"maxRunTime": 7200,
"image": "mozillamobile/mentat:1.1",
"command": [
"/bin/bash",
"--login",
"-cx",
"export TERM=dumb && git fetch %s %s && git config advice.detachedHead false && git checkout %s && cd sdks/android/Mentat && ./gradlew --no-daemon clean %s" % (REPO_URL, BRANCH, COMMIT, command)
],
"artifacts": {},
"deadline": taskcluster.stringDate(deadline)
},
"provisionerId": "aws-provisioner-v1",
"metadata": {
"name": name,
"description": description,
"owner": "*****@*****.**",
"source": "https://github.com/mozilla/mentat"
}
}
def build_signing_task(self, build_task_id, name, description, apks=[], scopes=[], routes=[]):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType": 'mobile-signing-v1',
"taskGroupId": self.task_id,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"schedulerId": "taskcluster-github",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [ self.task_id, build_task_id],
"routes": routes,
"scopes": scopes,
"requires": "all-completed",
"payload": {
"maxRunTime": 3600,
"upstreamArtifacts": [
{
"paths": apks,
"formats": [
"focus-jar"
],
"taskId": build_task_id,
"taskType": "build"
}
]
},
"provisionerId": "scriptworker-prov-v1",
"metadata": {
"name": name,
"description": description,
"owner": self.owner,
"source": self.source
}
}
def craft_push_task(self, signing_task_id, name, description, is_staging,
apks, scopes, commit):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType":
'mobile-pushapk-dep-v1' if is_staging else 'mobile-pushapk-v1',
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [self.task_id, signing_task_id],
"routes": [],
"scopes": scopes,
"requires": "all-completed",
"payload": {
"commit":
commit,
"google_play_track":
'nightly',
"upstreamArtifacts": [{
"paths": apks,
"taskId": signing_task_id,
"taskType": "signing"
}]
},
"provisionerId": "scriptworker-prov-v1",
"metadata": {
"name": name,
"description": description,
"owner": self.owner,
"source": self.source
}
}
def craft_build_task(self,
module_name,
gradle_tasks,
build_docker_image_task_id,
subtitle='',
run_coverage=False,
is_snapshot=False,
artifact_info=None):
artifacts = {} if artifact_info is None else {
artifact_info['artifact']: {
'type':
'file',
'expires':
taskcluster.stringDate(
taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
'path':
artifact_info['path']
}
}
scopes = [
"secrets:get:project/mobile/android-components/public-tokens"
] if run_coverage else []
snapshot_flag = '-Psnapshot ' if is_snapshot else ''
coverage_flag = '-Pcoverage ' if run_coverage else ''
gradle_command = ('./gradlew --no-daemon clean ' + coverage_flag +
snapshot_flag + gradle_tasks)
post_gradle_command = 'automation/taskcluster/action/upload_coverage_report.sh' if run_coverage else ''
command = ' && '.join(cmd
for cmd in (gradle_command, post_gradle_command)
if cmd)
features = {}
if artifact_info is not None:
features['chainOfTrust'] = True
elif any(scope.startswith('secrets:') for scope in scopes):
features['taskclusterProxy'] = True
return self._craft_build_ish_task(
name='Android Components - Module {} {}'.format(
module_name, subtitle),
description='Execure Gradle tasks for module {}'.format(
module_name),
command=command,
features=features,
scopes=scopes,
artifacts=artifacts,
build_docker_image_task_id=build_docker_image_task_id,
)
def craft_assemble_release_task(self, is_staging=False):
artifacts = {
_ARCH_APK_LOCATION_PATTERN.format(arch): {
"type":
'file',
"path":
"/build/reference-browser/app/build/outputs/apk/geckoNightly{}/release/"
"app-geckoNightly-{}-release-unsigned.apk".format(
arch.capitalize(), arch),
"expires":
taskcluster.stringDate(
taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
}
for arch in _SUPPORTED_ARCHITECTURES
}
sentry_secret = '{}project/mobile/reference-browser/sentry'.format(
'garbage/staging/' if is_staging else '')
pre_gradle_commands = (
'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'
.format(sentry_secret, 'dsn', '.sentry_token'), )
gradle_commands = (
'./gradlew --no-daemon -PcrashReportEnabled=true -Ptelemetry=true clean test assembleRelease',
)
command = ' && '.join(cmd for commands in (pre_gradle_commands,
gradle_commands)
for cmd in commands if cmd)
routes = [] if is_staging else [
"[email protected]"
]
return self._craft_build_ish_task(
name='Build task',
description='Build Reference-Browser from source code',
command=command,
scopes=["secrets:get:{}".format(sentry_secret)],
artifacts=artifacts,
routes=routes,
treeherder={
'jobKind': 'build',
'machine': {
'platform': 'android-all',
},
'symbol': 'NA',
'tier': 1,
},
)
def _craft_default_task_definition(
self, worker_type, provisioner_id, dependencies, routes, scopes, name, description,
payload, treeherder=None
):
treeherder = {} if treeherder is None else treeherder
created = datetime.datetime.now()
deadline = taskcluster.fromNow('1 day')
expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN)
return {
"provisionerId": provisioner_id,
"workerType": worker_type,
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"created": taskcluster.stringDate(created),
"deadline": taskcluster.stringDate(deadline),
"expires": taskcluster.stringDate(expires),
"retries": 5,
"tags": {},
"priority": self.tasks_priority,
"dependencies": [self.task_id] + dependencies,
"requires": "all-completed",
"routes": routes + [
"tc-treeherder.v2.fenix.{}".format(self.commit)
],
"scopes": scopes,
"payload": payload,
"extra": {
"treeherder": treeherder,
},
"metadata": {
"name": "Fenix - {}".format(name),
"description": description,
"owner": self.owner,
"source": self.source,
},
}
def beetmover_task(self, name, description, version, artifact_id,
dependencies=[], upstreamArtifacts=[], scopes=[]):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType": "mobile-beetmover-v1",
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"schedulerId": "taskcluster-github",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [self.task_id] + dependencies,
"routes": [],
"scopes": scopes,
"requires": "all-completed",
"payload": {
"maxRunTime": 600,
"upstreamArtifacts": upstreamArtifacts,
"releaseProperties": {
"appName": "components",
},
"version": version,
"artifact_id": artifact_id
},
"provisionerId": "scriptworker-prov-v1",
"metadata": {
"name": name,
"description": description,
"owner": self.owner,
"source": self.source
}
}
def build_push_task(self, signing_task_id, name, description, apks=[], scopes=[], track='internal', commit=False):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType": 'mobile-pushapk-v1',
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [ self.task_id, signing_task_id],
"routes": [],
"scopes": scopes,
"requires": "all-completed",
"payload": {
"commit": commit,
"google_play_track": track,
"upstreamArtifacts": [
{
"paths": apks,
"taskId": signing_task_id,
"taskType": "signing"
}
]
},
"provisionerId": "scriptworker-prov-v1",
"metadata": {
"name": name,
"description": description,
"owner": "*****@*****.**",
"source": "https://github.com/mozilla-mobile/focus-android/tree/master/tools/taskcluster"
}
}
def craft_ui_tests_task(self):
artifacts = {
"public": {
"type": "directory",
"path": "/build/fenix/results",
"expires": taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN))
}
}
env_vars = {
"GOOGLE_PROJECT": "moz-fenix",
"GOOGLE_APPLICATION_CREDENTIALS": ".firebase_token.json"
}
gradle_commands = (
'./gradlew --no-daemon clean assemble assembleAndroidTest',
)
test_commands = (
'automation/taskcluster/androidTest/ui-test.sh arm64-v8a -1',
'automation/taskcluster/androidTest/ui-test.sh armeabi-v7a -1',
)
command = ' && '.join(
cmd
for commands in (gradle_commands, test_commands)
for cmd in commands
if cmd
)
treeherder = {
'jobKind': 'test',
'machine': {
'platform': 'ui-test',
},
'symbol': 'ui-test',
'tier': 2,
}
return self._craft_build_ish_task(
name='Fenix - UI test',
description='Execute Gradle tasks for UI tests',
command=command,
scopes=[
'secrets:get:project/mobile/fenix/firebase'
],
artifacts=artifacts,
env_vars=env_vars,
treeherder=treeherder,
)
def craft_build_task(self,
module_name,
gradle_tasks,
subtitle='',
run_coverage=False,
is_snapshot=False,
component=None,
artifacts=None,
timestamp=None):
taskcluster_artifacts = {}
# component is not None when this is a release build, in which case artifacts is defined too
if component is not None:
taskcluster_artifacts = {
artifact['taskcluster_path']: {
'type':
'file',
'expires':
taskcluster.stringDate(
taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
'path':
artifact['build_fs_path'],
}
for artifact in artifacts
}
scopes = [
"secrets:get:project/mobile/android-components/public-tokens"
] if run_coverage else []
snapshot_flag = '-Psnapshot -Ptimestamp={} '.format(
timestamp) if is_snapshot else ''
coverage_flag = '-Pcoverage ' if run_coverage else ''
gradle_command = ('./gradlew --no-daemon clean ' + coverage_flag +
snapshot_flag + gradle_tasks)
post_gradle_command = 'automation/taskcluster/action/upload_coverage_report.sh' if run_coverage else ''
command = ' && '.join(cmd
for cmd in (gradle_command, post_gradle_command)
if cmd)
return self._craft_build_ish_task(
name='Android Components - Module {} {}'.format(
module_name, subtitle),
description='Execute Gradle tasks for module {}'.format(
module_name),
command=command,
scopes=scopes,
artifacts=taskcluster_artifacts)
def generate_compare_locales_task():
return taskcluster.slugId(), generate_task(
name="(Focus for Android) String validation",
description="Check Focus/Klar for Android for errors in en-US and l10n.",
command=('pip install "compare-locales>=5.0.2,<6.0"'
' && mkdir -p /opt/focus-android/test_artifacts'
' && compare-locales --validate l10n.toml .'
' && compare-locales --json=/opt/focus-android/test_artifacts/data.json l10n.toml .'),
artifacts={
"public": {
"type": "directory",
"path": "/opt/focus-android/test_artifacts",
"expires": taskcluster.stringDate(taskcluster.fromNow('1 week'))
}
})
def _craft_default_task_definition(self, worker_type, provisioner_id,
dependencies, routes, scopes, name,
description, payload):
created = datetime.datetime.now()
deadline = taskcluster.fromNow('1 day')
expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN)
routes.append('checks')
return {
"attributes": {},
"dependencies": dependencies,
"label": name,
"task": {
"provisionerId": provisioner_id,
"workerType": worker_type,
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"created": taskcluster.stringDate(created),
"deadline": taskcluster.stringDate(deadline),
"expires": taskcluster.stringDate(expires),
"retries": 5,
"tags": {},
"priority": self.tasks_priority,
"requires": "all-completed",
"routes": routes,
"scopes": scopes,
"payload": payload,
"metadata": {
"name": name,
"description": description,
"owner": self.owner,
"source": self.source,
},
}
}
def from_template(self, platform, revision, branch, update_number,
chunk_name, subchunk, extra):
"""Reads and populates graph template.
:param platform: buildbot platform (linux, macosx64)
:param locale: en-US, de, ka, etc.
:param from_mar: "from" MAR URL
:param to_mar: "to" MAR URL
:return: graph definition dictionary
"""
template_file = os.path.join(os.path.dirname(__file__), "tasks",
"funsize.yml")
extra_balrog_submitter_params = None
if branch in STAGING_BRANCHES:
extra_balrog_submitter_params = "--dummy"
template_vars = {
# Stable slugId
"stableSlugId": stableSlugId(),
# Now in ISO format
"now": stringDate(datetime.datetime.utcnow()),
# Now in ms
"now_ms": time.time() * 1000,
"fromNow": fromNow,
"platform": platform,
"s3_bucket": self.s3_info["s3_bucket"],
"aws_access_key_id": self.s3_info["aws_access_key_id"],
"aws_secret_access_key": self.s3_info["aws_secret_access_key"],
"balrog_api_root": self.balrog_worker_api_root,
"balrog_username": self.balrog_client.auth[0],
"balrog_password": self.balrog_client.auth[1],
"encryptEnvVar": encryptEnvVar_wrapper,
"revision": revision,
"branch": branch,
"treeherder_platform": buildbot_to_treeherder(platform),
"revision_hash": revision_to_revision_hash(self.th_api_root,
branch, revision),
"update_number": update_number,
"extra_balrog_submitter_params": extra_balrog_submitter_params,
"extra": extra,
"chunk_name": chunk_name,
"subchunk": subchunk,
"sign_task": partial(sign_task, pvt_key=self.pvt_key),
}
with open(template_file) as f:
template = Template(f.read(), undefined=StrictUndefined)
rendered = template.render(**template_vars)
return yaml.safe_load(rendered)
def generate_ui_test_task(dependencies):
return taskcluster.slugId(), generate_task(
name = "(Focus for Android) UI tests",
description = "Run UI tests for Focus/Klar for Android.",
command = ('echo "--" > .adjust_token'
' && ./gradlew --no-daemon clean assembleFocusWebviewUniversalDebug assembleFocusWebviewUniversalDebugAndroidTest'
' && tools/taskcluster/execute-firebase-test.sh'),
dependencies = dependencies,
scopes = [ 'secrets:get:project/focus/firebase' ],
artifacts = {
"public": {
"type": "directory",
"path": "/opt/focus-android/test_artifacts",
"expires": taskcluster.stringDate(taskcluster.fromNow('1 week'))
}
})
def generate_gecko_X86_ui_test_task(dependencies):
return taskcluster.slugId(), generate_task(
name = "(Focus for Android) UI tests - Gecko X86",
description = "Run UI tests for Klar Gecko X86 for Android.",
command = ('echo "--" > .adjust_token'
' && ./gradlew --no-daemon clean assembleKlarX86Debug assembleKlarX86DebugAndroidTest'
' && ./tools/taskcluster/google-firebase-testlab-login.sh'
' && tools/taskcluster/execute-firebase-test.sh klarX86 app-klar-x86-debug model=Nexus9,version=25'),
dependencies = dependencies,
scopes = [ 'secrets:get:project/focus/firebase' ],
artifacts = {
"public": {
"type": "directory",
"path": "/opt/focus-android/test_artifacts",
"expires": taskcluster.stringDate(taskcluster.fromNow('1 week'))
}
})
def generate_ui_test_task(dependencies, engine="Klar", device="arm"):
'''
:param str engine: Klar, Webview
:param str device: ARM, X86
:return: uiWebviewARMTestTaskId, uiWebviewARMTestTask
'''
if engine is "Klar":
engine = "geckoview"
assemble_engine = engine
elif engine is "Webview":
engine = "webview"
assemble_engine = "Focus"
else:
raise Exception("ERROR: unknown engine type --> Aborting!")
task_name = "(Focus for Android) UI tests - {0} {1}".format(engine, device)
task_description = "Run UI tests for {0} build for Android.".format(
engine, device)
build_dir = "assemble{0}{1}Debug".format(assemble_engine,
device.capitalize())
build_dir_test = "assemble{0}{1}DebugAndroidTest".format(
assemble_engine, device.capitalize())
print('BUILD_DIR: {0}'.format(build_dir))
print('BUILD_DIR_TEST: {0}'.format(build_dir_test))
device = device.lower()
return taskcluster.slugId(), generate_task(
name=task_name,
description=task_description,
command=('echo "--" > .adjust_token'
' && ./gradlew --no-daemon clean ' + build_dir + ' ' +
build_dir_test + ' '
' && ./tools/taskcluster/google-firebase-testlab-login.sh'
' && tools/taskcluster/execute-firebase-tests.sh ' + device +
' ' + engine),
dependencies=dependencies,
scopes=['secrets:get:project/focus/firebase'],
routes=['notify.irc-channel.#android-ci.on-any'],
artifacts={
"public": {
"type": "directory",
"path": "/opt/focus-android/test_artifacts",
"expires":
taskcluster.stringDate(taskcluster.fromNow('1 week'))
}
})
def generate_webview_ARM_ui_test_task(dependencies):
return taskcluster.slugId(), generate_task(
name = "(Focus for Android) UI tests - Webview ARM",
description = "Run UI tests for Focus/Klar for Android.",
command = ('echo "--" > .adjust_token'
' && ./gradlew --no-daemon clean assembleFocusArmDebug assembleFocusArmDebugAndroidTest'
' && ./tools/taskcluster/google-firebase-testlab-login.sh'
' && tools/taskcluster/execute-firebase-test.sh focusArm app-focus-arm-debug model=walleye,version=26 model=shamu,version=23'),
dependencies = dependencies,
scopes = [ 'secrets:get:project/focus/firebase' ],
artifacts = {
"public": {
"type": "directory",
"path": "/opt/focus-android/test_artifacts",
"expires": taskcluster.stringDate(taskcluster.fromNow('1 week'))
}
})
def generate_demo_test_task():
slug_id = taskcluster.slugId()
task_json = generate_task(
name = "(Demo Taskcluster) Dummy tests",
description = "Run a demo for the heck of it.",
command = ('./tools/demo.sh'),
routes=['notify.irc-channel.#demo-ci.on-any',
'/repository/kglazko/demo-taskcluster/issues/1/comments'],
#dependencies = dependencies,
scopes = ['github:create-comment:kglazko/*',
'queue:route:notify.irc-channel.*'],
artifacts = {
"public": {
"type": "directory",
"path": "/opt/demo-taskcluster/test_artifacts",
"expires": taskcluster.stringDate(taskcluster.fromNow('1 week'))
}
})
print(task_json)
post_github_comment('1', payload_source)
return slug_id, task_json
def generate_release_task(dependencies):
return taskcluster.slugId(), generate_task(
name = "(Focus for Android) Preview release",
description = "Build preview versions for testing Focus/Klar for Android.",
command = ('echo "--" > .adjust_token'
' && ./gradlew --no-daemon clean assembleBeta'
' && python tools/taskcluster/sign-preview-builds.py'
' && touch /opt/focus-android/builds/`date +"%Y-%m-%d-%H-%M"`'
' && touch /opt/focus-android/builds/' + COMMIT),
dependencies = dependencies,
scopes = [
"secrets:get:project/focus/preview-key-store",
"queue:route:index.project.focus.android.preview-builds"],
routes = [ "index.project.focus.android.preview-builds" ],
artifacts = {
"public": {
"type": "directory",
"path": "/opt/focus-android/builds",
"expires": taskcluster.stringDate(taskcluster.fromNow('1 month'))
}
})
def craft_ui_tests_task(self):
artifacts = {
"public": {
"type": "directory",
"path": "/build/reference-browser/results",
"expires": taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN))
}
}
env_vars = {
"GOOGLE_PROJECT": "moz-reference-browser-230023",
"GOOGLE_APPLICATION_CREDENTIALS": ".firebase_token.json"
}
gradle_commands = (
'./gradlew --no-daemon clean assembleDebug assembleAndroidTest',
)
test_commands = (
'automation/taskcluster/androidTest/ui-test.sh aarch64 -1',
'automation/taskcluster/androidTest/ui-test.sh arm -1',
)
command = ' && '.join(
cmd
for commands in (gradle_commands, test_commands)
for cmd in commands
if cmd
)
return self._craft_build_ish_task(
name='UI tests',
description='Execute Gradle tasks for UI tests',
command=command,
scopes=[
'secrets:get:project/mobile/reference-browser/firebase'
],
artifacts=artifacts,
env_vars=env_vars,
)
def generate_massager_task(build_task_id, massager_task_id, artifacts_info,
version):
command = (
"git fetch origin --tags && "
"git config advice.detachedHead false && "
"git checkout {} && "
"apt-get install -y python3-pip && "
"pip3 install scriptworker && "
"python3 automation/taskcluster/release/convert_group_and_artifact_ids.py {}"
.format(version, massager_task_id))
scopes = []
upstreamZip = []
for artifact, info in artifacts_info.items():
new_artifact_id = None if info['name'] == info[
'old_artifact_id'] else info['name']
new_group_id = info['new_group_id'] if new_artifact_id else None
upstreamZip.append({
"path": artifact,
"taskId": build_task_id,
"newGroupId": new_group_id,
"newArtifactId": new_artifact_id,
})
return BUILDER.massager_task(
name="Android Components - Massager",
description="Task to massage the group/artifact ids",
dependencies=[build_task_id],
command=command,
scopes=scopes,
features={"chainOfTrust": True},
artifacts={
"public/build": {
"expires":
taskcluster.stringDate(taskcluster.fromNow('1 year')),
"path": "/build/android-components/work_dir/public/build",
"type": "directory"
}
},
upstreamZip=upstreamZip,
)
def generate_build_task(apks, is_staging):
artifacts = {
'public/{}'.format(os.path.basename(apk)): {
"type": 'file',
"path": apk,
"expires": taskcluster.stringDate(taskcluster.fromNow('1 year')),
}
for apk in apks
}
checkout = ("export TERM=dumb && git fetch {} {} --tags && "
"git config advice.detachedHead false && "
"git checkout {}".format(GITHUB_HTTP_REPOSITORY, HEAD_BRANCH,
HEAD_REV))
sentry_secret = '{}project/mobile/fenix/sentry'.format(
'garbage/staging/' if is_staging else '')
leanplum_secret = '{}project/mobile/fenix/leanplum'.format(
'garbage/staging/' if is_staging else '')
return taskcluster.slugId(), BUILDER.build_task(
name="(Fenix) Build task",
description="Build Fenix from source code.",
command=
(checkout + ' && python automation/taskcluster/helper/get-secret.py'
' -s {} -k dsn -f .sentry_token'.format(sentry_secret) +
' && python automation/taskcluster/helper/get-secret.py'
' -s {} -k production -f .leanplum_token'.format(leanplum_secret) +
' && ./gradlew --no-daemon -PcrashReports=true clean test assembleRelease'
),
features={
"chainOfTrust": True,
"taskclusterProxy": True
},
artifacts=artifacts,
scopes=[
"secrets:get:{}".format(sentry_secret),
"secrets:get:{}".format(leanplum_secret)
],
routes=["[email protected]"])
def generate_ui_test_task(dependencies):
task_name = "(Focus for Android) UI tests"
task_description = "Run UI tests for Focus for Android."
build_dir = "assembleFocusDebug"
build_dir_test = "assembleFocusDebugAndroidTest"
return taskcluster.slugId(), generate_task(
name=task_name,
description=task_description,
command=('echo "--" > .adjust_token'
' && ./gradlew --no-daemon clean ' + build_dir + ' ' + build_dir_test + ' '
' && ./tools/taskcluster/google-firebase-testlab-login.sh'
' && ./tools/taskcluster/execute-firebase-tests.sh'),
dependencies=dependencies,
scopes=['secrets:get:project/focus/firebase'],
routes=['notify.irc-channel.#android-ci.on-any'],
artifacts={
"public": {
"type": "directory",
"path": "/opt/focus-android/test_artifacts",
"expires": taskcluster.stringDate(taskcluster.fromNow('1 week'))
}
})
def generate_build_task(apks, tag, is_staging):
artifacts = {}
for apk in apks:
artifact = {
"type": 'file',
"path": apk,
"expires": taskcluster.stringDate(taskcluster.fromNow('1 year'))
}
artifacts["public/%s" % os.path.basename(apk)] = artifact
checkout = "cd .. && git clone {} repository && cd repository".format(
GITHUB_HTTP_REPOSITORY)
if tag is not None:
checkout += " && git checkout {}".format(tag)
assemble_task = 'assembleNightly'
if tag:
# Non-tagged (nightly) builds should contain all languages
checkout = checkout + ' && python tools/l10n/filter-release-translations.py'
assemble_task = 'assembleRelease'
return taskcluster.slugId(), BUILDER.build_task(
name="(Focus for Android) Build task",
description="Build Focus/Klar from source code.",
command=(checkout +
' && python tools/taskcluster/get-adjust-token.py {}'.format(
'--staging' if is_staging else '') +
' && python tools/taskcluster/get-sentry-token.py {}'.format(
'--staging' if is_staging else '') +
' && ./gradlew --no-daemon clean test ' + assemble_task),
features={"chainOfTrust": True},
artifacts=artifacts,
worker_type='gecko-focus',
scopes=[
"secrets:get:garbage/staging/project/focus/tokens"
if is_staging else "secrets:get:project/focus/tokens"
])
def generate_build_task(apks, tag):
artifacts = {}
for apk in apks:
artifact = {
"type": 'file',
"path": apk,
"expires": taskcluster.stringDate(taskcluster.fromNow('1 year'))
}
artifacts["public/%s" % os.path.basename(apk)] = artifact
checkout = "git fetch origin && git reset --hard origin/master" if tag is None else "git fetch origin && git checkout %s" % (tag)
checkout = checkout + ' && tools/taskcluster/accept-license.sh'
assemble_task = 'assembleNightly'
if tag:
# Non-tagged (nightly) builds should contain all languages
checkout = checkout + ' && python tools/l10n/filter-release-translations.py'
assemble_task = 'assembleRelease'
return taskcluster.slugId(), BUILDER.build_task(
name="(Focus for Android) Build task",
description="Build Focus/Klar from source code.",
command=(checkout +
' && python tools/taskcluster/get-adjust-token.py'
' && python tools/taskcluster/get-sentry-token.py'
' && ./gradlew --no-daemon clean test ' + assemble_task),
features = {
"chainOfTrust": True
},
artifacts = artifacts,
worker_type='gecko-focus',
scopes=[
"secrets:get:project/focus/tokens"
])
def retrigger_task(task_id, dry_run=False):
""" Given a task id (our uuid) we query it and build
a new task based on the old one which we schedule on TaskCluster.
We don't call the rerun API since we can't rerun a task past
its deadline, instead we create a new task with a new taskGroupId,
expiration, creation and deadline values.
task_id (int) - ID that identifies a task on Taskcluster
dry_run (bool) - Default to False. If True, it won't trigger
a task.
returns - 0 for dry_run case, -1 for any failure or the task id (int)
of a succesful retriggered task.
http://docs.taskcluster.net/queue/api-docs/#createTask
"""
one_year = 365
new_task_id = 0
try:
queue = taskcluster_client.Queue()
task = queue.task(task_id)
LOG.debug("Original task: (Limit 1024 char)")
LOG.debug(str(json.dumps(task))[:1024])
new_task_id = taskcluster_client.slugId()
artifacts = task['payload'].get('artifacts', {})
for artifact, definition in artifacts.iteritems():
definition['expires'] = taskcluster_client.fromNow('%s days' % one_year)
# The task group will be identified by the ID of the only
# task in the group
task['taskGroupId'] = new_task_id
# https://bugzilla.mozilla.org/show_bug.cgi?id=1190660
# TC workers create public logs which are 365 days; if the task expiration
# date is the same or less than that we won't have logs for the task
task['expires'] = taskcluster_client.fromNow('%s days' % (one_year + 1))
now = datetime.datetime.utcnow()
tomorrow = now + datetime.timedelta(hours=24)
task['created'] = taskcluster_client.stringDate(now)
task['deadline'] = taskcluster_client.stringDate(tomorrow)
LOG.debug("Contents of new task: (Limit 1024 char)")
LOG.debug(str(task)[:1024])
if not dry_run:
LOG.info("Attempting to schedule new task with task_id: {}".format(new_task_id))
result = queue.createTask(new_task_id, task)
LOG.debug(json.dumps(result))
LOG.info("{}/task-inspector/#{}".format(TASKCLUSTER_TOOLS_HOST, new_task_id))
else:
LOG.info("Dry-run mode: Nothing was retriggered.")
except taskcluster_client.exceptions.TaskclusterRestFailure as e:
traceback.print_exc()
new_task_id = -1
except taskcluster_client.exceptions.TaskclusterAuthFailure as e:
# Hack until we fix it in the issue
if str(e) == "Authorization Failed":
LOG.error("The taskclaster client that you specified is lacking "
"the right set of scopes.")
LOG.error("Run this same command with --debug and you will see "
"the missing scopes (the output comes from the "
"taskcluster python client)")
elif str(e) == "Authentication Error":
LOG.error("Make sure that you create permanent credentials and you "
"set these environment variables: TASKCLUSTER_CLIENT_ID & "
"TASKCLUSTER_ACCESS_TOKEN")
new_task_id = -1
return new_task_id
def _craft_raptor_task(
self,
signing_task_id,
mozharness_task_id,
variant,
gecko_revision,
name_prefix,
description,
test_name,
job_symbol,
group_symbol=None,
extra_test_args=None,
force_run_on_64_bit_device=False,
):
extra_test_args = [] if extra_test_args is None else extra_test_args
apk_location = '{}/{}/artifacts/{}'.format(
_DEFAULT_TASK_URL, signing_task_id, DEFAULT_APK_ARTIFACT_LOCATION)
worker_type = 'gecko-t-bitbar-gw-perf-p2' if force_run_on_64_bit_device or variant.abi == 'aarch64' else 'gecko-t-bitbar-gw-perf-g5'
if force_run_on_64_bit_device:
treeherder_platform = 'android-hw-p2-8-0-arm7-api-16'
elif variant.abi == 'arm':
treeherder_platform = 'android-hw-g5-7-0-arm7-api-16'
elif variant.abi == 'aarch64':
treeherder_platform = 'android-hw-p2-8-0-android-aarch64'
else:
raise ValueError('Unsupported architecture "{}"'.format(
variant.abi))
task_name = '{}: {} {}'.format(
name_prefix, variant.raw,
'(on 64-bit-device)' if force_run_on_64_bit_device else '')
apk_url = '{}/{}/artifacts/{}'.format(_DEFAULT_TASK_URL,
signing_task_id,
DEFAULT_APK_ARTIFACT_LOCATION)
return self._craft_default_task_definition(
worker_type=worker_type,
provisioner_id='proj-autophone',
dependencies=[signing_task_id],
name=task_name,
description=description,
payload={
"maxRunTime":
2700,
"artifacts": [{
'path':
worker_path,
'expires':
taskcluster.stringDate(
taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
'type':
'directory',
'name':
'public/{}/'.format(public_folder)
} for worker_path, public_folder in (
('artifacts/public', 'test'),
('workspace/logs', 'logs'),
('workspace/build/blobber_upload_dir', 'test_info'),
)],
"command": [[
"/builds/taskcluster/script.py", "bash", "./test-linux.sh",
"--cfg=mozharness/configs/raptor/android_hw_config.py",
"--test={}".format(test_name), "--app=refbrow",
"--binary=org.mozilla.reference.browser.raptor",
"--activity=org.mozilla.reference.browser.GeckoViewActivity",
"--download-symbols=ondemand"
] + extra_test_args],
"env": {
"EXTRA_MOZHARNESS_CONFIG":
json.dumps({
"test_packages_url":
"{}/{}/artifacts/public/build/en-US/target.test_packages.json"
.format(_DEFAULT_TASK_URL, mozharness_task_id),
"installer_url":
apk_url,
}),
"GECKO_HEAD_REPOSITORY":
"https://hg.mozilla.org/mozilla-central",
"GECKO_HEAD_REV":
gecko_revision,
"MOZ_AUTOMATION":
"1",
"MOZ_HIDE_RESULTS_TABLE":
"1",
"MOZ_NO_REMOTE":
"1",
"MOZ_NODE_PATH":
"/usr/local/bin/node",
"MOZHARNESS_CONFIG":
"raptor/android_hw_config.py",
"MOZHARNESS_SCRIPT":
"raptor_script.py",
"MOZHARNESS_URL":
"{}/{}/artifacts/public/build/en-US/mozharness.zip".format(
_DEFAULT_TASK_URL, mozharness_task_id),
"MOZILLA_BUILD_URL":
apk_location,
"NEED_XVFB":
"false",
"NO_FAIL_ON_TEST_ERRORS":
"1",
"SCCACHE_DISABLE":
"1",
"TASKCLUSTER_WORKER_TYPE":
worker_type[len('gecko-'):],
"XPCOM_DEBUG_BREAK":
"warn",
},
"mounts": [{
"content": {
"url":
"https://hg.mozilla.org/mozilla-central/raw-file/{}/taskcluster/scripts/tester/test-linux.sh"
.format(gecko_revision),
},
"file": "test-linux.sh",
}]
},
treeherder={
'jobKind': 'test',
'groupSymbol': 'Rap' if group_symbol is None else group_symbol,
'machine': {
'platform': treeherder_platform,
},
'symbol': job_symbol,
'tier': 2,
})
def from_now_json(self, offset):
"""
Same as `taskcluster.fromNowJSON`, but uses the creation time of `self` for “now”.
"""
return taskcluster.stringDate(taskcluster.fromNow(offset, dateObj=self.now))
def from_now_json(self, offset):
"""
Same as `taskcluster.fromNowJSON`, but uses the creation time of `self` for “now”.
"""
return taskcluster.stringDate(
taskcluster.fromNow(offset, dateObj=self.now))
"iat": iat,
"exp": exp,
"taskId": task_id,
"version": "1",
}
return jws.sign(claims, pvt_key, algorithm=algorithm)
config = json.load(open("secrets.json"))
config["credentials"]["certificate"] = json.dumps(config["credentials"]["certificate"])
scheduler = taskcluster.Scheduler(config)
# funsize scheduler key
pvt_key = open("id_rsa").read()
template_vars = {
"stableSlugId": stableSlugId(),
"now": stringDate(datetime.datetime.utcnow()),
"now_ms": time.time() * 1000,
"fromNow": fromNow,
"sign_task": partial(sign_task, pvt_key=pvt_key),
### TODO: change these
"url": "http://people.mozilla.org/~raliiev/bouncer.apk",
"filename": "bouncer.apk",
"signign_format": "jar",
"hash": "8f4210c62cf533322b09237a3741bc3e9bb52582b8d0b88c52a0d78a6eabe08e29b636d5c9668e8db721c9dead36736db643c53231e966c86dbc28d86b9eb699",
}
template_file = os.path.join(os.path.dirname(__file__), "graph.yml")
with open(template_file) as f:
template = Template(f.read(), undefined=StrictUndefined)
rendered = template.render(**template_vars)
graph = yaml.safe_load(rendered)
