def refresh_timestamps(task):
''' It refreshes the timestamps of the task. '''
# XXX split this function
LOG.debug("Updating timestamps of task.")
LOG.debug("Original task: (Limit 1024 char)")
LOG.debug(str(json.dumps(task))[:1024])
artifacts = task['payload'].get('artifacts', {})
for artifact, definition in artifacts.iteritems():
definition['expires'] = taskcluster_client.fromNow('%s days' % 365)
# https://bugzilla.mozilla.org/show_bug.cgi?id=1190660
# TC workers create public logs which are 365 days; if the task expiration
# date is the same or less than that we won't have logs for the task
task['expires'] = taskcluster_client.fromNow('%s days' % (365 + 1))
now = datetime.datetime.utcnow()
tomorrow = now + datetime.timedelta(hours=24)
task['created'] = taskcluster_client.stringDate(now)
task['deadline'] = taskcluster_client.stringDate(tomorrow)
LOG.debug("Contents of new task: (Limit 1024 char)")
LOG.debug(str(task)[:1024])
return task
def _recreate_task(task_id):
one_year = 365
queue = taskcluster_client.Queue()
task = queue.task(task_id)
LOG.debug("Original task: (Limit 1024 char)")
LOG.debug(str(json.dumps(task))[:1024])
# Start updating the task
task["taskId"] = taskcluster_client.slugId()
artifacts = task["payload"].get("artifacts", {})
for artifact, definition in artifacts.iteritems():
definition["expires"] = taskcluster_client.fromNow("%s days" % one_year)
# https://bugzilla.mozilla.org/show_bug.cgi?id=1190660
# TC workers create public logs which are 365 days; if the task expiration
# date is the same or less than that we won't have logs for the task
task["expires"] = taskcluster_client.fromNow("%s days" % (one_year + 1))
now = datetime.datetime.utcnow()
tomorrow = now + datetime.timedelta(hours=24)
task["created"] = taskcluster_client.stringDate(now)
task["deadline"] = taskcluster_client.stringDate(tomorrow)
LOG.debug("Contents of new task: (Limit 1024 char)")
LOG.debug(str(task)[:1024])
return task
def _craft_default_task_definition(
self,
worker_type,
provisioner_id,
name,
description,
payload,
dependencies=None,
routes=None,
scopes=None,
treeherder=None,
):
dependencies = [] if dependencies is None else dependencies
scopes = [] if scopes is None else scopes
routes = [] if routes is None else routes
treeherder = {} if treeherder is None else treeherder
created = datetime.datetime.now()
deadline = taskcluster.fromNow('1 day')
expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN)
if self.trust_level == 3:
routes.append("tc-treeherder.v2.reference-browser.{}".format(
self.commit))
return {
"provisionerId": provisioner_id,
"workerType": worker_type,
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"created": taskcluster.stringDate(created),
"deadline": taskcluster.stringDate(deadline),
"expires": taskcluster.stringDate(expires),
"retries": 5,
"tags": {},
"priority": self.tasks_priority,
"dependencies": [self.task_id] + dependencies,
"requires": "all-completed",
"routes": routes,
"scopes": scopes,
"payload": payload,
"extra": {
"treeherder": treeherder,
},
"metadata": {
"name": "Reference-Browser - {}".format(name),
"description": description,
"owner": self.owner,
"source": self.source,
},
}
def generate_task(name,
description,
command,
dependencies=[],
artifacts={},
scopes=[],
routes=[]):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 month')
deadline = taskcluster.fromNow('1 day')
return {
"workerType": "b-linux",
"taskGroupId": TASK_ID,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"schedulerId": "taskcluster-github",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [TASK_ID] + dependencies,
"routes": routes,
"scopes": scopes,
"requires": "all-completed",
"payload": {
"features": {
"taskclusterProxy": True
},
"maxRunTime":
7200,
"image":
"mozillamobile/focus-android:1.6",
"command": [
"/bin/bash", "--login", "-c",
"git fetch %s %s && git config advice.detachedHead false && git checkout %s && %s"
% (REPO_URL, BRANCH, COMMIT, command)
],
"artifacts":
artifacts,
"deadline":
taskcluster.stringDate(deadline)
},
"provisionerId": "mobile-3",
"metadata": {
"name": name,
"description": description,
"owner": OWNER,
"source": SOURCE
}
}
def create_fxaclient_task():
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType": 'github-worker',
"taskGroupId": TASK_ID,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"schedulerId": "taskcluster-github",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [TASK_ID],
"routes": [],
"scopes": [],
"requires": "all-completed",
"payload": {
"features": {},
"maxRunTime":
7200,
"image":
"mozillamobile/rust-component:buildtools-27.0.3-ndk-r15c-ndk-version-21-rust-stable-rust-beta",
"command": [
"/bin/bash", "--login", "-cx",
"export TERM=dumb && git clone %s && cd application-services && git fetch %s %s && git config advice.detachedHead false && git checkout %s && ./scripts/taskcluster-android.sh"
% (REPO_URL, REPO_URL, BRANCH, COMMIT)
],
"artifacts": {
"public/bin/mozilla/fxa_client_android.zip": {
"type":
"file",
"path":
"/build/application-services/fxa-client/fxa_client_android.zip",
},
},
"deadline":
taskcluster.stringDate(deadline)
},
"provisionerId": "aws-provisioner-v1",
"metadata": {
"name": "application-services - FxA client library",
"description":
"Building FxA client Rust library and native code dependencies",
"owner": "*****@*****.**",
"source": "https://github.com/mozilla/application-services"
}
}
def build_push_task(self,
signing_task_id,
name,
description,
apks=[],
scopes=[],
channel='internal',
commit=False):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType": 'mobile-3-pushapk',
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [self.task_id, signing_task_id],
"routes": [],
"scopes": scopes,
"requires": "all-completed",
"payload": {
"commit":
commit,
"channel":
channel,
"upstreamArtifacts": [{
"paths": apks,
"taskId": signing_task_id,
"taskType": "signing"
}]
},
"provisionerId": "scriptworker-k8s",
"metadata": {
"name":
name,
"description":
description,
"owner":
"*****@*****.**",
"source":
"https://github.com/mozilla-mobile/focus-android/tree/master/tools/taskcluster"
}
}
def massager_task(self,
name,
description,
command,
dependencies=[],
scopes=[],
features={},
artifacts={},
upstreamZip=[]):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType": "gecko-focus",
"taskGroupId": self.task_id,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"schedulerId": "taskcluster-github",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [self.task_id] + dependencies,
"routes": [],
"scopes": scopes,
"requires": "all-completed",
"payload": {
"maxRunTime": 3600,
"features": features,
"image": "mozillamobile/android-components:1.4",
"command": [
"/bin/bash",
"--login",
"-cx",
command,
],
"artifacts": artifacts,
"upstreamZip": upstreamZip,
"deadline": taskcluster.stringDate(deadline)
},
"provisionerId": "aws-provisioner-v1",
"metadata": {
"name": name,
"description": description,
"owner": self.owner,
"source": self.source
}
}
def craft_assemble_release_task(self, architectures, channel, is_staging,
version_name):
artifacts = {
'public/target.{}.apk'.format(arch): {
"type":
'file',
"path":
'/opt/fenix/app/build/outputs/apk/'
'{arch}/{channel}/app-{arch}-{channel}-unsigned.apk'.format(
arch=arch, channel=channel),
"expires":
taskcluster.stringDate(
taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
}
for arch in architectures
}
if is_staging:
secret_index = 'garbage/staging/project/mobile/fenix'
else:
secret_index = 'project/mobile/fenix/{}'.format(channel)
pre_gradle_commands = (
'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'
.format(secret_index, key, target_file) for key, target_file in (
('sentry_dsn', '.sentry_token'),
('leanplum', '.leanplum_token'),
('adjust', '.adjust_token'),
))
capitalized_channel = upper_case_first_letter(channel)
gradle_commands = (
'./gradlew --no-daemon -PversionName={} clean test assemble{}'.
format(version_name, capitalized_channel), )
command = ' && '.join(cmd for commands in (pre_gradle_commands,
gradle_commands)
for cmd in commands if cmd)
routes = [] if is_staging else [
"[email protected]"
]
return self._craft_build_ish_task(
name='Build {} task'.format(capitalized_channel),
description='Build Fenix {} from source code'.format(
capitalized_channel),
command=command,
scopes=["secrets:get:{}".format(secret_index)],
artifacts=artifacts,
routes=routes,
treeherder={
'jobKind': 'build',
'machine': {
'platform': 'android-all',
},
'symbol': '{}-A'.format(channel),
'tier': 1,
},
)
def _craft_shell_task(self, name, script, scopes, artifacts):
single_command = ' && '.join([line.strip() for line in script.split('\n') if line.strip()])
bash_command = [
'/bin/bash',
'--login',
'-cx',
f'export TERM=dumb && {single_command}'
]
return {
'taskGroupId': self.task_group_id,
'provisionerId': 'aws-provisioner-v1',
'schedulerId': 'taskcluster-github',
'workerType': 'github-worker',
'created': taskcluster.stringDate(datetime.datetime.now()),
'deadline': taskcluster.stringDate(taskcluster.fromNow('1 day')),
'scopes': scopes,
'payload': {
'maxRunTime': 3600,
'image': 'mozillamobile/firefox-tv:2.3',
'command': bash_command,
'artifacts': artifacts,
'features': {
'taskclusterProxy': True,
}
},
'metadata': {
'name': name,
'description': '',
'owner': self.owner,
'source': f'{self.repo_url}/raw/{self.commit}/.taskcluster.yml',
}
}
def generate_build_task(apks, is_staging):
artifacts = {
'public/{}'.format(os.path.basename(apk)): {
"type": 'file',
"path": apk,
"expires": taskcluster.stringDate(taskcluster.fromNow('1 year')),
}
for apk in apks
}
checkout = ("export TERM=dumb && git fetch {} {} --tags && "
"git config advice.detachedHead false && "
"git checkout {}".format(GITHUB_HTTP_REPOSITORY, HEAD_BRANCH,
HEAD_REV))
sentry_secret = '{}project/mobile/fenix/sentry'.format(
'garbage/staging/' if is_staging else '')
return taskcluster.slugId(), BUILDER.build_task(
name="(Fenix) Build task",
description="Build Fenix from source code.",
command=
(checkout + ' && python automation/taskcluster/helper/get-secret.py'
' -s {} -k dsn -f .sentry_token'.format(sentry_secret) +
' && ./gradlew --no-daemon -PcrashReports=true clean test assembleRelease'
),
features={
"chainOfTrust": True,
"taskclusterProxy": True
},
artifacts=artifacts,
scopes=["secrets:get:{}".format(sentry_secret)])
def generate_ui_test_task(dependencies):
task_name = "(Focus for Android) UI tests"
task_description = "Run UI tests for Focus for Android."
build_dir = "assembleFocusDebug"
build_dir_test = "assembleFocusDebugAndroidTest"
return taskcluster.slugId(), generate_task(
name=task_name,
description=task_description,
command=('echo "--" > .adjust_token'
' && ./gradlew --no-daemon clean ' + build_dir + ' ' +
build_dir_test + ' '
' && ./tools/taskcluster/google-firebase-testlab-login.sh'
' && ./tools/taskcluster/execute-firebase-tests.sh'),
dependencies=dependencies,
scopes=['secrets:get:project/focus/firebase'],
routes=['notify.irc-channel.#android-ci.on-any'],
artifacts={
"public": {
"type": "directory",
"path": "/opt/focus-android/test_artifacts",
"expires":
taskcluster.stringDate(taskcluster.fromNow('1 week'))
}
})
def generate_build_task(apks):
artifacts = {
'public/{}'.format(os.path.basename(apk)): {
"type": 'file',
"path": "/build/reference-browser/{}".format(apk),
"expires": taskcluster.stringDate(taskcluster.fromNow('1 year')),
}
for apk in apks
}
checkout = 'git clone {} && cd reference-browser && git checkout {}'.format(
GITHUB_HTTP_REPOSITORY, HEAD_REV)
return taskcluster.slugId(), BUILDER.build_task(
name="(Reference Browser) Build task",
description="Build Reference Browser from source code.",
command=
('cd .. && ' + checkout +
' && python automation/taskcluster/helper/get-secret.py'
' -s project/mobile/reference-browser/sentry -k dsn -f .sentry_token'
' && ./gradlew --no-daemon -PcrashReportEnabled=true -Ptelemetry=true clean test assembleRelease'
),
features={
"chainOfTrust": True,
"taskclusterProxy": True
},
artifacts=artifacts,
scopes=["secrets:get:project/mobile/reference-browser/sentry"])
def generate_build_task(info):
checkout = ("export TERM=dumb && git fetch {} {} && "
"git config advice.detachedHead false && "
"git checkout {} ".format(REPO_URL, BRANCH, HEAD_REV))
artifacts = {
info["artifact"]: {
'type': 'file',
'expires': taskcluster.stringDate(taskcluster.fromNow('1 year')),
'path': info['path']
}
}
module_name = ":{}".format(info['name'])
return taskcluster.slugId(), BUILDER.build_task(
name='Android Components - Module {}'.format(module_name),
description='Building and testing module {}'.format(module_name),
command=(checkout + ' && ./gradlew --no-daemon clean ' +
"-Psnapshot " + " ".join([
"{}:{}".format(module_name, gradle_task)
for gradle_task in ['assemble', 'test', 'lint']
]) + " uploadArchives zipMavenArtifacts"),
features={
"chainOfTrust": True,
"taskclusterProxy": True
},
worker_type='gecko-focus',
scopes=[],
artifacts=artifacts,
)
def generate_release_task(uiTestTaskId):
return taskcluster.slugId(), generate_task(
name="(Focus for Android) Preview release",
description=
"Build preview versions for testing Focus/Klar for Android.",
command=(
'echo "--" > .adjust_token'
' && ./gradlew clean assembleBeta'
' && python tools/taskcluster/sign-preview-builds.py'
' && touch /opt/focus-android/builds/`date +"%%Y-%%m-%%d-%%-H-%%M"`'
' && touch /opt/focus-android/builds/' + COMMIT),
dependencies=[uiTestTaskId],
scopes=[
"secrets:get:project/focus/preview-key-store",
"queue:route:index.project.focus.android.preview-builds"
],
routes=["index.project.focus.android.preview-builds"],
artifacts={
"public": {
"type": "directory",
"path": "/opt/focus-android/builds",
"expires":
taskcluster.stringDate(taskcluster.fromNow('1 week'))
}
})
def test_putUrl_upload(randbytes):
"""Test that upload works with a custom sync reader factory."""
data = randbytes(10240) # >8k to avoid using dataInline
uploaded_data = b''
class Server(httptest.Handler):
def do_PUT(self):
nonlocal uploaded_data
uploaded_data = self.rfile.read(len(data))
self.send_response(200)
self.end_headers()
def readerFactory():
return io.BytesIO(data)
with httptest.Server(Server) as ts:
objectService = FakeObject(ts)
upload.upload(
projectId="taskcluster",
expires=taskcluster.fromNow('1 hour'),
contentType="text/plain",
contentLength=len(data),
name="some/object",
readerFactory=readerFactory,
objectService=objectService)
assert uploaded_data == data
async def test_file_upload_download(objectService, tmp_path):
src = tmp_path / "src"
dest = tmp_path / "dest"
data = secrets.token_bytes(102400)
with open(src, "wb") as f:
f.write(data)
name = f"taskcluster/test/client-py/{taskcluster.slugid.v4()}"
with open(src, "rb") as file:
await upload.uploadFromFile(
projectId="taskcluster",
name=name,
contentType="text/plain",
contentLength=len(data),
expires=taskcluster.fromNow('1 hour'),
file=file,
objectService=objectService)
with open(dest, "wb") as file:
contentType = await download.downloadToFile(
name=name,
file=file,
objectService=objectService)
with open(dest, "rb") as f:
got = f.read()
assert got == data
assert contentType == 'text/plain'
def craft_dependencies_task(self):
# Output the dependencies to an artifact. This is used by the
# telemetry probe scraper to determine all of the metrics that
# Fenix might send (both from itself and any of its dependent
# libraries that use Glean).
return self._craft_clean_gradle_task(
name='dependencies',
description='Write dependencies to a build artifact',
gradle_task=
'app:dependencies --configuration implementation > dependencies.txt',
treeherder={
'jobKind': 'test',
'machine': {
'platform': 'lint',
},
'symbol': 'dependencies',
'tier': 1,
},
routes=[
'index.project.mobile.fenix.v2.branch.master.revision.{}'.
format(self.commit)
],
artifacts={
'public/dependencies.txt': {
"type":
'file',
"path":
'/opt/fenix/dependencies.txt',
"expires":
taskcluster.stringDate(
taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
}
},
)
async def test_putUrl_upload_fails_retried_succeeds(randbytes):
"When a putUrl upload's PUT fails with a 500, an exception is raised"
data = randbytes(10240) # >8k to avoid using dataInline
attempts = 0
class Server(httptest.Handler):
def do_PUT(self):
nonlocal attempts
attempts += 1
if attempts > 2:
self.send_response(200)
self.end_headers()
else:
self.send_response(500)
self.end_headers()
self.wfile.write(b'uhoh')
with httptest.Server(Server) as ts:
objectService = FakeObject(ts)
await upload.upload_from_buf(
projectId="taskcluster",
expires=taskcluster.fromNow('1 hour'),
contentType="text/plain",
contentLength=len(data),
name="some/object",
data=data,
objectService=objectService)
assert attempts == 3
def craft_build_task(self, module_name, gradle_tasks, subtitle='', run_coverage=False,
is_snapshot=False, component=None, artifacts=None, timestamp=None):
taskcluster_artifacts = {}
# component is not None when this is a release build, in which case artifacts is defined too
if component is not None:
taskcluster_artifacts = {
artifact['taskcluster_path']: {
'type': 'file',
'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
'path': artifact['build_fs_path'],
} for artifact in artifacts
}
scopes = [
"secrets:get:project/mobile/android-components/public-tokens"
] if run_coverage else []
snapshot_flag = '-Psnapshot -Ptimestamp={} '.format(timestamp) if is_snapshot else ''
coverage_flag = '-Pcoverage ' if run_coverage else ''
gradle_command = (
'./gradlew --no-daemon clean ' + coverage_flag + snapshot_flag + gradle_tasks
)
post_gradle_command = 'automation/taskcluster/action/upload_coverage_report.sh' if run_coverage else ''
command = ' && '.join(cmd for cmd in (gradle_command, post_gradle_command) if cmd)
return self._craft_build_ish_task(
name='Android Components - Module {} {}'.format(module_name, subtitle),
description='Execute Gradle tasks for module {}'.format(module_name),
command=command,
scopes=scopes,
artifacts=taskcluster_artifacts
)
def generate_build_task(apks, tag):
artifacts = {}
for apk in apks:
artifact = {
"type": 'file',
"path": apk,
"expires": taskcluster.stringDate(taskcluster.fromNow('1 year'))
}
artifacts["public/%s" % os.path.basename(apk)] = artifact
checkout = "git fetch origin && git reset --hard origin/master" if tag is None else "git fetch origin && git checkout %s" % (tag)
assemble_task = 'assembleNightly'
if tag:
# Non-tagged (nightly) builds should contain all languages
checkout = checkout + ' && python tools/l10n/filter-release-translations.py'
assemble_task = 'assembleRelease'
return taskcluster.slugId(), BUILDER.build_task(
name="(Focus for Android) Build task",
description="Build Focus/Klar from source code.",
command=(checkout +
' && python tools/taskcluster/get-adjust-token.py'
' && python tools/taskcluster/get-sentry-token.py'
' && ./gradlew --no-daemon clean test ' + assemble_task),
features = {
"chainOfTrust": True
},
artifacts = artifacts,
worker_type='gecko-focus',
scopes=[
"secrets:get:project/focus/tokens"
])
def craft_ui_tests_task(self):
artifacts = {
"public": {
"type":
"directory",
"path":
"/build/reference-browser/results",
"expires":
taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN))
}
}
env_vars = {
"GOOGLE_PROJECT": "moz-reference-browser-230023",
"GOOGLE_APPLICATION_CREDENTIALS": ".firebase_token.json"
}
gradle_commands = (
'./gradlew --no-daemon clean assembleDebug assembleAndroidTest', )
test_commands = (
'automation/taskcluster/androidTest/ui-test.sh arm -1', )
command = ' && '.join(cmd
for commands in (gradle_commands, test_commands)
for cmd in commands if cmd)
return self._craft_build_ish_task(
name='UI tests',
description='Execute Gradle tasks for UI tests',
command=command,
scopes=['secrets:get:project/mobile/reference-browser/firebase'],
artifacts=artifacts,
env_vars=env_vars,
)
def generate_build_task(version, artifact_info, is_snapshot):
checkout = ("export TERM=dumb && git fetch {} {} --tags && "
"git config advice.detachedHead false && "
"git checkout {}".format(REPO_URL, BRANCH, HEAD_REV))
artifacts = {
artifact_info['artifact']: {
'type': 'file',
'expires': taskcluster.stringDate(taskcluster.fromNow('1 year')),
'path': artifact_info['path']
}
}
snapshot_flag = '-Psnapshot ' if is_snapshot else ''
module_name = _get_gradle_module_name(artifact_info['name'])
gradle_tasks_for_this_module_only = ' '.join(
'{}:{}{}'.format(module_name, gradle_task,
'' if is_snapshot else 'Release')
for gradle_task in BUILD_GRADLE_TASK_NAMES)
return taskcluster.slugId(), BUILDER.craft_build_ish_task(
name='Android Components - Module {} ({})'.format(
module_name, version),
description='Building and testing module {}'.format(module_name),
command=(checkout + ' && ./gradlew --no-daemon clean ' +
snapshot_flag + gradle_tasks_for_this_module_only +
' uploadArchives zipMavenArtifacts'),
features={
'chainOfTrust': True,
},
scopes=[],
artifacts=artifacts)
def test_putUrl_upload_fails_retried(randbytes):
"When a putUrl upload's PUT fails with a 500, an exception is raised"
data = randbytes(10240) # >8k to avoid using dataInline
attempts = 0
class Server(httptest.Handler):
def do_PUT(self):
nonlocal attempts
attempts += 1
self.rfile.read(len(data))
self.send_response(500)
self.end_headers()
self.wfile.write(b'uhoh')
with httptest.Server(Server) as ts:
objectService = FakeObject(ts)
with pytest.raises(aiohttp.ClientResponseError):
upload.uploadFromBuf(
projectId="taskcluster",
expires=taskcluster.fromNow('1 hour'),
contentType="text/plain",
contentLength=len(data),
name="some/object",
data=data,
objectService=objectService)
assert attempts == 6 # one try plus five retries
def craft_push_task(self,
signing_task_id,
name,
description,
is_staging,
apks=[],
scopes=[],
track='internal',
commit=False):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType":
'mobile-pushapk-dep-v1' if is_staging else 'mobile-pushapk-v1',
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [self.task_id, signing_task_id],
"routes": [],
"scopes": scopes,
"requires": "all-completed",
"payload": {
"commit":
commit,
"google_play_track":
track,
"upstreamArtifacts": [{
"paths": apks,
"taskId": signing_task_id,
"taskType": "signing"
}]
},
"provisionerId": "scriptworker-prov-v1",
"metadata": {
"name": name,
"description": description,
"owner": self.owner,
"source": self.source
}
}
def build_task(self,
name,
description,
command,
dependencies=[],
artifacts={},
scopes=[],
routes=[],
features={},
worker_type='github-worker'):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
features = features.copy()
features.update({"taskclusterProxy": True})
return {
"workerType": worker_type,
"taskGroupId": self.task_id,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"schedulerId": "taskcluster-github",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [self.task_id] + dependencies,
"routes": routes,
"scopes": scopes,
"requires": "all-completed",
"payload": {
"features": features,
"maxRunTime": 7200,
"image": "mozillamobile/android-components:1.4",
"command": ["/bin/bash", "--login", "-cx", command],
"artifacts": artifacts,
"deadline": taskcluster.stringDate(deadline)
},
"provisionerId": "aws-provisioner-v1",
"metadata": {
"name": name,
"description": description,
"owner": self.owner,
"source": self.source
}
}
def beetmover_task(self,
name,
description,
version,
artifact_id,
dependencies=[],
upstreamArtifacts=[],
scopes=[],
worker_type='mobile-beetmover-v1',
is_snapshot=False):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType": worker_type,
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [self.task_id] + dependencies,
"routes": [],
"scopes": scopes,
"requires": "all-completed",
"payload": {
"maxRunTime": 600,
"upstreamArtifacts": upstreamArtifacts,
"releaseProperties": {
"appName":
"snapshot_components" if is_snapshot else "components",
},
"version":
"{}-SNAPSHOT".format(version) if is_snapshot else version,
"artifact_id": artifact_id
},
"provisionerId": "scriptworker-prov-v1",
"metadata": {
"name": name,
"description": description,
"owner": self.owner,
"source": self.source
}
}
def craft_default_task_definition(self, worker_type, provisioner_id,
dependencies, routes, scopes, name,
description, payload, treeherder):
created = datetime.datetime.now()
deadline = taskcluster.fromNow('1 day')
expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN)
return {
"provisionerId":
provisioner_id,
"workerType":
worker_type,
"taskGroupId":
self.task_id,
"schedulerId":
self.scheduler_id,
"created":
taskcluster.stringDate(created),
"deadline":
taskcluster.stringDate(deadline),
"expires":
taskcluster.stringDate(expires),
"retries":
5,
"tags": {},
"priority":
self.tasks_priority,
"dependencies": [self.task_id] + dependencies,
"requires":
"all-completed",
"routes":
["tc-treeherder.v2.reference-browser.{}".format(self.commit)] +
routes,
"scopes":
scopes,
"payload":
payload,
"extra": {
"treeherder": treeherder,
},
"metadata": {
"name": name,
"description": description,
"owner": self.owner,
"source": self.source,
},
}
def craft_assemble_release_task(self, architectures, is_staging=False):
artifacts = {
'public/target.{}.apk'.format(arch): {
"type":
'file',
"path":
'/opt/fenix/app/build/outputs/apk/'
'{}Greenfield/release/app-{}-greenfield-release-unsigned.apk'.
format(arch, arch),
"expires":
taskcluster.stringDate(
taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
}
for arch in architectures
}
sentry_secret = '{}project/mobile/fenix/sentry'.format(
'garbage/staging/' if is_staging else '')
leanplum_secret = '{}project/mobile/fenix/leanplum'.format(
'garbage/staging/' if is_staging else '')
pre_gradle_commands = (
'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'
.format(secret, key, target_file) for secret, key, target_file in (
(sentry_secret, 'dsn', '.sentry_token'),
(leanplum_secret, 'production', '.leanplum_token'),
))
gradle_commands = (
'./gradlew --no-daemon -PcrashReports=true -Ptelemetry=true clean test assembleRelease',
)
command = ' && '.join(cmd for commands in (pre_gradle_commands,
gradle_commands)
for cmd in commands if cmd)
routes = [] if is_staging else [
"[email protected]"
]
return self._craft_build_ish_task(
name='Build task',
description='Build Fenix from source code',
command=command,
scopes=[
"secrets:get:{}".format(secret)
for secret in (sentry_secret, leanplum_secret)
],
artifacts=artifacts,
routes=routes,
treeherder={
'jobKind': 'build',
'machine': {
'platform': 'android-all',
},
'symbol': 'NA',
'tier': 1,
},
)
def _craft_artifacts_from_variant(variant):
return {
'public/target.apk': {
'type': 'file',
'path': _craft_apk_full_path_from_variant(variant),
'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
}
}
def _craft_artifacts_from_variant(variant):
return {
DEFAULT_APK_ARTIFACT_LOCATION: {
'type': 'file',
'path': variant.apk_absolute_path(),
'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
}
}
def _craft_artifacts_from_variant(variant):
return {
apk.taskcluster_path: {
'type': 'file',
'path': apk.absolute_path(variant.build_type),
'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
} for apk in variant.apks
}
def craft_signing_task(self,
build_task_id,
name,
description,
signing_format,
is_staging,
apks=[],
scopes=[],
routes=[]):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType":
'mobile-signing-dep-v1' if is_staging else 'mobile-signing-v1',
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [self.task_id, build_task_id],
"routes": routes,
"scopes": scopes,
"requires": "all-completed",
"payload": {
"maxRunTime":
3600,
"upstreamArtifacts": [{
"paths": apks,
"formats": [signing_format],
"taskId": build_task_id,
"taskType": "build"
}]
},
"provisionerId": "scriptworker-prov-v1",
"metadata": {
"name": name,
"description": description,
"owner": self.owner,
"source": self.source
}
}
def create_task(name, description, command):
created = datetime.datetime.now()
expires = taskcluster.fromNow('1 year')
deadline = taskcluster.fromNow('1 day')
return {
"workerType": 'github-worker',
"taskGroupId": TASK_ID,
"expires": taskcluster.stringDate(expires),
"retries": 5,
"created": taskcluster.stringDate(created),
"tags": {},
"priority": "lowest",
"schedulerId": "taskcluster-github",
"deadline": taskcluster.stringDate(deadline),
"dependencies": [ TASK_ID ],
"routes": [],
"scopes": [],
"requires": "all-completed",
"payload": {
"features": {},
"maxRunTime": 7200,
"image": "mozillamobile/mentat:1.1",
"command": [
"/bin/bash",
"--login",
"-cx",
"export TERM=dumb && git fetch %s %s && git config advice.detachedHead false && git checkout %s && cd sdks/android/Mentat && ./gradlew --no-daemon clean %s" % (REPO_URL, BRANCH, COMMIT, command)
],
"artifacts": {},
"deadline": taskcluster.stringDate(deadline)
},
"provisionerId": "aws-provisioner-v1",
"metadata": {
"name": name,
"description": description,
"owner": "*****@*****.**",
"source": "https://github.com/mozilla/mentat"
}
}
def retrigger_task(task_id, dry_run=False):
""" Given a task id (our uuid) we query it and build
a new task based on the old one which we schedule on TaskCluster.
We don't call the rerun API since we can't rerun a task past
its deadline, instead we create a new task with a new taskGroupId,
expiration, creation and deadline values.
task_id (int) - ID that identifies a task on Taskcluster
dry_run (bool) - Default to False. If True, it won't trigger
a task.
returns - 0 for dry_run case, -1 for any failure or the task id (int)
of a succesful retriggered task.
http://docs.taskcluster.net/queue/api-docs/#createTask
"""
one_year = 365
new_task_id = 0
try:
queue = taskcluster_client.Queue()
task = queue.task(task_id)
LOG.debug("Original task: (Limit 1024 char)")
LOG.debug(str(json.dumps(task))[:1024])
new_task_id = taskcluster_client.slugId()
artifacts = task['payload'].get('artifacts', {})
for artifact, definition in artifacts.iteritems():
definition['expires'] = taskcluster_client.fromNow('%s days' % one_year)
# The task group will be identified by the ID of the only
# task in the group
task['taskGroupId'] = new_task_id
# https://bugzilla.mozilla.org/show_bug.cgi?id=1190660
# TC workers create public logs which are 365 days; if the task expiration
# date is the same or less than that we won't have logs for the task
task['expires'] = taskcluster_client.fromNow('%s days' % (one_year + 1))
now = datetime.datetime.utcnow()
tomorrow = now + datetime.timedelta(hours=24)
task['created'] = taskcluster_client.stringDate(now)
task['deadline'] = taskcluster_client.stringDate(tomorrow)
LOG.debug("Contents of new task: (Limit 1024 char)")
LOG.debug(str(task)[:1024])
if not dry_run:
LOG.info("Attempting to schedule new task with task_id: {}".format(new_task_id))
result = queue.createTask(new_task_id, task)
LOG.debug(json.dumps(result))
LOG.info("{}/task-inspector/#{}".format(TASKCLUSTER_TOOLS_HOST, new_task_id))
else:
LOG.info("Dry-run mode: Nothing was retriggered.")
except taskcluster_client.exceptions.TaskclusterRestFailure as e:
traceback.print_exc()
new_task_id = -1
except taskcluster_client.exceptions.TaskclusterAuthFailure as e:
# Hack until we fix it in the issue
if str(e) == "Authorization Failed":
LOG.error("The taskclaster client that you specified is lacking "
"the right set of scopes.")
LOG.error("Run this same command with --debug and you will see "
"the missing scopes (the output comes from the "
"taskcluster python client)")
elif str(e) == "Authentication Error":
LOG.error("Make sure that you create permanent credentials and you "
"set these environment variables: TASKCLUSTER_CLIENT_ID & "
"TASKCLUSTER_ACCESS_TOKEN")
new_task_id = -1
return new_task_id
def from_now_json(self, offset):
"""
Same as `taskcluster.fromNowJSON`, but uses the creation time of `self` for “now”.
"""
return taskcluster.stringDate(taskcluster.fromNow(offset, dateObj=self.now))