def make_release_generate_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "SGenChcks")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
job_template = "{}-{}".format(dep_job.label, "signing")
label = job.get("label", job_template)
description = "Signing of the overall release-related checksums"
dependencies = {dep_job.kind: dep_job.label}
upstream_artifacts = [{
"taskId": {
"task-reference": "<{}>".format(str(dep_job.kind))
},
"taskType":
"build",
"paths": [
get_artifact_path(dep_job, "SHA256SUMS"),
get_artifact_path(dep_job, "SHA512SUMS"),
],
"formats": ["autograph_gpg"],
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"scopes": [
signing_cert_scope,
],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
}
yield task
def make_release_generate_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'SGenChcks')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
job_template = "{}-{}".format(dep_job.label, "signing")
label = job.get("label", job_template)
description = "Signing of the overall release-related checksums"
dependencies = {"build": dep_job.label}
upstream_artifacts = [{
"taskId": {
"task-reference": "<build>"
},
"taskType":
"build",
"paths": [
get_artifact_path(dep_job, "SHA256SUMS"),
get_artifact_path(dep_job, "SHA512SUMS"),
],
"formats": ["gpg"]
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config,
signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes':
[signing_cert_scope, "project:releng:signing:format:gpg"],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def define_upstream_artifacts(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
build_platform = dep_job.attributes.get('build_platform')
job['attributes'] = copy_attributes_from_dependent_job(dep_job)
artifacts_specifications = generate_specifications_of_artifacts_to_sign(
config,
job,
keep_locale_template=False,
kind=config.kind,
)
if 'android' in build_platform:
# We're in the job that creates both multilocale and en-US APKs
artifacts_specifications[0]['artifacts'].append(
get_artifact_path(dep_job, 'en-US/target.apk'))
job['upstream-artifacts'] = [{
'taskId': {
'task-reference': '<build>'
},
'taskType': 'build',
'paths': spec['artifacts'],
'formats': spec['formats'],
} for spec in artifacts_specifications]
yield job
def set_target(config, tests):
for test in tests:
build_platform = test['build-platform']
target = None
if 'target' in test:
resolve_keyed_by(test, 'target', item_name=test['test-name'])
target = test['target']
if not target:
if build_platform.startswith('macosx'):
target = 'target.dmg'
elif build_platform.startswith('android'):
target = 'target.apk'
elif build_platform.startswith('win'):
target = 'target.zip'
else:
target = 'target.tar.bz2'
if isinstance(target, dict):
# TODO Remove hardcoded mobile artifact prefix
index_url = get_index_url(target['index'])
installer_url = '{}/artifacts/public/{}'.format(
index_url, target['name'])
test['mozharness']['installer-url'] = installer_url
else:
test['mozharness']['build-artifact-name'] = get_artifact_path(
test, target)
yield test
def test_packages_url(taskdesc):
"""Account for different platforms that name their test packages differently"""
artifact_url = get_artifact_url('<build>', get_artifact_path(taskdesc,
'target.test_packages.json'))
# for android nightly we need to add 'en-US' to the artifact url
test = taskdesc['run']['test']
if get_variant(test['test-platform']) == "nightly" and 'android' in test['test-platform']:
head, tail = os.path.split(artifact_url)
artifact_url = os.path.join(head, 'en-US', tail)
return artifact_url
def test_packages_url(taskdesc):
"""Account for different platforms that name their test packages differently"""
artifact_url = get_artifact_url(
"<build>", get_artifact_path(taskdesc, "target.test_packages.json"))
# for android shippable we need to add 'en-US' to the artifact url
test = taskdesc["run"]["test"]
if "android" in test["test-platform"] and (get_variant(
test["test-platform"]) in ("shippable", "shippable-qr")):
head, tail = os.path.split(artifact_url)
artifact_url = os.path.join(head, "en-US", tail)
return artifact_url
def set_target(config, tests):
for test in tests:
build_platform = test['build-platform']
target = None
if 'target' in test:
resolve_keyed_by(test, 'target', item_name=test['test-name'])
target = test['target']
if not target:
if build_platform.startswith('macosx'):
target = 'target.dmg'
elif build_platform.startswith('android'):
target = 'target.apk'
elif build_platform.startswith('win'):
target = 'target.zip'
else:
target = 'target.tar.bz2'
test['mozharness']['build-artifact-name'] = get_artifact_path(test, target)
yield test
def mozharness_test_on_native_engine(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
is_talos = test['suite'] == 'talos' or test['suite'] == 'raptor'
is_macosx = worker['os'] == 'macosx'
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
mozharness_url = get_artifact_url(
'<build>', get_artifact_path(taskdesc, 'mozharness.zip'))
worker['artifacts'] = [
{
'name': prefix.rstrip('/'),
'path': path.rstrip('/'),
'type': 'directory',
} for (prefix, path) in [
# (artifact name prefix, in-image path relative to homedir)
("public/logs/", "workspace/build/upload/logs/"),
("public/test", "artifacts/"),
("public/test_info/", "workspace/build/blobber_upload_dir/"),
]
]
if test['reboot']:
worker['reboot'] = test['reboot']
if test['max-run-time']:
worker['max-run-time'] = test['max-run-time']
env = worker.setdefault('env', {})
env.update({
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZHARNESS_URL': {
'task-reference': mozharness_url
},
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
"MOZ_NO_REMOTE": '1',
"XPCOM_DEBUG_BREAK": 'warn',
"NO_FAIL_ON_TEST_ERRORS": '1',
"MOZ_HIDE_RESULTS_TABLE": '1',
"MOZ_NODE_PATH": "/usr/local/bin/node",
'MOZ_AUTOMATION': '1',
})
# talos tests don't need Xvfb
if is_talos:
env['NEED_XVFB'] = 'false'
script = 'test-macosx.sh' if is_macosx else 'test-linux.sh'
worker['context'] = '{}/raw-file/{}/taskcluster/scripts/tester/{}'.format(
config.params['head_repository'], config.params['head_rev'], script)
command = worker['command'] = ["./{}".format(script)]
command.extend([
{
"task-reference": "--installer-url=" + installer_url
},
{
"task-reference":
"--test-packages-url=" + test_packages_url(taskdesc)
},
])
if mozharness.get('include-blob-upload-branch'):
command.append('--blob-upload-branch=' + config.params['project'])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
command[i] += suffix
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
is_macosx = worker['os'] == 'macosx'
is_windows = worker['os'] == 'windows'
is_linux = worker['os'] == 'linux'
assert is_macosx or is_windows or is_linux
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
upstream_task = '<build-signing>' if mozharness[
'requires-signed-builds'] else '<build>'
installer_url = get_artifact_url(upstream_task,
mozharness['build-artifact-name'])
taskdesc['scopes'].extend([
'generic-worker:os-group:{}/{}'.format(job['worker-type'], group)
for group in test['os-groups']
])
worker['os-groups'] = test['os-groups']
# run-as-administrator is a feature for workers with UAC enabled and as such should not be
# included in tasks on workers that have UAC disabled. Currently UAC is only enabled on
# gecko Windows 10 workers, however this may be subject to change. Worker type
# environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig
# See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control
# for more information about UAC.
if test.get('run-as-administrator', False):
if job['worker-type'].startswith(
'aws-provisioner-v1/gecko-t-win10-64'):
taskdesc['scopes'].extend([
'generic-worker:run-as-administrator:{}'.format(
job['worker-type'])
])
worker['run-as-administrator'] = True
else:
raise Exception('run-as-administrator not supported on {}'.format(
job['worker-type']))
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
env = worker.setdefault('env', {})
env['MOZ_AUTOMATION'] = '1'
env['GECKO_HEAD_REPOSITORY'] = config.params['head_repository']
env['GECKO_HEAD_REV'] = config.params['head_rev']
# this list will get cleaned up / reduced / removed in bug 1354088
if is_macosx:
env.update({
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0',
})
if is_windows:
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
else:
# is_linux or is_macosx
mh_command = [
'python2.7', '-u', 'mozharness/scripts/' + mozharness['script']
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if is_windows:
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url(taskdesc)])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
mh_command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': get_artifact_path(taskdesc, 'mozharness.zip'),
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if is_windows:
worker['command'] = [{'task-reference': ' '.join(mh_command)}]
else: # is_macosx
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [mh_command_task_ref]
def fill_template(config, tasks):
dummy_tasks = {}
for task in tasks:
name = task['name']
deps = {}
urls = {}
previous_artifact = None
for k in ('original', 'new'):
value = task[k]
if isinstance(value, basestring):
deps[k] = value
task_id = '<{}>'.format(k)
os_hint = value
else:
index = value['index-search']
if index not in dummy_tasks:
dummy_tasks[index] = {
'label': 'index-search-' + index,
'description': index,
'worker-type': 'invalid/always-optimized',
'run': {
'using': 'always-optimized',
},
'optimization': {
'index-search': [index],
}
}
yield dummy_tasks[index]
deps[index] = 'index-search-' + index
task_id = '<{}>'.format(index)
os_hint = index.split('.')[-1]
if 'linux' in os_hint:
artifact = 'target.tar.bz2'
elif 'macosx' in os_hint:
artifact = 'target.dmg'
elif 'android' in os_hint:
artifact = 'target.apk'
elif 'win' in os_hint:
artifact = 'target.zip'
else:
raise Exception(
'Cannot figure out the OS for {!r}'.format(value))
if previous_artifact is not None and previous_artifact != artifact:
raise Exception('Cannot compare builds from different OSes')
url = get_artifact_url(task_id, get_artifact_path(task, artifact))
urls[k] = {'task-reference': url}
previous_artifact = artifact
taskdesc = {
'label':
'diff-' + name,
'description':
name,
'treeherder': {
'symbol': task['symbol'],
'platform': 'diff/opt',
'kind': 'other',
'tier': 2,
},
'worker-type':
'aws-provisioner-v1/gecko-{}-b-linux'.format(
config.params['level']),
'worker': {
'docker-image': {
'in-tree': 'diffoscope'
},
'artifacts': [{
'type': 'file',
'path': '/builds/worker/diff.html',
'name': 'public/diff.html',
}, {
'type': 'file',
'path': '/builds/worker/diff.txt',
'name': 'public/diff.txt',
}],
'env': {
'ORIG_URL':
urls['original'],
'NEW_URL':
urls['new'],
'DIFFOSCOPE_ARGS':
' '.join(task[k] for k in ('args', 'extra-args')
if k in task)
},
'max-run-time':
1800,
},
'run': {
'using':
'run-task',
'checkout':
False,
'command':
'/builds/worker/bin/get_and_diffoscope '
'"$ORIG_URL" "$NEW_URL"',
},
'dependencies':
deps,
}
if artifact.endswith('.dmg'):
taskdesc['toolchains'] = [
'linux64-cctools-port',
'linux64-libdmg',
]
yield taskdesc
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
treeherder = job.get('treeherder', {})
if attributes.get('nightly'):
treeherder.setdefault('symbol', 'rs(N)')
else:
treeherder.setdefault('symbol', 'rs(B)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)
)
treeherder.setdefault('kind', 'build')
label = job['label']
dependencies = {"repackage": dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update({k: v for k, v in signing_dependencies.items()
if k != 'docker-image'})
locale_str = ""
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'rs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
locale_str = "{}/".format(dep_job.attributes.get('locale'))
description = (
"Signing of repackaged artifacts for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get('nightly')
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
scopes = [signing_cert_scope, add_scope_prefix(config, 'signing:format:mar_sha384')]
upstream_artifacts = [{
"taskId": {"task-reference": "<repackage>"},
"taskType": "repackage",
"paths": [
get_artifact_path(dep_job, "{}target.complete.mar".format(locale_str)),
],
"formats": ["mar_sha384"]
}]
if 'win' in build_platform:
upstream_artifacts.append({
"taskId": {"task-reference": "<repackage>"},
"taskType": "repackage",
"paths": [
get_artifact_path(dep_job, "{}target.installer.exe".format(locale_str)),
],
"formats": ["sha2signcode"]
})
scopes.append(add_scope_prefix(config, "signing:format:sha2signcode"))
use_stub = attributes.get('stub-installer')
if use_stub:
upstream_artifacts.append({
"taskId": {"task-reference": "<repackage>"},
"taskType": "repackage",
"paths": [
get_artifact_path(
dep_job, "{}target.stub-installer.exe".format(locale_str)
),
],
"formats": ["sha2signcodestub"]
})
scopes.append(add_scope_prefix(config, "signing:format:sha2signcodestub"))
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, signing_cert_scope),
'worker': {'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
}
yield task
def fill_template(config, tasks):
dummy_tasks = {}
for task in tasks:
name = task["name"]
deps = {}
urls = {}
previous_artifact = None
artifact = task.get("artifact")
for k in ("original", "new"):
value = task[k]
if isinstance(value, text_type):
deps[k] = value
dep_name = k
os_hint = value
else:
index = value["index-search"]
if index not in dummy_tasks:
dummy_tasks[index] = {
"label": "index-search-" + index,
"description": index,
"worker-type": "invalid/always-optimized",
"run": {
"using": "always-optimized",
},
"optimization": {
"index-search": [index],
},
}
yield dummy_tasks[index]
deps[index] = "index-search-" + index
dep_name = index
os_hint = index.split(".")[-1]
if artifact:
pass
elif "linux" in os_hint:
artifact = "target.tar.bz2"
elif "macosx" in os_hint:
artifact = "target.dmg"
elif "android" in os_hint:
artifact = "target.apk"
elif "win" in os_hint:
artifact = "target.zip"
else:
raise Exception("Cannot figure out the OS for {!r}".format(value))
if previous_artifact is not None and previous_artifact != artifact:
raise Exception("Cannot compare builds from different OSes")
urls[k] = {
"artifact-reference": "<{}/{}>".format(
dep_name, get_artifact_path(task, artifact)
),
}
previous_artifact = artifact
taskdesc = {
"label": "diff-" + name,
"description": name,
"treeherder": {
"symbol": task["symbol"],
"platform": "diff/opt",
"kind": "other",
"tier": task["tier"],
},
"worker-type": "b-linux",
"worker": {
"docker-image": {"in-tree": "diffoscope"},
"artifacts": [
{
"type": "file",
"path": "/builds/worker/{}".format(f),
"name": "public/{}".format(f),
}
for f in (
"diff.html",
"diff.txt",
)
],
"env": {
"ORIG_URL": urls["original"],
"NEW_URL": urls["new"],
"DIFFOSCOPE_ARGS": " ".join(
task[k] for k in ("args", "extra-args") if k in task
),
"PRE_DIFF": "; ".join(task.get("pre-diff-commands", [])),
},
"max-run-time": 1800,
},
"run": {
"using": "run-task",
"checkout": task.get("unpack", False),
"command": "/builds/worker/bin/get_and_diffoscope{}{}".format(
" --unpack" if task.get("unpack") else "",
" --fail" if task.get("fail-on-diff") else "",
),
},
"dependencies": deps,
"optimization": task.get("optimization"),
}
if "run-on-projects" in task:
taskdesc["run-on-projects"] = task["run-on-projects"]
if artifact.endswith(".dmg"):
taskdesc.setdefault("fetches", {}).setdefault("toolchain", []).extend(
[
"linux64-cctools-port",
"linux64-libdmg",
]
)
yield taskdesc
def generate_specifications_of_artifacts_to_sign(task,
keep_locale_template=True,
kind=None):
build_platform = task.attributes.get('build_platform')
use_stub = task.attributes.get('stub-installer')
if kind == 'release-source-signing':
artifacts_specifications = [{
'artifacts': [get_artifact_path(task, 'source.tar.xz')],
'formats': ['gpg'],
}]
elif 'android' in build_platform:
artifacts_specifications = [{
'artifacts': [
get_artifact_path(task, '{locale}/target.apk'),
],
'formats': ['jar'],
}]
# XXX: Mars aren't signed here (on any platform) because internals will be
# signed at after this stage of the release
elif 'macosx' in build_platform:
if is_partner_kind(kind):
extension = 'tar.gz'
else:
extension = 'dmg'
artifacts_specifications = [{
'artifacts': [
get_artifact_path(task,
'{{locale}}/target.{}'.format(extension))
],
'formats': ['macapp', 'widevine'],
}]
elif 'win' in build_platform:
artifacts_specifications = [{
'artifacts': [
get_artifact_path(task, '{locale}/setup.exe'),
],
'formats': ['sha2signcode'],
}, {
'artifacts': [
get_artifact_path(task, '{locale}/target.zip'),
],
'formats': ['sha2signcode', 'widevine'],
}]
if use_stub:
artifacts_specifications[0]['artifacts'] += [
get_artifact_path(task, '{locale}/setup-stub.exe')
]
elif 'linux' in build_platform:
artifacts_specifications = [{
'artifacts': [get_artifact_path(task, '{locale}/target.tar.bz2')],
'formats': ['gpg', 'widevine'],
}]
else:
raise Exception("Platform not implemented for signing")
if not keep_locale_template:
artifacts_specifications = _strip_locale_template(
artifacts_specifications)
if is_partner_kind(kind):
artifacts_specifications = _strip_widevine_for_partners(
artifacts_specifications)
return artifacts_specifications
def fill_template(config, tasks):
dummy_tasks = {}
for task in tasks:
name = task['name']
deps = {}
urls = {}
previous_artifact = None
for k in ('original', 'new'):
value = task[k]
if isinstance(value, text_type):
deps[k] = value
dep_name = k
os_hint = value
else:
index = value['index-search']
if index not in dummy_tasks:
dummy_tasks[index] = {
'label': 'index-search-' + index,
'description': index,
'worker-type': 'invalid/always-optimized',
'run': {
'using': 'always-optimized',
},
'optimization': {
'index-search': [index],
}
}
yield dummy_tasks[index]
deps[index] = 'index-search-' + index
dep_name = index
os_hint = index.split('.')[-1]
if 'linux' in os_hint:
artifact = 'target.tar.bz2'
elif 'macosx' in os_hint:
artifact = 'target.dmg'
elif 'android' in os_hint:
artifact = 'target.apk'
elif 'win' in os_hint:
artifact = 'target.zip'
else:
raise Exception(
'Cannot figure out the OS for {!r}'.format(value))
if previous_artifact is not None and previous_artifact != artifact:
raise Exception(
'Cannot compare builds from different OSes')
urls[k] = {
'artifact-reference': '<{}/{}>'.format(
dep_name, get_artifact_path(task, artifact)),
}
previous_artifact = artifact
taskdesc = {
'label': 'diff-' + name,
'description': name,
'treeherder': {
'symbol': task['symbol'],
'platform': 'diff/opt',
'kind': 'other',
'tier': 2,
},
'worker-type': 'b-linux',
'worker': {
'docker-image': {'in-tree': 'diffoscope'},
'artifacts': [{
'type': 'file',
'path': '/builds/worker/{}'.format(f),
'name': 'public/{}'.format(f),
} for f in (
'diff.html',
'diff.txt',
'generated-files.diff.html',
'generated-files.diff.txt',
)],
'env': {
'ORIG_URL': urls['original'],
'NEW_URL': urls['new'],
'DIFFOSCOPE_ARGS': ' '.join(
task[k] for k in ('args', 'extra-args') if k in task),
'PRE_DIFF': '; '.join(task.get('pre-diff-commands', [])),
},
'max-run-time': 1800,
},
'run': {
'using': 'run-task',
'checkout': task.get('unpack', False),
'command': '/builds/worker/bin/get_and_diffoscope{}{}'.format(
' --unpack' if task.get('unpack') else '',
' --fail' if task.get('fail-on-diff') else '',
),
},
'dependencies': deps,
}
if 'run-on-projects' in task:
taskdesc['run-on-projects'] = task['run-on-projects']
if artifact.endswith('.dmg'):
taskdesc.setdefault('fetches', {}).setdefault('toolchain', []).extend([
'linux64-cctools-port',
'linux64-libdmg',
])
yield taskdesc
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
repack_id = dep_job.task['extra']['repack_id']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_nightly = dep_job.attributes.get(
'nightly', dep_job.attributes.get('shippable'))
# Mac & windows
label = dep_job.label.replace("repackage-", "repackage-signing-")
# Linux
label = label.replace("chunking-dummy-", "repackage-signing-")
description = (
"Signing of repackaged artifacts for partner repack id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
if 'linux' in build_platform:
# we want the repack job, via the dependencies for the the chunking-dummy dep_job
for dep in dep_job.dependencies.values():
if dep.startswith('release-partner-repack'):
dependencies = {"repack": dep}
break
else:
# we have a genuine repackage job as our parent
dependencies = {"repackage": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage-signing'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config)
scopes = [signing_cert_scope]
if 'win' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(
dep_job, "{}/target.installer.exe".format(repack_id)),
],
"formats": ["autograph_authenticode", "autograph_gpg"]
}]
partner_config = get_partner_config_by_kind(config, config.kind)
partner, subpartner, _ = repack_id.split('/')
repack_stub_installer = partner_config[partner][subpartner].get(
'repack_stub_installer')
if build_platform.startswith('win32') and repack_stub_installer:
upstream_artifacts.append({
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(
dep_job,
"{}/target.stub-installer.exe".format(repack_id)),
],
"formats": ["autograph_authenticode", "autograph_gpg"]
})
elif 'mac' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.dmg".format(repack_id)),
],
"formats": ["autograph_gpg"]
}]
elif 'linux' in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repack>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
"{}/target.tar.bz2".format(repack_id)),
],
"formats": ["autograph_gpg"]
}]
task = {
'label': label,
'description': description,
'worker-type': 'linux-signing',
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': scopes,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'extra': {
'repack_id': repack_id,
}
}
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get('priority'):
task['priority'] = job['priority']
yield task
def mozharness_test_on_docker(config, job, taskdesc):
run = job['run']
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker'] = job['worker']
# apply some defaults
worker['docker-image'] = test['docker-image']
worker['allow-ptrace'] = True # required for all tests, for crashreporter
worker['loopback-video'] = test['loopback-video']
worker['loopback-audio'] = test['loopback-audio']
worker['max-run-time'] = test['max-run-time']
worker['retry-exit-status'] = test['retry-exit-status']
if 'android-em-7.0-x86' in test['test-platform']:
worker['privileged'] = True
artifacts = [
# (artifact name prefix, in-image path)
("public/logs/", "{workdir}/workspace/logs/".format(**run)),
("public/test", "{workdir}/artifacts/".format(**run)),
("public/test_info/",
"{workdir}/workspace/build/blobber_upload_dir/".format(**run)),
]
if 'installer-url' in mozharness:
installer_url = mozharness['installer-url']
else:
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
mozharness_url = get_artifact_url(
'<build>', get_artifact_path(taskdesc, 'mozharness.zip'))
worker['artifacts'] = [{
'name':
prefix,
'path':
os.path.join('{workdir}/workspace'.format(**run), path),
'type':
'directory',
} for (prefix, path) in artifacts]
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
'NEED_PULSEAUDIO': 'true',
'NEED_WINDOW_MANAGER': 'true',
'NEED_COMPIZ': 'true',
'ENABLE_E10S': str(bool(test.get('e10s'))).lower(),
'WORKING_DIR': '/builds/worker',
})
# by default, require compiz unless proven otherwise, hence a whitelist.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=1552563
# if using regex this list can be shortened greatly.
suites_not_need_compiz = [
'mochitest-webgl1-core', 'mochitest-webgl1-ext', 'mochitest-plain-gpu',
'mochitest-browser-chrome-screenshots', 'gtest', 'cppunittest',
'jsreftest', 'crashtest', 'reftest', 'reftest-no-accel',
'web-platform-tests', 'web-platform-tests-reftests', 'xpcshell'
]
if job['run']['test']['suite'] in suites_not_need_compiz or (
job['run']['test']['suite'] == 'mochitest-plain-chunked'
and job['run']['test']['try-name'] == 'mochitest-plain-headless'):
env['NEED_COMPIZ'] = 'false'
if mozharness.get('mochitest-flavor'):
env['MOCHITEST_FLAVOR'] = mozharness['mochitest-flavor']
if mozharness['set-moz-node-path']:
env['MOZ_NODE_PATH'] = '/usr/local/bin/node'
if 'actions' in mozharness:
env['MOZHARNESS_ACTIONS'] = ' '.join(mozharness['actions'])
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
# handle some of the mozharness-specific options
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
# Support vcs checkouts regardless of whether the task runs from
# source or not in case it is needed on an interactive loaner.
support_vcs_checkout(config, job, taskdesc)
# If we have a source checkout, run mozharness from it instead of
# downloading a zip file with the same content.
if test['checkout']:
env['MOZHARNESS_PATH'] = '{workdir}/checkouts/gecko/testing/mozharness'.format(
**run)
else:
env['MOZHARNESS_URL'] = {'task-reference': mozharness_url}
extra_config = {
'installer_url': installer_url,
'test_packages_url': test_packages_url(taskdesc),
}
env['EXTRA_MOZHARNESS_CONFIG'] = {
'task-reference': json.dumps(extra_config)
}
command = [
'{workdir}/bin/test-linux.sh'.format(**run),
]
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
job['run'] = {
'workdir': run['workdir'],
'tooltool-downloads': mozharness['tooltool-downloads'],
'checkout': test['checkout'],
'command': command,
'using': 'run-task',
}
configure_taskdesc_for_run(config, job, taskdesc, worker['implementation'])
def mozharness_test_on_generic_worker(config, job, taskdesc):
run = job['run']
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker'] = job['worker']
bitbar_script = 'test-linux.sh'
is_macosx = worker['os'] == 'macosx'
is_windows = worker['os'] == 'windows'
is_linux = worker['os'] == 'linux' or worker['os'] == 'linux-bitbar'
is_bitbar = worker['os'] == 'linux-bitbar'
assert is_macosx or is_windows or is_linux
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
if is_bitbar:
artifacts = [
{
'name': 'public/test/',
'path': 'artifacts/public',
'type': 'directory'
},
{
'name': 'public/logs/',
'path': 'workspace/logs',
'type': 'directory'
},
{
'name': 'public/test_info/',
'path': 'workspace/build/blobber_upload_dir',
'type': 'directory'
},
]
if 'installer-url' in mozharness:
installer_url = mozharness['installer-url']
else:
upstream_task = '<build-signing>' if mozharness[
'requires-signed-builds'] else '<build>'
installer_url = get_artifact_url(upstream_task,
mozharness['build-artifact-name'])
worker['os-groups'] = test['os-groups']
# run-as-administrator is a feature for workers with UAC enabled and as such should not be
# included in tasks on workers that have UAC disabled. Currently UAC is only enabled on
# gecko Windows 10 workers, however this may be subject to change. Worker type
# environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig
# See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control
# for more information about UAC.
if test.get('run-as-administrator', False):
if job['worker-type'].startswith('t-win10-64'):
worker['run-as-administrator'] = True
else:
raise Exception('run-as-administrator not supported on {}'.format(
job['worker-type']))
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
worker['max-run-time'] = test['max-run-time']
worker['retry-exit-status'] = test['retry-exit-status']
worker['artifacts'] = artifacts
env = worker.setdefault('env', {})
env['GECKO_HEAD_REPOSITORY'] = config.params['head_repository']
env['GECKO_HEAD_REV'] = config.params['head_rev']
# this list will get cleaned up / reduced / removed in bug 1354088
if is_macosx:
env.update({
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0',
})
elif is_bitbar:
env.update({
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZHARNESS_URL': {
'artifact-reference': '<build/public/build/mozharness.zip>'
},
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
"MOZ_NO_REMOTE": '1',
"NEED_XVFB": "false",
"XPCOM_DEBUG_BREAK": 'warn',
"NO_FAIL_ON_TEST_ERRORS": '1',
"MOZ_HIDE_RESULTS_TABLE": '1',
"MOZ_NODE_PATH": "/usr/local/bin/node",
'TASKCLUSTER_WORKER_TYPE': job['worker-type'],
})
extra_config = {
'installer_url': installer_url,
'test_packages_url': test_packages_url(taskdesc),
}
env['EXTRA_MOZHARNESS_CONFIG'] = {
'task-reference': json.dumps(extra_config)
}
if is_windows:
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
elif is_bitbar:
mh_command = ['bash', "./{}".format(bitbar_script)]
elif is_macosx and 'macosx1014-64' in test['test-platform']:
mh_command = [
'/usr/local/bin/python2', '-u',
'mozharness/scripts/' + mozharness['script']
]
else:
# is_linux or is_macosx
mh_command = [
# Using /usr/bin/python2.7 rather than python2.7 because
# /usr/local/bin/python2.7 is broken on the mac workers.
# See bug #1547903.
'/usr/bin/python2.7',
'-u',
'mozharness/scripts/' + mozharness['script']
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if is_windows:
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': get_artifact_path(taskdesc, 'mozharness.zip'),
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if is_bitbar:
a_url = config.params.file_url(
'taskcluster/scripts/tester/{}'.format(bitbar_script), )
worker['mounts'] = [{
'file': bitbar_script,
'content': {
'url': a_url,
},
}]
job['run'] = {
'workdir': run['workdir'],
'tooltool-downloads': mozharness['tooltool-downloads'],
'checkout': test['checkout'],
'command': mh_command,
'using': 'run-task',
}
if is_bitbar:
job['run']['run-as-root'] = True
# FIXME: The bitbar config incorrectly requests internal tooltool downloads
# so force it off here.
job['run']['tooltool-downloads'] = False
configure_taskdesc_for_run(config, job, taskdesc, worker['implementation'])
def mozharness_test_on_docker(config, job, taskdesc):
run = job["run"]
test = taskdesc["run"]["test"]
mozharness = test["mozharness"]
worker = taskdesc["worker"] = job["worker"]
# apply some defaults
worker["docker-image"] = test["docker-image"]
worker["allow-ptrace"] = True # required for all tests, for crashreporter
worker["loopback-video"] = test["loopback-video"]
worker["loopback-audio"] = test["loopback-audio"]
worker["max-run-time"] = test["max-run-time"]
worker["retry-exit-status"] = test["retry-exit-status"]
if "android-em-7.0-x86" in test["test-platform"]:
worker["privileged"] = True
artifacts = [
# (artifact name prefix, in-image path)
("public/logs/", "{workdir}/workspace/logs/".format(**run)),
("public/test", "{workdir}/artifacts/".format(**run)),
(
"public/test_info/",
"{workdir}/workspace/build/blobber_upload_dir/".format(**run),
),
]
installer = installer_url(taskdesc)
mozharness_url = get_artifact_url(
"<build>", get_artifact_path(taskdesc, "mozharness.zip"))
worker.setdefault("artifacts", [])
worker["artifacts"].extend([{
"name":
prefix,
"path":
os.path.join("{workdir}/workspace".format(**run), path),
"type":
"directory",
} for (prefix, path) in artifacts])
env = worker.setdefault("env", {})
env.update({
"MOZHARNESS_CONFIG": " ".join(mozharness["config"]),
"MOZHARNESS_SCRIPT": mozharness["script"],
"MOZILLA_BUILD_URL": {
"task-reference": installer
},
"NEED_PULSEAUDIO": "true",
"NEED_WINDOW_MANAGER": "true",
"ENABLE_E10S": text_type(bool(test.get("e10s"))).lower(),
"WORKING_DIR": "/builds/worker",
})
if test.get("python-3"):
env["PYTHON"] = "python3"
# Legacy linux64 tests rely on compiz.
if test.get("docker-image", {}).get("in-tree") == "desktop1604-test":
env.update({"NEED_COMPIZ": "true"})
# Bug 1602701/1601828 - use compiz on ubuntu1804 due to GTK asynchiness
# when manipulating windows.
if test.get("docker-image", {}).get("in-tree") == "ubuntu1804-test":
if "wdspec" in job["run"]["test"]["suite"] or (
"marionette" in job["run"]["test"]["suite"]
and "headless" not in job["label"]):
env.update({"NEED_COMPIZ": "true"})
if mozharness.get("mochitest-flavor"):
env["MOCHITEST_FLAVOR"] = mozharness["mochitest-flavor"]
if mozharness["set-moz-node-path"]:
env["MOZ_NODE_PATH"] = "/usr/local/bin/node"
if "actions" in mozharness:
env["MOZHARNESS_ACTIONS"] = " ".join(mozharness["actions"])
if config.params.is_try():
env["TRY_COMMIT_MSG"] = config.params["message"]
# handle some of the mozharness-specific options
if test["reboot"]:
raise Exception("reboot: {} not supported on generic-worker".format(
test["reboot"]))
# Support vcs checkouts regardless of whether the task runs from
# source or not in case it is needed on an interactive loaner.
support_vcs_checkout(config, job, taskdesc)
# If we have a source checkout, run mozharness from it instead of
# downloading a zip file with the same content.
if test["checkout"]:
env["MOZHARNESS_PATH"] = "{workdir}/checkouts/gecko/testing/mozharness".format(
**run)
else:
env["MOZHARNESS_URL"] = {"task-reference": mozharness_url}
extra_config = {
"installer_url": installer,
"test_packages_url": test_packages_url(taskdesc),
}
env["EXTRA_MOZHARNESS_CONFIG"] = {
"task-reference":
six.ensure_text(json.dumps(extra_config, sort_keys=True))
}
# Bug 1634554 - pass in decision task artifact URL to mozharness for WPT.
# Bug 1645974 - test-verify-wpt and test-coverage-wpt need artifact URL.
if "web-platform-tests" in test["suite"] or re.match(
"test-(coverage|verify)-wpt", test["suite"]):
env["TESTS_BY_MANIFEST_URL"] = {
"artifact-reference": "<decision/public/tests-by-manifest.json.gz>"
}
command = [
"{workdir}/bin/test-linux.sh".format(**run),
]
command.extend(mozharness.get("extra-options", []))
if test.get("test-manifests"):
env["MOZHARNESS_TEST_PATHS"] = six.ensure_text(
json.dumps({test["suite"]: test["test-manifests"]},
sort_keys=True))
# TODO: remove the need for run['chunked']
elif mozharness.get("chunked") or test["chunks"] > 1:
command.append("--total-chunk={}".format(test["chunks"]))
command.append("--this-chunk={}".format(test["this-chunk"]))
if "download-symbols" in mozharness:
download_symbols = mozharness["download-symbols"]
download_symbols = {
True: "true",
False: "false"
}.get(download_symbols, download_symbols)
command.append("--download-symbols=" + download_symbols)
job["run"] = {
"workdir": run["workdir"],
"tooltool-downloads": mozharness["tooltool-downloads"],
"checkout": test["checkout"],
"command": command,
"using": "run-task",
}
configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"])
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc["run"]["test"]
mozharness = test["mozharness"]
worker = taskdesc["worker"] = job["worker"]
bitbar_script = "test-linux.sh"
is_macosx = worker["os"] == "macosx"
is_windows = worker["os"] == "windows"
is_linux = worker["os"] == "linux" or worker["os"] == "linux-bitbar"
is_bitbar = worker["os"] == "linux-bitbar"
assert is_macosx or is_windows or is_linux
artifacts = [
{
"name": "public/logs",
"path": "logs",
"type": "directory"
},
]
# jittest doesn't have blob_upload_dir
if test["test-name"] != "jittest":
artifacts.append({
"name": "public/test_info",
"path": "build/blobber_upload_dir",
"type": "directory",
})
if is_bitbar:
artifacts = [
{
"name": "public/test/",
"path": "artifacts/public",
"type": "directory"
},
{
"name": "public/logs/",
"path": "workspace/logs",
"type": "directory"
},
{
"name": "public/test_info/",
"path": "workspace/build/blobber_upload_dir",
"type": "directory",
},
]
installer = installer_url(taskdesc)
worker["os-groups"] = test["os-groups"]
# run-as-administrator is a feature for workers with UAC enabled and as such should not be
# included in tasks on workers that have UAC disabled. Currently UAC is only enabled on
# gecko Windows 10 workers, however this may be subject to change. Worker type
# environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig
# See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control
# for more information about UAC.
if test.get("run-as-administrator", False):
if job["worker-type"].startswith("t-win10-64"):
worker["run-as-administrator"] = True
else:
raise Exception("run-as-administrator not supported on {}".format(
job["worker-type"]))
if test["reboot"]:
raise Exception("reboot: {} not supported on generic-worker".format(
test["reboot"]))
worker["max-run-time"] = test["max-run-time"]
worker["retry-exit-status"] = test["retry-exit-status"]
worker.setdefault("artifacts", [])
worker["artifacts"].extend(artifacts)
env = worker.setdefault("env", {})
env["GECKO_HEAD_REPOSITORY"] = config.params["head_repository"]
env["GECKO_HEAD_REV"] = config.params["head_rev"]
# this list will get cleaned up / reduced / removed in bug 1354088
if is_macosx:
env.update({
"LC_ALL": "en_US.UTF-8",
"LANG": "en_US.UTF-8",
"MOZ_NODE_PATH": "/usr/local/bin/node",
"PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",
"SHELL": "/bin/bash",
})
elif is_bitbar:
env.update({
"MOZHARNESS_CONFIG": " ".join(mozharness["config"]),
"MOZHARNESS_SCRIPT": mozharness["script"],
"MOZHARNESS_URL": {
"artifact-reference": "<build/public/build/mozharness.zip>"
},
"MOZILLA_BUILD_URL": {
"task-reference": installer
},
"MOZ_NO_REMOTE": "1",
"NEED_XVFB": "false",
"XPCOM_DEBUG_BREAK": "warn",
"NO_FAIL_ON_TEST_ERRORS": "1",
"MOZ_HIDE_RESULTS_TABLE": "1",
"MOZ_NODE_PATH": "/usr/local/bin/node",
"TASKCLUSTER_WORKER_TYPE": job["worker-type"],
})
extra_config = {
"installer_url": installer,
"test_packages_url": test_packages_url(taskdesc),
}
env["EXTRA_MOZHARNESS_CONFIG"] = {
"task-reference":
six.ensure_text(json.dumps(extra_config, sort_keys=True))
}
# Bug 1634554 - pass in decision task artifact URL to mozharness for WPT.
# Bug 1645974 - test-verify-wpt and test-coverage-wpt need artifact URL.
if "web-platform-tests" in test["suite"] or re.match(
"test-(coverage|verify)-wpt", test["suite"]):
env["TESTS_BY_MANIFEST_URL"] = {
"artifact-reference": "<decision/public/tests-by-manifest.json.gz>"
}
py_3 = test.get("python-3", False)
if is_windows:
py_binary = "c:\\mozilla-build\\{python}\\{python}.exe".format(
python="python3" if py_3 else "python")
mh_command = [
py_binary,
"-u",
"mozharness\\scripts\\" + normpath(mozharness["script"]),
]
elif is_bitbar:
py_binary = "python3" if py_3 else "python"
mh_command = ["bash", "./{}".format(bitbar_script)]
elif is_macosx and "macosx1014-64" in test["test-platform"]:
py_binary = "/usr/local/bin/{}".format(
"python3" if py_3 else "python2")
mh_command = [
py_binary,
"-u",
"mozharness/scripts/" + mozharness["script"],
]
else:
# is_linux or is_macosx
py_binary = "/usr/bin/{}".format("python3" if py_3 else "python2")
mh_command = [
# Using /usr/bin/python2.7 rather than python2.7 because
# /usr/local/bin/python2.7 is broken on the mac workers.
# See bug #1547903.
py_binary,
"-u",
"mozharness/scripts/" + mozharness["script"],
]
if py_3:
env["PYTHON"] = py_binary
for mh_config in mozharness["config"]:
cfg_path = "mozharness/configs/" + mh_config
if is_windows:
cfg_path = normpath(cfg_path)
mh_command.extend(["--cfg", cfg_path])
mh_command.extend(mozharness.get("extra-options", []))
if mozharness.get("download-symbols"):
if isinstance(mozharness["download-symbols"], text_type):
mh_command.extend(
["--download-symbols", mozharness["download-symbols"]])
else:
mh_command.extend(["--download-symbols", "true"])
if mozharness.get("include-blob-upload-branch"):
mh_command.append("--blob-upload-branch=" + config.params["project"])
if test.get("test-manifests"):
env["MOZHARNESS_TEST_PATHS"] = six.ensure_text(
json.dumps({test["suite"]: test["test-manifests"]},
sort_keys=True))
# TODO: remove the need for run['chunked']
elif mozharness.get("chunked") or test["chunks"] > 1:
mh_command.append("--total-chunk={}".format(test["chunks"]))
mh_command.append("--this-chunk={}".format(test["this-chunk"]))
if config.params.is_try():
env["TRY_COMMIT_MSG"] = config.params["message"]
worker["mounts"] = [{
"directory": "mozharness",
"content": {
"artifact": get_artifact_path(taskdesc, "mozharness.zip"),
"task-id": {
"task-reference": "<build>"
},
},
"format": "zip",
}]
if is_bitbar:
a_url = config.params.file_url(
"taskcluster/scripts/tester/{}".format(bitbar_script), )
worker["mounts"] = [{
"file": bitbar_script,
"content": {
"url": a_url,
},
}]
job["run"] = {
"tooltool-downloads": mozharness["tooltool-downloads"],
"checkout": test["checkout"],
"command": mh_command,
"using": "run-task",
}
if is_bitbar:
job["run"]["run-as-root"] = True
configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"])
def test_packages_url(taskdesc):
"""Account for different platforms that name their test packages differently"""
return get_artifact_url(
'<build>', get_artifact_path(taskdesc, 'target.test_packages.json'))
def mozharness_test_on_docker(config, job, taskdesc):
run = job['run']
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
# apply some defaults
worker['docker-image'] = test['docker-image']
worker['allow-ptrace'] = True # required for all tests, for crashreporter
worker['loopback-video'] = test['loopback-video']
worker['loopback-audio'] = test['loopback-audio']
worker['max-run-time'] = test['max-run-time']
worker['retry-exit-status'] = test['retry-exit-status']
if 'android-em-7.0-x86' in test['test-platform']:
worker['privileged'] = True
artifacts = [
# (artifact name prefix, in-image path)
("public/logs/", "{workdir}/workspace/build/upload/logs/".format(**run)
),
("public/test", "{workdir}/artifacts/".format(**run)),
("public/test_info/",
"{workdir}/workspace/build/blobber_upload_dir/".format(**run)),
]
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
mozharness_url = get_artifact_url(
'<build>', get_artifact_path(taskdesc, 'mozharness.zip'))
worker['artifacts'] = [{
'name':
prefix,
'path':
os.path.join('{workdir}/workspace'.format(**run), path),
'type':
'directory',
} for (prefix, path) in artifacts]
worker['caches'] = [{
'type':
'persistent',
'name':
'level-{}-{}-test-workspace'.format(config.params['level'],
config.params['project']),
'mount-point':
"{workdir}/workspace".format(**run),
}]
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
'NEED_PULSEAUDIO': 'true',
'NEED_WINDOW_MANAGER': 'true',
'ENABLE_E10S': str(bool(test.get('e10s'))).lower(),
'MOZ_AUTOMATION': '1',
'WORKING_DIR': '/builds/worker',
})
if mozharness.get('mochitest-flavor'):
env['MOCHITEST_FLAVOR'] = mozharness['mochitest-flavor']
if mozharness['set-moz-node-path']:
env['MOZ_NODE_PATH'] = '/usr/local/bin/node'
if 'actions' in mozharness:
env['MOZHARNESS_ACTIONS'] = ' '.join(mozharness['actions'])
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
# handle some of the mozharness-specific options
if mozharness['tooltool-downloads']:
docker_worker_add_tooltool(config, job, taskdesc, internal=True)
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
# assemble the command line
command = [
'{workdir}/bin/run-task'.format(**run),
]
# Support vcs checkouts regardless of whether the task runs from
# source or not in case it is needed on an interactive loaner.
support_vcs_checkout(config, job, taskdesc)
# If we have a source checkout, run mozharness from it instead of
# downloading a zip file with the same content.
if test['checkout']:
command.extend(
['--vcs-checkout', '{workdir}/checkouts/gecko'.format(**run)])
env['MOZHARNESS_PATH'] = '{workdir}/checkouts/gecko/testing/mozharness'.format(
**run)
else:
env['MOZHARNESS_URL'] = {'task-reference': mozharness_url}
command.extend([
'--',
'{workdir}/bin/test-linux.sh'.format(**run),
])
command.extend([
{
"task-reference": "--installer-url=" + installer_url
},
{
"task-reference":
"--test-packages-url=" + test_packages_url(taskdesc)
},
])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
command[i] += suffix
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
worker['command'] = command
def mozharness_test_on_docker(config, job, taskdesc):
run = job['run']
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker'] = job['worker']
# apply some defaults
worker['docker-image'] = test['docker-image']
worker['allow-ptrace'] = True # required for all tests, for crashreporter
worker['loopback-video'] = test['loopback-video']
worker['loopback-audio'] = test['loopback-audio']
worker['max-run-time'] = test['max-run-time']
worker['retry-exit-status'] = test['retry-exit-status']
if 'android-em-7.0-x86' in test['test-platform']:
worker['privileged'] = True
artifacts = [
# (artifact name prefix, in-image path)
("public/logs/", "{workdir}/workspace/logs/".format(**run)),
("public/test", "{workdir}/artifacts/".format(**run)),
("public/test_info/",
"{workdir}/workspace/build/blobber_upload_dir/".format(**run)),
]
if 'installer-url' in mozharness:
installer_url = mozharness['installer-url']
else:
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
mozharness_url = get_artifact_url(
'<build>', get_artifact_path(taskdesc, 'mozharness.zip'))
worker['artifacts'] = [{
'name':
prefix,
'path':
os.path.join('{workdir}/workspace'.format(**run), path),
'type':
'directory',
} for (prefix, path) in artifacts]
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
'NEED_PULSEAUDIO': 'true',
'NEED_WINDOW_MANAGER': 'true',
'ENABLE_E10S': text_type(bool(test.get('e10s'))).lower(),
'WORKING_DIR': '/builds/worker',
})
# Legacy linux64 tests rely on compiz.
if test.get('docker-image', {}).get('in-tree') == 'desktop1604-test':
env.update({'NEED_COMPIZ': 'true'})
# Bug 1602701/1601828 - use compiz on ubuntu1804 due to GTK asynchiness
# when manipulating windows.
if test.get('docker-image', {}).get('in-tree') == 'ubuntu1804-test':
if ('wdspec' in job['run']['test']['suite']
or ('marionette' in job['run']['test']['suite']
and 'headless' not in job['label'])):
env.update({'NEED_COMPIZ': 'true'})
if mozharness.get('mochitest-flavor'):
env['MOCHITEST_FLAVOR'] = mozharness['mochitest-flavor']
if mozharness['set-moz-node-path']:
env['MOZ_NODE_PATH'] = '/usr/local/bin/node'
if 'actions' in mozharness:
env['MOZHARNESS_ACTIONS'] = ' '.join(mozharness['actions'])
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
# handle some of the mozharness-specific options
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
# Support vcs checkouts regardless of whether the task runs from
# source or not in case it is needed on an interactive loaner.
support_vcs_checkout(config, job, taskdesc)
# If we have a source checkout, run mozharness from it instead of
# downloading a zip file with the same content.
if test['checkout']:
env['MOZHARNESS_PATH'] = '{workdir}/checkouts/gecko/testing/mozharness'.format(
**run)
else:
env['MOZHARNESS_URL'] = {'task-reference': mozharness_url}
extra_config = {
'installer_url': installer_url,
'test_packages_url': test_packages_url(taskdesc),
}
env['EXTRA_MOZHARNESS_CONFIG'] = {
'task-reference': six.ensure_text(json.dumps(extra_config))
}
command = [
'{workdir}/bin/test-linux.sh'.format(**run),
]
command.extend(mozharness.get('extra-options', []))
if test.get('test-manifests'):
env['MOZHARNESS_TEST_PATHS'] = six.ensure_text(
json.dumps({test['suite']: test['test-manifests']}))
# TODO: remove the need for run['chunked']
elif mozharness.get('chunked') or test['chunks'] > 1:
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
job['run'] = {
'workdir': run['workdir'],
'tooltool-downloads': mozharness['tooltool-downloads'],
'checkout': test['checkout'],
'command': command,
'using': 'run-task',
}
configure_taskdesc_for_run(config, job, taskdesc, worker['implementation'])
