def load_parameters_file(options):
"""
Load parameters from the --parameters option
"""
import urllib
from taskgraph.util.taskcluster import get_artifact_url
filename = options['parameters']
if not filename:
return Parameters()
try:
# reading parameters from a local parameters.yml file
f = open(filename)
except IOError:
# fetching parameters.yml using task task-id or supplied url
if filename.startswith("task-id="):
task_id = filename.split("=")[1]
filename = get_artifact_url(task_id, 'public/parameters.yml')
f = urllib.urlopen(filename)
if filename.endswith('.yml'):
return Parameters(**yaml.safe_load(f))
elif filename.endswith('.json'):
return Parameters(**json.load(f))
else:
raise TypeError("Parameters file `{}` is not JSON or YAML".format(filename))
def load_image_by_task_id(task_id, tag=None):
artifact_url = get_artifact_url(task_id, 'public/image.tar.zst')
result = load_image(artifact_url, tag)
print("Found docker image: {}:{}".format(result['image'], result['tag']))
if tag:
print("Re-tagged as: {}".format(tag))
else:
tag = '{}:{}'.format(result['image'], result['tag'])
print("Try: docker run -ti --rm {} bash".format(tag))
return True
def artifact_urls(self, tree, job, rev, download_symbols):
try:
artifact_job = get_job_details(job, log=self._log, download_symbols=download_symbols)
except KeyError:
self.log(logging.INFO, 'artifact',
{'job': job},
'Unknown job {job}')
raise KeyError("Unknown job")
# Grab the second part of the repo name, which is generally how things
# are indexed. Eg: 'integration/mozilla-inbound' is indexed as
# 'mozilla-inbound'
tree = tree.split('/')[1] if '/' in tree else tree
namespace = 'gecko.v2.{tree}.revision.{rev}.{product}.{job}'.format(
rev=rev,
tree=tree,
product=artifact_job.product,
job=job,
)
self.log(logging.DEBUG, 'artifact',
{'namespace': namespace},
'Searching Taskcluster index with namespace: {namespace}')
try:
taskId = find_task_id(namespace)
except Exception:
# Not all revisions correspond to pushes that produce the job we
# care about; and even those that do may not have completed yet.
raise ValueError('Task for {namespace} does not exist (yet)!'.format(namespace=namespace))
artifacts = list_artifacts(taskId)
urls = []
for artifact_name in artifact_job.find_candidate_artifacts(artifacts):
# We can easily extract the task ID from the URL. We can't easily
# extract the build ID; we use the .ini files embedded in the
# downloaded artifact for this. We could also use the uploaded
# public/build/buildprops.json for this purpose.
url = get_artifact_url(taskId, artifact_name)
urls.append(url)
if not urls:
raise ValueError('Task for {namespace} existed, but no artifacts found!'.format(namespace=namespace))
return urls
def add_task_info(config, jobs):
for job in jobs:
dep_task = job['dependent-task']
del job['dependent-task']
# Add a dependency on the build task.
job['dependencies'] = {'build': dep_task.label}
# Label the job to match the build task it's uploading from.
job['label'] = dep_task.label.replace("build-", "upload-generated-sources-")
# Copy over some bits of metdata from the build task.
dep_th = dep_task.task['extra']['treeherder']
job.setdefault('attributes', {})
job['attributes']['build_platform'] = dep_task.attributes.get('build_platform')
plat = '{}/{}'.format(dep_th['machine']['platform'], dep_task.attributes.get('build_type'))
job['treeherder']['platform'] = plat
job['treeherder']['tier'] = dep_th['tier']
# Add an environment variable pointing at the artifact from the build.
artifact_url = get_artifact_url('<build>',
'public/build/target.generated-files.tar.gz')
job['worker'].setdefault('env', {})['ARTIFACT_URL'] = {
'task-reference': artifact_url
}
yield job
def load_parameters_file(filename, strict=True):
"""
Load parameters from a path, url, decision task-id or project.
Examples:
task-id=fdtgsD5DQUmAQZEaGMvQ4Q
project=mozilla-central
"""
import urllib
from taskgraph.util.taskcluster import get_artifact_url, find_task_id
if not filename:
return Parameters(strict=strict)
try:
# reading parameters from a local parameters.yml file
f = open(filename)
except IOError:
# fetching parameters.yml using task task-id, project or supplied url
task_id = None
if filename.startswith("task-id="):
task_id = filename.split("=")[1]
elif filename.startswith("project="):
index = "gecko.v2.{}.latest.firefox.decision".format(filename.split("=")[1])
task_id = find_task_id(index)
if task_id:
filename = get_artifact_url(task_id, 'public/parameters.yml')
f = urllib.urlopen(filename)
if filename.endswith('.yml'):
return Parameters(strict=strict, **yaml.safe_load(f))
elif filename.endswith('.json'):
return Parameters(strict=strict, **json.load(f))
else:
raise TypeError("Parameters file `{}` is not JSON or YAML".format(filename))
def test_packages_url(taskdesc):
"""Account for different platforms that name their test packages differently"""
return get_artifact_url(
'<build>', get_artifact_path(taskdesc, 'target.test_packages.json'))
def mozharness_test_on_native_engine(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
is_talos = test['suite'] == 'talos' or test['suite'] == 'raptor'
is_macosx = worker['os'] == 'macosx'
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
mozharness_url = get_artifact_url(
'<build>', get_artifact_path(taskdesc, 'mozharness.zip'))
worker['artifacts'] = [
{
'name': prefix.rstrip('/'),
'path': path.rstrip('/'),
'type': 'directory',
} for (prefix, path) in [
# (artifact name prefix, in-image path relative to homedir)
("public/logs/", "workspace/build/upload/logs/"),
("public/test", "artifacts/"),
("public/test_info/", "workspace/build/blobber_upload_dir/"),
]
]
if test['reboot']:
worker['reboot'] = test['reboot']
if test['max-run-time']:
worker['max-run-time'] = test['max-run-time']
env = worker.setdefault('env', {})
env.update({
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZHARNESS_URL': {
'task-reference': mozharness_url
},
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
"MOZ_NO_REMOTE": '1',
"XPCOM_DEBUG_BREAK": 'warn',
"NO_FAIL_ON_TEST_ERRORS": '1',
"MOZ_HIDE_RESULTS_TABLE": '1',
"MOZ_NODE_PATH": "/usr/local/bin/node",
'MOZ_AUTOMATION': '1',
})
# talos tests don't need Xvfb
if is_talos:
env['NEED_XVFB'] = 'false'
script = 'test-macosx.sh' if is_macosx else 'test-linux.sh'
worker['context'] = '{}/raw-file/{}/taskcluster/scripts/tester/{}'.format(
config.params['head_repository'], config.params['head_rev'], script)
command = worker['command'] = ["./{}".format(script)]
command.extend([
{
"task-reference": "--installer-url=" + installer_url
},
{
"task-reference":
"--test-packages-url=" + test_packages_url(taskdesc)
},
])
if mozharness.get('include-blob-upload-branch'):
command.append('--blob-upload-branch=' + config.params['project'])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
command[i] += suffix
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
def fill_template(config, tasks):
dummy_tasks = {}
for task in tasks:
name = task['name']
deps = {}
urls = {}
previous_artifact = None
for k in ('original', 'new'):
value = task[k]
if isinstance(value, basestring):
deps[k] = value
task_id = '<{}>'.format(k)
os_hint = value
else:
index = value['index-search']
if index not in dummy_tasks:
dummy_tasks[index] = {
'label': 'index-search-' + index,
'description': index,
'worker-type': 'invalid/always-optimized',
'run': {
'using': 'always-optimized',
},
'optimization': {
'index-search': [index],
}
}
yield dummy_tasks[index]
deps[index] = 'index-search-' + index
task_id = '<{}>'.format(index)
os_hint = index.split('.')[-1]
if 'linux' in os_hint:
artifact = 'target.tar.bz2'
elif 'macosx' in os_hint:
artifact = 'target.dmg'
elif 'android' in os_hint:
artifact = 'target.apk'
elif 'win' in os_hint:
artifact = 'target.zip'
else:
raise Exception(
'Cannot figure out the OS for {!r}'.format(value))
if previous_artifact is not None and previous_artifact != artifact:
raise Exception('Cannot compare builds from different OSes')
url = get_artifact_url(task_id, get_artifact_path(task, artifact))
urls[k] = {'task-reference': url}
previous_artifact = artifact
taskdesc = {
'label':
'diff-' + name,
'description':
name,
'treeherder': {
'symbol': task['symbol'],
'platform': 'diff/opt',
'kind': 'other',
'tier': 2,
},
'worker-type':
'aws-provisioner-v1/gecko-{}-b-linux'.format(
config.params['level']),
'worker': {
'docker-image': {
'in-tree': 'diffoscope'
},
'artifacts': [{
'type': 'file',
'path': '/builds/worker/diff.html',
'name': 'public/diff.html',
}, {
'type': 'file',
'path': '/builds/worker/diff.txt',
'name': 'public/diff.txt',
}],
'env': {
'ORIG_URL':
urls['original'],
'NEW_URL':
urls['new'],
'DIFFOSCOPE_ARGS':
' '.join(task[k] for k in ('args', 'extra-args')
if k in task)
},
'max-run-time':
1800,
},
'run': {
'using':
'run-task',
'checkout':
False,
'command':
'/builds/worker/bin/get_and_diffoscope '
'"$ORIG_URL" "$NEW_URL"',
},
'dependencies':
deps,
}
if artifact.endswith('.dmg'):
taskdesc['toolchains'] = [
'linux64-cctools-port',
'linux64-libdmg',
]
yield taskdesc
def mozharness_test_on_docker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
# apply some defaults
worker['docker-image'] = test['docker-image']
worker['allow-ptrace'] = True # required for all tests, for crashreporter
worker['loopback-video'] = test['loopback-video']
worker['loopback-audio'] = test['loopback-audio']
worker['max-run-time'] = test['max-run-time']
worker['retry-exit-status'] = test['retry-exit-status']
artifacts = [
# (artifact name prefix, in-image path)
("public/logs/", "/builds/worker/workspace/build/upload/logs/"),
("public/test", "/builds/worker/artifacts/"),
("public/test_info/", "/builds/worker/workspace/build/blobber_upload_dir/"),
]
installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
mozharness_url = get_artifact_url('<build>',
'public/build/mozharness.zip')
worker['artifacts'] = [{
'name': prefix,
'path': os.path.join('/builds/worker/workspace', path),
'type': 'directory',
} for (prefix, path) in artifacts]
worker['caches'] = [{
'type': 'persistent',
'name': 'level-{}-{}-test-workspace'.format(
config.params['level'], config.params['project']),
'mount-point': "/builds/worker/workspace",
}]
env = worker['env'] = {
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZILLA_BUILD_URL': {'task-reference': installer_url},
'NEED_PULSEAUDIO': 'true',
'NEED_WINDOW_MANAGER': 'true',
'ENABLE_E10S': str(bool(test.get('e10s'))).lower(),
'MOZ_AUTOMATION': '1',
}
if mozharness.get('mochitest-flavor'):
env['MOCHITEST_FLAVOR'] = mozharness['mochitest-flavor']
if mozharness['set-moz-node-path']:
env['MOZ_NODE_PATH'] = '/usr/local/bin/node'
if 'actions' in mozharness:
env['MOZHARNESS_ACTIONS'] = ' '.join(mozharness['actions'])
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
# handle some of the mozharness-specific options
if mozharness['tooltool-downloads']:
docker_worker_add_tooltool(config, job, taskdesc, internal=True)
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(test['reboot']))
# assemble the command line
command = [
'/builds/worker/bin/run-task',
]
# Support vcs checkouts regardless of whether the task runs from
# source or not in case it is needed on an interactive loaner.
support_vcs_checkout(config, job, taskdesc)
# If we have a source checkout, run mozharness from it instead of
# downloading a zip file with the same content.
if test['checkout']:
command.extend(['--vcs-checkout', '/builds/worker/checkouts/gecko'])
env['MOZHARNESS_PATH'] = '/builds/worker/checkouts/gecko/testing/mozharness'
else:
env['MOZHARNESS_URL'] = {'task-reference': mozharness_url}
command.extend([
'--',
'/builds/worker/bin/test-linux.sh',
])
if mozharness.get('no-read-buildbot-config'):
command.append("--no-read-buildbot-config")
command.extend([
{"task-reference": "--installer-url=" + installer_url},
{"task-reference": "--test-packages-url=" + test_packages_url(taskdesc)},
])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
command[i] += suffix
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
worker['command'] = command
def mozharness_test_buildbot_bridge(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
branch = config.params['project']
build_platform, build_type = test['build-platform'].split('/')
test_platform = test['test-platform'].split('/')[0]
test_name = test.get('try-name', test['test-name'])
mozharness = test['mozharness']
# mochitest e10s follows the pattern mochitest-e10s-<suffix>
# in buildbot, except for these special cases
buildbot_specials = [
'mochitest-webgl',
'mochitest-clipboard',
'mochitest-media',
'mochitest-gpu',
'mochitest-e10s',
]
test_name = test.get('try-name', test['test-name'])
if test['e10s'] and 'e10s' not in test_name:
test_name += '-e10s'
if test_name.startswith('mochitest') \
and test_name.endswith('e10s') \
and not any(map(
lambda name: test_name.startswith(name),
buildbot_specials
)):
split_mochitest = test_name.split('-')
test_name = '-'.join([
split_mochitest[0],
split_mochitest[-1],
'-'.join(split_mochitest[1:-1])
])
# in buildbot, mochitest-webgl is called mochitest-gl
test_name = test_name.replace('webgl', 'gl')
if mozharness.get('chunked', False):
this_chunk = test.get('this-chunk')
test_name = '{}-{}'.format(test_name, this_chunk)
elif test.get('this-chunk', 1) != 1:
raise Exception("Unexpected chunking when 'chunked' attribute is 'false'"
" for {}".format(test_name))
if test.get('suite', '') == 'talos':
variant = get_variant(test['test-platform'])
# On beta and release, we run nightly builds on-push; the talos
# builders need to run against non-nightly buildernames
if variant == 'nightly':
variant = ''
# this variant name has branch after the variant type in BBB bug 1338871
if variant in ('qr', 'stylo', 'stylo-sequential', 'devedition', 'stylo-disabled'):
name = '{prefix} {variant} {branch} talos {test_name}'
elif variant:
name = '{prefix} {branch} {variant} talos {test_name}'
else:
name = '{prefix} {branch} talos {test_name}'
buildername = name.format(
prefix=BUILDER_NAME_PREFIX[test_platform],
variant=variant,
branch=branch,
test_name=test_name
)
if buildername.startswith('Ubuntu'):
buildername = buildername.replace('VM', 'HW')
else:
variant = get_variant(test['test-platform'])
# If we are a pgo type, munge the build_type for the
# Unittest builder name generation
if 'pgo' in variant:
build_type = variant
prefix = BUILDER_NAME_PREFIX.get(
(test_platform, test.get('virtualization')),
BUILDER_NAME_PREFIX[test_platform])
if variant in ['stylo-disabled']:
buildername = '{prefix} {variant} {branch} {build_type} test {test_name}'.format(
prefix=prefix,
variant=variant,
branch=branch,
build_type=build_type,
test_name=test_name
)
else:
buildername = '{prefix} {branch} {build_type} test {test_name}'.format(
prefix=prefix,
branch=branch,
build_type=build_type,
test_name=test_name
)
worker.update({
'buildername': buildername,
'sourcestamp': {
'branch': branch,
'repository': config.params['head_repository'],
'revision': config.params['head_rev'],
},
'properties': {
'product': test.get('product', 'firefox'),
'who': config.params['owner'],
'installer_path': mozharness['build-artifact-name'],
}
})
if mozharness['requires-signed-builds']:
upstream_task = '<build-signing>'
installer_url = get_artifact_url(upstream_task, mozharness['build-artifact-name'])
worker['properties']['signed_installer_url'] = {'task-reference': installer_url}
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
is_macosx = worker['os'] == 'macosx'
is_windows = worker['os'] == 'windows'
assert is_macosx or is_windows
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
upstream_task = '<build-signing>' if mozharness['requires-signed-builds'] else '<build>'
installer_url = get_artifact_url(upstream_task, mozharness['build-artifact-name'])
taskdesc['scopes'].extend(
['generic-worker:os-group:{}'.format(group) for group in test['os-groups']])
worker['os-groups'] = test['os-groups']
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(test['reboot']))
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
env = worker.setdefault('env', {})
env['MOZ_AUTOMATION'] = '1'
env['GECKO_HEAD_REPOSITORY'] = config.params['head_repository']
env['GECKO_HEAD_REV'] = config.params['head_rev']
# this list will get cleaned up / reduced / removed in bug 1354088
if is_macosx:
env.update({
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_EM_RESTART': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0',
})
if is_macosx:
mh_command = [
'python2.7',
'-u',
'mozharness/scripts/' + mozharness['script']
]
elif is_windows:
mh_command = [
'c:\\mozilla-build\\python\\python.exe',
'-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if is_windows:
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url(taskdesc)])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
mh_command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': 'public/build/mozharness.zip',
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if is_windows:
worker['command'] = [
{'task-reference': ' '.join(mh_command)}
]
else: # is_macosx
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [
mh_command_task_ref
]
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
build_platform = taskdesc['attributes']['build_platform']
build_type = taskdesc['attributes']['build_type']
if build_platform == 'macosx64' and build_type == 'opt':
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac')
else:
target = 'target'
installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
test_packages_url = get_artifact_url(
'<build>', 'public/build/{}.test_packages.json'.format(target))
taskdesc['scopes'].extend(
['generic-worker:os-group:{}'.format(group) for group in test['os-groups']])
worker['os-groups'] = test['os-groups']
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
# this list will get cleaned up / reduced / removed in bug 1354088
if build_platform.startswith('macosx'):
worker['env'] = {
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_EM_RESTART': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0'
}
if build_platform.startswith('macosx'):
mh_command = [
'python2.7',
'-u',
'mozharness/scripts/' + mozharness['script']
]
elif build_platform.startswith('win'):
mh_command = [
'c:\\mozilla-build\\python\\python.exe',
'-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
else:
mh_command = [
'python',
'-u',
'mozharness/scripts/' + mozharness['script']
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if build_platform.startswith('win'):
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
mh_command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': 'public/build/mozharness.zip',
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if build_platform.startswith('win'):
worker['command'] = [
{'task-reference': ' '.join(mh_command)}
]
else:
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [
mh_command_task_ref
]
def mozharness_test_on_docker(config, job, taskdesc):
run = job['run']
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker'] = job['worker']
# apply some defaults
worker['docker-image'] = test['docker-image']
worker['allow-ptrace'] = True # required for all tests, for crashreporter
worker['loopback-video'] = test['loopback-video']
worker['loopback-audio'] = test['loopback-audio']
worker['max-run-time'] = test['max-run-time']
worker['retry-exit-status'] = test['retry-exit-status']
if 'android-em-7.0-x86' in test['test-platform']:
worker['privileged'] = True
artifacts = [
# (artifact name prefix, in-image path)
("public/logs/", "{workdir}/workspace/logs/".format(**run)),
("public/test", "{workdir}/artifacts/".format(**run)),
("public/test_info/",
"{workdir}/workspace/build/blobber_upload_dir/".format(**run)),
]
if 'installer-url' in mozharness:
installer_url = mozharness['installer-url']
else:
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
mozharness_url = get_artifact_url(
'<build>', get_artifact_path(taskdesc, 'mozharness.zip'))
worker['artifacts'] = [{
'name':
prefix,
'path':
os.path.join('{workdir}/workspace'.format(**run), path),
'type':
'directory',
} for (prefix, path) in artifacts]
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
'NEED_PULSEAUDIO': 'true',
'NEED_WINDOW_MANAGER': 'true',
'ENABLE_E10S': text_type(bool(test.get('e10s'))).lower(),
'WORKING_DIR': '/builds/worker',
})
# Legacy linux64 tests rely on compiz.
if test.get('docker-image', {}).get('in-tree') == 'desktop1604-test':
env.update({'NEED_COMPIZ': 'true'})
# Bug 1602701/1601828 - use compiz on ubuntu1804 due to GTK asynchiness
# when manipulating windows.
if test.get('docker-image', {}).get('in-tree') == 'ubuntu1804-test':
if ('wdspec' in job['run']['test']['suite']
or ('marionette' in job['run']['test']['suite']
and 'headless' not in job['label'])):
env.update({'NEED_COMPIZ': 'true'})
if mozharness.get('mochitest-flavor'):
env['MOCHITEST_FLAVOR'] = mozharness['mochitest-flavor']
if mozharness['set-moz-node-path']:
env['MOZ_NODE_PATH'] = '/usr/local/bin/node'
if 'actions' in mozharness:
env['MOZHARNESS_ACTIONS'] = ' '.join(mozharness['actions'])
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
# handle some of the mozharness-specific options
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
# Support vcs checkouts regardless of whether the task runs from
# source or not in case it is needed on an interactive loaner.
support_vcs_checkout(config, job, taskdesc)
# If we have a source checkout, run mozharness from it instead of
# downloading a zip file with the same content.
if test['checkout']:
env['MOZHARNESS_PATH'] = '{workdir}/checkouts/gecko/testing/mozharness'.format(
**run)
else:
env['MOZHARNESS_URL'] = {'task-reference': mozharness_url}
extra_config = {
'installer_url': installer_url,
'test_packages_url': test_packages_url(taskdesc),
}
env['EXTRA_MOZHARNESS_CONFIG'] = {
'task-reference': six.ensure_text(json.dumps(extra_config))
}
command = [
'{workdir}/bin/test-linux.sh'.format(**run),
]
command.extend(mozharness.get('extra-options', []))
if test.get('test-manifests'):
env['MOZHARNESS_TEST_PATHS'] = six.ensure_text(
json.dumps({test['suite']: test['test-manifests']}))
# TODO: remove the need for run['chunked']
elif mozharness.get('chunked') or test['chunks'] > 1:
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
job['run'] = {
'workdir': run['workdir'],
'tooltool-downloads': mozharness['tooltool-downloads'],
'checkout': test['checkout'],
'command': command,
'using': 'run-task',
}
configure_taskdesc_for_run(config, job, taskdesc, worker['implementation'])
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
build_platform = dep_job.attributes.get("build_platform")
treeherder = None
if "partner" not in config.kind and "eme-free" not in config.kind:
treeherder = job.get("treeherder", {})
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
build_type = dep_job.attributes.get("build_type")
treeherder.setdefault("platform",
"{}/{}".format(dep_th_platform, build_type))
dep_treeherder = dep_job.task.get("extra",
{}).get("treeherder", {})
treeherder.setdefault("tier", dep_treeherder.get("tier", 1))
treeherder.setdefault(
"symbol",
_generate_treeherder_symbol(
dep_treeherder.get("groupSymbol", "?"),
dep_treeherder.get("symbol")),
)
treeherder.setdefault("kind", "build")
label = dep_job.label.replace("part-1", "poller")
description = ("Mac Notarization Poller for build '"
"{build_platform}/{build_type}'".format(
build_platform=build_platform,
build_type=attributes.get("build_type")))
attributes = (job["attributes"] if job.get("attributes") else
copy_attributes_from_dependent_job(dep_job))
attributes["signed"] = True
if dep_job.attributes.get("chunk_locales"):
# Used for l10n attribute passthrough
attributes["chunk_locales"] = dep_job.attributes.get(
"chunk_locales")
uuid_manifest_url = get_artifact_url("<part1>",
"public/uuid_manifest.json")
task = {
"label": label,
"description": description,
"worker": {
"implementation": "notarization-poller",
"uuid-manifest": {
"task-reference": uuid_manifest_url
},
},
"worker-type": "mac-notarization-poller",
"dependencies": {
"part1": dep_job.label
},
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"optimization": dep_job.optimization,
"routes": job.get("routes", []),
"shipping-product": job.get("shipping-product"),
"shipping-phase": job.get("shipping-phase"),
}
if treeherder:
task["treeherder"] = treeherder
if job.get("extra"):
task["extra"] = job["extra"]
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get("priority"):
task["priority"] = job["priority"]
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
treeherder = None
if 'partner' not in config.kind and 'eme-free' not in config.kind:
treeherder = job.get('treeherder', {})
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
build_type = dep_job.attributes.get('build_type')
treeherder.setdefault(
'platform',
'{}/{}'.format(dep_th_platform, build_type)
)
dep_treeherder = dep_job.task.get('extra', {}).get('treeherder', {})
treeherder.setdefault('tier', dep_treeherder.get('tier', 1))
treeherder.setdefault('symbol', _generate_treeherder_symbol(
dep_treeherder.get('groupSymbol', '?'), dep_treeherder.get('symbol')
))
treeherder.setdefault('kind', 'build')
label = dep_job.label.replace('part-1', 'poller')
description = (
"Mac Notarization Poller for build '"
"{build_platform}/{build_type}'".format(
build_platform=build_platform,
build_type=attributes.get('build_type')
)
)
attributes = job['attributes'] if job.get('attributes') else \
copy_attributes_from_dependent_job(dep_job)
attributes['signed'] = True
if dep_job.attributes.get('chunk_locales'):
# Used for l10n attribute passthrough
attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
uuid_manifest_url = get_artifact_url('<part1>', 'public/uuid_manifest.json')
task = {
'label': label,
'description': description,
'worker': {
'implementation': 'notarization-poller',
'uuid-manifest': {'task-reference': uuid_manifest_url},
},
'worker-type': 'mac-notarization-poller',
'dependencies': {'part1': dep_job.label},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'optimization': dep_job.optimization,
'routes': job.get('routes', []),
'shipping-product': job.get('shipping-product'),
'shipping-phase': job.get('shipping-phase'),
}
if treeherder:
task['treeherder'] = treeherder
if job.get('extra'):
task['extra'] = job['extra']
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get('priority'):
task['priority'] = job['priority']
yield task
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
artifacts = [
{
'name': 'public/logs/localconfig.json',
'path': 'logs/localconfig.json',
'type': 'file'
},
{
'name': 'public/logs/log_critical.log',
'path': 'logs/log_critical.log',
'type': 'file'
},
{
'name': 'public/logs/log_error.log',
'path': 'logs/log_error.log',
'type': 'file'
},
{
'name': 'public/logs/log_fatal.log',
'path': 'logs/log_fatal.log',
'type': 'file'
},
{
'name': 'public/logs/log_info.log',
'path': 'logs/log_info.log',
'type': 'file'
},
{
'name': 'public/logs/log_raw.log',
'path': 'logs/log_raw.log',
'type': 'file'
},
{
'name': 'public/logs/log_warning.log',
'path': 'logs/log_warning.log',
'type': 'file'
},
{
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
}
]
build_platform = taskdesc['attributes']['build_platform']
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform) \
if build_platform.startswith('win') else 'target'
installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
test_packages_url = get_artifact_url(
'<build>', 'public/build/{}.test_packages.json'.format(target))
taskdesc['scopes'].extend(
['generic-worker:os-group:{}'.format(group) for group in test['os-groups']])
worker['os-groups'] = test['os-groups']
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
# this list will get cleaned up / reduced / removed in bug 1354088
if build_platform.startswith('macosx'):
worker['env'] = {
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_EM_RESTART': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0'
}
if build_platform.startswith('macosx'):
mh_command = [
'python2.7',
'-u',
'mozharness/scripts/' + mozharness['script']
]
elif build_platform.startswith('win'):
mh_command = [
'c:\\mozilla-build\\python\\python.exe',
'-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
else:
mh_command = [
'python',
'-u',
'mozharness/scripts/' + mozharness['script']
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if build_platform.startswith('win'):
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': 'public/build/mozharness.zip',
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if build_platform.startswith('win'):
worker['command'] = [
{'task-reference': ' '.join(mh_command)}
]
else:
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [
mh_command_task_ref
]
def mozharness_test_on_docker(config, job, taskdesc):
run = job['run']
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker'] = job['worker']
# apply some defaults
worker['docker-image'] = test['docker-image']
worker['allow-ptrace'] = True # required for all tests, for crashreporter
worker['loopback-video'] = test['loopback-video']
worker['loopback-audio'] = test['loopback-audio']
worker['max-run-time'] = test['max-run-time']
worker['retry-exit-status'] = test['retry-exit-status']
if 'android-em-7.0-x86' in test['test-platform']:
worker['privileged'] = True
artifacts = [
# (artifact name prefix, in-image path)
("public/logs/", "{workdir}/workspace/logs/".format(**run)),
("public/test", "{workdir}/artifacts/".format(**run)),
("public/test_info/",
"{workdir}/workspace/build/blobber_upload_dir/".format(**run)),
]
if 'installer-url' in mozharness:
installer_url = mozharness['installer-url']
else:
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
mozharness_url = get_artifact_url(
'<build>', get_artifact_path(taskdesc, 'mozharness.zip'))
worker['artifacts'] = [{
'name':
prefix,
'path':
os.path.join('{workdir}/workspace'.format(**run), path),
'type':
'directory',
} for (prefix, path) in artifacts]
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
'NEED_PULSEAUDIO': 'true',
'NEED_WINDOW_MANAGER': 'true',
'NEED_COMPIZ': 'true',
'ENABLE_E10S': str(bool(test.get('e10s'))).lower(),
'WORKING_DIR': '/builds/worker',
})
# by default, require compiz unless proven otherwise, hence a whitelist.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=1552563
# if using regex this list can be shortened greatly.
suites_not_need_compiz = [
'mochitest-webgl1-core', 'mochitest-webgl1-ext', 'mochitest-plain-gpu',
'mochitest-browser-chrome-screenshots', 'gtest', 'cppunittest',
'jsreftest', 'crashtest', 'reftest', 'reftest-no-accel',
'web-platform-tests', 'web-platform-tests-reftests', 'xpcshell'
]
if job['run']['test']['suite'] in suites_not_need_compiz or (
job['run']['test']['suite'] == 'mochitest-plain-chunked'
and job['run']['test']['try-name'] == 'mochitest-plain-headless'):
env['NEED_COMPIZ'] = 'false'
if mozharness.get('mochitest-flavor'):
env['MOCHITEST_FLAVOR'] = mozharness['mochitest-flavor']
if mozharness['set-moz-node-path']:
env['MOZ_NODE_PATH'] = '/usr/local/bin/node'
if 'actions' in mozharness:
env['MOZHARNESS_ACTIONS'] = ' '.join(mozharness['actions'])
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
# handle some of the mozharness-specific options
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
# Support vcs checkouts regardless of whether the task runs from
# source or not in case it is needed on an interactive loaner.
support_vcs_checkout(config, job, taskdesc)
# If we have a source checkout, run mozharness from it instead of
# downloading a zip file with the same content.
if test['checkout']:
env['MOZHARNESS_PATH'] = '{workdir}/checkouts/gecko/testing/mozharness'.format(
**run)
else:
env['MOZHARNESS_URL'] = {'task-reference': mozharness_url}
extra_config = {
'installer_url': installer_url,
'test_packages_url': test_packages_url(taskdesc),
}
env['EXTRA_MOZHARNESS_CONFIG'] = {
'task-reference': json.dumps(extra_config)
}
command = [
'{workdir}/bin/test-linux.sh'.format(**run),
]
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
job['run'] = {
'workdir': run['workdir'],
'tooltool-downloads': mozharness['tooltool-downloads'],
'checkout': test['checkout'],
'command': command,
'using': 'run-task',
}
configure_taskdesc_for_run(config, job, taskdesc, worker['implementation'])
def mozharness_test_on_script_engine_autophone(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
is_talos = test['suite'] == 'talos' or test['suite'] == 'raptor'
if worker['os'] != 'linux':
raise Exception(
'os: {} not supported on script-engine-autophone'.format(
worker['os']))
if 'installer-url' in mozharness:
installer_url = mozharness['installer-url']
else:
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
mozharness_url = get_artifact_url('<build>', 'public/build/mozharness.zip')
artifacts = [
# (artifact name prefix, in-image path)
("public/test/", "/builds/worker/artifacts"),
("public/logs/", "/builds/worker/workspace/build/logs"),
("public/test_info/",
"/builds/worker/workspace/build/blobber_upload_dir"),
]
worker['artifacts'] = [{
'name': prefix,
'path': path,
'type': 'directory',
} for (prefix, path) in artifacts]
if test['reboot']:
worker['reboot'] = test['reboot']
worker['env'] = env = {
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZHARNESS_URL': {
'task-reference': mozharness_url
},
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
"MOZ_NO_REMOTE": '1',
"XPCOM_DEBUG_BREAK": 'warn',
"NO_FAIL_ON_TEST_ERRORS": '1',
"MOZ_HIDE_RESULTS_TABLE": '1',
"MOZ_NODE_PATH": "/usr/local/bin/node",
'WORKING_DIR': '/builds/worker',
'WORKSPACE': '/builds/worker/workspace',
'TASKCLUSTER_WORKER_TYPE': job['worker-type'],
}
# for fetch tasks on mobile
if 'env' in job['worker'] and 'MOZ_FETCHES' in job['worker']['env']:
env['MOZ_FETCHES'] = job['worker']['env']['MOZ_FETCHES']
env['MOZ_FETCHES_DIR'] = job['worker']['env']['MOZ_FETCHES_DIR']
# talos tests don't need Xvfb
if is_talos:
env['NEED_XVFB'] = 'false'
extra_config = {
'installer_url': installer_url,
'test_packages_url': test_packages_url(taskdesc),
}
env['EXTRA_MOZHARNESS_CONFIG'] = {
'task-reference': six.ensure_text(json.dumps(extra_config))
}
script = 'test-linux.sh'
worker['context'] = config.params.file_url(
'taskcluster/scripts/tester/{}'.format(script), )
command = worker['command'] = ["./{}".format(script)]
if mozharness.get('include-blob-upload-branch'):
command.append('--blob-upload-branch=' + config.params['project'])
command.extend(mozharness.get('extra-options', []))
if test.get('test-manifests'):
env['MOZHARNESS_TEST_PATHS'] = six.ensure_text(
json.dumps({test['suite']: test['test-manifests']}))
# TODO: remove the need for run['chunked']
elif mozharness.get('chunked') or test['chunks'] > 1:
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
def mozharness_test_on_native_engine(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
build_platform = taskdesc['attributes']['build_platform']
build_type = taskdesc['attributes']['build_type']
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') \
if build_platform == 'macosx64' and build_type == 'opt' else 'target'
installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
test_packages_url = get_artifact_url('<build>',
'public/build/{}.test_packages.json'.format(target))
mozharness_url = get_artifact_url('<build>',
'public/build/mozharness.zip')
worker['artifacts'] = [{
'name': prefix.rstrip('/'),
'path': path.rstrip('/'),
'type': 'directory',
} for (prefix, path) in ARTIFACTS]
worker['reboot'] = test['reboot']
worker['env'] = {
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZHARNESS_URL': {'task-reference': mozharness_url},
'MOZILLA_BUILD_URL': {'task-reference': installer_url},
"MOZ_NO_REMOTE": '1',
"NO_EM_RESTART": '1',
"XPCOM_DEBUG_BREAK": 'warn',
"NO_FAIL_ON_TEST_ERRORS": '1',
"MOZ_HIDE_RESULTS_TABLE": '1',
"MOZ_NODE_PATH": "/usr/local/bin/node",
}
worker['context'] = '{}/raw-file/{}/taskcluster/scripts/tester/test-macosx.sh'.format(
config.params['head_repository'], config.params['head_rev']
)
command = worker['command'] = ["./test-macosx.sh"]
if mozharness.get('no-read-buildbot-config'):
command.append("--no-read-buildbot-config")
command.extend([
{"task-reference": "--installer-url=" + installer_url},
{"task-reference": "--test-packages-url=" + test_packages_url},
])
if mozharness.get('include-blob-upload-branch'):
command.append('--blob-upload-branch=' + config.params['project'])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
command[i] += suffix
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
def mozharness_test_on_windows(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
artifacts = [{
'path': 'public\\logs\\localconfig.json',
'type': 'file'
}, {
'path': 'public\\logs\\log_critical.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_error.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_fatal.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_info.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_raw.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_warning.log',
'type': 'file'
}, {
'path': 'public\\test_info',
'type': 'directory'
}]
build_platform = taskdesc['attributes']['build_platform']
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(),
build_platform)
installer_url = get_artifact_url('<build>',
'public/build/{}.zip'.format(target))
test_packages_url = get_artifact_url(
'<build>', 'public/build/{}.test_packages.json'.format(target))
mozharness_url = get_artifact_url('<build>', 'public/build/mozharness.zip')
taskdesc['scopes'].extend([
'generic-worker:os-group:{}'.format(group)
for group in test['os-groups']
])
worker['os-groups'] = test['os-groups']
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
# assemble the command line
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
for mh_config in mozharness['config']:
mh_command.extend(
['--cfg', 'mozharness\\configs\\' + normpath(mh_config)])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
# bug 1311966 - symlink to artifacts until generic worker supports virtual artifact paths
artifact_link_commands = [
'mklink /d %cd%\\public\\test_info %cd%\\build\\blobber_upload_dir'
]
for link in [
a['path'] for a in artifacts
if a['path'].startswith('public\\logs\\')
]:
artifact_link_commands.append('mklink %cd%\\{} %cd%\\{}'.format(
link, link[7:]))
worker['command'] = artifact_link_commands + [{
'task-reference':
'c:\\mozilla-build\\wget\\wget.exe {}'.format(mozharness_url)
}, 'c:\\mozilla-build\\info-zip\\unzip.exe mozharness.zip', {
'task-reference':
' '.join(mh_command)
}]
def mozharness_test_on_native_engine(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
is_talos = test['suite'] == 'talos'
is_macosx = worker['os'] == 'macosx'
installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
mozharness_url = get_artifact_url('<build>',
'public/build/mozharness.zip')
worker['artifacts'] = [{
'name': prefix.rstrip('/'),
'path': path.rstrip('/'),
'type': 'directory',
} for (prefix, path) in [
# (artifact name prefix, in-image path relative to homedir)
("public/logs/", "workspace/build/upload/logs/"),
("public/test", "artifacts/"),
("public/test_info/", "workspace/build/blobber_upload_dir/"),
]]
if test['reboot']:
worker['reboot'] = test['reboot']
worker['env'] = env = {
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZHARNESS_URL': {'task-reference': mozharness_url},
'MOZILLA_BUILD_URL': {'task-reference': installer_url},
"MOZ_NO_REMOTE": '1',
"NO_EM_RESTART": '1',
"XPCOM_DEBUG_BREAK": 'warn',
"NO_FAIL_ON_TEST_ERRORS": '1',
"MOZ_HIDE_RESULTS_TABLE": '1',
"MOZ_NODE_PATH": "/usr/local/bin/node",
'MOZ_AUTOMATION': '1',
}
# talos tests don't need Xvfb
if is_talos:
env['NEED_XVFB'] = 'false'
script = 'test-macosx.sh' if is_macosx else 'test-linux.sh'
worker['context'] = '{}/raw-file/{}/taskcluster/scripts/tester/{}'.format(
config.params['head_repository'], config.params['head_rev'], script
)
command = worker['command'] = ["./{}".format(script)]
if mozharness.get('no-read-buildbot-config'):
command.append("--no-read-buildbot-config")
command.extend([
{"task-reference": "--installer-url=" + installer_url},
{"task-reference": "--test-packages-url=" + test_packages_url(taskdesc)},
])
if mozharness.get('include-blob-upload-branch'):
command.append('--blob-upload-branch=' + config.params['project'])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
command[i] += suffix
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
def mozharness_test_on_native_engine(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
test_packages_url = get_artifact_url(
'<build>', 'public/build/target.test_packages.json')
mozharness_url = get_artifact_url('<build>', 'public/build/mozharness.zip')
worker['artifacts'] = [{
'name': prefix.rstrip('/'),
'path': path.rstrip('/'),
'type': 'directory',
} for (prefix, path) in ARTIFACTS]
worker['reboot'] = test['reboot']
worker['env'] = {
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZHARNESS_URL': {
'task-reference': mozharness_url
},
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
"MOZ_NO_REMOTE": '1',
"NO_EM_RESTART": '1',
"XPCOM_DEBUG_BREAK": 'warn',
"NO_FAIL_ON_TEST_ERRORS": '1',
"MOZ_HIDE_RESULTS_TABLE": '1',
"MOZ_NODE_PATH": "/usr/local/bin/node",
}
worker[
'context'] = '{}/raw-file/{}/taskcluster/scripts/tester/test-macosx.sh'.format(
config.params['head_repository'], config.params['head_rev'])
command = worker['command'] = ["./test-macosx.sh"]
if mozharness.get('no-read-buildbot-config'):
command.append("--no-read-buildbot-config")
command.extend([
{
"task-reference": "--installer-url=" + installer_url
},
{
"task-reference": "--test-packages-url=" + test_packages_url
},
])
if mozharness.get('include-blob-upload-branch'):
command.append('--blob-upload-branch=' + config.params['project'])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
command[i] += suffix
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
def test_packages_url(taskdesc):
"""Account for different platforms that name their test packages differently"""
return get_artifact_url('<build>', 'public/build/target.test_packages.json')
def mozharness_test_on_docker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
artifacts = [
# (artifact name prefix, in-image path)
("public/logs/", "/home/worker/workspace/build/upload/logs/"),
("public/test", "/home/worker/artifacts/"),
("public/test_info/",
"/home/worker/workspace/build/blobber_upload_dir/"),
]
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
test_packages_url = get_artifact_url(
'<build>', 'public/build/target.test_packages.json')
mozharness_url = get_artifact_url('<build>', 'public/build/mozharness.zip')
worker['artifacts'] = [{
'name': prefix,
'path': os.path.join('/home/worker/workspace', path),
'type': 'directory',
} for (prefix, path) in artifacts]
worker['caches'] = [{
'type':
'persistent',
'name':
'level-{}-{}-test-workspace'.format(config.params['level'],
config.params['project']),
'mount-point':
"/home/worker/workspace",
}]
env = worker['env'] = {
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
'NEED_PULSEAUDIO': 'true',
'NEED_WINDOW_MANAGER': 'true',
'ENABLE_E10S': str(bool(test.get('e10s'))).lower(),
}
if mozharness.get('mochitest-flavor'):
env['MOCHITEST_FLAVOR'] = mozharness['mochitest-flavor']
if mozharness['set-moz-node-path']:
env['MOZ_NODE_PATH'] = '/usr/local/bin/node'
if 'actions' in mozharness:
env['MOZHARNESS_ACTIONS'] = ' '.join(mozharness['actions'])
if config.params['project'] == 'try':
env['TRY_COMMIT_MSG'] = config.params['message']
# handle some of the mozharness-specific options
if mozharness['tooltool-downloads']:
worker['relengapi-proxy'] = True
worker['caches'].append({
'type': 'persistent',
'name': 'tooltool-cache',
'mount-point': '/home/worker/tooltool-cache',
})
taskdesc['scopes'].extend([
'docker-worker:relengapi-proxy:tooltool.download.internal',
'docker-worker:relengapi-proxy:tooltool.download.public',
])
# assemble the command line
command = [
'/home/worker/bin/run-task',
# The workspace cache/volume is default owned by root:root.
'--chown',
'/home/worker/workspace',
]
# Support vcs checkouts regardless of whether the task runs from
# source or not in case it is needed on an interactive loaner.
support_vcs_checkout(config, job, taskdesc)
# If we have a source checkout, run mozharness from it instead of
# downloading a zip file with the same content.
if test['checkout']:
command.extend(['--vcs-checkout', '/home/worker/checkouts/gecko'])
env['MOZHARNESS_PATH'] = '/home/worker/checkouts/gecko/testing/mozharness'
else:
env['MOZHARNESS_URL'] = {'task-reference': mozharness_url}
command.extend([
'--',
'/home/worker/bin/test-linux.sh',
])
if mozharness.get('no-read-buildbot-config'):
command.append("--no-read-buildbot-config")
command.extend([
{
"task-reference": "--installer-url=" + installer_url
},
{
"task-reference": "--test-packages-url=" + test_packages_url
},
])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
command[i] += suffix
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
worker['command'] = command
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
is_macosx = worker['os'] == 'macosx'
is_windows = worker['os'] == 'windows'
assert is_macosx or is_windows
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
upstream_task = '<build-signing>' if mozharness[
'requires-signed-builds'] else '<build>'
installer_url = get_artifact_url(upstream_task,
mozharness['build-artifact-name'])
taskdesc['scopes'].extend([
'generic-worker:os-group:{}'.format(group)
for group in test['os-groups']
])
worker['os-groups'] = test['os-groups']
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
env = worker.setdefault('env', {})
env['MOZ_AUTOMATION'] = '1'
# this list will get cleaned up / reduced / removed in bug 1354088
if is_macosx:
env.update({
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_EM_RESTART': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0',
})
if is_macosx:
mh_command = [
'python2.7', '-u', 'mozharness/scripts/' + mozharness['script']
]
elif is_windows:
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if is_windows:
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url(taskdesc)])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
mh_command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
if config.params['project'] == 'try':
env['TRY_COMMIT_MSG'] = config.params['message']
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': 'public/build/mozharness.zip',
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if is_windows:
worker['command'] = [{'task-reference': ' '.join(mh_command)}]
else: # is_macosx
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [mh_command_task_ref]
def mozharness_test_buildbot_bridge(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
branch = config.params['project']
build_platform, build_type = test['build-platform'].split('/')
test_platform = test['test-platform'].split('/')[0]
test_name = test.get('try-name', test['test-name'])
mozharness = test['mozharness']
# mochitest e10s follows the pattern mochitest-e10s-<suffix>
# in buildbot, except for these special cases
buildbot_specials = [
'mochitest-webgl',
'mochitest-clipboard',
'mochitest-media',
'mochitest-gpu',
'mochitest-e10s',
]
test_name = test.get('try-name', test['test-name'])
if test['e10s'] and 'e10s' not in test_name:
test_name += '-e10s'
if test_name.startswith('mochitest') \
and test_name.endswith('e10s') \
and not any(map(
lambda name: test_name.startswith(name),
buildbot_specials
)):
split_mochitest = test_name.split('-')
test_name = '-'.join([
split_mochitest[0], split_mochitest[-1],
'-'.join(split_mochitest[1:-1])
])
# in buildbot, mochitest-webgl is called mochitest-gl
test_name = test_name.replace('webgl', 'gl')
if mozharness.get('chunked', False):
this_chunk = test.get('this-chunk')
test_name = '{}-{}'.format(test_name, this_chunk)
elif test.get('this-chunk', 1) != 1:
raise Exception(
"Unexpected chunking when 'chunked' attribute is 'false'"
" for {}".format(test_name))
if test.get('suite', '') == 'talos':
variant = get_variant(test['test-platform'])
# On beta and release, we run nightly builds on-push; the talos
# builders need to run against non-nightly buildernames
if variant == 'nightly':
variant = ''
# this variant name has branch after the variant type in BBB bug 1338871
if variant in ('stylo', 'stylo-sequential', 'devedition'):
name = '{prefix} {variant} {branch} talos {test_name}'
elif variant:
name = '{prefix} {branch} {variant} talos {test_name}'
else:
name = '{prefix} {branch} talos {test_name}'
buildername = name.format(prefix=BUILDER_NAME_PREFIX[test_platform],
variant=variant,
branch=branch,
test_name=test_name)
if buildername.startswith('Ubuntu'):
buildername = buildername.replace('VM', 'HW')
else:
variant = get_variant(test['test-platform'])
# If we are a pgo type, munge the build_type for the
# Unittest builder name generation
if 'pgo' in variant:
build_type = variant
prefix = BUILDER_NAME_PREFIX.get(
(test_platform, test.get('virtualization')),
BUILDER_NAME_PREFIX[test_platform])
buildername = '{prefix} {branch} {build_type} test {test_name}'.format(
prefix=prefix,
branch=branch,
build_type=build_type,
test_name=test_name)
worker.update({
'buildername': buildername,
'sourcestamp': {
'branch': branch,
'repository': config.params['head_repository'],
'revision': config.params['head_rev'],
},
'properties': {
'product': test.get('product', 'firefox'),
'who': config.params['owner'],
'installer_path': mozharness['build-artifact-name'],
}
})
if mozharness['requires-signed-builds']:
upstream_task = '<build-signing>'
installer_url = get_artifact_url(upstream_task,
mozharness['build-artifact-name'])
worker['properties']['signed_installer_url'] = {
'task-reference': installer_url
}
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
is_macosx = worker['os'] == 'macosx'
is_windows = worker['os'] == 'windows'
is_linux = worker['os'] == 'linux'
assert is_macosx or is_windows or is_linux
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
upstream_task = '<build-signing>' if mozharness[
'requires-signed-builds'] else '<build>'
installer_url = get_artifact_url(upstream_task,
mozharness['build-artifact-name'])
taskdesc['scopes'].extend([
'generic-worker:os-group:{}/{}'.format(job['worker-type'], group)
for group in test['os-groups']
])
worker['os-groups'] = test['os-groups']
# run-as-administrator is a feature for workers with UAC enabled and as such should not be
# included in tasks on workers that have UAC disabled. Currently UAC is only enabled on
# gecko Windows 10 workers, however this may be subject to change. Worker type
# environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig
# See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control
# for more information about UAC.
if test.get('run-as-administrator', False):
if job['worker-type'].startswith(
'aws-provisioner-v1/gecko-t-win10-64'):
taskdesc['scopes'].extend([
'generic-worker:run-as-administrator:{}'.format(
job['worker-type'])
])
worker['run-as-administrator'] = True
else:
raise Exception('run-as-administrator not supported on {}'.format(
job['worker-type']))
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
env = worker.setdefault('env', {})
env['MOZ_AUTOMATION'] = '1'
env['GECKO_HEAD_REPOSITORY'] = config.params['head_repository']
env['GECKO_HEAD_REV'] = config.params['head_rev']
# this list will get cleaned up / reduced / removed in bug 1354088
if is_macosx:
env.update({
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0',
})
if is_windows:
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
else:
# is_linux or is_macosx
mh_command = [
'python2.7', '-u', 'mozharness/scripts/' + mozharness['script']
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if is_windows:
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url(taskdesc)])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
mh_command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': get_artifact_path(taskdesc, 'mozharness.zip'),
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if is_windows:
worker['command'] = [{'task-reference': ' '.join(mh_command)}]
else: # is_macosx
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [mh_command_task_ref]
def mozharness_test_on_docker(config, job, taskdesc):
run = job["run"]
test = taskdesc["run"]["test"]
mozharness = test["mozharness"]
worker = taskdesc["worker"] = job["worker"]
# apply some defaults
worker["docker-image"] = test["docker-image"]
worker["allow-ptrace"] = True # required for all tests, for crashreporter
worker["loopback-video"] = test["loopback-video"]
worker["loopback-audio"] = test["loopback-audio"]
worker["max-run-time"] = test["max-run-time"]
worker["retry-exit-status"] = test["retry-exit-status"]
if "android-em-7.0-x86" in test["test-platform"]:
worker["privileged"] = True
artifacts = [
# (artifact name prefix, in-image path)
("public/logs/", "{workdir}/workspace/logs/".format(**run)),
("public/test", "{workdir}/artifacts/".format(**run)),
(
"public/test_info/",
"{workdir}/workspace/build/blobber_upload_dir/".format(**run),
),
]
installer = installer_url(taskdesc)
mozharness_url = get_artifact_url(
"<build>", get_artifact_path(taskdesc, "mozharness.zip"))
worker.setdefault("artifacts", [])
worker["artifacts"].extend([{
"name":
prefix,
"path":
os.path.join("{workdir}/workspace".format(**run), path),
"type":
"directory",
} for (prefix, path) in artifacts])
env = worker.setdefault("env", {})
env.update({
"MOZHARNESS_CONFIG": " ".join(mozharness["config"]),
"MOZHARNESS_SCRIPT": mozharness["script"],
"MOZILLA_BUILD_URL": {
"task-reference": installer
},
"NEED_PULSEAUDIO": "true",
"NEED_WINDOW_MANAGER": "true",
"ENABLE_E10S": text_type(bool(test.get("e10s"))).lower(),
"WORKING_DIR": "/builds/worker",
})
if test.get("python-3"):
env["PYTHON"] = "python3"
# Legacy linux64 tests rely on compiz.
if test.get("docker-image", {}).get("in-tree") == "desktop1604-test":
env.update({"NEED_COMPIZ": "true"})
# Bug 1602701/1601828 - use compiz on ubuntu1804 due to GTK asynchiness
# when manipulating windows.
if test.get("docker-image", {}).get("in-tree") == "ubuntu1804-test":
if "wdspec" in job["run"]["test"]["suite"] or (
"marionette" in job["run"]["test"]["suite"]
and "headless" not in job["label"]):
env.update({"NEED_COMPIZ": "true"})
if mozharness.get("mochitest-flavor"):
env["MOCHITEST_FLAVOR"] = mozharness["mochitest-flavor"]
if mozharness["set-moz-node-path"]:
env["MOZ_NODE_PATH"] = "/usr/local/bin/node"
if "actions" in mozharness:
env["MOZHARNESS_ACTIONS"] = " ".join(mozharness["actions"])
if config.params.is_try():
env["TRY_COMMIT_MSG"] = config.params["message"]
# handle some of the mozharness-specific options
if test["reboot"]:
raise Exception("reboot: {} not supported on generic-worker".format(
test["reboot"]))
# Support vcs checkouts regardless of whether the task runs from
# source or not in case it is needed on an interactive loaner.
support_vcs_checkout(config, job, taskdesc)
# If we have a source checkout, run mozharness from it instead of
# downloading a zip file with the same content.
if test["checkout"]:
env["MOZHARNESS_PATH"] = "{workdir}/checkouts/gecko/testing/mozharness".format(
**run)
else:
env["MOZHARNESS_URL"] = {"task-reference": mozharness_url}
extra_config = {
"installer_url": installer,
"test_packages_url": test_packages_url(taskdesc),
}
env["EXTRA_MOZHARNESS_CONFIG"] = {
"task-reference":
six.ensure_text(json.dumps(extra_config, sort_keys=True))
}
# Bug 1634554 - pass in decision task artifact URL to mozharness for WPT.
# Bug 1645974 - test-verify-wpt and test-coverage-wpt need artifact URL.
if "web-platform-tests" in test["suite"] or re.match(
"test-(coverage|verify)-wpt", test["suite"]):
env["TESTS_BY_MANIFEST_URL"] = {
"artifact-reference": "<decision/public/tests-by-manifest.json.gz>"
}
command = [
"{workdir}/bin/test-linux.sh".format(**run),
]
command.extend(mozharness.get("extra-options", []))
if test.get("test-manifests"):
env["MOZHARNESS_TEST_PATHS"] = six.ensure_text(
json.dumps({test["suite"]: test["test-manifests"]},
sort_keys=True))
# TODO: remove the need for run['chunked']
elif mozharness.get("chunked") or test["chunks"] > 1:
command.append("--total-chunk={}".format(test["chunks"]))
command.append("--this-chunk={}".format(test["this-chunk"]))
if "download-symbols" in mozharness:
download_symbols = mozharness["download-symbols"]
download_symbols = {
True: "true",
False: "false"
}.get(download_symbols, download_symbols)
command.append("--download-symbols=" + download_symbols)
job["run"] = {
"workdir": run["workdir"],
"tooltool-downloads": mozharness["tooltool-downloads"],
"checkout": test["checkout"],
"command": command,
"using": "run-task",
}
configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"])
def mozharness_test_on_script_engine_autophone(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
is_talos = test['suite'] == 'talos' or test['suite'] == 'raptor'
if worker['os'] != 'linux':
raise Exception(
'os: {} not supported on script-engine-autophone'.format(
worker['os']))
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
mozharness_url = get_artifact_url('<build>', 'public/build/mozharness.zip')
artifacts = [
# (artifact name prefix, in-image path)
("public/test/", "/builds/worker/artifacts"),
("public/logs/", "/builds/worker/workspace/build/upload/logs"),
("public/test_info/",
"/builds/worker/workspace/build/blobber_upload_dir"),
]
worker['artifacts'] = [{
'name': prefix,
'path': path,
'type': 'directory',
} for (prefix, path) in artifacts]
if test['reboot']:
worker['reboot'] = test['reboot']
worker['env'] = env = {
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZHARNESS_URL': {
'task-reference': mozharness_url
},
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
"MOZ_NO_REMOTE": '1',
"XPCOM_DEBUG_BREAK": 'warn',
"NO_FAIL_ON_TEST_ERRORS": '1',
"MOZ_HIDE_RESULTS_TABLE": '1',
"MOZ_NODE_PATH": "/usr/local/bin/node",
'MOZ_AUTOMATION': '1',
'WORKING_DIR': '/builds/worker',
'WORKSPACE': '/builds/worker/workspace',
'TASKCLUSTER_WORKER_TYPE': job['worker-type'],
}
# for fetch tasks on mobile
if 'env' in job['worker'] and 'MOZ_FETCHES' in job['worker']['env']:
env['MOZ_FETCHES'] = job['worker']['env']['MOZ_FETCHES']
env['MOZ_FETCHES_DIR'] = job['worker']['env']['MOZ_FETCHES_DIR']
# talos tests don't need Xvfb
if is_talos:
env['NEED_XVFB'] = 'false'
script = 'test-linux.sh'
worker['context'] = '{}/raw-file/{}/taskcluster/scripts/tester/{}'.format(
config.params['head_repository'], config.params['head_rev'], script)
command = worker['command'] = ["./{}".format(script)]
command.extend([
{
"task-reference": "--installer-url=" + installer_url
},
{
"task-reference":
"--test-packages-url=" + test_packages_url(taskdesc)
},
])
if mozharness.get('include-blob-upload-branch'):
command.append('--blob-upload-branch=' + config.params['project'])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
command[i] += suffix
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
def mozharness_test_on_generic_worker(config, job, taskdesc):
run = job['run']
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker'] = job['worker']
bitbar_script = 'test-linux.sh'
is_macosx = worker['os'] == 'macosx'
is_windows = worker['os'] == 'windows'
is_linux = worker['os'] == 'linux' or worker['os'] == 'linux-bitbar'
is_bitbar = worker['os'] == 'linux-bitbar'
assert is_macosx or is_windows or is_linux
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
if is_bitbar:
artifacts = [
{
'name': 'public/test/',
'path': 'artifacts/public',
'type': 'directory'
},
{
'name': 'public/logs/',
'path': 'workspace/logs',
'type': 'directory'
},
{
'name': 'public/test_info/',
'path': 'workspace/build/blobber_upload_dir',
'type': 'directory'
},
]
if 'installer-url' in mozharness:
installer_url = mozharness['installer-url']
else:
upstream_task = '<build-signing>' if mozharness[
'requires-signed-builds'] else '<build>'
installer_url = get_artifact_url(upstream_task,
mozharness['build-artifact-name'])
worker['os-groups'] = test['os-groups']
# run-as-administrator is a feature for workers with UAC enabled and as such should not be
# included in tasks on workers that have UAC disabled. Currently UAC is only enabled on
# gecko Windows 10 workers, however this may be subject to change. Worker type
# environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig
# See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control
# for more information about UAC.
if test.get('run-as-administrator', False):
if job['worker-type'].startswith('t-win10-64'):
worker['run-as-administrator'] = True
else:
raise Exception('run-as-administrator not supported on {}'.format(
job['worker-type']))
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
worker['max-run-time'] = test['max-run-time']
worker['retry-exit-status'] = test['retry-exit-status']
worker['artifacts'] = artifacts
env = worker.setdefault('env', {})
env['GECKO_HEAD_REPOSITORY'] = config.params['head_repository']
env['GECKO_HEAD_REV'] = config.params['head_rev']
# this list will get cleaned up / reduced / removed in bug 1354088
if is_macosx:
env.update({
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0',
})
elif is_bitbar:
env.update({
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZHARNESS_URL': {
'artifact-reference': '<build/public/build/mozharness.zip>'
},
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
"MOZ_NO_REMOTE": '1',
"NEED_XVFB": "false",
"XPCOM_DEBUG_BREAK": 'warn',
"NO_FAIL_ON_TEST_ERRORS": '1',
"MOZ_HIDE_RESULTS_TABLE": '1',
"MOZ_NODE_PATH": "/usr/local/bin/node",
'TASKCLUSTER_WORKER_TYPE': job['worker-type'],
})
extra_config = {
'installer_url': installer_url,
'test_packages_url': test_packages_url(taskdesc),
}
env['EXTRA_MOZHARNESS_CONFIG'] = {
'task-reference': json.dumps(extra_config)
}
if is_windows:
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
elif is_bitbar:
mh_command = ['bash', "./{}".format(bitbar_script)]
elif is_macosx and 'macosx1014-64' in test['test-platform']:
mh_command = [
'/usr/local/bin/python2', '-u',
'mozharness/scripts/' + mozharness['script']
]
else:
# is_linux or is_macosx
mh_command = [
# Using /usr/bin/python2.7 rather than python2.7 because
# /usr/local/bin/python2.7 is broken on the mac workers.
# See bug #1547903.
'/usr/bin/python2.7',
'-u',
'mozharness/scripts/' + mozharness['script']
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if is_windows:
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': get_artifact_path(taskdesc, 'mozharness.zip'),
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if is_bitbar:
a_url = config.params.file_url(
'taskcluster/scripts/tester/{}'.format(bitbar_script), )
worker['mounts'] = [{
'file': bitbar_script,
'content': {
'url': a_url,
},
}]
job['run'] = {
'workdir': run['workdir'],
'tooltool-downloads': mozharness['tooltool-downloads'],
'checkout': test['checkout'],
'command': mh_command,
'using': 'run-task',
}
if is_bitbar:
job['run']['run-as-root'] = True
# FIXME: The bitbar config incorrectly requests internal tooltool downloads
# so force it off here.
job['run']['tooltool-downloads'] = False
configure_taskdesc_for_run(config, job, taskdesc, worker['implementation'])
def mozharness_test_on_docker(config, job, taskdesc):
run = job['run']
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
# apply some defaults
worker['docker-image'] = test['docker-image']
worker['allow-ptrace'] = True # required for all tests, for crashreporter
worker['loopback-video'] = test['loopback-video']
worker['loopback-audio'] = test['loopback-audio']
worker['max-run-time'] = test['max-run-time']
worker['retry-exit-status'] = test['retry-exit-status']
if 'android-em-7.0-x86' in test['test-platform']:
worker['privileged'] = True
artifacts = [
# (artifact name prefix, in-image path)
("public/logs/", "{workdir}/workspace/build/upload/logs/".format(**run)
),
("public/test", "{workdir}/artifacts/".format(**run)),
("public/test_info/",
"{workdir}/workspace/build/blobber_upload_dir/".format(**run)),
]
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
mozharness_url = get_artifact_url(
'<build>', get_artifact_path(taskdesc, 'mozharness.zip'))
worker['artifacts'] = [{
'name':
prefix,
'path':
os.path.join('{workdir}/workspace'.format(**run), path),
'type':
'directory',
} for (prefix, path) in artifacts]
worker['caches'] = [{
'type':
'persistent',
'name':
'level-{}-{}-test-workspace'.format(config.params['level'],
config.params['project']),
'mount-point':
"{workdir}/workspace".format(**run),
}]
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
'NEED_PULSEAUDIO': 'true',
'NEED_WINDOW_MANAGER': 'true',
'ENABLE_E10S': str(bool(test.get('e10s'))).lower(),
'MOZ_AUTOMATION': '1',
'WORKING_DIR': '/builds/worker',
})
if mozharness.get('mochitest-flavor'):
env['MOCHITEST_FLAVOR'] = mozharness['mochitest-flavor']
if mozharness['set-moz-node-path']:
env['MOZ_NODE_PATH'] = '/usr/local/bin/node'
if 'actions' in mozharness:
env['MOZHARNESS_ACTIONS'] = ' '.join(mozharness['actions'])
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
# handle some of the mozharness-specific options
if mozharness['tooltool-downloads']:
docker_worker_add_tooltool(config, job, taskdesc, internal=True)
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
# assemble the command line
command = [
'{workdir}/bin/run-task'.format(**run),
]
# Support vcs checkouts regardless of whether the task runs from
# source or not in case it is needed on an interactive loaner.
support_vcs_checkout(config, job, taskdesc)
# If we have a source checkout, run mozharness from it instead of
# downloading a zip file with the same content.
if test['checkout']:
command.extend(
['--vcs-checkout', '{workdir}/checkouts/gecko'.format(**run)])
env['MOZHARNESS_PATH'] = '{workdir}/checkouts/gecko/testing/mozharness'.format(
**run)
else:
env['MOZHARNESS_URL'] = {'task-reference': mozharness_url}
command.extend([
'--',
'{workdir}/bin/test-linux.sh'.format(**run),
])
command.extend([
{
"task-reference": "--installer-url=" + installer_url
},
{
"task-reference":
"--test-packages-url=" + test_packages_url(taskdesc)
},
])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
command[i] += suffix
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
worker['command'] = command
def fill_template(config, tasks):
dummy_tasks = {}
for task in tasks:
name = task['name']
deps = {}
urls = {}
previous_artifact = None
for k in ('original', 'new'):
value = task[k]
if isinstance(value, basestring):
deps[k] = value
task_id = '<{}>'.format(k)
os_hint = value
else:
index = value['index-search']
if index not in dummy_tasks:
dummy_tasks[index] = {
'label': 'index-search-' + index,
'description': index,
'worker-type': 'invalid/always-optimized',
'run': {
'using': 'always-optimized',
},
'optimization': {
'index-search': [index],
}
}
yield dummy_tasks[index]
deps[index] = 'index-search-' + index
task_id = '<{}>'.format(index)
os_hint = index.split('.')[-1]
if 'linux' in os_hint:
artifact = 'target.tar.bz2'
elif 'macosx' in os_hint:
artifact = 'target.dmg'
elif 'android' in os_hint:
artifact = 'target.apk'
elif 'win' in os_hint:
artifact = 'target.zip'
else:
raise Exception(
'Cannot figure out the OS for {!r}'.format(value))
if previous_artifact is not None and previous_artifact != artifact:
raise Exception(
'Cannot compare builds from different OSes')
url = get_artifact_url(task_id, 'public/build/{}'.format(artifact))
urls[k] = {'task-reference': url}
previous_artifact = artifact
taskdesc = {
'label': 'diff-' + name,
'description': name,
'treeherder': {
'symbol': task['symbol'],
'platform': 'diff/opt',
'kind': 'other',
'tier': 2,
},
'worker-type': 'aws-provisioner-v1/gecko-{}-b-linux'.format(
config.params['level']),
'worker': {
'docker-image': {'in-tree': 'diffoscope'},
'artifacts': [{
'type': 'file',
'path': '/builds/worker/diff.html',
'name': 'public/diff.html',
}, {
'type': 'file',
'path': '/builds/worker/diff.txt',
'name': 'public/diff.txt',
}],
'env': {
'ORIG_URL': urls['original'],
'NEW_URL': urls['new'],
'DIFFOSCOPE_ARGS': ' '.join(
task[k] for k in ('args', 'extra-args') if k in task)
},
'max-run-time': 1800,
},
'run': {
'using': 'run-task',
'checkout': False,
'command': '/builds/worker/bin/get_and_diffoscope '
'"$ORIG_URL" "$NEW_URL"',
},
'dependencies': deps,
}
if artifact.endswith('.dmg'):
taskdesc['toolchains'] = [
'linux64-cctools-port',
'linux64-libdmg',
]
yield taskdesc