def test_packages_url(taskdesc):
"""Account for different platforms that name their test packages differently"""
build_platform = taskdesc['attributes']['build_platform']
build_type = taskdesc['attributes']['build_type']
if build_platform == 'macosx64' and build_type == 'opt':
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac')
else:
target = 'target'
return get_artifact_url(
'<build>', 'public/build/{}.test_packages.json'.format(target))
def mozharness_test_on_native_engine(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
build_platform = taskdesc['attributes']['build_platform']
build_type = taskdesc['attributes']['build_type']
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') \
if build_platform == 'macosx64' and build_type == 'opt' else 'target'
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
test_packages_url = get_artifact_url(
'<build>', 'public/build/{}.test_packages.json'.format(target))
mozharness_url = get_artifact_url('<build>', 'public/build/mozharness.zip')
worker['artifacts'] = [{
'name': prefix.rstrip('/'),
'path': path.rstrip('/'),
'type': 'directory',
} for (prefix, path) in ARTIFACTS]
worker['reboot'] = test['reboot']
worker['env'] = {
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZHARNESS_URL': {
'task-reference': mozharness_url
},
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
"MOZ_NO_REMOTE": '1',
"NO_EM_RESTART": '1',
"XPCOM_DEBUG_BREAK": 'warn',
"NO_FAIL_ON_TEST_ERRORS": '1',
"MOZ_HIDE_RESULTS_TABLE": '1',
"MOZ_NODE_PATH": "/usr/local/bin/node",
}
worker[
'context'] = '{}/raw-file/{}/taskcluster/scripts/tester/test-macosx.sh'.format(
config.params['head_repository'], config.params['head_rev'])
command = worker['command'] = ["./test-macosx.sh"]
if mozharness.get('no-read-buildbot-config'):
command.append("--no-read-buildbot-config")
command.extend([
{
"task-reference": "--installer-url=" + installer_url
},
{
"task-reference": "--test-packages-url=" + test_packages_url
},
])
if mozharness.get('include-blob-upload-branch'):
command.append('--blob-upload-branch=' + config.params['project'])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
command[i] += suffix
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {
True: 'true',
False: 'false'
}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
build_platform = taskdesc['attributes']['build_platform']
build_type = taskdesc['attributes']['build_type']
if build_platform == 'macosx64' and build_type == 'opt':
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac')
else:
target = 'target'
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
test_packages_url = get_artifact_url(
'<build>', 'public/build/{}.test_packages.json'.format(target))
taskdesc['scopes'].extend([
'generic-worker:os-group:{}'.format(group)
for group in test['os-groups']
])
worker['os-groups'] = test['os-groups']
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
# this list will get cleaned up / reduced / removed in bug 1354088
if build_platform.startswith('macosx'):
worker['env'] = {
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_EM_RESTART': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0'
}
if build_platform.startswith('macosx'):
mh_command = [
'python2.7', '-u', 'mozharness/scripts/' + mozharness['script']
]
elif build_platform.startswith('win'):
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
else:
mh_command = [
'python', '-u', 'mozharness/scripts/' + mozharness['script']
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if build_platform.startswith('win'):
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
mh_command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': 'public/build/mozharness.zip',
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if build_platform.startswith('win'):
worker['command'] = [{'task-reference': ' '.join(mh_command)}]
else:
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [mh_command_task_ref]
def mozharness_test_on_native_engine(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
build_platform = taskdesc['attributes']['build_platform']
build_type = taskdesc['attributes']['build_type']
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') \
if build_platform == 'macosx64' and build_type == 'opt' else 'target'
installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
test_packages_url = get_artifact_url('<build>',
'public/build/{}.test_packages.json'.format(target))
mozharness_url = get_artifact_url('<build>',
'public/build/mozharness.zip')
worker['artifacts'] = [{
'name': prefix.rstrip('/'),
'path': path.rstrip('/'),
'type': 'directory',
} for (prefix, path) in ARTIFACTS]
worker['reboot'] = test['reboot']
worker['env'] = {
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZHARNESS_URL': {'task-reference': mozharness_url},
'MOZILLA_BUILD_URL': {'task-reference': installer_url},
"MOZ_NO_REMOTE": '1',
"NO_EM_RESTART": '1',
"XPCOM_DEBUG_BREAK": 'warn',
"NO_FAIL_ON_TEST_ERRORS": '1',
"MOZ_HIDE_RESULTS_TABLE": '1',
"MOZ_NODE_PATH": "/usr/local/bin/node",
}
worker['context'] = '{}/raw-file/{}/taskcluster/scripts/tester/test-macosx.sh'.format(
config.params['head_repository'], config.params['head_rev']
)
command = worker['command'] = ["./test-macosx.sh"]
if mozharness.get('no-read-buildbot-config'):
command.append("--no-read-buildbot-config")
command.extend([
{"task-reference": "--installer-url=" + installer_url},
{"task-reference": "--test-packages-url=" + test_packages_url},
])
if mozharness.get('include-blob-upload-branch'):
command.append('--blob-upload-branch=' + config.params['project'])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
command[i] += suffix
if 'download-symbols' in mozharness:
download_symbols = mozharness['download-symbols']
download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols)
command.append('--download-symbols=' + download_symbols)
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
build_platform = taskdesc['attributes']['build_platform']
build_type = taskdesc['attributes']['build_type']
if build_platform == 'macosx64' and build_type == 'opt':
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac')
else:
target = 'target'
installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
test_packages_url = get_artifact_url(
'<build>', 'public/build/{}.test_packages.json'.format(target))
taskdesc['scopes'].extend(
['generic-worker:os-group:{}'.format(group) for group in test['os-groups']])
worker['os-groups'] = test['os-groups']
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
# this list will get cleaned up / reduced / removed in bug 1354088
if build_platform.startswith('macosx'):
worker['env'] = {
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_EM_RESTART': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0'
}
if build_platform.startswith('macosx'):
mh_command = [
'python2.7',
'-u',
'mozharness/scripts/' + mozharness['script']
]
elif build_platform.startswith('win'):
mh_command = [
'c:\\mozilla-build\\python\\python.exe',
'-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
else:
mh_command = [
'python',
'-u',
'mozharness/scripts/' + mozharness['script']
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if build_platform.startswith('win'):
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
mh_command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': 'public/build/mozharness.zip',
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if build_platform.startswith('win'):
worker['command'] = [
{'task-reference': ' '.join(mh_command)}
]
else:
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [
mh_command_task_ref
]
def mozharness_test_on_windows(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
artifacts = [{
'path': 'public\\logs\\localconfig.json',
'type': 'file'
}, {
'path': 'public\\logs\\log_critical.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_error.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_fatal.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_info.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_raw.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_warning.log',
'type': 'file'
}, {
'path': 'public\\test_info',
'type': 'directory'
}]
build_platform = taskdesc['attributes']['build_platform']
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(),
build_platform)
installer_url = get_artifact_url('<build>',
'public/build/{}.zip'.format(target))
test_packages_url = get_artifact_url(
'<build>', 'public/build/{}.test_packages.json'.format(target))
mozharness_url = get_artifact_url('<build>', 'public/build/mozharness.zip')
taskdesc['scopes'].extend([
'generic-worker:os-group:{}'.format(group)
for group in test['os-groups']
])
worker['os-groups'] = test['os-groups']
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
# assemble the command line
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
for mh_config in mozharness['config']:
mh_command.extend(
['--cfg', 'mozharness\\configs\\' + normpath(mh_config)])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
# bug 1311966 - symlink to artifacts until generic worker supports virtual artifact paths
artifact_link_commands = [
'mklink /d %cd%\\public\\test_info %cd%\\build\\blobber_upload_dir'
]
for link in [
a['path'] for a in artifacts
if a['path'].startswith('public\\logs\\')
]:
artifact_link_commands.append('mklink %cd%\\{} %cd%\\{}'.format(
link, link[7:]))
worker['command'] = artifact_link_commands + [{
'task-reference':
'c:\\mozilla-build\\wget\\wget.exe {}'.format(mozharness_url)
}, 'c:\\mozilla-build\\info-zip\\unzip.exe mozharness.zip', {
'task-reference':
' '.join(mh_command)
}]