def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
is_macosx = worker['os'] == 'macosx'
is_windows = worker['os'] == 'windows'
assert is_macosx or is_windows
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
upstream_task = '<build-signing>' if mozharness[
'requires-signed-builds'] else '<build>'
installer_url = get_artifact_url(upstream_task,
mozharness['build-artifact-name'])
taskdesc['scopes'].extend([
'generic-worker:os-group:{}'.format(group)
for group in test['os-groups']
])
worker['os-groups'] = test['os-groups']
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
env = worker.setdefault('env', {})
env['MOZ_AUTOMATION'] = '1'
# this list will get cleaned up / reduced / removed in bug 1354088
if is_macosx:
env.update({
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_EM_RESTART': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0',
})
if is_macosx:
mh_command = [
'python2.7', '-u', 'mozharness/scripts/' + mozharness['script']
]
elif is_windows:
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if is_windows:
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url(taskdesc)])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
mh_command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
if config.params['project'] == 'try':
env['TRY_COMMIT_MSG'] = config.params['message']
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': 'public/build/mozharness.zip',
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if is_windows:
worker['command'] = [{'task-reference': ' '.join(mh_command)}]
else: # is_macosx
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [mh_command_task_ref]
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
is_macosx = worker['os'] == 'macosx'
is_windows = worker['os'] == 'windows'
is_linux = worker['os'] == 'linux'
assert is_macosx or is_windows or is_linux
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
upstream_task = '<build-signing>' if mozharness[
'requires-signed-builds'] else '<build>'
installer_url = get_artifact_url(upstream_task,
mozharness['build-artifact-name'])
taskdesc['scopes'].extend([
'generic-worker:os-group:{}/{}'.format(job['worker-type'], group)
for group in test['os-groups']
])
worker['os-groups'] = test['os-groups']
# run-as-administrator is a feature for workers with UAC enabled and as such should not be
# included in tasks on workers that have UAC disabled. Currently UAC is only enabled on
# gecko Windows 10 workers, however this may be subject to change. Worker type
# environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig
# See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control
# for more information about UAC.
if test.get('run-as-administrator', False):
if job['worker-type'].startswith(
'aws-provisioner-v1/gecko-t-win10-64'):
taskdesc['scopes'].extend([
'generic-worker:run-as-administrator:{}'.format(
job['worker-type'])
])
worker['run-as-administrator'] = True
else:
raise Exception('run-as-administrator not supported on {}'.format(
job['worker-type']))
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
env = worker.setdefault('env', {})
env['MOZ_AUTOMATION'] = '1'
env['GECKO_HEAD_REPOSITORY'] = config.params['head_repository']
env['GECKO_HEAD_REV'] = config.params['head_rev']
# this list will get cleaned up / reduced / removed in bug 1354088
if is_macosx:
env.update({
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0',
})
if is_windows:
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
else:
# is_linux or is_macosx
mh_command = [
'python2.7', '-u', 'mozharness/scripts/' + mozharness['script']
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if is_windows:
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url(taskdesc)])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
mh_command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': get_artifact_path(taskdesc, 'mozharness.zip'),
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if is_windows:
worker['command'] = [{'task-reference': ' '.join(mh_command)}]
else: # is_macosx
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [mh_command_task_ref]
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
build_platform = taskdesc['attributes']['build_platform']
build_type = taskdesc['attributes']['build_type']
if build_platform == 'macosx64' and build_type == 'opt':
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac')
else:
target = 'target'
installer_url = get_artifact_url('<build>',
mozharness['build-artifact-name'])
test_packages_url = get_artifact_url(
'<build>', 'public/build/{}.test_packages.json'.format(target))
taskdesc['scopes'].extend([
'generic-worker:os-group:{}'.format(group)
for group in test['os-groups']
])
worker['os-groups'] = test['os-groups']
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
# this list will get cleaned up / reduced / removed in bug 1354088
if build_platform.startswith('macosx'):
worker['env'] = {
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_EM_RESTART': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0'
}
if build_platform.startswith('macosx'):
mh_command = [
'python2.7', '-u', 'mozharness/scripts/' + mozharness['script']
]
elif build_platform.startswith('win'):
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
else:
mh_command = [
'python', '-u', 'mozharness/scripts/' + mozharness['script']
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if build_platform.startswith('win'):
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
mh_command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': 'public/build/mozharness.zip',
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if build_platform.startswith('win'):
worker['command'] = [{'task-reference': ' '.join(mh_command)}]
else:
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [mh_command_task_ref]
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
is_macosx = worker['os'] == 'macosx'
is_windows = worker['os'] == 'windows'
assert is_macosx or is_windows
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
upstream_task = '<build-signing>' if mozharness['requires-signed-builds'] else '<build>'
installer_url = get_artifact_url(upstream_task, mozharness['build-artifact-name'])
taskdesc['scopes'].extend(
['generic-worker:os-group:{}'.format(group) for group in test['os-groups']])
worker['os-groups'] = test['os-groups']
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(test['reboot']))
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
env = worker.setdefault('env', {})
env['MOZ_AUTOMATION'] = '1'
env['GECKO_HEAD_REPOSITORY'] = config.params['head_repository']
env['GECKO_HEAD_REV'] = config.params['head_rev']
# this list will get cleaned up / reduced / removed in bug 1354088
if is_macosx:
env.update({
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_EM_RESTART': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0',
})
if is_macosx:
mh_command = [
'python2.7',
'-u',
'mozharness/scripts/' + mozharness['script']
]
elif is_windows:
mh_command = [
'c:\\mozilla-build\\python\\python.exe',
'-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if is_windows:
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url(taskdesc)])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
mh_command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': 'public/build/mozharness.zip',
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if is_windows:
worker['command'] = [
{'task-reference': ' '.join(mh_command)}
]
else: # is_macosx
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [
mh_command_task_ref
]
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
build_platform = taskdesc['attributes']['build_platform']
build_type = taskdesc['attributes']['build_type']
if build_platform == 'macosx64' and build_type == 'opt':
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac')
else:
target = 'target'
installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
test_packages_url = get_artifact_url(
'<build>', 'public/build/{}.test_packages.json'.format(target))
taskdesc['scopes'].extend(
['generic-worker:os-group:{}'.format(group) for group in test['os-groups']])
worker['os-groups'] = test['os-groups']
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
# this list will get cleaned up / reduced / removed in bug 1354088
if build_platform.startswith('macosx'):
worker['env'] = {
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_EM_RESTART': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0'
}
if build_platform.startswith('macosx'):
mh_command = [
'python2.7',
'-u',
'mozharness/scripts/' + mozharness['script']
]
elif build_platform.startswith('win'):
mh_command = [
'c:\\mozilla-build\\python\\python.exe',
'-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
else:
mh_command = [
'python',
'-u',
'mozharness/scripts/' + mozharness['script']
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if build_platform.startswith('win'):
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
mh_command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': 'public/build/mozharness.zip',
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if build_platform.startswith('win'):
worker['command'] = [
{'task-reference': ' '.join(mh_command)}
]
else:
mh_command_task_ref = []
for token in mh_command:
mh_command_task_ref.append({'task-reference': token})
worker['command'] = [
mh_command_task_ref
]
def mozharness_test_on_generic_worker(config, job, taskdesc):
run = job['run']
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker'] = job['worker']
bitbar_script = 'test-linux.sh'
is_macosx = worker['os'] == 'macosx'
is_windows = worker['os'] == 'windows'
is_linux = worker['os'] == 'linux' or worker['os'] == 'linux-bitbar'
is_bitbar = worker['os'] == 'linux-bitbar'
assert is_macosx or is_windows or is_linux
artifacts = [
{
'name': 'public/logs',
'path': 'logs',
'type': 'directory'
},
]
# jittest doesn't have blob_upload_dir
if test['test-name'] != 'jittest':
artifacts.append({
'name': 'public/test_info',
'path': 'build/blobber_upload_dir',
'type': 'directory'
})
if is_bitbar:
artifacts = [
{
'name': 'public/test/',
'path': 'artifacts/public',
'type': 'directory'
},
{
'name': 'public/logs/',
'path': 'workspace/logs',
'type': 'directory'
},
{
'name': 'public/test_info/',
'path': 'workspace/build/blobber_upload_dir',
'type': 'directory'
},
]
if 'installer-url' in mozharness:
installer_url = mozharness['installer-url']
else:
upstream_task = '<build-signing>' if mozharness[
'requires-signed-builds'] else '<build>'
installer_url = get_artifact_url(upstream_task,
mozharness['build-artifact-name'])
worker['os-groups'] = test['os-groups']
# run-as-administrator is a feature for workers with UAC enabled and as such should not be
# included in tasks on workers that have UAC disabled. Currently UAC is only enabled on
# gecko Windows 10 workers, however this may be subject to change. Worker type
# environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig
# See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control
# for more information about UAC.
if test.get('run-as-administrator', False):
if job['worker-type'].startswith('t-win10-64'):
worker['run-as-administrator'] = True
else:
raise Exception('run-as-administrator not supported on {}'.format(
job['worker-type']))
if test['reboot']:
raise Exception('reboot: {} not supported on generic-worker'.format(
test['reboot']))
worker['max-run-time'] = test['max-run-time']
worker['retry-exit-status'] = test['retry-exit-status']
worker['artifacts'] = artifacts
env = worker.setdefault('env', {})
env['GECKO_HEAD_REPOSITORY'] = config.params['head_repository']
env['GECKO_HEAD_REV'] = config.params['head_rev']
# this list will get cleaned up / reduced / removed in bug 1354088
if is_macosx:
env.update({
'IDLEIZER_DISABLE_SHUTDOWN': 'true',
'LANG': 'en_US.UTF-8',
'LC_ALL': 'en_US.UTF-8',
'MOZ_HIDE_RESULTS_TABLE': '1',
'MOZ_NODE_PATH': '/usr/local/bin/node',
'MOZ_NO_REMOTE': '1',
'NO_FAIL_ON_TEST_ERRORS': '1',
'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL': '/bin/bash',
'XPCOM_DEBUG_BREAK': 'warn',
'XPC_FLAGS': '0x0',
'XPC_SERVICE_NAME': '0',
})
elif is_bitbar:
env.update({
'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
'MOZHARNESS_SCRIPT': mozharness['script'],
'MOZHARNESS_URL': {
'artifact-reference': '<build/public/build/mozharness.zip>'
},
'MOZILLA_BUILD_URL': {
'task-reference': installer_url
},
"MOZ_NO_REMOTE": '1',
"NEED_XVFB": "false",
"XPCOM_DEBUG_BREAK": 'warn',
"NO_FAIL_ON_TEST_ERRORS": '1',
"MOZ_HIDE_RESULTS_TABLE": '1',
"MOZ_NODE_PATH": "/usr/local/bin/node",
'TASKCLUSTER_WORKER_TYPE': job['worker-type'],
})
extra_config = {
'installer_url': installer_url,
'test_packages_url': test_packages_url(taskdesc),
}
env['EXTRA_MOZHARNESS_CONFIG'] = {
'task-reference': json.dumps(extra_config)
}
if is_windows:
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
elif is_bitbar:
mh_command = ['bash', "./{}".format(bitbar_script)]
elif is_macosx and 'macosx1014-64' in test['test-platform']:
mh_command = [
'/usr/local/bin/python2', '-u',
'mozharness/scripts/' + mozharness['script']
]
else:
# is_linux or is_macosx
mh_command = [
# Using /usr/bin/python2.7 rather than python2.7 because
# /usr/local/bin/python2.7 is broken on the mac workers.
# See bug #1547903.
'/usr/bin/python2.7',
'-u',
'mozharness/scripts/' + mozharness['script']
]
for mh_config in mozharness['config']:
cfg_path = 'mozharness/configs/' + mh_config
if is_windows:
cfg_path = normpath(cfg_path)
mh_command.extend(['--cfg', cfg_path])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
if mozharness.get('include-blob-upload-branch'):
mh_command.append('--blob-upload-branch=' + config.params['project'])
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
worker['mounts'] = [{
'directory': '.',
'content': {
'artifact': get_artifact_path(taskdesc, 'mozharness.zip'),
'task-id': {
'task-reference': '<build>'
}
},
'format': 'zip'
}]
if is_bitbar:
a_url = config.params.file_url(
'taskcluster/scripts/tester/{}'.format(bitbar_script), )
worker['mounts'] = [{
'file': bitbar_script,
'content': {
'url': a_url,
},
}]
job['run'] = {
'workdir': run['workdir'],
'tooltool-downloads': mozharness['tooltool-downloads'],
'checkout': test['checkout'],
'command': mh_command,
'using': 'run-task',
}
if is_bitbar:
job['run']['run-as-root'] = True
# FIXME: The bitbar config incorrectly requests internal tooltool downloads
# so force it off here.
job['run']['tooltool-downloads'] = False
configure_taskdesc_for_run(config, job, taskdesc, worker['implementation'])
def mozharness_test_on_generic_worker(config, job, taskdesc):
test = taskdesc["run"]["test"]
mozharness = test["mozharness"]
worker = taskdesc["worker"] = job["worker"]
bitbar_script = "test-linux.sh"
is_macosx = worker["os"] == "macosx"
is_windows = worker["os"] == "windows"
is_linux = worker["os"] == "linux" or worker["os"] == "linux-bitbar"
is_bitbar = worker["os"] == "linux-bitbar"
assert is_macosx or is_windows or is_linux
artifacts = [
{
"name": "public/logs",
"path": "logs",
"type": "directory"
},
]
# jittest doesn't have blob_upload_dir
if test["test-name"] != "jittest":
artifacts.append({
"name": "public/test_info",
"path": "build/blobber_upload_dir",
"type": "directory",
})
if is_bitbar:
artifacts = [
{
"name": "public/test/",
"path": "artifacts/public",
"type": "directory"
},
{
"name": "public/logs/",
"path": "workspace/logs",
"type": "directory"
},
{
"name": "public/test_info/",
"path": "workspace/build/blobber_upload_dir",
"type": "directory",
},
]
installer = installer_url(taskdesc)
worker["os-groups"] = test["os-groups"]
# run-as-administrator is a feature for workers with UAC enabled and as such should not be
# included in tasks on workers that have UAC disabled. Currently UAC is only enabled on
# gecko Windows 10 workers, however this may be subject to change. Worker type
# environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig
# See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control
# for more information about UAC.
if test.get("run-as-administrator", False):
if job["worker-type"].startswith("t-win10-64"):
worker["run-as-administrator"] = True
else:
raise Exception("run-as-administrator not supported on {}".format(
job["worker-type"]))
if test["reboot"]:
raise Exception("reboot: {} not supported on generic-worker".format(
test["reboot"]))
worker["max-run-time"] = test["max-run-time"]
worker["retry-exit-status"] = test["retry-exit-status"]
worker.setdefault("artifacts", [])
worker["artifacts"].extend(artifacts)
env = worker.setdefault("env", {})
env["GECKO_HEAD_REPOSITORY"] = config.params["head_repository"]
env["GECKO_HEAD_REV"] = config.params["head_rev"]
# this list will get cleaned up / reduced / removed in bug 1354088
if is_macosx:
env.update({
"LC_ALL": "en_US.UTF-8",
"LANG": "en_US.UTF-8",
"MOZ_NODE_PATH": "/usr/local/bin/node",
"PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",
"SHELL": "/bin/bash",
})
elif is_bitbar:
env.update({
"MOZHARNESS_CONFIG": " ".join(mozharness["config"]),
"MOZHARNESS_SCRIPT": mozharness["script"],
"MOZHARNESS_URL": {
"artifact-reference": "<build/public/build/mozharness.zip>"
},
"MOZILLA_BUILD_URL": {
"task-reference": installer
},
"MOZ_NO_REMOTE": "1",
"NEED_XVFB": "false",
"XPCOM_DEBUG_BREAK": "warn",
"NO_FAIL_ON_TEST_ERRORS": "1",
"MOZ_HIDE_RESULTS_TABLE": "1",
"MOZ_NODE_PATH": "/usr/local/bin/node",
"TASKCLUSTER_WORKER_TYPE": job["worker-type"],
})
extra_config = {
"installer_url": installer,
"test_packages_url": test_packages_url(taskdesc),
}
env["EXTRA_MOZHARNESS_CONFIG"] = {
"task-reference":
six.ensure_text(json.dumps(extra_config, sort_keys=True))
}
# Bug 1634554 - pass in decision task artifact URL to mozharness for WPT.
# Bug 1645974 - test-verify-wpt and test-coverage-wpt need artifact URL.
if "web-platform-tests" in test["suite"] or re.match(
"test-(coverage|verify)-wpt", test["suite"]):
env["TESTS_BY_MANIFEST_URL"] = {
"artifact-reference": "<decision/public/tests-by-manifest.json.gz>"
}
py_3 = test.get("python-3", False)
if is_windows:
py_binary = "c:\\mozilla-build\\{python}\\{python}.exe".format(
python="python3" if py_3 else "python")
mh_command = [
py_binary,
"-u",
"mozharness\\scripts\\" + normpath(mozharness["script"]),
]
elif is_bitbar:
py_binary = "python3" if py_3 else "python"
mh_command = ["bash", "./{}".format(bitbar_script)]
elif is_macosx and "macosx1014-64" in test["test-platform"]:
py_binary = "/usr/local/bin/{}".format(
"python3" if py_3 else "python2")
mh_command = [
py_binary,
"-u",
"mozharness/scripts/" + mozharness["script"],
]
else:
# is_linux or is_macosx
py_binary = "/usr/bin/{}".format("python3" if py_3 else "python2")
mh_command = [
# Using /usr/bin/python2.7 rather than python2.7 because
# /usr/local/bin/python2.7 is broken on the mac workers.
# See bug #1547903.
py_binary,
"-u",
"mozharness/scripts/" + mozharness["script"],
]
if py_3:
env["PYTHON"] = py_binary
for mh_config in mozharness["config"]:
cfg_path = "mozharness/configs/" + mh_config
if is_windows:
cfg_path = normpath(cfg_path)
mh_command.extend(["--cfg", cfg_path])
mh_command.extend(mozharness.get("extra-options", []))
if mozharness.get("download-symbols"):
if isinstance(mozharness["download-symbols"], text_type):
mh_command.extend(
["--download-symbols", mozharness["download-symbols"]])
else:
mh_command.extend(["--download-symbols", "true"])
if mozharness.get("include-blob-upload-branch"):
mh_command.append("--blob-upload-branch=" + config.params["project"])
if test.get("test-manifests"):
env["MOZHARNESS_TEST_PATHS"] = six.ensure_text(
json.dumps({test["suite"]: test["test-manifests"]},
sort_keys=True))
# TODO: remove the need for run['chunked']
elif mozharness.get("chunked") or test["chunks"] > 1:
mh_command.append("--total-chunk={}".format(test["chunks"]))
mh_command.append("--this-chunk={}".format(test["this-chunk"]))
if config.params.is_try():
env["TRY_COMMIT_MSG"] = config.params["message"]
worker["mounts"] = [{
"directory": "mozharness",
"content": {
"artifact": get_artifact_path(taskdesc, "mozharness.zip"),
"task-id": {
"task-reference": "<build>"
},
},
"format": "zip",
}]
if is_bitbar:
a_url = config.params.file_url(
"taskcluster/scripts/tester/{}".format(bitbar_script), )
worker["mounts"] = [{
"file": bitbar_script,
"content": {
"url": a_url,
},
}]
job["run"] = {
"tooltool-downloads": mozharness["tooltool-downloads"],
"checkout": test["checkout"],
"command": mh_command,
"using": "run-task",
}
if is_bitbar:
job["run"]["run-as-root"] = True
configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"])
def mozharness_test_on_windows(config, job, taskdesc):
test = taskdesc['run']['test']
mozharness = test['mozharness']
worker = taskdesc['worker']
artifacts = [{
'path': 'public\\logs\\localconfig.json',
'type': 'file'
}, {
'path': 'public\\logs\\log_critical.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_error.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_fatal.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_info.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_raw.log',
'type': 'file'
}, {
'path': 'public\\logs\\log_warning.log',
'type': 'file'
}, {
'path': 'public\\test_info',
'type': 'directory'
}]
build_platform = taskdesc['attributes']['build_platform']
target = 'firefox-{}.en-US.{}'.format(get_firefox_version(),
build_platform)
installer_url = get_artifact_url('<build>',
'public/build/{}.zip'.format(target))
test_packages_url = get_artifact_url(
'<build>', 'public/build/{}.test_packages.json'.format(target))
mozharness_url = get_artifact_url('<build>', 'public/build/mozharness.zip')
taskdesc['scopes'].extend([
'generic-worker:os-group:{}'.format(group)
for group in test['os-groups']
])
worker['os-groups'] = test['os-groups']
worker['max-run-time'] = test['max-run-time']
worker['artifacts'] = artifacts
# assemble the command line
mh_command = [
'c:\\mozilla-build\\python\\python.exe', '-u',
'mozharness\\scripts\\' + normpath(mozharness['script'])
]
for mh_config in mozharness['config']:
mh_command.extend(
['--cfg', 'mozharness\\configs\\' + normpath(mh_config)])
mh_command.extend(mozharness.get('extra-options', []))
if mozharness.get('no-read-buildbot-config'):
mh_command.append('--no-read-buildbot-config')
mh_command.extend(['--installer-url', installer_url])
mh_command.extend(['--test-packages-url', test_packages_url])
if mozharness.get('download-symbols'):
if isinstance(mozharness['download-symbols'], basestring):
mh_command.extend(
['--download-symbols', mozharness['download-symbols']])
else:
mh_command.extend(['--download-symbols', 'true'])
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
mh_command.append('--total-chunk={}'.format(test['chunks']))
mh_command.append('--this-chunk={}'.format(test['this-chunk']))
elif mozharness['chunking-args'] == 'test-suite-suffix':
suffix = mozharness['chunk-suffix'].replace(
'<CHUNK>', str(test['this-chunk']))
for i, c in enumerate(mh_command):
if isinstance(c, basestring) and c.startswith('--test-suite'):
mh_command[i] += suffix
# bug 1311966 - symlink to artifacts until generic worker supports virtual artifact paths
artifact_link_commands = [
'mklink /d %cd%\\public\\test_info %cd%\\build\\blobber_upload_dir'
]
for link in [
a['path'] for a in artifacts
if a['path'].startswith('public\\logs\\')
]:
artifact_link_commands.append('mklink %cd%\\{} %cd%\\{}'.format(
link, link[7:]))
worker['command'] = artifact_link_commands + [{
'task-reference':
'c:\\mozilla-build\\wget\\wget.exe {}'.format(mozharness_url)
}, 'c:\\mozilla-build\\info-zip\\unzip.exe mozharness.zip', {
'task-reference':
' '.join(mh_command)
}]
