def mozharness_test_on_generic_worker(config, job, taskdesc): test = taskdesc['run']['test'] mozharness = test['mozharness'] worker = taskdesc['worker'] is_macosx = worker['os'] == 'macosx' is_windows = worker['os'] == 'windows' assert is_macosx or is_windows artifacts = [ { 'name': 'public/logs', 'path': 'logs', 'type': 'directory' }, ] # jittest doesn't have blob_upload_dir if test['test-name'] != 'jittest': artifacts.append({ 'name': 'public/test_info', 'path': 'build/blobber_upload_dir', 'type': 'directory' }) upstream_task = '<build-signing>' if mozharness[ 'requires-signed-builds'] else '<build>' installer_url = get_artifact_url(upstream_task, mozharness['build-artifact-name']) taskdesc['scopes'].extend([ 'generic-worker:os-group:{}'.format(group) for group in test['os-groups'] ]) worker['os-groups'] = test['os-groups'] if test['reboot']: raise Exception('reboot: {} not supported on generic-worker'.format( test['reboot'])) worker['max-run-time'] = test['max-run-time'] worker['artifacts'] = artifacts env = worker.setdefault('env', {}) env['MOZ_AUTOMATION'] = '1' # this list will get cleaned up / reduced / removed in bug 1354088 if is_macosx: env.update({ 'IDLEIZER_DISABLE_SHUTDOWN': 'true', 'LANG': 'en_US.UTF-8', 'LC_ALL': 'en_US.UTF-8', 'MOZ_HIDE_RESULTS_TABLE': '1', 'MOZ_NODE_PATH': '/usr/local/bin/node', 'MOZ_NO_REMOTE': '1', 'NO_EM_RESTART': '1', 'NO_FAIL_ON_TEST_ERRORS': '1', 'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', 'SHELL': '/bin/bash', 'XPCOM_DEBUG_BREAK': 'warn', 'XPC_FLAGS': '0x0', 'XPC_SERVICE_NAME': '0', }) if is_macosx: mh_command = [ 'python2.7', '-u', 'mozharness/scripts/' + mozharness['script'] ] elif is_windows: mh_command = [ 'c:\\mozilla-build\\python\\python.exe', '-u', 'mozharness\\scripts\\' + normpath(mozharness['script']) ] for mh_config in mozharness['config']: cfg_path = 'mozharness/configs/' + mh_config if is_windows: cfg_path = normpath(cfg_path) mh_command.extend(['--cfg', cfg_path]) mh_command.extend(mozharness.get('extra-options', [])) if mozharness.get('no-read-buildbot-config'): mh_command.append('--no-read-buildbot-config') mh_command.extend(['--installer-url', installer_url]) mh_command.extend(['--test-packages-url', test_packages_url(taskdesc)]) if mozharness.get('download-symbols'): if isinstance(mozharness['download-symbols'], basestring): mh_command.extend( ['--download-symbols', mozharness['download-symbols']]) else: mh_command.extend(['--download-symbols', 'true']) if mozharness.get('include-blob-upload-branch'): mh_command.append('--blob-upload-branch=' + config.params['project']) mh_command.extend(mozharness.get('extra-options', [])) # TODO: remove the need for run['chunked'] if mozharness.get('chunked') or test['chunks'] > 1: # Implement mozharness['chunking-args'], modifying command in place if mozharness['chunking-args'] == 'this-chunk': mh_command.append('--total-chunk={}'.format(test['chunks'])) mh_command.append('--this-chunk={}'.format(test['this-chunk'])) elif mozharness['chunking-args'] == 'test-suite-suffix': suffix = mozharness['chunk-suffix'].replace( '<CHUNK>', str(test['this-chunk'])) for i, c in enumerate(mh_command): if isinstance(c, basestring) and c.startswith('--test-suite'): mh_command[i] += suffix if config.params['project'] == 'try': env['TRY_COMMIT_MSG'] = config.params['message'] worker['mounts'] = [{ 'directory': '.', 'content': { 'artifact': 'public/build/mozharness.zip', 'task-id': { 'task-reference': '<build>' } }, 'format': 'zip' }] if is_windows: worker['command'] = [{'task-reference': ' '.join(mh_command)}] else: # is_macosx mh_command_task_ref = [] for token in mh_command: mh_command_task_ref.append({'task-reference': token}) worker['command'] = [mh_command_task_ref]
def mozharness_test_on_generic_worker(config, job, taskdesc): test = taskdesc['run']['test'] mozharness = test['mozharness'] worker = taskdesc['worker'] is_macosx = worker['os'] == 'macosx' is_windows = worker['os'] == 'windows' is_linux = worker['os'] == 'linux' assert is_macosx or is_windows or is_linux artifacts = [ { 'name': 'public/logs', 'path': 'logs', 'type': 'directory' }, ] # jittest doesn't have blob_upload_dir if test['test-name'] != 'jittest': artifacts.append({ 'name': 'public/test_info', 'path': 'build/blobber_upload_dir', 'type': 'directory' }) upstream_task = '<build-signing>' if mozharness[ 'requires-signed-builds'] else '<build>' installer_url = get_artifact_url(upstream_task, mozharness['build-artifact-name']) taskdesc['scopes'].extend([ 'generic-worker:os-group:{}/{}'.format(job['worker-type'], group) for group in test['os-groups'] ]) worker['os-groups'] = test['os-groups'] # run-as-administrator is a feature for workers with UAC enabled and as such should not be # included in tasks on workers that have UAC disabled. Currently UAC is only enabled on # gecko Windows 10 workers, however this may be subject to change. Worker type # environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig # See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control # for more information about UAC. if test.get('run-as-administrator', False): if job['worker-type'].startswith( 'aws-provisioner-v1/gecko-t-win10-64'): taskdesc['scopes'].extend([ 'generic-worker:run-as-administrator:{}'.format( job['worker-type']) ]) worker['run-as-administrator'] = True else: raise Exception('run-as-administrator not supported on {}'.format( job['worker-type'])) if test['reboot']: raise Exception('reboot: {} not supported on generic-worker'.format( test['reboot'])) worker['max-run-time'] = test['max-run-time'] worker['artifacts'] = artifacts env = worker.setdefault('env', {}) env['MOZ_AUTOMATION'] = '1' env['GECKO_HEAD_REPOSITORY'] = config.params['head_repository'] env['GECKO_HEAD_REV'] = config.params['head_rev'] # this list will get cleaned up / reduced / removed in bug 1354088 if is_macosx: env.update({ 'IDLEIZER_DISABLE_SHUTDOWN': 'true', 'LANG': 'en_US.UTF-8', 'LC_ALL': 'en_US.UTF-8', 'MOZ_HIDE_RESULTS_TABLE': '1', 'MOZ_NODE_PATH': '/usr/local/bin/node', 'MOZ_NO_REMOTE': '1', 'NO_FAIL_ON_TEST_ERRORS': '1', 'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', 'SHELL': '/bin/bash', 'XPCOM_DEBUG_BREAK': 'warn', 'XPC_FLAGS': '0x0', 'XPC_SERVICE_NAME': '0', }) if is_windows: mh_command = [ 'c:\\mozilla-build\\python\\python.exe', '-u', 'mozharness\\scripts\\' + normpath(mozharness['script']) ] else: # is_linux or is_macosx mh_command = [ 'python2.7', '-u', 'mozharness/scripts/' + mozharness['script'] ] for mh_config in mozharness['config']: cfg_path = 'mozharness/configs/' + mh_config if is_windows: cfg_path = normpath(cfg_path) mh_command.extend(['--cfg', cfg_path]) mh_command.extend(mozharness.get('extra-options', [])) mh_command.extend(['--installer-url', installer_url]) mh_command.extend(['--test-packages-url', test_packages_url(taskdesc)]) if mozharness.get('download-symbols'): if isinstance(mozharness['download-symbols'], basestring): mh_command.extend( ['--download-symbols', mozharness['download-symbols']]) else: mh_command.extend(['--download-symbols', 'true']) if mozharness.get('include-blob-upload-branch'): mh_command.append('--blob-upload-branch=' + config.params['project']) mh_command.extend(mozharness.get('extra-options', [])) # TODO: remove the need for run['chunked'] if mozharness.get('chunked') or test['chunks'] > 1: # Implement mozharness['chunking-args'], modifying command in place if mozharness['chunking-args'] == 'this-chunk': mh_command.append('--total-chunk={}'.format(test['chunks'])) mh_command.append('--this-chunk={}'.format(test['this-chunk'])) elif mozharness['chunking-args'] == 'test-suite-suffix': suffix = mozharness['chunk-suffix'].replace( '<CHUNK>', str(test['this-chunk'])) for i, c in enumerate(mh_command): if isinstance(c, basestring) and c.startswith('--test-suite'): mh_command[i] += suffix if config.params.is_try(): env['TRY_COMMIT_MSG'] = config.params['message'] worker['mounts'] = [{ 'directory': '.', 'content': { 'artifact': get_artifact_path(taskdesc, 'mozharness.zip'), 'task-id': { 'task-reference': '<build>' } }, 'format': 'zip' }] if is_windows: worker['command'] = [{'task-reference': ' '.join(mh_command)}] else: # is_macosx mh_command_task_ref = [] for token in mh_command: mh_command_task_ref.append({'task-reference': token}) worker['command'] = [mh_command_task_ref]
def mozharness_test_on_generic_worker(config, job, taskdesc): test = taskdesc['run']['test'] mozharness = test['mozharness'] worker = taskdesc['worker'] artifacts = [ { 'name': 'public/logs', 'path': 'logs', 'type': 'directory' }, ] # jittest doesn't have blob_upload_dir if test['test-name'] != 'jittest': artifacts.append({ 'name': 'public/test_info', 'path': 'build/blobber_upload_dir', 'type': 'directory' }) build_platform = taskdesc['attributes']['build_platform'] build_type = taskdesc['attributes']['build_type'] if build_platform == 'macosx64' and build_type == 'opt': target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') else: target = 'target' installer_url = get_artifact_url('<build>', mozharness['build-artifact-name']) test_packages_url = get_artifact_url( '<build>', 'public/build/{}.test_packages.json'.format(target)) taskdesc['scopes'].extend([ 'generic-worker:os-group:{}'.format(group) for group in test['os-groups'] ]) worker['os-groups'] = test['os-groups'] worker['max-run-time'] = test['max-run-time'] worker['artifacts'] = artifacts # this list will get cleaned up / reduced / removed in bug 1354088 if build_platform.startswith('macosx'): worker['env'] = { 'IDLEIZER_DISABLE_SHUTDOWN': 'true', 'LANG': 'en_US.UTF-8', 'LC_ALL': 'en_US.UTF-8', 'MOZ_HIDE_RESULTS_TABLE': '1', 'MOZ_NODE_PATH': '/usr/local/bin/node', 'MOZ_NO_REMOTE': '1', 'NO_EM_RESTART': '1', 'NO_FAIL_ON_TEST_ERRORS': '1', 'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', 'SHELL': '/bin/bash', 'XPCOM_DEBUG_BREAK': 'warn', 'XPC_FLAGS': '0x0', 'XPC_SERVICE_NAME': '0' } if build_platform.startswith('macosx'): mh_command = [ 'python2.7', '-u', 'mozharness/scripts/' + mozharness['script'] ] elif build_platform.startswith('win'): mh_command = [ 'c:\\mozilla-build\\python\\python.exe', '-u', 'mozharness\\scripts\\' + normpath(mozharness['script']) ] else: mh_command = [ 'python', '-u', 'mozharness/scripts/' + mozharness['script'] ] for mh_config in mozharness['config']: cfg_path = 'mozharness/configs/' + mh_config if build_platform.startswith('win'): cfg_path = normpath(cfg_path) mh_command.extend(['--cfg', cfg_path]) mh_command.extend(mozharness.get('extra-options', [])) if mozharness.get('no-read-buildbot-config'): mh_command.append('--no-read-buildbot-config') mh_command.extend(['--installer-url', installer_url]) mh_command.extend(['--test-packages-url', test_packages_url]) if mozharness.get('download-symbols'): if isinstance(mozharness['download-symbols'], basestring): mh_command.extend( ['--download-symbols', mozharness['download-symbols']]) else: mh_command.extend(['--download-symbols', 'true']) if mozharness.get('include-blob-upload-branch'): mh_command.append('--blob-upload-branch=' + config.params['project']) mh_command.extend(mozharness.get('extra-options', [])) # TODO: remove the need for run['chunked'] if mozharness.get('chunked') or test['chunks'] > 1: # Implement mozharness['chunking-args'], modifying command in place if mozharness['chunking-args'] == 'this-chunk': mh_command.append('--total-chunk={}'.format(test['chunks'])) mh_command.append('--this-chunk={}'.format(test['this-chunk'])) elif mozharness['chunking-args'] == 'test-suite-suffix': suffix = mozharness['chunk-suffix'].replace( '<CHUNK>', str(test['this-chunk'])) for i, c in enumerate(mh_command): if isinstance(c, basestring) and c.startswith('--test-suite'): mh_command[i] += suffix worker['mounts'] = [{ 'directory': '.', 'content': { 'artifact': 'public/build/mozharness.zip', 'task-id': { 'task-reference': '<build>' } }, 'format': 'zip' }] if build_platform.startswith('win'): worker['command'] = [{'task-reference': ' '.join(mh_command)}] else: mh_command_task_ref = [] for token in mh_command: mh_command_task_ref.append({'task-reference': token}) worker['command'] = [mh_command_task_ref]
def mozharness_test_on_generic_worker(config, job, taskdesc): test = taskdesc['run']['test'] mozharness = test['mozharness'] worker = taskdesc['worker'] is_macosx = worker['os'] == 'macosx' is_windows = worker['os'] == 'windows' assert is_macosx or is_windows artifacts = [ { 'name': 'public/logs', 'path': 'logs', 'type': 'directory' }, ] # jittest doesn't have blob_upload_dir if test['test-name'] != 'jittest': artifacts.append({ 'name': 'public/test_info', 'path': 'build/blobber_upload_dir', 'type': 'directory' }) upstream_task = '<build-signing>' if mozharness['requires-signed-builds'] else '<build>' installer_url = get_artifact_url(upstream_task, mozharness['build-artifact-name']) taskdesc['scopes'].extend( ['generic-worker:os-group:{}'.format(group) for group in test['os-groups']]) worker['os-groups'] = test['os-groups'] if test['reboot']: raise Exception('reboot: {} not supported on generic-worker'.format(test['reboot'])) worker['max-run-time'] = test['max-run-time'] worker['artifacts'] = artifacts env = worker.setdefault('env', {}) env['MOZ_AUTOMATION'] = '1' env['GECKO_HEAD_REPOSITORY'] = config.params['head_repository'] env['GECKO_HEAD_REV'] = config.params['head_rev'] # this list will get cleaned up / reduced / removed in bug 1354088 if is_macosx: env.update({ 'IDLEIZER_DISABLE_SHUTDOWN': 'true', 'LANG': 'en_US.UTF-8', 'LC_ALL': 'en_US.UTF-8', 'MOZ_HIDE_RESULTS_TABLE': '1', 'MOZ_NODE_PATH': '/usr/local/bin/node', 'MOZ_NO_REMOTE': '1', 'NO_EM_RESTART': '1', 'NO_FAIL_ON_TEST_ERRORS': '1', 'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', 'SHELL': '/bin/bash', 'XPCOM_DEBUG_BREAK': 'warn', 'XPC_FLAGS': '0x0', 'XPC_SERVICE_NAME': '0', }) if is_macosx: mh_command = [ 'python2.7', '-u', 'mozharness/scripts/' + mozharness['script'] ] elif is_windows: mh_command = [ 'c:\\mozilla-build\\python\\python.exe', '-u', 'mozharness\\scripts\\' + normpath(mozharness['script']) ] for mh_config in mozharness['config']: cfg_path = 'mozharness/configs/' + mh_config if is_windows: cfg_path = normpath(cfg_path) mh_command.extend(['--cfg', cfg_path]) mh_command.extend(mozharness.get('extra-options', [])) if mozharness.get('no-read-buildbot-config'): mh_command.append('--no-read-buildbot-config') mh_command.extend(['--installer-url', installer_url]) mh_command.extend(['--test-packages-url', test_packages_url(taskdesc)]) if mozharness.get('download-symbols'): if isinstance(mozharness['download-symbols'], basestring): mh_command.extend(['--download-symbols', mozharness['download-symbols']]) else: mh_command.extend(['--download-symbols', 'true']) if mozharness.get('include-blob-upload-branch'): mh_command.append('--blob-upload-branch=' + config.params['project']) mh_command.extend(mozharness.get('extra-options', [])) # TODO: remove the need for run['chunked'] if mozharness.get('chunked') or test['chunks'] > 1: # Implement mozharness['chunking-args'], modifying command in place if mozharness['chunking-args'] == 'this-chunk': mh_command.append('--total-chunk={}'.format(test['chunks'])) mh_command.append('--this-chunk={}'.format(test['this-chunk'])) elif mozharness['chunking-args'] == 'test-suite-suffix': suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk'])) for i, c in enumerate(mh_command): if isinstance(c, basestring) and c.startswith('--test-suite'): mh_command[i] += suffix if config.params.is_try(): env['TRY_COMMIT_MSG'] = config.params['message'] worker['mounts'] = [{ 'directory': '.', 'content': { 'artifact': 'public/build/mozharness.zip', 'task-id': { 'task-reference': '<build>' } }, 'format': 'zip' }] if is_windows: worker['command'] = [ {'task-reference': ' '.join(mh_command)} ] else: # is_macosx mh_command_task_ref = [] for token in mh_command: mh_command_task_ref.append({'task-reference': token}) worker['command'] = [ mh_command_task_ref ]
def mozharness_test_on_generic_worker(config, job, taskdesc): test = taskdesc['run']['test'] mozharness = test['mozharness'] worker = taskdesc['worker'] artifacts = [ { 'name': 'public/logs', 'path': 'logs', 'type': 'directory' }, ] # jittest doesn't have blob_upload_dir if test['test-name'] != 'jittest': artifacts.append({ 'name': 'public/test_info', 'path': 'build/blobber_upload_dir', 'type': 'directory' }) build_platform = taskdesc['attributes']['build_platform'] build_type = taskdesc['attributes']['build_type'] if build_platform == 'macosx64' and build_type == 'opt': target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') else: target = 'target' installer_url = get_artifact_url('<build>', mozharness['build-artifact-name']) test_packages_url = get_artifact_url( '<build>', 'public/build/{}.test_packages.json'.format(target)) taskdesc['scopes'].extend( ['generic-worker:os-group:{}'.format(group) for group in test['os-groups']]) worker['os-groups'] = test['os-groups'] worker['max-run-time'] = test['max-run-time'] worker['artifacts'] = artifacts # this list will get cleaned up / reduced / removed in bug 1354088 if build_platform.startswith('macosx'): worker['env'] = { 'IDLEIZER_DISABLE_SHUTDOWN': 'true', 'LANG': 'en_US.UTF-8', 'LC_ALL': 'en_US.UTF-8', 'MOZ_HIDE_RESULTS_TABLE': '1', 'MOZ_NODE_PATH': '/usr/local/bin/node', 'MOZ_NO_REMOTE': '1', 'NO_EM_RESTART': '1', 'NO_FAIL_ON_TEST_ERRORS': '1', 'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', 'SHELL': '/bin/bash', 'XPCOM_DEBUG_BREAK': 'warn', 'XPC_FLAGS': '0x0', 'XPC_SERVICE_NAME': '0' } if build_platform.startswith('macosx'): mh_command = [ 'python2.7', '-u', 'mozharness/scripts/' + mozharness['script'] ] elif build_platform.startswith('win'): mh_command = [ 'c:\\mozilla-build\\python\\python.exe', '-u', 'mozharness\\scripts\\' + normpath(mozharness['script']) ] else: mh_command = [ 'python', '-u', 'mozharness/scripts/' + mozharness['script'] ] for mh_config in mozharness['config']: cfg_path = 'mozharness/configs/' + mh_config if build_platform.startswith('win'): cfg_path = normpath(cfg_path) mh_command.extend(['--cfg', cfg_path]) mh_command.extend(mozharness.get('extra-options', [])) if mozharness.get('no-read-buildbot-config'): mh_command.append('--no-read-buildbot-config') mh_command.extend(['--installer-url', installer_url]) mh_command.extend(['--test-packages-url', test_packages_url]) if mozharness.get('download-symbols'): if isinstance(mozharness['download-symbols'], basestring): mh_command.extend(['--download-symbols', mozharness['download-symbols']]) else: mh_command.extend(['--download-symbols', 'true']) if mozharness.get('include-blob-upload-branch'): mh_command.append('--blob-upload-branch=' + config.params['project']) mh_command.extend(mozharness.get('extra-options', [])) # TODO: remove the need for run['chunked'] if mozharness.get('chunked') or test['chunks'] > 1: # Implement mozharness['chunking-args'], modifying command in place if mozharness['chunking-args'] == 'this-chunk': mh_command.append('--total-chunk={}'.format(test['chunks'])) mh_command.append('--this-chunk={}'.format(test['this-chunk'])) elif mozharness['chunking-args'] == 'test-suite-suffix': suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk'])) for i, c in enumerate(mh_command): if isinstance(c, basestring) and c.startswith('--test-suite'): mh_command[i] += suffix worker['mounts'] = [{ 'directory': '.', 'content': { 'artifact': 'public/build/mozharness.zip', 'task-id': { 'task-reference': '<build>' } }, 'format': 'zip' }] if build_platform.startswith('win'): worker['command'] = [ {'task-reference': ' '.join(mh_command)} ] else: mh_command_task_ref = [] for token in mh_command: mh_command_task_ref.append({'task-reference': token}) worker['command'] = [ mh_command_task_ref ]
def mozharness_test_on_generic_worker(config, job, taskdesc): run = job['run'] test = taskdesc['run']['test'] mozharness = test['mozharness'] worker = taskdesc['worker'] = job['worker'] bitbar_script = 'test-linux.sh' is_macosx = worker['os'] == 'macosx' is_windows = worker['os'] == 'windows' is_linux = worker['os'] == 'linux' or worker['os'] == 'linux-bitbar' is_bitbar = worker['os'] == 'linux-bitbar' assert is_macosx or is_windows or is_linux artifacts = [ { 'name': 'public/logs', 'path': 'logs', 'type': 'directory' }, ] # jittest doesn't have blob_upload_dir if test['test-name'] != 'jittest': artifacts.append({ 'name': 'public/test_info', 'path': 'build/blobber_upload_dir', 'type': 'directory' }) if is_bitbar: artifacts = [ { 'name': 'public/test/', 'path': 'artifacts/public', 'type': 'directory' }, { 'name': 'public/logs/', 'path': 'workspace/logs', 'type': 'directory' }, { 'name': 'public/test_info/', 'path': 'workspace/build/blobber_upload_dir', 'type': 'directory' }, ] if 'installer-url' in mozharness: installer_url = mozharness['installer-url'] else: upstream_task = '<build-signing>' if mozharness[ 'requires-signed-builds'] else '<build>' installer_url = get_artifact_url(upstream_task, mozharness['build-artifact-name']) worker['os-groups'] = test['os-groups'] # run-as-administrator is a feature for workers with UAC enabled and as such should not be # included in tasks on workers that have UAC disabled. Currently UAC is only enabled on # gecko Windows 10 workers, however this may be subject to change. Worker type # environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig # See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control # for more information about UAC. if test.get('run-as-administrator', False): if job['worker-type'].startswith('t-win10-64'): worker['run-as-administrator'] = True else: raise Exception('run-as-administrator not supported on {}'.format( job['worker-type'])) if test['reboot']: raise Exception('reboot: {} not supported on generic-worker'.format( test['reboot'])) worker['max-run-time'] = test['max-run-time'] worker['retry-exit-status'] = test['retry-exit-status'] worker['artifacts'] = artifacts env = worker.setdefault('env', {}) env['GECKO_HEAD_REPOSITORY'] = config.params['head_repository'] env['GECKO_HEAD_REV'] = config.params['head_rev'] # this list will get cleaned up / reduced / removed in bug 1354088 if is_macosx: env.update({ 'IDLEIZER_DISABLE_SHUTDOWN': 'true', 'LANG': 'en_US.UTF-8', 'LC_ALL': 'en_US.UTF-8', 'MOZ_HIDE_RESULTS_TABLE': '1', 'MOZ_NODE_PATH': '/usr/local/bin/node', 'MOZ_NO_REMOTE': '1', 'NO_FAIL_ON_TEST_ERRORS': '1', 'PATH': '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', 'SHELL': '/bin/bash', 'XPCOM_DEBUG_BREAK': 'warn', 'XPC_FLAGS': '0x0', 'XPC_SERVICE_NAME': '0', }) elif is_bitbar: env.update({ 'MOZHARNESS_CONFIG': ' '.join(mozharness['config']), 'MOZHARNESS_SCRIPT': mozharness['script'], 'MOZHARNESS_URL': { 'artifact-reference': '<build/public/build/mozharness.zip>' }, 'MOZILLA_BUILD_URL': { 'task-reference': installer_url }, "MOZ_NO_REMOTE": '1', "NEED_XVFB": "false", "XPCOM_DEBUG_BREAK": 'warn', "NO_FAIL_ON_TEST_ERRORS": '1', "MOZ_HIDE_RESULTS_TABLE": '1', "MOZ_NODE_PATH": "/usr/local/bin/node", 'TASKCLUSTER_WORKER_TYPE': job['worker-type'], }) extra_config = { 'installer_url': installer_url, 'test_packages_url': test_packages_url(taskdesc), } env['EXTRA_MOZHARNESS_CONFIG'] = { 'task-reference': json.dumps(extra_config) } if is_windows: mh_command = [ 'c:\\mozilla-build\\python\\python.exe', '-u', 'mozharness\\scripts\\' + normpath(mozharness['script']) ] elif is_bitbar: mh_command = ['bash', "./{}".format(bitbar_script)] elif is_macosx and 'macosx1014-64' in test['test-platform']: mh_command = [ '/usr/local/bin/python2', '-u', 'mozharness/scripts/' + mozharness['script'] ] else: # is_linux or is_macosx mh_command = [ # Using /usr/bin/python2.7 rather than python2.7 because # /usr/local/bin/python2.7 is broken on the mac workers. # See bug #1547903. '/usr/bin/python2.7', '-u', 'mozharness/scripts/' + mozharness['script'] ] for mh_config in mozharness['config']: cfg_path = 'mozharness/configs/' + mh_config if is_windows: cfg_path = normpath(cfg_path) mh_command.extend(['--cfg', cfg_path]) mh_command.extend(mozharness.get('extra-options', [])) if mozharness.get('download-symbols'): if isinstance(mozharness['download-symbols'], basestring): mh_command.extend( ['--download-symbols', mozharness['download-symbols']]) else: mh_command.extend(['--download-symbols', 'true']) if mozharness.get('include-blob-upload-branch'): mh_command.append('--blob-upload-branch=' + config.params['project']) # TODO: remove the need for run['chunked'] if mozharness.get('chunked') or test['chunks'] > 1: mh_command.append('--total-chunk={}'.format(test['chunks'])) mh_command.append('--this-chunk={}'.format(test['this-chunk'])) if config.params.is_try(): env['TRY_COMMIT_MSG'] = config.params['message'] worker['mounts'] = [{ 'directory': '.', 'content': { 'artifact': get_artifact_path(taskdesc, 'mozharness.zip'), 'task-id': { 'task-reference': '<build>' } }, 'format': 'zip' }] if is_bitbar: a_url = config.params.file_url( 'taskcluster/scripts/tester/{}'.format(bitbar_script), ) worker['mounts'] = [{ 'file': bitbar_script, 'content': { 'url': a_url, }, }] job['run'] = { 'workdir': run['workdir'], 'tooltool-downloads': mozharness['tooltool-downloads'], 'checkout': test['checkout'], 'command': mh_command, 'using': 'run-task', } if is_bitbar: job['run']['run-as-root'] = True # FIXME: The bitbar config incorrectly requests internal tooltool downloads # so force it off here. job['run']['tooltool-downloads'] = False configure_taskdesc_for_run(config, job, taskdesc, worker['implementation'])
def mozharness_test_on_generic_worker(config, job, taskdesc): test = taskdesc["run"]["test"] mozharness = test["mozharness"] worker = taskdesc["worker"] = job["worker"] bitbar_script = "test-linux.sh" is_macosx = worker["os"] == "macosx" is_windows = worker["os"] == "windows" is_linux = worker["os"] == "linux" or worker["os"] == "linux-bitbar" is_bitbar = worker["os"] == "linux-bitbar" assert is_macosx or is_windows or is_linux artifacts = [ { "name": "public/logs", "path": "logs", "type": "directory" }, ] # jittest doesn't have blob_upload_dir if test["test-name"] != "jittest": artifacts.append({ "name": "public/test_info", "path": "build/blobber_upload_dir", "type": "directory", }) if is_bitbar: artifacts = [ { "name": "public/test/", "path": "artifacts/public", "type": "directory" }, { "name": "public/logs/", "path": "workspace/logs", "type": "directory" }, { "name": "public/test_info/", "path": "workspace/build/blobber_upload_dir", "type": "directory", }, ] installer = installer_url(taskdesc) worker["os-groups"] = test["os-groups"] # run-as-administrator is a feature for workers with UAC enabled and as such should not be # included in tasks on workers that have UAC disabled. Currently UAC is only enabled on # gecko Windows 10 workers, however this may be subject to change. Worker type # environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig # See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control # for more information about UAC. if test.get("run-as-administrator", False): if job["worker-type"].startswith("t-win10-64"): worker["run-as-administrator"] = True else: raise Exception("run-as-administrator not supported on {}".format( job["worker-type"])) if test["reboot"]: raise Exception("reboot: {} not supported on generic-worker".format( test["reboot"])) worker["max-run-time"] = test["max-run-time"] worker["retry-exit-status"] = test["retry-exit-status"] worker.setdefault("artifacts", []) worker["artifacts"].extend(artifacts) env = worker.setdefault("env", {}) env["GECKO_HEAD_REPOSITORY"] = config.params["head_repository"] env["GECKO_HEAD_REV"] = config.params["head_rev"] # this list will get cleaned up / reduced / removed in bug 1354088 if is_macosx: env.update({ "LC_ALL": "en_US.UTF-8", "LANG": "en_US.UTF-8", "MOZ_NODE_PATH": "/usr/local/bin/node", "PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", "SHELL": "/bin/bash", }) elif is_bitbar: env.update({ "MOZHARNESS_CONFIG": " ".join(mozharness["config"]), "MOZHARNESS_SCRIPT": mozharness["script"], "MOZHARNESS_URL": { "artifact-reference": "<build/public/build/mozharness.zip>" }, "MOZILLA_BUILD_URL": { "task-reference": installer }, "MOZ_NO_REMOTE": "1", "NEED_XVFB": "false", "XPCOM_DEBUG_BREAK": "warn", "NO_FAIL_ON_TEST_ERRORS": "1", "MOZ_HIDE_RESULTS_TABLE": "1", "MOZ_NODE_PATH": "/usr/local/bin/node", "TASKCLUSTER_WORKER_TYPE": job["worker-type"], }) extra_config = { "installer_url": installer, "test_packages_url": test_packages_url(taskdesc), } env["EXTRA_MOZHARNESS_CONFIG"] = { "task-reference": six.ensure_text(json.dumps(extra_config, sort_keys=True)) } # Bug 1634554 - pass in decision task artifact URL to mozharness for WPT. # Bug 1645974 - test-verify-wpt and test-coverage-wpt need artifact URL. if "web-platform-tests" in test["suite"] or re.match( "test-(coverage|verify)-wpt", test["suite"]): env["TESTS_BY_MANIFEST_URL"] = { "artifact-reference": "<decision/public/tests-by-manifest.json.gz>" } py_3 = test.get("python-3", False) if is_windows: py_binary = "c:\\mozilla-build\\{python}\\{python}.exe".format( python="python3" if py_3 else "python") mh_command = [ py_binary, "-u", "mozharness\\scripts\\" + normpath(mozharness["script"]), ] elif is_bitbar: py_binary = "python3" if py_3 else "python" mh_command = ["bash", "./{}".format(bitbar_script)] elif is_macosx and "macosx1014-64" in test["test-platform"]: py_binary = "/usr/local/bin/{}".format( "python3" if py_3 else "python2") mh_command = [ py_binary, "-u", "mozharness/scripts/" + mozharness["script"], ] else: # is_linux or is_macosx py_binary = "/usr/bin/{}".format("python3" if py_3 else "python2") mh_command = [ # Using /usr/bin/python2.7 rather than python2.7 because # /usr/local/bin/python2.7 is broken on the mac workers. # See bug #1547903. py_binary, "-u", "mozharness/scripts/" + mozharness["script"], ] if py_3: env["PYTHON"] = py_binary for mh_config in mozharness["config"]: cfg_path = "mozharness/configs/" + mh_config if is_windows: cfg_path = normpath(cfg_path) mh_command.extend(["--cfg", cfg_path]) mh_command.extend(mozharness.get("extra-options", [])) if mozharness.get("download-symbols"): if isinstance(mozharness["download-symbols"], text_type): mh_command.extend( ["--download-symbols", mozharness["download-symbols"]]) else: mh_command.extend(["--download-symbols", "true"]) if mozharness.get("include-blob-upload-branch"): mh_command.append("--blob-upload-branch=" + config.params["project"]) if test.get("test-manifests"): env["MOZHARNESS_TEST_PATHS"] = six.ensure_text( json.dumps({test["suite"]: test["test-manifests"]}, sort_keys=True)) # TODO: remove the need for run['chunked'] elif mozharness.get("chunked") or test["chunks"] > 1: mh_command.append("--total-chunk={}".format(test["chunks"])) mh_command.append("--this-chunk={}".format(test["this-chunk"])) if config.params.is_try(): env["TRY_COMMIT_MSG"] = config.params["message"] worker["mounts"] = [{ "directory": "mozharness", "content": { "artifact": get_artifact_path(taskdesc, "mozharness.zip"), "task-id": { "task-reference": "<build>" }, }, "format": "zip", }] if is_bitbar: a_url = config.params.file_url( "taskcluster/scripts/tester/{}".format(bitbar_script), ) worker["mounts"] = [{ "file": bitbar_script, "content": { "url": a_url, }, }] job["run"] = { "tooltool-downloads": mozharness["tooltool-downloads"], "checkout": test["checkout"], "command": mh_command, "using": "run-task", } if is_bitbar: job["run"]["run-as-root"] = True configure_taskdesc_for_run(config, job, taskdesc, worker["implementation"])
def mozharness_test_on_windows(config, job, taskdesc): test = taskdesc['run']['test'] mozharness = test['mozharness'] worker = taskdesc['worker'] artifacts = [{ 'path': 'public\\logs\\localconfig.json', 'type': 'file' }, { 'path': 'public\\logs\\log_critical.log', 'type': 'file' }, { 'path': 'public\\logs\\log_error.log', 'type': 'file' }, { 'path': 'public\\logs\\log_fatal.log', 'type': 'file' }, { 'path': 'public\\logs\\log_info.log', 'type': 'file' }, { 'path': 'public\\logs\\log_raw.log', 'type': 'file' }, { 'path': 'public\\logs\\log_warning.log', 'type': 'file' }, { 'path': 'public\\test_info', 'type': 'directory' }] build_platform = taskdesc['attributes']['build_platform'] target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform) installer_url = get_artifact_url('<build>', 'public/build/{}.zip'.format(target)) test_packages_url = get_artifact_url( '<build>', 'public/build/{}.test_packages.json'.format(target)) mozharness_url = get_artifact_url('<build>', 'public/build/mozharness.zip') taskdesc['scopes'].extend([ 'generic-worker:os-group:{}'.format(group) for group in test['os-groups'] ]) worker['os-groups'] = test['os-groups'] worker['max-run-time'] = test['max-run-time'] worker['artifacts'] = artifacts # assemble the command line mh_command = [ 'c:\\mozilla-build\\python\\python.exe', '-u', 'mozharness\\scripts\\' + normpath(mozharness['script']) ] for mh_config in mozharness['config']: mh_command.extend( ['--cfg', 'mozharness\\configs\\' + normpath(mh_config)]) mh_command.extend(mozharness.get('extra-options', [])) if mozharness.get('no-read-buildbot-config'): mh_command.append('--no-read-buildbot-config') mh_command.extend(['--installer-url', installer_url]) mh_command.extend(['--test-packages-url', test_packages_url]) if mozharness.get('download-symbols'): if isinstance(mozharness['download-symbols'], basestring): mh_command.extend( ['--download-symbols', mozharness['download-symbols']]) else: mh_command.extend(['--download-symbols', 'true']) # TODO: remove the need for run['chunked'] if mozharness.get('chunked') or test['chunks'] > 1: # Implement mozharness['chunking-args'], modifying command in place if mozharness['chunking-args'] == 'this-chunk': mh_command.append('--total-chunk={}'.format(test['chunks'])) mh_command.append('--this-chunk={}'.format(test['this-chunk'])) elif mozharness['chunking-args'] == 'test-suite-suffix': suffix = mozharness['chunk-suffix'].replace( '<CHUNK>', str(test['this-chunk'])) for i, c in enumerate(mh_command): if isinstance(c, basestring) and c.startswith('--test-suite'): mh_command[i] += suffix # bug 1311966 - symlink to artifacts until generic worker supports virtual artifact paths artifact_link_commands = [ 'mklink /d %cd%\\public\\test_info %cd%\\build\\blobber_upload_dir' ] for link in [ a['path'] for a in artifacts if a['path'].startswith('public\\logs\\') ]: artifact_link_commands.append('mklink %cd%\\{} %cd%\\{}'.format( link, link[7:])) worker['command'] = artifact_link_commands + [{ 'task-reference': 'c:\\mozilla-build\\wget\\wget.exe {}'.format(mozharness_url) }, 'c:\\mozilla-build\\info-zip\\unzip.exe mozharness.zip', { 'task-reference': ' '.join(mh_command) }]