def docker_worker_toolchain(config, job, taskdesc): run = job['run'] worker = taskdesc['worker'] worker['artifacts'] = [] worker['caches'] = [] worker['artifacts'].append({ 'name': 'public', 'path': '/home/worker/workspace/artifacts/', 'type': 'directory', }) docker_worker_add_tc_vcs_cache(config, job, taskdesc) docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) support_vcs_checkout(config, job, taskdesc) env = worker['env'] env.update({ 'MOZ_BUILD_DATE': config.params['moz_build_date'], 'MOZ_SCM_LEVEL': config.params['level'], 'TOOLS_DISABLE': 'true', }) # tooltool downloads. By default we download using the API endpoint, but # the job can optionally request relengapi-proxy (for example when downloading # internal tooltool resources. So we define the tooltool cache unconditionally. worker['caches'].append({ 'type': 'persistent', 'name': 'tooltool-cache', 'mount-point': '/home/worker/tooltool-cache', }) env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache' # tooltool downloads worker['relengapi-proxy'] = False # but maybe enabled for tooltool below if run['tooltool-downloads']: worker['relengapi-proxy'] = True taskdesc['scopes'].extend([ 'docker-worker:relengapi-proxy:tooltool.download.public', ]) if run['tooltool-downloads'] == 'internal': taskdesc['scopes'].append( 'docker-worker:relengapi-proxy:tooltool.download.internal') worker['command'] = [ '/home/worker/bin/run-task', # Various caches/volumes are default owned by root:root. '--chown-recursive', '/home/worker/workspace', '--chown-recursive', '/home/worker/tooltool-cache', '--vcs-checkout=/home/worker/workspace/build/src', '--', 'bash', '-c', 'cd /home/worker && ' './workspace/build/src/taskcluster/scripts/misc/{}'.format( run['script']) ] add_optimizations(config, run, taskdesc)
def docker_worker_phone_builder(config, job, taskdesc): run = job['run'] worker = taskdesc.get('worker') worker['artifacts'] = [{ 'name': 'private/build', 'path': '/home/worker/artifacts/', 'type': 'directory', }, { 'name': 'public/build', 'path': '/home/worker/artifacts-public/', 'type': 'directory', }] docker_worker_add_tc_vcs_cache(config, job, taskdesc) docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) if config.params['project'] != 'try': taskdesc['worker']['caches'].append({ 'type': 'persistent', 'name': 'level-{}-{}-build-{}-{}'.format( config.params['level'], config.params['project'], taskdesc['attributes']['build_platform'], taskdesc['attributes']['build_type'],), 'mount-point': "/home/worker/workspace", }) taskdesc['worker']['caches'].append({ 'type': 'persistent', 'name': 'level-{}-{}-build-{}-{}-objdir-gecko'.format( config.params['level'], config.params['project'], taskdesc['attributes']['build_platform'], taskdesc['attributes']['build_type'],), 'mount-point': "/home/worker/objdir-gecko", }) env = worker.setdefault('env', {}) env.update({ 'MOZHARNESS_CONFIG': 'b2g/taskcluster-phone-eng.py', 'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])), 'TARGET': run['target'], }) if run['debug']: env['B2G_DEBUG'] = '1' # tooltool downloads worker['relengapi-proxy'] = True taskdesc['scopes'].extend([ 'docker-worker:relengapi-proxy:tooltool.download.internal', 'docker-worker:relengapi-proxy:tooltool.download.public', ]) worker['command'] = [ "/bin/bash", "-c", "checkout-gecko workspace" " && cd ./workspace/gecko/taskcluster/scripts/phone-builder" " && buildbot_step 'Build' ./build-phone.sh $HOME/workspace", ]
def docker_worker_spidermonkey(config, job, taskdesc, schema=sm_run_schema): run = job['run'] worker = taskdesc['worker'] worker['artifacts'] = [] worker['caches'] = [] if int(config.params['level']) > 1: worker['caches'].append({ 'type': 'persistent', 'name': 'level-{}-{}-build-spidermonkey-workspace'.format( config.params['level'], config.params['project']), 'mount-point': "/home/worker/workspace", }) docker_worker_add_tc_vcs_cache(config, job, taskdesc) docker_worker_add_public_artifacts(config, job, taskdesc) docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) env = worker['env'] env.update({ 'MOZHARNESS_DISABLE': 'true', 'TOOLS_DISABLE': 'true', 'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'], 'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])), 'MOZ_SCM_LEVEL': config.params['level'], }) # tooltool downloads; note that this script downloads using the API # endpoiint directly, rather than via relengapi-proxy worker['caches'].append({ 'type': 'persistent', 'name': 'tooltool-cache', 'mount-point': '/home/worker/tooltool-cache', }) env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache' env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool' env['TOOLTOOL_REV'] = 'master' if run.get('tooltool-manifest'): env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest'] script = "build-sm.sh" if run['using'] == 'spidermonkey-package': script = "build-sm-package.sh" worker['command'] = [ "/bin/bash", "-c", "cd /home/worker/ " "&& ./bin/checkout-sources.sh " "&& ./workspace/build/src/taskcluster/scripts/builder/" + script ]
def docker_worker_toolchain(config, job, taskdesc): run = job['run'] worker = taskdesc['worker'] worker['artifacts'] = [] worker['caches'] = [] worker['artifacts'].append({ 'name': 'public', 'path': '/home/worker/workspace/artifacts/', 'type': 'directory', }) docker_worker_add_tc_vcs_cache(config, job, taskdesc) docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) support_vcs_checkout(config, job, taskdesc) env = worker['env'] env.update({ 'MOZ_BUILD_DATE': config.params['moz_build_date'], 'MOZ_SCM_LEVEL': config.params['level'], 'TOOLS_DISABLE': 'true', }) # tooltool downloads. By default we download using the API endpoint, but # the job can optionally request relengapi-proxy (for example when downloading # internal tooltool resources. So we define the tooltool cache unconditionally. worker['caches'].append({ 'type': 'persistent', 'name': 'tooltool-cache', 'mount-point': '/home/worker/tooltool-cache', }) env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache' # tooltool downloads worker['relengapi-proxy'] = False # but maybe enabled for tooltool below if run['tooltool-downloads']: worker['relengapi-proxy'] = True taskdesc['scopes'].extend([ 'docker-worker:relengapi-proxy:tooltool.download.public', ]) if run['tooltool-downloads'] == 'internal': taskdesc['scopes'].append( 'docker-worker:relengapi-proxy:tooltool.download.internal') worker['command'] = [ '/home/worker/bin/run-task', '--vcs-checkout=/home/worker/workspace/build/src', '--', 'bash', '-c', 'cd /home/worker && ' './workspace/build/src/taskcluster/scripts/misc/{}'.format( run['script']) ] add_optimizations(config, run, taskdesc)
def docker_worker_make_via_build_mulet_linux_sh(config, job, taskdesc): run = job['run'] worker = taskdesc.get('worker') # assumes the builder image (which contains the gecko checkout command) taskdesc['worker']['docker-image'] = {"in-tree": "builder"} worker['taskcluster-proxy'] = False docker_worker_add_public_artifacts(config, job, taskdesc) docker_worker_add_tc_vcs_cache(config, job, taskdesc) docker_worker_add_workspace_cache(config, job, taskdesc) docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) env = worker.setdefault('env', {}) env.update({ 'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])), 'MOZ_SCM_LEVEL': config.params['level'], }) env['MOZCONFIG'] = run['mozconfig'] # tooltool downloads (not via relengapi proxy) worker['caches'].append({ 'type': 'persistent', 'name': 'tooltool-cache', 'mount-point': '/home/worker/tooltool-cache', }) env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache' env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest'] env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool' env['TOOLTOOL_REV'] = 'master' worker['command'] = [ "/bin/bash", "-c", "checkout-gecko workspace" " && cd ./workspace/gecko/taskcluster/scripts/builder" " && buildbot_step 'Build' ./build-mulet-linux.sh $HOME/workspace", ]
def docker_worker_hazard(config, job, taskdesc): run = job['run'] worker = taskdesc['worker'] worker['artifacts'] = [] worker['caches'] = [] docker_worker_add_tc_vcs_cache(config, job, taskdesc) docker_worker_add_public_artifacts(config, job, taskdesc) docker_worker_add_workspace_cache(config, job, taskdesc) docker_worker_setup_secrets(config, job, taskdesc) docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) env = worker['env'] env.update({ 'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])), 'MOZ_SCM_LEVEL': config.params['level'], }) # script parameters if run.get('tooltool-manifest'): env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest'] if run.get('mozconfig'): env['MOZCONFIG'] = run['mozconfig'] # tooltool downloads worker['caches'].append({ 'type': 'persistent', 'name': 'tooltool-cache', 'mount-point': '/home/worker/tooltool-cache', }) worker['relengapi-proxy'] = True taskdesc['scopes'].extend([ 'docker-worker:relengapi-proxy:tooltool.download.public', ]) env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache' env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool' env['TOOLTOOL_REV'] = 'master' worker['command'] = ["/bin/bash", "-c", run['command']]
def docker_worker_toolchain(config, job, taskdesc): run = job['run'] worker = taskdesc['worker'] worker['artifacts'] = [] worker['caches'] = [] worker['artifacts'].append({ 'name': 'public', 'path': '/home/worker/workspace/artifacts/', 'type': 'directory', }) docker_worker_add_tc_vcs_cache(config, job, taskdesc) docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) env = worker['env'] env.update({ 'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])), 'MOZ_SCM_LEVEL': config.params['level'], 'TOOLS_DISABLE': 'true', }) # tooltool downloads; note that this downloads using the API endpoint directly, # rather than via relengapi-proxy worker['caches'].append({ 'type': 'persistent', 'name': 'tooltool-cache', 'mount-point': '/home/worker/tooltool-cache', }) env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache' env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool' env['TOOLTOOL_REV'] = 'master' command = ' && '.join([ "cd /home/worker/", "./bin/checkout-sources.sh", "./workspace/build/src/taskcluster/scripts/misc/" + run['script'], ]) worker['command'] = ["/bin/bash", "-c", command]
def docker_worker_toolchain(config, job, taskdesc): run = job['run'] worker = taskdesc['worker'] worker['artifacts'] = [] worker['caches'] = [] worker['artifacts'].append({ 'name': 'public', 'path': '/home/worker/workspace/artifacts/', 'type': 'directory', }) docker_worker_add_tc_vcs_cache(config, job, taskdesc) docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) env = worker['env'] env.update({ 'MOZ_BUILD_DATE': config.params['moz_build_date'], 'MOZ_SCM_LEVEL': config.params['level'], 'TOOLS_DISABLE': 'true', }) # tooltool downloads; note that this downloads using the API endpoint directly, # rather than via relengapi-proxy worker['caches'].append({ 'type': 'persistent', 'name': 'tooltool-cache', 'mount-point': '/home/worker/tooltool-cache', }) env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache' env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool' env['TOOLTOOL_REV'] = 'master' command = ' && '.join([ "cd /home/worker/", "./bin/checkout-sources.sh", "./workspace/build/src/taskcluster/scripts/misc/" + run['script'], ]) worker['command'] = ["/bin/bash", "-c", command]
def docker_worker_toolchain(config, job, taskdesc): run = job['run'] taskdesc['run-on-projects'] = ['trunk', 'try'] worker = taskdesc['worker'] worker['artifacts'] = [] worker['chain-of-trust'] = True docker_worker_add_public_artifacts(config, job, taskdesc) docker_worker_add_tc_vcs_cache(config, job, taskdesc) docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) support_vcs_checkout(config, job, taskdesc) env = worker['env'] env.update({ 'MOZ_BUILD_DATE': config.params['moz_build_date'], 'MOZ_SCM_LEVEL': config.params['level'], 'TOOLS_DISABLE': 'true', 'MOZ_AUTOMATION': '1', }) if run['tooltool-downloads']: internal = run['tooltool-downloads'] == 'internal' docker_worker_add_tooltool(config, job, taskdesc, internal=internal) worker['command'] = [ '/builds/worker/bin/run-task', '--vcs-checkout=/builds/worker/workspace/build/src', '--', 'bash', '-c', 'cd /builds/worker && ' './workspace/build/src/taskcluster/scripts/misc/{}'.format( run['script']) ] attributes = taskdesc.setdefault('attributes', {}) attributes['toolchain-artifact'] = run['toolchain-artifact'] if 'toolchain-alias' in run: attributes['toolchain-alias'] = run['toolchain-alias'] add_optimizations(config, run, taskdesc)
def docker_worker_toolchain(config, job, taskdesc): run = job["run"] worker = taskdesc["worker"] worker["artifacts"] = [] worker["caches"] = [] worker["artifacts"].append({"name": "public", "path": "/home/worker/workspace/artifacts/", "type": "directory"}) docker_worker_add_tc_vcs_cache(config, job, taskdesc) docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) env = worker["env"] env.update( { "MOZ_BUILD_DATE": time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params["pushdate"])), "MOZ_SCM_LEVEL": config.params["level"], "TOOLS_DISABLE": "true", } ) # tooltool downloads; note that this downloads using the API endpoint directly, # rather than via relengapi-proxy worker["caches"].append( {"type": "persistent", "name": "tooltool-cache", "mount-point": "/home/worker/tooltool-cache"} ) env["TOOLTOOL_CACHE"] = "/home/worker/tooltool-cache" env["TOOLTOOL_REPO"] = "https://github.com/mozilla/build-tooltool" env["TOOLTOOL_REV"] = "master" command = " && ".join( [ "cd /home/worker/", "./bin/checkout-sources.sh", "./workspace/build/src/taskcluster/scripts/misc/" + run["script"], ] ) worker["command"] = ["/bin/bash", "-c", command]
def mozharness_on_docker_worker_setup(config, job, taskdesc): run = job['run'] worker = taskdesc['worker'] worker['implementation'] = job['worker']['implementation'] # running via mozharness assumes desktop-build (which contains build.sh) taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"} worker['relengapi-proxy'] = False # but maybe enabled for tooltool below worker['taskcluster-proxy'] = run.get('taskcluster-proxy') docker_worker_add_public_artifacts(config, job, taskdesc) docker_worker_add_tc_vcs_cache(config, job, taskdesc) docker_worker_add_workspace_cache(config, job, taskdesc) docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) env = worker.setdefault('env', {}) env.update({ 'MOZHARNESS_CONFIG': ' '.join(run['config']), 'MOZHARNESS_SCRIPT': run['script'], 'MH_BRANCH': config.params['project'], 'MH_BUILD_POOL': 'taskcluster', 'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])), 'MOZ_SCM_LEVEL': config.params['level'], }) if 'actions' in run: env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions']) if 'options' in run: env['MOZHARNESS_OPTIONS'] = ' '.join(run['options']) if 'custom-build-variant-cfg' in run: env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg'] # if we're not keeping artifacts, set some env variables to empty values # that will cause the build process to skip copying the results to the # artifacts directory. This will have no effect for operations that are # not builds. if not run['keep-artifacts']: env['DIST_TARGET_UPLOADS'] = '' env['DIST_UPLOADS'] = '' # Xvfb if run['need-xvfb']: env['NEED_XVFB'] = 'true' # tooltool downloads if run['tooltool-downloads']: worker['relengapi-proxy'] = True worker['caches'].append({ 'type': 'persistent', 'name': 'tooltool-cache', 'mount-point': '/home/worker/tooltool-cache', }) taskdesc['scopes'].extend([ 'docker-worker:relengapi-proxy:tooltool.download.public', ]) if run['tooltool-downloads'] == 'internal': taskdesc['scopes'].append( 'docker-worker:relengapi-proxy:tooltool.download.internal') env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache' env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool' env['TOOLTOOL_REV'] = 'master' docker_worker_setup_secrets(config, job, taskdesc) worker['command'] = ["/bin/bash", "bin/build.sh"]
def docker_worker_phone_builder(config, job, taskdesc): run = job['run'] worker = taskdesc.get('worker') worker['artifacts'] = [{ 'name': 'private/build', 'path': '/home/worker/artifacts/', 'type': 'directory', }, { 'name': 'public/build', 'path': '/home/worker/artifacts-public/', 'type': 'directory', }] docker_worker_add_tc_vcs_cache(config, job, taskdesc) docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) if config.params['project'] != 'try': taskdesc['worker']['caches'].append({ 'type': 'persistent', 'name': 'level-{}-{}-build-{}-{}'.format( config.params['level'], config.params['project'], taskdesc['attributes']['build_platform'], taskdesc['attributes']['build_type'], ), 'mount-point': "/home/worker/workspace", }) taskdesc['worker']['caches'].append({ 'type': 'persistent', 'name': 'level-{}-{}-build-{}-{}-objdir-gecko'.format( config.params['level'], config.params['project'], taskdesc['attributes']['build_platform'], taskdesc['attributes']['build_type'], ), 'mount-point': "/home/worker/objdir-gecko", }) env = worker.setdefault('env', {}) env.update({ 'MOZHARNESS_CONFIG': 'b2g/taskcluster-phone-eng.py', 'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])), 'TARGET': run['target'], }) if run['debug']: env['B2G_DEBUG'] = '1' # tooltool downloads worker['relengapi-proxy'] = True taskdesc['scopes'].extend([ 'docker-worker:relengapi-proxy:tooltool.download.internal', 'docker-worker:relengapi-proxy:tooltool.download.public', ]) worker['command'] = [ "/bin/bash", "-c", "checkout-gecko workspace" " && cd ./workspace/gecko/taskcluster/scripts/phone-builder" " && buildbot_step 'Build' ./build-phone.sh $HOME/workspace", ]