def docker_worker_hazard(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['artifacts'] = []
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_workspace_cache(config, job, taskdesc)
docker_worker_add_tooltool(config, job, taskdesc)
docker_worker_setup_secrets(config, job, taskdesc)
support_vcs_checkout(config, job, taskdesc)
env = worker['env']
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
})
# script parameters
if run.get('mozconfig'):
env['MOZCONFIG'] = run['mozconfig']
# build-haz-linux.sh needs this otherwise it assumes the checkout is in
# the workspace.
env['GECKO_DIR'] = '/builds/worker/checkouts/gecko'
worker['command'] = [
'/builds/worker/bin/run-task',
'--vcs-checkout', '/builds/worker/checkouts/gecko',
'--',
'/bin/bash', '-c', run['command']
]
def docker_worker_spidermonkey(config, job, taskdesc, schema=sm_run_schema):
run = job['run']
worker = taskdesc['worker']
worker['artifacts'] = []
worker['caches'] = []
if int(config.params['level']) > 1:
worker['caches'].append({
'type': 'persistent',
'name': 'level-{}-{}-build-spidermonkey-workspace'.format(
config.params['level'], config.params['project']),
'mount-point': "/home/worker/workspace",
})
docker_worker_add_public_artifacts(config, job, taskdesc)
env = worker['env']
env.update({
'MOZHARNESS_DISABLE': 'true',
'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'],
'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
'MOZ_SCM_LEVEL': config.params['level'],
})
# tooltool downloads; note that this script downloads using the API
# endpoiint directly, rather than via relengapi-proxy
worker['caches'].append({
'type': 'persistent',
'name': 'tooltool-cache',
'mount-point': '/home/worker/tooltool-cache',
})
env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
if run.get('tooltool-manifest'):
env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
docker_worker_support_vcs_checkout(config, job, taskdesc)
script = "build-sm.sh"
if run['using'] == 'spidermonkey-package':
script = "build-sm-package.sh"
elif run['using'] == 'spidermonkey-mozjs-crate':
script = "build-sm-mozjs-crate.sh"
worker['command'] = [
'/home/worker/bin/run-task',
'--chown-recursive', '/home/worker/workspace',
'--chown-recursive', '/home/worker/tooltool-cache',
'--vcs-checkout', '/home/worker/workspace/build/src',
'--',
'/bin/bash',
'-c',
'cd /home/worker && workspace/build/src/taskcluster/scripts/builder/%s' % script
]
def docker_worker_spidermonkey(config, job, taskdesc, schema=sm_run_schema):
run = job['run']
worker = taskdesc['worker']
worker['artifacts'] = []
worker['caches'] = []
if int(config.params['level']) > 1:
worker['caches'].append({
'type': 'persistent',
'name': 'level-{}-{}-build-spidermonkey-workspace'.format(
config.params['level'], config.params['project']),
'mount-point': "/home/worker/workspace",
})
docker_worker_add_tc_vcs_cache(config, job, taskdesc)
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
env = worker['env']
env.update({
'MOZHARNESS_DISABLE': 'true',
'TOOLS_DISABLE': 'true',
'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'],
'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
'MOZ_SCM_LEVEL': config.params['level'],
})
# tooltool downloads; note that this script downloads using the API
# endpoiint directly, rather than via relengapi-proxy
worker['caches'].append({
'type': 'persistent',
'name': 'tooltool-cache',
'mount-point': '/home/worker/tooltool-cache',
})
env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
env['TOOLTOOL_REV'] = 'master'
if run.get('tooltool-manifest'):
env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
script = "build-sm.sh"
if run['using'] == 'spidermonkey-package':
script = "build-sm-package.sh"
worker['command'] = [
"/bin/bash",
"-c",
"cd /home/worker/ "
"&& ./bin/checkout-sources.sh "
"&& ./workspace/build/src/taskcluster/scripts/builder/" + script
]
def docker_worker_hazard(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['artifacts'] = []
worker['caches'] = []
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_workspace_cache(config, job, taskdesc)
docker_worker_setup_secrets(config, job, taskdesc)
docker_worker_support_vcs_checkout(config, job, taskdesc)
env = worker['env']
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
})
# script parameters
if run.get('tooltool-manifest'):
env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
if run.get('mozconfig'):
env['MOZCONFIG'] = run['mozconfig']
# tooltool downloads
worker['caches'].append({
'type': 'persistent',
'name': 'tooltool-cache',
'mount-point': '/home/worker/tooltool-cache',
})
worker['relengapi-proxy'] = True
taskdesc['scopes'].extend([
'docker-worker:relengapi-proxy:tooltool.download.public',
])
env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
env['TOOLTOOL_REV'] = 'master'
# build-haz-linux.sh needs this otherwise it assumes the checkout is in
# the workspace.
env['GECKO_DIR'] = '/home/worker/checkouts/gecko'
worker['command'] = [
'/home/worker/bin/run-task',
'--chown-recursive', '/home/worker/tooltool-cache',
'--chown-recursive', '/home/worker/workspace',
'--vcs-checkout', '/home/worker/checkouts/gecko',
'--',
'/bin/bash', '-c', run['command']
]
def docker_worker_mulet_simulator(config, job, taskdesc):
run = job['run']
worker = taskdesc.get('worker')
# assumes the builder image (which contains the gecko checkout command)
taskdesc['worker']['docker-image'] = {"in-tree": "builder"}
worker['taskcluster-proxy'] = False
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
shell_command = run['shell-command'].format(**config.params)
worker['command'] = [
"/bin/bash",
"-exc",
{'task-reference': shell_command},
]
def docker_worker_spidermonkey(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['artifacts'] = []
worker.setdefault('caches', []).append({
'type': 'persistent',
'name': 'level-{}-{}-build-spidermonkey-workspace'.format(
config.params['level'], config.params['project']),
'mount-point': "/builds/worker/workspace",
'skip-untrusted': True,
})
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_tooltool(config, job, taskdesc)
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_DISABLE': 'true',
'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'],
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
})
support_vcs_checkout(config, job, taskdesc)
script = "build-sm.sh"
if run['using'] == 'spidermonkey-package':
script = "build-sm-package.sh"
elif run['using'] == 'spidermonkey-mozjs-crate':
script = "build-sm-mozjs-crate.sh"
elif run['using'] == 'spidermonkey-rust-bindings':
script = "build-sm-rust-bindings.sh"
worker['command'] = [
'/builds/worker/bin/run-task',
'--vcs-checkout', '/builds/worker/workspace/build/src',
'--',
'/bin/bash',
'-c',
'cd /builds/worker && workspace/build/src/taskcluster/scripts/builder/%s' % script
]
def docker_worker_make_via_build_mulet_linux_sh(config, job, taskdesc):
run = job['run']
worker = taskdesc.get('worker')
# assumes the builder image (which contains the gecko checkout command)
taskdesc['worker']['docker-image'] = {"in-tree": "builder"}
worker['taskcluster-proxy'] = False
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_tc_vcs_cache(config, job, taskdesc)
docker_worker_add_workspace_cache(config, job, taskdesc)
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
env = worker.setdefault('env', {})
env.update({
'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
'MOZ_SCM_LEVEL': config.params['level'],
})
env['MOZCONFIG'] = run['mozconfig']
# tooltool downloads (not via relengapi proxy)
worker['caches'].append({
'type': 'persistent',
'name': 'tooltool-cache',
'mount-point': '/home/worker/tooltool-cache',
})
env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
env['TOOLTOOL_REV'] = 'master'
worker['command'] = [
"/bin/bash",
"-c",
"checkout-gecko workspace"
" && cd ./workspace/gecko/taskcluster/scripts/builder"
" && buildbot_step 'Build' ./build-mulet-linux.sh $HOME/workspace",
]
def docker_worker_hazard(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['artifacts'] = []
worker['caches'] = []
docker_worker_add_tc_vcs_cache(config, job, taskdesc)
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_workspace_cache(config, job, taskdesc)
docker_worker_setup_secrets(config, job, taskdesc)
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
env = worker['env']
env.update({
'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
'MOZ_SCM_LEVEL': config.params['level'],
})
# script parameters
if run.get('tooltool-manifest'):
env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
if run.get('mozconfig'):
env['MOZCONFIG'] = run['mozconfig']
# tooltool downloads
worker['caches'].append({
'type': 'persistent',
'name': 'tooltool-cache',
'mount-point': '/home/worker/tooltool-cache',
})
worker['relengapi-proxy'] = True
taskdesc['scopes'].extend([
'docker-worker:relengapi-proxy:tooltool.download.public',
])
env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
env['TOOLTOOL_REV'] = 'master'
worker['command'] = ["/bin/bash", "-c", run['command']]
def docker_worker_toolchain(config, job, taskdesc):
run = job['run']
taskdesc['run-on-projects'] = ['trunk', 'try']
worker = taskdesc['worker']
worker['artifacts'] = []
worker['caches'] = []
worker['chain-of-trust'] = True
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_tc_vcs_cache(config, job, taskdesc)
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
support_vcs_checkout(config, job, taskdesc)
env = worker['env']
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'TOOLS_DISABLE': 'true',
'MOZ_AUTOMATION': '1',
})
# tooltool downloads. By default we download using the API endpoint, but
# the job can optionally request relengapi-proxy (for example when downloading
# internal tooltool resources. So we define the tooltool cache unconditionally.
worker['caches'].append({
'type': 'persistent',
'name': 'tooltool-cache',
'mount-point': '/home/worker/tooltool-cache',
})
env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
# tooltool downloads
worker['relengapi-proxy'] = False # but maybe enabled for tooltool below
if run['tooltool-downloads']:
worker['relengapi-proxy'] = True
taskdesc['scopes'].extend([
'docker-worker:relengapi-proxy:tooltool.download.public',
])
if run['tooltool-downloads'] == 'internal':
taskdesc['scopes'].append(
'docker-worker:relengapi-proxy:tooltool.download.internal')
worker['command'] = [
'/home/worker/bin/run-task',
# Various caches/volumes are default owned by root:root.
'--chown-recursive',
'/home/worker/workspace',
'--chown-recursive',
'/home/worker/tooltool-cache',
'--vcs-checkout=/home/worker/workspace/build/src',
'--',
'bash',
'-c',
'cd /home/worker && '
'./workspace/build/src/taskcluster/scripts/misc/{}'.format(
run['script'])
]
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run['toolchain-artifact']
add_optimizations(config, run, taskdesc)
def mozharness_on_docker_worker_setup(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['implementation'] = job['worker']['implementation']
# running via mozharness assumes desktop-build (which contains build.sh)
taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"}
worker['relengapi-proxy'] = False # but maybe enabled for tooltool below
worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_workspace_cache(config, job, taskdesc)
support_vcs_checkout(config, job, taskdesc)
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG': ' '.join(run['config']),
'MOZHARNESS_SCRIPT': run['script'],
'MH_BRANCH': config.params['project'],
'MH_BUILD_POOL': 'taskcluster',
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'MOZ_AUTOMATION': '1',
})
if 'actions' in run:
env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
if 'options' in run:
env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
if 'custom-build-variant-cfg' in run:
env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
if 'job-script' in run:
env['JOB_SCRIPT'] = run['job-script']
# if we're not keeping artifacts, set some env variables to empty values
# that will cause the build process to skip copying the results to the
# artifacts directory. This will have no effect for operations that are
# not builds.
if not run['keep-artifacts']:
env['DIST_TARGET_UPLOADS'] = ''
env['DIST_UPLOADS'] = ''
# Xvfb
if run['need-xvfb']:
env['NEED_XVFB'] = 'true'
# tooltool downloads
if run['tooltool-downloads']:
worker['relengapi-proxy'] = True
worker['caches'].append({
'type': 'persistent',
'name': 'tooltool-cache',
'mount-point': '/home/worker/tooltool-cache',
})
taskdesc['scopes'].extend([
'docker-worker:relengapi-proxy:tooltool.download.public',
])
if run['tooltool-downloads'] == 'internal':
taskdesc['scopes'].append(
'docker-worker:relengapi-proxy:tooltool.download.internal')
env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
# Retry if mozharness returns TBPL_RETRY
worker['retry-exit-status'] = 4
docker_worker_setup_secrets(config, job, taskdesc)
command = [
'/home/worker/bin/run-task',
# Various caches/volumes are default owned by root:root.
'--chown-recursive',
'/home/worker/workspace',
'--chown-recursive',
'/home/worker/tooltool-cache',
'--vcs-checkout',
'/home/worker/workspace/build/src',
'--tools-checkout',
'/home/worker/workspace/build/tools',
'--',
]
command.append("/home/worker/workspace/build/src/{}".format(
run.get('job-script', "taskcluster/scripts/builder/build-linux.sh")))
worker['command'] = command
def mozharness_on_docker_worker_setup(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['implementation'] = job['worker']['implementation']
# running via mozharness assumes desktop-build (which contains build.sh)
taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"}
worker['relengapi-proxy'] = False # but maybe enabled for tooltool below
worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_tc_vcs_cache(config, job, taskdesc)
docker_worker_add_workspace_cache(config, job, taskdesc)
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG': ' '.join(run['config']),
'MOZHARNESS_SCRIPT': run['script'],
'MH_BRANCH': config.params['project'],
'MH_BUILD_POOL': 'taskcluster',
'MOZ_BUILD_DATE': time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
'MOZ_SCM_LEVEL': config.params['level'],
})
if 'actions' in run:
env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
if 'options' in run:
env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
if 'custom-build-variant-cfg' in run:
env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
# if we're not keeping artifacts, set some env variables to empty values
# that will cause the build process to skip copying the results to the
# artifacts directory. This will have no effect for operations that are
# not builds.
if not run['keep-artifacts']:
env['DIST_TARGET_UPLOADS'] = ''
env['DIST_UPLOADS'] = ''
# Xvfb
if run['need-xvfb']:
env['NEED_XVFB'] = 'true'
# tooltool downloads
if run['tooltool-downloads']:
worker['relengapi-proxy'] = True
worker['caches'].append({
'type': 'persistent',
'name': 'tooltool-cache',
'mount-point': '/home/worker/tooltool-cache',
})
taskdesc['scopes'].extend([
'docker-worker:relengapi-proxy:tooltool.download.public',
])
if run['tooltool-downloads'] == 'internal':
taskdesc['scopes'].append(
'docker-worker:relengapi-proxy:tooltool.download.internal')
env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
env['TOOLTOOL_REV'] = 'master'
docker_worker_setup_secrets(config, job, taskdesc)
worker['command'] = ["/bin/bash", "bin/build.sh"]
def mozharness_on_docker_worker_setup(config, job, taskdesc):
run = job["run"]
worker = taskdesc["worker"]
worker["implementation"] = job["worker"]["implementation"]
# running via mozharness assumes desktop-build (which contains build.sh)
taskdesc["worker"]["docker-image"] = {"in-tree": "desktop-build"}
worker["relengapi-proxy"] = False # but maybe enabled for tooltool below
worker["taskcluster-proxy"] = run.get("taskcluster-proxy")
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_workspace_cache(config, job, taskdesc)
docker_worker_support_vcs_checkout(config, job, taskdesc)
env = worker.setdefault("env", {})
env.update(
{
"MOZHARNESS_CONFIG": " ".join(run["config"]),
"MOZHARNESS_SCRIPT": run["script"],
"MH_BRANCH": config.params["project"],
"MH_BUILD_POOL": "taskcluster",
"MOZ_BUILD_DATE": config.params["moz_build_date"],
"MOZ_SCM_LEVEL": config.params["level"],
}
)
if "actions" in run:
env["MOZHARNESS_ACTIONS"] = " ".join(run["actions"])
if "options" in run:
env["MOZHARNESS_OPTIONS"] = " ".join(run["options"])
if "custom-build-variant-cfg" in run:
env["MH_CUSTOM_BUILD_VARIANT_CFG"] = run["custom-build-variant-cfg"]
if "job-script" in run:
env["JOB_SCRIPT"] = run["job-script"]
# if we're not keeping artifacts, set some env variables to empty values
# that will cause the build process to skip copying the results to the
# artifacts directory. This will have no effect for operations that are
# not builds.
if not run["keep-artifacts"]:
env["DIST_TARGET_UPLOADS"] = ""
env["DIST_UPLOADS"] = ""
# Xvfb
if run["need-xvfb"]:
env["NEED_XVFB"] = "true"
# tooltool downloads
if run["tooltool-downloads"]:
worker["relengapi-proxy"] = True
worker["caches"].append(
{"type": "persistent", "name": "tooltool-cache", "mount-point": "/home/worker/tooltool-cache"}
)
taskdesc["scopes"].extend(["docker-worker:relengapi-proxy:tooltool.download.public"])
if run["tooltool-downloads"] == "internal":
taskdesc["scopes"].append("docker-worker:relengapi-proxy:tooltool.download.internal")
env["TOOLTOOL_CACHE"] = "/home/worker/tooltool-cache"
env["TOOLTOOL_REPO"] = "https://github.com/mozilla/build-tooltool"
env["TOOLTOOL_REV"] = "master"
# Retry if mozharness returns TBPL_RETRY
worker["retry-exit-status"] = 4
docker_worker_setup_secrets(config, job, taskdesc)
command = [
"/home/worker/bin/run-task",
# Various caches/volumes are default owned by root:root.
"--chown-recursive",
"/home/worker/workspace",
"--chown-recursive",
"/home/worker/tooltool-cache",
"--vcs-checkout",
"/home/worker/workspace/build/src",
"--tools-checkout",
"/home/worker/workspace/build/tools",
"--",
]
command.append(
"/home/worker/workspace/build/src/{}".format(
run.get("job-script", "taskcluster/scripts/builder/build-linux.sh")
)
)
worker["command"] = command
def mozharness_on_docker_worker_setup(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['implementation'] = job['worker']['implementation']
# running via mozharness assumes desktop-build (which contains build.sh)
taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"}
worker['relengapi-proxy'] = False # but maybe enabled for tooltool below
worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_workspace_cache(config, job, taskdesc)
support_vcs_checkout(config, job, taskdesc)
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG': ' '.join(run['config']),
'MOZHARNESS_SCRIPT': run['script'],
'MH_BRANCH': config.params['project'],
'MH_BUILD_POOL': 'taskcluster',
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
})
if 'actions' in run:
env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
if 'options' in run:
env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
if 'custom-build-variant-cfg' in run:
env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
if 'job-script' in run:
env['JOB_SCRIPT'] = run['job-script']
# if we're not keeping artifacts, set some env variables to empty values
# that will cause the build process to skip copying the results to the
# artifacts directory. This will have no effect for operations that are
# not builds.
if not run['keep-artifacts']:
env['DIST_TARGET_UPLOADS'] = ''
env['DIST_UPLOADS'] = ''
# Xvfb
if run['need-xvfb']:
env['NEED_XVFB'] = 'true'
# tooltool downloads
if run['tooltool-downloads']:
worker['relengapi-proxy'] = True
worker['caches'].append({
'type': 'persistent',
'name': 'tooltool-cache',
'mount-point': '/home/worker/tooltool-cache',
})
taskdesc['scopes'].extend([
'docker-worker:relengapi-proxy:tooltool.download.public',
])
if run['tooltool-downloads'] == 'internal':
taskdesc['scopes'].append(
'docker-worker:relengapi-proxy:tooltool.download.internal')
env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
# Retry if mozharness returns TBPL_RETRY
worker['retry-exit-status'] = 4
docker_worker_setup_secrets(config, job, taskdesc)
command = [
'/home/worker/bin/run-task',
# Various caches/volumes are default owned by root:root.
'--chown-recursive', '/home/worker/workspace',
'--chown-recursive', '/home/worker/tooltool-cache',
'--vcs-checkout', '/home/worker/workspace/build/src',
'--tools-checkout', '/home/worker/workspace/build/tools',
'--',
]
command.append("/home/worker/workspace/build/src/{}".format(
run.get('job-script',
"taskcluster/scripts/builder/build-linux.sh"
)))
worker['command'] = command
def docker_worker_toolchain(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['chain-of-trust'] = True
# Allow the job to specify where artifacts come from, but add
# public/build if it's not there already.
artifacts = worker.setdefault('artifacts', [])
if not any(artifact.get('name') == 'public/build' for artifact in artifacts):
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
support_vcs_checkout(config, job, taskdesc, sparse=True)
env = worker['env']
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'TOOLS_DISABLE': 'true',
'MOZ_AUTOMATION': '1',
})
if run['tooltool-downloads']:
internal = run['tooltool-downloads'] == 'internal'
docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
# Use `mach` to invoke python scripts so in-tree libraries are available.
if run['script'].endswith('.py'):
wrapper = 'workspace/build/src/mach python '
else:
wrapper = ''
args = run.get('arguments', '')
if args:
args = ' ' + shell_quote(*args)
sparse_profile = []
if run.get('sparse-profile'):
sparse_profile = ['--sparse-profile',
'build/sparse-profiles/{}'.format(run['sparse-profile'])]
worker['command'] = [
'/builds/worker/bin/run-task',
'--vcs-checkout=/builds/worker/workspace/build/src',
] + sparse_profile + [
'--',
'bash',
'-c',
'cd /builds/worker && '
'{}workspace/build/src/taskcluster/scripts/misc/{}{}'.format(
wrapper, run['script'], args)
]
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run['toolchain-artifact']
if 'toolchain-alias' in run:
attributes['toolchain-alias'] = run['toolchain-alias']
if not taskgraph.fast:
name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
add_optimization(
config, taskdesc,
cache_type=CACHE_TYPE,
cache_name=name,
digest_data=get_digest_data(config, run, taskdesc),
)
def docker_worker_toolchain(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['chain-of-trust'] = True
# Allow the job to specify where artifacts come from, but add
# public/build if it's not there already.
artifacts = worker.setdefault('artifacts', [])
if not any(
artifact.get('name') == 'public/build' for artifact in artifacts):
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
support_vcs_checkout(config, job, taskdesc, sparse=True)
env = worker['env']
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'TOOLS_DISABLE': 'true',
'MOZ_AUTOMATION': '1',
})
if run['tooltool-downloads']:
internal = run['tooltool-downloads'] == 'internal'
docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
# Use `mach` to invoke python scripts so in-tree libraries are available.
if run['script'].endswith('.py'):
wrapper = 'workspace/build/src/mach python '
else:
wrapper = ''
args = run.get('arguments', '')
if args:
args = ' ' + shell_quote(*args)
sparse_profile = []
if run.get('sparse-profile'):
sparse_profile = [
'--sparse-profile',
'build/sparse-profiles/{}'.format(run['sparse-profile'])
]
worker['command'] = [
'/builds/worker/bin/run-task',
'--vcs-checkout=/builds/worker/workspace/build/src',
] + sparse_profile + [
'--', 'bash', '-c', 'cd /builds/worker && '
'{}workspace/build/src/taskcluster/scripts/misc/{}{}'.format(
wrapper, run['script'], args)
]
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run['toolchain-artifact']
if 'toolchain-alias' in run:
attributes['toolchain-alias'] = run['toolchain-alias']
if not taskgraph.fast:
name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
add_optimization(
config,
taskdesc,
cache_type=CACHE_TYPE,
cache_name=name,
digest_data=get_digest_data(config, run, taskdesc),
)
def mozharness_on_docker_worker_setup(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['implementation'] = job['worker']['implementation']
# running via mozharness assumes desktop-build (which contains build.sh)
taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"}
worker['relengapi-proxy'] = False # but maybe enabled for tooltool below
worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_tc_vcs_cache(config, job, taskdesc)
docker_worker_add_workspace_cache(config, job, taskdesc)
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG':
' '.join(run['config']),
'MOZHARNESS_SCRIPT':
run['script'],
'MH_BRANCH':
config.params['project'],
'MH_BUILD_POOL':
'taskcluster',
'MOZ_BUILD_DATE':
time.strftime("%Y%m%d%H%M%S", time.gmtime(config.params['pushdate'])),
'MOZ_SCM_LEVEL':
config.params['level'],
})
if 'actions' in run:
env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
if 'options' in run:
env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
if 'custom-build-variant-cfg' in run:
env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
# if we're not keeping artifacts, set some env variables to empty values
# that will cause the build process to skip copying the results to the
# artifacts directory. This will have no effect for operations that are
# not builds.
if not run['keep-artifacts']:
env['DIST_TARGET_UPLOADS'] = ''
env['DIST_UPLOADS'] = ''
# Xvfb
if run['need-xvfb']:
env['NEED_XVFB'] = 'true'
# tooltool downloads
if run['tooltool-downloads']:
worker['relengapi-proxy'] = True
worker['caches'].append({
'type': 'persistent',
'name': 'tooltool-cache',
'mount-point': '/home/worker/tooltool-cache',
})
taskdesc['scopes'].extend([
'docker-worker:relengapi-proxy:tooltool.download.public',
])
if run['tooltool-downloads'] == 'internal':
taskdesc['scopes'].append(
'docker-worker:relengapi-proxy:tooltool.download.internal')
env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
env['TOOLTOOL_REV'] = 'master'
docker_worker_setup_secrets(config, job, taskdesc)
worker['command'] = ["/bin/bash", "bin/build.sh"]
def mozharness_on_docker_worker_setup(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['implementation'] = job['worker']['implementation']
if not run['use-simple-package']:
raise NotImplementedError("Simple packaging cannot be disabled via"
"'use-simple-package' on docker-workers")
if not run['use-magic-mh-args']:
raise NotImplementedError("Cannot disabled mh magic arg passing via"
"'use-magic-mh-args' on docker-workers")
# Running via mozharness assumes an image that contains build.sh:
# by default, desktop-build, but it could be another image (like
# android-gradle-build) that "inherits" from desktop-build.
if not taskdesc['worker']['docker-image']:
taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"}
worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_workspace_cache(
config, job, taskdesc, extra=run.get('extra-workspace-cache-key'))
support_vcs_checkout(config, job, taskdesc)
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG': ' '.join(run['config']),
'MOZHARNESS_SCRIPT': run['script'],
'MH_BRANCH': config.params['project'],
'MH_BUILD_POOL': 'taskcluster',
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'MOZ_AUTOMATION': '1',
})
if 'actions' in run:
env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
if 'options' in run:
env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
if 'custom-build-variant-cfg' in run:
env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
if 'job-script' in run:
env['JOB_SCRIPT'] = run['job-script']
# if we're not keeping artifacts, set some env variables to empty values
# that will cause the build process to skip copying the results to the
# artifacts directory. This will have no effect for operations that are
# not builds.
if not run['keep-artifacts']:
env['DIST_TARGET_UPLOADS'] = ''
env['DIST_UPLOADS'] = ''
# Xvfb
if run['need-xvfb']:
env['NEED_XVFB'] = 'true'
if run['tooltool-downloads']:
internal = run['tooltool-downloads'] == 'internal'
docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
# Retry if mozharness returns TBPL_RETRY
worker['retry-exit-status'] = 4
docker_worker_setup_secrets(config, job, taskdesc)
command = [
'/builds/worker/bin/run-task',
'--vcs-checkout',
'/builds/worker/workspace/build/src',
'--tools-checkout',
'/builds/worker/workspace/build/tools',
'--',
'/builds/worker/workspace/build/src/{}'.format(
run.get('job-script',
'taskcluster/scripts/builder/build-linux.sh')),
]
worker['command'] = command
def mozharness_on_docker_worker_setup(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['implementation'] = job['worker']['implementation']
if not run['use-simple-package']:
raise NotImplementedError("Simple packaging cannot be disabled via"
"'use-simple-package' on docker-workers")
if not run['use-magic-mh-args']:
raise NotImplementedError("Cannot disabled mh magic arg passing via"
"'use-magic-mh-args' on docker-workers")
# Running via mozharness assumes an image that contains build.sh:
# by default, debian7-amd64-build, but it could be another image (like
# android-build).
taskdesc['worker'].setdefault('docker-image', {'in-tree': 'debian7-amd64-build'})
worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_workspace_cache(config, job, taskdesc,
extra=run.get('extra-workspace-cache-key'))
support_vcs_checkout(config, job, taskdesc)
env = worker.setdefault('env', {})
env.update({
'MOZHARNESS_CONFIG': ' '.join(run['config']),
'MOZHARNESS_SCRIPT': run['script'],
'MH_BRANCH': config.params['project'],
'MH_BUILD_POOL': 'taskcluster',
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'MOZ_AUTOMATION': '1',
})
if 'actions' in run:
env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
if 'options' in run:
env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
if 'config-paths' in run:
env['MOZHARNESS_CONFIG_PATHS'] = ' '.join(run['config-paths'])
if 'custom-build-variant-cfg' in run:
env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
if 'extra-config' in run:
env['EXTRA_MOZHARNESS_CONFIG'] = json.dumps(run['extra-config'])
if 'job-script' in run:
env['JOB_SCRIPT'] = run['job-script']
if config.params.is_try():
env['TRY_COMMIT_MSG'] = config.params['message']
# if we're not keeping artifacts, set some env variables to empty values
# that will cause the build process to skip copying the results to the
# artifacts directory. This will have no effect for operations that are
# not builds.
if not run['keep-artifacts']:
env['DIST_TARGET_UPLOADS'] = ''
env['DIST_UPLOADS'] = ''
# Xvfb
if run['need-xvfb']:
env['NEED_XVFB'] = 'true'
if run['tooltool-downloads']:
internal = run['tooltool-downloads'] == 'internal'
docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
# Retry if mozharness returns TBPL_RETRY
worker['retry-exit-status'] = 4
docker_worker_setup_secrets(config, job, taskdesc)
command = [
'/builds/worker/bin/run-task',
'--vcs-checkout', '/builds/worker/workspace/build/src',
'--tools-checkout', '/builds/worker/workspace/build/tools',
]
if run['comm-checkout']:
command.append('--comm-checkout=/builds/worker/workspace/build/src/comm')
command += [
'--',
'/builds/worker/workspace/build/src/{}'.format(
run.get('job-script', 'taskcluster/scripts/builder/build-linux.sh')
),
]
worker['command'] = command
