def cache_task(config, tasks):
if taskgraph.fast:
for task in tasks:
yield task
return
digests = {}
for task in config.kind_dependencies_tasks.values():
if 'cached_task' in task.attributes:
digests[task.label] = format_task_digest(task.attributes['cached_task'])
for task in order_tasks(config, tasks):
cache = task.pop('cache', None)
if cache is None:
yield task
continue
dependency_digests = []
for p in task.get('dependencies', {}).values():
if p in digests:
dependency_digests.append(digests[p])
else:
raise Exception('Cached task {} has uncached parent task: {}'.format(
task['label'], p))
digest_data = cache['digest-data'] + sorted(dependency_digests)
add_optimization(config, task, cache_type=cache['type'],
cache_name=cache['name'], digest_data=digest_data)
digests[task['label']] = format_task_digest(task['attributes']['cached_task'])
yield task
def windows_toolchain(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['artifacts'] = [{
'path': r'public\build',
'type': 'directory',
}]
worker['chain-of-trust'] = True
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
env = worker['env']
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'MOZ_AUTOMATION': '1',
})
hg = r'c:\Program Files\Mercurial\hg.exe'
hg_command = ['"{}"'.format(hg)]
hg_command.append('robustcheckout')
hg_command.extend(['--sharebase', 'y:\\hg-shared'])
hg_command.append('--purge')
hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
hg_command.extend(['--revision', '%GECKO_HEAD_REV%'])
hg_command.append('%GECKO_HEAD_REPOSITORY%')
hg_command.append('.\\build\\src')
# Use `mach` to invoke python scripts so in-tree libraries are available.
if run['script'].endswith('.py'):
raise NotImplementedError("Python scripts don't work on Windows")
args = run.get('arguments', '')
if args:
args = ' ' + shell_quote(*args)
bash = r'c:\mozilla-build\msys\bin\bash'
worker['command'] = [
' '.join(hg_command),
# do something intelligent.
r'{} build/src/taskcluster/scripts/misc/{}{}'.format(
bash, run['script'], args)
]
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run['toolchain-artifact']
if 'toolchain-alias' in run:
attributes['toolchain-alias'] = run['toolchain-alias']
if not taskgraph.fast:
name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
add_optimization(
config,
taskdesc,
cache_type=CACHE_TYPE,
cache_name=name,
digest_data=get_digest_data(config, run, taskdesc),
)
def windows_toolchain(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['artifacts'] = [{
'path': r'public\build',
'type': 'directory',
}]
worker['chain-of-trust'] = True
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
env = worker['env']
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'MOZ_AUTOMATION': '1',
})
hg = r'c:\Program Files\Mercurial\hg.exe'
hg_command = ['"{}"'.format(hg)]
hg_command.append('robustcheckout')
hg_command.extend(['--sharebase', 'y:\\hg-shared'])
hg_command.append('--purge')
hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
hg_command.extend(['--revision', '%GECKO_HEAD_REV%'])
hg_command.append('%GECKO_HEAD_REPOSITORY%')
hg_command.append('.\\build\\src')
# Use `mach` to invoke python scripts so in-tree libraries are available.
if run['script'].endswith('.py'):
raise NotImplementedError("Python scripts don't work on Windows")
args = run.get('arguments', '')
if args:
args = ' ' + shell_quote(*args)
bash = r'c:\mozilla-build\msys\bin\bash'
worker['command'] = [
' '.join(hg_command),
# do something intelligent.
r'{} build/src/taskcluster/scripts/misc/{}{}'.format(
bash, run['script'], args)
]
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run['toolchain-artifact']
if 'toolchain-alias' in run:
attributes['toolchain-alias'] = run['toolchain-alias']
if not taskgraph.fast:
name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
add_optimization(
config, taskdesc,
cache_type=CACHE_TYPE,
cache_name=name,
digest_data=get_digest_data(config, run, taskdesc),
)
def windows_toolchain(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['artifacts'] = [{
'path': r'public\build',
'type': 'directory',
}]
worker['chain-of-trust'] = True
support_vcs_checkout(config, job, taskdesc)
env = worker['env']
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'MOZ_AUTOMATION': '1',
})
hg_command = generic_worker_hg_commands(
'https://hg.mozilla.org/mozilla-unified',
env['GECKO_HEAD_REPOSITORY'],
env['GECKO_HEAD_REV'],
r'.\build\src')[0]
# Use `mach` to invoke python scripts so in-tree libraries are available.
if run['script'].endswith('.py'):
raise NotImplementedError("Python scripts don't work on Windows")
args = run.get('arguments', '')
if args:
args = ' ' + shell_quote(*args)
bash = r'c:\mozilla-build\msys\bin\bash'
worker['command'] = [
hg_command,
# do something intelligent.
r'{} build/src/taskcluster/scripts/misc/{}{}'.format(
bash, run['script'], args)
]
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run['toolchain-artifact']
if 'toolchain-alias' in run:
attributes['toolchain-alias'] = run['toolchain-alias']
if not taskgraph.fast:
name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
add_optimization(
config, taskdesc,
cache_type=CACHE_TYPE,
cache_name=name,
digest_data=get_digest_data(config, run, taskdesc),
)
def cache_task(config, tasks):
if taskgraph.fast:
for task in tasks:
yield task
return
digests = {}
for task in config.kind_dependencies_tasks.values():
if ("cached_task" in task.attributes
and task.attributes["cached_task"] is not False):
digests[task.label] = format_task_digest(
task.attributes["cached_task"])
for task in order_tasks(config, tasks):
cache = task.pop("cache", None)
if cache is None:
yield task
continue
dependency_digests = []
for p in task.get("dependencies", {}).values():
if p in digests:
dependency_digests.append(digests[p])
else:
raise Exception(
"Cached task {} has uncached parent task: {}".format(
task["label"], p))
digest_data = cache["digest-data"] + sorted(dependency_digests)
add_optimization(
config,
task,
cache_type=cache["type"],
cache_name=cache["name"],
digest_data=digest_data,
)
digests[task["label"]] = format_task_digest(
task["attributes"]["cached_task"])
yield task
def fill_template(config, tasks):
available_packages = {}
for task in config.kind_dependencies_tasks:
if task.kind != 'packages':
continue
name = task.label.replace('packages-', '')
for route in task.task.get('routes', []):
if route.startswith('index.') and '.hash.' in route:
# Only keep the hash part of the route.
h = route.rsplit('.', 1)[1]
assert DIGEST_RE.match(h)
available_packages[name] = h
break
context_hashes = {}
for task in order_image_tasks(config, tasks):
image_name = task.pop('name')
job_symbol = task.pop('symbol')
args = task.pop('args', {})
definition = task.pop('definition', image_name)
packages = task.pop('packages', [])
parent = task.pop('parent', None)
for p in packages:
if p not in available_packages:
raise Exception('Missing package job for {}-{}: {}'.format(
config.kind, image_name, p))
# Generating the context hash relies on arguments being set, so we
# set this now, although it's not the final value (it's a
# task-reference value, see further below). We add the package routes
# containing a hash to get the overall docker image hash, so changes
# to packages will be reflected in the docker image hash.
args['DOCKER_IMAGE_PACKAGES'] = ' '.join('<{}>'.format(p)
for p in packages)
if parent:
args['DOCKER_IMAGE_PARENT'] = '{}:{}'.format(
parent, context_hashes[parent])
context_path = os.path.join('taskcluster', 'docker', definition)
context_hash = generate_context_hash(GECKO, context_path, image_name,
args)
digest_data = [context_hash]
context_hashes[image_name] = context_hash
description = 'Build the docker image {} for use by dependent tasks'.format(
image_name)
# Adjust the zstandard compression level based on the execution level.
# We use faster compression for level 1 because we care more about
# end-to-end times. We use slower/better compression for other levels
# because images are read more often and it is worth the trade-off to
# burn more CPU once to reduce image size.
zstd_level = '3' if int(config.params['level']) == 1 else '10'
# include some information that is useful in reconstructing this task
# from JSON
taskdesc = {
'label':
'build-docker-image-' + image_name,
'description':
description,
'attributes': {
'image_name': image_name
},
'expires-after':
'28 days' if config.params.is_try() else '1 year',
'scopes': ['secrets:get:project/taskcluster/gecko/hgfingerprint'],
'treeherder': {
'symbol': job_symbol,
'platform': 'taskcluster-images/opt',
'kind': 'other',
'tier': 1,
},
'run-on-projects': [],
'worker-type':
'aws-provisioner-v1/gecko-{}-images'.format(
config.params['level']),
'worker': {
'implementation':
'docker-worker',
'os':
'linux',
'artifacts': [{
'type': 'file',
'path': '/builds/worker/workspace/artifacts/image.tar.zst',
'name': 'public/image.tar.zst',
}],
'env': {
'HG_STORE_PATH': '/builds/worker/checkouts/hg-store',
'HASH': context_hash,
'PROJECT': config.params['project'],
'IMAGE_NAME': image_name,
'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level,
'GECKO_BASE_REPOSITORY': config.params['base_repository'],
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
},
'chain-of-trust':
True,
'docker-in-docker':
True,
'taskcluster-proxy':
True,
'max-run-time':
7200,
},
}
# Retry for 'funsize-update-generator' if exit status code is -1
if image_name in ['funsize-update-generator']:
taskdesc['worker']['retry-exit-status'] = [-1]
worker = taskdesc['worker']
# We use the in-tree image_builder image to build docker images, but
# that can't be used to build the image_builder image itself,
# obviously. So we fall back to the last snapshot of the image that
# was uploaded to docker hub.
if image_name == 'image_builder':
worker['docker-image'] = 'taskcluster/image_builder@sha256:' + \
'24ce54a1602453bc93515aecd9d4ad25a22115fbc4b209ddb5541377e9a37315'
# Keep in sync with the Dockerfile used to generate the
# docker image whose digest is referenced above.
worker['volumes'] = [
'/builds/worker/checkouts',
'/builds/worker/workspace',
]
cache_name = 'imagebuilder-v1'
else:
worker['docker-image'] = {'in-tree': 'image_builder'}
cache_name = 'imagebuilder-sparse-{}'.format(_run_task_suffix())
# Force images built against the in-tree image builder to
# have a different digest by adding a fixed string to the
# hashed data.
digest_data.append('image_builder')
worker['caches'] = [{
'type':
'persistent',
'name':
'level-{}-{}'.format(config.params['level'], cache_name),
'mount-point':
'/builds/worker/checkouts',
}]
for k, v in args.items():
if k == 'DOCKER_IMAGE_PACKAGES':
worker['env'][k] = {'task-reference': v}
else:
worker['env'][k] = v
if packages:
deps = taskdesc.setdefault('dependencies', {})
for p in sorted(packages):
deps[p] = 'packages-{}'.format(p)
digest_data.append(available_packages[p])
if parent:
deps = taskdesc.setdefault('dependencies', {})
deps[parent] = 'build-docker-image-{}'.format(parent)
worker['env']['DOCKER_IMAGE_PARENT_TASK'] = {
'task-reference': '<{}>'.format(parent)
}
if len(digest_data) > 1:
kwargs = {'digest_data': digest_data}
else:
kwargs = {'digest': digest_data[0]}
add_optimization(config,
taskdesc,
cache_type="docker-images.v1",
cache_name=image_name,
**kwargs)
yield taskdesc
def docker_worker_toolchain(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['chain-of-trust'] = True
# Allow the job to specify where artifacts come from, but add
# public/build if it's not there already.
artifacts = worker.setdefault('artifacts', [])
if not any(
artifact.get('name') == 'public/build' for artifact in artifacts):
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
support_vcs_checkout(config, job, taskdesc, sparse=True)
env = worker['env']
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'TOOLS_DISABLE': 'true',
'MOZ_AUTOMATION': '1',
})
if run['tooltool-downloads']:
internal = run['tooltool-downloads'] == 'internal'
docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
# Use `mach` to invoke python scripts so in-tree libraries are available.
if run['script'].endswith('.py'):
wrapper = 'workspace/build/src/mach python '
else:
wrapper = ''
args = run.get('arguments', '')
if args:
args = ' ' + shell_quote(*args)
sparse_profile = []
if run.get('sparse-profile'):
sparse_profile = [
'--sparse-profile',
'build/sparse-profiles/{}'.format(run['sparse-profile'])
]
worker['command'] = [
'/builds/worker/bin/run-task',
'--vcs-checkout=/builds/worker/workspace/build/src',
] + sparse_profile + [
'--', 'bash', '-c', 'cd /builds/worker && '
'{}workspace/build/src/taskcluster/scripts/misc/{}{}'.format(
wrapper, run['script'], args)
]
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run['toolchain-artifact']
if 'toolchain-alias' in run:
attributes['toolchain-alias'] = run['toolchain-alias']
if not taskgraph.fast:
name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
add_optimization(
config,
taskdesc,
cache_type=CACHE_TYPE,
cache_name=name,
digest_data=get_digest_data(config, run, taskdesc),
)
def docker_worker_debian_package(config, job, taskdesc):
run = job['run']
name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
docker_repo = 'debian'
arch = run.get('arch', 'amd64')
if arch != 'amd64':
docker_repo = '{}/{}'.format(arch, docker_repo)
worker = taskdesc['worker']
worker['artifacts'] = []
worker['docker-image'] = '{repo}:{dist}-{date}'.format(
repo=docker_repo, dist=run['dist'], date=run['snapshot'][:8])
add_artifacts(config, job, taskdesc, path='/tmp/artifacts')
env = worker.setdefault('env', {})
env['DEBFULLNAME'] = 'Mozilla build team'
env['DEBEMAIL'] = '*****@*****.**'
if 'dsc' in run:
src = run['dsc']
unpack = 'dpkg-source -x {src_file} {package}'
package_re = DSC_PACKAGE_RE
elif 'tarball' in run:
src = run['tarball']
unpack = ('mkdir {package} && '
'tar -C {package} -axf {src_file} --strip-components=1')
package_re = SOURCE_PACKAGE_RE
else:
raise RuntimeError('Unreachable')
src_url = src['url']
src_file = os.path.basename(src_url)
src_sha256 = src['sha256']
package = package_re.match(src_file).group(0)
unpack = unpack.format(src_file=src_file, package=package)
base_deps = [
'apt-utils',
'build-essential',
'devscripts',
'fakeroot',
]
resolver = run.get('resolver', 'apt-get')
if resolver == 'apt-get':
resolver = 'apt-get -yyq --no-install-recommends'
elif resolver == 'aptitude':
resolver = ('aptitude -y --without-recommends -o '
'Aptitude::ProblemResolver::Hints::KeepBuildDeps='
'"reject {}-build-deps :UNINST"').format(package)
base_deps.append('aptitude')
else:
raise RuntimeError('Unreachable')
adjust = ''
if 'patch' in run:
# We can't depend on docker images, so we don't have robustcheckout or
# or run-task to get a checkout. So for this one file we'd need
# from a checkout, download it.
env['PATCH_URL'] = '{head_repo}/raw-file/{head_rev}/build/debian-packages/{patch}'.format(
head_repo=config.params['head_repository'],
head_rev=config.params['head_rev'],
patch=run['patch'],
)
adjust += 'curl -sL $PATCH_URL | patch -p1 && '
if 'pre-build-command' in run:
adjust += run['pre-build-command'] + ' && '
if 'tarball' in run:
adjust += 'mv ../{src_file} ../{package}_{ver}.orig.tar.gz && '.format(
src_file=src_file,
package=package,
ver=
'$(dpkg-parsechangelog | awk \'$1=="Version:"{print $2}\' | cut -f 1 -d -)',
)
if 'patch' not in run and 'pre-build-command' not in run:
adjust += ('debchange -l ".{prefix}moz" --distribution "{dist}"'
' "Mozilla backport for {dist}." < /dev/null && ').format(
prefix=name.split('-', 1)[0],
dist=run['dist'],
)
# We can't depend on docker images (since docker images depend on packages),
# so we inline the whole script here.
worker['command'] = [
'sh',
'-x',
'-c',
# Fill /etc/apt/sources.list with the relevant snapshot repository.
'echo "deb http://snapshot.debian.org/archive/debian'
'/{snapshot}/ {dist} main" > /etc/apt/sources.list && '
'echo "deb http://snapshot.debian.org/archive/debian'
'/{snapshot}/ {dist}-updates main" >> /etc/apt/sources.list && '
'echo "deb http://snapshot.debian.org/archive/debian'
'/{snapshot}/ {dist}-backports main" >> /etc/apt/sources.list && '
'echo "deb http://snapshot.debian.org/archive/debian-security'
'/{snapshot}/ {dist}/updates main" >> /etc/apt/sources.list && '
'apt-get update -o Acquire::Check-Valid-Until=false -q && '
# Add sources for packages coming from other package tasks.
'apt-get install -yyq apt-transport-https ca-certificates && '
'for task in $PACKAGES; do '
' echo "deb [trusted=yes] https://queue.taskcluster.net/v1/task'
'/$task/artifacts/public/build/ debian/" '
'>> /etc/apt/sources.list; '
'done && '
# Install the base utilities required to build debian packages.
'apt-get update -o Acquire::Check-Valid-Until=false -q && '
'apt-get install -yyq {base_deps} && '
'cd /tmp && '
# Get, validate and extract the package source.
'dget -d -u {src_url} && '
'echo "{src_sha256} {src_file}" | sha256sum -c && '
'{unpack} && '
'cd {package} && '
# Optionally apply patch and/or pre-build command.
'{adjust}'
# Install the necessary build dependencies.
'mk-build-deps -i -r debian/control -t \'{resolver}\' && '
# Build the package
'DEB_BUILD_OPTIONS="parallel=$(nproc) nocheck" dpkg-buildpackage && '
# Copy the artifacts
'mkdir -p {artifacts}/debian && '
'dcmd cp ../{package}_*.changes {artifacts}/debian/ && '
'cd {artifacts} && '
# Make the artifacts directory usable as an APT repository.
'apt-ftparchive sources debian | gzip -c9 > debian/Sources.gz && '
'apt-ftparchive packages debian | gzip -c9 > debian/Packages.gz'.
format(
package=package,
snapshot=run['snapshot'],
dist=run['dist'],
src_url=src_url,
src_file=src_file,
src_sha256=src_sha256,
unpack=unpack,
adjust=adjust,
artifacts='/tmp/artifacts',
base_deps=' '.join(base_deps),
resolver=resolver,
)
]
# Use the command generated above as the base for the index hash.
# We rely on it not varying depending on the head_repository or head_rev.
data = list(worker['command'])
if 'patch' in run:
data.append(
hash_path(
os.path.join(GECKO, 'build', 'debian-packages', run['patch'])))
if docker_repo != 'debian':
data.append(docker_repo)
if run.get('packages'):
env = worker.setdefault('env', {})
env['PACKAGES'] = {
'task-reference':
' '.join('<{}>'.format(p) for p in run['packages'])
}
deps = taskdesc.setdefault('dependencies', {})
for p in run['packages']:
deps[p] = 'packages-{}'.format(p)
data.append(p)
add_optimization(config,
taskdesc,
cache_type='packages.v1',
cache_name=name,
digest_data=data)
def fill_template(config, tasks):
available_packages = {}
for task in config.kind_dependencies_tasks:
if task.kind != 'packages':
continue
name = task.label.replace('packages-', '')
for route in task.task.get('routes', []):
if route.startswith('index.') and '.hash.' in route:
# Only keep the hash part of the route.
h = route.rsplit('.', 1)[1]
assert DIGEST_RE.match(h)
available_packages[name] = h
break
context_hashes = {}
for task in order_image_tasks(config, tasks):
image_name = task.pop('name')
job_symbol = task.pop('symbol')
args = task.pop('args', {})
definition = task.pop('definition', image_name)
packages = task.pop('packages', [])
parent = task.pop('parent', None)
for p in packages:
if p not in available_packages:
raise Exception('Missing package job for {}-{}: {}'.format(
config.kind, image_name, p))
# Generating the context hash relies on arguments being set, so we
# set this now, although it's not the final value (it's a
# task-reference value, see further below). We add the package routes
# containing a hash to get the overall docker image hash, so changes
# to packages will be reflected in the docker image hash.
args['DOCKER_IMAGE_PACKAGES'] = ' '.join('<{}>'.format(p)
for p in packages)
if parent:
args['DOCKER_IMAGE_PARENT'] = '{}:{}'.format(parent, context_hashes[parent])
context_path = os.path.join('taskcluster', 'docker', definition)
context_hash = generate_context_hash(
GECKO, context_path, image_name, args)
digest_data = [context_hash]
context_hashes[image_name] = context_hash
description = 'Build the docker image {} for use by dependent tasks'.format(
image_name)
# Adjust the zstandard compression level based on the execution level.
# We use faster compression for level 1 because we care more about
# end-to-end times. We use slower/better compression for other levels
# because images are read more often and it is worth the trade-off to
# burn more CPU once to reduce image size.
zstd_level = '3' if int(config.params['level']) == 1 else '10'
# include some information that is useful in reconstructing this task
# from JSON
taskdesc = {
'label': 'build-docker-image-' + image_name,
'description': description,
'attributes': {'image_name': image_name},
'expires-after': '28 days' if config.params.is_try() else '1 year',
'scopes': ['secrets:get:project/taskcluster/gecko/hgfingerprint'],
'treeherder': {
'symbol': job_symbol,
'platform': 'taskcluster-images/opt',
'kind': 'other',
'tier': 1,
},
'run-on-projects': [],
'worker-type': 'aws-provisioner-v1/gecko-{}-images'.format(
config.params['level']),
'worker': {
'implementation': 'docker-worker',
'os': 'linux',
'artifacts': [{
'type': 'file',
'path': '/builds/worker/workspace/artifacts/image.tar.zst',
'name': 'public/image.tar.zst',
}],
'env': {
'HG_STORE_PATH': '/builds/worker/checkouts/hg-store',
'HASH': context_hash,
'PROJECT': config.params['project'],
'IMAGE_NAME': image_name,
'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level,
'GECKO_BASE_REPOSITORY': config.params['base_repository'],
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
'GECKO_HEAD_REV': config.params['head_rev'],
},
'chain-of-trust': True,
'docker-in-docker': True,
'taskcluster-proxy': True,
'max-run-time': 7200,
},
}
worker = taskdesc['worker']
# We use the in-tree image_builder image to build docker images, but
# that can't be used to build the image_builder image itself,
# obviously. So we fall back to the last snapshot of the image that
# was uploaded to docker hub.
if image_name == 'image_builder':
worker['docker-image'] = 'taskcluster/image_builder@sha256:' + \
'24ce54a1602453bc93515aecd9d4ad25a22115fbc4b209ddb5541377e9a37315'
# Keep in sync with the Dockerfile used to generate the
# docker image whose digest is referenced above.
worker['volumes'] = [
'/builds/worker/checkouts',
'/builds/worker/workspace',
]
cache_name = 'imagebuilder-v1'
else:
worker['docker-image'] = {'in-tree': 'image_builder'}
cache_name = 'imagebuilder-sparse-{}'.format(_run_task_suffix())
# Force images built against the in-tree image builder to
# have a different digest by adding a fixed string to the
# hashed data.
digest_data.append('image_builder')
worker['caches'] = [{
'type': 'persistent',
'name': 'level-{}-{}'.format(config.params['level'], cache_name),
'mount-point': '/builds/worker/checkouts',
}]
for k, v in args.items():
if k == 'DOCKER_IMAGE_PACKAGES':
worker['env'][k] = {'task-reference': v}
else:
worker['env'][k] = v
if packages:
deps = taskdesc.setdefault('dependencies', {})
for p in sorted(packages):
deps[p] = 'packages-{}'.format(p)
digest_data.append(available_packages[p])
if parent:
deps = taskdesc.setdefault('dependencies', {})
deps[parent] = 'build-docker-image-{}'.format(parent)
worker['env']['DOCKER_IMAGE_PARENT_TASK'] = {
'task-reference': '<{}>'.format(parent)
}
if len(digest_data) > 1:
kwargs = {'digest_data': digest_data}
else:
kwargs = {'digest': digest_data[0]}
add_optimization(
config, taskdesc,
cache_type="docker-images.v1",
cache_name=image_name,
**kwargs
)
yield taskdesc
def docker_worker_debian_package(config, job, taskdesc):
run = job['run']
name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
worker = taskdesc['worker']
worker['artifacts'] = []
worker['docker-image'] = 'debian:{dist}-{date}'.format(
dist=run['dist'],
date=run['snapshot'][:8])
add_public_artifacts(config, job, taskdesc, path='/tmp/artifacts')
env = worker.setdefault('env', {})
env['DEBFULLNAME'] = 'Mozilla build team'
env['DEBEMAIL'] = '*****@*****.**'
if 'dsc' in run:
src = run['dsc']
unpack = 'dpkg-source -x {src_file} {package}'
package_re = DSC_PACKAGE_RE
elif 'tarball' in run:
src = run['tarball']
unpack = ('mkdir {package} && '
'tar -C {package} -axf {src_file} --strip-components=1')
package_re = SOURCE_PACKAGE_RE
else:
raise RuntimeError('Unreachable')
src_url = src['url']
src_file = os.path.basename(src_url)
src_sha256 = src['sha256']
package = package_re.match(src_file).group(0)
unpack = unpack.format(src_file=src_file, package=package)
base_deps = [
'apt-utils',
'build-essential',
'devscripts',
'fakeroot',
]
resolver = run.get('resolver', 'apt-get')
if resolver == 'apt-get':
resolver = 'apt-get -yyq --no-install-recommends'
elif resolver == 'aptitude':
resolver = ('aptitude -y --without-recommends -o '
'Aptitude::ProblemResolver::Hints::KeepBuildDeps='
'"reject {}-build-deps :UNINST"').format(package)
base_deps.append('aptitude')
else:
raise RuntimeError('Unreachable')
adjust = ''
if 'patch' in run:
# We can't depend on docker images, so we don't have robustcheckout or
# or run-task to get a checkout. So for this one file we'd need
# from a checkout, download it.
env['PATCH_URL'] = '{head_repo}/raw-file/{head_rev}/build/debian-packages/{patch}'.format(
head_repo=config.params['head_repository'],
head_rev=config.params['head_rev'],
patch=run['patch'],
)
adjust += 'curl -sL $PATCH_URL | patch -p1 && '
if 'pre-build-command' in run:
adjust += run['pre-build-command'] + ' && '
if 'tarball' in run:
adjust += 'mv ../{src_file} ../{package}_{ver}.orig.tar.gz && '.format(
src_file=src_file,
package=package,
ver='$(dpkg-parsechangelog | awk \'$1=="Version:"{print $2}\' | cut -f 1 -d -)',
)
if 'patch' not in run and 'pre-build-command' not in run:
adjust += ('debchange -l ".{prefix}moz" --distribution "{dist}"'
' "Mozilla backport for {dist}." < /dev/null && ').format(
prefix=name.split('-', 1)[0],
dist=run['dist'],
)
# We can't depend on docker images (since docker images depend on packages),
# so we inline the whole script here.
worker['command'] = [
'sh',
'-x',
'-c',
# Fill /etc/apt/sources.list with the relevant snapshot repository.
'echo "deb http://snapshot.debian.org/archive/debian'
'/{snapshot}/ {dist} main" > /etc/apt/sources.list && '
'echo "deb http://snapshot.debian.org/archive/debian'
'/{snapshot}/ {dist}-updates main" >> /etc/apt/sources.list && '
'echo "deb http://snapshot.debian.org/archive/debian'
'/{snapshot}/ {dist}-backports main" >> /etc/apt/sources.list && '
'echo "deb http://snapshot.debian.org/archive/debian-security'
'/{snapshot}/ {dist}/updates main" >> /etc/apt/sources.list && '
'apt-get update -o Acquire::Check-Valid-Until=false -q && '
# Add sources for packages coming from other package tasks.
'apt-get install -yyq apt-transport-https ca-certificates && '
'for task in $PACKAGES; do '
' echo "deb [trusted=yes] https://queue.taskcluster.net/v1/task'
'/$task/runs/0/artifacts/public/build/ debian/" '
'>> /etc/apt/sources.list; '
'done && '
# Install the base utilities required to build debian packages.
'apt-get update -o Acquire::Check-Valid-Until=false -q && '
'apt-get install -yyq {base_deps} && '
'cd /tmp && '
# Get, validate and extract the package source.
'dget -d -u {src_url} && '
'echo "{src_sha256} {src_file}" | sha256sum -c && '
'{unpack} && '
'cd {package} && '
# Optionally apply patch and/or pre-build command.
'{adjust}'
# Install the necessary build dependencies.
'mk-build-deps -i -r debian/control -t \'{resolver}\' && '
# Build the package
'DEB_BUILD_OPTIONS="parallel=$(nproc) nocheck" dpkg-buildpackage && '
# Copy the artifacts
'mkdir -p {artifacts}/debian && '
'dcmd cp ../{package}_*.changes {artifacts}/debian/ && '
'cd {artifacts} && '
# Make the artifacts directory usable as an APT repository.
'apt-ftparchive sources debian | gzip -c9 > debian/Sources.gz && '
'apt-ftparchive packages debian | gzip -c9 > debian/Packages.gz'
.format(
package=package,
snapshot=run['snapshot'],
dist=run['dist'],
src_url=src_url,
src_file=src_file,
src_sha256=src_sha256,
unpack=unpack,
adjust=adjust,
artifacts='/tmp/artifacts',
base_deps=' '.join(base_deps),
resolver=resolver,
)
]
# Use the command generated above as the base for the index hash.
# We rely on it not varying depending on the head_repository or head_rev.
data = list(worker['command'])
if 'patch' in run:
data.append(hash_path(os.path.join(GECKO, 'build', 'debian-packages', run['patch'])))
if run.get('packages'):
env = worker.setdefault('env', {})
env['PACKAGES'] = {
'task-reference': ' '.join('<{}>'.format(p)
for p in run['packages'])
}
deps = taskdesc.setdefault('dependencies', {})
for p in run['packages']:
deps[p] = 'packages-{}'.format(p)
data.append(p)
add_optimization(config, taskdesc, cache_type='packages.v1',
cache_name=name, digest_data=data)
def docker_worker_toolchain(config, job, taskdesc):
run = job['run']
worker = taskdesc['worker']
worker['chain-of-trust'] = True
# Allow the job to specify where artifacts come from, but add
# public/build if it's not there already.
artifacts = worker.setdefault('artifacts', [])
if not any(artifact.get('name') == 'public/build' for artifact in artifacts):
docker_worker_add_public_artifacts(config, job, taskdesc)
docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
support_vcs_checkout(config, job, taskdesc, sparse=True)
env = worker['env']
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'TOOLS_DISABLE': 'true',
'MOZ_AUTOMATION': '1',
})
if run['tooltool-downloads']:
internal = run['tooltool-downloads'] == 'internal'
docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
# Use `mach` to invoke python scripts so in-tree libraries are available.
if run['script'].endswith('.py'):
wrapper = 'workspace/build/src/mach python '
else:
wrapper = ''
args = run.get('arguments', '')
if args:
args = ' ' + shell_quote(*args)
sparse_profile = []
if run.get('sparse-profile'):
sparse_profile = ['--sparse-profile',
'build/sparse-profiles/{}'.format(run['sparse-profile'])]
worker['command'] = [
'/builds/worker/bin/run-task',
'--vcs-checkout=/builds/worker/workspace/build/src',
] + sparse_profile + [
'--',
'bash',
'-c',
'cd /builds/worker && '
'{}workspace/build/src/taskcluster/scripts/misc/{}{}'.format(
wrapper, run['script'], args)
]
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run['toolchain-artifact']
if 'toolchain-alias' in run:
attributes['toolchain-alias'] = run['toolchain-alias']
if not taskgraph.fast:
name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
add_optimization(
config, taskdesc,
cache_type=CACHE_TYPE,
cache_name=name,
digest_data=get_digest_data(config, run, taskdesc),
)
def process_thirdparty_build(config, jobs):
"""
Set up a thirdparty library build, caching the built artifacts.
"""
for job in jobs:
name = job['name']
thirdparty = job['thirdparty']
artifact_name = thirdparty['artifact']
script = os.path.join(COMM_SCRIPTS, thirdparty['script'])
args = thirdparty.get('args', [])
command = [script] + args
task = make_base_task(config, name, job, script, command)
merge_to(job['index'], task['index'])
merge_to(job['treeherder'], task['treeherder'])
merge_to(job['worker'], task['worker'])
if 'run' in job:
merge_to(job['run'], task['run'])
if 'when' in job:
merge_to(job['when'], task['when'])
if 'toolchain' in job:
task['fetches']['toolchain'] = job['toolchain']
when = task.pop('when')
if 'when' in job:
merge_to(job['when'], when)
# The files-changed optimization is not actually used because it
# conflicts with the indexing optimization, but the same list of files
# is used to look up the revision with the most recent changes in
# order to calculate a hash for the index.
files_changed = when['files-changed']
task['worker'].setdefault('artifacts', []).append({
'name': 'public/build',
'path': '/builds/worker/artifacts',
'type': 'directory',
})
if not taskgraph.fast:
project = config.params['project']
# Get the most recent revision with changes. files-changed paths
# are relative to GECKO, so strip 'comm/' off first.
files_changed = frozenset(map(lambda p: strip_comm_prefix(p), files_changed))
last_changed_rev = get_last_modified_revision(COMM, files_changed)
logger.info("Using artifact from rev {}.".format(last_changed_rev))
cache_name = task['label'].replace('{}-'.format(config.kind), '', 1)
# This adds the level to the index path automatically.
add_optimization(
config,
task,
cache_type=CACHE_TYPE,
cache_name=cache_name,
# Digest is based on the repo name and revision
digest_data=command + [project, last_changed_rev, artifact_name],
)
yield task
