def make_task_description(config, jobs):
"""Given a build description, create a task description"""
# import plugin modules first, before iterating over jobs
import_all()
for job in jobs:
if 'label' not in job:
if 'name' not in job:
raise Exception("job has neither a name nor a label")
job['label'] = '{}-{}'.format(config.kind, job['name'])
if job.get('name'):
del job['name']
impl, os = worker_type_implementation(job['worker-type'])
worker = job.setdefault('worker', {})
assert 'implementation' not in worker
worker['implementation'] = impl
if os:
worker['os'] = os
taskdesc = copy.deepcopy(job)
# fill in some empty defaults to make run implementations easier
taskdesc.setdefault('attributes', {})
taskdesc.setdefault('dependencies', {})
taskdesc.setdefault('routes', [])
taskdesc.setdefault('scopes', [])
taskdesc.setdefault('extra', {})
# give the function for job.run.using on this worker implementation a
# chance to set up the task description.
configure_taskdesc_for_run(config, job, taskdesc, impl)
del taskdesc['run']
# yield only the task description, discarding the job description
yield taskdesc
def use_profile_data(config, jobs):
for job in jobs:
use_pgo = job.pop('use-pgo', False)
disable_pgo = config.params['try_task_config'].get(
'disable-pgo', False)
artifact_build = job['attributes'].get('artifact-build')
if not use_pgo or disable_pgo or artifact_build:
yield job
continue
# If use_pgo is True, the task uses the generate-profile task of the
# same name. Otherwise a task can specify a specific generate-profile
# task to use in the use_pgo field.
if use_pgo is True:
name = job['name']
else:
name = use_pgo
dependencies = 'generate-profile-{}'.format(name)
job.setdefault('dependencies', {})['generate-profile'] = dependencies
job.setdefault('fetches', {})['generate-profile'] = ['profdata.tar.xz']
job['worker']['env'].update({"TASKCLUSTER_PGO_PROFILE_USE": "1"})
_, worker_os = worker_type_implementation(config.graph_config,
job['worker-type'])
if worker_os == "linux":
# LTO linkage needs more open files than the default from run-task.
job['worker']['env'].update({"MOZ_LIMIT_NOFILE": "8192"})
yield job
def use_profile_data(config, jobs):
for job in jobs:
use_pgo = job.pop("use-pgo", False)
disable_pgo = config.params["try_task_config"].get(
"disable-pgo", False)
artifact_build = job["attributes"].get("artifact-build")
if not use_pgo or disable_pgo or artifact_build:
yield job
continue
# If use_pgo is True, the task uses the generate-profile task of the
# same name. Otherwise a task can specify a specific generate-profile
# task to use in the use_pgo field.
if use_pgo is True:
name = job["name"]
else:
name = use_pgo
dependencies = "generate-profile-{}".format(name)
job.setdefault("dependencies", {})["generate-profile"] = dependencies
job.setdefault("fetches", {})["generate-profile"] = ["profdata.tar.xz"]
job["worker"]["env"].update({"TASKCLUSTER_PGO_PROFILE_USE": "1"})
_, worker_os = worker_type_implementation(config.graph_config,
job["worker-type"])
if worker_os == "linux":
# LTO linkage needs more open files than the default from run-task.
job["worker"]["env"].update({"MOZ_LIMIT_NOFILE": "8192"})
yield job
def add_resource_monitor(config, jobs):
for job in jobs:
if job.get("attributes", {}).get("resource-monitor"):
worker_implementation, worker_os = worker_type_implementation(
config.graph_config, job["worker-type"])
# Normalise worker os so that linux-bitbar and similar use linux tools.
worker_os = worker_os.split("-")[0]
if "win7" in job["worker-type"]:
arch = "32"
else:
arch = "64"
job.setdefault("fetches", {})
job["fetches"].setdefault("toolchain", [])
job["fetches"]["toolchain"].append(
f"{worker_os}{arch}-resource-monitor")
if worker_implementation == "docker-worker":
artifact_source = "/builds/worker/monitoring/resource-monitor.json"
else:
artifact_source = "monitoring/resource-monitor.json"
job["worker"].setdefault("artifacts", [])
job["worker"]["artifacts"].append({
"name": "public/monitoring/resource-monitor.json",
"type": "file",
"path": artifact_source,
})
# Set env for output file
job["worker"].setdefault("env", {})
job["worker"]["env"]["RESOURCE_MONITOR_OUTPUT"] = artifact_source
yield job
def disable_non_linux_workers(config, jobs):
"""
Never try to run tasks on macosx or windows workers.
"""
for job in jobs:
impl, os = worker_type_implementation(config.graph_config,
job["worker-type"])
if os in ("macosx", "windows"):
job["optimization"] = {"always": None}
yield job
def set_implementation(config, jobs):
for job in jobs:
impl, os = worker_type_implementation(config.graph_config, job["worker-type"])
if os:
job.setdefault("tags", {})["os"] = os
if impl:
job.setdefault("tags", {})["worker-implementation"] = impl
worker = job.setdefault("worker", {})
assert "implementation" not in worker
worker["implementation"] = impl
if os:
worker["os"] = os
yield job
def set_implementation(config, jobs):
for job in jobs:
impl, os = worker_type_implementation(config.graph_config, job['worker-type'])
if os:
job.setdefault('tags', {})['os'] = os
if impl:
job.setdefault('tags', {})['worker-implementation'] = impl
worker = job.setdefault('worker', {})
assert 'implementation' not in worker
worker['implementation'] = impl
if os:
worker['os'] = os
yield job
def set_defaults(config, jobs):
"""Set defaults, including those that differ per worker implementation"""
for job in jobs:
job['treeherder'].setdefault('kind', 'build')
job['treeherder'].setdefault('tier', 1)
_, worker_os = worker_type_implementation(config.graph_config, job['worker-type'])
worker = job.setdefault('worker', {})
worker.setdefault('env', {})
if worker_os == "linux":
worker.setdefault('docker-image', {'in-tree': 'debian7-amd64-build'})
worker['chain-of-trust'] = True
elif worker_os == "windows":
worker['chain-of-trust'] = True
yield job
def set_defaults(config, jobs):
"""Set defaults, including those that differ per worker implementation"""
for job in jobs:
job['treeherder'].setdefault('kind', 'build')
job['treeherder'].setdefault('tier', 1)
_, worker_os = worker_type_implementation(job['worker-type'])
worker = job.setdefault('worker', {})
worker.setdefault('env', {})
if worker_os == "linux":
worker.setdefault('docker-image', {'in-tree': 'debian7-amd64-build'})
worker['chain-of-trust'] = True
elif worker_os == "windows":
worker['chain-of-trust'] = True
yield job
def set_defaults(config, jobs):
"""Set defaults, including those that differ per worker implementation"""
for job in jobs:
job["treeherder"].setdefault("kind", "build")
job["treeherder"].setdefault("tier", 1)
_, worker_os = worker_type_implementation(config.graph_config,
job["worker-type"])
worker = job.setdefault("worker", {})
worker.setdefault("env", {})
worker["chain-of-trust"] = True
if worker_os == "linux":
worker.setdefault("docker-image",
{"in-tree": "debian8-amd64-build"})
yield job
def set_defaults(config, jobs):
"""Set defaults, including those that differ per worker implementation"""
for job in jobs:
job['treeherder'].setdefault('kind', 'build')
job['treeherder'].setdefault('tier', 1)
job.setdefault('needs-sccache', True)
_, worker_os = worker_type_implementation(job['worker-type'])
if worker_os == "linux":
worker = job.setdefault('worker')
worker.setdefault('docker-image', {'in-tree': 'desktop-build'})
worker['chain-of-trust'] = True
extra = job.setdefault('extra', {})
extra.setdefault('chainOfTrust', {})
extra['chainOfTrust'].setdefault('inputs', {})
extra['chainOfTrust']['inputs']['docker-image'] = {
"task-reference": "<docker-image>"
}
elif worker_os in set(["macosx", "windows"]):
job['worker'].setdefault('env', {})
yield job
def set_implementation(config, tasks):
"""
Set the worker implementation based on the worker-type alias.
"""
for task in tasks:
worker = task.setdefault("worker", {})
if "implementation" in task["worker"]:
yield task
continue
impl, os = worker_type_implementation(config.graph_config, task["worker-type"])
tags = task.setdefault("tags", {})
tags["worker-implementation"] = impl
if os:
task["tags"]["os"] = os
worker["implementation"] = impl
if os:
worker["os"] = os
yield task
def add_resource_monitor(config, jobs):
for job in jobs:
if job.get("attributes", {}).get("resource-monitor"):
worker_implementation, worker_os = worker_type_implementation(
config.graph_config, job["worker-type"]
)
# Normalise worker os so that linux-bitbar and similar use linux tools.
worker_os = worker_os.split("-")[0]
# We don't currently support an Arm worker, due to gopsutil's indirect
# dependencies (go-ole)
if "aarch64" in job["worker-type"]:
yield job
continue
elif "win7" in job["worker-type"]:
arch = "32"
else:
arch = "64"
job.setdefault("fetches", {})
job["fetches"].setdefault("toolchain", [])
job["fetches"]["toolchain"].append(
"{}{}-resource-monitor".format(worker_os, arch)
)
if worker_implementation == "docker-worker":
artifact_source = "/builds/worker/monitoring/resource-monitor.json"
else:
artifact_source = "monitoring/resource-monitor.json"
job["worker"].setdefault("artifacts", [])
job["worker"]["artifacts"].append(
{
"name": "public/monitoring/resource-monitor.json",
"type": "file",
"path": artifact_source,
}
)
# Set env for output file
job["worker"].setdefault("env", {})
job["worker"]["env"]["RESOURCE_MONITOR_OUTPUT"] = artifact_source
yield job
def make_task_description(config, jobs):
"""Given a build description, create a task description"""
# import plugin modules first, before iterating over jobs
import_all()
for job in jobs:
if 'label' not in job:
if 'name' not in job:
raise Exception("job has neither a name nor a label")
job['label'] = '{}-{}'.format(config.kind, job['name'])
if job.get('name'):
del job['name']
impl, os = worker_type_implementation(job['worker-type'])
if os:
job.setdefault('tags', {})['os'] = os
if impl:
job.setdefault('tags', {})['worker-implementation'] = impl
worker = job.setdefault('worker', {})
assert 'implementation' not in worker
worker['implementation'] = impl
if os:
worker['os'] = os
taskdesc = copy.deepcopy(job)
# fill in some empty defaults to make run implementations easier
taskdesc.setdefault('attributes', {})
taskdesc.setdefault('dependencies', {})
taskdesc.setdefault('routes', [])
taskdesc.setdefault('scopes', [])
taskdesc.setdefault('extra', {})
# give the function for job.run.using on this worker implementation a
# chance to set up the task description.
configure_taskdesc_for_run(config, job, taskdesc, impl)
del taskdesc['run']
# yield only the task description, discarding the job description
yield taskdesc
def make_job_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
build_platform = attributes['build_platform']
if job['build-platform'].startswith('win'):
if dep_job.kind.endswith('signing'):
continue
if job['build-platform'].startswith('macosx'):
if dep_job.kind.endswith('repack'):
continue
dependencies = {dep_job.attributes.get('kind'): dep_job.label}
dependencies.update(dep_job.dependencies)
signing_task = None
for dependency in dependencies.keys():
if build_platform.startswith('macosx') and dependency.endswith(
'signing'):
signing_task = dependency
elif build_platform.startswith('win') and dependency.endswith(
'repack'):
signing_task = dependency
attributes['repackage_type'] = 'repackage'
level = config.params['level']
repack_id = job['extra']['repack_id']
repackage_config = []
for format in job.get('package-formats'):
command = copy.deepcopy(PACKAGE_FORMATS[format])
substs = {
'archive_format': archive_format(build_platform),
'executable_extension': executable_extension(build_platform),
}
command['inputs'] = {
name: filename.format(**substs)
for name, filename in command['inputs'].items()
}
repackage_config.append(command)
run = job.get('mozharness', {})
run.update({
'using': 'mozharness',
'script': 'mozharness/scripts/repackage.py',
'job-script': 'taskcluster/scripts/builder/repackage.sh',
'actions': ['setup', 'repackage'],
'extra-workspace-cache-key': 'repackage',
'extra-config': {
'repackage_config': repackage_config,
},
})
worker = {
'chain-of-trust': True,
'max-run-time': 7200 if build_platform.startswith('win') else 3600,
'taskcluster-proxy':
True if get_artifact_prefix(dep_job) else False,
'env': {
'REPACK_ID': repack_id,
},
# Don't add generic artifact directory.
'skip-artifacts': True,
}
if build_platform.startswith('win'):
worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
run['use-magic-mh-args'] = False
else:
if build_platform.startswith('macosx'):
worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
else:
raise NotImplementedError(
'Unsupported build_platform: "{}"'.format(build_platform))
run['tooltool-downloads'] = 'internal'
worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
worker['artifacts'] = _generate_task_output_files(
dep_job,
worker_type_implementation(worker_type),
repackage_config,
partner=repack_id,
)
description = ("Repackaging for repack_id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=job['extra']['repack_id'],
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
task = {
'label':
job['label'],
'description':
description,
'worker-type':
worker_type,
'dependencies':
dependencies,
'attributes':
attributes,
'scopes': ['queue:get-artifact:releng/partner/*'],
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'routes':
job.get('routes', []),
'extra':
job.get('extra', {}),
'worker':
worker,
'run':
run,
'fetches':
_generate_download_config(dep_job,
build_platform,
signing_task,
partner=repack_id,
project=config.params["project"]),
}
if build_platform.startswith('macosx'):
task['toolchains'] = [
'linux64-libdmg',
'linux64-hfsplus',
'linux64-node',
]
yield task
return frozen_args
return inner
@pytest.mark.parametrize(
'task', [
{
'worker-type': 'aws-provisioner-v1/gecko-1-b-linux'
},
{
'worker-type': 'releng-hardware/gecko-t-win10-64-hw'
},
],
ids=lambda t: worker_type_implementation(t['worker-type'])[0])
def test_worker_caches(task, transform):
config, job, taskdesc, impl = transform(task)
add_cache(job, taskdesc, 'cache1', '/cache1')
add_cache(job, taskdesc, 'cache2', '/cache2', skip_untrusted=True)
if impl not in ('docker-worker', 'generic-worker'):
pytest.xfail("caches not implemented for '{}'".format(impl))
key = 'caches' if impl == 'docker-worker' else 'mounts'
assert key in taskdesc['worker']
assert len(taskdesc['worker'][key]) == 2
# Create a new schema object with just the part relevant to caches.
partial_schema = Schema(payload_builders[impl].schema.schema.schema[key])
validate_schema(partial_schema, taskdesc['worker'][key],
def make_job_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
dependencies = {dep_job.attributes.get('kind'): dep_job.label}
if len(dep_job.dependencies) > 1 and not config.kind == 'repackage-msi':
# repackage-signing can end up with multiple deps...
raise NotImplementedError(
"Can't repackage a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
# This is so we get the build task in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage'
locale = attributes.get('locale', job.get('locale'))
if locale:
attributes['locale'] = locale
treeherder = job.get('treeherder', {})
if attributes.get('nightly'):
treeherder.setdefault('symbol', 'Nr')
else:
treeherder.setdefault('symbol', 'Rpk')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
if config.kind == 'repackage-msi':
treeherder['symbol'] = 'MSI({})'.format(locale or 'N')
build_task = None
signing_task = None
repackage_signing_task = None
for dependency in dependencies.keys():
if 'repackage-signing' in dependency:
repackage_signing_task = dependency
elif 'signing' in dependency:
signing_task = dependency
else:
build_task = dependency
_fetch_subst_locale = 'en-US'
if locale:
# XXXCallek: todo: rewrite dependency finding
# Use string splice to strip out 'nightly-l10n-' .. '-<chunk>/opt'
# We need this additional dependency to support finding the mar binary
# Which is needed in order to generate a new complete.mar
dependencies['build'] = "build-{}/opt".format(
dependencies[build_task][13:dependencies[build_task].rfind('-')])
build_task = 'build'
_fetch_subst_locale = locale
level = config.params['level']
build_platform = attributes['build_platform']
use_stub = attributes.get('stub-installer')
repackage_config = []
package_formats = job.get('package-formats')
if use_stub and not repackage_signing_task:
# if repackage_signing_task doesn't exists, generate the stub installer
package_formats += ['installer-stub']
for format in package_formats:
command = copy.deepcopy(PACKAGE_FORMATS[format])
substs = {
'archive_format': archive_format(build_platform),
'executable_extension': executable_extension(build_platform),
'_locale': _fetch_subst_locale,
'architecture': architecture(build_platform),
'version_display': config.params['version'],
}
# Allow us to replace args a well, but specifying things expanded in mozharness
# Without breaking .format and without allowing unknown through
substs.update({name: '{{{}}}'.format(name)
for name in MOZHARNESS_EXPANSIONS})
command['inputs'] = {
name: filename.format(**substs)
for name, filename in command['inputs'].items()
}
command['args'] = [
arg.format(**substs) for arg in command['args']
]
if 'installer' in format and 'aarch64' not in build_platform:
command['args'].append('--use-upx')
repackage_config.append(command)
run = job.get('mozharness', {})
run.update({
'using': 'mozharness',
'script': 'mozharness/scripts/repackage.py',
'job-script': 'taskcluster/scripts/builder/repackage.sh',
'actions': ['setup', 'repackage'],
'extra-workspace-cache-key': 'repackage',
'extra-config': {
'repackage_config': repackage_config,
},
})
worker = {
'chain-of-trust': True,
'max-run-time': 7200 if build_platform.startswith('win') else 3600,
# Don't add generic artifact directory.
'skip-artifacts': True,
}
if locale:
# Make sure we specify the locale-specific upload dir
worker.setdefault('env', {}).update(LOCALE=locale)
if build_platform.startswith('win'):
worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
run['use-magic-mh-args'] = False
else:
if build_platform.startswith(('linux', 'macosx')):
worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
else:
raise NotImplementedError(
'Unsupported build_platform: "{}"'.format(build_platform)
)
run['tooltool-downloads'] = 'internal'
worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
worker['artifacts'] = _generate_task_output_files(
dep_job, worker_type_implementation(worker_type),
repackage_config=repackage_config,
locale=locale,
)
description = (
"Repackaging for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
task = {
'label': job['label'],
'description': description,
'worker-type': worker_type,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'routes': job.get('routes', []),
'extra': job.get('extra', {}),
'worker': worker,
'run': run,
'fetches': _generate_download_config(dep_job, build_platform, build_task,
signing_task, repackage_signing_task,
locale=locale,
project=config.params["project"],
existing_fetch=job.get('fetches')),
'release-artifacts': [artifact['name'] for artifact in worker['artifacts']]
}
if build_platform.startswith('macosx'):
task['toolchains'] = [
'linux64-libdmg',
'linux64-hfsplus',
'linux64-node',
]
yield task
def use_fetches(config, jobs):
artifact_names = {}
for task in config.kind_dependencies_tasks:
if task.kind in ('fetch', 'toolchain'):
get_attribute(
artifact_names,
task.label,
task.attributes,
'{kind}-artifact'.format(kind=task.kind),
)
for job in jobs:
fetches = job.pop('fetches', None)
if not fetches:
yield job
continue
# Hack added for `mach artifact toolchain` to support reading toolchain
# kinds in isolation.
if 'fetch' in fetches and config.params.get('ignore_fetches'):
fetches['fetch'][:] = []
job_fetches = []
name = job.get('name', job.get('label'))
dependencies = job.setdefault('dependencies', {})
prefix = get_artifact_prefix(job)
for kind, artifacts in fetches.items():
if kind in ('fetch', 'toolchain'):
for fetch_name in artifacts:
label = '{kind}-{name}'.format(kind=kind, name=fetch_name)
if label not in artifact_names:
raise Exception(
'Missing fetch job for {kind}-{name}: {fetch}'.
format(kind=config.kind,
name=name,
fetch=fetch_name))
path = artifact_names[label]
if not path.startswith('public/'):
raise Exception(
'Non-public artifacts not supported for {kind}-{name}: '
'{fetch}'.format(kind=config.kind,
name=name,
fetch=fetch_name))
dependencies[label] = label
job_fetches.append({
'artifact': path,
'task': '<{label}>'.format(label=label),
'extract': True,
})
else:
if kind not in dependencies:
raise Exception(
"{name} can't fetch {kind} artifacts because "
"it has no {kind} dependencies!".format(name=name,
kind=kind))
for artifact in artifacts:
if isinstance(artifact, basestring):
path = artifact
dest = None
extract = True
else:
path = artifact['artifact']
dest = artifact.get('dest')
extract = artifact.get('extract', True)
fetch = {
'artifact':
'{prefix}/{path}'.format(prefix=prefix, path=path),
'task':
'<{dep}>'.format(dep=kind),
'extract':
extract,
}
if dest is not None:
fetch['dest'] = dest
job_fetches.append(fetch)
env = job.setdefault('worker', {}).setdefault('env', {})
env['MOZ_FETCHES'] = {
'task-reference': json.dumps(job_fetches, sort_keys=True)
}
impl, os = worker_type_implementation(job['worker-type'])
if os == 'windows':
env.setdefault('MOZ_FETCHES_DIR', 'fetches')
else:
workdir = job['run'].get('workdir', '/builds/worker')
env.setdefault('MOZ_FETCHES_DIR', '{}/fetches'.format(workdir))
yield job
def make_job_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["repackage_type"] = "repackage"
locale = attributes.get("locale", job.get("locale"))
if locale:
attributes["locale"] = locale
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "Rpk")
dep_th_platform = dep_job.task.get("extra",
{}).get("treeherder-platform")
treeherder.setdefault("platform", dep_th_platform)
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
if config.kind == "repackage-msi":
treeherder["symbol"] = "MSI({})".format(locale or "N")
signing_task = None
repackage_signing_task = None
for dependency in dependencies.keys():
if "repackage-signing" in dependency:
repackage_signing_task = dependency
elif "signing" in dependency:
signing_task = dependency
_fetch_subst_locale = "en-US"
if locale:
_fetch_subst_locale = locale
worker_type = job["worker-type"]
build_platform = attributes["build_platform"]
use_stub = attributes.get("stub-installer")
repackage_config = []
package_formats = job.get("package-formats")
if use_stub and not repackage_signing_task:
# if repackage_signing_task doesn't exists, generate the stub installer
package_formats += ["installer-stub"]
for format in package_formats:
command = copy.deepcopy(PACKAGE_FORMATS[format])
substs = {
"archive_format": archive_format(build_platform),
"_locale": _fetch_subst_locale,
"architecture": architecture(build_platform),
"version_display": config.params["version"],
"mar-channel-id": attributes["mar-channel-id"],
}
# Allow us to replace args a well, but specifying things expanded in mozharness
# Without breaking .format and without allowing unknown through
substs.update({
name: "{{{}}}".format(name)
for name in MOZHARNESS_EXPANSIONS
})
command["inputs"] = {
name: filename.format(**substs)
for name, filename in command["inputs"].items()
}
command["args"] = [arg.format(**substs) for arg in command["args"]]
if "installer" in format and "aarch64" not in build_platform:
command["args"].append("--use-upx")
repackage_config.append(command)
run = job.get("mozharness", {})
run.update({
"using": "mozharness",
"script": "mozharness/scripts/repackage.py",
"job-script": "taskcluster/scripts/builder/repackage.sh",
"actions": ["setup", "repackage"],
"extra-config": {
"repackage_config": repackage_config,
},
})
worker = job.get("worker", {})
worker.update({
"chain-of-trust":
True,
"max-run-time":
7200 if build_platform.startswith("win") else 3600,
# Don't add generic artifact directory.
"skip-artifacts":
True,
})
if locale:
# Make sure we specify the locale-specific upload dir
worker.setdefault("env", {})["LOCALE"] = locale
worker["artifacts"] = _generate_task_output_files(
dep_job,
worker_type_implementation(config.graph_config, worker_type),
repackage_config=repackage_config,
locale=locale,
)
description = ("Repackaging for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
task = {
"label":
job["label"],
"description":
description,
"worker-type":
worker_type,
"dependencies":
dependencies,
"if-dependencies": [dep_job.kind],
"attributes":
attributes,
"run-on-projects":
dep_job.attributes.get("run_on_projects"),
"optimization":
dep_job.optimization,
"treeherder":
treeherder,
"routes":
job.get("routes", []),
"extra":
job.get("extra", {}),
"worker":
worker,
"run":
run,
"fetches":
_generate_download_config(
dep_job,
build_platform,
signing_task,
repackage_signing_task,
locale=locale,
project=config.params["project"],
existing_fetch=job.get("fetches"),
),
"release-artifacts":
[artifact["name"] for artifact in worker["artifacts"]],
}
if build_platform.startswith("macosx"):
task.setdefault("fetches", {}).setdefault("toolchain", []).extend([
"linux64-libdmg",
"linux64-hfsplus",
"linux64-node",
])
yield task
def make_job_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
build_platform = attributes['build_platform']
if job['build-platform'].startswith('win'):
if dep_job.kind.endswith('signing'):
continue
if job['build-platform'].startswith('macosx'):
if dep_job.kind.endswith('repack'):
continue
dependencies = {dep_job.attributes.get('kind'): dep_job.label}
dependencies.update(dep_job.dependencies)
signing_task = None
for dependency in dependencies.keys():
if build_platform.startswith('macosx') and dependency.endswith(
'signing'):
signing_task = dependency
elif build_platform.startswith('win') and dependency.endswith(
'repack'):
signing_task = dependency
attributes['repackage_type'] = 'repackage'
repack_id = job['extra']['repack_id']
partner_config = get_partner_config_by_kind(config, config.kind)
partner, subpartner, _ = repack_id.split('/')
repack_stub_installer = partner_config[partner][subpartner].get(
'repack_stub_installer')
if build_platform.startswith('win32') and repack_stub_installer:
job['package-formats'].append('installer-stub')
repackage_config = []
for format in job.get('package-formats'):
command = copy.deepcopy(PACKAGE_FORMATS[format])
substs = {
'archive_format': archive_format(build_platform),
'executable_extension': executable_extension(build_platform),
}
command['inputs'] = {
name: filename.format(**substs)
for name, filename in command['inputs'].items()
}
repackage_config.append(command)
run = job.get('mozharness', {})
run.update({
'using': 'mozharness',
'script': 'mozharness/scripts/repackage.py',
'job-script': 'taskcluster/scripts/builder/repackage.sh',
'actions': ['setup', 'repackage'],
'extra-config': {
'repackage_config': repackage_config,
},
})
worker = {
'chain-of-trust': True,
'max-run-time': 7200 if build_platform.startswith('win') else 3600,
'taskcluster-proxy':
True if get_artifact_prefix(dep_job) else False,
'env': {
'REPACK_ID': repack_id,
},
# Don't add generic artifact directory.
'skip-artifacts': True,
}
worker_type = 'b-linux'
worker['docker-image'] = {"in-tree": "debian8-amd64-build"}
worker['artifacts'] = _generate_task_output_files(
dep_job,
worker_type_implementation(config.graph_config, worker_type),
repackage_config,
partner=repack_id,
)
description = ("Repackaging for repack_id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=job['extra']['repack_id'],
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
task = {
'label':
job['label'],
'description':
description,
'worker-type':
worker_type,
'dependencies':
dependencies,
'attributes':
attributes,
'scopes': ['queue:get-artifact:releng/partner/*'],
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'routes':
job.get('routes', []),
'extra':
job.get('extra', {}),
'worker':
worker,
'run':
run,
'fetches':
_generate_download_config(
dep_job,
build_platform,
signing_task,
partner=repack_id,
project=config.params["project"],
repack_stub_installer=repack_stub_installer),
}
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get('priority'):
task['priority'] = job['priority']
if build_platform.startswith('macosx'):
task.setdefault('fetches', {}).setdefault('toolchain', []).extend([
'linux64-libdmg',
'linux64-hfsplus',
'linux64-node',
])
yield task
def make_job_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
build_platform = attributes["build_platform"]
if job["build-platform"].startswith("win"):
if dep_job.kind.endswith("signing"):
continue
if job["build-platform"].startswith("macosx"):
if dep_job.kind.endswith("repack"):
continue
dependencies = {dep_job.attributes.get("kind"): dep_job.label}
dependencies.update(dep_job.dependencies)
signing_task = None
for dependency in dependencies.keys():
if build_platform.startswith("macosx") and dependency.endswith(
"signing"):
signing_task = dependency
elif build_platform.startswith("win") and dependency.endswith(
"repack"):
signing_task = dependency
attributes["repackage_type"] = "repackage"
repack_id = job["extra"]["repack_id"]
partner_config = get_partner_config_by_kind(config, config.kind)
partner, subpartner, _ = repack_id.split("/")
repack_stub_installer = partner_config[partner][subpartner].get(
"repack_stub_installer")
if build_platform.startswith("win32") and repack_stub_installer:
job["package-formats"].append("installer-stub")
repackage_config = []
for format in job.get("package-formats"):
command = copy.deepcopy(PACKAGE_FORMATS[format])
substs = {
"archive_format": archive_format(build_platform),
"executable_extension": executable_extension(build_platform),
}
command["inputs"] = {
name: filename.format(**substs)
for name, filename in command["inputs"].items()
}
repackage_config.append(command)
run = job.get("mozharness", {})
run.update({
"using": "mozharness",
"script": "mozharness/scripts/repackage.py",
"job-script": "taskcluster/scripts/builder/repackage.sh",
"actions": ["setup", "repackage"],
"extra-config": {
"repackage_config": repackage_config,
},
})
worker = {
"chain-of-trust": True,
"max-run-time": 7200 if build_platform.startswith("win") else 3600,
"taskcluster-proxy":
True if get_artifact_prefix(dep_job) else False,
"env": {
"REPACK_ID": repack_id,
},
# Don't add generic artifact directory.
"skip-artifacts": True,
}
worker_type = "b-linux"
worker["docker-image"] = {"in-tree": "debian10-amd64-build"}
worker["artifacts"] = _generate_task_output_files(
dep_job,
worker_type_implementation(config.graph_config, worker_type),
repackage_config,
partner=repack_id,
)
description = ("Repackaging for repack_id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=job["extra"]["repack_id"],
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
task = {
"label":
job["label"],
"description":
description,
"worker-type":
worker_type,
"dependencies":
dependencies,
"attributes":
attributes,
"scopes": ["queue:get-artifact:releng/partner/*"],
"run-on-projects":
dep_job.attributes.get("run_on_projects"),
"routes":
job.get("routes", []),
"extra":
job.get("extra", {}),
"worker":
worker,
"run":
run,
"fetches":
_generate_download_config(
dep_job,
build_platform,
signing_task,
partner=repack_id,
project=config.params["project"],
repack_stub_installer=repack_stub_installer,
),
}
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get("priority"):
task["priority"] = job["priority"]
if build_platform.startswith("macosx"):
task.setdefault("fetches", {}).setdefault("toolchain", []).extend([
"linux64-libdmg",
"linux64-hfsplus",
"linux64-node",
])
yield task
def make_job_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
dependencies = {dep_job.attributes.get('kind'): dep_job.label}
if len(dep_job.dependencies) > 1 and not config.kind == 'repackage-msi':
# repackage-signing can end up with multiple deps...
raise NotImplementedError(
"Can't repackage a signing task with multiple dependencies")
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage'
locale = attributes.get('locale', job.get('locale'))
if locale:
attributes['locale'] = locale
treeherder = job.get('treeherder', {})
if attributes.get('nightly'):
treeherder.setdefault('symbol', 'Nr')
else:
treeherder.setdefault('symbol', 'Rpk')
dep_th_platform = dep_job.task.get('extra', {}).get('treeherder-platform')
treeherder.setdefault('platform', dep_th_platform)
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
if config.kind == 'repackage-msi':
treeherder['symbol'] = 'MSI({})'.format(locale or 'N')
signing_task = None
repackage_signing_task = None
for dependency in dependencies.keys():
if 'repackage-signing' in dependency:
repackage_signing_task = dependency
elif 'signing' in dependency:
signing_task = dependency
_fetch_subst_locale = 'en-US'
if locale:
_fetch_subst_locale = locale
worker_type = job['worker-type']
build_platform = attributes['build_platform']
use_stub = attributes.get('stub-installer')
repackage_config = []
package_formats = job.get('package-formats')
if use_stub and not repackage_signing_task:
# if repackage_signing_task doesn't exists, generate the stub installer
package_formats += ['installer-stub']
for format in package_formats:
command = copy.deepcopy(PACKAGE_FORMATS[format])
substs = {
'archive_format': archive_format(build_platform),
'_locale': _fetch_subst_locale,
'architecture': architecture(build_platform),
'version_display': config.params['version'],
'mar-channel-id': attributes['mar-channel-id'],
}
# Allow us to replace args a well, but specifying things expanded in mozharness
# Without breaking .format and without allowing unknown through
substs.update({name: '{{{}}}'.format(name)
for name in MOZHARNESS_EXPANSIONS})
command['inputs'] = {
name: filename.format(**substs)
for name, filename in command['inputs'].items()
}
command['args'] = [
arg.format(**substs) for arg in command['args']
]
if 'installer' in format and 'aarch64' not in build_platform:
command['args'].append('--use-upx')
repackage_config.append(command)
run = job.get('mozharness', {})
run.update({
'using': 'mozharness',
'script': 'mozharness/scripts/repackage.py',
'job-script': 'taskcluster/scripts/builder/repackage.sh',
'actions': ['setup', 'repackage'],
'extra-workspace-cache-key': 'repackage',
'extra-config': {
'repackage_config': repackage_config,
},
})
worker = job.get('worker', {})
worker.update({
'chain-of-trust': True,
'max-run-time': 7200 if build_platform.startswith('win') else 3600,
# Don't add generic artifact directory.
'skip-artifacts': True,
})
if locale:
# Make sure we specify the locale-specific upload dir
worker.setdefault('env', {}).update(LOCALE=locale)
worker['artifacts'] = _generate_task_output_files(
dep_job, worker_type_implementation(config.graph_config, worker_type),
repackage_config=repackage_config,
locale=locale,
)
description = (
"Repackaging for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
task = {
'label': job['label'],
'description': description,
'worker-type': worker_type,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'routes': job.get('routes', []),
'extra': job.get('extra', {}),
'worker': worker,
'run': run,
'fetches': _generate_download_config(dep_job, build_platform,
signing_task, repackage_signing_task,
locale=locale,
project=config.params["project"],
existing_fetch=job.get('fetches')),
'release-artifacts': [artifact['name'] for artifact in worker['artifacts']]
}
if build_platform.startswith('macosx'):
task.setdefault('fetches', {}).setdefault('toolchain', []).extend([
'linux64-libdmg',
'linux64-hfsplus',
'linux64-node',
])
yield task
