def make_job_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
dependencies = {dep_job.attributes.get('kind'): dep_job.label}
if len(dep_job.dependencies) > 1 and not config.kind == 'repackage-msi':
# repackage-signing can end up with multiple deps...
raise NotImplementedError(
"Can't repackage a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
# This is so we get the build task in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage'
locale = attributes.get('locale', job.get('locale'))
if locale:
attributes['locale'] = locale
treeherder = job.get('treeherder', {})
if attributes.get('nightly'):
treeherder.setdefault('symbol', 'Nr')
else:
treeherder.setdefault('symbol', 'Rpk')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
if config.kind == 'repackage-msi':
treeherder['symbol'] = 'MSI({})'.format(locale or 'N')
build_task = None
signing_task = None
repackage_signing_task = None
for dependency in dependencies.keys():
if 'repackage-signing' in dependency:
repackage_signing_task = dependency
elif 'signing' in dependency:
signing_task = dependency
else:
build_task = dependency
_fetch_subst_locale = 'en-US'
if locale:
# XXXCallek: todo: rewrite dependency finding
# Use string splice to strip out 'nightly-l10n-' .. '-<chunk>/opt'
# We need this additional dependency to support finding the mar binary
# Which is needed in order to generate a new complete.mar
dependencies['build'] = "build-{}/opt".format(
dependencies[build_task][13:dependencies[build_task].rfind('-')])
build_task = 'build'
_fetch_subst_locale = locale
level = config.params['level']
build_platform = attributes['build_platform']
use_stub = attributes.get('stub-installer')
repackage_config = []
package_formats = job.get('package-formats')
if use_stub and not repackage_signing_task:
# if repackage_signing_task doesn't exists, generate the stub installer
package_formats += ['installer-stub']
for format in package_formats:
command = copy.deepcopy(PACKAGE_FORMATS[format])
substs = {
'archive_format': archive_format(build_platform),
'executable_extension': executable_extension(build_platform),
'_locale': _fetch_subst_locale,
'architecture': architecture(build_platform),
'version_display': config.params['version'],
}
# Allow us to replace args a well, but specifying things expanded in mozharness
# Without breaking .format and without allowing unknown through
substs.update({name: '{{{}}}'.format(name)
for name in MOZHARNESS_EXPANSIONS})
command['inputs'] = {
name: filename.format(**substs)
for name, filename in command['inputs'].items()
}
command['args'] = [
arg.format(**substs) for arg in command['args']
]
if 'installer' in format and 'aarch64' not in build_platform:
command['args'].append('--use-upx')
repackage_config.append(command)
run = job.get('mozharness', {})
run.update({
'using': 'mozharness',
'script': 'mozharness/scripts/repackage.py',
'job-script': 'taskcluster/scripts/builder/repackage.sh',
'actions': ['setup', 'repackage'],
'extra-workspace-cache-key': 'repackage',
'extra-config': {
'repackage_config': repackage_config,
},
})
worker = {
'chain-of-trust': True,
'max-run-time': 7200 if build_platform.startswith('win') else 3600,
# Don't add generic artifact directory.
'skip-artifacts': True,
}
if locale:
# Make sure we specify the locale-specific upload dir
worker.setdefault('env', {}).update(LOCALE=locale)
if build_platform.startswith('win'):
worker_type = 'aws-provisioner-v1/gecko-%s-b-win2012' % level
run['use-magic-mh-args'] = False
else:
if build_platform.startswith(('linux', 'macosx')):
worker_type = 'aws-provisioner-v1/gecko-%s-b-linux' % level
else:
raise NotImplementedError(
'Unsupported build_platform: "{}"'.format(build_platform)
)
run['tooltool-downloads'] = 'internal'
worker['docker-image'] = {"in-tree": "debian7-amd64-build"}
worker['artifacts'] = _generate_task_output_files(
dep_job, worker_type_implementation(worker_type),
repackage_config=repackage_config,
locale=locale,
)
description = (
"Repackaging for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
task = {
'label': job['label'],
'description': description,
'worker-type': worker_type,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'routes': job.get('routes', []),
'extra': job.get('extra', {}),
'worker': worker,
'run': run,
'fetches': _generate_download_config(dep_job, build_platform, build_task,
signing_task, repackage_signing_task,
locale=locale,
project=config.params["project"],
existing_fetch=job.get('fetches')),
'release-artifacts': [artifact['name'] for artifact in worker['artifacts']]
}
if build_platform.startswith('macosx'):
task['toolchains'] = [
'linux64-libdmg',
'linux64-hfsplus',
'linux64-node',
]
yield task
def make_job_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
dependencies = {dep_job.attributes.get('kind'): dep_job.label}
if len(dep_job.dependencies) > 1 and not config.kind == 'repackage-msi':
# repackage-signing can end up with multiple deps...
raise NotImplementedError(
"Can't repackage a signing task with multiple dependencies")
attributes = copy_attributes_from_dependent_job(dep_job)
attributes['repackage_type'] = 'repackage'
locale = attributes.get('locale', job.get('locale'))
if locale:
attributes['locale'] = locale
treeherder = job.get('treeherder', {})
if attributes.get('nightly'):
treeherder.setdefault('symbol', 'Nr')
else:
treeherder.setdefault('symbol', 'Rpk')
dep_th_platform = dep_job.task.get('extra', {}).get('treeherder-platform')
treeherder.setdefault('platform', dep_th_platform)
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
if config.kind == 'repackage-msi':
treeherder['symbol'] = 'MSI({})'.format(locale or 'N')
signing_task = None
repackage_signing_task = None
for dependency in dependencies.keys():
if 'repackage-signing' in dependency:
repackage_signing_task = dependency
elif 'signing' in dependency:
signing_task = dependency
_fetch_subst_locale = 'en-US'
if locale:
_fetch_subst_locale = locale
worker_type = job['worker-type']
build_platform = attributes['build_platform']
use_stub = attributes.get('stub-installer')
repackage_config = []
package_formats = job.get('package-formats')
if use_stub and not repackage_signing_task:
# if repackage_signing_task doesn't exists, generate the stub installer
package_formats += ['installer-stub']
for format in package_formats:
command = copy.deepcopy(PACKAGE_FORMATS[format])
substs = {
'archive_format': archive_format(build_platform),
'_locale': _fetch_subst_locale,
'architecture': architecture(build_platform),
'version_display': config.params['version'],
'mar-channel-id': attributes['mar-channel-id'],
}
# Allow us to replace args a well, but specifying things expanded in mozharness
# Without breaking .format and without allowing unknown through
substs.update({name: '{{{}}}'.format(name)
for name in MOZHARNESS_EXPANSIONS})
command['inputs'] = {
name: filename.format(**substs)
for name, filename in command['inputs'].items()
}
command['args'] = [
arg.format(**substs) for arg in command['args']
]
if 'installer' in format and 'aarch64' not in build_platform:
command['args'].append('--use-upx')
repackage_config.append(command)
run = job.get('mozharness', {})
run.update({
'using': 'mozharness',
'script': 'mozharness/scripts/repackage.py',
'job-script': 'taskcluster/scripts/builder/repackage.sh',
'actions': ['setup', 'repackage'],
'extra-workspace-cache-key': 'repackage',
'extra-config': {
'repackage_config': repackage_config,
},
})
worker = job.get('worker', {})
worker.update({
'chain-of-trust': True,
'max-run-time': 7200 if build_platform.startswith('win') else 3600,
# Don't add generic artifact directory.
'skip-artifacts': True,
})
if locale:
# Make sure we specify the locale-specific upload dir
worker.setdefault('env', {}).update(LOCALE=locale)
worker['artifacts'] = _generate_task_output_files(
dep_job, worker_type_implementation(config.graph_config, worker_type),
repackage_config=repackage_config,
locale=locale,
)
description = (
"Repackaging for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')
)
)
task = {
'label': job['label'],
'description': description,
'worker-type': worker_type,
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'routes': job.get('routes', []),
'extra': job.get('extra', {}),
'worker': worker,
'run': run,
'fetches': _generate_download_config(dep_job, build_platform,
signing_task, repackage_signing_task,
locale=locale,
project=config.params["project"],
existing_fetch=job.get('fetches')),
'release-artifacts': [artifact['name'] for artifact in worker['artifacts']]
}
if build_platform.startswith('macosx'):
task.setdefault('fetches', {}).setdefault('toolchain', []).extend([
'linux64-libdmg',
'linux64-hfsplus',
'linux64-node',
])
yield task
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get('release_history'):
return
for job in jobs:
dep_job = job['primary-dependency']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'p(N)')
label = job.get('label', "partials-{}".format(dep_job.label))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('kind', 'build')
treeherder.setdefault('tier', 1)
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = "p({})".format(locale)
attributes['shipping_phase'] = job['shipping-phase']
build_locale = locale or 'en-US'
builds = get_builds(config.params['release_history'], dep_th_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
extra = {'funsize': {'partials': list()}}
update_number = 1
locale_suffix = ''
if locale:
locale_suffix = '{}/'.format(locale)
artifact_path = "<{}/{}/{}target.complete.mar>".format(
dep_job.kind,
get_artifact_prefix(dep_job),
locale_suffix,
)
for build in sorted(builds):
partial_info = {
'locale': build_locale,
'from_mar': builds[build]['mar_url'],
'to_mar': {
'artifact-reference': artifact_path
},
'platform': get_balrog_platform_name(dep_th_platform),
'branch': config.params['project'],
'update_number': update_number,
'dest_mar': build,
}
if 'product' in builds[build]:
partial_info['product'] = builds[build]['product']
if 'previousVersion' in builds[build]:
partial_info['previousVersion'] = builds[build][
'previousVersion']
if 'previousBuildNumber' in builds[build]:
partial_info['previousBuildNumber'] = builds[build][
'previousBuildNumber']
extra['funsize']['partials'].append(partial_info)
update_number += 1
mar_channel_id = None
if config.params['project'] == 'mozilla-beta':
if 'devedition' in label:
mar_channel_id = 'firefox-mozilla-aurora'
else:
mar_channel_id = 'firefox-mozilla-beta'
elif config.params['project'] == 'mozilla-release':
mar_channel_id = 'firefox-mozilla-release'
elif 'esr' in config.params['project']:
mar_channel_id = 'firefox-mozilla-esr'
level = config.params['level']
worker = {
'artifacts':
_generate_task_output_files(dep_job, builds.keys(), locale),
'implementation':
'docker-worker',
'docker-image': {
'in-tree': 'funsize-update-generator'
},
'os':
'linux',
'max-run-time':
3600 if 'asan' in dep_job.label else 600,
'chain-of-trust':
True,
'taskcluster-proxy':
True,
'env': {
'SHA1_SIGNING_CERT':
'nightly_sha1',
'SHA384_SIGNING_CERT':
'nightly_sha384',
'DATADOG_API_SECRET':
'project/releng/gecko/build/level-{}/datadog-api-key'.format(
level),
'EXTRA_PARAMS':
'--arch={}'.format(architecture(attributes['build_platform'])),
}
}
if mar_channel_id:
worker['env']['ACCEPTED_MAR_CHANNEL_IDS'] = mar_channel_id
if config.params.release_level() == 'staging':
worker['env']['FUNSIZE_ALLOW_STAGING_PREFIXES'] = 'true'
task = {
'label':
label,
'description':
"{} Partials".format(dep_job.task["metadata"]["description"]),
'worker-type':
'aws-provisioner-v1/gecko-%s-b-linux' % level,
'dependencies':
dependencies,
'scopes': [
'secrets:get:project/releng/gecko/build/level-%s/datadog-api-key'
% level
],
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
'extra':
extra,
'worker':
worker,
}
# We only want caching on linux/windows due to bug 1436977
if int(level) == 3 \
and any([platform in dep_th_platform for platform in ['linux', 'windows']]):
task['scopes'].append(
'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/'
)
yield task
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get('release_history'):
return
for job in jobs:
dep_job = job['primary-dependency']
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault('symbol', 'p(N)')
label = job.get('label', "partials-{}".format(dep_job.label))
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get('locale')
if locale:
attributes['locale'] = locale
treeherder['symbol'] = "p({})".format(locale)
attributes['shipping_phase'] = job['shipping-phase']
build_locale = locale or 'en-US'
build_platform = attributes['build_platform']
builds = get_builds(config.params['release_history'], build_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
extra = {'funsize': {'partials': list()}}
update_number = 1
locale_suffix = ''
if locale:
locale_suffix = '{}/'.format(locale)
artifact_path = "<{}/{}/{}target.complete.mar>".format(
dep_job.kind,
get_artifact_prefix(dep_job),
locale_suffix,
)
for build in sorted(builds):
partial_info = {
'locale': build_locale,
'from_mar': builds[build]['mar_url'],
'to_mar': {
'artifact-reference': artifact_path
},
'branch': config.params['project'],
'update_number': update_number,
'dest_mar': build,
}
if 'product' in builds[build]:
partial_info['product'] = builds[build]['product']
if 'previousVersion' in builds[build]:
partial_info['previousVersion'] = builds[build][
'previousVersion']
if 'previousBuildNumber' in builds[build]:
partial_info['previousBuildNumber'] = builds[build][
'previousBuildNumber']
extra['funsize']['partials'].append(partial_info)
update_number += 1
level = config.params['level']
worker = {
'artifacts':
_generate_task_output_files(dep_job, builds.keys(), locale),
'implementation':
'docker-worker',
'docker-image': {
'in-tree': 'funsize-update-generator'
},
'os':
'linux',
'max-run-time':
3600 if 'asan' in dep_job.label else 900,
'chain-of-trust':
True,
'taskcluster-proxy':
True,
'env': {
'SIGNING_CERT':
identify_desired_signing_keys(
config.params["project"],
config.params['release_product']),
'EXTRA_PARAMS':
'--arch={}'.format(architecture(build_platform)),
'MAR_CHANNEL_ID':
attributes['mar-channel-id']
}
}
if config.params.release_level() == 'staging':
worker['env']['FUNSIZE_ALLOW_STAGING_PREFIXES'] = 'true'
task = {
'label':
label,
'description':
"{} Partials".format(dep_job.task["metadata"]["description"]),
'worker-type':
'b-linux',
'dependencies':
dependencies,
'scopes': [],
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
'extra':
extra,
'worker':
worker,
}
# We only want caching on linux/windows due to bug 1436977
if int(level) == 3 \
and any([build_platform.startswith(prefix) for prefix in ['linux', 'win']]):
task['scopes'].append(
'auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/'
)
yield task
def make_job_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["repackage_type"] = "repackage"
locale = attributes.get("locale", job.get("locale"))
if locale:
attributes["locale"] = locale
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "Rpk")
dep_th_platform = dep_job.task.get("extra",
{}).get("treeherder-platform")
treeherder.setdefault("platform", dep_th_platform)
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
if config.kind == "repackage-msi":
treeherder["symbol"] = "MSI({})".format(locale or "N")
signing_task = None
repackage_signing_task = None
for dependency in dependencies.keys():
if "repackage-signing" in dependency:
repackage_signing_task = dependency
elif "signing" in dependency:
signing_task = dependency
_fetch_subst_locale = "en-US"
if locale:
_fetch_subst_locale = locale
worker_type = job["worker-type"]
build_platform = attributes["build_platform"]
use_stub = attributes.get("stub-installer")
repackage_config = []
package_formats = job.get("package-formats")
if use_stub and not repackage_signing_task:
# if repackage_signing_task doesn't exists, generate the stub installer
package_formats += ["installer-stub"]
for format in package_formats:
command = copy.deepcopy(PACKAGE_FORMATS[format])
substs = {
"archive_format": archive_format(build_platform),
"_locale": _fetch_subst_locale,
"architecture": architecture(build_platform),
"version_display": config.params["version"],
"mar-channel-id": attributes["mar-channel-id"],
}
# Allow us to replace args a well, but specifying things expanded in mozharness
# Without breaking .format and without allowing unknown through
substs.update({
name: "{{{}}}".format(name)
for name in MOZHARNESS_EXPANSIONS
})
command["inputs"] = {
name: filename.format(**substs)
for name, filename in command["inputs"].items()
}
command["args"] = [arg.format(**substs) for arg in command["args"]]
if "installer" in format and "aarch64" not in build_platform:
command["args"].append("--use-upx")
repackage_config.append(command)
run = job.get("mozharness", {})
run.update({
"using": "mozharness",
"script": "mozharness/scripts/repackage.py",
"job-script": "taskcluster/scripts/builder/repackage.sh",
"actions": ["setup", "repackage"],
"extra-config": {
"repackage_config": repackage_config,
},
})
worker = job.get("worker", {})
worker.update({
"chain-of-trust":
True,
"max-run-time":
7200 if build_platform.startswith("win") else 3600,
# Don't add generic artifact directory.
"skip-artifacts":
True,
})
if locale:
# Make sure we specify the locale-specific upload dir
worker.setdefault("env", {})["LOCALE"] = locale
worker["artifacts"] = _generate_task_output_files(
dep_job,
worker_type_implementation(config.graph_config, worker_type),
repackage_config=repackage_config,
locale=locale,
)
description = ("Repackaging for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
task = {
"label":
job["label"],
"description":
description,
"worker-type":
worker_type,
"dependencies":
dependencies,
"if-dependencies": [dep_job.kind],
"attributes":
attributes,
"run-on-projects":
dep_job.attributes.get("run_on_projects"),
"optimization":
dep_job.optimization,
"treeherder":
treeherder,
"routes":
job.get("routes", []),
"extra":
job.get("extra", {}),
"worker":
worker,
"run":
run,
"fetches":
_generate_download_config(
dep_job,
build_platform,
signing_task,
repackage_signing_task,
locale=locale,
project=config.params["project"],
existing_fetch=job.get("fetches"),
),
"release-artifacts":
[artifact["name"] for artifact in worker["artifacts"]],
}
if build_platform.startswith("macosx"):
task.setdefault("fetches", {}).setdefault("toolchain", []).extend([
"linux64-libdmg",
"linux64-hfsplus",
"linux64-node",
])
yield task
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get("release_history"):
return
for job in jobs:
dep_job = job["primary-dependency"]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault("symbol", "p(N)")
label = job.get("label", "partials-{}".format(dep_job.label))
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get("locale")
if locale:
attributes["locale"] = locale
treeherder["symbol"] = "p({})".format(locale)
attributes["shipping_phase"] = job["shipping-phase"]
build_locale = locale or "en-US"
build_platform = attributes["build_platform"]
builds = get_builds(config.params["release_history"], build_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
extra = {"funsize": {"partials": list()}}
update_number = 1
locale_suffix = ""
if locale:
locale_suffix = "{}/".format(locale)
artifact_path = "<{}/{}/{}target.complete.mar>".format(
dep_job.kind,
get_artifact_prefix(dep_job),
locale_suffix,
)
for build in sorted(builds):
partial_info = {
"locale": build_locale,
"from_mar": builds[build]["mar_url"],
"to_mar": {
"artifact-reference": artifact_path
},
"branch": config.params["project"],
"update_number": update_number,
"dest_mar": build,
}
if "product" in builds[build]:
partial_info["product"] = builds[build]["product"]
if "previousVersion" in builds[build]:
partial_info["previousVersion"] = builds[build][
"previousVersion"]
if "previousBuildNumber" in builds[build]:
partial_info["previousBuildNumber"] = builds[build][
"previousBuildNumber"]
extra["funsize"]["partials"].append(partial_info)
update_number += 1
level = config.params["level"]
worker = {
"artifacts":
_generate_task_output_files(dep_job, builds.keys(), locale),
"implementation":
"docker-worker",
"docker-image": {
"in-tree": "funsize-update-generator"
},
"os":
"linux",
"max-run-time":
3600 if "asan" in dep_job.label else 900,
"chain-of-trust":
True,
"taskcluster-proxy":
True,
"env": {
"SIGNING_CERT":
identify_desired_signing_keys(
config.params["project"],
config.params["release_product"]),
"EXTRA_PARAMS":
"--arch={}".format(architecture(build_platform)),
"MAR_CHANNEL_ID":
attributes["mar-channel-id"],
},
}
if config.params.release_level() == "staging":
worker["env"]["FUNSIZE_ALLOW_STAGING_PREFIXES"] = "true"
task = {
"label": label,
"description": "{} Partials".format(dep_job.description),
"worker-type": "b-linux",
"dependencies": dependencies,
"scopes": [],
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
"worker": worker,
}
# We only want caching on linux/windows due to bug 1436977
if int(level) == 3 and any(
[build_platform.startswith(prefix)
for prefix in ["linux", "win"]]):
task["scopes"].append(
"auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/"
)
yield task