def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'c-Up(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder_job_symbol = dep_job.task['extra']['treeherder']['symbol']
treeherder['symbol'] = replace_group(treeherder_job_symbol, 'c-Up')
if dep_job.attributes.get('locale'):
attributes['locale'] = dep_job.attributes.get('locale')
label = job['label']
description = ("Balrog submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": ["public/manifest.json"],
}]
task = {
'label': label,
'description': description,
'worker-type': 'balrog',
'worker': {
'implementation': 'balrog',
'upstream-artifacts': upstream_artifacts,
'balrog-action': 'submit-locale',
'suffixes':
['', '-No-WNP'] if job.get('update-no-wnp') else [''],
},
'dependencies': {
'beetmover': dep_job.label
},
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job.get('shipping-phase', 'promote'),
'shipping-product': job.get('shipping-product'),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
replace_group(dep_job.task['extra']['treeherder']['symbol'], 'BM'))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {dep_job.kind: dep_job.label}
# XXX release snap-repackage has a variable number of dependencies, depending on how many
# "post-beetmover-dummy" jobs there are in the graph.
if dep_job.kind != 'release-snap-repackage' and len(
dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(
config, job_release_type=attributes.get('release-type'))
action_scope = get_beetmover_action_scope(
config, job_release_type=attributes.get('release-type'))
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job['shipping-phase'],
'shipping-product': job.get('shipping-product'),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
upstream_symbol = dep_job.task['extra']['treeherder']['symbol']
if 'build' in job['dependent-tasks']:
upstream_symbol = job['dependent-tasks']['build'].task['extra'][
'treeherder']['symbol']
treeherder.setdefault('symbol', replace_group(upstream_symbol, 'BMR'))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
label = job['label']
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
upstream_deps = job['dependent-tasks']
signing_name = "build-signing"
build_name = "build"
repackage_name = "repackage"
repackage_signing_name = "repackage-signing"
msi_signing_name = "repackage-signing-msi"
mar_signing_name = "mar-signing"
if job.get('locale'):
signing_name = "nightly-l10n-signing"
build_name = "nightly-l10n"
repackage_name = "repackage-l10n"
repackage_signing_name = "repackage-signing-l10n"
mar_signing_name = "mar-signing-l10n"
dependencies = {
"build": upstream_deps[build_name],
"repackage": upstream_deps[repackage_name],
"signing": upstream_deps[signing_name],
"mar-signing": upstream_deps[mar_signing_name],
}
if 'partials-signing' in upstream_deps:
dependencies['partials-signing'] = upstream_deps[
'partials-signing']
if msi_signing_name in upstream_deps:
dependencies[msi_signing_name] = upstream_deps[msi_signing_name]
if repackage_signing_name in upstream_deps:
dependencies["repackage-signing"] = upstream_deps[
repackage_signing_name]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if job.get('locale'):
attributes['locale'] = job['locale']
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'shipping-phase': job['shipping-phase'],
'shipping-product': job.get('shipping-product'),
}
yield task
def make_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
replace_group(dep_job.task['extra']['treeherder']['symbol'], 'cs'))
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1))
treeherder.setdefault('kind', 'build')
label = job['label']
description = (
"Signing of Checksums file for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=attributes.get('build_platform'),
build_type=attributes.get('build_type')))
dependencies = {"beetmover": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'cs({})'.format(
dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": [
"public/target.checksums",
],
"formats": ["gpg"]
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
'label':
label,
'description':
description,
'worker-type':
get_worker_type_for_scope(config, signing_cert_scope),
'worker': {
'implementation': 'scriptworker-signing',
'upstream-artifacts': upstream_artifacts,
'max-run-time': 3600
},
'scopes': [
signing_cert_scope,
add_scope_prefix(config, 'signing:format:gpg'),
],
'dependencies':
dependencies,
'attributes':
attributes,
'run-on-projects':
dep_job.attributes.get('run_on_projects'),
'treeherder':
treeherder,
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault(
'symbol',
replace_group(dep_job.task['extra']['treeherder']['symbol'], 'BMcs')
)
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault(
'tier',
dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)
)
treeherder.setdefault('kind', 'build')
label = job['label']
build_platform = attributes.get('build_platform')
description = (
"Beetmover submission of checksums for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get('locale', 'en-US'),
build_platform=build_platform,
build_type=attributes.get('build_type')
)
)
extra = {}
if build_platform.startswith("android"):
extra['product'] = 'fennec'
elif 'devedition' in build_platform:
extra['product'] = 'devedition'
else:
extra['product'] = 'firefox'
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
if dep_job.attributes.get('locale'):
treeherder['symbol'] = 'BMcs({})'.format(dep_job.attributes.get('locale'))
attributes['locale'] = dep_job.attributes.get('locale')
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
'label': label,
'description': description,
'worker-type': get_worker_type_for_scope(config, bucket_scope),
'scopes': [bucket_scope, action_scope],
'dependencies': dependencies,
'attributes': attributes,
'run-on-projects': dep_job.attributes.get('run_on_projects'),
'treeherder': treeherder,
'extra': extra,
}
if 'shipping-phase' in job:
task['shipping-phase'] = job['shipping-phase']
if 'shipping-product' in job:
task['shipping-product'] = job['shipping-product']
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "c-Up(N)")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault(
"tier",
dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1))
treeherder.setdefault("kind", "build")
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder_job_symbol = dep_job.task["extra"]["treeherder"]["symbol"]
treeherder["symbol"] = replace_group(treeherder_job_symbol, "c-Up")
if dep_job.attributes.get("locale"):
attributes["locale"] = dep_job.attributes.get("locale")
label = job["label"]
description = ("Balrog submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": ["public/manifest.json"],
}]
dependencies = {"beetmover": dep_job.label}
for kind_dep in config.kind_dependencies_tasks.values():
if (kind_dep.kind == "startup-test"
and kind_dep.attributes["build_platform"]
== attributes.get("build_platform")
and kind_dep.attributes["build_type"]
== attributes.get("build_type")
and kind_dep.attributes.get("shipping_product")
== job.get("shipping-product")):
dependencies["startup-test"] = kind_dep.label
task = {
"label": label,
"description": description,
"worker-type": "balrog",
"worker": {
"implementation": "balrog",
"upstream-artifacts": upstream_artifacts,
"balrog-action": "v2-submit-locale",
"suffixes":
["", "-No-WNP"] if job.get("update-no-wnp") else [""],
},
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": job.get("shipping-phase", "promote"),
"shipping-product": job.get("shipping-product"),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
upstream_symbol = dep_job.task["extra"]["treeherder"]["symbol"]
if "build" in job["dependent-tasks"]:
upstream_symbol = job["dependent-tasks"]["build"].task["extra"][
"treeherder"]["symbol"]
treeherder.setdefault("symbol", replace_group(upstream_symbol, "BMR"))
label = job["label"]
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
upstream_deps = job["dependent-tasks"]
signing_name = "build-signing"
build_name = "build"
repackage_name = "repackage"
repackage_signing_name = "repackage-signing"
msi_signing_name = "repackage-signing-msi"
mar_signing_name = "mar-signing"
if job.get("locale"):
signing_name = "shippable-l10n-signing"
build_name = "shippable-l10n"
repackage_name = "repackage-l10n"
repackage_signing_name = "repackage-signing-l10n"
mar_signing_name = "mar-signing-l10n"
dependencies = {
"build": upstream_deps[build_name],
"repackage": upstream_deps[repackage_name],
"signing": upstream_deps[signing_name],
"mar-signing": upstream_deps[mar_signing_name],
}
if "partials-signing" in upstream_deps:
dependencies["partials-signing"] = upstream_deps[
"partials-signing"]
if msi_signing_name in upstream_deps:
dependencies[msi_signing_name] = upstream_deps[msi_signing_name]
if repackage_signing_name in upstream_deps:
dependencies["repackage-signing"] = upstream_deps[
repackage_signing_name]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if job.get("locale"):
attributes["locale"] = job["locale"]
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": job["shipping-phase"],
"shipping-product": job.get("shipping-product"),
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault(
"symbol",
replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BMcs"),
)
dep_th_platform = (
dep_job.task.get("extra", {})
.get("treeherder", {})
.get("machine", {})
.get("platform", "")
)
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault(
"tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1)
)
treeherder.setdefault("kind", "build")
label = job["label"]
build_platform = attributes.get("build_platform")
description = (
"Beetmover submission of checksums for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=build_platform,
build_type=attributes.get("build_type"),
)
)
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if dep_job.attributes.get("locale"):
treeherder["symbol"] = "BMcs({})".format(dep_job.attributes.get("locale"))
attributes["locale"] = dep_job.attributes.get("locale")
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault(
"symbol", replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BM")
)
dep_th_platform = (
dep_job.task.get("extra", {})
.get("treeherder", {})
.get("machine", {})
.get("platform", "")
)
treeherder.setdefault("platform", "{}/opt".format(dep_th_platform))
treeherder.setdefault(
"tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1)
)
treeherder.setdefault("kind", "build")
label = job["label"]
description = (
"Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
)
)
dependencies = {dep_job.kind: dep_job.label}
# XXX release snap-repackage has a variable number of dependencies, depending on how many
# "post-beetmover-dummy" jobs there are in the graph.
if dep_job.kind != "release-snap-repackage" and len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies"
)
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if job.get("locale"):
attributes["locale"] = job["locale"]
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": job["shipping-phase"],
}
yield task
