def make_beetmover_push_to_release_description(config, jobs):
for job in jobs:
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "Rel(BM-C)")
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
treeherder.setdefault("platform", job["treeherder-platform"])
label = job["name"]
description = "Beetmover push to release for '{product}'".format(
product=job["product"])
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = add_scope_prefix(config,
"beetmover:action:push-to-releases")
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"product": job["product"],
"dependencies": job["dependencies"],
"attributes": job.get("attributes", {}),
"run-on-projects": job.get("run-on-projects"),
"treeherder": treeherder,
"shipping-phase": job.get("shipping-phase", "push"),
"shipping-product": job.get("shipping-product"),
"routes": job.get("routes", []),
"extra": job.get("extra", {}),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "langpack(BM{})".format(attributes.get("l10n_chunk",
"")))
job["attributes"].update(copy_attributes_from_dependent_job(dep_job))
job["attributes"]["chunk_locales"] = dep_job.attributes.get(
"chunk_locales", ["en-US"])
job["description"] = job["description"].format(
locales="/".join(job["attributes"]["chunk_locales"]),
platform=job["attributes"]["build_platform"],
)
job["scopes"] = [
get_beetmover_bucket_scope(config),
get_beetmover_action_scope(config),
]
job["dependencies"] = {"langpack-copy": dep_job.label}
job["run-on-projects"] = job.get("run_on_projects",
dep_job.attributes["run_on_projects"])
job["treeherder"] = treeherder
job["shipping-phase"] = job.get("shipping-phase",
dep_job.attributes["shipping_phase"])
job["shipping-product"] = dep_job.attributes["shipping_product"]
yield job
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault(
"symbol",
replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BM"))
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault(
"tier",
dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1))
treeherder.setdefault("kind", "build")
label = job["label"]
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
dependencies = {dep_job.kind: dep_job.label}
# XXX release snap-repackage has a variable number of dependencies, depending on how many
# "post-beetmover-dummy" jobs there are in the graph.
if dep_job.kind != "release-snap-repackage" and len(
dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if job.get("locale"):
attributes["locale"] = job["locale"]
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": job["shipping-phase"],
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "BMcss(N)")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
label = job["label"]
build_platform = attributes.get("build_platform")
description = "Beetmover submission of checksums for source file"
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith("beetmover"):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "BMcslang(N{})".format(attributes.get("l10n_chunk", ""))
)
label = job["label"]
build_platform = attributes.get("build_platform")
description = "Beetmover submission of checksums for langpack files"
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith("beetmover"):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if "chunk_locales" in dep_job.attributes:
attributes["chunk_locales"] = dep_job.attributes["chunk_locales"]
attributes.update(job.get("attributes", {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "BM-SGenChcks")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
job_template = f"{dep_job.label}"
label = job_template.replace("signing", "beetmover")
description = "Transfer *SUMS and *SUMMARY checksums file to S3."
# first dependency is the signing task for the *SUMS files
dependencies = {dep_job.kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
# update the dependencies with the dependencies of the signing task
dependencies.update(dep_job.dependencies)
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": "promote",
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
upstream_symbol = dep_job.task["extra"]["treeherder"]["symbol"]
if "build" in job["dependent-tasks"]:
upstream_symbol = job["dependent-tasks"]["build"].task["extra"][
"treeherder"]["symbol"]
treeherder.setdefault("symbol", replace_group(upstream_symbol, "BMR"))
label = job["label"]
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
upstream_deps = job["dependent-tasks"]
signing_name = "build-signing"
build_name = "build"
repackage_name = "repackage"
repackage_signing_name = "repackage-signing"
msi_signing_name = "repackage-signing-msi"
msix_signing_name = "repackage-signing-shippable-l10n-msix"
mar_signing_name = "mar-signing"
if job.get("locale"):
signing_name = "shippable-l10n-signing"
build_name = "shippable-l10n"
repackage_name = "repackage-l10n"
repackage_signing_name = "repackage-signing-l10n"
mar_signing_name = "mar-signing-l10n"
dependencies = {
"build": upstream_deps[build_name],
"repackage": upstream_deps[repackage_name],
"signing": upstream_deps[signing_name],
"mar-signing": upstream_deps[mar_signing_name],
}
if "partials-signing" in upstream_deps:
dependencies["partials-signing"] = upstream_deps[
"partials-signing"]
if msi_signing_name in upstream_deps:
dependencies[msi_signing_name] = upstream_deps[msi_signing_name]
if msix_signing_name in upstream_deps:
dependencies[msix_signing_name] = upstream_deps[msix_signing_name]
if repackage_signing_name in upstream_deps:
dependencies["repackage-signing"] = upstream_deps[
repackage_signing_name]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if job.get("locale"):
attributes["locale"] = job["locale"]
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": job["shipping-phase"],
"shipping-product": job.get("shipping-product"),
}
yield task
def split_public_and_private(config, jobs):
# we need to separate private vs public destinations because beetmover supports one
# in a single task. Only use a single task for each type though.
partner_config = get_partner_config_by_kind(config, config.kind)
for job in jobs:
upstream_artifacts = job["primary-dependency"].release_artifacts
attribution_task_ref = "<{}>".format(job["primary-dependency"].label)
prefix = get_artifact_prefix(job["primary-dependency"])
artifacts = defaultdict(list)
for artifact in upstream_artifacts:
partner, sub_partner, platform, locale, _ = artifact.replace(
prefix + "/", "").split("/", 4)
destination = "private"
this_config = [
p for p in partner_config["configs"]
if (p["campaign"] == partner and p["content"] == sub_partner)
]
if this_config[0].get("upload_to_candidates"):
destination = "public"
artifacts[destination].append(
(artifact, partner, sub_partner, platform, locale))
action_scope = add_scope_prefix(config,
"beetmover:action:push-to-partner")
public_bucket_scope = get_beetmover_bucket_scope(config)
partner_bucket_scope = add_scope_prefix(config,
job["partner-bucket-scope"])
repl_dict = {
"build_number":
config.params["build_number"],
"release_partner_build_number":
config.params["release_partner_build_number"],
"version":
config.params["version"],
"partner":
"{partner}", # we'll replace these later, per artifact
"subpartner":
"{subpartner}",
"platform":
"{platform}",
"locale":
"{locale}",
}
for destination, destination_artifacts in artifacts.items():
this_job = deepcopy(job)
if destination == "public":
this_job["scopes"] = [public_bucket_scope, action_scope]
this_job["partner_public"] = True
else:
this_job["scopes"] = [partner_bucket_scope, action_scope]
this_job["partner_public"] = False
partner_path_key = "partner-{destination}-path".format(
destination=destination)
partner_path = this_job[partner_path_key].format(**repl_dict)
this_job.setdefault(
"worker",
{})["upstream-artifacts"] = generate_upstream_artifacts(
attribution_task_ref, destination_artifacts, partner_path)
yield this_job
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault(
"symbol",
replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BMcs"),
)
dep_th_platform = (
dep_job.task.get("extra", {})
.get("treeherder", {})
.get("machine", {})
.get("platform", "")
)
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault(
"tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1)
)
treeherder.setdefault("kind", "build")
label = job["label"]
build_platform = attributes.get("build_platform")
description = (
"Beetmover submission of checksums for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=build_platform,
build_type=attributes.get("build_type"),
)
)
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if dep_job.attributes.get("locale"):
treeherder["symbol"] = "BMcs({})".format(dep_job.attributes.get("locale"))
attributes["locale"] = dep_job.attributes.get("locale")
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task