def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "langpack(BM{})".format(attributes.get("l10n_chunk",
"")))
job["attributes"].update(copy_attributes_from_dependent_job(dep_job))
job["attributes"]["chunk_locales"] = dep_job.attributes.get(
"chunk_locales", ["en-US"])
job["description"] = job["description"].format(
locales="/".join(job["attributes"]["chunk_locales"]),
platform=job["attributes"]["build_platform"],
)
job["scopes"] = [
get_beetmover_bucket_scope(config),
get_beetmover_action_scope(config),
]
job["dependencies"] = {"langpack-copy": dep_job.label}
job["run-on-projects"] = job.get("run_on_projects",
dep_job.attributes["run_on_projects"])
job["treeherder"] = treeherder
job["shipping-phase"] = job.get("shipping-phase",
dep_job.attributes["shipping_phase"])
job["shipping-product"] = dep_job.attributes["shipping_product"]
yield job
def define_upstream_artifacts(config, jobs):
partner_configs = get_partner_config_by_kind(config, config.kind)
if not partner_configs:
return
for job in jobs:
dep_job = job["primary-dependency"]
job["depname"] = dep_job.label
job["attributes"] = copy_attributes_from_dependent_job(dep_job)
repack_ids = job["extra"]["repack_ids"]
artifacts_specifications = generate_specifications_of_artifacts_to_sign(
config,
job,
keep_locale_template=True,
kind=config.kind,
)
task_type = "build"
if "notarization" in job["depname"]:
task_type = "scriptworker"
job["upstream-artifacts"] = [
{
"taskId": {"task-reference": f"<{dep_job.kind}>"},
"taskType": task_type,
"paths": [
path_template.format(locale=repack_id)
for path_template in spec["artifacts"]
for repack_id in repack_ids
],
"formats": spec["formats"],
}
for spec in artifacts_specifications
]
yield job
def copy_attributes(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
job["attributes"] = copy_attributes_from_dependent_job(dep_job)
job["attributes"]["chunk_locales"] = dep_job.attributes.get(
"chunk_locales", ["en-US"]
)
yield job
def copy_in_useful_magic(config, jobs):
for job in jobs:
dep = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep)
attributes.update(job.get("attributes", {}))
# build-platform is needed on `job` for by-build-platform
job["build-platform"] = attributes.get("build_platform")
job["attributes"] = attributes
yield job
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault(
"symbol",
replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BM"))
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault(
"tier",
dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1))
treeherder.setdefault("kind", "build")
label = job["label"]
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
dependencies = {dep_job.kind: dep_job.label}
# XXX release snap-repackage has a variable number of dependencies, depending on how many
# "post-beetmover-dummy" jobs there are in the graph.
if dep_job.kind != "release-snap-repackage" and len(
dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
signing_dependencies = dep_job.dependencies
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if job.get("locale"):
attributes["locale"] = job["locale"]
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": job["shipping-phase"],
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
build_platform = attributes.get("build_platform")
if not build_platform:
raise Exception("Cannot find build platform!")
repack_id = dep_job.task.get("extra", {}).get("repack_id")
if not repack_id:
raise Exception("Cannot find repack id!")
label = dep_job.label.replace("beetmover-", "beetmover-checksums-")
description = (
"Beetmove checksums for repack_id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=repack_id,
build_platform=build_platform,
build_type=attributes.get("build_type"),
)
)
extra = {}
extra["partner_path"] = dep_job.task["payload"]["upstreamArtifacts"][0][
"locale"
]
extra["repack_id"] = repack_id
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith("beetmover"):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
task = {
"label": label,
"description": description,
"worker-type": "{}/{}".format(
dep_job.task["provisionerId"],
dep_job.task["workerType"],
),
"scopes": dep_job.task["scopes"],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "BMcss(N)")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
label = job["label"]
build_platform = attributes.get("build_platform")
description = "Beetmover submission of checksums for source file"
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith("beetmover"):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_release_generate_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "SGenChcks")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
job_template = "{}-{}".format(dep_job.label, "signing")
label = job.get("label", job_template)
description = "Signing of the overall release-related checksums"
dependencies = {dep_job.kind: dep_job.label}
upstream_artifacts = [{
"taskId": {
"task-reference": f"<{str(dep_job.kind)}>"
},
"taskType":
"build",
"paths": [
get_artifact_path(dep_job, "SHA256SUMS"),
get_artifact_path(dep_job, "SHA512SUMS"),
],
"formats": ["autograph_gpg"],
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"scopes": [
signing_cert_scope,
],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
}
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "BMcslang(N{})".format(attributes.get("l10n_chunk", ""))
)
label = job["label"]
build_platform = attributes.get("build_platform")
description = "Beetmover submission of checksums for langpack files"
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith("beetmover"):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if "chunk_locales" in dep_job.attributes:
attributes["chunk_locales"] = dep_job.attributes["chunk_locales"]
attributes.update(job.get("attributes", {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "BM-gv")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault("tier", 2)
treeherder.setdefault("kind", "build")
label = job["label"]
description = ("Beetmover submission for geckoview"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
dependencies = deepcopy(dep_job.dependencies)
dependencies[dep_job.kind] = dep_job.label
if job.get("locale"):
attributes["locale"] = job["locale"]
attributes["run_on_hg_branches"] = job["run-on-hg-branches"]
task = {
"label":
label,
"description":
description,
"worker-type":
"beetmover",
"scopes": [
job["bucket-scope"],
"project:releng:beetmover:action:push-to-maven",
],
"dependencies":
dependencies,
"attributes":
attributes,
"run-on-projects":
job["run-on-projects"],
"treeherder":
treeherder,
"shipping-phase":
job["shipping-phase"],
}
yield task
def make_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "css(N)")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
label = job["label"]
description = "Signing of release-source checksums file"
dependencies = {"beetmover": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": [
"public/target-source.checksums",
],
"formats": ["autograph_gpg"],
}]
signing_cert_scope = get_signing_cert_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"scopes": [
signing_cert_scope,
],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
}
yield task
def fill_template(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task.pop("dependent-tasks", None)
# Fill out the dynamic fields in the task description
task["label"] = dep.label + "-upload-symbols"
# Skip tasks where we don't have the full crashsymbols enabled
if not dep.attributes.get("enable-full-crashsymbols") or dep.attributes.get(
"skip-upload-crashsymbols"
):
logger.debug("Skipping upload symbols task for %s", task["label"])
continue
task["dependencies"] = {"build": dep.label}
task["worker"]["env"]["GECKO_HEAD_REPOSITORY"] = config.params[
"head_repository"
]
task["worker"]["env"]["GECKO_HEAD_REV"] = config.params["head_rev"]
task["worker"]["env"]["SYMBOL_SECRET"] = task["worker"]["env"][
"SYMBOL_SECRET"
].format(level=config.params["level"])
attributes = copy_attributes_from_dependent_job(dep)
attributes.update(task.get("attributes", {}))
task["attributes"] = attributes
treeherder = inherit_treeherder_from_dep(task, dep)
th = dep.task.get("extra")["treeherder"]
th_symbol = th.get("symbol")
th_groupsymbol = th.get("groupSymbol", "?")
# Disambiguate the treeherder symbol.
sym = "Sym" + (th_symbol[1:] if th_symbol.startswith("B") else th_symbol)
treeherder.setdefault("symbol", join_symbol(th_groupsymbol, sym))
task["treeherder"] = treeherder
# We only want to run these tasks if the build is run.
# XXX Better to run this on promote phase instead?
task["run-on-projects"] = dep.attributes.get("run_on_projects")
task["optimization"] = {"upload-symbols": None}
task["if-dependencies"] = ["build"]
# clear out the stuff that's not part of a task description
del task["primary-dependency"]
yield task
def define_upstream_artifacts(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
upstream_artifact_task = job.pop("upstream-artifact-task", dep_job)
job["attributes"] = copy_attributes_from_dependent_job(dep_job)
if dep_job.attributes.get("chunk_locales"):
# Used for l10n attribute passthrough
job["attributes"]["chunk_locales"] = dep_job.attributes.get(
"chunk_locales")
locale_specifications = generate_specifications_of_artifacts_to_sign(
config,
job,
keep_locale_template=True,
dep_kind=upstream_artifact_task.kind,
)
upstream_artifacts = []
for spec in locale_specifications:
task_type = "l10n"
if "notarization" in upstream_artifact_task.kind:
task_type = "scriptworker"
upstream_artifacts.append({
"taskId": {
"task-reference": f"<{upstream_artifact_task.kind}>"
},
"taskType":
task_type,
# Set paths based on artifacts in the specs (above) one per
# locale present in the chunk this is signing stuff for.
# Pass paths through set and sorted() so we get a list back
# and we remove any duplicates (e.g. hardcoded ja-JP-mac langpack)
"paths":
sorted({
path_template.format(locale=locale)
for locale in upstream_artifact_task.attributes.get(
"chunk_locales", [])
for path_template in spec["artifacts"]
}),
"formats":
spec["formats"],
})
job["upstream-artifacts"] = upstream_artifacts
yield job
def one_task_per_product_and_platform(config, jobs):
unique_products_and_platforms = set()
for job in jobs:
dep_task = job["primary-dependency"]
if "primary-dependency" in job:
del job["primary-dependency"]
product = dep_task.attributes.get("shipping_product")
platform = dep_task.attributes.get("build_platform")
if (product, platform) not in unique_products_and_platforms:
attr_denylist = ("l10n_chunk", "locale", "artifact_map",
"artifact_prefix")
attributes = copy_attributes_from_dependent_job(
dep_task, denylist=attr_denylist)
attributes.update(job.get("attributes", {}))
job["attributes"] = attributes
job["name"] = f"{product}-{platform}"
yield job
unique_products_and_platforms.add((product, platform))
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "BM-SGenChcks")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
job_template = f"{dep_job.label}"
label = job_template.replace("signing", "beetmover")
description = "Transfer *SUMS and *SUMMARY checksums file to S3."
# first dependency is the signing task for the *SUMS files
dependencies = {dep_job.kind: dep_job.label}
if len(dep_job.dependencies) > 1:
raise NotImplementedError(
"Can't beetmove a signing task with multiple dependencies")
# update the dependencies with the dependencies of the signing task
dependencies.update(dep_job.dependencies)
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": "promote",
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
build_platform = attributes.get("build_platform")
if not build_platform:
raise Exception("Cannot find build platform!")
label = config.kind
description = "Beetmover for partner attribution"
if job["partner_public"]:
label = f"{label}-public"
description = f"{description} public"
else:
label = f"{label}-private"
description = f"{description} private"
attributes = copy_attributes_from_dependent_job(dep_job)
task = {
"label": label,
"description": description,
"dependencies": {
dep_job.kind: dep_job.label
},
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"shipping-phase": job["shipping-phase"],
"shipping-product": job.get("shipping-product"),
"partner_public": job["partner_public"],
"worker": job["worker"],
"scopes": job["scopes"],
}
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get("priority"):
task["priority"] = job["priority"]
yield task
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault(
"symbol",
replace_group(dep_job.task["extra"]["treeherder"]["symbol"], "BMcs"),
)
dep_th_platform = (
dep_job.task.get("extra", {})
.get("treeherder", {})
.get("machine", {})
.get("platform", "")
)
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault(
"tier", dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1)
)
treeherder.setdefault("kind", "build")
label = job["label"]
build_platform = attributes.get("build_platform")
description = (
"Beetmover submission of checksums for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=build_platform,
build_type=attributes.get("build_type"),
)
)
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if dep_job.attributes.get("locale"):
treeherder["symbol"] = "BMcs({})".format(dep_job.attributes.get("locale"))
attributes["locale"] = dep_job.attributes.get("locale")
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def make_job_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
build_platform = attributes["build_platform"]
if job["build-platform"].startswith("win"):
if dep_job.kind.endswith("signing"):
continue
if job["build-platform"].startswith("macosx"):
if dep_job.kind.endswith("repack"):
continue
dependencies = {dep_job.attributes.get("kind"): dep_job.label}
dependencies.update(dep_job.dependencies)
signing_task = None
for dependency in dependencies.keys():
if build_platform.startswith("macosx") and dependency.endswith("signing"):
signing_task = dependency
elif build_platform.startswith("win") and dependency.endswith("repack"):
signing_task = dependency
attributes["repackage_type"] = "repackage"
repack_id = job["extra"]["repack_id"]
partner_config = get_partner_config_by_kind(config, config.kind)
partner, subpartner, _ = repack_id.split("/")
repack_stub_installer = partner_config[partner][subpartner].get(
"repack_stub_installer"
)
if build_platform.startswith("win32") and repack_stub_installer:
job["package-formats"].append("installer-stub")
repackage_config = []
for format in job.get("package-formats"):
command = copy.deepcopy(PACKAGE_FORMATS[format])
substs = {
"archive_format": archive_format(build_platform),
"executable_extension": executable_extension(build_platform),
}
command["inputs"] = {
name: filename.format(**substs)
for name, filename in command["inputs"].items()
}
repackage_config.append(command)
run = job.get("mozharness", {})
run.update(
{
"using": "mozharness",
"script": "mozharness/scripts/repackage.py",
"job-script": "taskcluster/scripts/builder/repackage.sh",
"actions": ["setup", "repackage"],
"extra-config": {
"repackage_config": repackage_config,
},
}
)
worker = {
"chain-of-trust": True,
"max-run-time": 7200 if build_platform.startswith("win") else 3600,
"taskcluster-proxy": True if get_artifact_prefix(dep_job) else False,
"env": {
"REPACK_ID": repack_id,
},
# Don't add generic artifact directory.
"skip-artifacts": True,
}
worker_type = "b-linux"
worker["docker-image"] = {"in-tree": "debian11-amd64-build"}
worker["artifacts"] = _generate_task_output_files(
dep_job,
worker_type_implementation(config.graph_config, worker_type),
repackage_config,
partner=repack_id,
)
description = (
"Repackaging for repack_id '{repack_id}' for build '"
"{build_platform}/{build_type}'".format(
repack_id=job["extra"]["repack_id"],
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
)
)
task = {
"label": job["label"],
"description": description,
"worker-type": worker_type,
"dependencies": dependencies,
"attributes": attributes,
"scopes": ["queue:get-artifact:releng/partner/*"],
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"routes": job.get("routes", []),
"extra": job.get("extra", {}),
"worker": worker,
"run": run,
"fetches": _generate_download_config(
dep_job,
build_platform,
signing_task,
partner=repack_id,
project=config.params["project"],
repack_stub_installer=repack_stub_installer,
),
}
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get("priority"):
task["priority"] = job["priority"]
if build_platform.startswith("macosx"):
task.setdefault("fetches", {}).setdefault("toolchain", []).extend(
[
"linux64-libdmg",
"linux64-hfsplus",
"linux64-node",
]
)
yield task
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get("release_history"):
return
for job in jobs:
dep_job = job["primary-dependency"]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault("symbol", "p(N)")
label = job.get("label", f"partials-{dep_job.label}")
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get("locale")
if locale:
attributes["locale"] = locale
treeherder["symbol"] = f"p({locale})"
attributes["shipping_phase"] = job["shipping-phase"]
build_locale = locale or "en-US"
build_platform = attributes["build_platform"]
builds = get_builds(config.params["release_history"], build_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
extra = {"funsize": {"partials": list()}}
update_number = 1
locale_suffix = ""
if locale:
locale_suffix = f"{locale}/"
artifact_path = "<{}/{}/{}target.complete.mar>".format(
dep_job.kind,
get_artifact_prefix(dep_job),
locale_suffix,
)
for build in sorted(builds):
partial_info = {
"locale": build_locale,
"from_mar": builds[build]["mar_url"],
"to_mar": {
"artifact-reference": artifact_path
},
"branch": config.params["project"],
"update_number": update_number,
"dest_mar": build,
}
if "product" in builds[build]:
partial_info["product"] = builds[build]["product"]
if "previousVersion" in builds[build]:
partial_info["previousVersion"] = builds[build][
"previousVersion"]
if "previousBuildNumber" in builds[build]:
partial_info["previousBuildNumber"] = builds[build][
"previousBuildNumber"]
extra["funsize"]["partials"].append(partial_info)
update_number += 1
level = config.params["level"]
worker = {
"artifacts":
_generate_task_output_files(dep_job, builds.keys(), locale),
"implementation":
"docker-worker",
"docker-image": {
"in-tree": "funsize-update-generator"
},
"os":
"linux",
"max-run-time":
3600 if "asan" in dep_job.label else 900,
"chain-of-trust":
True,
"taskcluster-proxy":
True,
"env": {
"SIGNING_CERT":
identify_desired_signing_keys(
config.params["project"],
config.params["release_product"]),
"EXTRA_PARAMS":
f"--arch={architecture(build_platform)}",
"MAR_CHANNEL_ID":
attributes["mar-channel-id"],
},
}
if config.params.release_level() == "staging":
worker["env"]["FUNSIZE_ALLOW_STAGING_PREFIXES"] = "true"
task = {
"label": label,
"description": f"{dep_job.description} Partials",
"worker-type": "b-linux",
"dependencies": dependencies,
"scopes": [],
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
"worker": worker,
}
# We only want caching on linux/windows due to bug 1436977
if int(level) == 3 and any(
[build_platform.startswith(prefix)
for prefix in ["linux", "win"]]):
task["scopes"].append(
"auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/"
)
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
locale = attributes.get("locale", dep_job.attributes.get("locale"))
attributes["repackage_type"] = "repackage-signing"
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "rs(B)")
dep_th_platform = dep_job.task.get("extra",
{}).get("treeherder-platform")
treeherder.setdefault("platform", dep_th_platform)
treeherder.setdefault(
"tier",
dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1))
treeherder.setdefault("kind", "build")
if locale:
treeherder["symbol"] = f"rs({locale})"
if config.kind == "repackage-signing-msi":
treeherder["symbol"] = "MSIs({})".format(locale or "N")
elif config.kind in (
"repackage-signing-msix",
"repackage-signing-shippable-l10n-msix",
):
# Like "MSIXs(Bs-multi)".
treeherder["symbol"] = "MSIXs({})".format(
dep_job.task.get("extra", {}).get("treeherder",
{}).get("symbol", "B"))
label = job["label"]
dep_kind = dep_job.kind
if "l10n" in dep_kind:
dep_kind = "repackage"
dependencies = {dep_kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to have better beetmover
# support. But for multi-locale MSIX packages, we don't want the signing task to directly
# depend on the langpack tasks.
dependencies.update({
k: v
for k, v in signing_dependencies.items() if k != "docker-image"
and not k.startswith("shippable-l10n-signing-linux64")
})
description = (
"Signing of repackaged artifacts for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
build_platform = dep_job.attributes.get("build_platform")
is_shippable = dep_job.attributes.get("shippable")
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config)
scopes = [signing_cert_scope]
upstream_artifacts = []
for artifact in sorted(dep_job.release_artifacts):
basename = os.path.basename(artifact)
if basename in SIGNING_FORMATS:
upstream_artifacts.append({
"taskId": {
"task-reference": f"<{dep_kind}>"
},
"taskType":
"repackage",
"paths": [artifact],
"formats":
SIGNING_FORMATS[os.path.basename(artifact)],
})
task = {
"label": label,
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"scopes": scopes,
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"optimization": dep_job.optimization,
"treeherder": treeherder,
}
yield task
def make_job_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["repackage_type"] = "repackage"
locale = attributes.get("locale", job.get("locale"))
if locale:
attributes["locale"] = locale
description = (
"Repackaging for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
)
)
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "Rpk")
dep_th_platform = dep_job.task.get("extra", {}).get("treeherder-platform")
treeherder.setdefault("platform", dep_th_platform)
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
# Search dependencies before adding langpack dependencies.
signing_task = None
repackage_signing_task = None
for dependency in dependencies.keys():
if "repackage-signing" in dependency:
repackage_signing_task = dependency
elif "signing" in dependency:
signing_task = dependency
if config.kind == "repackage-msi":
treeherder["symbol"] = "MSI({})".format(locale or "N")
elif config.kind == "repackage-msix":
assert not locale
# Like "MSIXs(Bs)".
treeherder["symbol"] = "MSIX({})".format(
dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol", "B")
)
elif config.kind == "repackage-shippable-l10n-msix":
assert not locale
if attributes.get("l10n_chunk") or attributes.get("chunk_locales"):
# We don't want to produce MSIXes for single-locale repack builds.
continue
description = (
"Repackaging with multiple locales for build '"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
)
)
# Like "MSIXs(Bs-multi)".
treeherder["symbol"] = "MSIX({}-multi)".format(
dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol", "B")
)
fetches = job.setdefault("fetches", {})
# The keys are unique, like `shippable-l10n-signing-linux64-shippable-1/opt`, so we
# can't ask for the tasks directly, we must filter for them.
for t in config.kind_dependencies_tasks.values():
if t.kind != "shippable-l10n-signing":
continue
if t.attributes["build_platform"] != "linux64-shippable":
continue
if t.attributes["build_type"] != "opt":
continue
dependencies.update({t.label: t.label})
fetches.update(
{
t.label: [
{
"artifact": f"{loc}/target.langpack.xpi",
"extract": False,
# Otherwise we can't disambiguate locales!
"dest": f"distribution/extensions/{loc}",
}
for loc in t.attributes["chunk_locales"]
]
}
)
_fetch_subst_locale = "en-US"
if locale:
_fetch_subst_locale = locale
worker_type = job["worker-type"]
build_platform = attributes["build_platform"]
use_stub = attributes.get("stub-installer")
repackage_config = []
package_formats = job.get("package-formats")
if use_stub and not repackage_signing_task and "msix" not in package_formats:
# if repackage_signing_task doesn't exists, generate the stub installer
package_formats += ["installer-stub"]
for format in package_formats:
command = copy.deepcopy(PACKAGE_FORMATS[format])
substs = {
"archive_format": archive_format(build_platform),
"_locale": _fetch_subst_locale,
"architecture": architecture(build_platform),
"version_display": config.params["version"],
"mar-channel-id": attributes["mar-channel-id"],
}
# Allow us to replace `args` as well, but specifying things expanded in mozharness
# without breaking .format and without allowing unknown through.
substs.update({name: f"{{{name}}}" for name in MOZHARNESS_EXPANSIONS})
# We need to resolve `msix.*` values keyed by `package-format` for each format, not
# just once, so we update a temporary copy just for extracting these values.
temp_job = copy.deepcopy(job)
for msix_key in (
"channel",
"identity-name",
"publisher",
"publisher-display-name",
):
resolve_keyed_by(
item=temp_job,
field=f"msix.{msix_key}",
item_name="?",
**{
"package-format": format,
"release-type": config.params["release_type"],
"level": config.params["level"],
},
)
# Turn `msix.channel` into `msix-channel`, etc.
value = temp_job.get("msix", {}).get(msix_key)
if value:
substs.update(
{f"msix-{msix_key}": value},
)
command["inputs"] = {
name: filename.format(**substs)
for name, filename in command["inputs"].items()
}
command["args"] = [arg.format(**substs) for arg in command["args"]]
if "installer" in format and "aarch64" not in build_platform:
command["args"].append("--use-upx")
repackage_config.append(command)
run = job.get("mozharness", {})
run.update(
{
"using": "mozharness",
"script": "mozharness/scripts/repackage.py",
"job-script": "taskcluster/scripts/builder/repackage.sh",
"actions": ["setup", "repackage"],
"extra-config": {
"repackage_config": repackage_config,
},
}
)
worker = job.get("worker", {})
worker.update(
{
"chain-of-trust": True,
"max-run-time": 7200 if build_platform.startswith("win") else 3600,
# Don't add generic artifact directory.
"skip-artifacts": True,
}
)
if locale:
# Make sure we specify the locale-specific upload dir
worker.setdefault("env", {})["LOCALE"] = locale
worker["artifacts"] = _generate_task_output_files(
dep_job,
worker_type_implementation(config.graph_config, worker_type),
repackage_config=repackage_config,
locale=locale,
)
task = {
"label": job["label"],
"description": description,
"worker-type": worker_type,
"dependencies": dependencies,
"if-dependencies": [dep_job.kind],
"attributes": attributes,
"run-on-projects": job.get(
"run-on-projects", dep_job.attributes.get("run_on_projects")
),
"optimization": dep_job.optimization,
"treeherder": treeherder,
"routes": job.get("routes", []),
"extra": job.get("extra", {}),
"worker": worker,
"run": run,
"fetches": _generate_download_config(
dep_job,
build_platform,
signing_task,
repackage_signing_task,
locale=locale,
project=config.params["project"],
existing_fetch=job.get("fetches"),
),
"release-artifacts": [artifact["name"] for artifact in worker["artifacts"]],
}
if build_platform.startswith("macosx"):
task.setdefault("fetches", {}).setdefault("toolchain", []).extend(
[
"linux64-libdmg",
"linux64-hfsplus",
"linux64-node",
]
)
yield task
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["repackage_type"] = "repackage-signing"
treeherder = job.get("treeherder", {})
dep_treeherder = dep_job.task.get("extra", {}).get("treeherder", {})
treeherder.setdefault(
"symbol", "{}(gd-s)".format(dep_treeherder["groupSymbol"])
)
treeherder.setdefault(
"platform", dep_job.task.get("extra", {}).get("treeherder-platform")
)
treeherder.setdefault("tier", dep_treeherder.get("tier", 1))
treeherder.setdefault("kind", "build")
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
dependencies.update(
{k: v for k, v in signing_dependencies.items() if k != "docker-image"}
)
description = "Signing Geckodriver for build '{}'".format(
attributes.get("build_platform"),
)
build_platform = dep_job.attributes.get("build_platform")
is_shippable = dep_job.attributes.get("shippable")
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config
)
upstream_artifacts = _craft_upstream_artifacts(
dep_job, dep_job.kind, build_platform
)
scopes = [signing_cert_scope]
platform = build_platform.rsplit("-", 1)[0]
task = {
"label": job["label"],
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
},
"scopes": scopes,
"dependencies": dependencies,
"attributes": attributes,
"treeherder": treeherder,
"run-on-projects": ["mozilla-central"],
"index": {"product": "geckodriver", "job-name": platform},
}
if build_platform.startswith("macosx"):
worker_type = task["worker-type"]
worker_type_alias_map = {
"linux-depsigning": "mac-depsigning",
"linux-signing": "mac-signing",
}
assert worker_type in worker_type_alias_map, (
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
" ({} not found in mapping)".format(worker_type)
)
worker_type = worker_type_alias_map[worker_type]
task["worker-type"] = worker_type_alias_map[task["worker-type"]]
task["worker"]["mac-behavior"] = "mac_geckodriver"
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
locale = dep_job.attributes.get("locale")
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", join_symbol(job.get("treeherder-group", "ms"), locale or "N")
)
label = job.get("label", f"{config.kind}-{dep_job.label}")
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["required_signoffs"] = sorted_unique_list(
attributes.get("required_signoffs", []), job.pop("required_signoffs")
)
attributes["shipping_phase"] = job["shipping-phase"]
if locale:
attributes["locale"] = locale
build_platform = attributes.get("build_platform")
if config.kind == "partials-signing":
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params["release_history"], build_platform, locale
)
else:
upstream_artifacts = generate_complete_artifacts(dep_job, config.kind)
is_shippable = job.get(
"shippable", dep_job.attributes.get("shippable") # First check current job
) # Then dep job for 'shippable'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config
)
scopes = [signing_cert_scope]
task = {
"label": label,
"description": "{} {}".format(
dep_job.description, job["description-suffix"]
),
"worker-type": job.get("worker-type", "linux-signing"),
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"dependencies": dependencies,
"attributes": attributes,
"scopes": scopes,
"run-on-projects": job.get(
"run-on-projects", dep_job.attributes.get("run_on_projects")
),
"treeherder": treeherder,
}
yield task
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
repack_id = dep_job.task["extra"]["repack_id"]
attributes = dep_job.attributes
build_platform = dep_job.attributes.get("build_platform")
is_shippable = dep_job.attributes.get("shippable")
# Mac & windows
label = dep_job.label.replace("repackage-", "repackage-signing-")
# Linux
label = label.replace("chunking-dummy-", "repackage-signing-")
description = "Signing of repackaged artifacts for partner repack id '{repack_id}' for build '" "{build_platform}/{build_type}'".format( # NOQA: E501
repack_id=repack_id,
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
)
if "linux" in build_platform:
# we want the repack job, via the dependencies for the the chunking-dummy dep_job
for dep in dep_job.dependencies.values():
if dep.startswith("release-partner-repack"):
dependencies = {"repack": dep}
break
else:
# we have a genuine repackage job as our parent
dependencies = {"repackage": dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["repackage_type"] = "repackage-signing"
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config)
scopes = [signing_cert_scope]
if "win" in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job,
f"{repack_id}/target.installer.exe"),
],
"formats": ["autograph_authenticode", "autograph_gpg"],
}]
partner_config = get_partner_config_by_kind(config, config.kind)
partner, subpartner, _ = repack_id.split("/")
repack_stub_installer = partner_config[partner][subpartner].get(
"repack_stub_installer")
if build_platform.startswith("win32") and repack_stub_installer:
upstream_artifacts.append({
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(
dep_job,
f"{repack_id}/target.stub-installer.exe",
),
],
"formats": ["autograph_authenticode", "autograph_gpg"],
})
elif "mac" in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repackage>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job, f"{repack_id}/target.dmg"),
],
"formats": ["autograph_gpg"],
}]
elif "linux" in build_platform:
upstream_artifacts = [{
"taskId": {
"task-reference": "<repack>"
},
"taskType":
"repackage",
"paths": [
get_artifact_path(dep_job, f"{repack_id}/target.tar.bz2"),
],
"formats": ["autograph_gpg"],
}]
task = {
"label": label,
"description": description,
"worker-type": "linux-signing",
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"scopes": scopes,
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"extra": {
"repack_id": repack_id,
},
}
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get("priority"):
task["priority"] = job["priority"]
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
signing_format_scopes = []
formats = set()
for artifacts in job["upstream-artifacts"]:
for f in artifacts["formats"]:
formats.add(f) # Add each format only once
is_shippable = dep_job.attributes.get("shippable", False)
build_platform = dep_job.attributes.get("build_platform")
treeherder = None
if "partner" not in config.kind and "eme-free" not in config.kind:
treeherder = job.get("treeherder", {})
dep_th_platform = (
dep_job.task.get("extra", {})
.get("treeherder", {})
.get("machine", {})
.get("platform", "")
)
build_type = dep_job.attributes.get("build_type")
treeherder.setdefault(
"platform",
_generate_treeherder_platform(
dep_th_platform, build_platform, build_type
),
)
# ccov builds are tier 2, so they cannot have tier 1 tasks
# depending on them.
treeherder.setdefault(
"tier",
dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1),
)
treeherder.setdefault(
"symbol",
_generate_treeherder_symbol(
dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol")
),
)
treeherder.setdefault("kind", "build")
label = job["label"]
description = (
"Initial Signing for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=build_platform,
build_type=attributes.get("build_type"),
)
)
attributes = (
job["attributes"]
if job.get("attributes")
else copy_attributes_from_dependent_job(dep_job)
)
attributes["signed"] = True
if dep_job.attributes.get("chunk_locales"):
# Used for l10n attribute passthrough
attributes["chunk_locales"] = dep_job.attributes.get("chunk_locales")
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config
)
worker_type_alias = "linux-signing" if is_shippable else "linux-depsigning"
mac_behavior = None
task = {
"label": label,
"description": description,
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": job["upstream-artifacts"],
"max-run-time": job.get("max-run-time", 3600),
},
"scopes": [signing_cert_scope] + signing_format_scopes,
"dependencies": _generate_dependencies(job),
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"optimization": dep_job.optimization,
"routes": job.get("routes", []),
"shipping-product": job.get("shipping-product"),
"shipping-phase": job.get("shipping-phase"),
}
if dep_job.kind in task["dependencies"]:
task["if-dependencies"] = [dep_job.kind]
if "macosx" in build_platform:
shippable = "false"
if "shippable" in attributes and attributes["shippable"]:
shippable = "true"
mac_behavior = evaluate_keyed_by(
config.graph_config["mac-notarization"]["mac-behavior"],
"mac behavior",
{
"project": config.params["project"],
"shippable": shippable,
},
)
if mac_behavior == "mac_notarize":
if "part-1" in config.kind:
mac_behavior = "mac_notarize_part_1"
elif config.kind.endswith("signing"):
mac_behavior = "mac_notarize_part_3"
else:
raise Exception(f"Unknown kind {config.kind} for mac_behavior!")
else:
if "part-1" in config.kind:
continue
task["worker"]["mac-behavior"] = mac_behavior
worker_type_alias_map = {
"linux-depsigning": "mac-depsigning",
"linux-signing": "mac-signing",
}
assert worker_type_alias in worker_type_alias_map, (
"Make sure to adjust the below worker_type_alias logic for "
"mac if you change the signing workerType aliases!"
" ({} not found in mapping)".format(worker_type_alias)
)
worker_type_alias = worker_type_alias_map[worker_type_alias]
if job.get("entitlements-url"):
task["worker"]["entitlements-url"] = job["entitlements-url"]
task["worker-type"] = worker_type_alias
if treeherder:
task["treeherder"] = treeherder
if job.get("extra"):
task["extra"] = job["extra"]
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get("priority"):
task["priority"] = job["priority"]
yield task
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
build_platform = dep_job.attributes.get("build_platform")
is_nightly = True # cert_scope_per_platform uses this to choose the right cert
description = (
"Signing of OpenH264 Binaries for '"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
)
)
# we have a genuine repackage job as our parent
dependencies = {"openh264": dep_job.label}
my_attributes = copy_attributes_from_dependent_job(dep_job)
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
scopes = [signing_cert_scope]
worker_type = "linux-signing"
worker = {
"implementation": "scriptworker-signing",
"max-run-time": 3600,
}
rev = attributes["openh264_rev"]
upstream_artifact = {
"taskId": {"task-reference": "<openh264>"},
"taskType": "build",
}
if "win" in build_platform:
# job['primary-dependency'].task['payload']['command']
upstream_artifact["formats"] = ["autograph_authenticode"]
elif "mac" in build_platform:
upstream_artifact["formats"] = ["mac_single_file"]
upstream_artifact["singleFileGlobs"] = ["libgmpopenh264.dylib"]
worker_type = "mac-signing"
worker["mac-behavior"] = "mac_single_file"
else:
upstream_artifact["formats"] = ["autograph_gpg"]
upstream_artifact["paths"] = [
f"private/openh264/openh264-{build_platform}-{rev}.zip",
]
worker["upstream-artifacts"] = [upstream_artifact]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol",
_generate_treeherder_symbol(
dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol")
),
)
task = {
"label": job["label"],
"description": description,
"worker-type": worker_type,
"worker": worker,
"scopes": scopes,
"dependencies": dependencies,
"attributes": my_attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
build_platform = dep_job.attributes.get("build_platform")
treeherder = None
if "partner" not in config.kind and "eme-free" not in config.kind:
treeherder = job.get("treeherder", {})
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
build_type = dep_job.attributes.get("build_type")
treeherder.setdefault("platform",
f"{dep_th_platform}/{build_type}")
dep_treeherder = dep_job.task.get("extra",
{}).get("treeherder", {})
treeherder.setdefault("tier", dep_treeherder.get("tier", 1))
treeherder.setdefault(
"symbol",
_generate_treeherder_symbol(
dep_treeherder.get("groupSymbol", "?"),
dep_treeherder.get("symbol")),
)
treeherder.setdefault("kind", "build")
label = dep_job.label.replace("part-1", "poller")
description = ("Mac Notarization Poller for build '"
"{build_platform}/{build_type}'".format(
build_platform=build_platform,
build_type=attributes.get("build_type")))
attributes = (job["attributes"] if job.get("attributes") else
copy_attributes_from_dependent_job(dep_job))
attributes["signed"] = True
if dep_job.attributes.get("chunk_locales"):
# Used for l10n attribute passthrough
attributes["chunk_locales"] = dep_job.attributes.get(
"chunk_locales")
uuid_manifest_url = get_artifact_url("<part1>",
"public/uuid_manifest.json")
task = {
"label": label,
"description": description,
"worker": {
"implementation": "notarization-poller",
"uuid-manifest": {
"task-reference": uuid_manifest_url
},
"max-run-time": 3600,
},
"worker-type": "mac-notarization-poller",
"dependencies": {
"part1": dep_job.label
},
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"optimization": dep_job.optimization,
"routes": job.get("routes", []),
"shipping-product": job.get("shipping-product"),
"shipping-phase": job.get("shipping-phase"),
}
if treeherder:
task["treeherder"] = treeherder
if job.get("extra"):
task["extra"] = job["extra"]
# we may have reduced the priority for partner jobs, otherwise task.py will set it
if job.get("priority"):
task["priority"] = job["priority"]
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "c-Up(N)")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault(
"tier",
dep_job.task.get("extra", {}).get("treeherder", {}).get("tier", 1))
treeherder.setdefault("kind", "build")
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder_job_symbol = dep_job.task["extra"]["treeherder"]["symbol"]
treeherder["symbol"] = replace_group(treeherder_job_symbol, "c-Up")
if dep_job.attributes.get("locale"):
attributes["locale"] = dep_job.attributes.get("locale")
label = job["label"]
description = ("Balrog submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
upstream_artifacts = [{
"taskId": {
"task-reference": "<beetmover>"
},
"taskType": "beetmover",
"paths": ["public/manifest.json"],
}]
dependencies = {"beetmover": dep_job.label}
for kind_dep in config.kind_dependencies_tasks.values():
if (kind_dep.kind == "startup-test"
and kind_dep.attributes["build_platform"]
== attributes.get("build_platform")
and kind_dep.attributes["build_type"]
== attributes.get("build_type")
and kind_dep.attributes.get("shipping_product")
== job.get("shipping-product")):
dependencies["startup-test"] = kind_dep.label
task = {
"label": label,
"description": description,
"worker-type": "balrog",
"worker": {
"implementation": "balrog",
"upstream-artifacts": upstream_artifacts,
"balrog-action": "v2-submit-locale",
"suffixes":
["", "-No-WNP"] if job.get("update-no-wnp") else [""],
},
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": job.get("shipping-phase", "promote"),
"shipping-product": job.get("shipping-product"),
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
upstream_symbol = dep_job.task["extra"]["treeherder"]["symbol"]
if "build" in job["dependent-tasks"]:
upstream_symbol = job["dependent-tasks"]["build"].task["extra"][
"treeherder"]["symbol"]
treeherder.setdefault("symbol", replace_group(upstream_symbol, "BMR"))
label = job["label"]
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
upstream_deps = job["dependent-tasks"]
signing_name = "build-signing"
build_name = "build"
repackage_name = "repackage"
repackage_signing_name = "repackage-signing"
msi_signing_name = "repackage-signing-msi"
msix_signing_name = "repackage-signing-shippable-l10n-msix"
mar_signing_name = "mar-signing"
if job.get("locale"):
signing_name = "shippable-l10n-signing"
build_name = "shippable-l10n"
repackage_name = "repackage-l10n"
repackage_signing_name = "repackage-signing-l10n"
mar_signing_name = "mar-signing-l10n"
dependencies = {
"build": upstream_deps[build_name],
"repackage": upstream_deps[repackage_name],
"signing": upstream_deps[signing_name],
"mar-signing": upstream_deps[mar_signing_name],
}
if "partials-signing" in upstream_deps:
dependencies["partials-signing"] = upstream_deps[
"partials-signing"]
if msi_signing_name in upstream_deps:
dependencies[msi_signing_name] = upstream_deps[msi_signing_name]
if msix_signing_name in upstream_deps:
dependencies[msix_signing_name] = upstream_deps[msix_signing_name]
if repackage_signing_name in upstream_deps:
dependencies["repackage-signing"] = upstream_deps[
repackage_signing_name]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if job.get("locale"):
attributes["locale"] = job["locale"]
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": job["shipping-phase"],
"shipping-product": job.get("shipping-product"),
}
yield task
