def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "langpack(BM{})".format(attributes.get("l10n_chunk",
"")))
job["attributes"].update(copy_attributes_from_dependent_job(dep_job))
job["attributes"]["chunk_locales"] = dep_job.attributes.get(
"chunk_locales", ["en-US"])
job["description"] = job["description"].format(
locales="/".join(job["attributes"]["chunk_locales"]),
platform=job["attributes"]["build_platform"],
)
job["scopes"] = [
get_beetmover_bucket_scope(config),
get_beetmover_action_scope(config),
]
job["dependencies"] = {"langpack-copy": dep_job.label}
job["run-on-projects"] = job.get("run_on_projects",
dep_job.attributes["run_on_projects"])
job["treeherder"] = treeherder
job["shipping-phase"] = job.get("shipping-phase",
dep_job.attributes["shipping_phase"])
job["shipping-product"] = dep_job.attributes["shipping_product"]
yield job
def make_beetmover_checksums_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "BMcslang(N{})".format(attributes.get("l10n_chunk", ""))
)
label = job["label"]
build_platform = attributes.get("build_platform")
description = "Beetmover submission of checksums for langpack files"
extra = {}
if "devedition" in build_platform:
extra["product"] = "devedition"
else:
extra["product"] = "firefox"
dependencies = {dep_job.kind: dep_job.label}
for k, v in dep_job.dependencies.items():
if k.startswith("beetmover"):
dependencies[k] = v
attributes = copy_attributes_from_dependent_job(dep_job)
if "chunk_locales" in dep_job.attributes:
attributes["chunk_locales"] = dep_job.attributes["chunk_locales"]
attributes.update(job.get("attributes", {}))
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
}
if "shipping-phase" in job:
task["shipping-phase"] = job["shipping-phase"]
if "shipping-product" in job:
task["shipping-product"] = job["shipping-product"]
yield task
def fill_template(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task.pop("dependent-tasks", None)
# Fill out the dynamic fields in the task description
task["label"] = dep.label + "-upload-symbols"
# Skip tasks where we don't have the full crashsymbols enabled
if not dep.attributes.get("enable-full-crashsymbols") or dep.attributes.get(
"skip-upload-crashsymbols"
):
logger.debug("Skipping upload symbols task for %s", task["label"])
continue
task["dependencies"] = {"build": dep.label}
task["worker"]["env"]["GECKO_HEAD_REPOSITORY"] = config.params[
"head_repository"
]
task["worker"]["env"]["GECKO_HEAD_REV"] = config.params["head_rev"]
task["worker"]["env"]["SYMBOL_SECRET"] = task["worker"]["env"][
"SYMBOL_SECRET"
].format(level=config.params["level"])
attributes = copy_attributes_from_dependent_job(dep)
attributes.update(task.get("attributes", {}))
task["attributes"] = attributes
treeherder = inherit_treeherder_from_dep(task, dep)
th = dep.task.get("extra")["treeherder"]
th_symbol = th.get("symbol")
th_groupsymbol = th.get("groupSymbol", "?")
# Disambiguate the treeherder symbol.
sym = "Sym" + (th_symbol[1:] if th_symbol.startswith("B") else th_symbol)
treeherder.setdefault("symbol", join_symbol(th_groupsymbol, sym))
task["treeherder"] = treeherder
# We only want to run these tasks if the build is run.
# XXX Better to run this on promote phase instead?
task["run-on-projects"] = dep.attributes.get("run_on_projects")
task["optimization"] = {"upload-symbols": None}
task["if-dependencies"] = ["build"]
# clear out the stuff that's not part of a task description
del task["primary-dependency"]
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", "langpack(SnP{})".format(job["attributes"].get("l10n_chunk", ""))
)
job["description"] = job["description"].format(
locales="/".join(job["attributes"]["chunk_locales"]),
)
job["dependencies"] = {dep_job.kind: dep_job.label}
job["treeherder"] = treeherder
yield job
def add_command(config, tasks):
config_tasks = {}
for dep in config.kind_dependencies_tasks.values():
if (
"update-verify-config" in dep.kind
or "update-verify-next-config" in dep.kind
):
config_tasks[dep.name] = dep
for task in tasks:
config_task = config_tasks[task["name"]]
total_chunks = task["extra"]["chunks"]
task["worker"].setdefault("env", {})["CHANNEL"] = config_task.task["extra"][
"channel"
]
task.setdefault("fetches", {})[config_task.label] = [
"update-verify.cfg",
]
task["treeherder"] = inherit_treeherder_from_dep(task, config_task)
for this_chunk in range(1, total_chunks + 1):
chunked = deepcopy(task)
chunked["treeherder"]["symbol"] = add_suffix(
chunked["treeherder"]["symbol"], this_chunk
)
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks
)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["run"] = {
"using": "run-task",
"cwd": "{checkout}",
"command": "tools/update-verify/scripts/chunked-verify.sh "
f"--total-chunks={total_chunks} --this-chunk={this_chunk}",
"sparse-profile": "update-verify",
}
yield chunked
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
build_platform = dep_job.attributes.get("build_platform")
is_nightly = True # cert_scope_per_platform uses this to choose the right cert
description = (
"Signing of OpenH264 Binaries for '"
"{build_platform}/{build_type}'".format(
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
)
)
# we have a genuine repackage job as our parent
dependencies = {"openh264": dep_job.label}
my_attributes = copy_attributes_from_dependent_job(dep_job)
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_nightly, config
)
scopes = [signing_cert_scope]
worker_type = "linux-signing"
worker = {
"implementation": "scriptworker-signing",
"max-run-time": 3600,
}
rev = attributes["openh264_rev"]
upstream_artifact = {
"taskId": {"task-reference": "<openh264>"},
"taskType": "build",
}
if "win" in build_platform:
# job['primary-dependency'].task['payload']['command']
upstream_artifact["formats"] = ["autograph_authenticode"]
elif "mac" in build_platform:
upstream_artifact["formats"] = ["mac_single_file"]
upstream_artifact["singleFileGlobs"] = ["libgmpopenh264.dylib"]
worker_type = "mac-signing"
worker["mac-behavior"] = "mac_single_file"
else:
upstream_artifact["formats"] = ["autograph_gpg"]
upstream_artifact["paths"] = [
f"private/openh264/openh264-{build_platform}-{rev}.zip",
]
worker["upstream-artifacts"] = [upstream_artifact]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol",
_generate_treeherder_symbol(
dep_job.task.get("extra", {}).get("treeherder", {}).get("symbol")
),
)
task = {
"label": job["label"],
"description": description,
"worker-type": worker_type,
"worker": worker,
"scopes": scopes,
"dependencies": dependencies,
"attributes": my_attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
locale = dep_job.attributes.get("locale")
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", join_symbol(job.get("treeherder-group", "ms"), locale or "N")
)
label = job.get("label", f"{config.kind}-{dep_job.label}")
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["required_signoffs"] = sorted_unique_list(
attributes.get("required_signoffs", []), job.pop("required_signoffs")
)
attributes["shipping_phase"] = job["shipping-phase"]
if locale:
attributes["locale"] = locale
build_platform = attributes.get("build_platform")
if config.kind == "partials-signing":
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params["release_history"], build_platform, locale
)
else:
upstream_artifacts = generate_complete_artifacts(dep_job, config.kind)
is_shippable = job.get(
"shippable", dep_job.attributes.get("shippable") # First check current job
) # Then dep job for 'shippable'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config
)
scopes = [signing_cert_scope]
task = {
"label": label,
"description": "{} {}".format(
dep_job.description, job["description-suffix"]
),
"worker-type": job.get("worker-type", "linux-signing"),
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"dependencies": dependencies,
"attributes": attributes,
"scopes": scopes,
"run-on-projects": job.get(
"run-on-projects", dep_job.attributes.get("run_on_projects")
),
"treeherder": treeherder,
}
yield task
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = inherit_treeherder_from_dep(job, dep_job)
upstream_symbol = dep_job.task["extra"]["treeherder"]["symbol"]
if "build" in job["dependent-tasks"]:
upstream_symbol = job["dependent-tasks"]["build"].task["extra"][
"treeherder"]["symbol"]
treeherder.setdefault("symbol", replace_group(upstream_symbol, "BMR"))
label = job["label"]
description = ("Beetmover submission for locale '{locale}' for build '"
"{build_platform}/{build_type}'".format(
locale=attributes.get("locale", "en-US"),
build_platform=attributes.get("build_platform"),
build_type=attributes.get("build_type"),
))
upstream_deps = job["dependent-tasks"]
signing_name = "build-signing"
build_name = "build"
repackage_name = "repackage"
repackage_signing_name = "repackage-signing"
msi_signing_name = "repackage-signing-msi"
msix_signing_name = "repackage-signing-shippable-l10n-msix"
mar_signing_name = "mar-signing"
if job.get("locale"):
signing_name = "shippable-l10n-signing"
build_name = "shippable-l10n"
repackage_name = "repackage-l10n"
repackage_signing_name = "repackage-signing-l10n"
mar_signing_name = "mar-signing-l10n"
dependencies = {
"build": upstream_deps[build_name],
"repackage": upstream_deps[repackage_name],
"signing": upstream_deps[signing_name],
"mar-signing": upstream_deps[mar_signing_name],
}
if "partials-signing" in upstream_deps:
dependencies["partials-signing"] = upstream_deps[
"partials-signing"]
if msi_signing_name in upstream_deps:
dependencies[msi_signing_name] = upstream_deps[msi_signing_name]
if msix_signing_name in upstream_deps:
dependencies[msix_signing_name] = upstream_deps[msix_signing_name]
if repackage_signing_name in upstream_deps:
dependencies["repackage-signing"] = upstream_deps[
repackage_signing_name]
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get("attributes", {}))
if job.get("locale"):
attributes["locale"] = job["locale"]
bucket_scope = get_beetmover_bucket_scope(config)
action_scope = get_beetmover_action_scope(config)
task = {
"label": label,
"description": description,
"worker-type": "beetmover",
"scopes": [bucket_scope, action_scope],
"dependencies": dependencies,
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"shipping-phase": job["shipping-phase"],
"shipping-product": job.get("shipping-product"),
}
yield task
def make_task_description(config, jobs):
# If no balrog release history, then don't generate partials
if not config.params.get("release_history"):
return
for job in jobs:
dep_job = job["primary-dependency"]
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault("symbol", "p(N)")
label = job.get("label", f"partials-{dep_job.label}")
dependencies = {dep_job.kind: dep_job.label}
attributes = copy_attributes_from_dependent_job(dep_job)
locale = dep_job.attributes.get("locale")
if locale:
attributes["locale"] = locale
treeherder["symbol"] = f"p({locale})"
attributes["shipping_phase"] = job["shipping-phase"]
build_locale = locale or "en-US"
build_platform = attributes["build_platform"]
builds = get_builds(config.params["release_history"], build_platform,
build_locale)
# If the list is empty there's no available history for this platform
# and locale combination, so we can't build any partials.
if not builds:
continue
extra = {"funsize": {"partials": list()}}
update_number = 1
locale_suffix = ""
if locale:
locale_suffix = f"{locale}/"
artifact_path = "<{}/{}/{}target.complete.mar>".format(
dep_job.kind,
get_artifact_prefix(dep_job),
locale_suffix,
)
for build in sorted(builds):
partial_info = {
"locale": build_locale,
"from_mar": builds[build]["mar_url"],
"to_mar": {
"artifact-reference": artifact_path
},
"branch": config.params["project"],
"update_number": update_number,
"dest_mar": build,
}
if "product" in builds[build]:
partial_info["product"] = builds[build]["product"]
if "previousVersion" in builds[build]:
partial_info["previousVersion"] = builds[build][
"previousVersion"]
if "previousBuildNumber" in builds[build]:
partial_info["previousBuildNumber"] = builds[build][
"previousBuildNumber"]
extra["funsize"]["partials"].append(partial_info)
update_number += 1
level = config.params["level"]
worker = {
"artifacts":
_generate_task_output_files(dep_job, builds.keys(), locale),
"implementation":
"docker-worker",
"docker-image": {
"in-tree": "funsize-update-generator"
},
"os":
"linux",
"max-run-time":
3600 if "asan" in dep_job.label else 900,
"chain-of-trust":
True,
"taskcluster-proxy":
True,
"env": {
"SIGNING_CERT":
identify_desired_signing_keys(
config.params["project"],
config.params["release_product"]),
"EXTRA_PARAMS":
f"--arch={architecture(build_platform)}",
"MAR_CHANNEL_ID":
attributes["mar-channel-id"],
},
}
if config.params.release_level() == "staging":
worker["env"]["FUNSIZE_ALLOW_STAGING_PREFIXES"] = "true"
task = {
"label": label,
"description": f"{dep_job.description} Partials",
"worker-type": "b-linux",
"dependencies": dependencies,
"scopes": [],
"attributes": attributes,
"run-on-projects": dep_job.attributes.get("run_on_projects"),
"treeherder": treeherder,
"extra": extra,
"worker": worker,
}
# We only want caching on linux/windows due to bug 1436977
if int(level) == 3 and any(
[build_platform.startswith(prefix)
for prefix in ["linux", "win"]]):
task["scopes"].append(
"auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/"
)
yield task