def yield_job(orig_job, deps, count):
job = deepcopy(orig_job)
job["dependencies"] = deps
job["name"] = "{}-{}".format(orig_job["name"], count)
if "treeherder" in job:
job["treeherder"]["symbol"] = add_suffix(job["treeherder"]["symbol"],
f"-{count}")
return job
def use_artifact(config, jobs):
if config.params.is_try():
use_artifact = config.params["try_task_config"].get(
"use-artifact-builds", False)
else:
use_artifact = False
for job in jobs:
if (config.kind == "build" and use_artifact
and job.get("index", {}).get("job-name") in ARTIFACT_JOBS
# If tests aren't packaged, then we are not able to rebuild all the packages
and job["worker"]["env"].get("MOZ_AUTOMATION_PACKAGE_TESTS")
== "1"):
job["treeherder"]["symbol"] = add_suffix(
job["treeherder"]["symbol"], "a")
job["worker"]["env"]["USE_ARTIFACT"] = "1"
job["attributes"]["artifact-build"] = True
yield job
def chunk_locales(config, jobs):
""" Utilizes chunking for l10n stuff """
for job in jobs:
locales_per_chunk = job.get("locales-per-chunk")
locales_with_changesets = job["attributes"][
"all_locales_with_changesets"]
if locales_per_chunk:
chunks, remainder = divmod(len(locales_with_changesets),
locales_per_chunk)
if remainder:
chunks = int(chunks + 1)
for this_chunk in range(1, chunks + 1):
chunked = copy.deepcopy(job)
chunked["name"] = chunked["name"].replace(
"/", f"-{this_chunk}/", 1)
chunked["mozharness"]["options"] = chunked["mozharness"].get(
"options", [])
# chunkify doesn't work with dicts
locales_with_changesets_as_list = sorted(
locales_with_changesets.items())
chunked_locales = chunkify(locales_with_changesets_as_list,
this_chunk, chunks)
chunked["mozharness"]["options"].extend([
f"locale={locale}:{changeset}"
for locale, changeset in chunked_locales
])
chunked["attributes"]["l10n_chunk"] = str(this_chunk)
# strip revision
chunked["attributes"]["chunk_locales"] = [
locale for locale, _ in chunked_locales
]
# add the chunk number to the TH symbol
chunked["treeherder"]["symbol"] = add_suffix(
chunked["treeherder"]["symbol"], this_chunk)
yield chunked
else:
job["mozharness"]["options"] = job["mozharness"].get("options", [])
job["mozharness"]["options"].extend([
f"locale={locale}:{changeset}" for locale, changeset in sorted(
locales_with_changesets.items())
])
yield job
def add_command(config, tasks):
config_tasks = {}
for dep in config.kind_dependencies_tasks.values():
if (
"update-verify-config" in dep.kind
or "update-verify-next-config" in dep.kind
):
config_tasks[dep.name] = dep
for task in tasks:
config_task = config_tasks[task["name"]]
total_chunks = task["extra"]["chunks"]
task["worker"].setdefault("env", {})["CHANNEL"] = config_task.task["extra"][
"channel"
]
task.setdefault("fetches", {})[config_task.label] = [
"update-verify.cfg",
]
task["treeherder"] = inherit_treeherder_from_dep(task, config_task)
for this_chunk in range(1, total_chunks + 1):
chunked = deepcopy(task)
chunked["treeherder"]["symbol"] = add_suffix(
chunked["treeherder"]["symbol"], this_chunk
)
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks
)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["run"] = {
"using": "run-task",
"cwd": "{checkout}",
"command": "tools/update-verify/scripts/chunked-verify.sh "
f"--total-chunks={total_chunks} --this-chunk={this_chunk}",
"sparse-profile": "update-verify",
}
yield chunked
def _generate_treeherder_symbol(group_symbol, build_symbol):
return join_symbol(group_symbol, add_suffix(build_symbol, "-poll"))
def test_add_suffix_with_group(self):
self.assertEqual(add_suffix("ab(xy)", 1), "ab(xy1)")
def test_add_suffix_no_group(self):
self.assertEqual(add_suffix("xy", 1), "xy1")