def split_btime_variants(config, jobs):
for job in jobs:
if job.get("perftest-btime-variants") is None:
yield job
continue
variants = job.pop("perftest-btime-variants")
if not variants:
yield job
continue
yield_existing = False
for suffix, options in variants:
if suffix is None:
# Append options to the existing job
job.setdefault("perftest-btime-variants", []).append(options)
yield_existing = True
else:
job_new = deepcopy(job)
group, symbol = split_symbol(job_new["treeherder"]["symbol"])
symbol += "-" + suffix
job_new["treeherder"]["symbol"] = join_symbol(group, symbol)
job_new["name"] += "-" + suffix
job_new.setdefault("perftest-perfherder-global",
{}).setdefault("extraOptions",
[]).append(suffix)
# Replace the existing options with the new ones
job_new["perftest-btime-variants"] = [options]
yield job_new
# The existing job has been modified so we should also return it
if yield_existing:
yield job
def make_beetmover_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
locale = dep_job.attributes.get("locale")
if not locale:
yield job
continue
group = "BMR"
# add the locale code
symbol = locale
treeherder = {
"symbol": join_symbol(group, symbol),
}
beet_description = {
"label": job["label"],
"primary-dependency": dep_job,
"dependent-tasks": job["dependent-tasks"],
"attributes": job["attributes"],
"treeherder": treeherder,
"locale": locale,
"shipping-phase": job["shipping-phase"],
"shipping-product": job["shipping-product"],
}
yield beet_description
def split_jsshell(config, jobs):
all_shells = {"sm": "Spidermonkey", "v8": "Google V8"}
for job in jobs:
if not job["name"].startswith("jsshell"):
yield job
continue
test = job.pop("test")
for shell in job.get("shell", all_shells.keys()):
assert shell in all_shells
new_job = copy.deepcopy(job)
new_job["name"] = "{}-{}".format(new_job["name"], shell)
new_job["description"] = "{} on {}".format(new_job["description"],
all_shells[shell])
new_job["shell"] = shell
group = f"js-bench-{shell}"
symbol = split_symbol(new_job["treeherder"]["symbol"])[1]
new_job["treeherder"]["symbol"] = join_symbol(group, symbol)
run = new_job["run"]
run["mach"] = run["mach"].format(shell=shell,
SHELL=shell.upper(),
test=test)
yield new_job
def split_apps(config, tests):
app_symbols = {
"chrome": "ChR",
"chrome-m": "ChR",
"chromium": "Cr",
"fenix": "fenix",
"refbrow": "refbrow",
}
for test in tests:
apps = test.pop("apps", None)
if not apps:
yield test
continue
for app in apps:
atest = deepcopy(test)
suffix = f"-{app}"
atest["app"] = app
atest["description"] += f" on {app.capitalize()}"
name = atest["test-name"] + suffix
atest["test-name"] = name
atest["try-name"] = name
if app in app_symbols:
group, symbol = split_symbol(atest["treeherder-symbol"])
group += f"-{app_symbols[app]}"
atest["treeherder-symbol"] = join_symbol(group, symbol)
yield atest
def fill_template(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task.pop("dependent-tasks", None)
# Fill out the dynamic fields in the task description
task["label"] = dep.label + "-upload-symbols"
# Skip tasks where we don't have the full crashsymbols enabled
if not dep.attributes.get("enable-full-crashsymbols") or dep.attributes.get(
"skip-upload-crashsymbols"
):
logger.debug("Skipping upload symbols task for %s", task["label"])
continue
task["dependencies"] = {"build": dep.label}
task["worker"]["env"]["GECKO_HEAD_REPOSITORY"] = config.params[
"head_repository"
]
task["worker"]["env"]["GECKO_HEAD_REV"] = config.params["head_rev"]
task["worker"]["env"]["SYMBOL_SECRET"] = task["worker"]["env"][
"SYMBOL_SECRET"
].format(level=config.params["level"])
attributes = copy_attributes_from_dependent_job(dep)
attributes.update(task.get("attributes", {}))
task["attributes"] = attributes
treeherder = inherit_treeherder_from_dep(task, dep)
th = dep.task.get("extra")["treeherder"]
th_symbol = th.get("symbol")
th_groupsymbol = th.get("groupSymbol", "?")
# Disambiguate the treeherder symbol.
sym = "Sym" + (th_symbol[1:] if th_symbol.startswith("B") else th_symbol)
treeherder.setdefault("symbol", join_symbol(th_groupsymbol, sym))
task["treeherder"] = treeherder
# We only want to run these tasks if the build is run.
# XXX Better to run this on promote phase instead?
task["run-on-projects"] = dep.attributes.get("run_on_projects")
task["optimization"] = {"upload-symbols": None}
task["if-dependencies"] = ["build"]
# clear out the stuff that's not part of a task description
del task["primary-dependency"]
yield task
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
job["depname"] = dep_job.label
# add the chunk number to the TH symbol
symbol = job.get("treeherder", {}).get("symbol", "Bs")
symbol = "{}{}".format(symbol, dep_job.attributes.get("l10n_chunk"))
group = "L10n"
job["treeherder"] = {
"symbol": join_symbol(group, symbol),
}
yield job
def split_python(config, jobs):
for job in jobs:
key = "python-version"
versions = job.pop(key, [])
if not versions:
yield job
continue
for version in versions:
group = f"py{version}"
pyjob = copy.deepcopy(job)
if "name" in pyjob:
pyjob["name"] += f"-{group}"
else:
pyjob["label"] += f"-{group}"
symbol = split_symbol(pyjob["treeherder"]["symbol"])[1]
pyjob["treeherder"]["symbol"] = join_symbol(group, symbol)
pyjob["run"][key] = version
yield pyjob
def split_perftest_variants(config, jobs):
for job in jobs:
if job.get("variants") is None:
yield job
continue
for variant in job.pop("variants"):
job_new = deepcopy(job)
group, symbol = split_symbol(job_new["treeherder"]["symbol"])
group += "-" + variant
job_new["treeherder"]["symbol"] = join_symbol(group, symbol)
job_new["name"] += "-" + variant
job_new.setdefault("perftest-perfherder-global",
{}).setdefault("extraOptions",
[]).append(variant)
job_new[variant] = True
yield job_new
yield job
def split_page_load_by_url(config, tests):
for test in tests:
# `chunk-number` and 'subtest' only exists when the task had a
# definition for `raptor-subtests`
chunk_number = test.pop("chunk-number", None)
subtest = test.get(
"subtest"
) # don't pop as some tasks need this value after splitting variants
subtest_symbol = test.pop("subtest-symbol", None)
if not chunk_number or not subtest:
yield test
continue
if len(subtest_symbol) > 10 and "ytp" not in subtest_symbol:
raise Exception(
"Treeherder symbol %s is lager than 10 char! Please use a different symbol."
% subtest_symbol)
if test["test-name"].startswith("browsertime-"):
test["raptor-test"] = subtest
# Remove youtube-playback in the test name to avoid duplication
test["test-name"] = test["test-name"].replace(
"youtube-playback-", "")
else:
# Use full test name if running on webextension
test["raptor-test"] = "raptor-tp6-" + subtest + "-{}".format(
test["app"])
# Only run the subtest/single URL
test["test-name"] += f"-{subtest}"
test["try-name"] += f"-{subtest}"
# Set treeherder symbol and description
group, _ = split_symbol(test["treeherder-symbol"])
test["treeherder-symbol"] = join_symbol(group, subtest_symbol)
test["description"] += f" on {subtest}"
yield test
def _generate_treeherder_symbol(group_symbol, build_symbol):
return join_symbol(group_symbol, add_suffix(build_symbol, "-poll"))
def test_join_with_group(self):
self.assertEqual(join_symbol("ab", "xy"), "ab(xy)")
def test_join_no_group(self):
self.assertEqual(join_symbol("?", "xy"), "xy")
def make_task_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
locale = dep_job.attributes.get("locale")
treeherder = inherit_treeherder_from_dep(job, dep_job)
treeherder.setdefault(
"symbol", join_symbol(job.get("treeherder-group", "ms"), locale or "N")
)
label = job.get("label", f"{config.kind}-{dep_job.label}")
dependencies = {dep_job.kind: dep_job.label}
signing_dependencies = dep_job.dependencies
# This is so we get the build task etc in our dependencies to
# have better beetmover support.
dependencies.update(signing_dependencies)
attributes = copy_attributes_from_dependent_job(dep_job)
attributes["required_signoffs"] = sorted_unique_list(
attributes.get("required_signoffs", []), job.pop("required_signoffs")
)
attributes["shipping_phase"] = job["shipping-phase"]
if locale:
attributes["locale"] = locale
build_platform = attributes.get("build_platform")
if config.kind == "partials-signing":
upstream_artifacts = generate_partials_artifacts(
dep_job, config.params["release_history"], build_platform, locale
)
else:
upstream_artifacts = generate_complete_artifacts(dep_job, config.kind)
is_shippable = job.get(
"shippable", dep_job.attributes.get("shippable") # First check current job
) # Then dep job for 'shippable'
signing_cert_scope = get_signing_cert_scope_per_platform(
build_platform, is_shippable, config
)
scopes = [signing_cert_scope]
task = {
"label": label,
"description": "{} {}".format(
dep_job.description, job["description-suffix"]
),
"worker-type": job.get("worker-type", "linux-signing"),
"worker": {
"implementation": "scriptworker-signing",
"upstream-artifacts": upstream_artifacts,
"max-run-time": 3600,
},
"dependencies": dependencies,
"attributes": attributes,
"scopes": scopes,
"run-on-projects": job.get(
"run-on-projects", dep_job.attributes.get("run_on_projects")
),
"treeherder": treeherder,
}
yield task