def yield_job(orig_job, deps, count):
job = deepcopy(orig_job)
job["dependencies"] = deps
job["name"] = "{}-{}".format(orig_job["name"], count)
if "treeherder" in job:
job["treeherder"]["symbol"] = add_suffix(job["treeherder"]["symbol"],
"-{}".format(count))
return job
def build_task_definition(orig_task, deps, count):
task = deepcopy(orig_task)
task['dependencies'] = deps
task['name'] = "{}-{}".format(orig_task['name'], count)
if 'treeherder' in task:
task['treeherder']['symbol'] = add_suffix(
task['treeherder']['symbol'], "-{}".format(count))
return task
def yield_job(orig_job, deps, count):
job = deepcopy(orig_job)
job['dependencies'] = deps
job['name'] = "{}-{}".format(orig_job['name'], count)
if 'treeherder' in job:
job['treeherder']['symbol'] = add_suffix(job['treeherder']['symbol'],
"-{}".format(count))
return job
def build_task_definition(orig_task, deps, count):
task = deepcopy(orig_task)
task['dependencies'] = deps
task['name'] = "{}-{}".format(orig_task['name'], count)
if 'treeherder' in task:
task['treeherder']['symbol'] = add_suffix(task['treeherder']['symbol'],
f"-{count}")
task["attributes"]["is_final_chunked_task"] = False
return task
def tests_drop_1proc(config, jobs):
"""
Remove the -1proc suffix from Treeherder group symbols.
Restore the -e10s suffix (because some day we will have them!)
Reverses the effects of bug 1541527. Thunderbird builds are all single
process.
"""
for job in jobs:
test = job["run"]["test"]
e10s = test["e10s"]
if not e10s: # test-name & friends end with '-1proc'
test["test-name"] = _remove_suffix(test["test-name"], "-1proc")
test["try-name"] = _remove_suffix(test["try-name"], "-1proc")
group, symbol = split_symbol(test["treeherder-symbol"])
if group != "?":
group = _remove_suffix(group, "-1proc")
test["treeherder-symbol"] = join_symbol(group, symbol)
job["label"] = job["label"].replace("-1proc", "")
job["name"] = _remove_suffix(job["name"], "-1proc")
job["treeherder"]["symbol"] = test["treeherder-symbol"]
else: # e10s in the future
test["test-name"] = add_suffix(test["test-name"], "-e10s")
test["try-name"] = add_suffix(test["try-name"], "-e10s")
group, symbol = split_symbol(test["treeherder-symbol"])
if group != "?":
group = add_suffix(group, "-e10s")
test["treeherder-symbol"] = join_symbol(group, symbol)
job["label"] += "-e10s"
job["name"] = add_suffix(job["name"], "-e10s")
job["treeherder"]["symbol"] = test["treeherder-symbol"]
yield job
def tests_drop_1proc(config, jobs):
"""
Remove the -1proc suffix from Treeherder group symbols.
Restore the -e10s suffix (because some day we will have them!)
Reverses the effects of bug 1541527. Thunderbird builds are all single
process.
"""
for job in jobs:
test = job['run']['test']
e10s = test['e10s']
if not e10s: # test-name & friends end with '-1proc'
test['test-name'] = _remove_suffix(test['test-name'], '-1proc')
test['try-name'] = _remove_suffix(test['try-name'], '-1proc')
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group = _remove_suffix(group, '-1proc')
test['treeherder-symbol'] = join_symbol(group, symbol)
job['label'] = job['label'].replace('-1proc', '')
job['name'] = _remove_suffix(job['name'], '-1proc')
job['treeherder']['symbol'] = test['treeherder-symbol']
else: # e10s in the future
test['test-name'] = add_suffix(test['test-name'], '-e10s')
test['try-name'] = add_suffix(test['try-name'], '-e10s')
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group = add_suffix(group, '-e10s')
test['treeherder-symbol'] = join_symbol(group, symbol)
job['label'] += '-e10s'
job['name'] = add_suffix(job['name'], '-e10s')
job['treeherder']['symbol'] = test['treeherder-symbol']
yield job
def add_command(config, tasks):
for task in tasks:
total_chunks = task["extra"]["chunks"]
for this_chunk in range(1, total_chunks+1):
chunked = deepcopy(task)
chunked["treeherder"]["symbol"] = add_suffix(
chunked["treeherder"]["symbol"], this_chunk)
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks
)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["worker"]["command"] = [
"/bin/bash",
"-c",
"hg clone $BUILD_TOOLS_REPO tools && " +
"tools/scripts/release/updates/chunked-verify.sh " +
"UNUSED UNUSED {} {}".format(
total_chunks,
this_chunk,
)
]
for thing in ("CHANNEL", "VERIFY_CONFIG", "BUILD_TOOLS_REPO"):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(
chunked, thing, thing,
**{
'project': config.params['project'],
'release-type': config.params['release_type'],
}
)
update_verify_config = None
for upstream in chunked.get("dependencies", {}).keys():
if 'update-verify-config' in upstream:
update_verify_config = "{}update-verify.cfg".format(
get_taskcluster_artifact_prefix(task, "<{}>".format(upstream))
)
if not update_verify_config:
raise Exception("Couldn't find upate verify config")
chunked["worker"]["env"]["TASKCLUSTER_VERIFY_CONFIG"] = {
"task-reference": update_verify_config
}
yield chunked
def use_artifact(config, jobs):
if config.params.is_try():
use_artifact = config.params['try_task_config'].get(
'use-artifact-builds', False)
else:
use_artifact = False
for job in jobs:
if (config.kind == "build" and use_artifact
and job.get("index", {}).get("job-name") in ARTIFACT_JOBS
# If tests aren't packaged, then we are not able to rebuild all the packages
and job['worker']['env'].get('MOZ_AUTOMATION_PACKAGE_TESTS')
== '1'):
job['treeherder']['symbol'] = add_suffix(
job['treeherder']['symbol'], 'a')
job['worker']['env']['USE_ARTIFACT'] = '1'
job['attributes']['artifact-build'] = True
yield job
def use_artifact(config, jobs):
if config.params.is_try():
use_artifact = config.params["try_task_config"].get(
"use-artifact-builds", False)
else:
use_artifact = False
for job in jobs:
if (config.kind == "build" and use_artifact
and job.get("index", {}).get("job-name") in ARTIFACT_JOBS
# If tests aren't packaged, then we are not able to rebuild all the packages
and job["worker"]["env"].get("MOZ_AUTOMATION_PACKAGE_TESTS")
== "1"):
job["treeherder"]["symbol"] = add_suffix(
job["treeherder"]["symbol"], "a")
job["worker"]["env"]["USE_ARTIFACT"] = "1"
job["attributes"]["artifact-build"] = True
yield job
def chunk_locales(config, jobs):
""" Utilizes chunking for l10n stuff """
for job in jobs:
locales_per_chunk = job.get("locales-per-chunk")
locales_with_changesets = job["attributes"][
"all_locales_with_changesets"]
if locales_per_chunk:
chunks, remainder = divmod(len(locales_with_changesets),
locales_per_chunk)
if remainder:
chunks = int(chunks + 1)
for this_chunk in range(1, chunks + 1):
chunked = copy.deepcopy(job)
chunked["name"] = chunked["name"].replace(
"/", "-{}/".format(this_chunk), 1)
chunked["mozharness"]["options"] = chunked["mozharness"].get(
"options", [])
# chunkify doesn't work with dicts
locales_with_changesets_as_list = sorted(
locales_with_changesets.items())
chunked_locales = chunkify(locales_with_changesets_as_list,
this_chunk, chunks)
chunked["mozharness"]["options"].extend([
"locale={}:{}".format(locale, changeset)
for locale, changeset in chunked_locales
])
chunked["attributes"]["l10n_chunk"] = str(this_chunk)
# strip revision
chunked["attributes"]["chunk_locales"] = [
locale for locale, _ in chunked_locales
]
# add the chunk number to the TH symbol
chunked["treeherder"]["symbol"] = add_suffix(
chunked["treeherder"]["symbol"], this_chunk)
yield chunked
else:
job["mozharness"]["options"] = job["mozharness"].get("options", [])
job["mozharness"]["options"].extend([
"locale={}:{}".format(locale, changeset) for locale, changeset
in sorted(locales_with_changesets.items())
])
yield job
def add_command(config, tasks):
for task in tasks:
total_chunks = task["extra"]["chunks"]
for this_chunk in range(1, total_chunks+1):
chunked = deepcopy(task)
chunked["treeherder"]["symbol"] = add_suffix(
chunked["treeherder"]["symbol"], this_chunk)
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks
)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["run"] = {
'using': 'run-task',
'command': 'cd /builds/worker/checkouts/gecko && '
'tools/update-verify/scripts/chunked-verify.sh '
'{} {}'.format(
total_chunks,
this_chunk,
),
'sparse-profile': 'update-verify',
}
for thing in ("CHANNEL", "VERIFY_CONFIG"):
thing = "worker.env.{}".format(thing)
resolve_keyed_by(
chunked, thing, thing,
**{
'project': config.params['project'],
'release-type': config.params['release_type'],
}
)
for upstream in chunked.get("dependencies", {}).keys():
if 'update-verify-config' in upstream:
chunked.setdefault('fetches', {})[upstream] = [
"update-verify.cfg",
]
break
else:
raise Exception("Couldn't find upate verify config")
yield chunked
def add_command(config, tasks):
config_tasks = {}
for dep in config.kind_dependencies_tasks.values():
if (
"update-verify-config" in dep.kind
or "update-verify-next-config" in dep.kind
):
config_tasks[dep.name] = dep
for task in tasks:
config_task = config_tasks[task["name"]]
total_chunks = task["extra"]["chunks"]
task["worker"].setdefault("env", {})["CHANNEL"] = config_task.task["extra"][
"channel"
]
task.setdefault("fetches", {})[config_task.label] = [
"update-verify.cfg",
]
task["treeherder"] = inherit_treeherder_from_dep(task, config_task)
for this_chunk in range(1, total_chunks + 1):
chunked = deepcopy(task)
chunked["treeherder"]["symbol"] = add_suffix(
chunked["treeherder"]["symbol"], this_chunk
)
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks
)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["run"] = {
"using": "run-task",
"cwd": "{checkout}",
"command": "tools/update-verify/scripts/chunked-verify.sh "
"{} {}".format(
total_chunks,
this_chunk,
),
"sparse-profile": "update-verify",
}
yield chunked
def add_command(config, tasks):
config_tasks = {}
for dep in config.kind_dependencies_tasks:
if 'update-verify-config' in dep.kind or 'update-verify-next-config' in dep.kind:
config_tasks[dep.name] = dep
for task in tasks:
config_task = config_tasks[task['name']]
total_chunks = task["extra"]["chunks"]
task['worker'].setdefault(
'env', {})['CHANNEL'] = (config_task.task['extra']['channel'])
task.setdefault('fetches', {})[config_task.label] = [
"update-verify.cfg",
]
task['treeherder'] = inherit_treeherder_from_dep(task, config_task)
for this_chunk in range(1, total_chunks + 1):
chunked = deepcopy(task)
chunked["treeherder"]["symbol"] = add_suffix(
chunked["treeherder"]["symbol"], this_chunk)
chunked["label"] = "release-update-verify-{}-{}/{}".format(
chunked["name"], this_chunk, total_chunks)
if not chunked["worker"].get("env"):
chunked["worker"]["env"] = {}
chunked["run"] = {
'using':
'run-task',
'cwd':
'{checkout}',
'command':
'tools/update-verify/scripts/chunked-verify.sh '
'{} {}'.format(
total_chunks,
this_chunk,
),
'sparse-profile':
'update-verify',
}
yield chunked
def split_chunks(config, tests):
"""Based on the 'chunks' key, split tests up into chunks by duplicating
them and assigning 'this-chunk' appropriately and updating the treeherder
symbol."""
for test in tests:
if test['suite'].startswith('test-verify') or \
test['suite'].startswith('test-coverage'):
env = config.params.get('try_task_config', {}) or {}
env = env.get('templates', {}).get('env', {})
test['chunks'] = perfile_number_of_chunks(config.params.is_try(),
env.get('MOZHARNESS_TEST_PATHS', ''),
config.params.get('head_repository', ''),
config.params.get('head_rev', ''),
test['test-name'])
# limit the number of chunks we run for test-verify mode because
# test-verify is comprehensive and takes a lot of time, if we have
# >30 tests changed, this is probably an import of external tests,
# or a patch renaming/moving files in bulk
maximum_number_verify_chunks = 3
if test['chunks'] > maximum_number_verify_chunks:
test['chunks'] = maximum_number_verify_chunks
if test['chunks'] <= 1:
test['this-chunk'] = 1
yield test
continue
for this_chunk in range(1, test['chunks'] + 1):
# copy the test and update with the chunk number
chunked = copy.deepcopy(test)
chunked['this-chunk'] = this_chunk
# add the chunk number to the TH symbol
chunked['treeherder-symbol'] = add_suffix(
chunked['treeherder-symbol'], this_chunk)
yield chunked
def test_add_suffix_no_group(self):
self.assertEqual(add_suffix('xy', 1), 'xy1')
def _generate_treeherder_symbol(group_symbol, build_symbol):
return join_symbol(group_symbol, add_suffix(build_symbol, "-poll"))
def test_add_suffix_with_group(self):
self.assertEqual(add_suffix('ab(xy)', 1), 'ab(xy1)')
def test_add_suffix_with_group(self):
self.assertEqual(add_suffix("ab(xy)", 1), "ab(xy1)")
def test_add_suffix_no_group(self):
self.assertEqual(add_suffix("xy", 1), "xy1")