def split_pageload(config, tests):
# Split test by pageload type (cold, warm)
for test in tests:
mozharness = test.setdefault("mozharness", {})
extra_options = mozharness.setdefault("extra-options", [])
pageload = test.pop("pageload", None)
if not pageload or "--chimera" in extra_options:
yield test
continue
if pageload in ("warm", "both"):
# make a deepcopy if 'both', otherwise use the test object itself
warmtest = deepcopy(test) if pageload == "both" else test
warmtest["warm"] = True
group, symbol = split_symbol(warmtest["treeherder-symbol"])
symbol += "-w"
warmtest["treeherder-symbol"] = join_symbol(group, symbol)
yield warmtest
if pageload in ("cold", "both"):
assert "subtest" in test
test["description"] += " using cold pageload"
test["cold"] = True
test["max-run-time"] = 3000
test["test-name"] += "-cold"
test["try-name"] += "-cold"
group, symbol = split_symbol(test["treeherder-symbol"])
symbol += "-c"
test["treeherder-symbol"] = join_symbol(group, symbol)
yield test
def split_chunks(config, tests):
"""Based on the 'chunks' key, split tests up into chunks by duplicating
them and assigning 'this-chunk' appropriately and updating the treeherder
symbol."""
for test in tests:
if test['chunks'] == 1:
test['this-chunk'] = 1
yield test
continue
# HACK: Bug 1373578 appears to pass with more chunks, non-e10s only though
if test['test-platform'] == 'windows7-32/debug' and test[
'test-name'] == 'reftest':
test['chunks'] = 32
if (test['test-platform'] == 'windows7-32/opt' or
test['test-platform'] == 'windows7-32-pgo/opt') and \
test['test-name'] in ['reftest-e10s', 'reftest-no-accel-e10s', 'reftest-gpu-e10s']:
test['chunks'] = 32
for this_chunk in range(1, test['chunks'] + 1):
# copy the test and update with the chunk number
chunked = copy.deepcopy(test)
chunked['this-chunk'] = this_chunk
# add the chunk number to the TH symbol
group, symbol = split_symbol(chunked['treeherder-symbol'])
symbol += str(this_chunk)
chunked['treeherder-symbol'] = join_symbol(group, symbol)
yield chunked
def split_browsertime_page_load_by_url(config, tests):
for test in tests:
# for tests that have 'raptor-subtests' listed, we want to create a separate
# test job for every subtest (i.e. split out each page-load URL into its own job)
subtests = test.pop('raptor-subtests', None)
if not subtests:
yield test
continue
chunk_number = 0
for subtest in subtests:
# create new test job
chunked = deepcopy(test)
# only run the subtest/single URL
chunked['test-name'] += "-{}".format(subtest)
chunked['try-name'] += "-{}".format(subtest)
chunked['raptor-test'] = subtest
# set treeherder symbol and description
chunk_number += 1
group, symbol = split_symbol(test['treeherder-symbol'])
symbol += "-{}".format(chunk_number)
chunked['treeherder-symbol'] = join_symbol(group, symbol)
chunked['description'] += "-{}".format(subtest)
yield chunked
def chunk_locales(config, jobs):
""" Utilizes chunking for l10n stuff """
for job in jobs:
chunks = job.get('chunks')
all_locales = job['attributes']['all_locales']
if chunks:
if chunks > len(all_locales):
# Reduce chunks down to the number of locales
chunks = len(all_locales)
for this_chunk in range(1, chunks + 1):
chunked = copy.deepcopy(job)
chunked['name'] = chunked['name'].replace(
'/', '-{}/'.format(this_chunk), 1)
chunked['mozharness']['options'] = chunked['mozharness'].get(
'options', [])
my_locales = []
my_locales = chunkify(all_locales, this_chunk, chunks)
chunked['mozharness']['options'].extend(
["locale={}".format(locale) for locale in my_locales])
chunked['attributes']['l10n_chunk'] = str(this_chunk)
chunked['attributes']['chunk_locales'] = my_locales
# add the chunk number to the TH symbol
group, symbol = split_symbol(
chunked.get('treeherder', {}).get('symbol', ''))
symbol += str(this_chunk)
chunked['treeherder']['symbol'] = join_symbol(group, symbol)
yield chunked
else:
job['mozharness']['options'] = job['mozharness'].get('options', [])
job['mozharness']['options'].extend(
["locale={}".format(locale) for locale in all_locales])
yield job
def split_variants(config, tests):
for test in tests:
variants = test.pop('variants')
yield copy.deepcopy(test)
for name in variants:
testv = copy.deepcopy(test)
variant = TEST_VARIANTS[name]
if 'filterfn' in variant and not variant['filterfn'](testv):
continue
testv['attributes']['unittest_variant'] = name
testv['description'] = variant['description'].format(**testv)
suffix = '-' + variant['suffix']
testv['test-name'] += suffix
testv['try-name'] += suffix
group, symbol = split_symbol(testv['treeherder-symbol'])
if group != '?':
group += suffix
else:
symbol += suffix
testv['treeherder-symbol'] = join_symbol(group, symbol)
testv.update(variant.get('replace', {}))
yield merge(testv, variant.get('merge', {}))
def split_jsshell(config, jobs):
all_shells = {'sm': "Spidermonkey", 'v8': "Google V8"}
for job in jobs:
if not job['name'].startswith('jsshell'):
yield job
continue
test = job.pop('test')
for shell in job.get('shell', all_shells.keys()):
assert shell in all_shells
new_job = copy.deepcopy(job)
new_job['name'] = '{}-{}'.format(new_job['name'], shell)
new_job['description'] = '{} on {}'.format(new_job['description'],
all_shells[shell])
new_job['shell'] = shell
group = 'js-bench-{}'.format(shell)
symbol = split_symbol(new_job['treeherder']['symbol'])[1]
new_job['treeherder']['symbol'] = join_symbol(group, symbol)
run = new_job['run']
run['mach'] = run['mach'].format(shell=shell,
SHELL=shell.upper(),
test=test)
yield new_job
def split_chunks(config, tests):
"""Based on the 'chunks' key, split tests up into chunks by duplicating
them and assigning 'this-chunk' appropriately and updating the treeherder
symbol."""
for test in tests:
if test['chunks'] == 1:
test['this-chunk'] = 1
yield test
continue
# HACK: Bug 1373578 appears to pass with more chunks, non-e10s only though
if test['test-platform'] == 'windows7-32/debug' and test['test-name'] == 'reftest':
test['chunks'] = 32
if (test['test-platform'] == 'windows7-32/opt' or
test['test-platform'] == 'windows7-32-pgo/opt') and \
test['test-name'] in ['reftest-e10s', 'reftest-no-accel-e10s', 'reftest-gpu-e10s']:
test['chunks'] = 32
for this_chunk in range(1, test['chunks'] + 1):
# copy the test and update with the chunk number
chunked = copy.deepcopy(test)
chunked['this-chunk'] = this_chunk
# add the chunk number to the TH symbol
group, symbol = split_symbol(chunked['treeherder-symbol'])
symbol += str(this_chunk)
chunked['treeherder-symbol'] = join_symbol(group, symbol)
yield chunked
def split_e10s(config, tests):
for test in tests:
e10s = test['e10s']
test['e10s'] = False
test['attributes']['e10s'] = False
if e10s == 'both':
yield copy.deepcopy(test)
e10s = True
if e10s:
test['test-name'] += '-e10s'
test['try-name'] += '-e10s'
test['e10s'] = True
test['attributes']['e10s'] = True
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-e10s'
test['treeherder-symbol'] = join_symbol(group, symbol)
if test['suite'] == 'talos':
for i, option in enumerate(test['mozharness']['extra-options']):
if option.startswith('--suite='):
test['mozharness']['extra-options'][i] += '-e10s'
else:
test['mozharness']['extra-options'].append('--e10s')
yield test
def split_e10s(config, tests):
for test in tests:
e10s = test['e10s']
test['e10s'] = False
test['attributes']['e10s'] = False
if e10s == 'both':
yield copy.deepcopy(test)
e10s = True
if e10s:
test['test-name'] += '-e10s'
test['try-name'] += '-e10s'
test['e10s'] = True
test['attributes']['e10s'] = True
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-e10s'
test['treeherder-symbol'] = join_symbol(group, symbol)
if test['suite'] == 'talos':
for i, option in enumerate(
test['mozharness']['extra-options']):
if option.startswith('--suite='):
test['mozharness']['extra-options'][i] += '-e10s'
else:
test['mozharness']['extra-options'].append('--e10s')
yield test
def build_task(config, tasks):
for task in tasks:
provisioner_id, worker_type = task['worker-type'].split('/', 1)
routes = task['routes']
scopes = task['scopes']
# set up extra
extra = task['extra']
extra['treeherderEnv'] = task['treeherder']['environments']
task_th = task['treeherder']
treeherder = extra.setdefault('treeherder', {})
machine_platform, collection = task_th['platform'].split('/', 1)
treeherder['machine'] = {'platform': machine_platform}
treeherder['collection'] = {collection: True}
groupSymbol, symbol = split_symbol(task_th['symbol'])
if groupSymbol != '?':
treeherder['groupSymbol'] = groupSymbol
if groupSymbol not in GROUP_NAMES:
raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
treeherder['groupName'] = GROUP_NAMES[groupSymbol]
treeherder['symbol'] = symbol
treeherder['jobKind'] = task_th['kind']
treeherder['tier'] = task_th['tier']
routes.extend([
'{}.v2.{}.{}.{}'.format(root,
config.params['project'],
config.params['head_rev'],
config.params['pushlog_id'])
for root in 'tc-treeherder', 'tc-treeherder-stage'
])
def split_jsshell(config, jobs):
all_shells = {"sm": "Spidermonkey", "v8": "Google V8"}
for job in jobs:
if not job["name"].startswith("jsshell"):
yield job
continue
test = job.pop("test")
for shell in job.get("shell", all_shells.keys()):
assert shell in all_shells
new_job = copy.deepcopy(job)
new_job["name"] = "{}-{}".format(new_job["name"], shell)
new_job["description"] = "{} on {}".format(
new_job["description"], all_shells[shell]
)
new_job["shell"] = shell
group = "js-bench-{}".format(shell)
symbol = split_symbol(new_job["treeherder"]["symbol"])[1]
new_job["treeherder"]["symbol"] = join_symbol(group, symbol)
run = new_job["run"]
run["mach"] = run["mach"].format(
shell=shell, SHELL=shell.upper(), test=test
)
yield new_job
def split_apps(config, tests):
app_symbols = {
'chrome': 'Chr',
'chromium': 'Cr',
'fenix': 'fenix',
'refbrow': 'refbrow',
}
for test in tests:
apps = test.pop('apps', None)
if not apps:
yield test
continue
for app in apps:
atest = deepcopy(test)
suffix = "-{}".format(app)
atest['app'] = app
atest['description'] += " on {}".format(app.capitalize())
name = atest['test-name']
if name.endswith('-cold'):
name = atest['test-name'][:-len('-cold')] + suffix + '-cold'
else:
name += suffix
atest['test-name'] = name
atest['try-name'] = name
if app in app_symbols:
group, symbol = split_symbol(atest['treeherder-symbol'])
group += "-{}".format(app_symbols[app])
atest['treeherder-symbol'] = join_symbol(group, symbol)
yield atest
def split_btime_variants(config, jobs):
for job in jobs:
if job.get("perftest-btime-variants") is None:
yield job
continue
variants = job.pop("perftest-btime-variants")
if not variants:
yield job
continue
yield_existing = False
for suffix, options in variants:
if suffix is None:
# Append options to the existing job
job.setdefault("perftest-btime-variants", []).append(options)
yield_existing = True
else:
job_new = deepcopy(job)
group, symbol = split_symbol(job_new["treeherder"]["symbol"])
symbol += "-" + suffix
job_new["treeherder"]["symbol"] = join_symbol(group, symbol)
job_new["name"] += "-" + suffix
job_new.setdefault("perftest-perfherder-global", {}).setdefault(
"extraOptions", []
).append(suffix)
# Replace the existing options with the new ones
job_new["perftest-btime-variants"] = [options]
yield job_new
# The existing job has been modified so we should also return it
if yield_existing:
yield job
def split_serviceworker_e10s(config, tests):
for test in tests:
sw = test.pop('serviceworker-e10s')
test['serviceworker-e10s'] = False
test['attributes']['serviceworker_e10s'] = False
if sw == 'both':
yield copy.deepcopy(test)
sw = True
if sw:
if not match_run_on_projects('mozilla-central',
test['run-on-projects']):
continue
test['description'] += " with serviceworker-e10s redesign enabled"
test['run-on-projects'] = ['mozilla-central']
test['test-name'] += '-sw'
test['try-name'] += '-sw'
test['attributes']['serviceworker_e10s'] = True
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-sw'
else:
symbol += '-sw'
test['treeherder-symbol'] = join_symbol(group, symbol)
test['mozharness']['extra-options'].append(
'--setpref="dom.serviceWorkers.parent_intercept=true"')
yield test
def split_pageload(config, tests):
# Split test by pageload type (cold, warm)
for test in tests:
pageload = test.pop('pageload', 'warm')
if pageload not in ('cold', 'both'):
yield test
continue
if pageload == 'both':
orig = deepcopy(test)
yield orig
assert 'subtest' in test
test['description'] += " using cold pageload"
test['cold'] = True
test['max-run-time'] = 3000
test['test-name'] += '-cold'
test['try-name'] += '-cold'
group, symbol = split_symbol(test['treeherder-symbol'])
symbol += '-c'
test['treeherder-symbol'] = join_symbol(group, symbol)
yield test
def chunk_locales(config, jobs):
""" Utilizes chunking for l10n stuff """
for job in jobs:
chunks = job.get('chunks')
all_locales = job['attributes']['all_locales']
if chunks:
for this_chunk in range(1, chunks + 1):
chunked = copy.deepcopy(job)
chunked['name'] = chunked['name'].replace(
'/', '-{}/'.format(this_chunk), 1
)
chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])
my_locales = []
my_locales = chunkify(all_locales, this_chunk, chunks)
chunked['mozharness']['options'].extend([
"locale={}".format(locale) for locale in my_locales
])
chunked['attributes']['l10n_chunk'] = str(this_chunk)
chunked['attributes']['chunk_locales'] = my_locales
# add the chunk number to the TH symbol
group, symbol = split_symbol(
chunked.get('treeherder', {}).get('symbol', ''))
symbol += str(this_chunk)
chunked['treeherder']['symbol'] = join_symbol(group, symbol)
yield chunked
else:
job['mozharness']['options'] = job['mozharness'].get('options', [])
job['mozharness']['options'].extend([
"locale={}".format(locale) for locale in all_locales
])
yield job
def split_socketprocess_e10s(config, tests):
for test in tests:
if test['attributes'].get('serviceworker_e10s'):
yield test
continue
sw = test.pop('socketprocess-e10s')
test['socketprocess-e10s'] = False
test['attributes']['socketprocess_e10s'] = False
if sw == 'both':
yield copy.deepcopy(test)
sw = True
if sw:
test['description'] += " with socket process enabled"
test['test-name'] += '-spi'
test['try-name'] += '-spi'
test['attributes']['socketprocess_e10s'] = True
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-spi'
else:
symbol += '-spi'
test['treeherder-symbol'] = join_symbol(group, symbol)
test['mozharness']['extra-options'].append(
'--setpref="media.peerconnection.mtransport_process=true"')
test['mozharness']['extra-options'].append(
'--setpref="network.process.enabled=true"')
yield test
def split_pageload(config, tests):
for test in tests:
pageload = test.pop('pageload', 'warm')
if pageload not in ('cold', 'both'):
yield test
continue
if pageload == 'both':
orig = deepcopy(test)
yield orig
assert 'raptor-test' in test
test['description'] += " using cold pageload"
# for raptor-webext to run cold we just call the corresponding '-cold' test name; but
# for raptor browsertime we leave the raptor test name as/is and will set the '--cold'
# command line argument instead via settting test['cold'] to true
if test['test-name'].startswith('browsertime-tp6'):
test['cold'] = True
else:
test['raptor-test'] += '-cold'
test['max-run-time'] = 3000
test['test-name'] += '-cold'
test['try-name'] += '-cold'
group, symbol = split_symbol(test['treeherder-symbol'])
symbol += '-c'
test['treeherder-symbol'] = join_symbol(group, symbol)
yield test
def split_chunks(config, tests):
"""Based on the 'chunks' key, split tests up into chunks by duplicating
them and assigning 'this-chunk' appropriately and updating the treeherder
symbol."""
for test in tests:
if test['suite'].startswith('test-verify'):
test['chunks'] = perfile_number_of_chunks(config, test['test-name'])
if test['chunks'] == 0:
continue
# limit the number of chunks we run for test-verify mode because
# test-verify is comprehensive and takes a lot of time, if we have
# >30 tests changed, this is probably an import of external tests,
# or a patch renaming/moving files in bulk
maximum_number_verify_chunks = 3
if test['chunks'] > maximum_number_verify_chunks:
test['chunks'] = maximum_number_verify_chunks
if test['chunks'] == 1:
test['this-chunk'] = 1
yield test
continue
for this_chunk in range(1, test['chunks'] + 1):
# copy the test and update with the chunk number
chunked = copy.deepcopy(test)
chunked['this-chunk'] = this_chunk
# add the chunk number to the TH symbol
group, symbol = split_symbol(chunked['treeherder-symbol'])
symbol += str(this_chunk)
chunked['treeherder-symbol'] = join_symbol(group, symbol)
yield chunked
def split_apps(config, tests):
app_symbols = {
"chrome": "ChR",
"chrome-m": "ChR",
"chromium": "Cr",
"fenix": "fenix",
"refbrow": "refbrow",
}
for test in tests:
apps = test.pop("apps", None)
if not apps:
yield test
continue
for app in apps:
atest = deepcopy(test)
suffix = "-{}".format(app)
atest["app"] = app
atest["description"] += " on {}".format(app.capitalize())
name = atest["test-name"] + suffix
atest["test-name"] = name
atest["try-name"] = name
if app in app_symbols:
group, symbol = split_symbol(atest["treeherder-symbol"])
group += "-{}".format(app_symbols[app])
atest["treeherder-symbol"] = join_symbol(group, symbol)
yield atest
def split_e10s(config, tests):
for test in tests:
e10s = get_keyed_by(item=test,
field='e10s',
item_name=test['test-name'])
test.setdefault('attributes', {})
test['e10s'] = False
test['attributes']['e10s'] = False
if e10s == 'both':
yield copy.deepcopy(test)
e10s = True
if e10s:
test['test-name'] += '-e10s'
test['e10s'] = True
test['attributes']['e10s'] = True
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-e10s'
test['treeherder-symbol'] = join_symbol(group, symbol)
test['mozharness']['extra-options'] = get_keyed_by(
item=test,
field='mozharness',
subfield='extra-options',
item_name=test['test-name'])
test['mozharness']['extra-options'].append('--e10s')
yield test
def yield_job(orig_job, deps, count):
job = deepcopy(orig_job)
job['dependencies'] = deps
job['name'] = "{}-{}".format(orig_job['name'], count)
if 'treeherder' in job:
groupSymbol, symbol = split_symbol(job['treeherder']['symbol'])
symbol += '-'
symbol += str(count)
job['treeherder']['symbol'] = join_symbol(groupSymbol, symbol)
return job
def tests_drop_1proc(config, jobs):
"""
Remove the -1proc suffix from Treeherder group symbols.
Restore the -e10s suffix (because some day we will have them!)
Reverses the effects of bug 1541527. Thunderbird builds are all single
process.
"""
for job in jobs:
test = job["run"]["test"]
e10s = test["e10s"]
if not e10s: # test-name & friends end with '-1proc'
test["test-name"] = _remove_suffix(test["test-name"], "-1proc")
test["try-name"] = _remove_suffix(test["try-name"], "-1proc")
group, symbol = split_symbol(test["treeherder-symbol"])
if group != "?":
group = _remove_suffix(group, "-1proc")
test["treeherder-symbol"] = join_symbol(group, symbol)
job["label"] = job["label"].replace("-1proc", "")
job["name"] = _remove_suffix(job["name"], "-1proc")
job["treeherder"]["symbol"] = test["treeherder-symbol"]
else: # e10s in the future
test["test-name"] = add_suffix(test["test-name"], "-e10s")
test["try-name"] = add_suffix(test["try-name"], "-e10s")
group, symbol = split_symbol(test["treeherder-symbol"])
if group != "?":
group = add_suffix(group, "-e10s")
test["treeherder-symbol"] = join_symbol(group, symbol)
job["label"] += "-e10s"
job["name"] = add_suffix(job["name"], "-e10s")
job["treeherder"]["symbol"] = test["treeherder-symbol"]
yield job
def tests_drop_1proc(config, jobs):
"""
Remove the -1proc suffix from Treeherder group symbols.
Restore the -e10s suffix (because some day we will have them!)
Reverses the effects of bug 1541527. Thunderbird builds are all single
process.
"""
for job in jobs:
test = job['run']['test']
e10s = test['e10s']
if not e10s: # test-name & friends end with '-1proc'
test['test-name'] = _remove_suffix(test['test-name'], '-1proc')
test['try-name'] = _remove_suffix(test['try-name'], '-1proc')
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group = _remove_suffix(group, '-1proc')
test['treeherder-symbol'] = join_symbol(group, symbol)
job['label'] = job['label'].replace('-1proc', '')
job['name'] = _remove_suffix(job['name'], '-1proc')
job['treeherder']['symbol'] = test['treeherder-symbol']
else: # e10s in the future
test['test-name'] = add_suffix(test['test-name'], '-e10s')
test['try-name'] = add_suffix(test['try-name'], '-e10s')
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group = add_suffix(group, '-e10s')
test['treeherder-symbol'] = join_symbol(group, symbol)
job['label'] += '-e10s'
job['name'] = add_suffix(job['name'], '-e10s')
job['treeherder']['symbol'] = test['treeherder-symbol']
yield job
def split_page_load_by_url(config, tests):
for test in tests:
# `chunk-number` and 'subtest' only exists when the task had a
# definition for `raptor-subtests`
chunk_number = test.pop("chunk-number", None)
subtest = test.pop("subtest", None)
subtest_symbol = test.pop("subtest-symbol", None)
if not chunk_number or not subtest:
yield test
continue
if len(subtest_symbol) > 10 and "ytp" not in subtest_symbol:
raise Exception(
"Treeherder symbol %s is lager than 10 char! Please use a different symbol."
% subtest_symbol)
if test["test-name"].startswith("browsertime-"):
test["raptor-test"] = subtest
# Remove youtube-playback in the test name to avoid duplication
test["test-name"] = test["test-name"].replace(
"youtube-playback-", "")
else:
# Use full test name if running on webextension
test["raptor-test"] = "raptor-tp6-" + subtest + "-{}".format(
test["app"])
# Only run the subtest/single URL
test["test-name"] += "-{}".format(subtest)
test["try-name"] += "-{}".format(subtest)
# Set treeherder symbol and description
group, symbol = split_symbol(test["treeherder-symbol"])
symbol = subtest_symbol
if test.get("cold"):
symbol += "-c"
elif test.pop("warm", False):
symbol += "-w"
test["treeherder-symbol"] = join_symbol(group, symbol)
test["description"] += " on {}".format(subtest)
yield test
def chunk_locales(config, jobs):
""" Utilizes chunking for l10n stuff """
for job in jobs:
locales_per_chunk = job.get('locales-per-chunk')
locales_with_changesets = job['attributes'][
'all_locales_with_changesets']
if locales_per_chunk:
chunks, remainder = divmod(len(locales_with_changesets),
locales_per_chunk)
if remainder:
chunks = int(chunks + 1)
for this_chunk in range(1, chunks + 1):
chunked = copy.deepcopy(job)
chunked['name'] = chunked['name'].replace(
'/', '-{}/'.format(this_chunk), 1)
chunked['mozharness']['options'] = chunked['mozharness'].get(
'options', [])
# chunkify doesn't work with dicts
locales_with_changesets_as_list = sorted(
locales_with_changesets.items())
chunked_locales = chunkify(locales_with_changesets_as_list,
this_chunk, chunks)
chunked['mozharness']['options'].extend([
'locale={}:{}'.format(locale, changeset)
for locale, changeset in chunked_locales
])
chunked['attributes']['l10n_chunk'] = str(this_chunk)
# strip revision
chunked['attributes']['chunk_locales'] = [
locale for locale, _ in chunked_locales
]
# add the chunk number to the TH symbol
group, symbol = split_symbol(
chunked.get('treeherder', {}).get('symbol', ''))
symbol += str(this_chunk)
chunked['treeherder']['symbol'] = join_symbol(group, symbol)
yield chunked
else:
job['mozharness']['options'] = job['mozharness'].get('options', [])
job['mozharness']['options'].extend([
'locale={}:{}'.format(locale, changeset) for locale, changeset
in sorted(locales_with_changesets.items())
])
yield job
def split_python(config, jobs):
for job in jobs:
key = 'python-version'
versions = job.pop(key, [])
if not versions:
yield job
continue
for version in versions:
group = 'py{0}'.format(version)
pyjob = copy.deepcopy(job)
if 'name' in pyjob:
pyjob['name'] += '-{0}'.format(group)
else:
pyjob['label'] += '-{0}'.format(group)
symbol = split_symbol(pyjob['treeherder']['symbol'])[1]
pyjob['treeherder']['symbol'] = join_symbol(group, symbol)
pyjob['run'][key] = version
yield pyjob
def split_python(config, jobs):
for job in jobs:
key = "python-version"
versions = job.pop(key, [])
if not versions:
yield job
continue
for version in versions:
group = "py{0}".format(version)
pyjob = copy.deepcopy(job)
if "name" in pyjob:
pyjob["name"] += "-{0}".format(group)
else:
pyjob["label"] += "-{0}".format(group)
symbol = split_symbol(pyjob["treeherder"]["symbol"])[1]
pyjob["treeherder"]["symbol"] = join_symbol(group, symbol)
pyjob["run"][key] = version
yield pyjob
def split_page_load_by_url(config, tests):
for test in tests:
# `chunk-number` and 'subtest' only exists when the task had a
# definition for `raptor-subtests`
chunk_number = test.pop('chunk-number', None)
subtest = test.pop('subtest', None)
if not chunk_number or not subtest:
yield test
continue
if isinstance(subtest, list):
subtest, subtest_symbol = subtest
else:
subtest_symbol = subtest
subtest = subtest
if len(subtest_symbol) > 10:
raise Exception(
"Treeherder symbol %s is lager than 10 char! Please use a different symbol."
% subtest_symbol)
if test['test-name'].startswith('browsertime-tp6'):
test['raptor-test'] = subtest
else:
# Use full test name if running on webextension
test['raptor-test'] = 'raptor-tp6-' + subtest + "-{}".format(
test['app'])
# Only run the subtest/single URL
test['test-name'] += "-{}".format(subtest)
test['try-name'] += "-{}".format(subtest)
# Set treeherder symbol and description
group, symbol = split_symbol(test['treeherder-symbol'])
symbol = subtest_symbol
if test.get("cold"):
symbol += "-c"
test['treeherder-symbol'] = join_symbol(group, symbol)
test['description'] += " on {}".format(subtest)
yield test
def split_browsertime_page_load_by_url(config, tests):
for test in tests:
# `chunk-number` only exists when the task had a
# definition for `raptor-subtests`
chunk_number = test.pop('chunk-number', None)
if not chunk_number:
yield test
continue
# only run the subtest/single URL
test['test-name'] += "-{}".format(test['raptor-test'])
test['try-name'] += "-{}".format(test['raptor-test'])
# set treeherder symbol and description
group, symbol = split_symbol(test['treeherder-symbol'])
symbol += "-{}".format(chunk_number)
test['treeherder-symbol'] = join_symbol(group, symbol)
test['description'] += "-{}".format(test['raptor-test'])
yield test
def split_e10s(config, tests):
for test in tests:
e10s = test['e10s']
test.setdefault('attributes', {})
test['e10s'] = False
test['attributes']['e10s'] = False
if e10s == 'both':
yield copy.deepcopy(test)
e10s = True
if e10s:
test['test-name'] += '-e10s'
test['e10s'] = True
test['attributes']['e10s'] = True
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-e10s'
test['treeherder-symbol'] = join_symbol(group, symbol)
test['mozharness']['extra-options'].append('--e10s')
yield test
def split_perftest_variants(config, jobs):
for job in jobs:
if job.get("variants") is None:
yield job
continue
for variant in job.pop("variants"):
job_new = deepcopy(job)
group, symbol = split_symbol(job_new["treeherder"]["symbol"])
group += "-" + variant
job_new["treeherder"]["symbol"] = join_symbol(group, symbol)
job_new["name"] += "-" + variant
job_new.setdefault("perftest-perfherder-global", {}).setdefault(
"extraOptions", []
).append(variant)
job_new[variant] = True
yield job_new
yield job
def split_chunks(config, tests):
"""Based on the 'chunks' key, split tests up into chunks by duplicating
them and assigning 'this-chunk' appropriately and updating the treeherder
symbol."""
for test in tests:
if test['chunks'] == 1:
test['this-chunk'] = 1
yield test
continue
for this_chunk in range(1, test['chunks'] + 1):
# copy the test and update with the chunk number
chunked = copy.deepcopy(test)
chunked['this-chunk'] = this_chunk
# add the chunk number to the TH symbol
group, symbol = split_symbol(chunked['treeherder-symbol'])
symbol += str(this_chunk)
chunked['treeherder-symbol'] = join_symbol(group, symbol)
yield chunked
def split_e10s(config, tests):
for test in tests:
e10s = get_keyed_by(item=test, field="e10s", item_name=test["test-name"])
test.setdefault("attributes", {})
test["e10s"] = False
test["attributes"]["e10s"] = False
if e10s == "both":
yield test
test = copy.deepcopy(test)
e10s = True
if e10s:
test["test-name"] += "-e10s"
test["e10s"] = True
test["attributes"]["e10s"] = True
group, symbol = split_symbol(test["treeherder-symbol"])
if group != "?":
group += "-e10s"
test["treeherder-symbol"] = join_symbol(group, symbol)
test["mozharness"].setdefault("extra-options", []).append("--e10s")
yield test
def split_chunks(config, tests):
"""Based on the 'chunks' key, split tests up into chunks by duplicating
them and assigning 'this-chunk' appropriately and updating the treeherder
symbol."""
for test in tests:
if test['chunks'] == 1:
test['this-chunk'] = 1
yield test
continue
for this_chunk in range(1, test['chunks'] + 1):
# copy the test and update with the chunk number
chunked = copy.deepcopy(test)
chunked['this-chunk'] = this_chunk
# add the chunk number to the TH symbol
group, symbol = split_symbol(chunked['treeherder-symbol'])
symbol += str(this_chunk)
chunked['treeherder-symbol'] = join_symbol(group, symbol)
yield chunked
def split_e10s(config, tests):
for test in tests:
e10s = test['e10s']
if e10s:
test_copy = copy.deepcopy(test)
test_copy['test-name'] += '-e10s'
test_copy['e10s'] = True
test_copy['attributes']['e10s'] = True
yield test_copy
if not e10s or e10s == 'both':
test['test-name'] += '-1proc'
test['try-name'] += '-1proc'
test['e10s'] = False
test['attributes']['e10s'] = False
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-1proc'
test['treeherder-symbol'] = join_symbol(group, symbol)
test['mozharness']['extra-options'].append('--disable-e10s')
yield test
def chunk_locales(config, jobs):
""" Utilizes chunking for l10n stuff """
for job in jobs:
chunks = job.get('chunks')
locales_with_changesets = job['attributes']['all_locales_with_changesets']
if chunks:
if chunks > len(locales_with_changesets):
# Reduce chunks down to the number of locales
chunks = len(locales_with_changesets)
for this_chunk in range(1, chunks + 1):
chunked = copy.deepcopy(job)
chunked['name'] = chunked['name'].replace(
'/', '-{}/'.format(this_chunk), 1
)
chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])
# chunkify doesn't work with dicts
locales_with_changesets_as_list = locales_with_changesets.items()
chunked_locales = chunkify(locales_with_changesets_as_list, this_chunk, chunks)
chunked['mozharness']['options'].extend([
'locale={}:{}'.format(locale, changeset)
for locale, changeset in chunked_locales
])
chunked['attributes']['l10n_chunk'] = str(this_chunk)
# strip revision
chunked['attributes']['chunk_locales'] = [locale for locale, _ in chunked_locales]
# add the chunk number to the TH symbol
group, symbol = split_symbol(
chunked.get('treeherder', {}).get('symbol', ''))
symbol += str(this_chunk)
chunked['treeherder']['symbol'] = join_symbol(group, symbol)
yield chunked
else:
job['mozharness']['options'] = job['mozharness'].get('options', [])
job['mozharness']['options'].extend([
'locale={}:{}'.format(locale, changeset)
for locale, changeset in locales_with_changesets.items()
])
yield job
def split_cold(config, tests):
for test in tests:
cold = test.pop('cold', False)
if not cold:
yield test
continue
orig = deepcopy(test)
yield orig
assert 'raptor-test' in test
test['description'] += " using cold pageload"
test['raptor-test'] += '-cold'
test['max-run-time'] = 3000
test['test-name'] += '-cold'
test['try-name'] += '-cold'
group, symbol = split_symbol(test['treeherder-symbol'])
symbol += '-c'
test['treeherder-symbol'] = join_symbol(group, symbol)
yield test
def build_task(config, tasks):
for task in tasks:
worker_type = task['worker-type'].format(
level=str(config.params['level']))
provisioner_id, worker_type = worker_type.split('/', 1)
routes = task.get('routes', [])
scopes = task.get('scopes', [])
# set up extra
extra = task.get('extra', {})
task_th = task.get('treeherder')
if task_th:
extra['treeherderEnv'] = task_th['environments']
treeherder = extra.setdefault('treeherder', {})
machine_platform, collection = task_th['platform'].split('/', 1)
treeherder['machine'] = {'platform': machine_platform}
treeherder['collection'] = {collection: True}
groupSymbol, symbol = split_symbol(task_th['symbol'])
if groupSymbol != '?':
treeherder['groupSymbol'] = groupSymbol
if groupSymbol not in GROUP_NAMES:
raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
treeherder['groupName'] = GROUP_NAMES[groupSymbol]
treeherder['symbol'] = symbol
treeherder['jobKind'] = task_th['kind']
treeherder['tier'] = task_th['tier']
routes.extend([
'{}.v2.{}.{}.{}'.format(TREEHERDER_ROUTE_ROOTS[env],
config.params['project'],
config.params['head_rev'],
config.params['pushlog_id'])
for env in task_th['environments']
])
if 'expires-after' not in task:
task['expires-after'] = '28 days' if config.params[
'project'] == 'try' else '1 year'
if 'deadline-after' not in task:
task['deadline-after'] = '1 day'
if 'coalesce-name' in task and int(config.params['level']) > 1:
key = COALESCE_KEY.format(project=config.params['project'],
name=task['coalesce-name'])
routes.append('coalesce.v1.' + key)
task_def = {
'provisionerId': provisioner_id,
'workerType': worker_type,
'routes': routes,
'created': {
'relative-datestamp': '0 seconds'
},
'deadline': {
'relative-datestamp': task['deadline-after']
},
'expires': {
'relative-datestamp': task['expires-after']
},
'scopes': scopes,
'metadata': {
'description':
task['description'],
'name':
task['label'],
'owner':
config.params['owner'],
'source':
'{}/file/{}/{}'.format(config.params['head_repository'],
config.params['head_rev'], config.path),
},
'extra': extra,
'tags': {
'createdForUser': config.params['owner']
},
}
# add the payload and adjust anything else as required (e.g., scopes)
payload_builders[task['worker']['implementation']](config, task,
task_def)
attributes = task.get('attributes', {})
attributes['run_on_projects'] = task.get('run-on-projects', ['all'])
yield {
'label': task['label'],
'task': task_def,
'dependencies': task.get('dependencies', {}),
'attributes': attributes,
'when': task.get('when', {}),
}
def build_task(config, tasks):
for task in tasks:
worker_type = task['worker-type'].format(level=str(config.params['level']))
provisioner_id, worker_type = worker_type.split('/', 1)
routes = task.get('routes', [])
scopes = task.get('scopes', [])
# set up extra
extra = task.get('extra', {})
task_th = task.get('treeherder')
if task_th:
extra['treeherderEnv'] = task_th['environments']
treeherder = extra.setdefault('treeherder', {})
machine_platform, collection = task_th['platform'].split('/', 1)
treeherder['machine'] = {'platform': machine_platform}
treeherder['collection'] = {collection: True}
groupSymbol, symbol = split_symbol(task_th['symbol'])
if groupSymbol != '?':
treeherder['groupSymbol'] = groupSymbol
if groupSymbol not in GROUP_NAMES:
raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
treeherder['groupName'] = GROUP_NAMES[groupSymbol]
treeherder['symbol'] = symbol
treeherder['jobKind'] = task_th['kind']
treeherder['tier'] = task_th['tier']
routes.extend([
'{}.v2.{}.{}.{}'.format(TREEHERDER_ROUTE_ROOTS[env],
config.params['project'],
config.params['head_rev'],
config.params['pushlog_id'])
for env in task_th['environments']
])
if 'expires-after' not in task:
task['expires-after'] = '14 days' if config.params['project'] == 'try' else '1 year'
if 'deadline-after' not in task:
task['deadline-after'] = '1 day'
if 'coalesce-name' in task and int(config.params['level']) > 1:
key = COALESCE_KEY.format(
project=config.params['project'],
name=task['coalesce-name'])
routes.append('coalesce.v1.' + key)
task_def = {
'provisionerId': provisioner_id,
'workerType': worker_type,
'routes': routes,
'created': {'relative-datestamp': '0 seconds'},
'deadline': {'relative-datestamp': task['deadline-after']},
'expires': {'relative-datestamp': task['expires-after']},
'scopes': scopes,
'metadata': {
'description': task['description'],
'name': task['label'],
'owner': config.params['owner'],
'source': '{}/file/{}/{}'.format(
config.params['head_repository'],
config.params['head_rev'],
config.path),
},
'extra': extra,
'tags': {'createdForUser': config.params['owner']},
}
# add the payload and adjust anything else as required (e.g., scopes)
payload_builders[task['worker']['implementation']](config, task, task_def)
attributes = task.get('attributes', {})
attributes['run_on_projects'] = task.get('run-on-projects', ['all'])
yield {
'label': task['label'],
'task': task_def,
'dependencies': task.get('dependencies', {}),
'attributes': attributes,
'when': task.get('when', {}),
}
def build_task(config, tasks):
for task in tasks:
worker_type = task["worker-type"].format(level=str(config.params["level"]))
provisioner_id, worker_type = worker_type.split("/", 1)
routes = task.get("routes", [])
scopes = task.get("scopes", [])
# set up extra
extra = task.get("extra", {})
task_th = task.get("treeherder")
if task_th:
extra["treeherderEnv"] = task_th["environments"]
treeherder = extra.setdefault("treeherder", {})
machine_platform, collection = task_th["platform"].split("/", 1)
treeherder["machine"] = {"platform": machine_platform}
treeherder["collection"] = {collection: True}
groupSymbol, symbol = split_symbol(task_th["symbol"])
if groupSymbol != "?":
treeherder["groupSymbol"] = groupSymbol
if groupSymbol not in GROUP_NAMES:
raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
treeherder["groupName"] = GROUP_NAMES[groupSymbol]
treeherder["symbol"] = symbol
treeherder["jobKind"] = task_th["kind"]
treeherder["tier"] = task_th["tier"]
routes.extend(
[
"{}.v2.{}.{}.{}".format(
TREEHERDER_ROUTE_ROOTS[env],
config.params["project"],
config.params["head_rev"],
config.params["pushlog_id"],
)
for env in task_th["environments"]
]
)
if "expires-after" not in task:
task["expires-after"] = "28 days" if config.params["project"] == "try" else "1 year"
if "deadline-after" not in task:
task["deadline-after"] = "1 day"
if "coalesce-name" in task and int(config.params["level"]) > 1:
key = COALESCE_KEY.format(project=config.params["project"], name=task["coalesce-name"])
routes.append("coalesce.v1." + key)
task_def = {
"provisionerId": provisioner_id,
"workerType": worker_type,
"routes": routes,
"created": {"relative-datestamp": "0 seconds"},
"deadline": {"relative-datestamp": task["deadline-after"]},
"expires": {"relative-datestamp": task["expires-after"]},
"scopes": scopes,
"metadata": {
"description": task["description"],
"name": task["label"],
"owner": config.params["owner"],
"source": "{}/file/{}/{}".format(
config.params["head_repository"], config.params["head_rev"], config.path
),
},
"extra": extra,
"tags": {"createdForUser": config.params["owner"]},
}
# add the payload and adjust anything else as required (e.g., scopes)
payload_builders[task["worker"]["implementation"]](config, task, task_def)
attributes = task.get("attributes", {})
attributes["run_on_projects"] = task.get("run-on-projects", ["all"])
yield {
"label": task["label"],
"task": task_def,
"dependencies": task.get("dependencies", {}),
"attributes": attributes,
"when": task.get("when", {}),
}
def test_split_no_group(self):
self.assertEqual(split_symbol('xy'), ('?', 'xy'))
def test_split_with_group(self):
self.assertEqual(split_symbol('ab(xy)'), ('ab', 'xy'))
def build_task(config, tasks):
for task in tasks:
worker_type = task['worker-type'].format(level=str(config.params['level']))
provisioner_id, worker_type = worker_type.split('/', 1)
routes = task.get('routes', [])
scopes = task.get('scopes', [])
# set up extra
extra = task.get('extra', {})
task_th = task.get('treeherder')
if task_th:
extra['treeherderEnv'] = task_th['environments']
treeherder = extra.setdefault('treeherder', {})
machine_platform, collection = task_th['platform'].split('/', 1)
treeherder['machine'] = {'platform': machine_platform}
treeherder['collection'] = {collection: True}
groupSymbol, symbol = split_symbol(task_th['symbol'])
if groupSymbol != '?':
treeherder['groupSymbol'] = groupSymbol
if groupSymbol not in GROUP_NAMES:
raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
treeherder['groupName'] = GROUP_NAMES[groupSymbol]
treeherder['symbol'] = symbol
treeherder['jobKind'] = task_th['kind']
treeherder['tier'] = task_th['tier']
routes.extend([
'{}.v2.{}.{}.{}'.format(TREEHERDER_ROUTE_ROOTS[env],
config.params['project'],
config.params['head_rev'],
config.params['pushlog_id'])
for env in task_th['environments']
])
if 'expires-after' not in task:
task['expires-after'] = '28 days' if config.params['project'] == 'try' else '1 year'
if 'deadline-after' not in task:
task['deadline-after'] = '1 day'
if 'coalesce-name' in task and int(config.params['level']) > 1:
key = COALESCE_KEY.format(
project=config.params['project'],
name=task['coalesce-name'])
routes.append('coalesce.v1.' + key)
if 'priority' not in task:
task['priority'] = BRANCH_PRIORITIES.get(
config.params['project'],
DEFAULT_BRANCH_PRIORITY)
tags = task.get('tags', {})
tags.update({'createdForUser': config.params['owner']})
task_def = {
'provisionerId': provisioner_id,
'workerType': worker_type,
'routes': routes,
'created': {'relative-datestamp': '0 seconds'},
'deadline': {'relative-datestamp': task['deadline-after']},
'expires': {'relative-datestamp': task['expires-after']},
'scopes': scopes,
'metadata': {
'description': task['description'],
'name': task['label'],
'owner': config.params['owner'],
'source': '{}/file/{}/{}'.format(
config.params['head_repository'],
config.params['head_rev'],
config.path),
},
'extra': extra,
'tags': tags,
'priority': task['priority'],
}
if task_th:
# link back to treeherder in description
th_push_link = 'https://treeherder.mozilla.org/#/jobs?repo={}&revision={}'.format(
config.params['project'], config.params['head_rev'])
task_def['metadata']['description'] += ' ([Treeherder push]({}))'.format(
th_push_link)
# add the payload and adjust anything else as required (e.g., scopes)
payload_builders[task['worker']['implementation']](config, task, task_def)
attributes = task.get('attributes', {})
attributes['run_on_projects'] = task.get('run-on-projects', ['all'])
# Set MOZ_AUTOMATION on all jobs.
if task['worker']['implementation'] in (
'generic-worker',
'docker-engine',
'native-engine',
'docker-worker',
):
payload = task_def.get('payload')
if payload:
env = payload.setdefault('env', {})
env['MOZ_AUTOMATION'] = '1'
yield {
'label': task['label'],
'task': task_def,
'dependencies': task.get('dependencies', {}),
'attributes': attributes,
'optimizations': task.get('optimizations', []),
}
