def split_pageload(config, tests):
# Split test by pageload type (cold, warm)
for test in tests:
mozharness = test.setdefault("mozharness", {})
extra_options = mozharness.setdefault("extra-options", [])
pageload = test.pop("pageload", None)
if not pageload or "--chimera" in extra_options:
yield test
continue
if pageload in ("warm", "both"):
# make a deepcopy if 'both', otherwise use the test object itself
warmtest = deepcopy(test) if pageload == "both" else test
warmtest["warm"] = True
group, symbol = split_symbol(warmtest["treeherder-symbol"])
symbol += "-w"
warmtest["treeherder-symbol"] = join_symbol(group, symbol)
yield warmtest
if pageload in ("cold", "both"):
assert "subtest" in test
test["description"] += " using cold pageload"
test["cold"] = True
test["max-run-time"] = 3000
test["test-name"] += "-cold"
test["try-name"] += "-cold"
group, symbol = split_symbol(test["treeherder-symbol"])
symbol += "-c"
test["treeherder-symbol"] = join_symbol(group, symbol)
yield test
def split_browsertime_page_load_by_url(config, tests):
for test in tests:
# for tests that have 'raptor-subtests' listed, we want to create a separate
# test job for every subtest (i.e. split out each page-load URL into its own job)
subtests = test.pop('raptor-subtests', None)
if not subtests:
yield test
continue
chunk_number = 0
for subtest in subtests:
# create new test job
chunked = deepcopy(test)
# only run the subtest/single URL
chunked['test-name'] += "-{}".format(subtest)
chunked['try-name'] += "-{}".format(subtest)
chunked['raptor-test'] = subtest
# set treeherder symbol and description
chunk_number += 1
group, symbol = split_symbol(test['treeherder-symbol'])
symbol += "-{}".format(chunk_number)
chunked['treeherder-symbol'] = join_symbol(group, symbol)
chunked['description'] += "-{}".format(subtest)
yield chunked
def chunk_locales(config, jobs):
""" Utilizes chunking for l10n stuff """
for job in jobs:
chunks = job.get('chunks')
all_locales = job['attributes']['all_locales']
if chunks:
if chunks > len(all_locales):
# Reduce chunks down to the number of locales
chunks = len(all_locales)
for this_chunk in range(1, chunks + 1):
chunked = copy.deepcopy(job)
chunked['name'] = chunked['name'].replace(
'/', '-{}/'.format(this_chunk), 1)
chunked['mozharness']['options'] = chunked['mozharness'].get(
'options', [])
my_locales = []
my_locales = chunkify(all_locales, this_chunk, chunks)
chunked['mozharness']['options'].extend(
["locale={}".format(locale) for locale in my_locales])
chunked['attributes']['l10n_chunk'] = str(this_chunk)
chunked['attributes']['chunk_locales'] = my_locales
# add the chunk number to the TH symbol
group, symbol = split_symbol(
chunked.get('treeherder', {}).get('symbol', ''))
symbol += str(this_chunk)
chunked['treeherder']['symbol'] = join_symbol(group, symbol)
yield chunked
else:
job['mozharness']['options'] = job['mozharness'].get('options', [])
job['mozharness']['options'].extend(
["locale={}".format(locale) for locale in all_locales])
yield job
def split_variants(config, tests):
for test in tests:
variants = test.pop('variants')
yield copy.deepcopy(test)
for name in variants:
testv = copy.deepcopy(test)
variant = TEST_VARIANTS[name]
if 'filterfn' in variant and not variant['filterfn'](testv):
continue
testv['attributes']['unittest_variant'] = name
testv['description'] = variant['description'].format(**testv)
suffix = '-' + variant['suffix']
testv['test-name'] += suffix
testv['try-name'] += suffix
group, symbol = split_symbol(testv['treeherder-symbol'])
if group != '?':
group += suffix
else:
symbol += suffix
testv['treeherder-symbol'] = join_symbol(group, symbol)
testv.update(variant.get('replace', {}))
yield merge(testv, variant.get('merge', {}))
def split_chunks(config, tests):
"""Based on the 'chunks' key, split tests up into chunks by duplicating
them and assigning 'this-chunk' appropriately and updating the treeherder
symbol."""
for test in tests:
if test['chunks'] == 1:
test['this-chunk'] = 1
yield test
continue
# HACK: Bug 1373578 appears to pass with more chunks, non-e10s only though
if test['test-platform'] == 'windows7-32/debug' and test['test-name'] == 'reftest':
test['chunks'] = 32
if (test['test-platform'] == 'windows7-32/opt' or
test['test-platform'] == 'windows7-32-pgo/opt') and \
test['test-name'] in ['reftest-e10s', 'reftest-no-accel-e10s', 'reftest-gpu-e10s']:
test['chunks'] = 32
for this_chunk in range(1, test['chunks'] + 1):
# copy the test and update with the chunk number
chunked = copy.deepcopy(test)
chunked['this-chunk'] = this_chunk
# add the chunk number to the TH symbol
group, symbol = split_symbol(chunked['treeherder-symbol'])
symbol += str(this_chunk)
chunked['treeherder-symbol'] = join_symbol(group, symbol)
yield chunked
def split_e10s(config, tests):
for test in tests:
e10s = test['e10s']
test['e10s'] = False
test['attributes']['e10s'] = False
if e10s == 'both':
yield copy.deepcopy(test)
e10s = True
if e10s:
test['test-name'] += '-e10s'
test['try-name'] += '-e10s'
test['e10s'] = True
test['attributes']['e10s'] = True
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-e10s'
test['treeherder-symbol'] = join_symbol(group, symbol)
if test['suite'] == 'talos':
for i, option in enumerate(test['mozharness']['extra-options']):
if option.startswith('--suite='):
test['mozharness']['extra-options'][i] += '-e10s'
else:
test['mozharness']['extra-options'].append('--e10s')
yield test
def split_apps(config, tests):
app_symbols = {
'chrome': 'Chr',
'chromium': 'Cr',
'fenix': 'fenix',
'refbrow': 'refbrow',
}
for test in tests:
apps = test.pop('apps', None)
if not apps:
yield test
continue
for app in apps:
atest = deepcopy(test)
suffix = "-{}".format(app)
atest['app'] = app
atest['description'] += " on {}".format(app.capitalize())
name = atest['test-name']
if name.endswith('-cold'):
name = atest['test-name'][:-len('-cold')] + suffix + '-cold'
else:
name += suffix
atest['test-name'] = name
atest['try-name'] = name
if app in app_symbols:
group, symbol = split_symbol(atest['treeherder-symbol'])
group += "-{}".format(app_symbols[app])
atest['treeherder-symbol'] = join_symbol(group, symbol)
yield atest
def split_btime_variants(config, jobs):
for job in jobs:
if job.get("perftest-btime-variants") is None:
yield job
continue
variants = job.pop("perftest-btime-variants")
if not variants:
yield job
continue
yield_existing = False
for suffix, options in variants:
if suffix is None:
# Append options to the existing job
job.setdefault("perftest-btime-variants", []).append(options)
yield_existing = True
else:
job_new = deepcopy(job)
group, symbol = split_symbol(job_new["treeherder"]["symbol"])
symbol += "-" + suffix
job_new["treeherder"]["symbol"] = join_symbol(group, symbol)
job_new["name"] += "-" + suffix
job_new.setdefault("perftest-perfherder-global", {}).setdefault(
"extraOptions", []
).append(suffix)
# Replace the existing options with the new ones
job_new["perftest-btime-variants"] = [options]
yield job_new
# The existing job has been modified so we should also return it
if yield_existing:
yield job
def make_beetmover_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
locale = dep_job.attributes.get("locale")
if not locale:
yield job
continue
group = "BMR"
# add the locale code
symbol = locale
treeherder = {
"symbol": join_symbol(group, symbol),
}
beet_description = {
"label": job["label"],
"primary-dependency": dep_job,
"dependent-tasks": job["dependent-tasks"],
"attributes": job["attributes"],
"treeherder": treeherder,
"locale": locale,
"shipping-phase": job["shipping-phase"],
"shipping-product": job["shipping-product"],
}
yield beet_description
def split_pageload(config, tests):
for test in tests:
pageload = test.pop('pageload', 'warm')
if pageload not in ('cold', 'both'):
yield test
continue
if pageload == 'both':
orig = deepcopy(test)
yield orig
assert 'raptor-test' in test
test['description'] += " using cold pageload"
# for raptor-webext to run cold we just call the corresponding '-cold' test name; but
# for raptor browsertime we leave the raptor test name as/is and will set the '--cold'
# command line argument instead via settting test['cold'] to true
if test['test-name'].startswith('browsertime-tp6'):
test['cold'] = True
else:
test['raptor-test'] += '-cold'
test['max-run-time'] = 3000
test['test-name'] += '-cold'
test['try-name'] += '-cold'
group, symbol = split_symbol(test['treeherder-symbol'])
symbol += '-c'
test['treeherder-symbol'] = join_symbol(group, symbol)
yield test
def split_e10s(config, tests):
for test in tests:
e10s = get_keyed_by(item=test,
field='e10s',
item_name=test['test-name'])
test.setdefault('attributes', {})
test['e10s'] = False
test['attributes']['e10s'] = False
if e10s == 'both':
yield copy.deepcopy(test)
e10s = True
if e10s:
test['test-name'] += '-e10s'
test['e10s'] = True
test['attributes']['e10s'] = True
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-e10s'
test['treeherder-symbol'] = join_symbol(group, symbol)
test['mozharness']['extra-options'] = get_keyed_by(
item=test,
field='mozharness',
subfield='extra-options',
item_name=test['test-name'])
test['mozharness']['extra-options'].append('--e10s')
yield test
def make_beetmover_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
locale = dep_job.attributes.get('locale')
if not locale:
yield job
continue
group = 'BMR-L10n'
# add the locale code
symbol = locale
treeherder = {
'symbol': join_symbol(group, symbol),
}
beet_description = {
'label': job['label'],
'primary-dependency': dep_job,
'dependent-tasks': job['dependent-tasks'],
'attributes': job['attributes'],
'treeherder': treeherder,
'locale': locale,
'shipping-phase': job['shipping-phase'],
'shipping-product': job['shipping-product'],
}
yield beet_description
def split_serviceworker_e10s(config, tests):
for test in tests:
sw = test.pop('serviceworker-e10s')
test['serviceworker-e10s'] = False
test['attributes']['serviceworker_e10s'] = False
if sw == 'both':
yield copy.deepcopy(test)
sw = True
if sw:
if not match_run_on_projects('mozilla-central',
test['run-on-projects']):
continue
test['description'] += " with serviceworker-e10s redesign enabled"
test['run-on-projects'] = ['mozilla-central']
test['test-name'] += '-sw'
test['try-name'] += '-sw'
test['attributes']['serviceworker_e10s'] = True
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-sw'
else:
symbol += '-sw'
test['treeherder-symbol'] = join_symbol(group, symbol)
test['mozharness']['extra-options'].append(
'--setpref="dom.serviceWorkers.parent_intercept=true"')
yield test
def make_beetmover_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
locale = dep_job.attributes.get('locale')
if not locale:
yield job
continue
label = job.get('label',
"beetmover-{}-{}".format(locale, dep_job.label))
group = 'tc-BMR-L10n'
# add the locale code
symbol = locale
treeherder = {
'symbol': join_symbol(group, symbol),
}
beet_description = {
'dependent-task': dep_job,
'treeherder': treeherder,
'label': label,
'locale': locale,
}
yield beet_description
def chunk_locales(config, jobs):
""" Utilizes chunking for l10n stuff """
for job in jobs:
chunks = job.get('chunks')
all_locales = job['attributes']['all_locales']
if chunks:
for this_chunk in range(1, chunks + 1):
chunked = copy.deepcopy(job)
chunked['name'] = chunked['name'].replace(
'/', '-{}/'.format(this_chunk), 1
)
chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])
my_locales = []
my_locales = chunkify(all_locales, this_chunk, chunks)
chunked['mozharness']['options'].extend([
"locale={}".format(locale) for locale in my_locales
])
chunked['attributes']['l10n_chunk'] = str(this_chunk)
chunked['attributes']['chunk_locales'] = my_locales
# add the chunk number to the TH symbol
group, symbol = split_symbol(
chunked.get('treeherder', {}).get('symbol', ''))
symbol += str(this_chunk)
chunked['treeherder']['symbol'] = join_symbol(group, symbol)
yield chunked
else:
job['mozharness']['options'] = job['mozharness'].get('options', [])
job['mozharness']['options'].extend([
"locale={}".format(locale) for locale in all_locales
])
yield job
def split_jsshell(config, jobs):
all_shells = {"sm": "Spidermonkey", "v8": "Google V8"}
for job in jobs:
if not job["name"].startswith("jsshell"):
yield job
continue
test = job.pop("test")
for shell in job.get("shell", all_shells.keys()):
assert shell in all_shells
new_job = copy.deepcopy(job)
new_job["name"] = "{}-{}".format(new_job["name"], shell)
new_job["description"] = "{} on {}".format(
new_job["description"], all_shells[shell]
)
new_job["shell"] = shell
group = "js-bench-{}".format(shell)
symbol = split_symbol(new_job["treeherder"]["symbol"])[1]
new_job["treeherder"]["symbol"] = join_symbol(group, symbol)
run = new_job["run"]
run["mach"] = run["mach"].format(
shell=shell, SHELL=shell.upper(), test=test
)
yield new_job
def split_e10s(config, tests):
for test in tests:
e10s = test['e10s']
test['e10s'] = False
test['attributes']['e10s'] = False
if e10s == 'both':
yield copy.deepcopy(test)
e10s = True
if e10s:
test['test-name'] += '-e10s'
test['try-name'] += '-e10s'
test['e10s'] = True
test['attributes']['e10s'] = True
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-e10s'
test['treeherder-symbol'] = join_symbol(group, symbol)
if test['suite'] == 'talos':
for i, option in enumerate(
test['mozharness']['extra-options']):
if option.startswith('--suite='):
test['mozharness']['extra-options'][i] += '-e10s'
else:
test['mozharness']['extra-options'].append('--e10s')
yield test
def split_chunks(config, tests):
"""Based on the 'chunks' key, split tests up into chunks by duplicating
them and assigning 'this-chunk' appropriately and updating the treeherder
symbol."""
for test in tests:
if test['suite'].startswith('test-verify'):
test['chunks'] = perfile_number_of_chunks(config, test['test-name'])
if test['chunks'] == 0:
continue
# limit the number of chunks we run for test-verify mode because
# test-verify is comprehensive and takes a lot of time, if we have
# >30 tests changed, this is probably an import of external tests,
# or a patch renaming/moving files in bulk
maximum_number_verify_chunks = 3
if test['chunks'] > maximum_number_verify_chunks:
test['chunks'] = maximum_number_verify_chunks
if test['chunks'] == 1:
test['this-chunk'] = 1
yield test
continue
for this_chunk in range(1, test['chunks'] + 1):
# copy the test and update with the chunk number
chunked = copy.deepcopy(test)
chunked['this-chunk'] = this_chunk
# add the chunk number to the TH symbol
group, symbol = split_symbol(chunked['treeherder-symbol'])
symbol += str(this_chunk)
chunked['treeherder-symbol'] = join_symbol(group, symbol)
yield chunked
def split_socketprocess_e10s(config, tests):
for test in tests:
if test['attributes'].get('serviceworker_e10s'):
yield test
continue
sw = test.pop('socketprocess-e10s')
test['socketprocess-e10s'] = False
test['attributes']['socketprocess_e10s'] = False
if sw == 'both':
yield copy.deepcopy(test)
sw = True
if sw:
test['description'] += " with socket process enabled"
test['test-name'] += '-spi'
test['try-name'] += '-spi'
test['attributes']['socketprocess_e10s'] = True
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-spi'
else:
symbol += '-spi'
test['treeherder-symbol'] = join_symbol(group, symbol)
test['mozharness']['extra-options'].append(
'--setpref="media.peerconnection.mtransport_process=true"')
test['mozharness']['extra-options'].append(
'--setpref="network.process.enabled=true"')
yield test
def split_jsshell(config, jobs):
all_shells = {'sm': "Spidermonkey", 'v8': "Google V8"}
for job in jobs:
if not job['name'].startswith('jsshell'):
yield job
continue
test = job.pop('test')
for shell in job.get('shell', all_shells.keys()):
assert shell in all_shells
new_job = copy.deepcopy(job)
new_job['name'] = '{}-{}'.format(new_job['name'], shell)
new_job['description'] = '{} on {}'.format(new_job['description'],
all_shells[shell])
new_job['shell'] = shell
group = 'js-bench-{}'.format(shell)
symbol = split_symbol(new_job['treeherder']['symbol'])[1]
new_job['treeherder']['symbol'] = join_symbol(group, symbol)
run = new_job['run']
run['mach'] = run['mach'].format(shell=shell,
SHELL=shell.upper(),
test=test)
yield new_job
def split_chunks(config, tests):
"""Based on the 'chunks' key, split tests up into chunks by duplicating
them and assigning 'this-chunk' appropriately and updating the treeherder
symbol."""
for test in tests:
if test['chunks'] == 1:
test['this-chunk'] = 1
yield test
continue
# HACK: Bug 1373578 appears to pass with more chunks, non-e10s only though
if test['test-platform'] == 'windows7-32/debug' and test[
'test-name'] == 'reftest':
test['chunks'] = 32
if (test['test-platform'] == 'windows7-32/opt' or
test['test-platform'] == 'windows7-32-pgo/opt') and \
test['test-name'] in ['reftest-e10s', 'reftest-no-accel-e10s', 'reftest-gpu-e10s']:
test['chunks'] = 32
for this_chunk in range(1, test['chunks'] + 1):
# copy the test and update with the chunk number
chunked = copy.deepcopy(test)
chunked['this-chunk'] = this_chunk
# add the chunk number to the TH symbol
group, symbol = split_symbol(chunked['treeherder-symbol'])
symbol += str(this_chunk)
chunked['treeherder-symbol'] = join_symbol(group, symbol)
yield chunked
def split_pageload(config, tests):
# Split test by pageload type (cold, warm)
for test in tests:
pageload = test.pop('pageload', 'warm')
if pageload not in ('cold', 'both'):
yield test
continue
if pageload == 'both':
orig = deepcopy(test)
yield orig
assert 'subtest' in test
test['description'] += " using cold pageload"
test['cold'] = True
test['max-run-time'] = 3000
test['test-name'] += '-cold'
test['try-name'] += '-cold'
group, symbol = split_symbol(test['treeherder-symbol'])
symbol += '-c'
test['treeherder-symbol'] = join_symbol(group, symbol)
yield test
def make_beetmover_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
locale = dep_job.attributes.get('locale')
if not locale:
yield job
continue
group = 'BMR-L10n'
# add the locale code
symbol = locale
treeherder = {
'symbol': join_symbol(group, symbol),
}
beet_description = {
'label': job['label'],
'dependent-task': dep_job,
'treeherder': treeherder,
'locale': locale,
'shipping-phase': job.get('shipping-phase'),
'shipping-product': job.get('shipping-product'),
}
yield beet_description
def split_apps(config, tests):
app_symbols = {
"chrome": "ChR",
"chrome-m": "ChR",
"chromium": "Cr",
"fenix": "fenix",
"refbrow": "refbrow",
}
for test in tests:
apps = test.pop("apps", None)
if not apps:
yield test
continue
for app in apps:
atest = deepcopy(test)
suffix = "-{}".format(app)
atest["app"] = app
atest["description"] += " on {}".format(app.capitalize())
name = atest["test-name"] + suffix
atest["test-name"] = name
atest["try-name"] = name
if app in app_symbols:
group, symbol = split_symbol(atest["treeherder-symbol"])
group += "-{}".format(app_symbols[app])
atest["treeherder-symbol"] = join_symbol(group, symbol)
yield atest
def make_beetmover_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
for locale in dep_job.attributes.get('chunk_locales', []):
label = job.get('label',
"beetmover-{}-{}".format(locale, dep_job.label))
group = 'tc-BM-L10n'
# add the locale code
if 'signing' in label:
symbol = 'S{}'.format(locale)
else:
symbol = locale
treeherder = {
'symbol': join_symbol(group, symbol),
}
beet_description = {
'dependent-task': dep_job,
'treeherder': treeherder,
'label': label,
'locale': locale,
}
yield beet_description
def build_treeherder_definition(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task["treeherder"] = inherit_treeherder_from_dep(task, dep)
treeherder_group = dep.task["extra"]["treeherder"]["groupSymbol"]
treeherder_symbol = join_symbol(treeherder_group, 'gp')
task["treeherder"]["symbol"] = treeherder_symbol
yield task
def fill_template(config, tasks):
for task in tasks:
dep = task['primary-dependency']
# Fill out the dynamic fields in the task description
task['label'] = dep.label + '-upload-symbols'
# Skip tasks where we don't have the full crashsymbols enabled
if not dep.attributes.get('enable-full-crashsymbols'):
logger.debug("Skipping upload symbols task for %s", task['label'])
continue
task['dependencies'] = {'build': dep.label}
task['worker']['env']['GECKO_HEAD_REPOSITORY'] = config.params[
'head_repository']
task['worker']['env']['GECKO_HEAD_REV'] = config.params['head_rev']
task['worker']['env']['SYMBOL_SECRET'] = task['worker']['env'][
'SYMBOL_SECRET'].format(level=config.params['level'])
attributes = copy_attributes_from_dependent_job(dep)
attributes.update(task.get('attributes', {}))
task['attributes'] = attributes
build_type = attributes.get('build_type')
treeherder = task.get('treeherder', {})
th = dep.task.get('extra')['treeherder']
th_platform = dep.task['extra'].get(
'treeherder-platform', "{}/{}".format(th['machine']['platform'],
build_type))
th_symbol = th.get('symbol')
th_groupsymbol = th.get('groupSymbol', '?')
treeherder.setdefault('platform', th_platform)
treeherder.setdefault('tier', th['tier'])
treeherder.setdefault('kind', th['jobKind'])
# Disambiguate the treeherder symbol.
sym = 'Sym' + (th_symbol[1:]
if th_symbol.startswith('B') else th_symbol)
treeherder.setdefault('symbol', join_symbol(th_groupsymbol, sym))
task['treeherder'] = treeherder
if dep.attributes.get('nightly'):
# For nightly builds, we want to run these tasks if the build is run.
task['run-on-projects'] = dep.attributes.get('run_on_projects')
elif dep.attributes.get('shippable'):
# For shippable builds, we want to run these tasks if the build is run.
# XXX Better to run this on promote phase instead?
task['run-on-projects'] = dep.attributes.get('run_on_projects')
else:
# For other builds, these can be requested to upload to the try symbol sever.
task['run-on-projects'] = ['try']
# clear out the stuff that's not part of a task description
del task['primary-dependency']
yield task
def build_treeherder_definition(config, tasks):
for task in tasks:
dep = task.pop("primary-dependency")
task.setdefault("treeherder", {}).update(inherit_treeherder_from_dep(task, dep))
job_group = dep.task["extra"]["treeherder"].get("groupSymbol", "?")
job_symbol = task["treeherder"].pop("job-symbol")
full_symbol = join_symbol(job_group, job_symbol)
task["treeherder"]["symbol"] = full_symbol
yield task
def yield_job(orig_job, deps, count):
job = deepcopy(orig_job)
job['dependencies'] = deps
job['name'] = "{}-{}".format(orig_job['name'], count)
if 'treeherder' in job:
groupSymbol, symbol = split_symbol(job['treeherder']['symbol'])
symbol += '-'
symbol += str(count)
job['treeherder']['symbol'] = join_symbol(groupSymbol, symbol)
return job
def make_signing_description(config, jobs):
for job in jobs:
job['depname'] = 'unsigned-repack'
dep_job = job['dependent-task']
dep_platform = dep_job.attributes.get('build_platform')
job['upstream-artifacts'] = []
if 'android' in dep_platform:
job_specs = [
{
'artifacts': ['public/build/{locale}/target.apk'],
'format': 'jar',
},
]
else:
job_specs = [
{
'artifacts': ['public/build/{locale}/target.tar.bz2',
'public/build/{locale}/target.checksums'],
'format': 'gpg',
}, {
'artifacts': ['public/build/{locale}/target.complete.mar'],
'format': 'mar',
}
]
upstream_artifacts = []
for spec in job_specs:
fmt = spec['format']
upstream_artifacts.append({
"taskId": {"task-reference": "<unsigned-repack>"},
"taskType": "l10n",
# Set paths based on artifacts in the specs (above) one per
# locale present in the chunk this is signing stuff for.
"paths": [f.format(locale=l)
for l in dep_job.attributes.get('chunk_locales', [])
for f in spec['artifacts']],
"formats": [fmt]
})
job['upstream-artifacts'] = upstream_artifacts
label = dep_job.label.replace("nightly-l10n-", "signing-l10n-")
job['label'] = label
# add the chunk number to the TH symbol
symbol = 'Ns{}'.format(dep_job.attributes.get('l10n_chunk'))
group = 'tc-L10n'
job['treeherder'] = {
'symbol': join_symbol(group, symbol),
}
yield job
def make_signing_description(config, jobs):
for job in jobs:
job['depname'] = 'unsigned-repack'
dep_job = job['dependent-task']
# add the chunk number to the TH symbol
symbol = 'Ns{}'.format(dep_job.attributes.get('l10n_chunk'))
group = 'L10n'
job['treeherder'] = {
'symbol': join_symbol(group, symbol),
}
yield job
def tests_drop_1proc(config, jobs):
"""
Remove the -1proc suffix from Treeherder group symbols.
Restore the -e10s suffix (because some day we will have them!)
Reverses the effects of bug 1541527. Thunderbird builds are all single
process.
"""
for job in jobs:
test = job["run"]["test"]
e10s = test["e10s"]
if not e10s: # test-name & friends end with '-1proc'
test["test-name"] = _remove_suffix(test["test-name"], "-1proc")
test["try-name"] = _remove_suffix(test["try-name"], "-1proc")
group, symbol = split_symbol(test["treeherder-symbol"])
if group != "?":
group = _remove_suffix(group, "-1proc")
test["treeherder-symbol"] = join_symbol(group, symbol)
job["label"] = job["label"].replace("-1proc", "")
job["name"] = _remove_suffix(job["name"], "-1proc")
job["treeherder"]["symbol"] = test["treeherder-symbol"]
else: # e10s in the future
test["test-name"] = add_suffix(test["test-name"], "-e10s")
test["try-name"] = add_suffix(test["try-name"], "-e10s")
group, symbol = split_symbol(test["treeherder-symbol"])
if group != "?":
group = add_suffix(group, "-e10s")
test["treeherder-symbol"] = join_symbol(group, symbol)
job["label"] += "-e10s"
job["name"] = add_suffix(job["name"], "-e10s")
job["treeherder"]["symbol"] = test["treeherder-symbol"]
yield job
def make_signing_description(config, jobs):
for job in jobs:
job['depname'] = 'unsigned-repack'
dep_job = job['dependent-task']
# add the chunk number to the TH symbol
symbol = 'Ns{}'.format(dep_job.attributes.get('l10n_chunk'))
group = 'L10n'
job['treeherder'] = {
'symbol': join_symbol(group, symbol),
}
yield job
def tests_drop_1proc(config, jobs):
"""
Remove the -1proc suffix from Treeherder group symbols.
Restore the -e10s suffix (because some day we will have them!)
Reverses the effects of bug 1541527. Thunderbird builds are all single
process.
"""
for job in jobs:
test = job['run']['test']
e10s = test['e10s']
if not e10s: # test-name & friends end with '-1proc'
test['test-name'] = _remove_suffix(test['test-name'], '-1proc')
test['try-name'] = _remove_suffix(test['try-name'], '-1proc')
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group = _remove_suffix(group, '-1proc')
test['treeherder-symbol'] = join_symbol(group, symbol)
job['label'] = job['label'].replace('-1proc', '')
job['name'] = _remove_suffix(job['name'], '-1proc')
job['treeherder']['symbol'] = test['treeherder-symbol']
else: # e10s in the future
test['test-name'] = add_suffix(test['test-name'], '-e10s')
test['try-name'] = add_suffix(test['try-name'], '-e10s')
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group = add_suffix(group, '-e10s')
test['treeherder-symbol'] = join_symbol(group, symbol)
job['label'] += '-e10s'
job['name'] = add_suffix(job['name'], '-e10s')
job['treeherder']['symbol'] = test['treeherder-symbol']
yield job
def fill_template(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task.pop("dependent-tasks", None)
# Fill out the dynamic fields in the task description
task["label"] = dep.label + "-upload-symbols"
# Skip tasks where we don't have the full crashsymbols enabled
if not dep.attributes.get("enable-full-crashsymbols") or dep.attributes.get(
"skip-upload-crashsymbols"
):
logger.debug("Skipping upload symbols task for %s", task["label"])
continue
task["dependencies"] = {"build": dep.label}
task["worker"]["env"]["GECKO_HEAD_REPOSITORY"] = config.params[
"head_repository"
]
task["worker"]["env"]["GECKO_HEAD_REV"] = config.params["head_rev"]
task["worker"]["env"]["SYMBOL_SECRET"] = task["worker"]["env"][
"SYMBOL_SECRET"
].format(level=config.params["level"])
attributes = copy_attributes_from_dependent_job(dep)
attributes.update(task.get("attributes", {}))
task["attributes"] = attributes
treeherder = inherit_treeherder_from_dep(task, dep)
th = dep.task.get("extra")["treeherder"]
th_symbol = th.get("symbol")
th_groupsymbol = th.get("groupSymbol", "?")
# Disambiguate the treeherder symbol.
sym = "Sym" + (th_symbol[1:] if th_symbol.startswith("B") else th_symbol)
treeherder.setdefault("symbol", join_symbol(th_groupsymbol, sym))
task["treeherder"] = treeherder
# We only want to run these tasks if the build is run.
# XXX Better to run this on promote phase instead?
task["run-on-projects"] = dep.attributes.get("run_on_projects")
task["optimization"] = {"upload-symbols": None}
task["if-dependencies"] = ["build"]
# clear out the stuff that's not part of a task description
del task["primary-dependency"]
yield task
def fill_template(config, tasks):
for task in tasks:
dep = task['dependent-task']
# Fill out the dynamic fields in the task description
task['label'] = dep.label + '-upload-symbols'
# Skip tasks where we don't have the full crashsymbols enabled
if not dep.attributes.get('enable-full-crashsymbols'):
logger.debug("Skipping upload symbols task for %s", task['label'])
continue
task['dependencies'] = {'build': dep.label}
task['worker']['env']['GECKO_HEAD_REPOSITORY'] = config.params[
'head_repository']
task['worker']['env']['GECKO_HEAD_REV'] = config.params['head_rev']
task['worker']['env']['SYMBOL_SECRET'] = task['worker']['env'][
'SYMBOL_SECRET'].format(level=config.params['level'])
build_platform = dep.attributes.get('build_platform')
build_type = dep.attributes.get('build_type')
attributes = task.setdefault('attributes', {})
attributes['build_platform'] = build_platform
attributes['build_type'] = build_type
if dep.attributes.get('nightly'):
attributes['nightly'] = True
treeherder = task.get('treeherder', {})
th = dep.task.get('extra')['treeherder']
th_platform = dep.task['extra'].get(
'treeherder-platform', "{}/{}".format(th['machine']['platform'],
build_type))
th_symbol = th.get('symbol')
th_groupsymbol = th.get('groupSymbol', '?')
treeherder.setdefault('platform', th_platform)
treeherder.setdefault('tier', th['tier'])
treeherder.setdefault('kind', th['jobKind'])
# Disambiguate the treeherder symbol.
sym = 'Sym' + (th_symbol[1:]
if th_symbol.startswith('B') else th_symbol)
treeherder.setdefault('symbol', join_symbol(th_groupsymbol, sym))
task['treeherder'] = treeherder
# clear out the stuff that's not part of a task description
del task['dependent-task']
yield task
def make_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
job['depname'] = dep_job.label
# add the chunk number to the TH symbol
symbol = job.get('treeherder', {}).get('symbol', 'Bs')
symbol = '{}{}'.format(symbol, dep_job.attributes.get('l10n_chunk'))
group = 'L10n'
job['treeherder'] = {
'symbol': join_symbol(group, symbol),
}
yield job
def make_beetmover_description(config, jobs):
for job in jobs:
dep_job = job['dependent-task']
for locale in dep_job.attributes.get('chunk_locales', []):
group = 'BM-L10n'
# add the locale code
symbol = locale
treeherder = {
'symbol': join_symbol(group, symbol),
}
beet_description = {
'dependent-task': dep_job,
'treeherder': treeherder,
'locale': locale,
}
yield beet_description
def split_chunks(config, tests):
"""Based on the 'chunks' key, split tests up into chunks by duplicating
them and assigning 'this-chunk' appropriately and updating the treeherder
symbol."""
for test in tests:
if test['chunks'] == 1:
test['this-chunk'] = 1
yield test
continue
for this_chunk in range(1, test['chunks'] + 1):
# copy the test and update with the chunk number
chunked = copy.deepcopy(test)
chunked['this-chunk'] = this_chunk
# add the chunk number to the TH symbol
group, symbol = split_symbol(chunked['treeherder-symbol'])
symbol += str(this_chunk)
chunked['treeherder-symbol'] = join_symbol(group, symbol)
yield chunked
def split_e10s(config, tests):
for test in tests:
e10s = test['e10s']
test.setdefault('attributes', {})
test['e10s'] = False
test['attributes']['e10s'] = False
if e10s == 'both':
yield copy.deepcopy(test)
e10s = True
if e10s:
test['test-name'] += '-e10s'
test['e10s'] = True
test['attributes']['e10s'] = True
group, symbol = split_symbol(test['treeherder-symbol'])
if group != '?':
group += '-e10s'
test['treeherder-symbol'] = join_symbol(group, symbol)
test['mozharness']['extra-options'].append('--e10s')
yield test
def split_e10s(config, tests):
for test in tests:
e10s = get_keyed_by(item=test, field="e10s", item_name=test["test-name"])
test.setdefault("attributes", {})
test["e10s"] = False
test["attributes"]["e10s"] = False
if e10s == "both":
yield test
test = copy.deepcopy(test)
e10s = True
if e10s:
test["test-name"] += "-e10s"
test["e10s"] = True
test["attributes"]["e10s"] = True
group, symbol = split_symbol(test["treeherder-symbol"])
if group != "?":
group += "-e10s"
test["treeherder-symbol"] = join_symbol(group, symbol)
test["mozharness"].setdefault("extra-options", []).append("--e10s")
yield test
def chunk_locales(config, jobs):
""" Utilizes chunking for l10n stuff """
for job in jobs:
chunks = job.get('chunks')
locales_with_changesets = job['attributes']['all_locales_with_changesets']
if chunks:
if chunks > len(locales_with_changesets):
# Reduce chunks down to the number of locales
chunks = len(locales_with_changesets)
for this_chunk in range(1, chunks + 1):
chunked = copy.deepcopy(job)
chunked['name'] = chunked['name'].replace(
'/', '-{}/'.format(this_chunk), 1
)
chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])
# chunkify doesn't work with dicts
locales_with_changesets_as_list = locales_with_changesets.items()
chunked_locales = chunkify(locales_with_changesets_as_list, this_chunk, chunks)
chunked['mozharness']['options'].extend([
'locale={}:{}'.format(locale, changeset)
for locale, changeset in chunked_locales
])
chunked['attributes']['l10n_chunk'] = str(this_chunk)
# strip revision
chunked['attributes']['chunk_locales'] = [locale for locale, _ in chunked_locales]
# add the chunk number to the TH symbol
group, symbol = split_symbol(
chunked.get('treeherder', {}).get('symbol', ''))
symbol += str(this_chunk)
chunked['treeherder']['symbol'] = join_symbol(group, symbol)
yield chunked
else:
job['mozharness']['options'] = job['mozharness'].get('options', [])
job['mozharness']['options'].extend([
'locale={}:{}'.format(locale, changeset)
for locale, changeset in locales_with_changesets.items()
])
yield job
def test_join_no_group(self):
self.assertEqual(join_symbol('?', 'xy'), 'xy')
def test_join_with_group(self):
self.assertEqual(join_symbol('ab', 'xy'), 'ab(xy)')
