def split_pageload(config, tests): # Split test by pageload type (cold, warm) for test in tests: mozharness = test.setdefault("mozharness", {}) extra_options = mozharness.setdefault("extra-options", []) pageload = test.pop("pageload", None) if not pageload or "--chimera" in extra_options: yield test continue if pageload in ("warm", "both"): # make a deepcopy if 'both', otherwise use the test object itself warmtest = deepcopy(test) if pageload == "both" else test warmtest["warm"] = True group, symbol = split_symbol(warmtest["treeherder-symbol"]) symbol += "-w" warmtest["treeherder-symbol"] = join_symbol(group, symbol) yield warmtest if pageload in ("cold", "both"): assert "subtest" in test test["description"] += " using cold pageload" test["cold"] = True test["max-run-time"] = 3000 test["test-name"] += "-cold" test["try-name"] += "-cold" group, symbol = split_symbol(test["treeherder-symbol"]) symbol += "-c" test["treeherder-symbol"] = join_symbol(group, symbol) yield test
def split_browsertime_page_load_by_url(config, tests): for test in tests: # for tests that have 'raptor-subtests' listed, we want to create a separate # test job for every subtest (i.e. split out each page-load URL into its own job) subtests = test.pop('raptor-subtests', None) if not subtests: yield test continue chunk_number = 0 for subtest in subtests: # create new test job chunked = deepcopy(test) # only run the subtest/single URL chunked['test-name'] += "-{}".format(subtest) chunked['try-name'] += "-{}".format(subtest) chunked['raptor-test'] = subtest # set treeherder symbol and description chunk_number += 1 group, symbol = split_symbol(test['treeherder-symbol']) symbol += "-{}".format(chunk_number) chunked['treeherder-symbol'] = join_symbol(group, symbol) chunked['description'] += "-{}".format(subtest) yield chunked
def chunk_locales(config, jobs): """ Utilizes chunking for l10n stuff """ for job in jobs: chunks = job.get('chunks') all_locales = job['attributes']['all_locales'] if chunks: if chunks > len(all_locales): # Reduce chunks down to the number of locales chunks = len(all_locales) for this_chunk in range(1, chunks + 1): chunked = copy.deepcopy(job) chunked['name'] = chunked['name'].replace( '/', '-{}/'.format(this_chunk), 1) chunked['mozharness']['options'] = chunked['mozharness'].get( 'options', []) my_locales = [] my_locales = chunkify(all_locales, this_chunk, chunks) chunked['mozharness']['options'].extend( ["locale={}".format(locale) for locale in my_locales]) chunked['attributes']['l10n_chunk'] = str(this_chunk) chunked['attributes']['chunk_locales'] = my_locales # add the chunk number to the TH symbol group, symbol = split_symbol( chunked.get('treeherder', {}).get('symbol', '')) symbol += str(this_chunk) chunked['treeherder']['symbol'] = join_symbol(group, symbol) yield chunked else: job['mozharness']['options'] = job['mozharness'].get('options', []) job['mozharness']['options'].extend( ["locale={}".format(locale) for locale in all_locales]) yield job
def split_variants(config, tests): for test in tests: variants = test.pop('variants') yield copy.deepcopy(test) for name in variants: testv = copy.deepcopy(test) variant = TEST_VARIANTS[name] if 'filterfn' in variant and not variant['filterfn'](testv): continue testv['attributes']['unittest_variant'] = name testv['description'] = variant['description'].format(**testv) suffix = '-' + variant['suffix'] testv['test-name'] += suffix testv['try-name'] += suffix group, symbol = split_symbol(testv['treeherder-symbol']) if group != '?': group += suffix else: symbol += suffix testv['treeherder-symbol'] = join_symbol(group, symbol) testv.update(variant.get('replace', {})) yield merge(testv, variant.get('merge', {}))
def split_chunks(config, tests): """Based on the 'chunks' key, split tests up into chunks by duplicating them and assigning 'this-chunk' appropriately and updating the treeherder symbol.""" for test in tests: if test['chunks'] == 1: test['this-chunk'] = 1 yield test continue # HACK: Bug 1373578 appears to pass with more chunks, non-e10s only though if test['test-platform'] == 'windows7-32/debug' and test['test-name'] == 'reftest': test['chunks'] = 32 if (test['test-platform'] == 'windows7-32/opt' or test['test-platform'] == 'windows7-32-pgo/opt') and \ test['test-name'] in ['reftest-e10s', 'reftest-no-accel-e10s', 'reftest-gpu-e10s']: test['chunks'] = 32 for this_chunk in range(1, test['chunks'] + 1): # copy the test and update with the chunk number chunked = copy.deepcopy(test) chunked['this-chunk'] = this_chunk # add the chunk number to the TH symbol group, symbol = split_symbol(chunked['treeherder-symbol']) symbol += str(this_chunk) chunked['treeherder-symbol'] = join_symbol(group, symbol) yield chunked
def split_e10s(config, tests): for test in tests: e10s = test['e10s'] test['e10s'] = False test['attributes']['e10s'] = False if e10s == 'both': yield copy.deepcopy(test) e10s = True if e10s: test['test-name'] += '-e10s' test['try-name'] += '-e10s' test['e10s'] = True test['attributes']['e10s'] = True group, symbol = split_symbol(test['treeherder-symbol']) if group != '?': group += '-e10s' test['treeherder-symbol'] = join_symbol(group, symbol) if test['suite'] == 'talos': for i, option in enumerate(test['mozharness']['extra-options']): if option.startswith('--suite='): test['mozharness']['extra-options'][i] += '-e10s' else: test['mozharness']['extra-options'].append('--e10s') yield test
def split_apps(config, tests): app_symbols = { 'chrome': 'Chr', 'chromium': 'Cr', 'fenix': 'fenix', 'refbrow': 'refbrow', } for test in tests: apps = test.pop('apps', None) if not apps: yield test continue for app in apps: atest = deepcopy(test) suffix = "-{}".format(app) atest['app'] = app atest['description'] += " on {}".format(app.capitalize()) name = atest['test-name'] if name.endswith('-cold'): name = atest['test-name'][:-len('-cold')] + suffix + '-cold' else: name += suffix atest['test-name'] = name atest['try-name'] = name if app in app_symbols: group, symbol = split_symbol(atest['treeherder-symbol']) group += "-{}".format(app_symbols[app]) atest['treeherder-symbol'] = join_symbol(group, symbol) yield atest
def split_btime_variants(config, jobs): for job in jobs: if job.get("perftest-btime-variants") is None: yield job continue variants = job.pop("perftest-btime-variants") if not variants: yield job continue yield_existing = False for suffix, options in variants: if suffix is None: # Append options to the existing job job.setdefault("perftest-btime-variants", []).append(options) yield_existing = True else: job_new = deepcopy(job) group, symbol = split_symbol(job_new["treeherder"]["symbol"]) symbol += "-" + suffix job_new["treeherder"]["symbol"] = join_symbol(group, symbol) job_new["name"] += "-" + suffix job_new.setdefault("perftest-perfherder-global", {}).setdefault( "extraOptions", [] ).append(suffix) # Replace the existing options with the new ones job_new["perftest-btime-variants"] = [options] yield job_new # The existing job has been modified so we should also return it if yield_existing: yield job
def make_beetmover_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] locale = dep_job.attributes.get("locale") if not locale: yield job continue group = "BMR" # add the locale code symbol = locale treeherder = { "symbol": join_symbol(group, symbol), } beet_description = { "label": job["label"], "primary-dependency": dep_job, "dependent-tasks": job["dependent-tasks"], "attributes": job["attributes"], "treeherder": treeherder, "locale": locale, "shipping-phase": job["shipping-phase"], "shipping-product": job["shipping-product"], } yield beet_description
def split_pageload(config, tests): for test in tests: pageload = test.pop('pageload', 'warm') if pageload not in ('cold', 'both'): yield test continue if pageload == 'both': orig = deepcopy(test) yield orig assert 'raptor-test' in test test['description'] += " using cold pageload" # for raptor-webext to run cold we just call the corresponding '-cold' test name; but # for raptor browsertime we leave the raptor test name as/is and will set the '--cold' # command line argument instead via settting test['cold'] to true if test['test-name'].startswith('browsertime-tp6'): test['cold'] = True else: test['raptor-test'] += '-cold' test['max-run-time'] = 3000 test['test-name'] += '-cold' test['try-name'] += '-cold' group, symbol = split_symbol(test['treeherder-symbol']) symbol += '-c' test['treeherder-symbol'] = join_symbol(group, symbol) yield test
def split_e10s(config, tests): for test in tests: e10s = get_keyed_by(item=test, field='e10s', item_name=test['test-name']) test.setdefault('attributes', {}) test['e10s'] = False test['attributes']['e10s'] = False if e10s == 'both': yield copy.deepcopy(test) e10s = True if e10s: test['test-name'] += '-e10s' test['e10s'] = True test['attributes']['e10s'] = True group, symbol = split_symbol(test['treeherder-symbol']) if group != '?': group += '-e10s' test['treeherder-symbol'] = join_symbol(group, symbol) test['mozharness']['extra-options'] = get_keyed_by( item=test, field='mozharness', subfield='extra-options', item_name=test['test-name']) test['mozharness']['extra-options'].append('--e10s') yield test
def make_beetmover_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] locale = dep_job.attributes.get('locale') if not locale: yield job continue group = 'BMR-L10n' # add the locale code symbol = locale treeherder = { 'symbol': join_symbol(group, symbol), } beet_description = { 'label': job['label'], 'primary-dependency': dep_job, 'dependent-tasks': job['dependent-tasks'], 'attributes': job['attributes'], 'treeherder': treeherder, 'locale': locale, 'shipping-phase': job['shipping-phase'], 'shipping-product': job['shipping-product'], } yield beet_description
def split_serviceworker_e10s(config, tests): for test in tests: sw = test.pop('serviceworker-e10s') test['serviceworker-e10s'] = False test['attributes']['serviceworker_e10s'] = False if sw == 'both': yield copy.deepcopy(test) sw = True if sw: if not match_run_on_projects('mozilla-central', test['run-on-projects']): continue test['description'] += " with serviceworker-e10s redesign enabled" test['run-on-projects'] = ['mozilla-central'] test['test-name'] += '-sw' test['try-name'] += '-sw' test['attributes']['serviceworker_e10s'] = True group, symbol = split_symbol(test['treeherder-symbol']) if group != '?': group += '-sw' else: symbol += '-sw' test['treeherder-symbol'] = join_symbol(group, symbol) test['mozharness']['extra-options'].append( '--setpref="dom.serviceWorkers.parent_intercept=true"') yield test
def make_beetmover_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] locale = dep_job.attributes.get('locale') if not locale: yield job continue label = job.get('label', "beetmover-{}-{}".format(locale, dep_job.label)) group = 'tc-BMR-L10n' # add the locale code symbol = locale treeherder = { 'symbol': join_symbol(group, symbol), } beet_description = { 'dependent-task': dep_job, 'treeherder': treeherder, 'label': label, 'locale': locale, } yield beet_description
def chunk_locales(config, jobs): """ Utilizes chunking for l10n stuff """ for job in jobs: chunks = job.get('chunks') all_locales = job['attributes']['all_locales'] if chunks: for this_chunk in range(1, chunks + 1): chunked = copy.deepcopy(job) chunked['name'] = chunked['name'].replace( '/', '-{}/'.format(this_chunk), 1 ) chunked['mozharness']['options'] = chunked['mozharness'].get('options', []) my_locales = [] my_locales = chunkify(all_locales, this_chunk, chunks) chunked['mozharness']['options'].extend([ "locale={}".format(locale) for locale in my_locales ]) chunked['attributes']['l10n_chunk'] = str(this_chunk) chunked['attributes']['chunk_locales'] = my_locales # add the chunk number to the TH symbol group, symbol = split_symbol( chunked.get('treeherder', {}).get('symbol', '')) symbol += str(this_chunk) chunked['treeherder']['symbol'] = join_symbol(group, symbol) yield chunked else: job['mozharness']['options'] = job['mozharness'].get('options', []) job['mozharness']['options'].extend([ "locale={}".format(locale) for locale in all_locales ]) yield job
def split_jsshell(config, jobs): all_shells = {"sm": "Spidermonkey", "v8": "Google V8"} for job in jobs: if not job["name"].startswith("jsshell"): yield job continue test = job.pop("test") for shell in job.get("shell", all_shells.keys()): assert shell in all_shells new_job = copy.deepcopy(job) new_job["name"] = "{}-{}".format(new_job["name"], shell) new_job["description"] = "{} on {}".format( new_job["description"], all_shells[shell] ) new_job["shell"] = shell group = "js-bench-{}".format(shell) symbol = split_symbol(new_job["treeherder"]["symbol"])[1] new_job["treeherder"]["symbol"] = join_symbol(group, symbol) run = new_job["run"] run["mach"] = run["mach"].format( shell=shell, SHELL=shell.upper(), test=test ) yield new_job
def split_e10s(config, tests): for test in tests: e10s = test['e10s'] test['e10s'] = False test['attributes']['e10s'] = False if e10s == 'both': yield copy.deepcopy(test) e10s = True if e10s: test['test-name'] += '-e10s' test['try-name'] += '-e10s' test['e10s'] = True test['attributes']['e10s'] = True group, symbol = split_symbol(test['treeherder-symbol']) if group != '?': group += '-e10s' test['treeherder-symbol'] = join_symbol(group, symbol) if test['suite'] == 'talos': for i, option in enumerate( test['mozharness']['extra-options']): if option.startswith('--suite='): test['mozharness']['extra-options'][i] += '-e10s' else: test['mozharness']['extra-options'].append('--e10s') yield test
def split_chunks(config, tests): """Based on the 'chunks' key, split tests up into chunks by duplicating them and assigning 'this-chunk' appropriately and updating the treeherder symbol.""" for test in tests: if test['suite'].startswith('test-verify'): test['chunks'] = perfile_number_of_chunks(config, test['test-name']) if test['chunks'] == 0: continue # limit the number of chunks we run for test-verify mode because # test-verify is comprehensive and takes a lot of time, if we have # >30 tests changed, this is probably an import of external tests, # or a patch renaming/moving files in bulk maximum_number_verify_chunks = 3 if test['chunks'] > maximum_number_verify_chunks: test['chunks'] = maximum_number_verify_chunks if test['chunks'] == 1: test['this-chunk'] = 1 yield test continue for this_chunk in range(1, test['chunks'] + 1): # copy the test and update with the chunk number chunked = copy.deepcopy(test) chunked['this-chunk'] = this_chunk # add the chunk number to the TH symbol group, symbol = split_symbol(chunked['treeherder-symbol']) symbol += str(this_chunk) chunked['treeherder-symbol'] = join_symbol(group, symbol) yield chunked
def split_socketprocess_e10s(config, tests): for test in tests: if test['attributes'].get('serviceworker_e10s'): yield test continue sw = test.pop('socketprocess-e10s') test['socketprocess-e10s'] = False test['attributes']['socketprocess_e10s'] = False if sw == 'both': yield copy.deepcopy(test) sw = True if sw: test['description'] += " with socket process enabled" test['test-name'] += '-spi' test['try-name'] += '-spi' test['attributes']['socketprocess_e10s'] = True group, symbol = split_symbol(test['treeherder-symbol']) if group != '?': group += '-spi' else: symbol += '-spi' test['treeherder-symbol'] = join_symbol(group, symbol) test['mozharness']['extra-options'].append( '--setpref="media.peerconnection.mtransport_process=true"') test['mozharness']['extra-options'].append( '--setpref="network.process.enabled=true"') yield test
def split_jsshell(config, jobs): all_shells = {'sm': "Spidermonkey", 'v8': "Google V8"} for job in jobs: if not job['name'].startswith('jsshell'): yield job continue test = job.pop('test') for shell in job.get('shell', all_shells.keys()): assert shell in all_shells new_job = copy.deepcopy(job) new_job['name'] = '{}-{}'.format(new_job['name'], shell) new_job['description'] = '{} on {}'.format(new_job['description'], all_shells[shell]) new_job['shell'] = shell group = 'js-bench-{}'.format(shell) symbol = split_symbol(new_job['treeherder']['symbol'])[1] new_job['treeherder']['symbol'] = join_symbol(group, symbol) run = new_job['run'] run['mach'] = run['mach'].format(shell=shell, SHELL=shell.upper(), test=test) yield new_job
def split_chunks(config, tests): """Based on the 'chunks' key, split tests up into chunks by duplicating them and assigning 'this-chunk' appropriately and updating the treeherder symbol.""" for test in tests: if test['chunks'] == 1: test['this-chunk'] = 1 yield test continue # HACK: Bug 1373578 appears to pass with more chunks, non-e10s only though if test['test-platform'] == 'windows7-32/debug' and test[ 'test-name'] == 'reftest': test['chunks'] = 32 if (test['test-platform'] == 'windows7-32/opt' or test['test-platform'] == 'windows7-32-pgo/opt') and \ test['test-name'] in ['reftest-e10s', 'reftest-no-accel-e10s', 'reftest-gpu-e10s']: test['chunks'] = 32 for this_chunk in range(1, test['chunks'] + 1): # copy the test and update with the chunk number chunked = copy.deepcopy(test) chunked['this-chunk'] = this_chunk # add the chunk number to the TH symbol group, symbol = split_symbol(chunked['treeherder-symbol']) symbol += str(this_chunk) chunked['treeherder-symbol'] = join_symbol(group, symbol) yield chunked
def split_pageload(config, tests): # Split test by pageload type (cold, warm) for test in tests: pageload = test.pop('pageload', 'warm') if pageload not in ('cold', 'both'): yield test continue if pageload == 'both': orig = deepcopy(test) yield orig assert 'subtest' in test test['description'] += " using cold pageload" test['cold'] = True test['max-run-time'] = 3000 test['test-name'] += '-cold' test['try-name'] += '-cold' group, symbol = split_symbol(test['treeherder-symbol']) symbol += '-c' test['treeherder-symbol'] = join_symbol(group, symbol) yield test
def make_beetmover_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] locale = dep_job.attributes.get('locale') if not locale: yield job continue group = 'BMR-L10n' # add the locale code symbol = locale treeherder = { 'symbol': join_symbol(group, symbol), } beet_description = { 'label': job['label'], 'dependent-task': dep_job, 'treeherder': treeherder, 'locale': locale, 'shipping-phase': job.get('shipping-phase'), 'shipping-product': job.get('shipping-product'), } yield beet_description
def split_apps(config, tests): app_symbols = { "chrome": "ChR", "chrome-m": "ChR", "chromium": "Cr", "fenix": "fenix", "refbrow": "refbrow", } for test in tests: apps = test.pop("apps", None) if not apps: yield test continue for app in apps: atest = deepcopy(test) suffix = "-{}".format(app) atest["app"] = app atest["description"] += " on {}".format(app.capitalize()) name = atest["test-name"] + suffix atest["test-name"] = name atest["try-name"] = name if app in app_symbols: group, symbol = split_symbol(atest["treeherder-symbol"]) group += "-{}".format(app_symbols[app]) atest["treeherder-symbol"] = join_symbol(group, symbol) yield atest
def make_beetmover_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] for locale in dep_job.attributes.get('chunk_locales', []): label = job.get('label', "beetmover-{}-{}".format(locale, dep_job.label)) group = 'tc-BM-L10n' # add the locale code if 'signing' in label: symbol = 'S{}'.format(locale) else: symbol = locale treeherder = { 'symbol': join_symbol(group, symbol), } beet_description = { 'dependent-task': dep_job, 'treeherder': treeherder, 'label': label, 'locale': locale, } yield beet_description
def build_treeherder_definition(config, tasks): for task in tasks: dep = task["primary-dependency"] task["treeherder"] = inherit_treeherder_from_dep(task, dep) treeherder_group = dep.task["extra"]["treeherder"]["groupSymbol"] treeherder_symbol = join_symbol(treeherder_group, 'gp') task["treeherder"]["symbol"] = treeherder_symbol yield task
def fill_template(config, tasks): for task in tasks: dep = task['primary-dependency'] # Fill out the dynamic fields in the task description task['label'] = dep.label + '-upload-symbols' # Skip tasks where we don't have the full crashsymbols enabled if not dep.attributes.get('enable-full-crashsymbols'): logger.debug("Skipping upload symbols task for %s", task['label']) continue task['dependencies'] = {'build': dep.label} task['worker']['env']['GECKO_HEAD_REPOSITORY'] = config.params[ 'head_repository'] task['worker']['env']['GECKO_HEAD_REV'] = config.params['head_rev'] task['worker']['env']['SYMBOL_SECRET'] = task['worker']['env'][ 'SYMBOL_SECRET'].format(level=config.params['level']) attributes = copy_attributes_from_dependent_job(dep) attributes.update(task.get('attributes', {})) task['attributes'] = attributes build_type = attributes.get('build_type') treeherder = task.get('treeherder', {}) th = dep.task.get('extra')['treeherder'] th_platform = dep.task['extra'].get( 'treeherder-platform', "{}/{}".format(th['machine']['platform'], build_type)) th_symbol = th.get('symbol') th_groupsymbol = th.get('groupSymbol', '?') treeherder.setdefault('platform', th_platform) treeherder.setdefault('tier', th['tier']) treeherder.setdefault('kind', th['jobKind']) # Disambiguate the treeherder symbol. sym = 'Sym' + (th_symbol[1:] if th_symbol.startswith('B') else th_symbol) treeherder.setdefault('symbol', join_symbol(th_groupsymbol, sym)) task['treeherder'] = treeherder if dep.attributes.get('nightly'): # For nightly builds, we want to run these tasks if the build is run. task['run-on-projects'] = dep.attributes.get('run_on_projects') elif dep.attributes.get('shippable'): # For shippable builds, we want to run these tasks if the build is run. # XXX Better to run this on promote phase instead? task['run-on-projects'] = dep.attributes.get('run_on_projects') else: # For other builds, these can be requested to upload to the try symbol sever. task['run-on-projects'] = ['try'] # clear out the stuff that's not part of a task description del task['primary-dependency'] yield task
def build_treeherder_definition(config, tasks): for task in tasks: dep = task.pop("primary-dependency") task.setdefault("treeherder", {}).update(inherit_treeherder_from_dep(task, dep)) job_group = dep.task["extra"]["treeherder"].get("groupSymbol", "?") job_symbol = task["treeherder"].pop("job-symbol") full_symbol = join_symbol(job_group, job_symbol) task["treeherder"]["symbol"] = full_symbol yield task
def yield_job(orig_job, deps, count): job = deepcopy(orig_job) job['dependencies'] = deps job['name'] = "{}-{}".format(orig_job['name'], count) if 'treeherder' in job: groupSymbol, symbol = split_symbol(job['treeherder']['symbol']) symbol += '-' symbol += str(count) job['treeherder']['symbol'] = join_symbol(groupSymbol, symbol) return job
def make_signing_description(config, jobs): for job in jobs: job['depname'] = 'unsigned-repack' dep_job = job['dependent-task'] dep_platform = dep_job.attributes.get('build_platform') job['upstream-artifacts'] = [] if 'android' in dep_platform: job_specs = [ { 'artifacts': ['public/build/{locale}/target.apk'], 'format': 'jar', }, ] else: job_specs = [ { 'artifacts': ['public/build/{locale}/target.tar.bz2', 'public/build/{locale}/target.checksums'], 'format': 'gpg', }, { 'artifacts': ['public/build/{locale}/target.complete.mar'], 'format': 'mar', } ] upstream_artifacts = [] for spec in job_specs: fmt = spec['format'] upstream_artifacts.append({ "taskId": {"task-reference": "<unsigned-repack>"}, "taskType": "l10n", # Set paths based on artifacts in the specs (above) one per # locale present in the chunk this is signing stuff for. "paths": [f.format(locale=l) for l in dep_job.attributes.get('chunk_locales', []) for f in spec['artifacts']], "formats": [fmt] }) job['upstream-artifacts'] = upstream_artifacts label = dep_job.label.replace("nightly-l10n-", "signing-l10n-") job['label'] = label # add the chunk number to the TH symbol symbol = 'Ns{}'.format(dep_job.attributes.get('l10n_chunk')) group = 'tc-L10n' job['treeherder'] = { 'symbol': join_symbol(group, symbol), } yield job
def make_signing_description(config, jobs): for job in jobs: job['depname'] = 'unsigned-repack' dep_job = job['dependent-task'] # add the chunk number to the TH symbol symbol = 'Ns{}'.format(dep_job.attributes.get('l10n_chunk')) group = 'L10n' job['treeherder'] = { 'symbol': join_symbol(group, symbol), } yield job
def tests_drop_1proc(config, jobs): """ Remove the -1proc suffix from Treeherder group symbols. Restore the -e10s suffix (because some day we will have them!) Reverses the effects of bug 1541527. Thunderbird builds are all single process. """ for job in jobs: test = job["run"]["test"] e10s = test["e10s"] if not e10s: # test-name & friends end with '-1proc' test["test-name"] = _remove_suffix(test["test-name"], "-1proc") test["try-name"] = _remove_suffix(test["try-name"], "-1proc") group, symbol = split_symbol(test["treeherder-symbol"]) if group != "?": group = _remove_suffix(group, "-1proc") test["treeherder-symbol"] = join_symbol(group, symbol) job["label"] = job["label"].replace("-1proc", "") job["name"] = _remove_suffix(job["name"], "-1proc") job["treeherder"]["symbol"] = test["treeherder-symbol"] else: # e10s in the future test["test-name"] = add_suffix(test["test-name"], "-e10s") test["try-name"] = add_suffix(test["try-name"], "-e10s") group, symbol = split_symbol(test["treeherder-symbol"]) if group != "?": group = add_suffix(group, "-e10s") test["treeherder-symbol"] = join_symbol(group, symbol) job["label"] += "-e10s" job["name"] = add_suffix(job["name"], "-e10s") job["treeherder"]["symbol"] = test["treeherder-symbol"] yield job
def tests_drop_1proc(config, jobs): """ Remove the -1proc suffix from Treeherder group symbols. Restore the -e10s suffix (because some day we will have them!) Reverses the effects of bug 1541527. Thunderbird builds are all single process. """ for job in jobs: test = job['run']['test'] e10s = test['e10s'] if not e10s: # test-name & friends end with '-1proc' test['test-name'] = _remove_suffix(test['test-name'], '-1proc') test['try-name'] = _remove_suffix(test['try-name'], '-1proc') group, symbol = split_symbol(test['treeherder-symbol']) if group != '?': group = _remove_suffix(group, '-1proc') test['treeherder-symbol'] = join_symbol(group, symbol) job['label'] = job['label'].replace('-1proc', '') job['name'] = _remove_suffix(job['name'], '-1proc') job['treeherder']['symbol'] = test['treeherder-symbol'] else: # e10s in the future test['test-name'] = add_suffix(test['test-name'], '-e10s') test['try-name'] = add_suffix(test['try-name'], '-e10s') group, symbol = split_symbol(test['treeherder-symbol']) if group != '?': group = add_suffix(group, '-e10s') test['treeherder-symbol'] = join_symbol(group, symbol) job['label'] += '-e10s' job['name'] = add_suffix(job['name'], '-e10s') job['treeherder']['symbol'] = test['treeherder-symbol'] yield job
def fill_template(config, tasks): for task in tasks: dep = task["primary-dependency"] task.pop("dependent-tasks", None) # Fill out the dynamic fields in the task description task["label"] = dep.label + "-upload-symbols" # Skip tasks where we don't have the full crashsymbols enabled if not dep.attributes.get("enable-full-crashsymbols") or dep.attributes.get( "skip-upload-crashsymbols" ): logger.debug("Skipping upload symbols task for %s", task["label"]) continue task["dependencies"] = {"build": dep.label} task["worker"]["env"]["GECKO_HEAD_REPOSITORY"] = config.params[ "head_repository" ] task["worker"]["env"]["GECKO_HEAD_REV"] = config.params["head_rev"] task["worker"]["env"]["SYMBOL_SECRET"] = task["worker"]["env"][ "SYMBOL_SECRET" ].format(level=config.params["level"]) attributes = copy_attributes_from_dependent_job(dep) attributes.update(task.get("attributes", {})) task["attributes"] = attributes treeherder = inherit_treeherder_from_dep(task, dep) th = dep.task.get("extra")["treeherder"] th_symbol = th.get("symbol") th_groupsymbol = th.get("groupSymbol", "?") # Disambiguate the treeherder symbol. sym = "Sym" + (th_symbol[1:] if th_symbol.startswith("B") else th_symbol) treeherder.setdefault("symbol", join_symbol(th_groupsymbol, sym)) task["treeherder"] = treeherder # We only want to run these tasks if the build is run. # XXX Better to run this on promote phase instead? task["run-on-projects"] = dep.attributes.get("run_on_projects") task["optimization"] = {"upload-symbols": None} task["if-dependencies"] = ["build"] # clear out the stuff that's not part of a task description del task["primary-dependency"] yield task
def fill_template(config, tasks): for task in tasks: dep = task['dependent-task'] # Fill out the dynamic fields in the task description task['label'] = dep.label + '-upload-symbols' # Skip tasks where we don't have the full crashsymbols enabled if not dep.attributes.get('enable-full-crashsymbols'): logger.debug("Skipping upload symbols task for %s", task['label']) continue task['dependencies'] = {'build': dep.label} task['worker']['env']['GECKO_HEAD_REPOSITORY'] = config.params[ 'head_repository'] task['worker']['env']['GECKO_HEAD_REV'] = config.params['head_rev'] task['worker']['env']['SYMBOL_SECRET'] = task['worker']['env'][ 'SYMBOL_SECRET'].format(level=config.params['level']) build_platform = dep.attributes.get('build_platform') build_type = dep.attributes.get('build_type') attributes = task.setdefault('attributes', {}) attributes['build_platform'] = build_platform attributes['build_type'] = build_type if dep.attributes.get('nightly'): attributes['nightly'] = True treeherder = task.get('treeherder', {}) th = dep.task.get('extra')['treeherder'] th_platform = dep.task['extra'].get( 'treeherder-platform', "{}/{}".format(th['machine']['platform'], build_type)) th_symbol = th.get('symbol') th_groupsymbol = th.get('groupSymbol', '?') treeherder.setdefault('platform', th_platform) treeherder.setdefault('tier', th['tier']) treeherder.setdefault('kind', th['jobKind']) # Disambiguate the treeherder symbol. sym = 'Sym' + (th_symbol[1:] if th_symbol.startswith('B') else th_symbol) treeherder.setdefault('symbol', join_symbol(th_groupsymbol, sym)) task['treeherder'] = treeherder # clear out the stuff that's not part of a task description del task['dependent-task'] yield task
def make_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] job['depname'] = dep_job.label # add the chunk number to the TH symbol symbol = job.get('treeherder', {}).get('symbol', 'Bs') symbol = '{}{}'.format(symbol, dep_job.attributes.get('l10n_chunk')) group = 'L10n' job['treeherder'] = { 'symbol': join_symbol(group, symbol), } yield job
def make_beetmover_description(config, jobs): for job in jobs: dep_job = job['dependent-task'] for locale in dep_job.attributes.get('chunk_locales', []): group = 'BM-L10n' # add the locale code symbol = locale treeherder = { 'symbol': join_symbol(group, symbol), } beet_description = { 'dependent-task': dep_job, 'treeherder': treeherder, 'locale': locale, } yield beet_description
def split_chunks(config, tests): """Based on the 'chunks' key, split tests up into chunks by duplicating them and assigning 'this-chunk' appropriately and updating the treeherder symbol.""" for test in tests: if test['chunks'] == 1: test['this-chunk'] = 1 yield test continue for this_chunk in range(1, test['chunks'] + 1): # copy the test and update with the chunk number chunked = copy.deepcopy(test) chunked['this-chunk'] = this_chunk # add the chunk number to the TH symbol group, symbol = split_symbol(chunked['treeherder-symbol']) symbol += str(this_chunk) chunked['treeherder-symbol'] = join_symbol(group, symbol) yield chunked
def split_e10s(config, tests): for test in tests: e10s = test['e10s'] test.setdefault('attributes', {}) test['e10s'] = False test['attributes']['e10s'] = False if e10s == 'both': yield copy.deepcopy(test) e10s = True if e10s: test['test-name'] += '-e10s' test['e10s'] = True test['attributes']['e10s'] = True group, symbol = split_symbol(test['treeherder-symbol']) if group != '?': group += '-e10s' test['treeherder-symbol'] = join_symbol(group, symbol) test['mozharness']['extra-options'].append('--e10s') yield test
def split_e10s(config, tests): for test in tests: e10s = get_keyed_by(item=test, field="e10s", item_name=test["test-name"]) test.setdefault("attributes", {}) test["e10s"] = False test["attributes"]["e10s"] = False if e10s == "both": yield test test = copy.deepcopy(test) e10s = True if e10s: test["test-name"] += "-e10s" test["e10s"] = True test["attributes"]["e10s"] = True group, symbol = split_symbol(test["treeherder-symbol"]) if group != "?": group += "-e10s" test["treeherder-symbol"] = join_symbol(group, symbol) test["mozharness"].setdefault("extra-options", []).append("--e10s") yield test
def chunk_locales(config, jobs): """ Utilizes chunking for l10n stuff """ for job in jobs: chunks = job.get('chunks') locales_with_changesets = job['attributes']['all_locales_with_changesets'] if chunks: if chunks > len(locales_with_changesets): # Reduce chunks down to the number of locales chunks = len(locales_with_changesets) for this_chunk in range(1, chunks + 1): chunked = copy.deepcopy(job) chunked['name'] = chunked['name'].replace( '/', '-{}/'.format(this_chunk), 1 ) chunked['mozharness']['options'] = chunked['mozharness'].get('options', []) # chunkify doesn't work with dicts locales_with_changesets_as_list = locales_with_changesets.items() chunked_locales = chunkify(locales_with_changesets_as_list, this_chunk, chunks) chunked['mozharness']['options'].extend([ 'locale={}:{}'.format(locale, changeset) for locale, changeset in chunked_locales ]) chunked['attributes']['l10n_chunk'] = str(this_chunk) # strip revision chunked['attributes']['chunk_locales'] = [locale for locale, _ in chunked_locales] # add the chunk number to the TH symbol group, symbol = split_symbol( chunked.get('treeherder', {}).get('symbol', '')) symbol += str(this_chunk) chunked['treeherder']['symbol'] = join_symbol(group, symbol) yield chunked else: job['mozharness']['options'] = job['mozharness'].get('options', []) job['mozharness']['options'].extend([ 'locale={}:{}'.format(locale, changeset) for locale, changeset in locales_with_changesets.items() ]) yield job
def test_join_no_group(self): self.assertEqual(join_symbol('?', 'xy'), 'xy')
def test_join_with_group(self): self.assertEqual(join_symbol('ab', 'xy'), 'ab(xy)')