コード例 #1
0
def split_pageload(config, tests):
    # Split test by pageload type (cold, warm)
    for test in tests:
        mozharness = test.setdefault("mozharness", {})
        extra_options = mozharness.setdefault("extra-options", [])

        pageload = test.pop("pageload", None)

        if not pageload or "--chimera" in extra_options:
            yield test
            continue

        if pageload in ("warm", "both"):
            # make a deepcopy if 'both', otherwise use the test object itself
            warmtest = deepcopy(test) if pageload == "both" else test

            warmtest["warm"] = True
            group, symbol = split_symbol(warmtest["treeherder-symbol"])
            symbol += "-w"
            warmtest["treeherder-symbol"] = join_symbol(group, symbol)
            yield warmtest

        if pageload in ("cold", "both"):
            assert "subtest" in test

            test["description"] += " using cold pageload"
            test["cold"] = True
            test["max-run-time"] = 3000
            test["test-name"] += "-cold"
            test["try-name"] += "-cold"

            group, symbol = split_symbol(test["treeherder-symbol"])
            symbol += "-c"
            test["treeherder-symbol"] = join_symbol(group, symbol)
            yield test
コード例 #2
0
ファイル: tests.py プロジェクト: masm11/gecko-dev
def split_chunks(config, tests):
    """Based on the 'chunks' key, split tests up into chunks by duplicating
    them and assigning 'this-chunk' appropriately and updating the treeherder
    symbol."""
    for test in tests:
        if test['chunks'] == 1:
            test['this-chunk'] = 1
            yield test
            continue

        # HACK: Bug 1373578 appears to pass with more chunks, non-e10s only though
        if test['test-platform'] == 'windows7-32/debug' and test[
                'test-name'] == 'reftest':
            test['chunks'] = 32

        if (test['test-platform'] == 'windows7-32/opt' or
            test['test-platform'] == 'windows7-32-pgo/opt') and \
                test['test-name'] in ['reftest-e10s', 'reftest-no-accel-e10s', 'reftest-gpu-e10s']:
            test['chunks'] = 32

        for this_chunk in range(1, test['chunks'] + 1):
            # copy the test and update with the chunk number
            chunked = copy.deepcopy(test)
            chunked['this-chunk'] = this_chunk

            # add the chunk number to the TH symbol
            group, symbol = split_symbol(chunked['treeherder-symbol'])
            symbol += str(this_chunk)
            chunked['treeherder-symbol'] = join_symbol(group, symbol)

            yield chunked
コード例 #3
0
ファイル: raptor.py プロジェクト: hewei-github/gecko-dev
def split_browsertime_page_load_by_url(config, tests):

    for test in tests:

        # for tests that have 'raptor-subtests' listed, we want to create a separate
        # test job for every subtest (i.e. split out each page-load URL into its own job)
        subtests = test.pop('raptor-subtests', None)
        if not subtests:
            yield test
            continue

        chunk_number = 0

        for subtest in subtests:

            # create new test job
            chunked = deepcopy(test)

            # only run the subtest/single URL
            chunked['test-name'] += "-{}".format(subtest)
            chunked['try-name'] += "-{}".format(subtest)
            chunked['raptor-test'] = subtest

            # set treeherder symbol and description
            chunk_number += 1
            group, symbol = split_symbol(test['treeherder-symbol'])
            symbol += "-{}".format(chunk_number)
            chunked['treeherder-symbol'] = join_symbol(group, symbol)
            chunked['description'] += "-{}".format(subtest)

            yield chunked
コード例 #4
0
def chunk_locales(config, jobs):
    """ Utilizes chunking for l10n stuff """
    for job in jobs:
        chunks = job.get('chunks')
        all_locales = job['attributes']['all_locales']
        if chunks:
            if chunks > len(all_locales):
                # Reduce chunks down to the number of locales
                chunks = len(all_locales)
            for this_chunk in range(1, chunks + 1):
                chunked = copy.deepcopy(job)
                chunked['name'] = chunked['name'].replace(
                    '/', '-{}/'.format(this_chunk), 1)
                chunked['mozharness']['options'] = chunked['mozharness'].get(
                    'options', [])
                my_locales = []
                my_locales = chunkify(all_locales, this_chunk, chunks)
                chunked['mozharness']['options'].extend(
                    ["locale={}".format(locale) for locale in my_locales])
                chunked['attributes']['l10n_chunk'] = str(this_chunk)
                chunked['attributes']['chunk_locales'] = my_locales

                # add the chunk number to the TH symbol
                group, symbol = split_symbol(
                    chunked.get('treeherder', {}).get('symbol', ''))
                symbol += str(this_chunk)
                chunked['treeherder']['symbol'] = join_symbol(group, symbol)
                yield chunked
        else:
            job['mozharness']['options'] = job['mozharness'].get('options', [])
            job['mozharness']['options'].extend(
                ["locale={}".format(locale) for locale in all_locales])
            yield job
コード例 #5
0
def split_variants(config, tests):
    for test in tests:
        variants = test.pop('variants')

        yield copy.deepcopy(test)

        for name in variants:
            testv = copy.deepcopy(test)
            variant = TEST_VARIANTS[name]

            if 'filterfn' in variant and not variant['filterfn'](testv):
                continue

            testv['attributes']['unittest_variant'] = name
            testv['description'] = variant['description'].format(**testv)

            suffix = '-' + variant['suffix']
            testv['test-name'] += suffix
            testv['try-name'] += suffix

            group, symbol = split_symbol(testv['treeherder-symbol'])
            if group != '?':
                group += suffix
            else:
                symbol += suffix
            testv['treeherder-symbol'] = join_symbol(group, symbol)

            testv.update(variant.get('replace', {}))
            yield merge(testv, variant.get('merge', {}))
コード例 #6
0
def split_jsshell(config, jobs):
    all_shells = {'sm': "Spidermonkey", 'v8': "Google V8"}

    for job in jobs:
        if not job['name'].startswith('jsshell'):
            yield job
            continue

        test = job.pop('test')
        for shell in job.get('shell', all_shells.keys()):
            assert shell in all_shells

            new_job = copy.deepcopy(job)
            new_job['name'] = '{}-{}'.format(new_job['name'], shell)
            new_job['description'] = '{} on {}'.format(new_job['description'],
                                                       all_shells[shell])
            new_job['shell'] = shell

            group = 'js-bench-{}'.format(shell)
            symbol = split_symbol(new_job['treeherder']['symbol'])[1]
            new_job['treeherder']['symbol'] = join_symbol(group, symbol)

            run = new_job['run']
            run['mach'] = run['mach'].format(shell=shell,
                                             SHELL=shell.upper(),
                                             test=test)
            yield new_job
コード例 #7
0
ファイル: tests.py プロジェクト: luke-chang/gecko-1
def split_chunks(config, tests):
    """Based on the 'chunks' key, split tests up into chunks by duplicating
    them and assigning 'this-chunk' appropriately and updating the treeherder
    symbol."""
    for test in tests:
        if test['chunks'] == 1:
            test['this-chunk'] = 1
            yield test
            continue

        # HACK: Bug 1373578 appears to pass with more chunks, non-e10s only though
        if test['test-platform'] == 'windows7-32/debug' and test['test-name'] == 'reftest':
            test['chunks'] = 32

        if (test['test-platform'] == 'windows7-32/opt' or
            test['test-platform'] == 'windows7-32-pgo/opt') and \
                test['test-name'] in ['reftest-e10s', 'reftest-no-accel-e10s', 'reftest-gpu-e10s']:
            test['chunks'] = 32

        for this_chunk in range(1, test['chunks'] + 1):
            # copy the test and update with the chunk number
            chunked = copy.deepcopy(test)
            chunked['this-chunk'] = this_chunk

            # add the chunk number to the TH symbol
            group, symbol = split_symbol(chunked['treeherder-symbol'])
            symbol += str(this_chunk)
            chunked['treeherder-symbol'] = join_symbol(group, symbol)

            yield chunked
コード例 #8
0
ファイル: tests.py プロジェクト: luke-chang/gecko-1
def split_e10s(config, tests):
    for test in tests:
        e10s = test['e10s']

        test['e10s'] = False
        test['attributes']['e10s'] = False

        if e10s == 'both':
            yield copy.deepcopy(test)
            e10s = True
        if e10s:
            test['test-name'] += '-e10s'
            test['try-name'] += '-e10s'
            test['e10s'] = True
            test['attributes']['e10s'] = True
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group += '-e10s'
            test['treeherder-symbol'] = join_symbol(group, symbol)
            if test['suite'] == 'talos':
                for i, option in enumerate(test['mozharness']['extra-options']):
                    if option.startswith('--suite='):
                        test['mozharness']['extra-options'][i] += '-e10s'
            else:
                test['mozharness']['extra-options'].append('--e10s')
        yield test
コード例 #9
0
ファイル: tests.py プロジェクト: masm11/gecko-dev
def split_e10s(config, tests):
    for test in tests:
        e10s = test['e10s']

        test['e10s'] = False
        test['attributes']['e10s'] = False

        if e10s == 'both':
            yield copy.deepcopy(test)
            e10s = True
        if e10s:
            test['test-name'] += '-e10s'
            test['try-name'] += '-e10s'
            test['e10s'] = True
            test['attributes']['e10s'] = True
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group += '-e10s'
            test['treeherder-symbol'] = join_symbol(group, symbol)
            if test['suite'] == 'talos':
                for i, option in enumerate(
                        test['mozharness']['extra-options']):
                    if option.startswith('--suite='):
                        test['mozharness']['extra-options'][i] += '-e10s'
            else:
                test['mozharness']['extra-options'].append('--e10s')
        yield test
コード例 #10
0
ファイル: make_task.py プロジェクト: brendandahl/positron
def build_task(config, tasks):
    for task in tasks:
        provisioner_id, worker_type = task['worker-type'].split('/', 1)
        routes = task['routes']
        scopes = task['scopes']

        # set up extra
        extra = task['extra']
        extra['treeherderEnv'] = task['treeherder']['environments']

        task_th = task['treeherder']
        treeherder = extra.setdefault('treeherder', {})

        machine_platform, collection = task_th['platform'].split('/', 1)
        treeherder['machine'] = {'platform': machine_platform}
        treeherder['collection'] = {collection: True}

        groupSymbol, symbol = split_symbol(task_th['symbol'])
        if groupSymbol != '?':
            treeherder['groupSymbol'] = groupSymbol
            if groupSymbol not in GROUP_NAMES:
                raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
            treeherder['groupName'] = GROUP_NAMES[groupSymbol]
        treeherder['symbol'] = symbol
        treeherder['jobKind'] = task_th['kind']
        treeherder['tier'] = task_th['tier']

        routes.extend([
            '{}.v2.{}.{}.{}'.format(root,
                                    config.params['project'],
                                    config.params['head_rev'],
                                    config.params['pushlog_id'])
            for root in 'tc-treeherder', 'tc-treeherder-stage'
        ])
コード例 #11
0
def split_jsshell(config, jobs):
    all_shells = {"sm": "Spidermonkey", "v8": "Google V8"}

    for job in jobs:
        if not job["name"].startswith("jsshell"):
            yield job
            continue

        test = job.pop("test")
        for shell in job.get("shell", all_shells.keys()):
            assert shell in all_shells

            new_job = copy.deepcopy(job)
            new_job["name"] = "{}-{}".format(new_job["name"], shell)
            new_job["description"] = "{} on {}".format(
                new_job["description"], all_shells[shell]
            )
            new_job["shell"] = shell

            group = "js-bench-{}".format(shell)
            symbol = split_symbol(new_job["treeherder"]["symbol"])[1]
            new_job["treeherder"]["symbol"] = join_symbol(group, symbol)

            run = new_job["run"]
            run["mach"] = run["mach"].format(
                shell=shell, SHELL=shell.upper(), test=test
            )
            yield new_job
コード例 #12
0
ファイル: raptor.py プロジェクト: soufianos01/juggler
def split_apps(config, tests):
    app_symbols = {
        'chrome': 'Chr',
        'chromium': 'Cr',
        'fenix': 'fenix',
        'refbrow': 'refbrow',
    }

    for test in tests:
        apps = test.pop('apps', None)
        if not apps:
            yield test
            continue

        for app in apps:
            atest = deepcopy(test)
            suffix = "-{}".format(app)
            atest['app'] = app
            atest['description'] += " on {}".format(app.capitalize())

            name = atest['test-name']
            if name.endswith('-cold'):
                name = atest['test-name'][:-len('-cold')] + suffix + '-cold'
            else:
                name += suffix

            atest['test-name'] = name
            atest['try-name'] = name

            if app in app_symbols:
                group, symbol = split_symbol(atest['treeherder-symbol'])
                group += "-{}".format(app_symbols[app])
                atest['treeherder-symbol'] = join_symbol(group, symbol)

            yield atest
コード例 #13
0
def split_btime_variants(config, jobs):
    for job in jobs:
        if job.get("perftest-btime-variants") is None:
            yield job
            continue

        variants = job.pop("perftest-btime-variants")
        if not variants:
            yield job
            continue

        yield_existing = False
        for suffix, options in variants:
            if suffix is None:
                # Append options to the existing job
                job.setdefault("perftest-btime-variants", []).append(options)
                yield_existing = True
            else:
                job_new = deepcopy(job)
                group, symbol = split_symbol(job_new["treeherder"]["symbol"])
                symbol += "-" + suffix
                job_new["treeherder"]["symbol"] = join_symbol(group, symbol)
                job_new["name"] += "-" + suffix
                job_new.setdefault("perftest-perfherder-global", {}).setdefault(
                    "extraOptions", []
                ).append(suffix)
                # Replace the existing options with the new ones
                job_new["perftest-btime-variants"] = [options]
                yield job_new

        # The existing job has been modified so we should also return it
        if yield_existing:
            yield job
コード例 #14
0
def split_serviceworker_e10s(config, tests):
    for test in tests:
        sw = test.pop('serviceworker-e10s')

        test['serviceworker-e10s'] = False
        test['attributes']['serviceworker_e10s'] = False

        if sw == 'both':
            yield copy.deepcopy(test)
            sw = True
        if sw:
            if not match_run_on_projects('mozilla-central',
                                         test['run-on-projects']):
                continue

            test['description'] += " with serviceworker-e10s redesign enabled"
            test['run-on-projects'] = ['mozilla-central']
            test['test-name'] += '-sw'
            test['try-name'] += '-sw'
            test['attributes']['serviceworker_e10s'] = True
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group += '-sw'
            else:
                symbol += '-sw'
            test['treeherder-symbol'] = join_symbol(group, symbol)
            test['mozharness']['extra-options'].append(
                '--setpref="dom.serviceWorkers.parent_intercept=true"')
        yield test
コード例 #15
0
def split_pageload(config, tests):
    # Split test by pageload type (cold, warm)
    for test in tests:
        pageload = test.pop('pageload', 'warm')

        if pageload not in ('cold', 'both'):
            yield test
            continue

        if pageload == 'both':
            orig = deepcopy(test)
            yield orig

        assert 'subtest' in test
        test['description'] += " using cold pageload"

        test['cold'] = True

        test['max-run-time'] = 3000
        test['test-name'] += '-cold'
        test['try-name'] += '-cold'

        group, symbol = split_symbol(test['treeherder-symbol'])
        symbol += '-c'
        test['treeherder-symbol'] = join_symbol(group, symbol)
        yield test
コード例 #16
0
def chunk_locales(config, jobs):
    """ Utilizes chunking for l10n stuff """
    for job in jobs:
        chunks = job.get('chunks')
        all_locales = job['attributes']['all_locales']
        if chunks:
            for this_chunk in range(1, chunks + 1):
                chunked = copy.deepcopy(job)
                chunked['name'] = chunked['name'].replace(
                    '/', '-{}/'.format(this_chunk), 1
                )
                chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])
                my_locales = []
                my_locales = chunkify(all_locales, this_chunk, chunks)
                chunked['mozharness']['options'].extend([
                    "locale={}".format(locale) for locale in my_locales
                    ])
                chunked['attributes']['l10n_chunk'] = str(this_chunk)
                chunked['attributes']['chunk_locales'] = my_locales

                # add the chunk number to the TH symbol
                group, symbol = split_symbol(
                    chunked.get('treeherder', {}).get('symbol', ''))
                symbol += str(this_chunk)
                chunked['treeherder']['symbol'] = join_symbol(group, symbol)
                yield chunked
        else:
            job['mozharness']['options'] = job['mozharness'].get('options', [])
            job['mozharness']['options'].extend([
                "locale={}".format(locale) for locale in all_locales
                ])
            yield job
コード例 #17
0
def split_socketprocess_e10s(config, tests):
    for test in tests:
        if test['attributes'].get('serviceworker_e10s'):
            yield test
            continue

        sw = test.pop('socketprocess-e10s')

        test['socketprocess-e10s'] = False
        test['attributes']['socketprocess_e10s'] = False

        if sw == 'both':
            yield copy.deepcopy(test)
            sw = True
        if sw:
            test['description'] += " with socket process enabled"
            test['test-name'] += '-spi'
            test['try-name'] += '-spi'
            test['attributes']['socketprocess_e10s'] = True
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group += '-spi'
            else:
                symbol += '-spi'
            test['treeherder-symbol'] = join_symbol(group, symbol)
            test['mozharness']['extra-options'].append(
                '--setpref="media.peerconnection.mtransport_process=true"')
            test['mozharness']['extra-options'].append(
                '--setpref="network.process.enabled=true"')
        yield test
コード例 #18
0
ファイル: raptor.py プロジェクト: tsl143/browser-f
def split_pageload(config, tests):
    for test in tests:
        pageload = test.pop('pageload', 'warm')

        if pageload not in ('cold', 'both'):
            yield test
            continue

        if pageload == 'both':
            orig = deepcopy(test)
            yield orig

        assert 'raptor-test' in test
        test['description'] += " using cold pageload"

        # for raptor-webext to run cold we just call the corresponding '-cold' test name; but
        # for raptor browsertime we leave the raptor test name as/is and will set the '--cold'
        # command line argument instead via settting test['cold'] to true
        if test['test-name'].startswith('browsertime-tp6'):
            test['cold'] = True
        else:
            test['raptor-test'] += '-cold'

        test['max-run-time'] = 3000
        test['test-name'] += '-cold'
        test['try-name'] += '-cold'

        group, symbol = split_symbol(test['treeherder-symbol'])
        symbol += '-c'
        test['treeherder-symbol'] = join_symbol(group, symbol)
        yield test
コード例 #19
0
ファイル: tests.py プロジェクト: rwaldron/gecko-dev
def split_chunks(config, tests):
    """Based on the 'chunks' key, split tests up into chunks by duplicating
    them and assigning 'this-chunk' appropriately and updating the treeherder
    symbol."""
    for test in tests:
        if test['suite'].startswith('test-verify'):
            test['chunks'] = perfile_number_of_chunks(config, test['test-name'])
            if test['chunks'] == 0:
                continue
            # limit the number of chunks we run for test-verify mode because
            # test-verify is comprehensive and takes a lot of time, if we have
            # >30 tests changed, this is probably an import of external tests,
            # or a patch renaming/moving files in bulk
            maximum_number_verify_chunks = 3
            if test['chunks'] > maximum_number_verify_chunks:
                test['chunks'] = maximum_number_verify_chunks

        if test['chunks'] == 1:
            test['this-chunk'] = 1
            yield test
            continue

        for this_chunk in range(1, test['chunks'] + 1):
            # copy the test and update with the chunk number
            chunked = copy.deepcopy(test)
            chunked['this-chunk'] = this_chunk

            # add the chunk number to the TH symbol
            group, symbol = split_symbol(chunked['treeherder-symbol'])
            symbol += str(this_chunk)
            chunked['treeherder-symbol'] = join_symbol(group, symbol)

            yield chunked
コード例 #20
0
def split_apps(config, tests):
    app_symbols = {
        "chrome": "ChR",
        "chrome-m": "ChR",
        "chromium": "Cr",
        "fenix": "fenix",
        "refbrow": "refbrow",
    }

    for test in tests:
        apps = test.pop("apps", None)
        if not apps:
            yield test
            continue

        for app in apps:
            atest = deepcopy(test)
            suffix = "-{}".format(app)
            atest["app"] = app
            atest["description"] += " on {}".format(app.capitalize())

            name = atest["test-name"] + suffix
            atest["test-name"] = name
            atest["try-name"] = name

            if app in app_symbols:
                group, symbol = split_symbol(atest["treeherder-symbol"])
                group += "-{}".format(app_symbols[app])
                atest["treeherder-symbol"] = join_symbol(group, symbol)

            yield atest
コード例 #21
0
def split_e10s(config, tests):
    for test in tests:
        e10s = get_keyed_by(item=test,
                            field='e10s',
                            item_name=test['test-name'])
        test.setdefault('attributes', {})
        test['e10s'] = False
        test['attributes']['e10s'] = False

        if e10s == 'both':
            yield copy.deepcopy(test)
            e10s = True
        if e10s:
            test['test-name'] += '-e10s'
            test['e10s'] = True
            test['attributes']['e10s'] = True
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group += '-e10s'
            test['treeherder-symbol'] = join_symbol(group, symbol)
            test['mozharness']['extra-options'] = get_keyed_by(
                item=test,
                field='mozharness',
                subfield='extra-options',
                item_name=test['test-name'])
            test['mozharness']['extra-options'].append('--e10s')
        yield test
コード例 #22
0
def yield_job(orig_job, deps, count):
    job = deepcopy(orig_job)
    job['dependencies'] = deps
    job['name'] = "{}-{}".format(orig_job['name'], count)
    if 'treeherder' in job:
        groupSymbol, symbol = split_symbol(job['treeherder']['symbol'])
        symbol += '-'
        symbol += str(count)
        job['treeherder']['symbol'] = join_symbol(groupSymbol, symbol)

    return job
コード例 #23
0
def tests_drop_1proc(config, jobs):
    """
    Remove the -1proc suffix from Treeherder group symbols.
    Restore the -e10s suffix (because some day we will have them!)

    Reverses the effects of bug 1541527. Thunderbird builds are all single
    process.
    """
    for job in jobs:
        test = job["run"]["test"]
        e10s = test["e10s"]

        if not e10s:  # test-name & friends end with '-1proc'
            test["test-name"] = _remove_suffix(test["test-name"], "-1proc")
            test["try-name"] = _remove_suffix(test["try-name"], "-1proc")
            group, symbol = split_symbol(test["treeherder-symbol"])
            if group != "?":
                group = _remove_suffix(group, "-1proc")
            test["treeherder-symbol"] = join_symbol(group, symbol)

            job["label"] = job["label"].replace("-1proc", "")
            job["name"] = _remove_suffix(job["name"], "-1proc")
            job["treeherder"]["symbol"] = test["treeherder-symbol"]
        else:  # e10s in the future
            test["test-name"] = add_suffix(test["test-name"], "-e10s")
            test["try-name"] = add_suffix(test["try-name"], "-e10s")
            group, symbol = split_symbol(test["treeherder-symbol"])
            if group != "?":
                group = add_suffix(group, "-e10s")
            test["treeherder-symbol"] = join_symbol(group, symbol)

            job["label"] += "-e10s"
            job["name"] = add_suffix(job["name"], "-e10s")
            job["treeherder"]["symbol"] = test["treeherder-symbol"]

        yield job
コード例 #24
0
def tests_drop_1proc(config, jobs):
    """
    Remove the -1proc suffix from Treeherder group symbols.
    Restore the -e10s suffix (because some day we will have them!)

    Reverses the effects of bug 1541527. Thunderbird builds are all single
    process.
    """
    for job in jobs:
        test = job['run']['test']
        e10s = test['e10s']

        if not e10s:  # test-name & friends end with '-1proc'
            test['test-name'] = _remove_suffix(test['test-name'], '-1proc')
            test['try-name'] = _remove_suffix(test['try-name'], '-1proc')
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group = _remove_suffix(group, '-1proc')
            test['treeherder-symbol'] = join_symbol(group, symbol)

            job['label'] = job['label'].replace('-1proc', '')
            job['name'] = _remove_suffix(job['name'], '-1proc')
            job['treeherder']['symbol'] = test['treeherder-symbol']
        else:  # e10s in the future
            test['test-name'] = add_suffix(test['test-name'], '-e10s')
            test['try-name'] = add_suffix(test['try-name'], '-e10s')
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group = add_suffix(group, '-e10s')
            test['treeherder-symbol'] = join_symbol(group, symbol)

            job['label'] += '-e10s'
            job['name'] = add_suffix(job['name'], '-e10s')
            job['treeherder']['symbol'] = test['treeherder-symbol']

        yield job
コード例 #25
0
def split_page_load_by_url(config, tests):
    for test in tests:
        # `chunk-number` and 'subtest' only exists when the task had a
        # definition for `raptor-subtests`
        chunk_number = test.pop("chunk-number", None)
        subtest = test.pop("subtest", None)
        subtest_symbol = test.pop("subtest-symbol", None)

        if not chunk_number or not subtest:
            yield test
            continue

        if len(subtest_symbol) > 10 and "ytp" not in subtest_symbol:
            raise Exception(
                "Treeherder symbol %s is lager than 10 char! Please use a different symbol."
                % subtest_symbol)

        if test["test-name"].startswith("browsertime-"):
            test["raptor-test"] = subtest

            # Remove youtube-playback in the test name to avoid duplication
            test["test-name"] = test["test-name"].replace(
                "youtube-playback-", "")
        else:
            # Use full test name if running on webextension
            test["raptor-test"] = "raptor-tp6-" + subtest + "-{}".format(
                test["app"])

        # Only run the subtest/single URL
        test["test-name"] += "-{}".format(subtest)
        test["try-name"] += "-{}".format(subtest)

        # Set treeherder symbol and description
        group, symbol = split_symbol(test["treeherder-symbol"])

        symbol = subtest_symbol
        if test.get("cold"):
            symbol += "-c"
        elif test.pop("warm", False):
            symbol += "-w"

        test["treeherder-symbol"] = join_symbol(group, symbol)
        test["description"] += " on {}".format(subtest)

        yield test
コード例 #26
0
ファイル: l10n.py プロジェクト: masm11/gecko-dev
def chunk_locales(config, jobs):
    """ Utilizes chunking for l10n stuff """
    for job in jobs:
        locales_per_chunk = job.get('locales-per-chunk')
        locales_with_changesets = job['attributes'][
            'all_locales_with_changesets']
        if locales_per_chunk:
            chunks, remainder = divmod(len(locales_with_changesets),
                                       locales_per_chunk)
            if remainder:
                chunks = int(chunks + 1)
            for this_chunk in range(1, chunks + 1):
                chunked = copy.deepcopy(job)
                chunked['name'] = chunked['name'].replace(
                    '/', '-{}/'.format(this_chunk), 1)
                chunked['mozharness']['options'] = chunked['mozharness'].get(
                    'options', [])
                # chunkify doesn't work with dicts
                locales_with_changesets_as_list = sorted(
                    locales_with_changesets.items())
                chunked_locales = chunkify(locales_with_changesets_as_list,
                                           this_chunk, chunks)
                chunked['mozharness']['options'].extend([
                    'locale={}:{}'.format(locale, changeset)
                    for locale, changeset in chunked_locales
                ])
                chunked['attributes']['l10n_chunk'] = str(this_chunk)
                # strip revision
                chunked['attributes']['chunk_locales'] = [
                    locale for locale, _ in chunked_locales
                ]

                # add the chunk number to the TH symbol
                group, symbol = split_symbol(
                    chunked.get('treeherder', {}).get('symbol', ''))
                symbol += str(this_chunk)
                chunked['treeherder']['symbol'] = join_symbol(group, symbol)
                yield chunked
        else:
            job['mozharness']['options'] = job['mozharness'].get('options', [])
            job['mozharness']['options'].extend([
                'locale={}:{}'.format(locale, changeset) for locale, changeset
                in sorted(locales_with_changesets.items())
            ])
            yield job
コード例 #27
0
def split_python(config, jobs):
    for job in jobs:
        key = 'python-version'
        versions = job.pop(key, [])
        if not versions:
            yield job
            continue
        for version in versions:
            group = 'py{0}'.format(version)
            pyjob = copy.deepcopy(job)
            if 'name' in pyjob:
                pyjob['name'] += '-{0}'.format(group)
            else:
                pyjob['label'] += '-{0}'.format(group)
            symbol = split_symbol(pyjob['treeherder']['symbol'])[1]
            pyjob['treeherder']['symbol'] = join_symbol(group, symbol)
            pyjob['run'][key] = version
            yield pyjob
コード例 #28
0
def split_python(config, jobs):
    for job in jobs:
        key = "python-version"
        versions = job.pop(key, [])
        if not versions:
            yield job
            continue
        for version in versions:
            group = "py{0}".format(version)
            pyjob = copy.deepcopy(job)
            if "name" in pyjob:
                pyjob["name"] += "-{0}".format(group)
            else:
                pyjob["label"] += "-{0}".format(group)
            symbol = split_symbol(pyjob["treeherder"]["symbol"])[1]
            pyjob["treeherder"]["symbol"] = join_symbol(group, symbol)
            pyjob["run"][key] = version
            yield pyjob
コード例 #29
0
def split_page_load_by_url(config, tests):
    for test in tests:
        # `chunk-number` and 'subtest' only exists when the task had a
        # definition for `raptor-subtests`
        chunk_number = test.pop('chunk-number', None)
        subtest = test.pop('subtest', None)

        if not chunk_number or not subtest:
            yield test
            continue

        if isinstance(subtest, list):
            subtest, subtest_symbol = subtest
        else:
            subtest_symbol = subtest
            subtest = subtest

        if len(subtest_symbol) > 10:
            raise Exception(
                "Treeherder symbol %s is lager than 10 char! Please use a different symbol."
                % subtest_symbol)

        if test['test-name'].startswith('browsertime-tp6'):
            test['raptor-test'] = subtest
        else:
            # Use full test name if running on webextension
            test['raptor-test'] = 'raptor-tp6-' + subtest + "-{}".format(
                test['app'])

        # Only run the subtest/single URL
        test['test-name'] += "-{}".format(subtest)
        test['try-name'] += "-{}".format(subtest)

        # Set treeherder symbol and description
        group, symbol = split_symbol(test['treeherder-symbol'])

        symbol = subtest_symbol
        if test.get("cold"):
            symbol += "-c"

        test['treeherder-symbol'] = join_symbol(group, symbol)
        test['description'] += " on {}".format(subtest)

        yield test
コード例 #30
0
ファイル: raptor.py プロジェクト: tsl143/browser-f
def split_browsertime_page_load_by_url(config, tests):
    for test in tests:
        # `chunk-number` only exists when the task had a
        # definition for `raptor-subtests`
        chunk_number = test.pop('chunk-number', None)
        if not chunk_number:
            yield test
            continue

        # only run the subtest/single URL
        test['test-name'] += "-{}".format(test['raptor-test'])
        test['try-name'] += "-{}".format(test['raptor-test'])

        # set treeherder symbol and description
        group, symbol = split_symbol(test['treeherder-symbol'])
        symbol += "-{}".format(chunk_number)
        test['treeherder-symbol'] = join_symbol(group, symbol)
        test['description'] += "-{}".format(test['raptor-test'])

        yield test
コード例 #31
0
def split_e10s(config, tests):
    for test in tests:
        e10s = test['e10s']

        test.setdefault('attributes', {})
        test['e10s'] = False
        test['attributes']['e10s'] = False

        if e10s == 'both':
            yield copy.deepcopy(test)
            e10s = True
        if e10s:
            test['test-name'] += '-e10s'
            test['e10s'] = True
            test['attributes']['e10s'] = True
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group += '-e10s'
            test['treeherder-symbol'] = join_symbol(group, symbol)
            test['mozharness']['extra-options'].append('--e10s')
        yield test
コード例 #32
0
def split_perftest_variants(config, jobs):
    for job in jobs:
        if job.get("variants") is None:
            yield job
            continue

        for variant in job.pop("variants"):
            job_new = deepcopy(job)

            group, symbol = split_symbol(job_new["treeherder"]["symbol"])
            group += "-" + variant
            job_new["treeherder"]["symbol"] = join_symbol(group, symbol)
            job_new["name"] += "-" + variant
            job_new.setdefault("perftest-perfherder-global", {}).setdefault(
                "extraOptions", []
            ).append(variant)
            job_new[variant] = True

            yield job_new

        yield job
コード例 #33
0
ファイル: all_kinds.py プロジェクト: brendandahl/positron
def split_chunks(config, tests):
    """Based on the 'chunks' key, split tests up into chunks by duplicating
    them and assigning 'this-chunk' appropriately and updating the treeherder
    symbol."""
    for test in tests:
        if test['chunks'] == 1:
            test['this-chunk'] = 1
            yield test
            continue

        for this_chunk in range(1, test['chunks'] + 1):
            # copy the test and update with the chunk number
            chunked = copy.deepcopy(test)
            chunked['this-chunk'] = this_chunk

            # add the chunk number to the TH symbol
            group, symbol = split_symbol(chunked['treeherder-symbol'])
            symbol += str(this_chunk)
            chunked['treeherder-symbol'] = join_symbol(group, symbol)

            yield chunked
コード例 #34
0
def split_e10s(config, tests):
    for test in tests:
        e10s = get_keyed_by(item=test, field="e10s", item_name=test["test-name"])
        test.setdefault("attributes", {})
        test["e10s"] = False
        test["attributes"]["e10s"] = False

        if e10s == "both":
            yield test
            test = copy.deepcopy(test)
            e10s = True
        if e10s:
            test["test-name"] += "-e10s"
            test["e10s"] = True
            test["attributes"]["e10s"] = True
            group, symbol = split_symbol(test["treeherder-symbol"])
            if group != "?":
                group += "-e10s"
            test["treeherder-symbol"] = join_symbol(group, symbol)
            test["mozharness"].setdefault("extra-options", []).append("--e10s")
        yield test
コード例 #35
0
def split_chunks(config, tests):
    """Based on the 'chunks' key, split tests up into chunks by duplicating
    them and assigning 'this-chunk' appropriately and updating the treeherder
    symbol."""
    for test in tests:
        if test['chunks'] == 1:
            test['this-chunk'] = 1
            yield test
            continue

        for this_chunk in range(1, test['chunks'] + 1):
            # copy the test and update with the chunk number
            chunked = copy.deepcopy(test)
            chunked['this-chunk'] = this_chunk

            # add the chunk number to the TH symbol
            group, symbol = split_symbol(chunked['treeherder-symbol'])
            symbol += str(this_chunk)
            chunked['treeherder-symbol'] = join_symbol(group, symbol)

            yield chunked
コード例 #36
0
def split_e10s(config, tests):
    for test in tests:
        e10s = test['e10s']

        if e10s:
            test_copy = copy.deepcopy(test)
            test_copy['test-name'] += '-e10s'
            test_copy['e10s'] = True
            test_copy['attributes']['e10s'] = True
            yield test_copy

        if not e10s or e10s == 'both':
            test['test-name'] += '-1proc'
            test['try-name'] += '-1proc'
            test['e10s'] = False
            test['attributes']['e10s'] = False
            group, symbol = split_symbol(test['treeherder-symbol'])
            if group != '?':
                group += '-1proc'
            test['treeherder-symbol'] = join_symbol(group, symbol)
            test['mozharness']['extra-options'].append('--disable-e10s')
            yield test
コード例 #37
0
ファイル: l10n.py プロジェクト: Wafflespeanut/gecko-dev
def chunk_locales(config, jobs):
    """ Utilizes chunking for l10n stuff """
    for job in jobs:
        chunks = job.get('chunks')
        locales_with_changesets = job['attributes']['all_locales_with_changesets']
        if chunks:
            if chunks > len(locales_with_changesets):
                # Reduce chunks down to the number of locales
                chunks = len(locales_with_changesets)
            for this_chunk in range(1, chunks + 1):
                chunked = copy.deepcopy(job)
                chunked['name'] = chunked['name'].replace(
                    '/', '-{}/'.format(this_chunk), 1
                )
                chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])
                # chunkify doesn't work with dicts
                locales_with_changesets_as_list = locales_with_changesets.items()
                chunked_locales = chunkify(locales_with_changesets_as_list, this_chunk, chunks)
                chunked['mozharness']['options'].extend([
                    'locale={}:{}'.format(locale, changeset)
                    for locale, changeset in chunked_locales
                ])
                chunked['attributes']['l10n_chunk'] = str(this_chunk)
                # strip revision
                chunked['attributes']['chunk_locales'] = [locale for locale, _ in chunked_locales]

                # add the chunk number to the TH symbol
                group, symbol = split_symbol(
                    chunked.get('treeherder', {}).get('symbol', ''))
                symbol += str(this_chunk)
                chunked['treeherder']['symbol'] = join_symbol(group, symbol)
                yield chunked
        else:
            job['mozharness']['options'] = job['mozharness'].get('options', [])
            job['mozharness']['options'].extend([
                'locale={}:{}'.format(locale, changeset)
                for locale, changeset in locales_with_changesets.items()
            ])
            yield job
コード例 #38
0
ファイル: raptor.py プロジェクト: soufianos01/juggler
def split_cold(config, tests):
    for test in tests:
        cold = test.pop('cold', False)

        if not cold:
            yield test
            continue

        orig = deepcopy(test)
        yield orig

        assert 'raptor-test' in test
        test['description'] += " using cold pageload"
        test['raptor-test'] += '-cold'
        test['max-run-time'] = 3000
        test['test-name'] += '-cold'
        test['try-name'] += '-cold'

        group, symbol = split_symbol(test['treeherder-symbol'])
        symbol += '-c'
        test['treeherder-symbol'] = join_symbol(group, symbol)
        yield test
コード例 #39
0
def build_task(config, tasks):
    for task in tasks:
        worker_type = task['worker-type'].format(
            level=str(config.params['level']))
        provisioner_id, worker_type = worker_type.split('/', 1)

        routes = task.get('routes', [])
        scopes = task.get('scopes', [])

        # set up extra
        extra = task.get('extra', {})
        task_th = task.get('treeherder')
        if task_th:
            extra['treeherderEnv'] = task_th['environments']

            treeherder = extra.setdefault('treeherder', {})

            machine_platform, collection = task_th['platform'].split('/', 1)
            treeherder['machine'] = {'platform': machine_platform}
            treeherder['collection'] = {collection: True}

            groupSymbol, symbol = split_symbol(task_th['symbol'])
            if groupSymbol != '?':
                treeherder['groupSymbol'] = groupSymbol
                if groupSymbol not in GROUP_NAMES:
                    raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
                treeherder['groupName'] = GROUP_NAMES[groupSymbol]
            treeherder['symbol'] = symbol
            treeherder['jobKind'] = task_th['kind']
            treeherder['tier'] = task_th['tier']

            routes.extend([
                '{}.v2.{}.{}.{}'.format(TREEHERDER_ROUTE_ROOTS[env],
                                        config.params['project'],
                                        config.params['head_rev'],
                                        config.params['pushlog_id'])
                for env in task_th['environments']
            ])

        if 'expires-after' not in task:
            task['expires-after'] = '28 days' if config.params[
                'project'] == 'try' else '1 year'

        if 'deadline-after' not in task:
            task['deadline-after'] = '1 day'

        if 'coalesce-name' in task and int(config.params['level']) > 1:
            key = COALESCE_KEY.format(project=config.params['project'],
                                      name=task['coalesce-name'])
            routes.append('coalesce.v1.' + key)

        task_def = {
            'provisionerId': provisioner_id,
            'workerType': worker_type,
            'routes': routes,
            'created': {
                'relative-datestamp': '0 seconds'
            },
            'deadline': {
                'relative-datestamp': task['deadline-after']
            },
            'expires': {
                'relative-datestamp': task['expires-after']
            },
            'scopes': scopes,
            'metadata': {
                'description':
                task['description'],
                'name':
                task['label'],
                'owner':
                config.params['owner'],
                'source':
                '{}/file/{}/{}'.format(config.params['head_repository'],
                                       config.params['head_rev'], config.path),
            },
            'extra': extra,
            'tags': {
                'createdForUser': config.params['owner']
            },
        }

        # add the payload and adjust anything else as required (e.g., scopes)
        payload_builders[task['worker']['implementation']](config, task,
                                                           task_def)

        attributes = task.get('attributes', {})
        attributes['run_on_projects'] = task.get('run-on-projects', ['all'])

        yield {
            'label': task['label'],
            'task': task_def,
            'dependencies': task.get('dependencies', {}),
            'attributes': attributes,
            'when': task.get('when', {}),
        }
コード例 #40
0
ファイル: task.py プロジェクト: nwgh/gecko-dev
def build_task(config, tasks):
    for task in tasks:
        worker_type = task['worker-type'].format(level=str(config.params['level']))
        provisioner_id, worker_type = worker_type.split('/', 1)

        routes = task.get('routes', [])
        scopes = task.get('scopes', [])

        # set up extra
        extra = task.get('extra', {})
        task_th = task.get('treeherder')
        if task_th:
            extra['treeherderEnv'] = task_th['environments']

            treeherder = extra.setdefault('treeherder', {})

            machine_platform, collection = task_th['platform'].split('/', 1)
            treeherder['machine'] = {'platform': machine_platform}
            treeherder['collection'] = {collection: True}

            groupSymbol, symbol = split_symbol(task_th['symbol'])
            if groupSymbol != '?':
                treeherder['groupSymbol'] = groupSymbol
                if groupSymbol not in GROUP_NAMES:
                    raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
                treeherder['groupName'] = GROUP_NAMES[groupSymbol]
            treeherder['symbol'] = symbol
            treeherder['jobKind'] = task_th['kind']
            treeherder['tier'] = task_th['tier']

            routes.extend([
                '{}.v2.{}.{}.{}'.format(TREEHERDER_ROUTE_ROOTS[env],
                                        config.params['project'],
                                        config.params['head_rev'],
                                        config.params['pushlog_id'])
                for env in task_th['environments']
            ])

        if 'expires-after' not in task:
            task['expires-after'] = '14 days' if config.params['project'] == 'try' else '1 year'

        if 'deadline-after' not in task:
            task['deadline-after'] = '1 day'

        if 'coalesce-name' in task and int(config.params['level']) > 1:
            key = COALESCE_KEY.format(
                project=config.params['project'],
                name=task['coalesce-name'])
            routes.append('coalesce.v1.' + key)

        task_def = {
            'provisionerId': provisioner_id,
            'workerType': worker_type,
            'routes': routes,
            'created': {'relative-datestamp': '0 seconds'},
            'deadline': {'relative-datestamp': task['deadline-after']},
            'expires': {'relative-datestamp': task['expires-after']},
            'scopes': scopes,
            'metadata': {
                'description': task['description'],
                'name': task['label'],
                'owner': config.params['owner'],
                'source': '{}/file/{}/{}'.format(
                    config.params['head_repository'],
                    config.params['head_rev'],
                    config.path),
            },
            'extra': extra,
            'tags': {'createdForUser': config.params['owner']},
        }

        # add the payload and adjust anything else as required (e.g., scopes)
        payload_builders[task['worker']['implementation']](config, task, task_def)

        attributes = task.get('attributes', {})
        attributes['run_on_projects'] = task.get('run-on-projects', ['all'])

        yield {
            'label': task['label'],
            'task': task_def,
            'dependencies': task.get('dependencies', {}),
            'attributes': attributes,
            'when': task.get('when', {}),
        }
コード例 #41
0
ファイル: task.py プロジェクト: subsevenx2001/gecko-dev
def build_task(config, tasks):
    for task in tasks:
        worker_type = task["worker-type"].format(level=str(config.params["level"]))
        provisioner_id, worker_type = worker_type.split("/", 1)

        routes = task.get("routes", [])
        scopes = task.get("scopes", [])

        # set up extra
        extra = task.get("extra", {})
        task_th = task.get("treeherder")
        if task_th:
            extra["treeherderEnv"] = task_th["environments"]

            treeherder = extra.setdefault("treeherder", {})

            machine_platform, collection = task_th["platform"].split("/", 1)
            treeherder["machine"] = {"platform": machine_platform}
            treeherder["collection"] = {collection: True}

            groupSymbol, symbol = split_symbol(task_th["symbol"])
            if groupSymbol != "?":
                treeherder["groupSymbol"] = groupSymbol
                if groupSymbol not in GROUP_NAMES:
                    raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
                treeherder["groupName"] = GROUP_NAMES[groupSymbol]
            treeherder["symbol"] = symbol
            treeherder["jobKind"] = task_th["kind"]
            treeherder["tier"] = task_th["tier"]

            routes.extend(
                [
                    "{}.v2.{}.{}.{}".format(
                        TREEHERDER_ROUTE_ROOTS[env],
                        config.params["project"],
                        config.params["head_rev"],
                        config.params["pushlog_id"],
                    )
                    for env in task_th["environments"]
                ]
            )

        if "expires-after" not in task:
            task["expires-after"] = "28 days" if config.params["project"] == "try" else "1 year"

        if "deadline-after" not in task:
            task["deadline-after"] = "1 day"

        if "coalesce-name" in task and int(config.params["level"]) > 1:
            key = COALESCE_KEY.format(project=config.params["project"], name=task["coalesce-name"])
            routes.append("coalesce.v1." + key)

        task_def = {
            "provisionerId": provisioner_id,
            "workerType": worker_type,
            "routes": routes,
            "created": {"relative-datestamp": "0 seconds"},
            "deadline": {"relative-datestamp": task["deadline-after"]},
            "expires": {"relative-datestamp": task["expires-after"]},
            "scopes": scopes,
            "metadata": {
                "description": task["description"],
                "name": task["label"],
                "owner": config.params["owner"],
                "source": "{}/file/{}/{}".format(
                    config.params["head_repository"], config.params["head_rev"], config.path
                ),
            },
            "extra": extra,
            "tags": {"createdForUser": config.params["owner"]},
        }

        # add the payload and adjust anything else as required (e.g., scopes)
        payload_builders[task["worker"]["implementation"]](config, task, task_def)

        attributes = task.get("attributes", {})
        attributes["run_on_projects"] = task.get("run-on-projects", ["all"])

        yield {
            "label": task["label"],
            "task": task_def,
            "dependencies": task.get("dependencies", {}),
            "attributes": attributes,
            "when": task.get("when", {}),
        }
コード例 #42
0
 def test_split_no_group(self):
     self.assertEqual(split_symbol('xy'), ('?', 'xy'))
コード例 #43
0
 def test_split_with_group(self):
     self.assertEqual(split_symbol('ab(xy)'), ('ab', 'xy'))
コード例 #44
0
ファイル: task.py プロジェクト: Wafflespeanut/gecko-dev
def build_task(config, tasks):
    for task in tasks:
        worker_type = task['worker-type'].format(level=str(config.params['level']))
        provisioner_id, worker_type = worker_type.split('/', 1)

        routes = task.get('routes', [])
        scopes = task.get('scopes', [])

        # set up extra
        extra = task.get('extra', {})
        task_th = task.get('treeherder')
        if task_th:
            extra['treeherderEnv'] = task_th['environments']

            treeherder = extra.setdefault('treeherder', {})

            machine_platform, collection = task_th['platform'].split('/', 1)
            treeherder['machine'] = {'platform': machine_platform}
            treeherder['collection'] = {collection: True}

            groupSymbol, symbol = split_symbol(task_th['symbol'])
            if groupSymbol != '?':
                treeherder['groupSymbol'] = groupSymbol
                if groupSymbol not in GROUP_NAMES:
                    raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
                treeherder['groupName'] = GROUP_NAMES[groupSymbol]
            treeherder['symbol'] = symbol
            treeherder['jobKind'] = task_th['kind']
            treeherder['tier'] = task_th['tier']

            routes.extend([
                '{}.v2.{}.{}.{}'.format(TREEHERDER_ROUTE_ROOTS[env],
                                        config.params['project'],
                                        config.params['head_rev'],
                                        config.params['pushlog_id'])
                for env in task_th['environments']
            ])

        if 'expires-after' not in task:
            task['expires-after'] = '28 days' if config.params['project'] == 'try' else '1 year'

        if 'deadline-after' not in task:
            task['deadline-after'] = '1 day'

        if 'coalesce-name' in task and int(config.params['level']) > 1:
            key = COALESCE_KEY.format(
                project=config.params['project'],
                name=task['coalesce-name'])
            routes.append('coalesce.v1.' + key)

        if 'priority' not in task:
            task['priority'] = BRANCH_PRIORITIES.get(
                config.params['project'],
                DEFAULT_BRANCH_PRIORITY)

        tags = task.get('tags', {})
        tags.update({'createdForUser': config.params['owner']})

        task_def = {
            'provisionerId': provisioner_id,
            'workerType': worker_type,
            'routes': routes,
            'created': {'relative-datestamp': '0 seconds'},
            'deadline': {'relative-datestamp': task['deadline-after']},
            'expires': {'relative-datestamp': task['expires-after']},
            'scopes': scopes,
            'metadata': {
                'description': task['description'],
                'name': task['label'],
                'owner': config.params['owner'],
                'source': '{}/file/{}/{}'.format(
                    config.params['head_repository'],
                    config.params['head_rev'],
                    config.path),
            },
            'extra': extra,
            'tags': tags,
            'priority': task['priority'],
        }

        if task_th:
            # link back to treeherder in description
            th_push_link = 'https://treeherder.mozilla.org/#/jobs?repo={}&revision={}'.format(
                config.params['project'], config.params['head_rev'])
            task_def['metadata']['description'] += ' ([Treeherder push]({}))'.format(
                th_push_link)

        # add the payload and adjust anything else as required (e.g., scopes)
        payload_builders[task['worker']['implementation']](config, task, task_def)

        attributes = task.get('attributes', {})
        attributes['run_on_projects'] = task.get('run-on-projects', ['all'])

        # Set MOZ_AUTOMATION on all jobs.
        if task['worker']['implementation'] in (
            'generic-worker',
            'docker-engine',
            'native-engine',
            'docker-worker',
        ):
            payload = task_def.get('payload')
            if payload:
                env = payload.setdefault('env', {})
                env['MOZ_AUTOMATION'] = '1'

        yield {
            'label': task['label'],
            'task': task_def,
            'dependencies': task.get('dependencies', {}),
            'attributes': attributes,
            'optimizations': task.get('optimizations', []),
        }