def test_extend_schema_twice(self): "Extending a schema twice applies taskgraph checks." with self.assertRaises(Exception): Schema({ "kebab-case": int }).extend({ 'more-kebab': int }).extend({"camelCase": int})
def payload_builder(name, schema): schema = Schema({Required("implementation"): name, Optional("os"): str}).extend( schema ) def wrap(func): payload_builders[name] = PayloadBuilder(schema, func) return func return wrap
def test_worker_caches(task, transform): config, job, taskdesc, impl = transform(task) add_cache(job, taskdesc, 'cache1', '/cache1') add_cache(job, taskdesc, 'cache2', '/cache2', skip_untrusted=True) if impl not in ('docker-worker', 'generic-worker'): pytest.xfail("caches not implemented for '{}'".format(impl)) key = 'caches' if impl == 'docker-worker' else 'mounts' assert key in taskdesc['worker'] assert len(taskdesc['worker'][key]) == 2 # Create a new schema object with just the part relevant to caches. partial_schema = Schema(payload_builders[impl].schema.schema[key]) validate_schema(partial_schema, taskdesc['worker'][key], "validation error")
def test_worker_caches(task, transform): config, job, taskdesc, impl = transform(task) add_cache(job, taskdesc, "cache1", "/cache1") add_cache(job, taskdesc, "cache2", "/cache2", skip_untrusted=True) if impl not in ("docker-worker", "generic-worker"): pytest.xfail("caches not implemented for '{}'".format(impl)) key = "caches" if impl == "docker-worker" else "mounts" assert key in taskdesc["worker"] assert len(taskdesc["worker"][key]) == 2 # Create a new schema object with just the part relevant to caches. partial_schema = Schema(payload_builders[impl].schema.schema[key]) validate_schema(partial_schema, taskdesc["worker"][key], "validation error")
from taskgraph.transforms.job import run_job_using from taskgraph.util.schema import Schema from voluptuous import Required, Optional, Extra, Any bare_schema = Schema({ Required("using"): "bare", Required("command"): Any(basestring, [basestring]), Optional("install"): Any(basestring, [basestring]), Optional("clone"): bool, Extra: object }) bare_defaults = { "clone": True, } @run_job_using("docker-worker", "bare", schema=bare_schema, defaults=bare_defaults) def bare_docker_worker(config, job, taskdesc): run = job["run"] worker = taskdesc['worker'] = job['worker'] params = config.params command = [] if run.get("clone"): command.extend([ "git clone --quiet --depth=20 --no-single-branch {} taskcluster && "
from voluptuous import Required transforms = TransformSequence() push_apk_breakpoint_description_schema = Schema({ # the dependent task (object) for this beetmover job, used to inform beetmover. Required('dependent-tasks'): object, Required('name'): basestring, Required('label'): basestring, Required('description'): basestring, Required('attributes'): object, Required('worker-type'): None, Required('worker'): object, Required('treeherder'): object, Required('run-on-projects'): list, Required('deadline-after'): basestring, }) validate_jobs_schema_transform = functools.partial( validate_jobs_schema_transform_partial, push_apk_breakpoint_description_schema, 'PushApkBreakpoint')
packaging_description_schema = Schema({ # the dependant task (object) for this job, used to inform repackaging. Required('dependent-task'): object, # depname is used in taskref's to identify the taskID of the signed things Required('depname', default='build'): basestring, # unique label to describe this repackaging task Optional('label'): basestring, # treeherder is allowed here to override any defaults we use for repackaging. See # taskcluster/taskgraph/transforms/task.py for the schema details, and the # below transforms for defaults of various values. Optional('treeherder'): task_description_schema['treeherder'], # If a l10n task, the corresponding locale Optional('locale'): basestring, # Routes specific to this task, if defined Optional('routes'): [basestring], # passed through directly to the job description Optional('extra'): task_description_schema['extra'], })
mozharness_run_schema = Schema({ Required('using'): 'mozharness', # the mozharness script used to run this task, relative to the testing/ # directory and using forward slashes even on Windows Required('script'): text_type, # Additional paths to look for mozharness configs in. These should be # relative to the base of the source checkout Optional('config-paths'): [text_type], # the config files required for the task, relative to # testing/mozharness/configs or one of the paths specified in # `config-paths` and using forward slashes even on Windows Required('config'): [text_type], # any additional actions to pass to the mozharness command Optional('actions'): [ Match('^[a-z0-9-]+$', "actions must be `-` seperated alphanumeric strings") ], # any additional options (without leading --) to be passed to mozharness Optional('options'): [ Match( '^[a-z0-9-]+(=[^ ]+)?$', "options must be `-` seperated alphanumeric strings (with optional argument)" ) ], # --custom-build-variant-cfg value Optional('custom-build-variant-cfg'): text_type, # Extra configuration options to pass to mozharness. Optional('extra-config'): dict, # If not false, tooltool downloads will be enabled via relengAPIProxy # for either just public files, or all files. Not supported on Windows Required('tooltool-downloads'): Any( False, 'public', 'internal', ), # The set of secret names to which the task has access; these are prefixed # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting # this will enable any worker features required and set the task's scopes # appropriately. `true` here means ['*'], all secrets. Not supported on # Windows Required('secrets'): Any(bool, [text_type]), # If true, taskcluster proxy will be enabled; note that it may also be enabled # automatically e.g., for secrets support. Not supported on Windows. Required('taskcluster-proxy'): bool, # If true, the build scripts will start Xvfb. Not supported on Windows. Required('need-xvfb'): bool, # If false, indicate that builds should skip producing artifacts. Not # supported on Windows. Required('keep-artifacts'): bool, # If specified, use the in-tree job script specified. Optional('job-script'): text_type, Required('requires-signed-builds'): bool, # Whether or not to use caches. Optional('use-caches'): bool, # If false, don't set MOZ_SIMPLE_PACKAGE_NAME # Only disableable on windows Required('use-simple-package'): bool, # If false don't pass --branch mozharness script # Only disableable on windows Required('use-magic-mh-args'): bool, # if true, perform a checkout of a comm-central based branch inside the # gecko checkout Required('comm-checkout'): bool, # Base work directory used to set up the task. Required('workdir'): text_type, })
for v in VARIANTS: if '-{}/'.format(v) in test_platform: return v return '' test_description_schema = { str(k): v for k, v in test_description_schema.schema.iteritems() } mozharness_test_run_schema = Schema({ Required('using'): 'mozharness-test', Required('test'): test_description_schema, # Base work directory used to set up the task. Required('workdir'): basestring, }) def test_packages_url(taskdesc): """Account for different platforms that name their test packages differently""" return get_artifact_url( '<build>', get_artifact_path(taskdesc, 'target.test_packages.json')) @run_job_using('docker-engine', 'mozharness-test', schema=mozharness_test_run_schema)
job_description_schema = Schema({ # The name of the job and the job's label. At least one must be specified, # and the label will be generated from the name if necessary, by prepending # the kind. Optional('name'): text_type, Optional('label'): text_type, # the following fields are passed directly through to the task description, # possibly modified by the run implementation. See # taskcluster/taskgraph/transforms/task.py for the schema details. Required('description'): task_description_schema['description'], Optional('attributes'): task_description_schema['attributes'], Optional('job-from'): task_description_schema['job-from'], Optional('dependencies'): task_description_schema['dependencies'], Optional('soft-dependencies'): task_description_schema['soft-dependencies'], Optional('requires'): task_description_schema['requires'], Optional('expires-after'): task_description_schema['expires-after'], Optional('routes'): task_description_schema['routes'], Optional('scopes'): task_description_schema['scopes'], Optional('tags'): task_description_schema['tags'], Optional('extra'): task_description_schema['extra'], Optional('treeherder'): task_description_schema['treeherder'], Optional('index'): task_description_schema['index'], Optional('run-on-projects'): task_description_schema['run-on-projects'], Optional('shipping-phase'): task_description_schema['shipping-phase'], Optional('shipping-product'): task_description_schema['shipping-product'], Optional('coalesce'): task_description_schema['coalesce'], Optional('always-target'): task_description_schema['always-target'], Exclusive('optimization', 'optimization'): task_description_schema['optimization'], Optional('needs-sccache'): task_description_schema['needs-sccache'], Optional('release-artifacts'): task_description_schema['release-artifacts'], Optional('priority'): task_description_schema['priority'], # The "when" section contains descriptions of the circumstances under which # this task should be included in the task graph. This will be converted # into an optimization, so it cannot be specified in a job description that # also gives 'optimization'. Exclusive('when', 'optimization'): { # This task only needs to be run if a file matching one of the given # patterns has changed in the push. The patterns use the mozpack # match function (python/mozbuild/mozpack/path.py). Optional('files-changed'): [text_type], }, # A list of artifacts to install from 'fetch' tasks. Optional('fetches'): { text_type: [text_type, { Required('artifact'): text_type, Optional('dest'): text_type, Optional('extract'): bool, }], }, # A description of how to run this job. 'run': { # The key to a job implementation in a peer module to this one 'using': text_type, # Base work directory used to set up the task. Optional('workdir'): text_type, # Any remaining content is verified against that job implementation's # own schema. Extra: object, }, Required('worker-type'): task_description_schema['worker-type'], # This object will be passed through to the task description, with additions # provided by the job's run-using function Optional('worker'): dict, })
test_description_schema = Schema({ # description of the suite, for the task metadata 'description': basestring, # test suite name, or <suite>/<flavor> Required('suite'): optionally_keyed_by( 'test-platform', basestring), # the name by which this test suite is addressed in try syntax; defaults to # the test-name Optional('unittest-try-name'): basestring, # the name by which this talos test is addressed in try syntax Optional('talos-try-name'): basestring, # additional tags to mark up this type of test Optional('tags'): {basestring: object}, # the symbol, or group(symbol), under which this task should appear in # treeherder. 'treeherder-symbol': basestring, # the value to place in task.extra.treeherder.machine.platform; ideally # this is the same as build-platform, and that is the default, but in # practice it's not always a match. Optional('treeherder-machine-platform'): basestring, # attributes to appear in the resulting task (later transforms will add the # common attributes) Optional('attributes'): {basestring: object}, # The `run_on_projects` attribute, defaulting to "all". This dictates the # projects on which this task should be included in the target task set. # See the attributes documentation for details. Optional('run-on-projects', default=['all']): optionally_keyed_by( 'test-platform', [basestring]), # the sheriffing tier for this task (default: set based on test platform) Optional('tier'): optionally_keyed_by( 'test-platform', Any(int, 'default')), # number of chunks to create for this task. This can be keyed by test # platform by passing a dictionary in the `by-test-platform` key. If the # test platform is not found, the key 'default' will be tried. Required('chunks', default=1): optionally_keyed_by( 'test-platform', int), # the time (with unit) after which this task is deleted; default depends on # the branch (see below) Optional('expires-after'): basestring, # Whether to run this task with e10s (desktop-test only). If false, run # without e10s; if true, run with e10s; if 'both', run one task with and # one task without e10s. E10s tasks have "-e10s" appended to the test name # and treeherder group. Required('e10s', default='both'): optionally_keyed_by( 'test-platform', 'project', Any(bool, 'both')), # The EC2 instance size to run these tests on. Required('instance-size', default='default'): optionally_keyed_by( 'test-platform', Any('default', 'large', 'xlarge', 'legacy')), # Whether the task requires loopback audio or video (whatever that may mean # on the platform) Required('loopback-audio', default=False): bool, Required('loopback-video', default=False): bool, # Whether the test can run using a software GL implementation on Linux # using the GL compositor. May not be used with "legacy" sized instances # due to poor LLVMPipe performance (bug 1296086). Defaults to true for # linux platforms and false otherwise Optional('allow-software-gl-layers'): bool, # The worker implementation for this test, as dictated by policy and by the # test platform. Optional('worker-implementation'): Any( 'docker-worker', 'native-engine', 'generic-worker', # coming soon: 'docker-engine', 'buildbot-bridge', ), # For tasks that will run in docker-worker or docker-engine, this is the # name of the docker image or in-tree docker image to run the task in. If # in-tree, then a dependency will be created automatically. This is # generally `desktop-test`, or an image that acts an awful lot like it. Required('docker-image', default={'in-tree': 'desktop-test'}): optionally_keyed_by( 'test-platform', 'test-platform-phylum', Any( # a raw Docker image path (repo/image:tag) basestring, # an in-tree generated docker image (from `taskcluster/docker/<name>`) {'in-tree': basestring} ) ), # seconds of runtime after which the task will be killed. Like 'chunks', # this can be keyed by test pltaform. Required('max-run-time', default=3600): optionally_keyed_by( 'test-platform', int), # the exit status code that indicates the task should be retried Optional('retry-exit-status'): int, # Whether to perform a gecko checkout. Required('checkout', default=False): bool, # Wheter to perform a machine reboot after test is done Optional('reboot', default=True): bool, # What to run Required('mozharness'): optionally_keyed_by( 'test-platform', 'test-platform-phylum', { # the mozharness script used to run this task Required('script'): basestring, # the config files required for the task Required('config'): optionally_keyed_by( 'test-platform', [basestring]), # mochitest flavor for mochitest runs Optional('mochitest-flavor'): basestring, # any additional actions to pass to the mozharness command Optional('actions'): [basestring], # additional command-line options for mozharness, beyond those # automatically added Required('extra-options', default=[]): optionally_keyed_by( 'test-platform', [basestring]), # the artifact name (including path) to test on the build task; this is # generally set in a per-kind transformation Optional('build-artifact-name'): basestring, # If true, tooltool downloads will be enabled via relengAPIProxy. Required('tooltool-downloads', default=False): bool, # This mozharness script also runs in Buildbot and tries to read a # buildbot config file, so tell it not to do so in TaskCluster Required('no-read-buildbot-config', default=False): bool, # Add --blob-upload-branch=<project> mozharness parameter Optional('include-blob-upload-branch'): bool, # The setting for --download-symbols (if omitted, the option will not # be passed to mozharness) Optional('download-symbols'): Any(True, 'ondemand'), # If set, then MOZ_NODE_PATH=/usr/local/bin/node is included in the # environment. This is more than just a helpful path setting -- it # causes xpcshell tests to start additional servers, and runs # additional tests. Required('set-moz-node-path', default=False): bool, # If true, include chunking information in the command even if the number # of chunks is 1 Required('chunked', default=False): optionally_keyed_by( 'test-platform', bool), # The chunking argument format to use Required('chunking-args', default='this-chunk'): Any( # Use the usual --this-chunk/--total-chunk arguments 'this-chunk', # Use --test-suite=<suite>-<chunk-suffix>; see chunk-suffix, below 'test-suite-suffix', ), # the string to append to the `--test-suite` arugment when # chunking-args = test-suite-suffix; "<CHUNK>" in this string will # be replaced with the chunk number. Optional('chunk-suffix'): basestring, } ), # The current chunk; this is filled in by `all_kinds.py` Optional('this-chunk'): int, # os user groups for test task workers; required scopes, will be # added automatically Optional('os-groups', default=[]): optionally_keyed_by( 'test-platform', [basestring]), # -- values supplied by the task-generation infrastructure # the platform of the build this task is testing 'build-platform': basestring, # the label of the build task generating the materials to test 'build-label': basestring, # the platform on which the tests will run 'test-platform': basestring, # the name of the test (the key in tests.yml) 'test-name': basestring, # the product name, defaults to firefox Optional('product'): basestring, # conditional files to determine when these tests should be run Optional('when'): Any({ Optional('files-changed'): [basestring], }), }, required=True)
l10n_description_schema = Schema({ # Name for this job, inferred from the dependent job before validation Required('name'): basestring, # build-platform, inferred from dependent job before validation Required('build-platform'): basestring, # max run time of the task Required('run-time'): _by_platform(int), # Locales not to repack for Required('ignore-locales'): _by_platform([basestring]), # All l10n jobs use mozharness Required('mozharness'): { # Script to invoke for mozharness Required('script'): _by_platform(basestring), # Config files passed to the mozharness script Required('config'): _by_platform([basestring]), # Options to pass to the mozharness script Required('options'): _by_platform([basestring]), # Action commands to provide to mozharness script Required('actions'): _by_platform([basestring]), }, # Items for the taskcluster index Optional('index'): { # Product to identify as in the taskcluster index Required('product'): _by_platform(basestring), # Job name to identify as in the taskcluster index Required('job-name'): _by_platform(basestring), # Type of index Optional('type'): basestring, }, # Description of the localized task Required('description'): _by_platform(basestring), Optional('run-on-projects'): job_description_schema['run-on-projects'], # task object of the dependent task Required('dependent-task'): object, # worker-type to utilize Required('worker-type'): _by_platform(basestring), # File which contains the used locales Required('locales-file'): _by_platform(basestring), # Tooltool visibility required for task. Required('tooltool'): _by_platform(Any('internal', 'public')), # Docker image required for task. We accept only in-tree images # -- generally desktop-build or android-build -- for now. Required('docker-image'): _by_platform( Any( # an in-tree generated docker image (from `taskcluster/docker/<name>`) {'in-tree': basestring}, None, )), Optional('toolchains'): _by_platform([basestring]), # The set of secret names to which the task has access; these are prefixed # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting # this will enable any worker features required and set the task's scopes # appropriately. `true` here means ['*'], all secrets. Not supported on # Windows Required('secrets', default=False): _by_platform(Any(bool, [basestring])), # Information for treeherder Required('treeherder'): { # Platform to display the task on in treeherder Required('platform'): _by_platform(basestring), # Symbol to use Required('symbol'): basestring, # Tier this task is Required('tier'): _by_platform(int), }, # Extra environment values to pass to the worker Optional('env'): _by_platform({basestring: taskref_or_string}), # Max number locales per chunk Optional('locales-per-chunk'): _by_platform(int), # Task deps to chain this task with, added in transforms from dependent-task # if this is a nightly Optional('dependencies'): { basestring: basestring }, # Run the task when the listed files change (if present). Optional('when'): { 'files-changed': [basestring] }, # passed through directly to the job description Optional('attributes'): job_description_schema['attributes'], Optional('extra'): job_description_schema['extra'], })
) from taskgraph.util.hash import hash_paths from taskgraph import GECKO TOOLCHAIN_INDEX = 'gecko.cache.level-{level}.toolchains.v1.{name}.{digest}' toolchain_run_schema = Schema({ Required('using'): 'toolchain-script', # the script (in taskcluster/scripts/misc) to run Required('script'): basestring, # If not false, tooltool downloads will be enabled via relengAPIProxy # for either just public files, or all files. Not supported on Windows Required('tooltool-downloads', default=False): Any( False, 'public', 'internal', ), # Paths/patterns pointing to files that influence the outcome of a # toolchain build. Optional('resources'): [basestring], }) def add_optimizations(config, run, taskdesc): files = list(run.get('resources', [])) # This file files.append('taskcluster/taskgraph/transforms/job/toolchain.py') # The script
from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by from taskgraph.util.scriptworker import get_release_config from voluptuous import Optional, Required, Any from taskgraph.transforms.job import run_job_using buildbot_run_schema = Schema({ Required('using'): 'buildbot', # the buildername to use for buildbot-bridge, will expand {branch} in name from # the current project. Required('buildername'): basestring, # the product to use Required('product'): Any('firefox', 'mobile', 'fennec', 'devedition', 'thunderbird'), Optional('channels'): optionally_keyed_by('project', basestring), Optional('release-promotion'): bool, }) def _get_balrog_api_root(branch): if branch in ('mozilla-beta', 'mozilla-release') or branch.startswith('mozilla-esr'): return 'https://aus4-admin.mozilla.org/api' else:
diff_description_schema = Schema( { # Name of the diff task. Required("name"): text_type, # Treeherder tier. Required("tier"): int, # Treeherder symbol. Required("symbol"): text_type, # relative path (from config.path) to the file the task was defined in. Optional("job-from"): text_type, # Original and new builds to compare. Required("original"): index_or_string, Required("new"): index_or_string, # Arguments to pass to diffoscope, used for job-defaults in # taskcluster/ci/diffoscope/kind.yml Optional("args"): text_type, # Extra arguments to pass to diffoscope, that can be set per job. Optional("extra-args"): text_type, # Fail the task when differences are detected. Optional("fail-on-diff"): bool, # What artifact to check the differences of. Defaults to target.tar.bz2 # for Linux, target.dmg for Mac, target.zip for Windows, target.apk for # Android. Optional("artifact"): text_type, # Whether to unpack first. Diffoscope can normally work without unpacking, # but when one needs to --exclude some contents, that doesn't work out well # if said content is packed (e.g. in omni.ja). Optional("unpack"): bool, # Commands to run before performing the diff. Optional("pre-diff-commands"): [text_type], # Only run the task on a set of projects/branches. Optional("run-on-projects"): task_description_schema["run-on-projects"], Optional("optimization"): task_description_schema["optimization"], } )
from voluptuous import Required, Any from taskgraph.transforms.job import run_job_using from taskgraph.transforms.job.common import ( docker_worker_add_artifacts, generic_worker_add_artifacts, docker_worker_add_gecko_vcs_env_vars, docker_worker_add_tooltool, support_vcs_checkout, ) sm_run_schema = Schema({ Required('using'): Any('spidermonkey', 'spidermonkey-package', 'spidermonkey-mozjs-crate', 'spidermonkey-rust-bindings'), # The SPIDERMONKEY_VARIANT Required('spidermonkey-variant'): basestring, }) @run_job_using("docker-worker", "spidermonkey", schema=sm_run_schema) @run_job_using("docker-worker", "spidermonkey-package", schema=sm_run_schema) @run_job_using("docker-worker", "spidermonkey-mozjs-crate", schema=sm_run_schema) @run_job_using("docker-worker", "spidermonkey-rust-bindings", schema=sm_run_schema) def docker_worker_spidermonkey(config, job, taskdesc):
'mozilla-release': '', 'mozilla-beta': '-beta', 'mozilla-central': '-nightly', 'try': '-nightly-try', 'maple': '-nightly-maple', } task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} transforms = TransformSequence() beetmover_description_schema = Schema({ Required('dependent-task'): object, Required('depname', default='build'): basestring, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], Optional('bucket-scope'): optionally_keyed_by('project', basestring), Optional('shipping-phase'): task_description_schema['shipping-phase'], Optional('shipping-product'): task_description_schema['shipping-product'], }) @transforms.add def validate(config, jobs): for job in jobs: label = job.get('dependent-task', object).__dict__.get('label', '?no-label?') validate_schema( beetmover_description_schema, job, "In beetmover-geckoview ({!r} kind) task for {!r}:".format(config.kind, label)) yield job
from taskgraph.transforms.job.common import ( docker_worker_add_artifacts, generic_worker_add_artifacts, ) sm_run_schema = Schema({ Required("using"): Any( "spidermonkey", "spidermonkey-package", ), # SPIDERMONKEY_VARIANT and SPIDERMONKEY_PLATFORM Required("spidermonkey-variant"): text_type, Optional("spidermonkey-platform"): text_type, # Base work directory used to set up the task. Optional("workdir"): text_type, Required("tooltool-downloads"): Any( False, "public", "internal", ), }) @run_job_using("docker-worker", "spidermonkey", schema=sm_run_schema) @run_job_using("docker-worker", "spidermonkey-package", schema=sm_run_schema) def docker_worker_spidermonkey(config, job, taskdesc):
""" from __future__ import absolute_import, print_function, unicode_literals from six import text_type from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run from taskgraph.util.schema import Schema from voluptuous import Required, Optional python_test_schema = Schema({ Required("using"): "python-test", # Python version to use Required("python-version"): int, # The subsuite to run Required("subsuite"): text_type, # Base work directory used to set up the task. Optional("workdir"): text_type, }) defaults = { "python-version": 2, "subsuite": "default", } @run_job_using("docker-worker", "python-test",
task_description_schema = { str(k): v for k, v in task_description_schema.schema.iteritems() } transforms = TransformSequence() taskref_or_string = Any(basestring, {Required('task-reference'): basestring}) checksums_signing_description_schema = Schema({ Required('dependent-task'): object, Required('depname', default='beetmover'): basestring, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], Optional('shipping-product'): task_description_schema['shipping-product'], Optional('shipping-phase'): task_description_schema['shipping-phase'], }) @transforms.add def validate(config, jobs): for job in jobs: label = job.get('dependent-task', object).__dict__.get('label', '?no-label?') validate_schema( checksums_signing_description_schema, job,
job_description_schema = Schema({ # The name of the job and the job's label. At least one must be specified, # and the label will be generated from the name if necessary, by prepending # the kind. Optional('name'): basestring, Optional('label'): basestring, # the following fields are passed directly through to the task description, # possibly modified by the run implementation. See # taskcluster/taskgraph/transforms/task.py for the schema details. Required('description'): task_description_schema['description'], Optional('attributes'): task_description_schema['attributes'], Optional('dependencies'): task_description_schema['dependencies'], Optional('expires-after'): task_description_schema['expires-after'], Optional('routes'): task_description_schema['routes'], Optional('scopes'): task_description_schema['scopes'], Optional('tags'): task_description_schema['tags'], Optional('extra'): task_description_schema['extra'], Optional('treeherder'): task_description_schema['treeherder'], Optional('index'): task_description_schema['index'], Optional('run-on-projects'): task_description_schema['run-on-projects'], Optional('coalesce-name'): task_description_schema['coalesce-name'], Optional('optimizations'): task_description_schema['optimizations'], Optional('needs-sccache'): task_description_schema['needs-sccache'], # The "when" section contains descriptions of the circumstances # under which this task should be included in the task graph. This # will be converted into an element in the `optimizations` list. Optional('when'): Any({ # This task only needs to be run if a file matching one of the given # patterns has changed in the push. The patterns use the mozpack # match function (python/mozbuild/mozpack/path.py). Optional('files-changed'): [basestring], }), # A description of how to run this job. 'run': { # The key to a job implementation in a peer module to this one 'using': basestring, # Any remaining content is verified against that job implementation's # own schema. Extra: object, }, Required('worker-type'): task_description_schema['worker-type'], # This object will be passed through to the task description, with additions # provided by the job's run-using function Optional('worker'): dict, })
mozharness_run_schema = Schema({ Required('using'): 'mozharness', # the mozharness script used to run this task, relative to the testing/ # directory and using forward slashes even on Windows Required('script'): basestring, # the config files required for the task, relative to # testing/mozharness/configs and using forward slashes even on Windows Required('config'): [basestring], # any additional actions to pass to the mozharness command; not supported # on Windows Optional('actions'): [basestring], # any additional options (without leading --) to be passed to mozharness; # not supported on Windows Optional('options'): [basestring], # --custom-build-variant-cfg value (not supported on Windows) Optional('custom-build-variant-cfg'): basestring, # Extra metadata to use toward the workspace caching. # Only supported on docker-worker Optional('extra-workspace-cache-key'): basestring, # If not false, tooltool downloads will be enabled via relengAPIProxy # for either just public files, or all files. Not supported on Windows Required('tooltool-downloads', default=False): Any( False, 'public', 'internal', ), # The set of secret names to which the task has access; these are prefixed # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting # this will enable any worker features required and set the task's scopes # appropriately. `true` here means ['*'], all secrets. Not supported on # Windows Required('secrets', default=False): Any(bool, [basestring]), # If true, taskcluster proxy will be enabled; note that it may also be enabled # automatically e.g., for secrets support. Not supported on Windows. Required('taskcluster-proxy', default=False): bool, # If true, the build scripts will start Xvfb. Not supported on Windows. Required('need-xvfb', default=False): bool, # If false, indicate that builds should skip producing artifacts. Not # supported on Windows. Required('keep-artifacts', default=True): bool, # If specified, use the in-tree job script specified. Optional('job-script'): basestring, Required('requires-signed-builds', default=False): bool, # If false, don't set MOZ_SIMPLE_PACKAGE_NAME # Only disableable on windows Required('use-simple-package', default=True): bool, # If false don't pass --branch or --skip-buildbot-actions to mozharness script # Only disableable on windows Required('use-magic-mh-args', default=True): bool, })
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import copy from taskgraph.task import Task from taskgraph.util.attributes import sorted_unique_list from taskgraph.util.schema import Schema from voluptuous import Required schema = Schema( { Required("primary-dependency"): Task, Required("dependent-tasks"): {str: Task}, }, ) GROUP_BY_MAP = {} def group_by(name): def wrapper(func): GROUP_BY_MAP[name] = func return func return wrapper @group_by("addon-type")
docker_worker_add_artifacts, docker_worker_add_tooltool, ) haz_run_schema = Schema({ Required('using'): 'hazard', # The command to run within the task image (passed through to the worker) Required('command'): basestring, # The mozconfig to use; default in the script is used if omitted Optional('mozconfig'): basestring, # The set of secret names to which the task has access; these are prefixed # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting # this will enable any worker features required and set the task's scopes # appropriately. `true` here means ['*'], all secrets. Not supported on # Windows Required('secrets', default=False): Any(bool, [basestring]), # Base work directory used to set up the task. Required('workdir'): basestring, }) @run_job_using("docker-worker", "hazard", schema=haz_run_schema) def docker_worker_hazard(config, job, taskdesc):
docker_image_schema = Schema({ # Name of the docker image. Required('name'): basestring, # Name of the parent docker image. Optional('parent'): basestring, # Treeherder symbol. Required('symbol'): basestring, # relative path (from config.path) to the file the docker image was defined # in. Optional('job-from'): basestring, # Arguments to use for the Dockerfile. Optional('args'): { basestring: basestring }, # Name of the docker image definition under taskcluster/docker, when # different from the docker image name. Optional('definition'): basestring, # List of package tasks this docker image depends on. Optional('packages'): [basestring], })
run_task_schema = Schema({ Required('using'): 'run-task', # if true, add a cache at ~worker/.cache, which is where things like pip # tend to hide their caches. This cache is never added for level-1 jobs. Required('cache-dotcache', default=False): bool, # if true (the default), perform a checkout in /builds/worker/checkouts/gecko Required('checkout', default=True): bool, # The sparse checkout profile to use. Value is the filename relative to the # directory where sparse profiles are defined (build/sparse-profiles/). Required('sparse-profile', default=None): basestring, # if true, perform a checkout of a comm-central based branch inside the # gecko checkout Required('comm-checkout', default=False): bool, # The command arguments to pass to the `run-task` script, after the # checkout arguments. If a list, it will be passed directly; otherwise # it will be included in a single argument to `bash -cx`. Required('command'): Any([basestring], basestring), })
toolchain_run_schema = Schema({ Required('using'): 'toolchain-script', # The script (in taskcluster/scripts/misc) to run. # Python scripts are invoked with `mach python` so vendored libraries # are available. Required('script'): text_type, # Arguments to pass to the script. Optional('arguments'): [text_type], # If not false, tooltool downloads will be enabled via relengAPIProxy # for either just public files, or all files. Not supported on Windows Required('tooltool-downloads'): Any( False, 'public', 'internal', ), # Sparse profile to give to checkout using `run-task`. If given, # a filename in `build/sparse-profiles`. Defaults to # "toolchain-build", i.e., to # `build/sparse-profiles/toolchain-build`. If `None`, instructs # `run-task` to not use a sparse profile at all. Required('sparse-profile'): Any(text_type, None), # Paths/patterns pointing to files that influence the outcome of a # toolchain build. Optional('resources'): [text_type], # Path to the artifact produced by the toolchain job Required('toolchain-artifact'): text_type, Optional( "toolchain-alias", description="An alias that can be used instead of the real toolchain job name in " "fetch stanzas for jobs.", ): text_type, # Base work directory used to set up the task. Required('workdir'): text_type, })
configure_taskdesc_for_run, ) from taskgraph.transforms.job.common import ( docker_worker_add_artifacts, generic_worker_add_artifacts, docker_worker_add_tooltool, ) sm_run_schema = Schema({ Required('using'): Any('spidermonkey', 'spidermonkey-package', 'spidermonkey-mozjs-crate', 'spidermonkey-rust-bindings'), # SPIDERMONKEY_VARIANT and SPIDERMONKEY_PLATFORM Required('spidermonkey-variant'): basestring, Optional('spidermonkey-platform'): basestring, # Base work directory used to set up the task. Required('workdir'): basestring, }) @run_job_using("docker-worker", "spidermonkey", schema=sm_run_schema) @run_job_using("docker-worker", "spidermonkey-package", schema=sm_run_schema) @run_job_using("docker-worker", "spidermonkey-mozjs-crate", schema=sm_run_schema) @run_job_using("docker-worker",
def get_variant(test_platform): for v in VARIANTS: if '-{}/'.format(v) in test_platform: return v return '' test_description_schema = { str(k): v for k, v in test_description_schema.schema.iteritems() } mozharness_test_run_schema = Schema({ Required('using'): 'mozharness-test', Required('test'): test_description_schema, }) def test_packages_url(taskdesc): """Account for different platforms that name their test packages differently""" return get_artifact_url('<build>', 'public/build/target.test_packages.json') @run_job_using('docker-engine', 'mozharness-test', schema=mozharness_test_run_schema) @run_job_using('docker-worker', 'mozharness-test', schema=mozharness_test_run_schema)
) diff_description_schema = Schema({ # Name of the diff task. Required('name'): basestring, # Treeherder symbol. Required('symbol'): basestring, # relative path (from config.path) to the file the task was defined in. Optional('job-from'): basestring, # Original and new builds to compare. Required('original'): index_or_string, Required('new'): index_or_string, # Arguments to pass to diffoscope, used for job-defaults in # taskcluster/ci/diffoscope/kind.yml Optional('args'): basestring, # Extra arguments to pass to diffoscope, that can be set per job. Optional('extra-args'): basestring, })