# file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, print_function, unicode_literals import copy from voluptuous import Required from taskgraph.task import Task from taskgraph.util.attributes import sorted_unique_list from taskgraph.util.schema import Schema schema = Schema({ Required('primary-dependency', 'primary dependency task'): Task, Required( 'dependent-tasks', 'dictionary of dependent tasks, keyed by kind', ): {basestring: Task}, }) # Define a collection of group_by functions GROUP_BY_MAP = {} def group_by(name): def wrapper(func): GROUP_BY_MAP[name] = func return func return wrapper
cron_yml_schema = Schema({ 'jobs': [{ # Name of the crontask (must be unique) Required('name'): basestring, # what to run # Description of the job to run, keyed by 'type' Required('job'): Any({ Required('type'): 'decision-task', # Treeherder symbol for the cron task Required('treeherder-symbol'): basestring, # --target-tasks-method './mach taskgraph decision' argument 'target-tasks-method': basestring, }), # when to run it # Optional set of projects on which this job should run; if omitted, this will # run on all projects for which cron tasks are set up. This works just like the # `run_on_projects` attribute, where strings like "release" and "integration" are # expanded to cover multiple repositories. (taskcluster/docs/attributes.rst) 'run-on-projects': [basestring], # Array of times at which this task should run. These *must* be a # multiple of 15 minutes, the minimum scheduling interval. This field # can be keyed by project so that each project has a different schedule # for the same job. 'when': optionally_keyed_by( 'project', [{'hour': int, 'minute': All(int, even_15_minutes)}]), }], })
signing_description_schema = Schema({ # the dependant task (object) for this signing job, used to inform signing. Required('dependent-task'): object, # Artifacts from dep task to sign - Sync with taskgraph/transforms/task.py # because this is passed directly into the signingscript worker Required('upstream-artifacts'): [{ # taskId of the task with the artifact Required('taskId'): taskref_or_string, # type of signing task (for CoT) Required('taskType'): basestring, # Paths to the artifacts to sign Required('paths'): [basestring], # Signing formats to use on each of the paths Required('formats'): [basestring], }], # depname is used in taskref's to identify the taskID of the unsigned things Required('depname'): basestring, # unique label to describe this signing task, defaults to {dep.label}-signing Optional('label'): basestring, # treeherder is allowed here to override any defaults we use for signing. See # taskcluster/taskgraph/transforms/task.py for the schema details, and the # below transforms for defaults of various values. Optional('treeherder'): task_description_schema['treeherder'], # Routes specific to this task, if defined Optional('routes'): [basestring], Optional('shipping-phase'): task_description_schema['shipping-phase'], Optional('shipping-product'): task_description_schema['shipping-product'], # Optional control for how long a task may run (aka maxRunTime) Optional('max-run-time'): int, Optional('extra'): { basestring: object }, })
job_description_schema = Schema({ # The name of the job and the job's label. At least one must be specified, # and the label will be generated from the name if necessary, by prepending # the kind. Optional('name'): basestring, Optional('label'): basestring, # the following fields are passed directly through to the task description, # possibly modified by the run implementation. See # taskcluster/taskgraph/transforms/task.py for the schema details. Required('description'): task_description_schema['description'], Optional('attributes'): task_description_schema['attributes'], Optional('dependencies'): task_description_schema['dependencies'], Optional('expires-after'): task_description_schema['expires-after'], Optional('routes'): task_description_schema['routes'], Optional('scopes'): task_description_schema['scopes'], Optional('tags'): task_description_schema['tags'], Optional('extra'): task_description_schema['extra'], Optional('treeherder'): task_description_schema['treeherder'], Optional('index'): task_description_schema['index'], Optional('run-on-projects'): task_description_schema['run-on-projects'], Optional('coalesce-name'): task_description_schema['coalesce-name'], Optional('optimizations'): task_description_schema['optimizations'], Optional('needs-sccache'): task_description_schema['needs-sccache'], # The "when" section contains descriptions of the circumstances # under which this task should be included in the task graph. This # will be converted into an element in the `optimizations` list. Optional('when'): Any({ # This task only needs to be run if a file matching one of the given # patterns has changed in the push. The patterns use the mozpack # match function (python/mozbuild/mozpack/path.py). Optional('files-changed'): [basestring], }), # A description of how to run this job. 'run': { # The key to a job implementation in a peer module to this one 'using': basestring, # Any remaining content is verified against that job implementation's # own schema. Extra: object, }, Required('worker-type'): Any( task_description_schema['worker-type'], {'by-platform': { basestring: task_description_schema['worker-type'] }}, ), Required('worker'): Any( task_description_schema['worker'], {'by-platform': { basestring: task_description_schema['worker'] }}, ), })
l10n_description_schema = Schema({ # Name for this job, inferred from the dependent job before validation Required('name'): basestring, # build-platform, inferred from dependent job before validation Required('build-platform'): basestring, # max run time of the task Required('run-time'): _by_platform(int), # Locales not to repack for Required('ignore-locales'): _by_platform([basestring]), # All l10n jobs use mozharness Required('mozharness'): { # Script to invoke for mozharness Required('script'): _by_platform(basestring), # Config files passed to the mozharness script Required('config'): _by_platform([basestring]), # Options to pass to the mozharness script Required('options'): _by_platform([basestring]), # Action commands to provide to mozharness script Required('actions'): _by_platform([basestring]), }, # Items for the taskcluster index Optional('index'): { # Product to identify as in the taskcluster index Required('product'): _by_platform(basestring), # Job name to identify as in the taskcluster index Required('job-name'): _by_platform(basestring), # Type of index Optional('type'): basestring, }, # Description of the localized task Required('description'): _by_platform(basestring), Optional('run-on-projects'): job_description_schema['run-on-projects'], # task object of the dependent task Required('dependent-task'): object, # worker-type to utilize Required('worker-type'): _by_platform(basestring), # File which contains the used locales Required('locales-file'): _by_platform(basestring), # Tooltool visibility required for task. Required('tooltool'): _by_platform(Any('internal', 'public')), # Docker image required for task. We accept only in-tree images # -- generally desktop-build or android-build -- for now. Required('docker-image'): _by_platform( Any( # an in-tree generated docker image (from `taskcluster/docker/<name>`) {'in-tree': basestring}, None, )), Optional('toolchains'): _by_platform([basestring]), # Information for treeherder Required('treeherder'): { # Platform to display the task on in treeherder Required('platform'): _by_platform(basestring), # Symbol to use Required('symbol'): basestring, # Tier this task is Required('tier'): _by_platform(int), }, # Extra environment values to pass to the worker Optional('env'): _by_platform({basestring: taskref_or_string}), # Max number locales per chunk Optional('locales-per-chunk'): _by_platform(int), # Task deps to chain this task with, added in transforms from dependent-task # if this is a nightly Optional('dependencies'): { basestring: basestring }, # Run the task when the listed files change (if present). Optional('when'): { 'files-changed': [basestring] }, # passed through directly to the job description Optional('attributes'): job_description_schema['attributes'], Optional('extra'): job_description_schema['extra'], })
run_task_schema = Schema({ Required('using'): 'run-task', # if true, add a cache at ~worker/.cache, which is where things like pip # tend to hide their caches. This cache is never added for level-1 jobs. # TODO Once bug 1526028 is fixed, this and 'use-caches' should be merged. Required('cache-dotcache'): bool, # Whether or not to use caches. Optional('use-caches'): bool, # if true (the default), perform a checkout of gecko on the worker Required('checkout'): bool, Optional( "cwd", description="Path to run command in. If a checkout is present, the path " "to the checkout will be interpolated with the key `checkout`", ): text_type, # The sparse checkout profile to use. Value is the filename relative to the # directory where sparse profiles are defined (build/sparse-profiles/). Required('sparse-profile'): Any(basestring, None), # if true, perform a checkout of a comm-central based branch inside the # gecko checkout Required('comm-checkout'): bool, # The command arguments to pass to the `run-task` script, after the # checkout arguments. If a list, it will be passed directly; otherwise # it will be included in a single argument to `bash -cx`. Required('command'): Any([taskref_or_string], taskref_or_string), # Base work directory used to set up the task. Required('workdir'): basestring, # If not false, tooltool downloads will be enabled via relengAPIProxy # for either just public files, or all files. Only supported on # docker-worker. Required('tooltool-downloads'): Any( False, 'public', 'internal', ), # Whether to run as root. (defaults to False) Optional('run-as-root'): bool, })
diff_description_schema = Schema({ # Name of the diff task. Required('name'): text_type, # Treeherder symbol. Required('symbol'): text_type, # relative path (from config.path) to the file the task was defined in. Optional('job-from'): text_type, # Original and new builds to compare. Required('original'): index_or_string, Required('new'): index_or_string, # Arguments to pass to diffoscope, used for job-defaults in # taskcluster/ci/diffoscope/kind.yml Optional('args'): text_type, # Extra arguments to pass to diffoscope, that can be set per job. Optional('extra-args'): text_type, # Fail the task when differences are detected. Optional('fail-on-diff'): bool, # What artifact to check the differences of. Defaults to target.tar.bz2 # for Linux, target.dmg for Mac, target.zip for Windows, target.apk for # Android. Optional('artifact'): text_type, # Whether to unpack first. Diffoscope can normally work without unpacking, # but when one needs to --exclude some contents, that doesn't work out well # if said content is packed (e.g. in omni.ja). Optional('unpack'): bool, # Commands to run before performing the diff. Optional('pre-diff-commands'): [text_type], # Only run the task on a set of projects/branches. Optional('run-on-projects'): task_description_schema['run-on-projects'], Optional('optimization'): task_description_schema['optimization'], })
docker_worker_add_artifacts, add_tooltool, ) haz_run_schema = Schema({ Required('using'): 'hazard', # The command to run within the task image (passed through to the worker) Required('command'): basestring, # The mozconfig to use; default in the script is used if omitted Optional('mozconfig'): basestring, # The set of secret names to which the task has access; these are prefixed # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting # this will enable any worker features required and set the task's scopes # appropriately. `true` here means ['*'], all secrets. Not supported on # Windows Required('secrets', default=False): Any(bool, [basestring]), # Base work directory used to set up the task. Required('workdir'): basestring, }) @run_job_using("docker-worker", "hazard", schema=haz_run_schema) def docker_worker_hazard(config, job, taskdesc):
raptor_description_schema = Schema({ # Raptor specific configs. Optional('apps'): optionally_keyed_by('test-platform', [basestring]), Optional('raptor-test'): basestring, Optional('activity'): optionally_keyed_by('app', basestring), Optional('binary-path'): optionally_keyed_by('app', basestring), Optional('pageload'): optionally_keyed_by( 'test-platform', 'app', Any('cold', 'warm', 'both'), ), # Configs defined in the 'test_description_schema'. Optional('max-run-time'): optionally_keyed_by('app', test_description_schema['max-run-time']), Optional('run-on-projects'): optionally_keyed_by('app', test_description_schema['run-on-projects']), Optional('variants'): optionally_keyed_by('app', test_description_schema['variants']), Optional('target'): optionally_keyed_by('app', test_description_schema['target']), Optional('run-visual-metrics'): optionally_keyed_by('app', bool), Required('test-name'): test_description_schema['test-name'], Required('test-platform'): test_description_schema['test-platform'], Required('require-signed-extensions'): test_description_schema['require-signed-extensions'], Required('treeherder-symbol'): test_description_schema['treeherder-symbol'], # Any unrecognized keys will be validated against the test_description_schema. Extra: object, })
diff_description_schema = Schema({ # Name of the diff task. Required('name'): basestring, # Treeherder symbol. Required('symbol'): basestring, # relative path (from config.path) to the file the task was defined in. Optional('job-from'): basestring, # Original and new builds to compare. Required('original'): index_or_string, Required('new'): index_or_string, # Arguments to pass to diffoscope, used for job-defaults in # taskcluster/ci/diffoscope/kind.yml Optional('args'): basestring, # Extra arguments to pass to diffoscope, that can be set per job. Optional('extra-args'): basestring, # Fail the task when differences are detected. Optional('fail-on-diff'): bool, # Whether to unpack first. Diffoscope can normally work without unpacking, # but when one needs to --exclude some contents, that doesn't work out well # if said content is packed (e.g. in omni.ja). Optional('unpack'): bool, # Commands to run before performing the diff. Optional('pre-diff-commands'): [basestring], })
packaging_description_schema = Schema({ # the dependant task (object) for this job, used to inform repackaging. Required('dependent-task'): object, # depname is used in taskref's to identify the taskID of the signed things Required('depname', default='build'): basestring, # unique label to describe this repackaging task Optional('label'): basestring, # treeherder is allowed here to override any defaults we use for repackaging. See # taskcluster/taskgraph/transforms/task.py for the schema details, and the # below transforms for defaults of various values. Optional('treeherder'): task_description_schema['treeherder'], # If a l10n task, the corresponding locale Optional('locale'): basestring, # Routes specific to this task, if defined Optional('routes'): [basestring], # passed through directly to the job description Optional('extra'): task_description_schema['extra'], })
Support for running mach python-test tasks (via run-task) """ from __future__ import absolute_import, print_function, unicode_literals from six import text_type from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run from taskgraph.util.schema import Schema from voluptuous import Required, Optional python_test_schema = Schema({ Required('using'): 'python-test', # Python version to use Required('python-version'): int, # The subsuite to run Required('subsuite'): text_type, # Base work directory used to set up the task. Optional('workdir'): text_type, }) defaults = { 'python-version': 2, 'subsuite': 'default', } @run_job_using('docker-worker', 'python-test', schema=python_test_schema, defaults=defaults) @run_job_using('generic-worker', 'python-test', schema=python_test_schema, defaults=defaults)
for k, v in task_description_schema.schema.iteritems() } transforms = TransformSequence() # shortcut for a string where task references are allowed taskref_or_string = Any(basestring, {Required('task-reference'): basestring}) balrog_description_schema = Schema({ # the dependent task (object) for this balrog job, used to inform balrogworker. Required('dependent-task'): object, # unique label to describe this balrog task, defaults to balrog-{dep.label} Optional('label'): basestring, # treeherder is allowed here to override any defaults we use for beetmover. See # taskcluster/taskgraph/transforms/task.py for the schema details, and the # below transforms for defaults of various values. Optional('treeherder'): task_description_schema['treeherder'], }) @transforms.add def validate(config, jobs): for job in jobs: label = job.get('dependent-task', object).__dict__.get('label', '?no-label?') yield validate_schema(
run_schema = Schema( { Required("using"): "debian-package", # Debian distribution Required("dist"): text_type, # Date of the snapshot (from snapshot.debian.org) to use, in the format # YYYYMMDDTHHMMSSZ. The same date is used for the base docker-image name # (only the YYYYMMDD part). Required("snapshot"): text_type, # URL/SHA256 of a source file to build, which can either be a source # control (.dsc), or a tarball. Required(Any("dsc", "tarball")): source_definition, # Package name. Normally derived from the source control or tarball file # name. Use in case the name doesn't match DSC_PACKAGE_RE or # SOURCE_PACKAGE_RE. Optional("name"): text_type, # Patch to apply to the extracted source. Optional("patch"): text_type, # Command to run before dpkg-buildpackage. Optional("pre-build-command"): text_type, # Architecture to build the package for. Optional("arch"): text_type, # List of package tasks to get build dependencies from. Optional("packages"): [text_type], # What resolver to use to install build dependencies. The default # (apt-get) is good in most cases, but in subtle cases involving # a *-backports archive, its solver might not be able to find a # solution that satisfies the build dependencies. Optional("resolver"): Any("apt-get", "aptitude"), # Base work directory used to set up the task. Required("workdir"): text_type, } )
Support for running mach tasks (via run-task) """ from __future__ import absolute_import, print_function, unicode_literals from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run from taskgraph.util.schema import Schema from voluptuous import Required mach_schema = Schema({ Required('using'): 'mach', # The mach command (omitting `./mach`) to run Required('mach'): basestring, # if true, perform a checkout of a comm-central based branch inside the # gecko checkout Required('comm-checkout'): bool, }) @run_job_using("docker-worker", "mach", schema=mach_schema, defaults={'comm-checkout': False}) @run_job_using("native-engine", "mach", schema=mach_schema,
from voluptuous import Optional, Required push_snap_description_schema = Schema({ Required('name'): basestring, Required('job-from'): task_description_schema['job-from'], Required('dependencies'): task_description_schema['dependencies'], Required('description'): task_description_schema['description'], Required('treeherder'): task_description_schema['treeherder'], Required('run-on-projects'): task_description_schema['run-on-projects'], Required('worker-type'): optionally_keyed_by('release-level', basestring), Required('worker'): object, Optional('scopes'): [basestring], Required('shipping-phase'): task_description_schema['shipping-phase'], Required('shipping-product'): task_description_schema['shipping-product'], Optional('extra'): task_description_schema['extra'], Optional('attributes'): task_description_schema['attributes'], }) transforms = TransformSequence()
'stylo-sequential', 'qr', 'ccov', ] def get_variant(test_platform): for v in VARIANTS: if '-{}/'.format(v) in test_platform: return v return '' mozharness_test_run_schema = Schema({ Required('using'): 'mozharness-test', Required('test'): test_description_schema, # Base work directory used to set up the task. Required('workdir'): basestring, }) def test_packages_url(taskdesc): """Account for different platforms that name their test packages differently""" artifact_url = get_artifact_url('<build>', get_artifact_path(taskdesc, 'target.test_packages.json')) # for android nightly we need to add 'en-US' to the artifact url test = taskdesc['run']['test'] if 'android' in test['test-platform'] and ( get_variant(test['test-platform']) in ("nightly", 'shippable')): head, tail = os.path.split(artifact_url) artifact_url = os.path.join(head, 'en-US', tail) return artifact_url
# Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = { str(k): v for k, v in task_description_schema.schema.iteritems() } transforms = TransformSequence() repackage_signing_description_schema = Schema({ Required('dependent-task'): object, Required('depname', default='repackage'): basestring, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], Optional('shipping-product'): task_description_schema['shipping-product'], Optional('shipping-phase'): task_description_schema['shipping-phase'], }) SIGNING_FORMATS = { 'target.complete.mar': ["autograph_mar384"], 'target.bz2.complete.mar': ["mar"], "target.installer.exe": ["sha2signcode"], "target.stub-installer.exe": ["sha2signcodestub"], }
current taskgraph. The transform takes a list of indexes, and the optimization phase will replace the task with the task from the other graph. """ from taskgraph.transforms.base import TransformSequence from taskgraph.util.schema import Schema from taskgraph.transforms.job import run_job_using from voluptuous import Required transforms = TransformSequence() run_task_schema = Schema({ Required("using"): "index-search", Required( "index-search", "A list of indexes in decreasing order of priority at which to lookup for this " "task. This is interpolated with the graph parameters.", ): [str], }) @run_job_using("always-optimized", "index-search", schema=run_task_schema) def fill_template(config, job, taskdesc): run = job["run"] taskdesc["optimization"] = { "index-search": [index.format(**config.params) for index in run["index-search"]] }
from taskgraph.util.schema import ( Schema, taskref_or_string, ) from voluptuous import Required, Optional, Any mach_schema = Schema( { Required("using"): "mach", # The mach command (omitting `./mach`) to run Required("mach"): taskref_or_string, # The version of Python to run with. Either an absolute path to the binary # on the worker, a version identifier (e.g python2.7 or 3.8). There is no # validation performed to ensure the specified binaries actually exist. Optional("python-version"): Any(text_type, int, float), # The sparse checkout profile to use. Value is the filename relative to the # directory where sparse profiles are defined (build/sparse-profiles/). Optional("sparse-profile"): Any(text_type, None), # if true, perform a checkout of a comm-central based branch inside the # gecko checkout Required("comm-checkout"): bool, # Base work directory used to set up the task. Optional("workdir"): text_type, } ) defaults = { "comm-checkout": False, }
from taskgraph.util import path from taskgraph.util.schema import Schema from voluptuous import Required, Optional from six import text_type from pipes import quote as shell_quote gradlew_schema = Schema({ Required("using"): "gradlew", Required("gradlew"): [text_type], Optional("post-gradlew"): [[text_type]], # Base work directory used to set up the task. Required("workdir"): text_type, Optional("use-caches"): bool, Optional("secrets"): [{ Required("name"): text_type, Required("path"): text_type, Required("key"): text_type, Optional("json"): bool, }], }) @run_job_using("docker-worker", "gradlew", schema=gradlew_schema) def configure_gradlew(config, job, taskdesc): run = job["run"] worker = taskdesc["worker"] = job["worker"]
mozharness_run_schema = Schema( { Required("using"): "mozharness", # the mozharness script used to run this task, relative to the testing/ # directory and using forward slashes even on Windows Required("script"): text_type, # Additional paths to look for mozharness configs in. These should be # relative to the base of the source checkout Optional("config-paths"): [text_type], # the config files required for the task, relative to # testing/mozharness/configs or one of the paths specified in # `config-paths` and using forward slashes even on Windows Required("config"): [text_type], # any additional actions to pass to the mozharness command Optional("actions"): [ Match("^[a-z0-9-]+$", "actions must be `-` seperated alphanumeric strings") ], # any additional options (without leading --) to be passed to mozharness Optional("options"): [ Match( "^[a-z0-9-]+(=[^ ]+)?$", "options must be `-` seperated alphanumeric strings (with optional argument)", ) ], # --custom-build-variant-cfg value Optional("custom-build-variant-cfg"): text_type, # Extra configuration options to pass to mozharness. Optional("extra-config"): dict, # If not false, tooltool downloads will be enabled via relengAPIProxy # for either just public files, or all files. Not supported on Windows Required("tooltool-downloads"): Any( False, "public", "internal", ), # The set of secret names to which the task has access; these are prefixed # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting # this will enable any worker features required and set the task's scopes # appropriately. `true` here means ['*'], all secrets. Not supported on # Windows Required("secrets"): Any(bool, [text_type]), # If true, taskcluster proxy will be enabled; note that it may also be enabled # automatically e.g., for secrets support. Not supported on Windows. Required("taskcluster-proxy"): bool, # If true, the build scripts will start Xvfb. Not supported on Windows. Required("need-xvfb"): bool, # If false, indicate that builds should skip producing artifacts. Not # supported on Windows. Required("keep-artifacts"): bool, # If specified, use the in-tree job script specified. Optional("job-script"): text_type, Required("requires-signed-builds"): bool, # Whether or not to use caches. Optional("use-caches"): bool, # If false, don't set MOZ_SIMPLE_PACKAGE_NAME # Only disableable on windows Required("use-simple-package"): bool, # If false don't pass --branch mozharness script # Only disableable on windows Required("use-magic-mh-args"): bool, # if true, perform a checkout of a comm-central based branch inside the # gecko checkout Required("comm-checkout"): bool, # Base work directory used to set up the task. Optional("workdir"): text_type, } )
""" from __future__ import absolute_import, print_function, unicode_literals from taskgraph.transforms.job import run_job_using from taskgraph.transforms.job.run_task import ( docker_worker_run_task, native_engine_run_task, ) from taskgraph.util.schema import Schema from voluptuous import Required mach_schema = Schema({ Required('using'): 'mach', # The mach command (omitting `./mach`) to run Required('mach'): basestring, }) @run_job_using("docker-worker", "mach", schema=mach_schema) @run_job_using("native-engine", "mach", schema=mach_schema) def docker_worker_mach(config, job, taskdesc): run = job['run'] # defer to the run_task implementation run['command'] = 'cd /builds/worker/checkouts/gecko && ./mach ' + run[ 'mach'] run['checkout'] = True run['sparse-profile'] = None
raptor_description_schema = Schema({ # Raptor specific configs. Optional("apps"): optionally_keyed_by("test-platform", "subtest", [text_type]), Optional("raptor-test"): text_type, Optional("raptor-subtests"): optionally_keyed_by("app", "test-platform", list), Optional("activity"): optionally_keyed_by("app", text_type), Optional("binary-path"): optionally_keyed_by("app", text_type), # Configs defined in the 'test_description_schema'. Optional("max-run-time"): optionally_keyed_by("app", "subtest", "test-platform", test_description_schema["max-run-time"]), Optional("run-on-projects"): optionally_keyed_by( "app", "test-name", "raptor-test", "subtest", "variant", test_description_schema["run-on-projects"], ), Optional("variants"): optionally_keyed_by("app", "subtest", test_description_schema["variants"]), Optional("target"): optionally_keyed_by("app", test_description_schema["target"]), Optional("tier"): optionally_keyed_by("app", "raptor-test", "subtest", "variant", test_description_schema["tier"]), Optional("test-url-param"): optionally_keyed_by("subtest", "test-platform", text_type), Optional("run-visual-metrics"): optionally_keyed_by("app", bool), Required("test-name"): test_description_schema["test-name"], Required("test-platform"): test_description_schema["test-platform"], Required("require-signed-extensions"): test_description_schema["require-signed-extensions"], Required("treeherder-symbol"): test_description_schema["treeherder-symbol"], # Any unrecognized keys will be validated against the test_description_schema. Extra: object, })
# Voluptuous uses marker objects as dictionary *keys*, but they are not # comparable, so we cast all of the keys back to regular strings task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} transforms = TransformSequence() taskref_or_string = Any( basestring, {Required('task-reference'): basestring}) beetmover_checksums_description_schema = Schema({ Required('dependent-task'): object, Required('depname', default='build'): basestring, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], Optional('locale'): basestring, Optional('shipping-phase'): task_description_schema['shipping-phase'], Optional('shipping-product'): task_description_schema['shipping-product'], }) @transforms.add def validate(config, jobs): for job in jobs: label = job.get('dependent-task', object).__dict__.get('label', '?no-label?') validate_schema( beetmover_checksums_description_schema, job, "In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label)) yield job
run_schema = Schema({ Required('using'): 'debian-package', # Debian distribution Required('dist'): basestring, # Date of the snapshot (from snapshot.debian.org) to use, in the format # YYYYMMDDTHHMMSSZ. The same date is used for the base docker-image name # (only the YYYYMMDD part). Required('snapshot'): basestring, # URL/SHA256 of a source file to build, which can either be a source # control (.dsc), or a tarball. Required(Any('dsc', 'tarball')): source_definition, # Patch to apply to the extracted source. Optional('patch'): basestring, # Command to run before dpkg-buildpackage. Optional('pre-build-command'): basestring, # List of package tasks to get build dependencies from. Optional('packages'): [basestring], # What resolver to use to install build dependencies. The default # (apt-get) is good in most cases, but in subtle cases involving # a *-backports archive, its solver might not be able to find a # solution that satisfies the build dependencies. Optional('resolver'): Any('apt-get', 'aptitude'), })
taskref_or_string = Any(basestring, {Required('task-reference'): basestring}) beetmover_description_schema = Schema({ # the dependent task (object) for this beetmover job, used to inform beetmover. Required('dependent-task'): object, # depname is used in taskref's to identify the taskID of the unsigned things Required('depname', default='build'): basestring, # unique label to describe this beetmover task, defaults to {dep.label}-beetmover Optional('label'): basestring, # treeherder is allowed here to override any defaults we use for beetmover. See # taskcluster/taskgraph/transforms/task.py for the schema details, and the # below transforms for defaults of various values. Optional('treeherder'): task_description_schema['treeherder'], # locale is passed only for l10n beetmoving Optional('locale'): basestring, Optional('shipping-phase'): task_description_schema['shipping-phase'], Optional('shipping-product'): task_description_schema['shipping-product'], })
test_description_schema = Schema( { # description of the suite, for the task metadata 'description': basestring, # test suite name, or <suite>/<flavor> Required('suite'): optionally_keyed_by('test-platform', basestring), # the name by which this test suite is addressed in try syntax; defaults to # the test-name. This will translate to the `unittest_try_name` or # `talos_try_name` attribute. Optional('try-name'): basestring, # additional tags to mark up this type of test Optional('tags'): { basestring: object }, # the symbol, or group(symbol), under which this task should appear in # treeherder. 'treeherder-symbol': basestring, # the value to place in task.extra.treeherder.machine.platform; ideally # this is the same as build-platform, and that is the default, but in # practice it's not always a match. Optional('treeherder-machine-platform'): basestring, # attributes to appear in the resulting task (later transforms will add the # common attributes) Optional('attributes'): { basestring: object }, # relative path (from config.path) to the file task was defined in Optional('job-from'): basestring, # The `run_on_projects` attribute, defaulting to "all". This dictates the # projects on which this task should be included in the target task set. # See the attributes documentation for details. # # Note that the special case 'built-projects', the default, uses the parent # build task's run-on-projects, meaning that tests run only on platforms # that are built. Optional('run-on-projects'): optionally_keyed_by('test-platform', Any([basestring], 'built-projects')), # the sheriffing tier for this task (default: set based on test platform) Optional('tier'): optionally_keyed_by('test-platform', Any(int, 'default')), # number of chunks to create for this task. This can be keyed by test # platform by passing a dictionary in the `by-test-platform` key. If the # test platform is not found, the key 'default' will be tried. Required('chunks'): optionally_keyed_by('test-platform', int), # the time (with unit) after which this task is deleted; default depends on # the branch (see below) Optional('expires-after'): basestring, # Whether to run this task with e10s (desktop-test only). If false, run # without e10s; if true, run with e10s; if 'both', run one task with and # one task without e10s. E10s tasks have "-e10s" appended to the test name # and treeherder group. Required('e10s'): optionally_keyed_by('test-platform', 'project', Any(bool, 'both')), # Whether the task should run with WebRender enabled or not. Optional('webrender'): bool, # The EC2 instance size to run these tests on. Required('instance-size'): optionally_keyed_by('test-platform', Any('default', 'large', 'xlarge')), # type of virtualization or hardware required by test. Required('virtualization'): optionally_keyed_by('test-platform', Any('virtual', 'virtual-with-gpu', 'hardware')), # Whether the task requires loopback audio or video (whatever that may mean # on the platform) Required('loopback-audio'): bool, Required('loopback-video'): bool, # Whether the test can run using a software GL implementation on Linux # using the GL compositor. May not be used with "legacy" sized instances # due to poor LLVMPipe performance (bug 1296086). Defaults to true for # unit tests on linux platforms and false otherwise Optional('allow-software-gl-layers'): bool, # For tasks that will run in docker-worker or docker-engine, this is the # name of the docker image or in-tree docker image to run the task in. If # in-tree, then a dependency will be created automatically. This is # generally `desktop-test`, or an image that acts an awful lot like it. Required('docker-image'): optionally_keyed_by( 'test-platform', Any( # a raw Docker image path (repo/image:tag) basestring, # an in-tree generated docker image (from `taskcluster/docker/<name>`) {'in-tree': basestring}, # an indexed docker image {'indexed': basestring}, )), # seconds of runtime after which the task will be killed. Like 'chunks', # this can be keyed by test pltaform. Required('max-run-time'): optionally_keyed_by('test-platform', int), # the exit status code that indicates the task should be retried Optional('retry-exit-status'): [int], # Whether to perform a gecko checkout. Required('checkout'): bool, # Wheter to perform a machine reboot after test is done Optional('reboot'): Any(False, 'always', 'on-exception', 'on-failure'), # What to run Required('mozharness'): { # the mozharness script used to run this task Required('script'): optionally_keyed_by('test-platform', basestring), # the config files required for the task Required('config'): optionally_keyed_by('test-platform', [basestring]), # mochitest flavor for mochitest runs Optional('mochitest-flavor'): basestring, # any additional actions to pass to the mozharness command Optional('actions'): [basestring], # additional command-line options for mozharness, beyond those # automatically added Required('extra-options'): optionally_keyed_by('test-platform', [basestring]), # the artifact name (including path) to test on the build task; this is # generally set in a per-kind transformation Optional('build-artifact-name'): basestring, # If true, tooltool downloads will be enabled via relengAPIProxy. Required('tooltool-downloads'): bool, # Add --blob-upload-branch=<project> mozharness parameter Optional('include-blob-upload-branch'): bool, # The setting for --download-symbols (if omitted, the option will not # be passed to mozharness) Optional('download-symbols'): Any(True, 'ondemand'), # If set, then MOZ_NODE_PATH=/usr/local/bin/node is included in the # environment. This is more than just a helpful path setting -- it # causes xpcshell tests to start additional servers, and runs # additional tests. Required('set-moz-node-path'): bool, # If true, include chunking information in the command even if the number # of chunks is 1 Required('chunked'): optionally_keyed_by('test-platform', bool), # The chunking argument format to use Required('chunking-args'): Any( # Use the usual --this-chunk/--total-chunk arguments 'this-chunk', # Use --test-suite=<suite>-<chunk-suffix>; see chunk-suffix, below 'test-suite-suffix', ), # the string to append to the `--test-suite` arugment when # chunking-args = test-suite-suffix; "<CHUNK>" in this string will # be replaced with the chunk number. Optional('chunk-suffix'): basestring, Required('requires-signed-builds'): optionally_keyed_by('test-platform', bool), }, # The current chunk; this is filled in by `all_kinds.py` Optional('this-chunk'): int, # os user groups for test task workers; required scopes, will be # added automatically Optional('os-groups'): optionally_keyed_by('test-platform', [basestring]), # -- values supplied by the task-generation infrastructure # the platform of the build this task is testing 'build-platform': basestring, # the label of the build task generating the materials to test 'build-label': basestring, # the label of the signing task generating the materials to test. # Signed builds are used in xpcshell tests on Windows, for instance. Optional('build-signing-label'): basestring, # the build's attributes 'build-attributes': { basestring: object }, # the platform on which the tests will run 'test-platform': basestring, # the name of the test (the key in tests.yml) 'test-name': basestring, # the product name, defaults to firefox Optional('product'): basestring, # conditional files to determine when these tests should be run Exclusive(Optional('when'), 'optimization'): Any({ Optional('files-changed'): [basestring], }), # Optimization to perform on this task during the optimization phase. # Optimizations are defined in taskcluster/taskgraph/optimize.py. Exclusive(Optional('optimization'), 'optimization'): OptimizationSchema, # The SCHEDULES component for this task; this defaults to the suite # (not including the flavor) but can be overridden here. Exclusive(Optional('schedules-component'), 'optimization'): basestring, Optional('worker-type'): optionally_keyed_by( 'test-platform', Any(basestring, None), ), # The target name, specifying the build artifact to be tested. # If None or not specified, a transform sets the target based on OS: # target.dmg (Mac), target.apk (Android), target.tar.bz2 (Linux), # or target.zip (Windows). Optional('target'): optionally_keyed_by( 'test-platform', Any(basestring, None), ), }, required=True)
push_apk_description_schema = Schema({ Required('dependent-tasks'): object, Required('name'): basestring, Required('label'): task_description_schema['label'], Required('description'): task_description_schema['description'], Required('job-from'): task_description_schema['job-from'], Required('attributes'): task_description_schema['attributes'], Required('treeherder'): task_description_schema['treeherder'], Required('run-on-projects'): task_description_schema['run-on-projects'], Required('worker-type'): optionally_keyed_by('project', basestring), Required('worker'): object, Required('scopes'): None, Required('requires'): task_description_schema['requires'], Required('deadline-after'): basestring, Required('shipping-phase'): task_description_schema['shipping-phase'], Required('shipping-product'): task_description_schema['shipping-product'], Optional('extra'): task_description_schema['extra'], })
beetmover_cdns_description_schema = Schema({ Required('name'): basestring, Required('product'): basestring, Required('treeherder-platform'): basestring, Optional('attributes'): { basestring: object }, Optional('job-from'): task_description_schema['job-from'], Optional('run'): { basestring: object }, Optional('run-on-projects'): task_description_schema['run-on-projects'], Required('worker-type'): optionally_keyed_by('project', basestring), Optional('dependencies'): { basestring: taskref_or_string }, Optional('index'): { basestring: basestring }, Optional('routes'): [basestring], Required('shipping-phase'): task_description_schema['shipping-phase'], Required('shipping-product'): task_description_schema['shipping-product'], Optional('extra'): task_description_schema['extra'], })