def test_extend_schema_twice(self):
"Extending a schema twice applies taskgraph checks."
with self.assertRaises(Exception):
Schema({
"kebab-case": int
}).extend({
'more-kebab': int
}).extend({"camelCase": int})
def payload_builder(name, schema):
schema = Schema({Required("implementation"): name, Optional("os"): str}).extend(
schema
)
def wrap(func):
payload_builders[name] = PayloadBuilder(schema, func)
return func
return wrap
def test_worker_caches(task, transform):
config, job, taskdesc, impl = transform(task)
add_cache(job, taskdesc, 'cache1', '/cache1')
add_cache(job, taskdesc, 'cache2', '/cache2', skip_untrusted=True)
if impl not in ('docker-worker', 'generic-worker'):
pytest.xfail("caches not implemented for '{}'".format(impl))
key = 'caches' if impl == 'docker-worker' else 'mounts'
assert key in taskdesc['worker']
assert len(taskdesc['worker'][key]) == 2
# Create a new schema object with just the part relevant to caches.
partial_schema = Schema(payload_builders[impl].schema.schema[key])
validate_schema(partial_schema, taskdesc['worker'][key], "validation error")
def test_worker_caches(task, transform):
config, job, taskdesc, impl = transform(task)
add_cache(job, taskdesc, "cache1", "/cache1")
add_cache(job, taskdesc, "cache2", "/cache2", skip_untrusted=True)
if impl not in ("docker-worker", "generic-worker"):
pytest.xfail("caches not implemented for '{}'".format(impl))
key = "caches" if impl == "docker-worker" else "mounts"
assert key in taskdesc["worker"]
assert len(taskdesc["worker"][key]) == 2
# Create a new schema object with just the part relevant to caches.
partial_schema = Schema(payload_builders[impl].schema.schema[key])
validate_schema(partial_schema, taskdesc["worker"][key],
"validation error")
from taskgraph.transforms.job import run_job_using
from taskgraph.util.schema import Schema
from voluptuous import Required, Optional, Extra, Any
bare_schema = Schema({
Required("using"): "bare",
Required("command"): Any(basestring, [basestring]),
Optional("install"): Any(basestring, [basestring]),
Optional("clone"): bool,
Extra: object
})
bare_defaults = {
"clone": True,
}
@run_job_using("docker-worker",
"bare",
schema=bare_schema,
defaults=bare_defaults)
def bare_docker_worker(config, job, taskdesc):
run = job["run"]
worker = taskdesc['worker'] = job['worker']
params = config.params
command = []
if run.get("clone"):
command.extend([
"git clone --quiet --depth=20 --no-single-branch {} taskcluster && "
from voluptuous import Required
transforms = TransformSequence()
push_apk_breakpoint_description_schema = Schema({
# the dependent task (object) for this beetmover job, used to inform beetmover.
Required('dependent-tasks'):
object,
Required('name'):
basestring,
Required('label'):
basestring,
Required('description'):
basestring,
Required('attributes'):
object,
Required('worker-type'):
None,
Required('worker'):
object,
Required('treeherder'):
object,
Required('run-on-projects'):
list,
Required('deadline-after'):
basestring,
})
validate_jobs_schema_transform = functools.partial(
validate_jobs_schema_transform_partial,
push_apk_breakpoint_description_schema, 'PushApkBreakpoint')
packaging_description_schema = Schema({
# the dependant task (object) for this job, used to inform repackaging.
Required('dependent-task'):
object,
# depname is used in taskref's to identify the taskID of the signed things
Required('depname', default='build'):
basestring,
# unique label to describe this repackaging task
Optional('label'):
basestring,
# treeherder is allowed here to override any defaults we use for repackaging. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'):
task_description_schema['treeherder'],
# If a l10n task, the corresponding locale
Optional('locale'):
basestring,
# Routes specific to this task, if defined
Optional('routes'): [basestring],
# passed through directly to the job description
Optional('extra'):
task_description_schema['extra'],
})
mozharness_run_schema = Schema({
Required('using'):
'mozharness',
# the mozharness script used to run this task, relative to the testing/
# directory and using forward slashes even on Windows
Required('script'):
text_type,
# Additional paths to look for mozharness configs in. These should be
# relative to the base of the source checkout
Optional('config-paths'): [text_type],
# the config files required for the task, relative to
# testing/mozharness/configs or one of the paths specified in
# `config-paths` and using forward slashes even on Windows
Required('config'): [text_type],
# any additional actions to pass to the mozharness command
Optional('actions'): [
Match('^[a-z0-9-]+$',
"actions must be `-` seperated alphanumeric strings")
],
# any additional options (without leading --) to be passed to mozharness
Optional('options'): [
Match(
'^[a-z0-9-]+(=[^ ]+)?$',
"options must be `-` seperated alphanumeric strings (with optional argument)"
)
],
# --custom-build-variant-cfg value
Optional('custom-build-variant-cfg'):
text_type,
# Extra configuration options to pass to mozharness.
Optional('extra-config'):
dict,
# If not false, tooltool downloads will be enabled via relengAPIProxy
# for either just public files, or all files. Not supported on Windows
Required('tooltool-downloads'):
Any(
False,
'public',
'internal',
),
# The set of secret names to which the task has access; these are prefixed
# with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting
# this will enable any worker features required and set the task's scopes
# appropriately. `true` here means ['*'], all secrets. Not supported on
# Windows
Required('secrets'):
Any(bool, [text_type]),
# If true, taskcluster proxy will be enabled; note that it may also be enabled
# automatically e.g., for secrets support. Not supported on Windows.
Required('taskcluster-proxy'):
bool,
# If true, the build scripts will start Xvfb. Not supported on Windows.
Required('need-xvfb'):
bool,
# If false, indicate that builds should skip producing artifacts. Not
# supported on Windows.
Required('keep-artifacts'):
bool,
# If specified, use the in-tree job script specified.
Optional('job-script'):
text_type,
Required('requires-signed-builds'):
bool,
# Whether or not to use caches.
Optional('use-caches'):
bool,
# If false, don't set MOZ_SIMPLE_PACKAGE_NAME
# Only disableable on windows
Required('use-simple-package'):
bool,
# If false don't pass --branch mozharness script
# Only disableable on windows
Required('use-magic-mh-args'):
bool,
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
Required('comm-checkout'):
bool,
# Base work directory used to set up the task.
Required('workdir'):
text_type,
})
for v in VARIANTS:
if '-{}/'.format(v) in test_platform:
return v
return ''
test_description_schema = {
str(k): v
for k, v in test_description_schema.schema.iteritems()
}
mozharness_test_run_schema = Schema({
Required('using'):
'mozharness-test',
Required('test'):
test_description_schema,
# Base work directory used to set up the task.
Required('workdir'):
basestring,
})
def test_packages_url(taskdesc):
"""Account for different platforms that name their test packages differently"""
return get_artifact_url(
'<build>', get_artifact_path(taskdesc, 'target.test_packages.json'))
@run_job_using('docker-engine',
'mozharness-test',
schema=mozharness_test_run_schema)
job_description_schema = Schema({
# The name of the job and the job's label. At least one must be specified,
# and the label will be generated from the name if necessary, by prepending
# the kind.
Optional('name'): text_type,
Optional('label'): text_type,
# the following fields are passed directly through to the task description,
# possibly modified by the run implementation. See
# taskcluster/taskgraph/transforms/task.py for the schema details.
Required('description'): task_description_schema['description'],
Optional('attributes'): task_description_schema['attributes'],
Optional('job-from'): task_description_schema['job-from'],
Optional('dependencies'): task_description_schema['dependencies'],
Optional('soft-dependencies'): task_description_schema['soft-dependencies'],
Optional('requires'): task_description_schema['requires'],
Optional('expires-after'): task_description_schema['expires-after'],
Optional('routes'): task_description_schema['routes'],
Optional('scopes'): task_description_schema['scopes'],
Optional('tags'): task_description_schema['tags'],
Optional('extra'): task_description_schema['extra'],
Optional('treeherder'): task_description_schema['treeherder'],
Optional('index'): task_description_schema['index'],
Optional('run-on-projects'): task_description_schema['run-on-projects'],
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('coalesce'): task_description_schema['coalesce'],
Optional('always-target'): task_description_schema['always-target'],
Exclusive('optimization', 'optimization'): task_description_schema['optimization'],
Optional('needs-sccache'): task_description_schema['needs-sccache'],
Optional('release-artifacts'): task_description_schema['release-artifacts'],
Optional('priority'): task_description_schema['priority'],
# The "when" section contains descriptions of the circumstances under which
# this task should be included in the task graph. This will be converted
# into an optimization, so it cannot be specified in a job description that
# also gives 'optimization'.
Exclusive('when', 'optimization'): {
# This task only needs to be run if a file matching one of the given
# patterns has changed in the push. The patterns use the mozpack
# match function (python/mozbuild/mozpack/path.py).
Optional('files-changed'): [text_type],
},
# A list of artifacts to install from 'fetch' tasks.
Optional('fetches'): {
text_type: [text_type, {
Required('artifact'): text_type,
Optional('dest'): text_type,
Optional('extract'): bool,
}],
},
# A description of how to run this job.
'run': {
# The key to a job implementation in a peer module to this one
'using': text_type,
# Base work directory used to set up the task.
Optional('workdir'): text_type,
# Any remaining content is verified against that job implementation's
# own schema.
Extra: object,
},
Required('worker-type'): task_description_schema['worker-type'],
# This object will be passed through to the task description, with additions
# provided by the job's run-using function
Optional('worker'): dict,
})
test_description_schema = Schema({
# description of the suite, for the task metadata
'description': basestring,
# test suite name, or <suite>/<flavor>
Required('suite'): optionally_keyed_by(
'test-platform',
basestring),
# the name by which this test suite is addressed in try syntax; defaults to
# the test-name
Optional('unittest-try-name'): basestring,
# the name by which this talos test is addressed in try syntax
Optional('talos-try-name'): basestring,
# additional tags to mark up this type of test
Optional('tags'): {basestring: object},
# the symbol, or group(symbol), under which this task should appear in
# treeherder.
'treeherder-symbol': basestring,
# the value to place in task.extra.treeherder.machine.platform; ideally
# this is the same as build-platform, and that is the default, but in
# practice it's not always a match.
Optional('treeherder-machine-platform'): basestring,
# attributes to appear in the resulting task (later transforms will add the
# common attributes)
Optional('attributes'): {basestring: object},
# The `run_on_projects` attribute, defaulting to "all". This dictates the
# projects on which this task should be included in the target task set.
# See the attributes documentation for details.
Optional('run-on-projects', default=['all']): optionally_keyed_by(
'test-platform',
[basestring]),
# the sheriffing tier for this task (default: set based on test platform)
Optional('tier'): optionally_keyed_by(
'test-platform',
Any(int, 'default')),
# number of chunks to create for this task. This can be keyed by test
# platform by passing a dictionary in the `by-test-platform` key. If the
# test platform is not found, the key 'default' will be tried.
Required('chunks', default=1): optionally_keyed_by(
'test-platform',
int),
# the time (with unit) after which this task is deleted; default depends on
# the branch (see below)
Optional('expires-after'): basestring,
# Whether to run this task with e10s (desktop-test only). If false, run
# without e10s; if true, run with e10s; if 'both', run one task with and
# one task without e10s. E10s tasks have "-e10s" appended to the test name
# and treeherder group.
Required('e10s', default='both'): optionally_keyed_by(
'test-platform', 'project',
Any(bool, 'both')),
# The EC2 instance size to run these tests on.
Required('instance-size', default='default'): optionally_keyed_by(
'test-platform',
Any('default', 'large', 'xlarge', 'legacy')),
# Whether the task requires loopback audio or video (whatever that may mean
# on the platform)
Required('loopback-audio', default=False): bool,
Required('loopback-video', default=False): bool,
# Whether the test can run using a software GL implementation on Linux
# using the GL compositor. May not be used with "legacy" sized instances
# due to poor LLVMPipe performance (bug 1296086). Defaults to true for
# linux platforms and false otherwise
Optional('allow-software-gl-layers'): bool,
# The worker implementation for this test, as dictated by policy and by the
# test platform.
Optional('worker-implementation'): Any(
'docker-worker',
'native-engine',
'generic-worker',
# coming soon:
'docker-engine',
'buildbot-bridge',
),
# For tasks that will run in docker-worker or docker-engine, this is the
# name of the docker image or in-tree docker image to run the task in. If
# in-tree, then a dependency will be created automatically. This is
# generally `desktop-test`, or an image that acts an awful lot like it.
Required('docker-image', default={'in-tree': 'desktop-test'}): optionally_keyed_by(
'test-platform', 'test-platform-phylum',
Any(
# a raw Docker image path (repo/image:tag)
basestring,
# an in-tree generated docker image (from `taskcluster/docker/<name>`)
{'in-tree': basestring}
)
),
# seconds of runtime after which the task will be killed. Like 'chunks',
# this can be keyed by test pltaform.
Required('max-run-time', default=3600): optionally_keyed_by(
'test-platform',
int),
# the exit status code that indicates the task should be retried
Optional('retry-exit-status'): int,
# Whether to perform a gecko checkout.
Required('checkout', default=False): bool,
# Wheter to perform a machine reboot after test is done
Optional('reboot', default=True): bool,
# What to run
Required('mozharness'): optionally_keyed_by(
'test-platform', 'test-platform-phylum', {
# the mozharness script used to run this task
Required('script'): basestring,
# the config files required for the task
Required('config'): optionally_keyed_by(
'test-platform',
[basestring]),
# mochitest flavor for mochitest runs
Optional('mochitest-flavor'): basestring,
# any additional actions to pass to the mozharness command
Optional('actions'): [basestring],
# additional command-line options for mozharness, beyond those
# automatically added
Required('extra-options', default=[]): optionally_keyed_by(
'test-platform',
[basestring]),
# the artifact name (including path) to test on the build task; this is
# generally set in a per-kind transformation
Optional('build-artifact-name'): basestring,
# If true, tooltool downloads will be enabled via relengAPIProxy.
Required('tooltool-downloads', default=False): bool,
# This mozharness script also runs in Buildbot and tries to read a
# buildbot config file, so tell it not to do so in TaskCluster
Required('no-read-buildbot-config', default=False): bool,
# Add --blob-upload-branch=<project> mozharness parameter
Optional('include-blob-upload-branch'): bool,
# The setting for --download-symbols (if omitted, the option will not
# be passed to mozharness)
Optional('download-symbols'): Any(True, 'ondemand'),
# If set, then MOZ_NODE_PATH=/usr/local/bin/node is included in the
# environment. This is more than just a helpful path setting -- it
# causes xpcshell tests to start additional servers, and runs
# additional tests.
Required('set-moz-node-path', default=False): bool,
# If true, include chunking information in the command even if the number
# of chunks is 1
Required('chunked', default=False): optionally_keyed_by(
'test-platform',
bool),
# The chunking argument format to use
Required('chunking-args', default='this-chunk'): Any(
# Use the usual --this-chunk/--total-chunk arguments
'this-chunk',
# Use --test-suite=<suite>-<chunk-suffix>; see chunk-suffix, below
'test-suite-suffix',
),
# the string to append to the `--test-suite` arugment when
# chunking-args = test-suite-suffix; "<CHUNK>" in this string will
# be replaced with the chunk number.
Optional('chunk-suffix'): basestring,
}
),
# The current chunk; this is filled in by `all_kinds.py`
Optional('this-chunk'): int,
# os user groups for test task workers; required scopes, will be
# added automatically
Optional('os-groups', default=[]): optionally_keyed_by(
'test-platform',
[basestring]),
# -- values supplied by the task-generation infrastructure
# the platform of the build this task is testing
'build-platform': basestring,
# the label of the build task generating the materials to test
'build-label': basestring,
# the platform on which the tests will run
'test-platform': basestring,
# the name of the test (the key in tests.yml)
'test-name': basestring,
# the product name, defaults to firefox
Optional('product'): basestring,
# conditional files to determine when these tests should be run
Optional('when'): Any({
Optional('files-changed'): [basestring],
}),
}, required=True)
l10n_description_schema = Schema({
# Name for this job, inferred from the dependent job before validation
Required('name'):
basestring,
# build-platform, inferred from dependent job before validation
Required('build-platform'):
basestring,
# max run time of the task
Required('run-time'):
_by_platform(int),
# Locales not to repack for
Required('ignore-locales'):
_by_platform([basestring]),
# All l10n jobs use mozharness
Required('mozharness'): {
# Script to invoke for mozharness
Required('script'):
_by_platform(basestring),
# Config files passed to the mozharness script
Required('config'):
_by_platform([basestring]),
# Options to pass to the mozharness script
Required('options'):
_by_platform([basestring]),
# Action commands to provide to mozharness script
Required('actions'):
_by_platform([basestring]),
},
# Items for the taskcluster index
Optional('index'): {
# Product to identify as in the taskcluster index
Required('product'):
_by_platform(basestring),
# Job name to identify as in the taskcluster index
Required('job-name'):
_by_platform(basestring),
# Type of index
Optional('type'):
basestring,
},
# Description of the localized task
Required('description'):
_by_platform(basestring),
Optional('run-on-projects'):
job_description_schema['run-on-projects'],
# task object of the dependent task
Required('dependent-task'):
object,
# worker-type to utilize
Required('worker-type'):
_by_platform(basestring),
# File which contains the used locales
Required('locales-file'):
_by_platform(basestring),
# Tooltool visibility required for task.
Required('tooltool'):
_by_platform(Any('internal', 'public')),
# Docker image required for task. We accept only in-tree images
# -- generally desktop-build or android-build -- for now.
Required('docker-image'):
_by_platform(
Any(
# an in-tree generated docker image (from `taskcluster/docker/<name>`)
{'in-tree': basestring},
None,
)),
Optional('toolchains'):
_by_platform([basestring]),
# The set of secret names to which the task has access; these are prefixed
# with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting
# this will enable any worker features required and set the task's scopes
# appropriately. `true` here means ['*'], all secrets. Not supported on
# Windows
Required('secrets', default=False):
_by_platform(Any(bool, [basestring])),
# Information for treeherder
Required('treeherder'): {
# Platform to display the task on in treeherder
Required('platform'):
_by_platform(basestring),
# Symbol to use
Required('symbol'):
basestring,
# Tier this task is
Required('tier'):
_by_platform(int),
},
# Extra environment values to pass to the worker
Optional('env'):
_by_platform({basestring: taskref_or_string}),
# Max number locales per chunk
Optional('locales-per-chunk'):
_by_platform(int),
# Task deps to chain this task with, added in transforms from dependent-task
# if this is a nightly
Optional('dependencies'): {
basestring: basestring
},
# Run the task when the listed files change (if present).
Optional('when'): {
'files-changed': [basestring]
},
# passed through directly to the job description
Optional('attributes'):
job_description_schema['attributes'],
Optional('extra'):
job_description_schema['extra'],
})
)
from taskgraph.util.hash import hash_paths
from taskgraph import GECKO
TOOLCHAIN_INDEX = 'gecko.cache.level-{level}.toolchains.v1.{name}.{digest}'
toolchain_run_schema = Schema({
Required('using'): 'toolchain-script',
# the script (in taskcluster/scripts/misc) to run
Required('script'): basestring,
# If not false, tooltool downloads will be enabled via relengAPIProxy
# for either just public files, or all files. Not supported on Windows
Required('tooltool-downloads', default=False): Any(
False,
'public',
'internal',
),
# Paths/patterns pointing to files that influence the outcome of a
# toolchain build.
Optional('resources'): [basestring],
})
def add_optimizations(config, run, taskdesc):
files = list(run.get('resources', []))
# This file
files.append('taskcluster/taskgraph/transforms/job/toolchain.py')
# The script
from taskgraph.util.schema import Schema, optionally_keyed_by, resolve_keyed_by
from taskgraph.util.scriptworker import get_release_config
from voluptuous import Optional, Required, Any
from taskgraph.transforms.job import run_job_using
buildbot_run_schema = Schema({
Required('using'):
'buildbot',
# the buildername to use for buildbot-bridge, will expand {branch} in name from
# the current project.
Required('buildername'):
basestring,
# the product to use
Required('product'):
Any('firefox', 'mobile', 'fennec', 'devedition', 'thunderbird'),
Optional('channels'):
optionally_keyed_by('project', basestring),
Optional('release-promotion'):
bool,
})
def _get_balrog_api_root(branch):
if branch in ('mozilla-beta',
'mozilla-release') or branch.startswith('mozilla-esr'):
return 'https://aus4-admin.mozilla.org/api'
else:
diff_description_schema = Schema(
{
# Name of the diff task.
Required("name"): text_type,
# Treeherder tier.
Required("tier"): int,
# Treeherder symbol.
Required("symbol"): text_type,
# relative path (from config.path) to the file the task was defined in.
Optional("job-from"): text_type,
# Original and new builds to compare.
Required("original"): index_or_string,
Required("new"): index_or_string,
# Arguments to pass to diffoscope, used for job-defaults in
# taskcluster/ci/diffoscope/kind.yml
Optional("args"): text_type,
# Extra arguments to pass to diffoscope, that can be set per job.
Optional("extra-args"): text_type,
# Fail the task when differences are detected.
Optional("fail-on-diff"): bool,
# What artifact to check the differences of. Defaults to target.tar.bz2
# for Linux, target.dmg for Mac, target.zip for Windows, target.apk for
# Android.
Optional("artifact"): text_type,
# Whether to unpack first. Diffoscope can normally work without unpacking,
# but when one needs to --exclude some contents, that doesn't work out well
# if said content is packed (e.g. in omni.ja).
Optional("unpack"): bool,
# Commands to run before performing the diff.
Optional("pre-diff-commands"): [text_type],
# Only run the task on a set of projects/branches.
Optional("run-on-projects"): task_description_schema["run-on-projects"],
Optional("optimization"): task_description_schema["optimization"],
}
)
from voluptuous import Required, Any
from taskgraph.transforms.job import run_job_using
from taskgraph.transforms.job.common import (
docker_worker_add_artifacts,
generic_worker_add_artifacts,
docker_worker_add_gecko_vcs_env_vars,
docker_worker_add_tooltool,
support_vcs_checkout,
)
sm_run_schema = Schema({
Required('using'):
Any('spidermonkey', 'spidermonkey-package', 'spidermonkey-mozjs-crate',
'spidermonkey-rust-bindings'),
# The SPIDERMONKEY_VARIANT
Required('spidermonkey-variant'):
basestring,
})
@run_job_using("docker-worker", "spidermonkey", schema=sm_run_schema)
@run_job_using("docker-worker", "spidermonkey-package", schema=sm_run_schema)
@run_job_using("docker-worker",
"spidermonkey-mozjs-crate",
schema=sm_run_schema)
@run_job_using("docker-worker",
"spidermonkey-rust-bindings",
schema=sm_run_schema)
def docker_worker_spidermonkey(config, job, taskdesc):
'mozilla-release': '',
'mozilla-beta': '-beta',
'mozilla-central': '-nightly',
'try': '-nightly-try',
'maple': '-nightly-maple',
}
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
beetmover_description_schema = Schema({
Required('dependent-task'): object,
Required('depname', default='build'): basestring,
Optional('label'): basestring,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('bucket-scope'): optionally_keyed_by('project', basestring),
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_description_schema, job,
"In beetmover-geckoview ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
from taskgraph.transforms.job.common import (
docker_worker_add_artifacts,
generic_worker_add_artifacts,
)
sm_run_schema = Schema({
Required("using"):
Any(
"spidermonkey",
"spidermonkey-package",
),
# SPIDERMONKEY_VARIANT and SPIDERMONKEY_PLATFORM
Required("spidermonkey-variant"):
text_type,
Optional("spidermonkey-platform"):
text_type,
# Base work directory used to set up the task.
Optional("workdir"):
text_type,
Required("tooltool-downloads"):
Any(
False,
"public",
"internal",
),
})
@run_job_using("docker-worker", "spidermonkey", schema=sm_run_schema)
@run_job_using("docker-worker", "spidermonkey-package", schema=sm_run_schema)
def docker_worker_spidermonkey(config, job, taskdesc):
"""
from __future__ import absolute_import, print_function, unicode_literals
from six import text_type
from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
from taskgraph.util.schema import Schema
from voluptuous import Required, Optional
python_test_schema = Schema({
Required("using"):
"python-test",
# Python version to use
Required("python-version"):
int,
# The subsuite to run
Required("subsuite"):
text_type,
# Base work directory used to set up the task.
Optional("workdir"):
text_type,
})
defaults = {
"python-version": 2,
"subsuite": "default",
}
@run_job_using("docker-worker",
"python-test",
task_description_schema = {
str(k): v
for k, v in task_description_schema.schema.iteritems()
}
transforms = TransformSequence()
taskref_or_string = Any(basestring, {Required('task-reference'): basestring})
checksums_signing_description_schema = Schema({
Required('dependent-task'):
object,
Required('depname', default='beetmover'):
basestring,
Optional('label'):
basestring,
Optional('treeherder'):
task_description_schema['treeherder'],
Optional('shipping-product'):
task_description_schema['shipping-product'],
Optional('shipping-phase'):
task_description_schema['shipping-phase'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('dependent-task',
object).__dict__.get('label', '?no-label?')
validate_schema(
checksums_signing_description_schema, job,
job_description_schema = Schema({
# The name of the job and the job's label. At least one must be specified,
# and the label will be generated from the name if necessary, by prepending
# the kind.
Optional('name'):
basestring,
Optional('label'):
basestring,
# the following fields are passed directly through to the task description,
# possibly modified by the run implementation. See
# taskcluster/taskgraph/transforms/task.py for the schema details.
Required('description'):
task_description_schema['description'],
Optional('attributes'):
task_description_schema['attributes'],
Optional('dependencies'):
task_description_schema['dependencies'],
Optional('expires-after'):
task_description_schema['expires-after'],
Optional('routes'):
task_description_schema['routes'],
Optional('scopes'):
task_description_schema['scopes'],
Optional('tags'):
task_description_schema['tags'],
Optional('extra'):
task_description_schema['extra'],
Optional('treeherder'):
task_description_schema['treeherder'],
Optional('index'):
task_description_schema['index'],
Optional('run-on-projects'):
task_description_schema['run-on-projects'],
Optional('coalesce-name'):
task_description_schema['coalesce-name'],
Optional('optimizations'):
task_description_schema['optimizations'],
Optional('needs-sccache'):
task_description_schema['needs-sccache'],
# The "when" section contains descriptions of the circumstances
# under which this task should be included in the task graph. This
# will be converted into an element in the `optimizations` list.
Optional('when'):
Any({
# This task only needs to be run if a file matching one of the given
# patterns has changed in the push. The patterns use the mozpack
# match function (python/mozbuild/mozpack/path.py).
Optional('files-changed'): [basestring],
}),
# A description of how to run this job.
'run': {
# The key to a job implementation in a peer module to this one
'using': basestring,
# Any remaining content is verified against that job implementation's
# own schema.
Extra: object,
},
Required('worker-type'):
task_description_schema['worker-type'],
# This object will be passed through to the task description, with additions
# provided by the job's run-using function
Optional('worker'):
dict,
})
mozharness_run_schema = Schema({
Required('using'):
'mozharness',
# the mozharness script used to run this task, relative to the testing/
# directory and using forward slashes even on Windows
Required('script'):
basestring,
# the config files required for the task, relative to
# testing/mozharness/configs and using forward slashes even on Windows
Required('config'): [basestring],
# any additional actions to pass to the mozharness command; not supported
# on Windows
Optional('actions'): [basestring],
# any additional options (without leading --) to be passed to mozharness;
# not supported on Windows
Optional('options'): [basestring],
# --custom-build-variant-cfg value (not supported on Windows)
Optional('custom-build-variant-cfg'):
basestring,
# Extra metadata to use toward the workspace caching.
# Only supported on docker-worker
Optional('extra-workspace-cache-key'):
basestring,
# If not false, tooltool downloads will be enabled via relengAPIProxy
# for either just public files, or all files. Not supported on Windows
Required('tooltool-downloads', default=False):
Any(
False,
'public',
'internal',
),
# The set of secret names to which the task has access; these are prefixed
# with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting
# this will enable any worker features required and set the task's scopes
# appropriately. `true` here means ['*'], all secrets. Not supported on
# Windows
Required('secrets', default=False):
Any(bool, [basestring]),
# If true, taskcluster proxy will be enabled; note that it may also be enabled
# automatically e.g., for secrets support. Not supported on Windows.
Required('taskcluster-proxy', default=False):
bool,
# If true, the build scripts will start Xvfb. Not supported on Windows.
Required('need-xvfb', default=False):
bool,
# If false, indicate that builds should skip producing artifacts. Not
# supported on Windows.
Required('keep-artifacts', default=True):
bool,
# If specified, use the in-tree job script specified.
Optional('job-script'):
basestring,
Required('requires-signed-builds', default=False):
bool,
# If false, don't set MOZ_SIMPLE_PACKAGE_NAME
# Only disableable on windows
Required('use-simple-package', default=True):
bool,
# If false don't pass --branch or --skip-buildbot-actions to mozharness script
# Only disableable on windows
Required('use-magic-mh-args', default=True):
bool,
})
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import copy
from taskgraph.task import Task
from taskgraph.util.attributes import sorted_unique_list
from taskgraph.util.schema import Schema
from voluptuous import Required
schema = Schema(
{
Required("primary-dependency"): Task,
Required("dependent-tasks"): {str: Task},
},
)
GROUP_BY_MAP = {}
def group_by(name):
def wrapper(func):
GROUP_BY_MAP[name] = func
return func
return wrapper
@group_by("addon-type")
docker_worker_add_artifacts,
docker_worker_add_tooltool,
)
haz_run_schema = Schema({
Required('using'):
'hazard',
# The command to run within the task image (passed through to the worker)
Required('command'):
basestring,
# The mozconfig to use; default in the script is used if omitted
Optional('mozconfig'):
basestring,
# The set of secret names to which the task has access; these are prefixed
# with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting
# this will enable any worker features required and set the task's scopes
# appropriately. `true` here means ['*'], all secrets. Not supported on
# Windows
Required('secrets', default=False):
Any(bool, [basestring]),
# Base work directory used to set up the task.
Required('workdir'):
basestring,
})
@run_job_using("docker-worker", "hazard", schema=haz_run_schema)
def docker_worker_hazard(config, job, taskdesc):
docker_image_schema = Schema({
# Name of the docker image.
Required('name'):
basestring,
# Name of the parent docker image.
Optional('parent'):
basestring,
# Treeherder symbol.
Required('symbol'):
basestring,
# relative path (from config.path) to the file the docker image was defined
# in.
Optional('job-from'):
basestring,
# Arguments to use for the Dockerfile.
Optional('args'): {
basestring: basestring
},
# Name of the docker image definition under taskcluster/docker, when
# different from the docker image name.
Optional('definition'):
basestring,
# List of package tasks this docker image depends on.
Optional('packages'): [basestring],
})
run_task_schema = Schema({
Required('using'):
'run-task',
# if true, add a cache at ~worker/.cache, which is where things like pip
# tend to hide their caches. This cache is never added for level-1 jobs.
Required('cache-dotcache', default=False):
bool,
# if true (the default), perform a checkout in /builds/worker/checkouts/gecko
Required('checkout', default=True):
bool,
# The sparse checkout profile to use. Value is the filename relative to the
# directory where sparse profiles are defined (build/sparse-profiles/).
Required('sparse-profile', default=None):
basestring,
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
Required('comm-checkout', default=False):
bool,
# The command arguments to pass to the `run-task` script, after the
# checkout arguments. If a list, it will be passed directly; otherwise
# it will be included in a single argument to `bash -cx`.
Required('command'):
Any([basestring], basestring),
})
toolchain_run_schema = Schema({
Required('using'):
'toolchain-script',
# The script (in taskcluster/scripts/misc) to run.
# Python scripts are invoked with `mach python` so vendored libraries
# are available.
Required('script'):
text_type,
# Arguments to pass to the script.
Optional('arguments'): [text_type],
# If not false, tooltool downloads will be enabled via relengAPIProxy
# for either just public files, or all files. Not supported on Windows
Required('tooltool-downloads'):
Any(
False,
'public',
'internal',
),
# Sparse profile to give to checkout using `run-task`. If given,
# a filename in `build/sparse-profiles`. Defaults to
# "toolchain-build", i.e., to
# `build/sparse-profiles/toolchain-build`. If `None`, instructs
# `run-task` to not use a sparse profile at all.
Required('sparse-profile'):
Any(text_type, None),
# Paths/patterns pointing to files that influence the outcome of a
# toolchain build.
Optional('resources'): [text_type],
# Path to the artifact produced by the toolchain job
Required('toolchain-artifact'):
text_type,
Optional(
"toolchain-alias",
description="An alias that can be used instead of the real toolchain job name in "
"fetch stanzas for jobs.",
):
text_type,
# Base work directory used to set up the task.
Required('workdir'):
text_type,
})
configure_taskdesc_for_run,
)
from taskgraph.transforms.job.common import (
docker_worker_add_artifacts,
generic_worker_add_artifacts,
docker_worker_add_tooltool,
)
sm_run_schema = Schema({
Required('using'):
Any('spidermonkey', 'spidermonkey-package', 'spidermonkey-mozjs-crate',
'spidermonkey-rust-bindings'),
# SPIDERMONKEY_VARIANT and SPIDERMONKEY_PLATFORM
Required('spidermonkey-variant'):
basestring,
Optional('spidermonkey-platform'):
basestring,
# Base work directory used to set up the task.
Required('workdir'):
basestring,
})
@run_job_using("docker-worker", "spidermonkey", schema=sm_run_schema)
@run_job_using("docker-worker", "spidermonkey-package", schema=sm_run_schema)
@run_job_using("docker-worker",
"spidermonkey-mozjs-crate",
schema=sm_run_schema)
@run_job_using("docker-worker",
def get_variant(test_platform):
for v in VARIANTS:
if '-{}/'.format(v) in test_platform:
return v
return ''
test_description_schema = {
str(k): v
for k, v in test_description_schema.schema.iteritems()
}
mozharness_test_run_schema = Schema({
Required('using'): 'mozharness-test',
Required('test'): test_description_schema,
})
def test_packages_url(taskdesc):
"""Account for different platforms that name their test packages differently"""
return get_artifact_url('<build>',
'public/build/target.test_packages.json')
@run_job_using('docker-engine',
'mozharness-test',
schema=mozharness_test_run_schema)
@run_job_using('docker-worker',
'mozharness-test',
schema=mozharness_test_run_schema)
)
diff_description_schema = Schema({
# Name of the diff task.
Required('name'):
basestring,
# Treeherder symbol.
Required('symbol'):
basestring,
# relative path (from config.path) to the file the task was defined in.
Optional('job-from'):
basestring,
# Original and new builds to compare.
Required('original'):
index_or_string,
Required('new'):
index_or_string,
# Arguments to pass to diffoscope, used for job-defaults in
# taskcluster/ci/diffoscope/kind.yml
Optional('args'):
basestring,
# Extra arguments to pass to diffoscope, that can be set per job.
Optional('extra-args'):
basestring,
})
