# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import copy
from voluptuous import Required
from taskgraph.task import Task
from taskgraph.util.attributes import sorted_unique_list
from taskgraph.util.schema import Schema
schema = Schema({
Required('primary-dependency', 'primary dependency task'): Task,
Required(
'dependent-tasks',
'dictionary of dependent tasks, keyed by kind',
): {basestring: Task},
})
# Define a collection of group_by functions
GROUP_BY_MAP = {}
def group_by(name):
def wrapper(func):
GROUP_BY_MAP[name] = func
return func
return wrapper
cron_yml_schema = Schema({
'jobs': [{
# Name of the crontask (must be unique)
Required('name'): basestring,
# what to run
# Description of the job to run, keyed by 'type'
Required('job'): Any({
Required('type'): 'decision-task',
# Treeherder symbol for the cron task
Required('treeherder-symbol'): basestring,
# --target-tasks-method './mach taskgraph decision' argument
'target-tasks-method': basestring,
}),
# when to run it
# Optional set of projects on which this job should run; if omitted, this will
# run on all projects for which cron tasks are set up. This works just like the
# `run_on_projects` attribute, where strings like "release" and "integration" are
# expanded to cover multiple repositories. (taskcluster/docs/attributes.rst)
'run-on-projects': [basestring],
# Array of times at which this task should run. These *must* be a
# multiple of 15 minutes, the minimum scheduling interval. This field
# can be keyed by project so that each project has a different schedule
# for the same job.
'when': optionally_keyed_by(
'project',
[{'hour': int, 'minute': All(int, even_15_minutes)}]),
}],
})
signing_description_schema = Schema({
# the dependant task (object) for this signing job, used to inform signing.
Required('dependent-task'):
object,
# Artifacts from dep task to sign - Sync with taskgraph/transforms/task.py
# because this is passed directly into the signingscript worker
Required('upstream-artifacts'): [{
# taskId of the task with the artifact
Required('taskId'):
taskref_or_string,
# type of signing task (for CoT)
Required('taskType'):
basestring,
# Paths to the artifacts to sign
Required('paths'): [basestring],
# Signing formats to use on each of the paths
Required('formats'): [basestring],
}],
# depname is used in taskref's to identify the taskID of the unsigned things
Required('depname'):
basestring,
# unique label to describe this signing task, defaults to {dep.label}-signing
Optional('label'):
basestring,
# treeherder is allowed here to override any defaults we use for signing. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'):
task_description_schema['treeherder'],
# Routes specific to this task, if defined
Optional('routes'): [basestring],
Optional('shipping-phase'):
task_description_schema['shipping-phase'],
Optional('shipping-product'):
task_description_schema['shipping-product'],
# Optional control for how long a task may run (aka maxRunTime)
Optional('max-run-time'):
int,
Optional('extra'): {
basestring: object
},
})
job_description_schema = Schema({
# The name of the job and the job's label. At least one must be specified,
# and the label will be generated from the name if necessary, by prepending
# the kind.
Optional('name'):
basestring,
Optional('label'):
basestring,
# the following fields are passed directly through to the task description,
# possibly modified by the run implementation. See
# taskcluster/taskgraph/transforms/task.py for the schema details.
Required('description'):
task_description_schema['description'],
Optional('attributes'):
task_description_schema['attributes'],
Optional('dependencies'):
task_description_schema['dependencies'],
Optional('expires-after'):
task_description_schema['expires-after'],
Optional('routes'):
task_description_schema['routes'],
Optional('scopes'):
task_description_schema['scopes'],
Optional('tags'):
task_description_schema['tags'],
Optional('extra'):
task_description_schema['extra'],
Optional('treeherder'):
task_description_schema['treeherder'],
Optional('index'):
task_description_schema['index'],
Optional('run-on-projects'):
task_description_schema['run-on-projects'],
Optional('coalesce-name'):
task_description_schema['coalesce-name'],
Optional('optimizations'):
task_description_schema['optimizations'],
Optional('needs-sccache'):
task_description_schema['needs-sccache'],
# The "when" section contains descriptions of the circumstances
# under which this task should be included in the task graph. This
# will be converted into an element in the `optimizations` list.
Optional('when'):
Any({
# This task only needs to be run if a file matching one of the given
# patterns has changed in the push. The patterns use the mozpack
# match function (python/mozbuild/mozpack/path.py).
Optional('files-changed'): [basestring],
}),
# A description of how to run this job.
'run': {
# The key to a job implementation in a peer module to this one
'using': basestring,
# Any remaining content is verified against that job implementation's
# own schema.
Extra: object,
},
Required('worker-type'):
Any(
task_description_schema['worker-type'],
{'by-platform': {
basestring: task_description_schema['worker-type']
}},
),
Required('worker'):
Any(
task_description_schema['worker'],
{'by-platform': {
basestring: task_description_schema['worker']
}},
),
})
l10n_description_schema = Schema({
# Name for this job, inferred from the dependent job before validation
Required('name'):
basestring,
# build-platform, inferred from dependent job before validation
Required('build-platform'):
basestring,
# max run time of the task
Required('run-time'):
_by_platform(int),
# Locales not to repack for
Required('ignore-locales'):
_by_platform([basestring]),
# All l10n jobs use mozharness
Required('mozharness'): {
# Script to invoke for mozharness
Required('script'):
_by_platform(basestring),
# Config files passed to the mozharness script
Required('config'):
_by_platform([basestring]),
# Options to pass to the mozharness script
Required('options'):
_by_platform([basestring]),
# Action commands to provide to mozharness script
Required('actions'):
_by_platform([basestring]),
},
# Items for the taskcluster index
Optional('index'): {
# Product to identify as in the taskcluster index
Required('product'):
_by_platform(basestring),
# Job name to identify as in the taskcluster index
Required('job-name'):
_by_platform(basestring),
# Type of index
Optional('type'):
basestring,
},
# Description of the localized task
Required('description'):
_by_platform(basestring),
Optional('run-on-projects'):
job_description_schema['run-on-projects'],
# task object of the dependent task
Required('dependent-task'):
object,
# worker-type to utilize
Required('worker-type'):
_by_platform(basestring),
# File which contains the used locales
Required('locales-file'):
_by_platform(basestring),
# Tooltool visibility required for task.
Required('tooltool'):
_by_platform(Any('internal', 'public')),
# Docker image required for task. We accept only in-tree images
# -- generally desktop-build or android-build -- for now.
Required('docker-image'):
_by_platform(
Any(
# an in-tree generated docker image (from `taskcluster/docker/<name>`)
{'in-tree': basestring},
None,
)),
Optional('toolchains'):
_by_platform([basestring]),
# Information for treeherder
Required('treeherder'): {
# Platform to display the task on in treeherder
Required('platform'):
_by_platform(basestring),
# Symbol to use
Required('symbol'):
basestring,
# Tier this task is
Required('tier'):
_by_platform(int),
},
# Extra environment values to pass to the worker
Optional('env'):
_by_platform({basestring: taskref_or_string}),
# Max number locales per chunk
Optional('locales-per-chunk'):
_by_platform(int),
# Task deps to chain this task with, added in transforms from dependent-task
# if this is a nightly
Optional('dependencies'): {
basestring: basestring
},
# Run the task when the listed files change (if present).
Optional('when'): {
'files-changed': [basestring]
},
# passed through directly to the job description
Optional('attributes'):
job_description_schema['attributes'],
Optional('extra'):
job_description_schema['extra'],
})
run_task_schema = Schema({
Required('using'):
'run-task',
# if true, add a cache at ~worker/.cache, which is where things like pip
# tend to hide their caches. This cache is never added for level-1 jobs.
# TODO Once bug 1526028 is fixed, this and 'use-caches' should be merged.
Required('cache-dotcache'):
bool,
# Whether or not to use caches.
Optional('use-caches'):
bool,
# if true (the default), perform a checkout of gecko on the worker
Required('checkout'):
bool,
Optional(
"cwd",
description="Path to run command in. If a checkout is present, the path "
"to the checkout will be interpolated with the key `checkout`",
):
text_type,
# The sparse checkout profile to use. Value is the filename relative to the
# directory where sparse profiles are defined (build/sparse-profiles/).
Required('sparse-profile'):
Any(basestring, None),
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
Required('comm-checkout'):
bool,
# The command arguments to pass to the `run-task` script, after the
# checkout arguments. If a list, it will be passed directly; otherwise
# it will be included in a single argument to `bash -cx`.
Required('command'):
Any([taskref_or_string], taskref_or_string),
# Base work directory used to set up the task.
Required('workdir'):
basestring,
# If not false, tooltool downloads will be enabled via relengAPIProxy
# for either just public files, or all files. Only supported on
# docker-worker.
Required('tooltool-downloads'):
Any(
False,
'public',
'internal',
),
# Whether to run as root. (defaults to False)
Optional('run-as-root'):
bool,
})
diff_description_schema = Schema({
# Name of the diff task.
Required('name'):
text_type,
# Treeherder symbol.
Required('symbol'):
text_type,
# relative path (from config.path) to the file the task was defined in.
Optional('job-from'):
text_type,
# Original and new builds to compare.
Required('original'):
index_or_string,
Required('new'):
index_or_string,
# Arguments to pass to diffoscope, used for job-defaults in
# taskcluster/ci/diffoscope/kind.yml
Optional('args'):
text_type,
# Extra arguments to pass to diffoscope, that can be set per job.
Optional('extra-args'):
text_type,
# Fail the task when differences are detected.
Optional('fail-on-diff'):
bool,
# What artifact to check the differences of. Defaults to target.tar.bz2
# for Linux, target.dmg for Mac, target.zip for Windows, target.apk for
# Android.
Optional('artifact'):
text_type,
# Whether to unpack first. Diffoscope can normally work without unpacking,
# but when one needs to --exclude some contents, that doesn't work out well
# if said content is packed (e.g. in omni.ja).
Optional('unpack'):
bool,
# Commands to run before performing the diff.
Optional('pre-diff-commands'): [text_type],
# Only run the task on a set of projects/branches.
Optional('run-on-projects'):
task_description_schema['run-on-projects'],
Optional('optimization'):
task_description_schema['optimization'],
})
docker_worker_add_artifacts,
add_tooltool,
)
haz_run_schema = Schema({
Required('using'):
'hazard',
# The command to run within the task image (passed through to the worker)
Required('command'):
basestring,
# The mozconfig to use; default in the script is used if omitted
Optional('mozconfig'):
basestring,
# The set of secret names to which the task has access; these are prefixed
# with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting
# this will enable any worker features required and set the task's scopes
# appropriately. `true` here means ['*'], all secrets. Not supported on
# Windows
Required('secrets', default=False):
Any(bool, [basestring]),
# Base work directory used to set up the task.
Required('workdir'):
basestring,
})
@run_job_using("docker-worker", "hazard", schema=haz_run_schema)
def docker_worker_hazard(config, job, taskdesc):
raptor_description_schema = Schema({
# Raptor specific configs.
Optional('apps'):
optionally_keyed_by('test-platform', [basestring]),
Optional('raptor-test'):
basestring,
Optional('activity'):
optionally_keyed_by('app', basestring),
Optional('binary-path'):
optionally_keyed_by('app', basestring),
Optional('pageload'):
optionally_keyed_by(
'test-platform',
'app',
Any('cold', 'warm', 'both'),
),
# Configs defined in the 'test_description_schema'.
Optional('max-run-time'):
optionally_keyed_by('app', test_description_schema['max-run-time']),
Optional('run-on-projects'):
optionally_keyed_by('app', test_description_schema['run-on-projects']),
Optional('variants'):
optionally_keyed_by('app', test_description_schema['variants']),
Optional('target'):
optionally_keyed_by('app', test_description_schema['target']),
Optional('run-visual-metrics'):
optionally_keyed_by('app', bool),
Required('test-name'):
test_description_schema['test-name'],
Required('test-platform'):
test_description_schema['test-platform'],
Required('require-signed-extensions'):
test_description_schema['require-signed-extensions'],
Required('treeherder-symbol'):
test_description_schema['treeherder-symbol'],
# Any unrecognized keys will be validated against the test_description_schema.
Extra:
object,
})
diff_description_schema = Schema({
# Name of the diff task.
Required('name'):
basestring,
# Treeherder symbol.
Required('symbol'):
basestring,
# relative path (from config.path) to the file the task was defined in.
Optional('job-from'):
basestring,
# Original and new builds to compare.
Required('original'):
index_or_string,
Required('new'):
index_or_string,
# Arguments to pass to diffoscope, used for job-defaults in
# taskcluster/ci/diffoscope/kind.yml
Optional('args'):
basestring,
# Extra arguments to pass to diffoscope, that can be set per job.
Optional('extra-args'):
basestring,
# Fail the task when differences are detected.
Optional('fail-on-diff'):
bool,
# Whether to unpack first. Diffoscope can normally work without unpacking,
# but when one needs to --exclude some contents, that doesn't work out well
# if said content is packed (e.g. in omni.ja).
Optional('unpack'):
bool,
# Commands to run before performing the diff.
Optional('pre-diff-commands'): [basestring],
})
packaging_description_schema = Schema({
# the dependant task (object) for this job, used to inform repackaging.
Required('dependent-task'):
object,
# depname is used in taskref's to identify the taskID of the signed things
Required('depname', default='build'):
basestring,
# unique label to describe this repackaging task
Optional('label'):
basestring,
# treeherder is allowed here to override any defaults we use for repackaging. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'):
task_description_schema['treeherder'],
# If a l10n task, the corresponding locale
Optional('locale'):
basestring,
# Routes specific to this task, if defined
Optional('routes'): [basestring],
# passed through directly to the job description
Optional('extra'):
task_description_schema['extra'],
})
Support for running mach python-test tasks (via run-task)
"""
from __future__ import absolute_import, print_function, unicode_literals
from six import text_type
from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
from taskgraph.util.schema import Schema
from voluptuous import Required, Optional
python_test_schema = Schema({
Required('using'): 'python-test',
# Python version to use
Required('python-version'): int,
# The subsuite to run
Required('subsuite'): text_type,
# Base work directory used to set up the task.
Optional('workdir'): text_type,
})
defaults = {
'python-version': 2,
'subsuite': 'default',
}
@run_job_using('docker-worker', 'python-test', schema=python_test_schema, defaults=defaults)
@run_job_using('generic-worker', 'python-test', schema=python_test_schema, defaults=defaults)
for k, v in task_description_schema.schema.iteritems()
}
transforms = TransformSequence()
# shortcut for a string where task references are allowed
taskref_or_string = Any(basestring, {Required('task-reference'): basestring})
balrog_description_schema = Schema({
# the dependent task (object) for this balrog job, used to inform balrogworker.
Required('dependent-task'):
object,
# unique label to describe this balrog task, defaults to balrog-{dep.label}
Optional('label'):
basestring,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'):
task_description_schema['treeherder'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('dependent-task',
object).__dict__.get('label', '?no-label?')
yield validate_schema(
run_schema = Schema(
{
Required("using"): "debian-package",
# Debian distribution
Required("dist"): text_type,
# Date of the snapshot (from snapshot.debian.org) to use, in the format
# YYYYMMDDTHHMMSSZ. The same date is used for the base docker-image name
# (only the YYYYMMDD part).
Required("snapshot"): text_type,
# URL/SHA256 of a source file to build, which can either be a source
# control (.dsc), or a tarball.
Required(Any("dsc", "tarball")): source_definition,
# Package name. Normally derived from the source control or tarball file
# name. Use in case the name doesn't match DSC_PACKAGE_RE or
# SOURCE_PACKAGE_RE.
Optional("name"): text_type,
# Patch to apply to the extracted source.
Optional("patch"): text_type,
# Command to run before dpkg-buildpackage.
Optional("pre-build-command"): text_type,
# Architecture to build the package for.
Optional("arch"): text_type,
# List of package tasks to get build dependencies from.
Optional("packages"): [text_type],
# What resolver to use to install build dependencies. The default
# (apt-get) is good in most cases, but in subtle cases involving
# a *-backports archive, its solver might not be able to find a
# solution that satisfies the build dependencies.
Optional("resolver"): Any("apt-get", "aptitude"),
# Base work directory used to set up the task.
Required("workdir"): text_type,
}
)
Support for running mach tasks (via run-task)
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
from taskgraph.util.schema import Schema
from voluptuous import Required
mach_schema = Schema({
Required('using'):
'mach',
# The mach command (omitting `./mach`) to run
Required('mach'):
basestring,
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
Required('comm-checkout'):
bool,
})
@run_job_using("docker-worker",
"mach",
schema=mach_schema,
defaults={'comm-checkout': False})
@run_job_using("native-engine",
"mach",
schema=mach_schema,
from voluptuous import Optional, Required
push_snap_description_schema = Schema({
Required('name'):
basestring,
Required('job-from'):
task_description_schema['job-from'],
Required('dependencies'):
task_description_schema['dependencies'],
Required('description'):
task_description_schema['description'],
Required('treeherder'):
task_description_schema['treeherder'],
Required('run-on-projects'):
task_description_schema['run-on-projects'],
Required('worker-type'):
optionally_keyed_by('release-level', basestring),
Required('worker'):
object,
Optional('scopes'): [basestring],
Required('shipping-phase'):
task_description_schema['shipping-phase'],
Required('shipping-product'):
task_description_schema['shipping-product'],
Optional('extra'):
task_description_schema['extra'],
Optional('attributes'):
task_description_schema['attributes'],
})
transforms = TransformSequence()
'stylo-sequential',
'qr',
'ccov',
]
def get_variant(test_platform):
for v in VARIANTS:
if '-{}/'.format(v) in test_platform:
return v
return ''
mozharness_test_run_schema = Schema({
Required('using'): 'mozharness-test',
Required('test'): test_description_schema,
# Base work directory used to set up the task.
Required('workdir'): basestring,
})
def test_packages_url(taskdesc):
"""Account for different platforms that name their test packages differently"""
artifact_url = get_artifact_url('<build>', get_artifact_path(taskdesc,
'target.test_packages.json'))
# for android nightly we need to add 'en-US' to the artifact url
test = taskdesc['run']['test']
if 'android' in test['test-platform'] and (
get_variant(test['test-platform']) in ("nightly", 'shippable')):
head, tail = os.path.split(artifact_url)
artifact_url = os.path.join(head, 'en-US', tail)
return artifact_url
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {
str(k): v
for k, v in task_description_schema.schema.iteritems()
}
transforms = TransformSequence()
repackage_signing_description_schema = Schema({
Required('dependent-task'):
object,
Required('depname', default='repackage'):
basestring,
Optional('label'):
basestring,
Optional('treeherder'):
task_description_schema['treeherder'],
Optional('shipping-product'):
task_description_schema['shipping-product'],
Optional('shipping-phase'):
task_description_schema['shipping-phase'],
})
SIGNING_FORMATS = {
'target.complete.mar': ["autograph_mar384"],
'target.bz2.complete.mar': ["mar"],
"target.installer.exe": ["sha2signcode"],
"target.stub-installer.exe": ["sha2signcodestub"],
}
current taskgraph. The transform takes a list of indexes, and the optimization
phase will replace the task with the task from the other graph.
"""
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import Schema
from taskgraph.transforms.job import run_job_using
from voluptuous import Required
transforms = TransformSequence()
run_task_schema = Schema({
Required("using"):
"index-search",
Required(
"index-search",
"A list of indexes in decreasing order of priority at which to lookup for this "
"task. This is interpolated with the graph parameters.",
): [str],
})
@run_job_using("always-optimized", "index-search", schema=run_task_schema)
def fill_template(config, job, taskdesc):
run = job["run"]
taskdesc["optimization"] = {
"index-search":
[index.format(**config.params) for index in run["index-search"]]
}
from taskgraph.util.schema import (
Schema,
taskref_or_string,
)
from voluptuous import Required, Optional, Any
mach_schema = Schema(
{
Required("using"): "mach",
# The mach command (omitting `./mach`) to run
Required("mach"): taskref_or_string,
# The version of Python to run with. Either an absolute path to the binary
# on the worker, a version identifier (e.g python2.7 or 3.8). There is no
# validation performed to ensure the specified binaries actually exist.
Optional("python-version"): Any(text_type, int, float),
# The sparse checkout profile to use. Value is the filename relative to the
# directory where sparse profiles are defined (build/sparse-profiles/).
Optional("sparse-profile"): Any(text_type, None),
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
Required("comm-checkout"): bool,
# Base work directory used to set up the task.
Optional("workdir"): text_type,
}
)
defaults = {
"comm-checkout": False,
}
from taskgraph.util import path
from taskgraph.util.schema import Schema
from voluptuous import Required, Optional
from six import text_type
from pipes import quote as shell_quote
gradlew_schema = Schema({
Required("using"):
"gradlew",
Required("gradlew"): [text_type],
Optional("post-gradlew"): [[text_type]],
# Base work directory used to set up the task.
Required("workdir"):
text_type,
Optional("use-caches"):
bool,
Optional("secrets"): [{
Required("name"): text_type,
Required("path"): text_type,
Required("key"): text_type,
Optional("json"): bool,
}],
})
@run_job_using("docker-worker", "gradlew", schema=gradlew_schema)
def configure_gradlew(config, job, taskdesc):
run = job["run"]
worker = taskdesc["worker"] = job["worker"]
mozharness_run_schema = Schema(
{
Required("using"): "mozharness",
# the mozharness script used to run this task, relative to the testing/
# directory and using forward slashes even on Windows
Required("script"): text_type,
# Additional paths to look for mozharness configs in. These should be
# relative to the base of the source checkout
Optional("config-paths"): [text_type],
# the config files required for the task, relative to
# testing/mozharness/configs or one of the paths specified in
# `config-paths` and using forward slashes even on Windows
Required("config"): [text_type],
# any additional actions to pass to the mozharness command
Optional("actions"): [
Match("^[a-z0-9-]+$", "actions must be `-` seperated alphanumeric strings")
],
# any additional options (without leading --) to be passed to mozharness
Optional("options"): [
Match(
"^[a-z0-9-]+(=[^ ]+)?$",
"options must be `-` seperated alphanumeric strings (with optional argument)",
)
],
# --custom-build-variant-cfg value
Optional("custom-build-variant-cfg"): text_type,
# Extra configuration options to pass to mozharness.
Optional("extra-config"): dict,
# If not false, tooltool downloads will be enabled via relengAPIProxy
# for either just public files, or all files. Not supported on Windows
Required("tooltool-downloads"): Any(
False,
"public",
"internal",
),
# The set of secret names to which the task has access; these are prefixed
# with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting
# this will enable any worker features required and set the task's scopes
# appropriately. `true` here means ['*'], all secrets. Not supported on
# Windows
Required("secrets"): Any(bool, [text_type]),
# If true, taskcluster proxy will be enabled; note that it may also be enabled
# automatically e.g., for secrets support. Not supported on Windows.
Required("taskcluster-proxy"): bool,
# If true, the build scripts will start Xvfb. Not supported on Windows.
Required("need-xvfb"): bool,
# If false, indicate that builds should skip producing artifacts. Not
# supported on Windows.
Required("keep-artifacts"): bool,
# If specified, use the in-tree job script specified.
Optional("job-script"): text_type,
Required("requires-signed-builds"): bool,
# Whether or not to use caches.
Optional("use-caches"): bool,
# If false, don't set MOZ_SIMPLE_PACKAGE_NAME
# Only disableable on windows
Required("use-simple-package"): bool,
# If false don't pass --branch mozharness script
# Only disableable on windows
Required("use-magic-mh-args"): bool,
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
Required("comm-checkout"): bool,
# Base work directory used to set up the task.
Optional("workdir"): text_type,
}
)
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.job import run_job_using
from taskgraph.transforms.job.run_task import (
docker_worker_run_task,
native_engine_run_task,
)
from taskgraph.util.schema import Schema
from voluptuous import Required
mach_schema = Schema({
Required('using'):
'mach',
# The mach command (omitting `./mach`) to run
Required('mach'):
basestring,
})
@run_job_using("docker-worker", "mach", schema=mach_schema)
@run_job_using("native-engine", "mach", schema=mach_schema)
def docker_worker_mach(config, job, taskdesc):
run = job['run']
# defer to the run_task implementation
run['command'] = 'cd /builds/worker/checkouts/gecko && ./mach ' + run[
'mach']
run['checkout'] = True
run['sparse-profile'] = None
raptor_description_schema = Schema({
# Raptor specific configs.
Optional("apps"):
optionally_keyed_by("test-platform", "subtest", [text_type]),
Optional("raptor-test"):
text_type,
Optional("raptor-subtests"):
optionally_keyed_by("app", "test-platform", list),
Optional("activity"):
optionally_keyed_by("app", text_type),
Optional("binary-path"):
optionally_keyed_by("app", text_type),
# Configs defined in the 'test_description_schema'.
Optional("max-run-time"):
optionally_keyed_by("app", "subtest", "test-platform",
test_description_schema["max-run-time"]),
Optional("run-on-projects"):
optionally_keyed_by(
"app",
"test-name",
"raptor-test",
"subtest",
"variant",
test_description_schema["run-on-projects"],
),
Optional("variants"):
optionally_keyed_by("app", "subtest", test_description_schema["variants"]),
Optional("target"):
optionally_keyed_by("app", test_description_schema["target"]),
Optional("tier"):
optionally_keyed_by("app", "raptor-test", "subtest", "variant",
test_description_schema["tier"]),
Optional("test-url-param"):
optionally_keyed_by("subtest", "test-platform", text_type),
Optional("run-visual-metrics"):
optionally_keyed_by("app", bool),
Required("test-name"):
test_description_schema["test-name"],
Required("test-platform"):
test_description_schema["test-platform"],
Required("require-signed-extensions"):
test_description_schema["require-signed-extensions"],
Required("treeherder-symbol"):
test_description_schema["treeherder-symbol"],
# Any unrecognized keys will be validated against the test_description_schema.
Extra:
object,
})
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
transforms = TransformSequence()
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
beetmover_checksums_description_schema = Schema({
Required('dependent-task'): object,
Required('depname', default='build'): basestring,
Optional('label'): basestring,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('locale'): basestring,
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def validate(config, jobs):
for job in jobs:
label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
validate_schema(
beetmover_checksums_description_schema, job,
"In checksums-signing ({!r} kind) task for {!r}:".format(config.kind, label))
yield job
run_schema = Schema({
Required('using'):
'debian-package',
# Debian distribution
Required('dist'):
basestring,
# Date of the snapshot (from snapshot.debian.org) to use, in the format
# YYYYMMDDTHHMMSSZ. The same date is used for the base docker-image name
# (only the YYYYMMDD part).
Required('snapshot'):
basestring,
# URL/SHA256 of a source file to build, which can either be a source
# control (.dsc), or a tarball.
Required(Any('dsc', 'tarball')):
source_definition,
# Patch to apply to the extracted source.
Optional('patch'):
basestring,
# Command to run before dpkg-buildpackage.
Optional('pre-build-command'):
basestring,
# List of package tasks to get build dependencies from.
Optional('packages'): [basestring],
# What resolver to use to install build dependencies. The default
# (apt-get) is good in most cases, but in subtle cases involving
# a *-backports archive, its solver might not be able to find a
# solution that satisfies the build dependencies.
Optional('resolver'):
Any('apt-get', 'aptitude'),
})
taskref_or_string = Any(basestring, {Required('task-reference'): basestring})
beetmover_description_schema = Schema({
# the dependent task (object) for this beetmover job, used to inform beetmover.
Required('dependent-task'):
object,
# depname is used in taskref's to identify the taskID of the unsigned things
Required('depname', default='build'):
basestring,
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional('label'):
basestring,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'):
task_description_schema['treeherder'],
# locale is passed only for l10n beetmoving
Optional('locale'):
basestring,
Optional('shipping-phase'):
task_description_schema['shipping-phase'],
Optional('shipping-product'):
task_description_schema['shipping-product'],
})
test_description_schema = Schema(
{
# description of the suite, for the task metadata
'description':
basestring,
# test suite name, or <suite>/<flavor>
Required('suite'):
optionally_keyed_by('test-platform', basestring),
# the name by which this test suite is addressed in try syntax; defaults to
# the test-name. This will translate to the `unittest_try_name` or
# `talos_try_name` attribute.
Optional('try-name'):
basestring,
# additional tags to mark up this type of test
Optional('tags'): {
basestring: object
},
# the symbol, or group(symbol), under which this task should appear in
# treeherder.
'treeherder-symbol':
basestring,
# the value to place in task.extra.treeherder.machine.platform; ideally
# this is the same as build-platform, and that is the default, but in
# practice it's not always a match.
Optional('treeherder-machine-platform'):
basestring,
# attributes to appear in the resulting task (later transforms will add the
# common attributes)
Optional('attributes'): {
basestring: object
},
# relative path (from config.path) to the file task was defined in
Optional('job-from'):
basestring,
# The `run_on_projects` attribute, defaulting to "all". This dictates the
# projects on which this task should be included in the target task set.
# See the attributes documentation for details.
#
# Note that the special case 'built-projects', the default, uses the parent
# build task's run-on-projects, meaning that tests run only on platforms
# that are built.
Optional('run-on-projects'):
optionally_keyed_by('test-platform', Any([basestring],
'built-projects')),
# the sheriffing tier for this task (default: set based on test platform)
Optional('tier'):
optionally_keyed_by('test-platform', Any(int, 'default')),
# number of chunks to create for this task. This can be keyed by test
# platform by passing a dictionary in the `by-test-platform` key. If the
# test platform is not found, the key 'default' will be tried.
Required('chunks'):
optionally_keyed_by('test-platform', int),
# the time (with unit) after which this task is deleted; default depends on
# the branch (see below)
Optional('expires-after'):
basestring,
# Whether to run this task with e10s (desktop-test only). If false, run
# without e10s; if true, run with e10s; if 'both', run one task with and
# one task without e10s. E10s tasks have "-e10s" appended to the test name
# and treeherder group.
Required('e10s'):
optionally_keyed_by('test-platform', 'project', Any(bool, 'both')),
# Whether the task should run with WebRender enabled or not.
Optional('webrender'):
bool,
# The EC2 instance size to run these tests on.
Required('instance-size'):
optionally_keyed_by('test-platform', Any('default', 'large',
'xlarge')),
# type of virtualization or hardware required by test.
Required('virtualization'):
optionally_keyed_by('test-platform',
Any('virtual', 'virtual-with-gpu', 'hardware')),
# Whether the task requires loopback audio or video (whatever that may mean
# on the platform)
Required('loopback-audio'):
bool,
Required('loopback-video'):
bool,
# Whether the test can run using a software GL implementation on Linux
# using the GL compositor. May not be used with "legacy" sized instances
# due to poor LLVMPipe performance (bug 1296086). Defaults to true for
# unit tests on linux platforms and false otherwise
Optional('allow-software-gl-layers'):
bool,
# For tasks that will run in docker-worker or docker-engine, this is the
# name of the docker image or in-tree docker image to run the task in. If
# in-tree, then a dependency will be created automatically. This is
# generally `desktop-test`, or an image that acts an awful lot like it.
Required('docker-image'):
optionally_keyed_by(
'test-platform',
Any(
# a raw Docker image path (repo/image:tag)
basestring,
# an in-tree generated docker image (from `taskcluster/docker/<name>`)
{'in-tree': basestring},
# an indexed docker image
{'indexed': basestring},
)),
# seconds of runtime after which the task will be killed. Like 'chunks',
# this can be keyed by test pltaform.
Required('max-run-time'):
optionally_keyed_by('test-platform', int),
# the exit status code that indicates the task should be retried
Optional('retry-exit-status'): [int],
# Whether to perform a gecko checkout.
Required('checkout'):
bool,
# Wheter to perform a machine reboot after test is done
Optional('reboot'):
Any(False, 'always', 'on-exception', 'on-failure'),
# What to run
Required('mozharness'): {
# the mozharness script used to run this task
Required('script'):
optionally_keyed_by('test-platform', basestring),
# the config files required for the task
Required('config'):
optionally_keyed_by('test-platform', [basestring]),
# mochitest flavor for mochitest runs
Optional('mochitest-flavor'):
basestring,
# any additional actions to pass to the mozharness command
Optional('actions'): [basestring],
# additional command-line options for mozharness, beyond those
# automatically added
Required('extra-options'):
optionally_keyed_by('test-platform', [basestring]),
# the artifact name (including path) to test on the build task; this is
# generally set in a per-kind transformation
Optional('build-artifact-name'):
basestring,
# If true, tooltool downloads will be enabled via relengAPIProxy.
Required('tooltool-downloads'):
bool,
# Add --blob-upload-branch=<project> mozharness parameter
Optional('include-blob-upload-branch'):
bool,
# The setting for --download-symbols (if omitted, the option will not
# be passed to mozharness)
Optional('download-symbols'):
Any(True, 'ondemand'),
# If set, then MOZ_NODE_PATH=/usr/local/bin/node is included in the
# environment. This is more than just a helpful path setting -- it
# causes xpcshell tests to start additional servers, and runs
# additional tests.
Required('set-moz-node-path'):
bool,
# If true, include chunking information in the command even if the number
# of chunks is 1
Required('chunked'):
optionally_keyed_by('test-platform', bool),
# The chunking argument format to use
Required('chunking-args'):
Any(
# Use the usual --this-chunk/--total-chunk arguments
'this-chunk',
# Use --test-suite=<suite>-<chunk-suffix>; see chunk-suffix, below
'test-suite-suffix',
),
# the string to append to the `--test-suite` arugment when
# chunking-args = test-suite-suffix; "<CHUNK>" in this string will
# be replaced with the chunk number.
Optional('chunk-suffix'):
basestring,
Required('requires-signed-builds'):
optionally_keyed_by('test-platform', bool),
},
# The current chunk; this is filled in by `all_kinds.py`
Optional('this-chunk'):
int,
# os user groups for test task workers; required scopes, will be
# added automatically
Optional('os-groups'):
optionally_keyed_by('test-platform', [basestring]),
# -- values supplied by the task-generation infrastructure
# the platform of the build this task is testing
'build-platform':
basestring,
# the label of the build task generating the materials to test
'build-label':
basestring,
# the label of the signing task generating the materials to test.
# Signed builds are used in xpcshell tests on Windows, for instance.
Optional('build-signing-label'):
basestring,
# the build's attributes
'build-attributes': {
basestring: object
},
# the platform on which the tests will run
'test-platform':
basestring,
# the name of the test (the key in tests.yml)
'test-name':
basestring,
# the product name, defaults to firefox
Optional('product'):
basestring,
# conditional files to determine when these tests should be run
Exclusive(Optional('when'), 'optimization'):
Any({
Optional('files-changed'): [basestring],
}),
# Optimization to perform on this task during the optimization phase.
# Optimizations are defined in taskcluster/taskgraph/optimize.py.
Exclusive(Optional('optimization'), 'optimization'):
OptimizationSchema,
# The SCHEDULES component for this task; this defaults to the suite
# (not including the flavor) but can be overridden here.
Exclusive(Optional('schedules-component'), 'optimization'):
basestring,
Optional('worker-type'):
optionally_keyed_by(
'test-platform',
Any(basestring, None),
),
# The target name, specifying the build artifact to be tested.
# If None or not specified, a transform sets the target based on OS:
# target.dmg (Mac), target.apk (Android), target.tar.bz2 (Linux),
# or target.zip (Windows).
Optional('target'):
optionally_keyed_by(
'test-platform',
Any(basestring, None),
),
},
required=True)
push_apk_description_schema = Schema({
Required('dependent-tasks'):
object,
Required('name'):
basestring,
Required('label'):
task_description_schema['label'],
Required('description'):
task_description_schema['description'],
Required('job-from'):
task_description_schema['job-from'],
Required('attributes'):
task_description_schema['attributes'],
Required('treeherder'):
task_description_schema['treeherder'],
Required('run-on-projects'):
task_description_schema['run-on-projects'],
Required('worker-type'):
optionally_keyed_by('project', basestring),
Required('worker'):
object,
Required('scopes'):
None,
Required('requires'):
task_description_schema['requires'],
Required('deadline-after'):
basestring,
Required('shipping-phase'):
task_description_schema['shipping-phase'],
Required('shipping-product'):
task_description_schema['shipping-product'],
Optional('extra'):
task_description_schema['extra'],
})
beetmover_cdns_description_schema = Schema({
Required('name'):
basestring,
Required('product'):
basestring,
Required('treeherder-platform'):
basestring,
Optional('attributes'): {
basestring: object
},
Optional('job-from'):
task_description_schema['job-from'],
Optional('run'): {
basestring: object
},
Optional('run-on-projects'):
task_description_schema['run-on-projects'],
Required('worker-type'):
optionally_keyed_by('project', basestring),
Optional('dependencies'): {
basestring: taskref_or_string
},
Optional('index'): {
basestring: basestring
},
Optional('routes'): [basestring],
Required('shipping-phase'):
task_description_schema['shipping-phase'],
Required('shipping-product'):
task_description_schema['shipping-product'],
Optional('extra'):
task_description_schema['extra'],
})