# Schema for a test description
#
# *****WARNING*****
#
# This is a great place for baffling cruft to accumulate, and that makes
# everyone move more slowly. Be considerate of your fellow hackers!
# See the warnings in taskcluster/docs/how-tos.rst
#
# *****WARNING*****
test_description_schema = Schema({
# description of the suite, for the task metadata
'description': basestring,
# test suite name, or <suite>/<flavor>
Required('suite'): optionally_keyed_by(
'test-platform',
basestring),
# the name by which this test suite is addressed in try syntax; defaults to
# the test-name
Optional('unittest-try-name'): basestring,
# the name by which this talos test is addressed in try syntax
Optional('talos-try-name'): basestring,
# additional tags to mark up this type of test
Optional('tags'): {basestring: object},
# the symbol, or group(symbol), under which this task should appear in
# treeherder.
'treeherder-symbol': basestring,
# *****WARNING*****
#
# This is a great place for baffling cruft to accumulate, and that makes
# everyone move more slowly. Be considerate of your fellow hackers!
# See the warnings in taskcluster/docs/how-tos.rst
#
# *****WARNING*****
test_description_schema = Schema(
{
# description of the suite, for the task metadata
'description':
basestring,
# test suite name, or <suite>/<flavor>
Required('suite'):
optionally_keyed_by('test-platform', basestring),
# base work directory used to set up the task.
Optional('workdir'):
optionally_keyed_by('test-platform', Any(basestring, 'default')),
# the name by which this test suite is addressed in try syntax; defaults to
# the test-name. This will translate to the `unittest_try_name` or
# `talos_try_name` attribute.
Optional('try-name'):
basestring,
# additional tags to mark up this type of test
Optional('tags'): {
basestring: object
},
# repeated here
Extra:
object,
# The platform on which this task runs. This will be used to set up attributes
# (for try selection) and treeherder metadata (for display). If given as a list,
# the job will be "split" into multiple tasks, one with each platform.
Required('platform'):
Any(text_type, [text_type]),
# Build labels required for the task. If this key is provided it must
# contain a build label for the task platform.
# The task will then depend on a build task, and the installer url will be
# saved to the GECKO_INSTALLER_URL environment variable.
Optional('require-build'):
optionally_keyed_by('project', {text_type: text_type}),
# These fields can be keyed by "platform", and are otherwise identical to
# job descriptions.
Required('worker-type'):
optionally_keyed_by('platform', job_description_schema['worker-type']),
Required('worker'):
optionally_keyed_by('platform', job_description_schema['worker']),
Optional('python-version'): [int],
# A list of artifacts to install from 'fetch' tasks.
Optional('fetches'): {
text_type:
optionally_keyed_by('platform',
job_description_schema['fetches'][text_type]),
},
from taskgraph.transforms.google_play_strings import set_worker_data
from taskgraph.transforms.job.mozharness_test import get_artifact_url
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
from voluptuous import Required
transforms = TransformSequence()
transforms.add_validate(Schema({
Required('dependent-tasks'): object,
Required('name'): text_type,
Required('label'): task_description_schema['label'],
Required('description'): task_description_schema['description'],
Required('job-from'): task_description_schema['job-from'],
Required('attributes'): task_description_schema['attributes'],
Required('treeherder'): task_description_schema['treeherder'],
Required('package-name'): optionally_keyed_by('project', text_type),
Required('run-on-projects'): task_description_schema['run-on-projects'],
Required('worker-type'): text_type,
Required('worker'): object,
Required('shipping-phase'): task_description_schema['shipping-phase'],
Required('shipping-product'): task_description_schema['shipping-product'],
}))
transforms.add(validate_dependent_tasks)
transforms.add(set_worker_data)
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
job['dependencies'] = generate_dependencies(job['dependent-tasks'])
dependencies_labels = job['dependencies'].keys()
'maple': '-nightly-maple',
}
beetmover_description_schema = schema.extend({
Required('depname', default='build'):
basestring,
Optional('label'):
basestring,
Optional('treeherder'):
task_description_schema['treeherder'],
Required('run-on-projects'):
task_description_schema['run-on-projects'],
Required('run-on-hg-branches'):
task_description_schema['run-on-hg-branches'],
Optional('bucket-scope'):
optionally_keyed_by('release-level', basestring),
Optional('shipping-phase'):
optionally_keyed_by('project', task_description_schema['shipping-phase']),
Optional('shipping-product'):
task_description_schema['shipping-product'],
Optional('attributes'):
task_description_schema['attributes'],
})
transforms = TransformSequence()
transforms.add_validate(beetmover_description_schema)
@transforms.add
def resolve_keys(config, jobs):
for job in jobs:
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
push_apk_description_schema = Schema({
Required('dependent-tasks'): object,
Required('name'): basestring,
Required('label'): task_description_schema['label'],
Required('description'): task_description_schema['description'],
Required('job-from'): task_description_schema['job-from'],
Required('attributes'): task_description_schema['attributes'],
Required('treeherder'): task_description_schema['treeherder'],
Required('run-on-projects'): task_description_schema['run-on-projects'],
Required('worker-type'): optionally_keyed_by('project', basestring),
Required('worker'): object,
Required('scopes'): None,
Required('requires'): task_description_schema['requires'],
Required('deadline-after'): basestring,
Required('shipping-phase'): task_description_schema['shipping-phase'],
Required('shipping-product'): task_description_schema['shipping-product'],
Optional('extra'): task_description_schema['extra'],
})
REQUIRED_ARCHITECTURES = {
'android-x86-nightly',
'android-api-16-nightly',
}
PLATFORM_REGEX = re.compile(r'build-signing-android-(\S+)-nightly')
# passed through directly to the job description
Optional('extra'):
job_description_schema['extra'],
# passed through to job description
Optional('fetches'):
job_description_schema['fetches'],
# Shipping product and phase
Optional('shipping-product'):
job_description_schema['shipping-product'],
Optional('shipping-phase'):
job_description_schema['shipping-phase'],
Required('package-formats'):
optionally_keyed_by('build-platform', 'release-type', [text_type]),
# All l10n jobs use mozharness
Required('mozharness'): {
Extra:
object,
# Config files passed to the mozharness script
Required('config'):
optionally_keyed_by('build-platform', [text_type]),
# Additional paths to look for mozharness configs in. These should be
# relative to the base of the source checkout
Optional('config-paths'): [text_type],
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
transforms = TransformSequence()
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
push_apk_breakpoint_description_schema = Schema({
# the dependent task (object) for this beetmover job, used to inform beetmover.
Required('dependent-tasks'): object,
Required('name'): basestring,
Required('label'): basestring,
Required('description'): basestring,
Required('job-from'): basestring,
Required('attributes'): object,
Required('worker-type'): optionally_keyed_by('project', basestring),
Required('worker'): object,
Required('treeherder'): object,
Required('run-on-projects'): list,
Required('deadline-after'): basestring,
Required('shipping-phase'): task_description_schema['shipping-phase'],
Required('shipping-product'): task_description_schema['shipping-product'],
})
validate_jobs_schema_transform = functools.partial(
validate_jobs_schema_transform_partial,
push_apk_breakpoint_description_schema,
'PushApkBreakpoint'
)
transforms.add(fill_labels_tranform)
optionally_keyed_by,
resolve_keyed_by,
)
from taskgraph.util.treeherder import replace_group
from taskgraph.transforms.task import task_description_schema
from voluptuous import Optional
balrog_description_schema = schema.extend({
# unique label to describe this balrog task, defaults to balrog-{dep.label}
Optional("label"):
text_type,
Optional(
"update-no-wnp",
description="Whether the parallel `-No-WNP` blob should be updated as well.",
):
optionally_keyed_by("release-type", bool),
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional("treeherder"):
task_description_schema["treeherder"],
Optional("attributes"):
task_description_schema["attributes"],
# Shipping product / phase
Optional("shipping-product"):
task_description_schema["shipping-product"],
Optional("shipping-phase"):
task_description_schema["shipping-phase"],
})
transforms = TransformSequence()
Optional,
Required,
Extra,
)
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.tests import test_description_schema
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
from taskgraph.util.treeherder import split_symbol, join_symbol
transforms = TransformSequence()
raptor_description_schema = Schema({
# Raptor specific configs.
Optional("apps"):
optionally_keyed_by("test-platform", "subtest", [text_type]),
Optional("raptor-test"):
text_type,
Optional("raptor-subtests"):
optionally_keyed_by("app", "test-platform", list),
Optional("activity"):
optionally_keyed_by("app", text_type),
Optional("binary-path"):
optionally_keyed_by("app", text_type),
# Configs defined in the 'test_description_schema'.
Optional("max-run-time"):
optionally_keyed_by("app", "subtest", "test-platform",
test_description_schema["max-run-time"]),
Optional("run-on-projects"):
optionally_keyed_by(
"app",
push_snap_description_schema = Schema({
Required('name'):
basestring,
Required('job-from'):
task_description_schema['job-from'],
Required('dependencies'):
task_description_schema['dependencies'],
Required('description'):
task_description_schema['description'],
Required('treeherder'):
task_description_schema['treeherder'],
Required('run-on-projects'):
task_description_schema['run-on-projects'],
Required('worker-type'):
optionally_keyed_by('release-level', basestring),
Required('worker'):
object,
Required('scopes'):
optionally_keyed_by('project', [basestring]),
Required('shipping-phase'):
task_description_schema['shipping-phase'],
Required('shipping-product'):
task_description_schema['shipping-product'],
Optional('extra'):
task_description_schema['extra'],
})
@transforms.add
def validate_jobs_schema_transform(config, jobs):
from voluptuous import Optional, Required, Any
from taskgraph.transforms.job import run_job_using
buildbot_run_schema = Schema({
Required('using'): 'buildbot',
# the buildername to use for buildbot-bridge, will expand {branch} in name from
# the current project.
Required('buildername'): basestring,
# the product to use
Required('product'): Any('firefox', 'mobile', 'fennec', 'devedition', 'thunderbird'),
Optional('channels'): optionally_keyed_by('project', basestring),
Optional('release-promotion'): bool,
Optional('release-eta'): basestring,
})
def _get_balrog_api_root(branch):
if branch in ('mozilla-beta', 'mozilla-release') or branch.startswith('mozilla-esr'):
return 'https://aus4-admin.mozilla.org/api'
else:
return 'https://balrog-admin.stage.mozaws.net/api'
def bb_release_worker(config, worker, run):
# match function (python/mozbuild/mozpack/path.py).
Optional('files-changed'): [basestring],
}),
# A description of how to run this job.
'run': {
# The key to a job implementation in a peer module to this one
'using': basestring,
# Any remaining content is verified against that job implementation's
# own schema.
Extra: object,
},
Required('worker-type'): optionally_keyed_by(
'platform',
task_description_schema['worker-type']),
# for `worker`, all we need is the implementation; the rest will be verified
# by the task description schema
Required('worker'): optionally_keyed_by(
'platform', {
Required('implementation'): basestring,
Extra: object,
}),
})
transforms = TransformSequence()
@transforms.add
# passed through directly to the job description
Optional('extra'):
job_description_schema['extra'],
# passed through to job description
Optional('fetches'):
job_description_schema['fetches'],
# Shipping product and phase
Optional('shipping-product'):
job_description_schema['shipping-product'],
Optional('shipping-phase'):
job_description_schema['shipping-phase'],
Required('package-formats'):
optionally_keyed_by('build-platform', 'project', [basestring]),
# All l10n jobs use mozharness
Required('mozharness'): {
# Config files passed to the mozharness script
Required('config'):
optionally_keyed_by('build-platform', [basestring]),
# Additional paths to look for mozharness configs in. These should be
# relative to the base of the source checkout
Optional('config-paths'): [basestring],
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
Required('comm-checkout', default=False):
bool,
Extra,
)
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.tests import test_description_schema
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
from taskgraph.util.treeherder import split_symbol, join_symbol
transforms = TransformSequence()
raptor_description_schema = Schema({
# Raptor specific configs.
Optional('apps'): optionally_keyed_by(
'test-platform',
'subtest',
[text_type]
),
Optional('raptor-test'): text_type,
Optional('raptor-subtests'): optionally_keyed_by(
'app',
'test-platform',
list
),
Optional('activity'): optionally_keyed_by(
'app',
text_type
),
Optional('binary-path'): optionally_keyed_by(
'app',
text_type
def _by_platform(arg):
return optionally_keyed_by('build-platform', arg)
text_type: object
},
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional('label'):
text_type,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'):
task_description_schema['treeherder'],
Required('description'):
text_type,
Required('worker-type'):
optionally_keyed_by('release-level', text_type),
Required('run-on-projects'): [],
# locale is passed only for l10n beetmoving
Optional('locale'):
text_type,
Optional('shipping-phase'):
task_description_schema['shipping-phase'],
Optional('shipping-product'):
task_description_schema['shipping-product'],
})
transforms.add_validate(beetmover_description_schema)
@transforms.add
# Schema for a test description
#
# *****WARNING*****
#
# This is a great place for baffling cruft to accumulate, and that makes
# everyone move more slowly. Be considerate of your fellow hackers!
# See the warnings in taskcluster/docs/how-tos.rst
#
# *****WARNING*****
test_description_schema = Schema({
# description of the suite, for the task metadata
'description': basestring,
# test suite name, or <suite>/<flavor>
Required('suite'): optionally_keyed_by(
'test-platform',
basestring),
# the name by which this test suite is addressed in try syntax; defaults to
# the test-name. This will translate to the `unittest_try_name` or
# `talos_try_name` attribute.
Optional('try-name'): basestring,
# additional tags to mark up this type of test
Optional('tags'): {basestring: object},
# the symbol, or group(symbol), under which this task should appear in
# treeherder.
'treeherder-symbol': basestring,
# the value to place in task.extra.treeherder.machine.platform; ideally
beetmover_description_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the unsigned things
Required('depname', default='build'): basestring,
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional('label'): basestring,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'): task_description_schema['treeherder'],
Required('description'): basestring,
Required('worker-type'): optionally_keyed_by('release-level', basestring),
Required('run-on-projects'): [],
# locale is passed only for l10n beetmoving
Optional('locale'): basestring,
Optional('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def set_label(config, jobs):
for job in jobs:
job['label'] = job['primary-dependency'].label.replace(
'sign-and-push-langpacks', 'beetmover-signed-langpacks'
)
# attributes is used for enabling artifact-map by declarative artifacts
Required("attributes"): {
text_type: object
},
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional("label"):
text_type,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional("treeherder"):
task_description_schema["treeherder"],
Required("description"):
text_type,
Required("worker-type"):
optionally_keyed_by("release-level", text_type),
Required("run-on-projects"): [],
# locale is passed only for l10n beetmoving
Optional("locale"):
text_type,
Optional("shipping-phase"):
task_description_schema["shipping-phase"],
Optional("shipping-product"):
task_description_schema["shipping-product"],
})
transforms.add_validate(beetmover_description_schema)
@transforms.add
def resolve_keys(config, jobs):
# passed through directly to the job description
Optional('extra'):
job_description_schema['extra'],
# passed through to job description
Optional('fetches'):
job_description_schema['fetches'],
# Shipping product and phase
Optional('shipping-product'):
job_description_schema['shipping-product'],
Optional('shipping-phase'):
job_description_schema['shipping-phase'],
Required('package-formats'):
optionally_keyed_by('build-platform', 'release-type', [basestring]),
# All l10n jobs use mozharness
Required('mozharness'): {
Extra:
object,
# Config files passed to the mozharness script
Required('config'):
optionally_keyed_by('build-platform', [basestring]),
# Additional paths to look for mozharness configs in. These should be
# relative to the base of the source checkout
Optional('config-paths'): [basestring],
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
Optional,
Required,
Extra,
)
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.tests import test_description_schema
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
from taskgraph.util.treeherder import split_symbol, join_symbol
transforms = TransformSequence()
raptor_description_schema = Schema({
# Raptor specific configs.
Optional('apps'):
optionally_keyed_by('test-platform', [text_type]),
Optional('raptor-test'):
text_type,
Optional('raptor-subtests'):
optionally_keyed_by('app', [text_type]),
Optional('activity'):
optionally_keyed_by('app', text_type),
Optional('binary-path'):
optionally_keyed_by('app', text_type),
Optional('pageload'):
optionally_keyed_by(
'test-platform',
'app',
Any('cold', 'warm', 'both'),
),
# Configs defined in the 'test_description_schema'.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required
transforms = TransformSequence()
langpack_sign_push_description_schema = schema.extend({
Required('label'): basestring,
Required('description'): basestring,
Required('worker-type'): optionally_keyed_by('release-level', basestring),
Required('worker'): {
Required('implementation'): 'sign-and-push-addons',
Required('channel'): optionally_keyed_by(
'project',
optionally_keyed_by('platform', Any('listed', 'unlisted'))),
Required('upstream-artifacts'): None, # Processed here below
},
Required('run-on-projects'): [],
Required('scopes'): optionally_keyed_by('release-level', [basestring]),
Required('shipping-phase'): task_description_schema['shipping-phase'],
Required('shipping-product'): task_description_schema['shipping-product'],
})
# Schema for a test description
#
# *****WARNING*****
#
# This is a great place for baffling cruft to accumulate, and that makes
# everyone move more slowly. Be considerate of your fellow hackers!
# See the warnings in taskcluster/docs/how-tos.rst
#
# *****WARNING*****
test_description_schema = Schema({
# description of the suite, for the task metadata
'description': basestring,
# test suite name, or <suite>/<flavor>
Required('suite'): optionally_keyed_by(
'test-platform',
basestring),
# base work directory used to set up the task.
Optional('workdir'): optionally_keyed_by(
'test-platform',
Any(basestring, 'default')),
# the name by which this test suite is addressed in try syntax; defaults to
# the test-name. This will translate to the `unittest_try_name` or
# `talos_try_name` attribute.
Optional('try-name'): basestring,
# additional tags to mark up this type of test
Optional('tags'): {basestring: object},
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
from taskgraph.util.treeherder import inherit_treeherder_from_dep
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required
transforms = TransformSequence()
langpack_sign_push_description_schema = schema.extend({
Required('label'):
text_type,
Required('description'):
text_type,
Required('worker-type'):
optionally_keyed_by('release-level', text_type),
Required('worker'): {
Required('implementation'):
'push-addons',
Required('channel'):
optionally_keyed_by(
'project',
optionally_keyed_by('platform', Any('listed', 'unlisted'))),
Required('upstream-artifacts'):
None, # Processed here below
},
Required('run-on-projects'): [],
Required('scopes'):
optionally_keyed_by('release-level', [text_type]),
Required('shipping-phase'):
task_description_schema['shipping-phase'],
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {
str(k): v
for k, v in task_description_schema.schema.iteritems()
}
balrog_description_schema = schema.extend({
# unique label to describe this balrog task, defaults to balrog-{dep.label}
Optional('label'):
basestring,
Optional(
'update-no-wnp',
description="Whether the parallel `-No-WNP` blob should be updated as well.",
):
optionally_keyed_by('release-type', bool),
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'):
task_description_schema['treeherder'],
Optional('attributes'):
task_description_schema['attributes'],
# Shipping product / phase
Optional('shipping-product'):
task_description_schema['shipping-product'],
Optional('shipping-phase'):
task_description_schema['shipping-phase'],
})
from voluptuous import Any, Required, Optional
from copy import deepcopy
import logging
logger = logging.getLogger(__name__)
beetmover_description_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the unsigned things
Required('depname', default='build'): basestring,
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional('label'): basestring,
Required('partner-bucket-scope'): optionally_keyed_by('release-level', basestring),
Required('partner-public-path'): Any(None, basestring),
Required('partner-private-path'): Any(None, basestring),
Optional('extra'): object,
Required('shipping-phase'): task_description_schema['shipping-phase'],
Optional('shipping-product'): task_description_schema['shipping-product'],
})
transforms = TransformSequence()
transforms.add(check_if_partners_enabled)
transforms.add_validate(beetmover_description_schema)
@transforms.add
def resolve_keys(config, jobs):
from copy import deepcopy
import logging
logger = logging.getLogger(__name__)
beetmover_description_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the unsigned things
Required('depname', default='build'):
basestring,
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional('label'):
basestring,
Required('partner-bucket-scope'):
optionally_keyed_by('release-level', basestring),
Required('partner-public-path'):
Any(None, basestring),
Required('partner-private-path'):
Any(None, basestring),
Optional('extra'):
object,
Required('shipping-phase'):
task_description_schema['shipping-phase'],
Optional('shipping-product'):
task_description_schema['shipping-product'],
})
transforms = TransformSequence()
transforms.add(check_if_partners_enabled)
transforms.add_validate(beetmover_description_schema)
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
from taskgraph.util.treeherder import inherit_treeherder_from_dep
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required
transforms = TransformSequence()
langpack_sign_push_description_schema = schema.extend({
Required('label'):
basestring,
Required('description'):
basestring,
Required('worker-type'):
optionally_keyed_by('release-level', basestring),
Required('worker'): {
Required('implementation'):
'push-addons',
Required('channel'):
optionally_keyed_by(
'project',
optionally_keyed_by('platform', Any('listed', 'unlisted'))),
Required('upstream-artifacts'):
None, # Processed here below
},
Required('run-on-projects'): [],
Required('scopes'):
optionally_keyed_by('release-level', [basestring]),
Required('shipping-phase'):
task_description_schema['shipping-phase'],
text_type,
# Routes specific to this task, if defined
Optional("routes"): [text_type],
# passed through directly to the job description
Optional("extra"):
job_description_schema["extra"],
# passed through to job description
Optional("fetches"):
job_description_schema["fetches"],
# Shipping product and phase
Optional("shipping-product"):
job_description_schema["shipping-product"],
Optional("shipping-phase"):
job_description_schema["shipping-phase"],
Required("package-formats"):
optionally_keyed_by("build-platform", "release-type", [text_type]),
# All l10n jobs use mozharness
Required("mozharness"): {
Extra:
object,
# Config files passed to the mozharness script
Required("config"):
optionally_keyed_by("build-platform", [text_type]),
# Additional paths to look for mozharness configs in. These should be
# relative to the base of the source checkout
Optional("config-paths"): [text_type],
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
Optional("comm-checkout"):
bool,
},
# run on all projects for which cron tasks are set up. This works just like the
# `run_on_projects` attribute, where strings like "release" and "integration" are
# expanded to cover multiple repositories. (taskcluster/docs/attributes.rst)
'run-on-projects': [basestring],
# Array of times at which this task should run. These *must* be a
# multiple of 15 minutes, the minimum scheduling interval. This field
# can be keyed by project so that each project has a different schedule
# for the same job.
'when':
optionally_keyed_by(
'project',
[{
'hour':
int,
'minute':
All(int, even_15_minutes),
# You probably don't want both day and weekday.
'day':
int, # Day of the month, as used by datetime.
'weekday':
Any('Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday',
'Saturday', 'Sunday')
}]),
}],
})
def validate(cron_yml):
validate_schema(cron_yml_schema, cron_yml, "Invalid .cron.yml:")
Required('product'):
basestring,
Required('treeherder-platform'):
basestring,
Optional('attributes'): {
basestring: object
},
Optional('job-from'):
task_description_schema['job-from'],
Optional('run'): {
basestring: object
},
Optional('run-on-projects'):
task_description_schema['run-on-projects'],
Required('worker-type'):
optionally_keyed_by('project', basestring),
Optional('dependencies'): {
basestring: taskref_or_string
},
Optional('index'): {
basestring: basestring
},
Optional('routes'): [basestring],
Required('shipping-phase'):
task_description_schema['shipping-phase'],
Required('shipping-product'):
task_description_schema['shipping-product'],
Optional('notifications'):
task_description_schema['notifications'],
})
)
from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
from taskgraph.util.scriptworker import generate_beetmover_artifact_map
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
beetmover_description_schema = schema.extend({
Required('depname', default='build'): text_type,
Optional('label'): text_type,
Optional('treeherder'): task_description_schema['treeherder'],
Required('run-on-projects'): task_description_schema['run-on-projects'],
Required('run-on-hg-branches'): task_description_schema['run-on-hg-branches'],
Optional('bucket-scope'): optionally_keyed_by('release-level', text_type),
Optional('shipping-phase'): optionally_keyed_by(
'project', task_description_schema['shipping-phase']
),
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('attributes'): task_description_schema['attributes'],
})
transforms = TransformSequence()
transforms.add_validate(beetmover_description_schema)
@transforms.add
def resolve_keys(config, jobs):
for job in jobs:
resolve_keyed_by(
Required('type'): 'decision-task',
# Treeherder symbol for the cron task
Required('treeherder-symbol'): basestring,
# --target-tasks-method './mach taskgraph decision' argument
'target-tasks-method': basestring,
}),
# when to run it
# Optional set of projects on which this job should run; if omitted, this will
# run on all projects for which cron tasks are set up. This works just like the
# `run_on_projects` attribute, where strings like "release" and "integration" are
# expanded to cover multiple repositories. (taskcluster/docs/attributes.rst)
'run-on-projects': [basestring],
# Array of times at which this task should run. These *must* be a
# multiple of 15 minutes, the minimum scheduling interval. This field
# can be keyed by project so that each project has a different schedule
# for the same job.
'when': optionally_keyed_by(
'project',
[{'hour': int, 'minute': All(int, even_15_minutes)}]),
}],
})
def validate(cron_yml):
validate_schema(cron_yml_schema, cron_yml, "Invalid .cron.yml:")
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
from taskgraph.util.treeherder import split_symbol, join_symbol
transforms = TransformSequence()
perftest_description_schema = Schema({
# The test names and the symbols to use for them: [test-symbol, test-path]
Optional("perftest"): [[text_type]],
# Metrics to gather for the test. These will be merged
# with options specified through perftest-perfherder-global
Optional("perftest-metrics"):
optionally_keyed_by(
"perftest",
Any(
[text_type],
{
text_type:
Any(None, {text_type: Any(None, text_type, [text_type])})
},
),
),
# Perfherder data options that will be applied to
# all metrics gathered.
Optional("perftest-perfherder-global"):
optionally_keyed_by("perftest",
{text_type: Any(None, text_type, [text_type])}),
# Extra options to add to the test's command
Optional("perftest-extra-options"):
optionally_keyed_by("perftest", [text_type]),
# Variants of the test to make based on extra browsertime
# arguments. Expecting:
# [variant-suffix, options-to-use]
def _by_platform(arg):
return optionally_keyed_by('build-platform', arg)
# Schema for a test description
#
# *****WARNING*****
#
# This is a great place for baffling cruft to accumulate, and that makes
# everyone move more slowly. Be considerate of your fellow hackers!
# See the warnings in taskcluster/docs/how-tos.rst
#
# *****WARNING*****
test_description_schema = Schema({
# description of the suite, for the task metadata
'description': basestring,
# test suite name, or <suite>/<flavor>
Required('suite'): optionally_keyed_by(
'test-platform',
basestring),
# the name by which this test suite is addressed in try syntax; defaults to
# the test-name. This will translate to the `unittest_try_name` or
# `talos_try_name` attribute.
Optional('try-name'): basestring,
# additional tags to mark up this type of test
Optional('tags'): {basestring: object},
# the symbol, or group(symbol), under which this task should appear in
# treeherder.
'treeherder-symbol': basestring,
# the value to place in task.extra.treeherder.machine.platform; ideally