# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'):
task_description_schema['treeherder'],
Optional('shipping-phase'):
task_description_schema['shipping-phase'],
Optional('shipping-product'):
task_description_schema['shipping-product'],
Optional('attributes'):
task_description_schema['attributes'],
})
transforms = TransformSequence()
transforms.add_validate(release_generate_checksums_beetmover_schema)
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
attributes.update(job.get('attributes', {}))
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'BM-SGenChcks')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
}},
),
Optional('python-version'): [int],
})
transforms = TransformSequence()
@transforms.add
def set_defaults(config, jobs):
for job in jobs:
job.setdefault('require-build', False)
yield job
transforms.add_validate(source_test_description_schema)
@transforms.add
def set_job_name(config, jobs):
for job in jobs:
if 'job-from' in job and job['job-from'] != 'kind.yml':
from_name = os.path.splitext(job['job-from'])[0]
job['name'] = '{}-{}'.format(from_name, job['name'])
yield job
@transforms.add
def expand_platforms(config, jobs):
for job in jobs:
if isinstance(job['platform'], basestring):
Required('scopes'): optionally_keyed_by('release-level', [basestring]),
Required('shipping-phase'): task_description_schema['shipping-phase'],
Required('shipping-product'): task_description_schema['shipping-product'],
})
@transforms.add
def set_label(config, jobs):
for job in jobs:
label = 'sign-and-push-langpacks-{}'.format(job['primary-dependency'].label)
job['label'] = label
yield job
transforms.add_validate(langpack_sign_push_description_schema)
@transforms.add
def resolve_keys(config, jobs):
for job in jobs:
resolve_keyed_by(
job, 'worker-type', item_name=job['label'],
**{'release-level': config.params.release_level()}
)
resolve_keyed_by(
job, 'scopes', item_name=job['label'],
**{'release-level': config.params.release_level()}
)
resolve_keyed_by(
job, 'worker.channel', item_name=job['label'],
Optional('config-paths'): [text_type],
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
Optional('comm-checkout'):
bool,
},
# Override the default priority for the project
Optional('priority'):
task_description_schema['priority'],
})
transforms = TransformSequence()
transforms.add(check_if_partners_enabled)
transforms.add_validate(packaging_description_schema)
@transforms.add
def copy_in_useful_magic(config, jobs):
"""Copy attributes from upstream task to be used for keyed configuration."""
for job in jobs:
dep = job['primary-dependency']
job['build-platform'] = dep.attributes.get("build_platform")
yield job
@transforms.add
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by platform, etc."""
fields = [
from taskgraph.task import Task
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import resolve_keyed_by
from voluptuous import Required, Schema
from xpi_taskgraph.xpi_manifest import get_manifest
transforms = TransformSequence()
schema = Schema(
{
Required("primary-dependency"): Task,
Required("worker-type"): str,
Required("attributes"): dict,
Required("bucket-scope"): dict,
Required("run-on-tasks-for"): [str],
}, )
transforms.add_validate(schema)
@transforms.add
def add_beetmover_worker_config(config, tasks):
manifest = get_manifest()
for task in tasks:
if not (config.params.get("version") and config.params.get("xpi_name")
and config.params.get("head_ref")
and config.params.get("build_number")
and config.params.get("level")):
continue
xpi_name = config.params["xpi_name"]
xpi_manifest = manifest[xpi_name]
xpi_addon_type = xpi_manifest["addon-type"]
build_number = config.params["build_number"]
transforms = TransformSequence()
transforms.add_validate(
Schema({
Required('dependent-tasks'):
object,
Required('name'):
basestring,
Required('label'):
task_description_schema['label'],
Required('description'):
task_description_schema['description'],
Required('job-from'):
task_description_schema['job-from'],
Required('attributes'):
task_description_schema['attributes'],
Required('treeherder'):
task_description_schema['treeherder'],
Required('package-name'):
optionally_keyed_by('project', basestring),
Required('run-on-projects'):
task_description_schema['run-on-projects'],
Required('worker-type'):
basestring,
Required('worker'):
object,
Required('shipping-phase'):
task_description_schema['shipping-phase'],
Required('shipping-product'):
task_description_schema['shipping-product'],
}))
transforms.add(validate_dependent_tasks)
Optional('max-run-time'):
int,
Optional('extra'): {
basestring: object
},
})
@transforms.add
def set_defaults(config, jobs):
for job in jobs:
job.setdefault('depname', 'build')
yield job
transforms.add_validate(signing_description_schema)
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
signing_format_scopes = []
formats = set([])
for artifacts in job['upstream-artifacts']:
for f in artifacts['formats']:
formats.add(f) # Add each format only once
for format in formats:
signing_format_scopes.append(
checksums_signing_description_schema = schema.extend({
Required('depname', default='beetmover'):
basestring,
Optional('label'):
basestring,
Optional('treeherder'):
task_description_schema['treeherder'],
Optional('shipping-product'):
task_description_schema['shipping-product'],
Optional('shipping-phase'):
task_description_schema['shipping-phase'],
})
transforms = TransformSequence()
transforms.add_validate(checksums_signing_description_schema)
@transforms.add
def make_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = dep_job.attributes
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'css(N)')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform', "{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
yield job
@transforms.add
def copy_in_useful_magic(config, jobs):
for job in jobs:
dep = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep)
attributes.update(job.get('attributes', {}))
# build-platform is needed on `job` for by-build-platform
job['build-platform'] = attributes.get("build_platform")
job['attributes'] = attributes
yield job
transforms.add_validate(l10n_description_schema)
@transforms.add
def setup_nightly_dependency(config, jobs):
""" Sets up a task dependency to the signing job this relates to """
for job in jobs:
job['dependencies'] = {'build': job['dependent-tasks']['build'].label}
if job['attributes']['build_platform'].startswith('win') or \
job['attributes']['build_platform'].startswith('linux'):
job['dependencies'].update({
'build-signing':
job['dependent-tasks']['build-signing'].label,
})
if job['attributes']['build_platform'].startswith('macosx'):
job['dependencies'].update(
task_description_schema["worker-type"],
Optional("scopes"):
task_description_schema["scopes"],
Optional("job-from"):
task_description_schema["job-from"],
Required("worker"):
object,
Required("run"): {
str: Any(str, bool)
},
Optional("requires-level"):
int,
Optional("release-artifacts"): [str],
})
transforms.add_validate(build_schema)
# Deletes a task when the current level is < then the required level
@transforms.add
def checkRequiredLevel(config, tasks):
for task in tasks:
if "requires-level" in task:
requiredLevel = int(task.pop("requires-level"))
currentLevel = int(config.params["level"])
if requiredLevel <= currentLevel:
yield task
else:
yield task
Optional('fail-on-diff'): bool,
# Whether to unpack first. Diffoscope can normally work without unpacking,
# but when one needs to --exclude some contents, that doesn't work out well
# if said content is packed (e.g. in omni.ja).
Optional('unpack'): bool,
# Commands to run before performing the diff.
Optional('pre-diff-commands'): [text_type],
# Only run the task on a set of projects/branches.
Optional('run-on-projects'): task_description_schema['run-on-projects'],
})
transforms = TransformSequence()
transforms.add_validate(diff_description_schema)
@transforms.add
def fill_template(config, tasks):
dummy_tasks = {}
for task in tasks:
name = task['name']
deps = {}
urls = {}
previous_artifact = None
for k in ('original', 'new'):
value = task[k]
if isinstance(value, text_type):
# arguments. Expecting:
# [variant-suffix, options-to-use]
# If variant-suffix is `null` then the options will be added
# to the existing task. Otherwise, a new variant is created
# with the given suffix and with its options replaced.
Optional('perftest-btime-variants'): optionally_keyed_by(
'perftest',
[[Any(None, text_type)]]
),
# These options will be parsed in the next schemas
Extra: object,
})
transforms.add_validate(perftest_description_schema)
@transforms.add
def split_tests(config, jobs):
for job in jobs:
if job.get("perftest") is None:
yield job
continue
for test_symbol, test_name in job.pop("perftest"):
job_new = deepcopy(job)
job_new["perftest"] = test_symbol
job_new["name"] += "-" + test_symbol
job_new["treeherder"]["symbol"] = job["treeherder"]["symbol"].format(
basestring: taskref_or_string
},
Optional('index'): {
basestring: basestring
},
Optional('routes'): [basestring],
Required('shipping-phase'):
task_description_schema['shipping-phase'],
Required('shipping-product'):
task_description_schema['shipping-product'],
Optional('extra'):
task_description_schema['extra'],
})
transforms = TransformSequence()
transforms.add_validate(beetmover_push_to_release_description_schema)
@transforms.add
def make_beetmover_push_to_release_description(config, jobs):
for job in jobs:
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'Rel(BM-C)')
treeherder.setdefault('tier', 1)
treeherder.setdefault('kind', 'build')
treeherder.setdefault('platform', job['treeherder-platform'])
label = job['name']
description = ("Beetmover push to release for '{product}'".format(
product=job['product']))
optionally_keyed_by("release-level", text_type),
Required("worker"):
object,
Optional("scopes"): [text_type],
Required("shipping-phase"):
task_description_schema["shipping-phase"],
Required("shipping-product"):
task_description_schema["shipping-product"],
Optional("extra"):
task_description_schema["extra"],
Optional("attributes"):
task_description_schema["attributes"],
})
transforms = TransformSequence()
transforms.add_validate(push_flatpak_description_schema)
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
if len(job["dependencies"]) != 1:
raise Exception("Exactly 1 dependency is required")
job["worker"]["upstream-artifacts"] = generate_upstream_artifacts(
job["dependencies"])
resolve_keyed_by(job,
"worker.channel",
item_name=job["name"],
**{"release-type": config.params["release_type"]})
'subtest',
test_description_schema['tier']
),
Optional('run-visual-metrics'): optionally_keyed_by(
'app',
bool
),
Required('test-name'): test_description_schema['test-name'],
Required('test-platform'): test_description_schema['test-platform'],
Required('require-signed-extensions'): test_description_schema['require-signed-extensions'],
Required('treeherder-symbol'): test_description_schema['treeherder-symbol'],
# Any unrecognized keys will be validated against the test_description_schema.
Extra: object,
})
transforms.add_validate(raptor_description_schema)
@transforms.add
def set_defaults(config, tests):
for test in tests:
test.setdefault('pageload', 'warm')
test.setdefault('run-visual-metrics', False)
yield test
@transforms.add
def split_apps(config, tests):
app_symbols = {
'chrome': 'ChR',
'chrome-m': 'ChR',
google_play_description_schema = Schema({
Required('name'): text_type,
Required('description'): task_description_schema['description'],
Required('job-from'): task_description_schema['job-from'],
Required('attributes'): task_description_schema['attributes'],
Required('treeherder'): task_description_schema['treeherder'],
Required('run-on-projects'): task_description_schema['run-on-projects'],
Required('shipping-phase'): task_description_schema['shipping-phase'],
Required('shipping-product'): task_description_schema['shipping-product'],
Required('worker-type'): task_description_schema['worker-type'],
Required('worker'): object,
})
transforms = TransformSequence()
transforms.add_validate(google_play_description_schema)
@transforms.add
def set_label(config, jobs):
for job in jobs:
job['label'] = job['name']
yield job
@transforms.add
def set_worker_data(config, jobs):
for job in jobs:
worker = job['worker']
env = worker.setdefault('env', {})
get_signing_cert_scope,
)
from taskgraph.util.taskcluster import get_artifact_path
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
release_generate_checksums_signing_schema = schema.extend({
Required('depname', default='release-generate-checksums'): text_type,
Optional('label'): text_type,
Optional('treeherder'): task_description_schema['treeherder'],
Optional('shipping-product'): task_description_schema['shipping-product'],
Optional('shipping-phase'): task_description_schema['shipping-phase'],
})
transforms = TransformSequence()
transforms.add_validate(release_generate_checksums_signing_schema)
@transforms.add
def make_release_generate_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get('treeherder', {})
treeherder.setdefault('symbol', 'SGenChcks')
dep_th_platform = dep_job.task.get('extra', {}).get(
'treeherder', {}).get('machine', {}).get('platform', '')
treeherder.setdefault('platform',
"{}/opt".format(dep_th_platform))
treeherder.setdefault('tier', 1)
# Any remaining content is verified against that job implementation's
# own schema.
Extra: object,
},
Required('worker-type'):
task_description_schema['worker-type'],
# This object will be passed through to the task description, with additions
# provided by the job's run-using function
Optional('worker'):
dict,
})
transforms = TransformSequence()
transforms.add_validate(job_description_schema)
@transforms.add
def rewrite_when_to_optimization(config, jobs):
for job in jobs:
when = job.pop('when', {})
if not when:
yield job
continue
files_changed = when.get('files-changed')
# implicitly add task config directory.
files_changed.append('{}/**'.format(config.path))
Required('run-on-projects'):
task_description_schema['run-on-projects'],
Required('run-on-hg-branches'):
task_description_schema['run-on-hg-branches'],
Optional('bucket-scope'):
optionally_keyed_by('release-level', text_type),
Optional('shipping-phase'):
optionally_keyed_by('project', task_description_schema['shipping-phase']),
Optional('shipping-product'):
task_description_schema['shipping-product'],
Optional('attributes'):
task_description_schema['attributes'],
})
transforms = TransformSequence()
transforms.add_validate(beetmover_description_schema)
@transforms.add
def resolve_keys(config, jobs):
for job in jobs:
resolve_keyed_by(job,
'run-on-hg-branches',
item_name=job['label'],
project=config.params['project'])
resolve_keyed_by(job,
'shipping-phase',
item_name=job['label'],
project=config.params['project'])
resolve_keyed_by(job,
'bucket-scope',
Required('job-from'): task_description_schema['job-from'],
Required('dependencies'): task_description_schema['dependencies'],
Required('description'): task_description_schema['description'],
Required('treeherder'): task_description_schema['treeherder'],
Required('run-on-projects'): task_description_schema['run-on-projects'],
Required('worker-type'): optionally_keyed_by('release-level', text_type),
Required('worker'): object,
Optional('scopes'): [text_type],
Required('shipping-phase'): task_description_schema['shipping-phase'],
Required('shipping-product'): task_description_schema['shipping-product'],
Optional('extra'): task_description_schema['extra'],
Optional('attributes'): task_description_schema['attributes'],
})
transforms = TransformSequence()
transforms.add_validate(push_snap_description_schema)
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
if len(job['dependencies']) != 1:
raise Exception('Exactly 1 dependency is required')
job['worker']['upstream-artifacts'] = generate_upstream_artifacts(job['dependencies'])
resolve_keyed_by(
job, 'worker.channel', item_name=job['name'],
**{'release-type': config.params['release_type']}
)
resolve_keyed_by(
text_type,
# List of package tasks this docker image depends on.
Optional("packages"): [text_type],
Optional(
"index",
description="information for indexing this build so its artifacts can be discovered",
):
task_description_schema["index"],
Optional(
"cache",
description="Whether this image should be cached based on inputs.",
):
bool,
})
transforms.add_validate(docker_image_schema)
@transforms.add
def fill_template(config, tasks):
if not taskgraph.fast and config.write_artifacts:
if not os.path.isdir(CONTEXTS_DIR):
os.makedirs(CONTEXTS_DIR)
for task in tasks:
image_name = task.pop("name")
job_symbol = task.pop("symbol")
args = task.pop("args", {})
packages = task.pop("packages", [])
parent = task.pop("parent", None)
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'):
task_description_schema['treeherder'],
Optional('attributes'):
task_description_schema['attributes'],
# Shipping product / phase
Optional('shipping-product'):
task_description_schema['shipping-product'],
Optional('shipping-phase'):
task_description_schema['shipping-phase'],
})
transforms = TransformSequence()
transforms.add_validate(balrog_description_schema)
@transforms.add
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by platform, etc."""
fields = [
"update-no-wnp",
]
for job in jobs:
label = job.get('dependent-task',
object).__dict__.get('label', '?no-label?')
for field in fields:
resolve_keyed_by(item=job,
field=field,
item_name=label,
Required('depname', default='repackage'):
text_type,
Optional('label'):
text_type,
Optional('extra'):
object,
Optional('shipping-product'):
task_description_schema['shipping-product'],
Optional('shipping-phase'):
task_description_schema['shipping-phase'],
Optional('priority'):
task_description_schema['priority'],
})
transforms.add(check_if_partners_enabled)
transforms.add_validate(repackage_signing_description_schema)
@transforms.add
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job['primary-dependency']
repack_id = dep_job.task['extra']['repack_id']
attributes = dep_job.attributes
build_platform = dep_job.attributes.get('build_platform')
is_shippable = dep_job.attributes.get('shippable')
# Mac & windows
label = dep_job.label.replace("repackage-", "repackage-signing-")
# Linux
label = label.replace("chunking-dummy-", "repackage-signing-")
# A description of how to run this job.
Optional('run'): dict,
Required('thirdparty'): {
Required('artifact'): text_type,
Required('script'): text_type,
Optional('args'): [text_type]
},
Optional('toolchain'): [text_type],
Required('when'): {
Required('files-changed'): [text_type],
},
})
transforms = TransformSequence()
transforms.add_validate(SCHEMA)
def make_base_task(config, name, job, script, command):
"""
Common config for thirdparty build tasks
"""
if config.params['level'] == '3':
expires = '1 year'
else:
expires = '28 days'
# To be consistent with how files-changed is used elsewhere, the
# path must be relative to GECKO,
script_rel = mozpath.relpath(script, GECKO)
