# treeherder is allowed here to override any defaults we use for beetmover. See # taskcluster/taskgraph/transforms/task.py for the schema details, and the # below transforms for defaults of various values. Optional('treeherder'): task_description_schema['treeherder'], Optional('shipping-phase'): task_description_schema['shipping-phase'], Optional('shipping-product'): task_description_schema['shipping-product'], Optional('attributes'): task_description_schema['attributes'], }) transforms = TransformSequence() transforms.add_validate(release_generate_checksums_beetmover_schema) @transforms.add def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) attributes.update(job.get('attributes', {})) treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'BM-SGenChcks') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1)
}}, ), Optional('python-version'): [int], }) transforms = TransformSequence() @transforms.add def set_defaults(config, jobs): for job in jobs: job.setdefault('require-build', False) yield job transforms.add_validate(source_test_description_schema) @transforms.add def set_job_name(config, jobs): for job in jobs: if 'job-from' in job and job['job-from'] != 'kind.yml': from_name = os.path.splitext(job['job-from'])[0] job['name'] = '{}-{}'.format(from_name, job['name']) yield job @transforms.add def expand_platforms(config, jobs): for job in jobs: if isinstance(job['platform'], basestring):
Required('scopes'): optionally_keyed_by('release-level', [basestring]), Required('shipping-phase'): task_description_schema['shipping-phase'], Required('shipping-product'): task_description_schema['shipping-product'], }) @transforms.add def set_label(config, jobs): for job in jobs: label = 'sign-and-push-langpacks-{}'.format(job['primary-dependency'].label) job['label'] = label yield job transforms.add_validate(langpack_sign_push_description_schema) @transforms.add def resolve_keys(config, jobs): for job in jobs: resolve_keyed_by( job, 'worker-type', item_name=job['label'], **{'release-level': config.params.release_level()} ) resolve_keyed_by( job, 'scopes', item_name=job['label'], **{'release-level': config.params.release_level()} ) resolve_keyed_by( job, 'worker.channel', item_name=job['label'],
Optional('config-paths'): [text_type], # if true, perform a checkout of a comm-central based branch inside the # gecko checkout Optional('comm-checkout'): bool, }, # Override the default priority for the project Optional('priority'): task_description_schema['priority'], }) transforms = TransformSequence() transforms.add(check_if_partners_enabled) transforms.add_validate(packaging_description_schema) @transforms.add def copy_in_useful_magic(config, jobs): """Copy attributes from upstream task to be used for keyed configuration.""" for job in jobs: dep = job['primary-dependency'] job['build-platform'] = dep.attributes.get("build_platform") yield job @transforms.add def handle_keyed_by(config, jobs): """Resolve fields that can be keyed by platform, etc.""" fields = [
from taskgraph.task import Task from taskgraph.transforms.base import TransformSequence from taskgraph.util.schema import resolve_keyed_by from voluptuous import Required, Schema from xpi_taskgraph.xpi_manifest import get_manifest transforms = TransformSequence() schema = Schema( { Required("primary-dependency"): Task, Required("worker-type"): str, Required("attributes"): dict, Required("bucket-scope"): dict, Required("run-on-tasks-for"): [str], }, ) transforms.add_validate(schema) @transforms.add def add_beetmover_worker_config(config, tasks): manifest = get_manifest() for task in tasks: if not (config.params.get("version") and config.params.get("xpi_name") and config.params.get("head_ref") and config.params.get("build_number") and config.params.get("level")): continue xpi_name = config.params["xpi_name"] xpi_manifest = manifest[xpi_name] xpi_addon_type = xpi_manifest["addon-type"] build_number = config.params["build_number"]
transforms = TransformSequence() transforms.add_validate( Schema({ Required('dependent-tasks'): object, Required('name'): basestring, Required('label'): task_description_schema['label'], Required('description'): task_description_schema['description'], Required('job-from'): task_description_schema['job-from'], Required('attributes'): task_description_schema['attributes'], Required('treeherder'): task_description_schema['treeherder'], Required('package-name'): optionally_keyed_by('project', basestring), Required('run-on-projects'): task_description_schema['run-on-projects'], Required('worker-type'): basestring, Required('worker'): object, Required('shipping-phase'): task_description_schema['shipping-phase'], Required('shipping-product'): task_description_schema['shipping-product'], })) transforms.add(validate_dependent_tasks)
Optional('max-run-time'): int, Optional('extra'): { basestring: object }, }) @transforms.add def set_defaults(config, jobs): for job in jobs: job.setdefault('depname', 'build') yield job transforms.add_validate(signing_description_schema) @transforms.add def make_task_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes signing_format_scopes = [] formats = set([]) for artifacts in job['upstream-artifacts']: for f in artifacts['formats']: formats.add(f) # Add each format only once for format in formats: signing_format_scopes.append(
checksums_signing_description_schema = schema.extend({ Required('depname', default='beetmover'): basestring, Optional('label'): basestring, Optional('treeherder'): task_description_schema['treeherder'], Optional('shipping-product'): task_description_schema['shipping-product'], Optional('shipping-phase'): task_description_schema['shipping-phase'], }) transforms = TransformSequence() transforms.add_validate(checksums_signing_description_schema) @transforms.add def make_checksums_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = dep_job.attributes treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'css(N)') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build')
yield job @transforms.add def copy_in_useful_magic(config, jobs): for job in jobs: dep = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep) attributes.update(job.get('attributes', {})) # build-platform is needed on `job` for by-build-platform job['build-platform'] = attributes.get("build_platform") job['attributes'] = attributes yield job transforms.add_validate(l10n_description_schema) @transforms.add def setup_nightly_dependency(config, jobs): """ Sets up a task dependency to the signing job this relates to """ for job in jobs: job['dependencies'] = {'build': job['dependent-tasks']['build'].label} if job['attributes']['build_platform'].startswith('win') or \ job['attributes']['build_platform'].startswith('linux'): job['dependencies'].update({ 'build-signing': job['dependent-tasks']['build-signing'].label, }) if job['attributes']['build_platform'].startswith('macosx'): job['dependencies'].update(
task_description_schema["worker-type"], Optional("scopes"): task_description_schema["scopes"], Optional("job-from"): task_description_schema["job-from"], Required("worker"): object, Required("run"): { str: Any(str, bool) }, Optional("requires-level"): int, Optional("release-artifacts"): [str], }) transforms.add_validate(build_schema) # Deletes a task when the current level is < then the required level @transforms.add def checkRequiredLevel(config, tasks): for task in tasks: if "requires-level" in task: requiredLevel = int(task.pop("requires-level")) currentLevel = int(config.params["level"]) if requiredLevel <= currentLevel: yield task else: yield task
Optional('fail-on-diff'): bool, # Whether to unpack first. Diffoscope can normally work without unpacking, # but when one needs to --exclude some contents, that doesn't work out well # if said content is packed (e.g. in omni.ja). Optional('unpack'): bool, # Commands to run before performing the diff. Optional('pre-diff-commands'): [text_type], # Only run the task on a set of projects/branches. Optional('run-on-projects'): task_description_schema['run-on-projects'], }) transforms = TransformSequence() transforms.add_validate(diff_description_schema) @transforms.add def fill_template(config, tasks): dummy_tasks = {} for task in tasks: name = task['name'] deps = {} urls = {} previous_artifact = None for k in ('original', 'new'): value = task[k] if isinstance(value, text_type):
# arguments. Expecting: # [variant-suffix, options-to-use] # If variant-suffix is `null` then the options will be added # to the existing task. Otherwise, a new variant is created # with the given suffix and with its options replaced. Optional('perftest-btime-variants'): optionally_keyed_by( 'perftest', [[Any(None, text_type)]] ), # These options will be parsed in the next schemas Extra: object, }) transforms.add_validate(perftest_description_schema) @transforms.add def split_tests(config, jobs): for job in jobs: if job.get("perftest") is None: yield job continue for test_symbol, test_name in job.pop("perftest"): job_new = deepcopy(job) job_new["perftest"] = test_symbol job_new["name"] += "-" + test_symbol job_new["treeherder"]["symbol"] = job["treeherder"]["symbol"].format(
basestring: taskref_or_string }, Optional('index'): { basestring: basestring }, Optional('routes'): [basestring], Required('shipping-phase'): task_description_schema['shipping-phase'], Required('shipping-product'): task_description_schema['shipping-product'], Optional('extra'): task_description_schema['extra'], }) transforms = TransformSequence() transforms.add_validate(beetmover_push_to_release_description_schema) @transforms.add def make_beetmover_push_to_release_description(config, jobs): for job in jobs: treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'Rel(BM-C)') treeherder.setdefault('tier', 1) treeherder.setdefault('kind', 'build') treeherder.setdefault('platform', job['treeherder-platform']) label = job['name'] description = ("Beetmover push to release for '{product}'".format( product=job['product']))
optionally_keyed_by("release-level", text_type), Required("worker"): object, Optional("scopes"): [text_type], Required("shipping-phase"): task_description_schema["shipping-phase"], Required("shipping-product"): task_description_schema["shipping-product"], Optional("extra"): task_description_schema["extra"], Optional("attributes"): task_description_schema["attributes"], }) transforms = TransformSequence() transforms.add_validate(push_flatpak_description_schema) @transforms.add def make_task_description(config, jobs): for job in jobs: if len(job["dependencies"]) != 1: raise Exception("Exactly 1 dependency is required") job["worker"]["upstream-artifacts"] = generate_upstream_artifacts( job["dependencies"]) resolve_keyed_by(job, "worker.channel", item_name=job["name"], **{"release-type": config.params["release_type"]})
'subtest', test_description_schema['tier'] ), Optional('run-visual-metrics'): optionally_keyed_by( 'app', bool ), Required('test-name'): test_description_schema['test-name'], Required('test-platform'): test_description_schema['test-platform'], Required('require-signed-extensions'): test_description_schema['require-signed-extensions'], Required('treeherder-symbol'): test_description_schema['treeherder-symbol'], # Any unrecognized keys will be validated against the test_description_schema. Extra: object, }) transforms.add_validate(raptor_description_schema) @transforms.add def set_defaults(config, tests): for test in tests: test.setdefault('pageload', 'warm') test.setdefault('run-visual-metrics', False) yield test @transforms.add def split_apps(config, tests): app_symbols = { 'chrome': 'ChR', 'chrome-m': 'ChR',
google_play_description_schema = Schema({ Required('name'): text_type, Required('description'): task_description_schema['description'], Required('job-from'): task_description_schema['job-from'], Required('attributes'): task_description_schema['attributes'], Required('treeherder'): task_description_schema['treeherder'], Required('run-on-projects'): task_description_schema['run-on-projects'], Required('shipping-phase'): task_description_schema['shipping-phase'], Required('shipping-product'): task_description_schema['shipping-product'], Required('worker-type'): task_description_schema['worker-type'], Required('worker'): object, }) transforms = TransformSequence() transforms.add_validate(google_play_description_schema) @transforms.add def set_label(config, jobs): for job in jobs: job['label'] = job['name'] yield job @transforms.add def set_worker_data(config, jobs): for job in jobs: worker = job['worker'] env = worker.setdefault('env', {})
get_signing_cert_scope, ) from taskgraph.util.taskcluster import get_artifact_path from taskgraph.transforms.task import task_description_schema from voluptuous import Required, Optional release_generate_checksums_signing_schema = schema.extend({ Required('depname', default='release-generate-checksums'): text_type, Optional('label'): text_type, Optional('treeherder'): task_description_schema['treeherder'], Optional('shipping-product'): task_description_schema['shipping-product'], Optional('shipping-phase'): task_description_schema['shipping-phase'], }) transforms = TransformSequence() transforms.add_validate(release_generate_checksums_signing_schema) @transforms.add def make_release_generate_checksums_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] attributes = copy_attributes_from_dependent_job(dep_job) treeherder = job.get('treeherder', {}) treeherder.setdefault('symbol', 'SGenChcks') dep_th_platform = dep_job.task.get('extra', {}).get( 'treeherder', {}).get('machine', {}).get('platform', '') treeherder.setdefault('platform', "{}/opt".format(dep_th_platform)) treeherder.setdefault('tier', 1)
# Any remaining content is verified against that job implementation's # own schema. Extra: object, }, Required('worker-type'): task_description_schema['worker-type'], # This object will be passed through to the task description, with additions # provided by the job's run-using function Optional('worker'): dict, }) transforms = TransformSequence() transforms.add_validate(job_description_schema) @transforms.add def rewrite_when_to_optimization(config, jobs): for job in jobs: when = job.pop('when', {}) if not when: yield job continue files_changed = when.get('files-changed') # implicitly add task config directory. files_changed.append('{}/**'.format(config.path))
Required('run-on-projects'): task_description_schema['run-on-projects'], Required('run-on-hg-branches'): task_description_schema['run-on-hg-branches'], Optional('bucket-scope'): optionally_keyed_by('release-level', text_type), Optional('shipping-phase'): optionally_keyed_by('project', task_description_schema['shipping-phase']), Optional('shipping-product'): task_description_schema['shipping-product'], Optional('attributes'): task_description_schema['attributes'], }) transforms = TransformSequence() transforms.add_validate(beetmover_description_schema) @transforms.add def resolve_keys(config, jobs): for job in jobs: resolve_keyed_by(job, 'run-on-hg-branches', item_name=job['label'], project=config.params['project']) resolve_keyed_by(job, 'shipping-phase', item_name=job['label'], project=config.params['project']) resolve_keyed_by(job, 'bucket-scope',
Required('job-from'): task_description_schema['job-from'], Required('dependencies'): task_description_schema['dependencies'], Required('description'): task_description_schema['description'], Required('treeherder'): task_description_schema['treeherder'], Required('run-on-projects'): task_description_schema['run-on-projects'], Required('worker-type'): optionally_keyed_by('release-level', text_type), Required('worker'): object, Optional('scopes'): [text_type], Required('shipping-phase'): task_description_schema['shipping-phase'], Required('shipping-product'): task_description_schema['shipping-product'], Optional('extra'): task_description_schema['extra'], Optional('attributes'): task_description_schema['attributes'], }) transforms = TransformSequence() transforms.add_validate(push_snap_description_schema) @transforms.add def make_task_description(config, jobs): for job in jobs: if len(job['dependencies']) != 1: raise Exception('Exactly 1 dependency is required') job['worker']['upstream-artifacts'] = generate_upstream_artifacts(job['dependencies']) resolve_keyed_by( job, 'worker.channel', item_name=job['name'], **{'release-type': config.params['release_type']} ) resolve_keyed_by(
text_type, # List of package tasks this docker image depends on. Optional("packages"): [text_type], Optional( "index", description="information for indexing this build so its artifacts can be discovered", ): task_description_schema["index"], Optional( "cache", description="Whether this image should be cached based on inputs.", ): bool, }) transforms.add_validate(docker_image_schema) @transforms.add def fill_template(config, tasks): if not taskgraph.fast and config.write_artifacts: if not os.path.isdir(CONTEXTS_DIR): os.makedirs(CONTEXTS_DIR) for task in tasks: image_name = task.pop("name") job_symbol = task.pop("symbol") args = task.pop("args", {}) packages = task.pop("packages", []) parent = task.pop("parent", None)
# taskcluster/taskgraph/transforms/task.py for the schema details, and the # below transforms for defaults of various values. Optional('treeherder'): task_description_schema['treeherder'], Optional('attributes'): task_description_schema['attributes'], # Shipping product / phase Optional('shipping-product'): task_description_schema['shipping-product'], Optional('shipping-phase'): task_description_schema['shipping-phase'], }) transforms = TransformSequence() transforms.add_validate(balrog_description_schema) @transforms.add def handle_keyed_by(config, jobs): """Resolve fields that can be keyed by platform, etc.""" fields = [ "update-no-wnp", ] for job in jobs: label = job.get('dependent-task', object).__dict__.get('label', '?no-label?') for field in fields: resolve_keyed_by(item=job, field=field, item_name=label,
Required('depname', default='repackage'): text_type, Optional('label'): text_type, Optional('extra'): object, Optional('shipping-product'): task_description_schema['shipping-product'], Optional('shipping-phase'): task_description_schema['shipping-phase'], Optional('priority'): task_description_schema['priority'], }) transforms.add(check_if_partners_enabled) transforms.add_validate(repackage_signing_description_schema) @transforms.add def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job['primary-dependency'] repack_id = dep_job.task['extra']['repack_id'] attributes = dep_job.attributes build_platform = dep_job.attributes.get('build_platform') is_shippable = dep_job.attributes.get('shippable') # Mac & windows label = dep_job.label.replace("repackage-", "repackage-signing-") # Linux label = label.replace("chunking-dummy-", "repackage-signing-")
# A description of how to run this job. Optional('run'): dict, Required('thirdparty'): { Required('artifact'): text_type, Required('script'): text_type, Optional('args'): [text_type] }, Optional('toolchain'): [text_type], Required('when'): { Required('files-changed'): [text_type], }, }) transforms = TransformSequence() transforms.add_validate(SCHEMA) def make_base_task(config, name, job, script, command): """ Common config for thirdparty build tasks """ if config.params['level'] == '3': expires = '1 year' else: expires = '28 days' # To be consistent with how files-changed is used elsewhere, the # path must be relative to GECKO, script_rel = mozpath.relpath(script, GECKO)