from __future__ import absolute_import, print_function, unicode_literals
import functools
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import Schema
from taskgraph.util.scriptworker import get_push_apk_scope, get_push_apk_track, \
get_push_apk_dry_run_option, get_push_apk_rollout_percentage
from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial, \
validate_dependent_tasks_transform, delete_non_required_fields_transform, generate_dependencies
from voluptuous import Required
transforms = TransformSequence()
push_apk_description_schema = Schema({
# the dependent task (object) for this beetmover job, used to inform beetmover.
Required('dependent-tasks'): object,
Required('name'): basestring,
Required('label'): basestring,
Required('description'): basestring,
Required('attributes'): object,
Required('treeherder'): object,
Required('run-on-projects'): list,
Required('worker-type'): basestring,
Required('worker'): object,
Required('scopes'): None,
Required('deadline-after'): basestring,
})
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import copy
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.declarative_artifacts import get_geckoview_upstream_artifacts
from taskgraph.util.taskcluster import get_artifact_prefix
transforms = TransformSequence()
MOZ_ANDROID_FAT_AAR_ENV_MAP = {
'android-api-16-nightly': 'MOZ_ANDROID_FAT_AAR_ARMEABI_V7A',
'android-aarch64-nightly': 'MOZ_ANDROID_FAT_AAR_ARM64_V8A',
'android-x86-nightly': 'MOZ_ANDROID_FAT_AAR_X86',
'android-x86_64-nightly': 'MOZ_ANDROID_FAT_AAR_X86_64',
'android-api-16-opt': 'MOZ_ANDROID_FAT_AAR_ARMEABI_V7A',
'android-aarch64-opt': 'MOZ_ANDROID_FAT_AAR_ARM64_V8A',
'android-x86-opt': 'MOZ_ANDROID_FAT_AAR_X86',
'android-x86_64-opt': 'MOZ_ANDROID_FAT_AAR_X86_64',
}
@transforms.add
def set_fetches_and_locations(config, jobs):
Transform the push-apk kind into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
import functools
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.task import task_description_schema
from taskgraph.util.schema import resolve_keyed_by, Schema
from taskgraph.util.push_apk import fill_labels_tranform, validate_jobs_schema_transform_partial
from voluptuous import Required
transforms = TransformSequence()
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
google_play_description_schema = Schema({
Required('name'): basestring,
Required('label'): task_description_schema['label'],
Required('description'): task_description_schema['description'],
Required('job-from'): task_description_schema['job-from'],
Required('attributes'): task_description_schema['attributes'],
Required('treeherder'): task_description_schema['treeherder'],
Required('run-on-projects'): task_description_schema['run-on-projects'],
Required('shipping-phase'): task_description_schema['shipping-phase'],
Required('shipping-product'): task_description_schema['shipping-product'],
import copy
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import (
validate_schema,
optionally_keyed_by,
resolve_keyed_by,
Schema,
)
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.util.partners import check_if_partners_enabled
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional
transforms = TransformSequence()
# Voluptuous uses marker objects as dictionary *keys*, but they are not
# comparable, so we cast all of the keys back to regular strings
task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
def _by_platform(arg):
return optionally_keyed_by('build-platform', arg)
# shortcut for a string where task references are allowed
taskref_or_string = Any(
basestring,
{Required('task-reference'): basestring})
"setupexe": "setup-stub.exe",
},
'output':
'target.stub-installer.exe',
},
}
MOZHARNESS_EXPANSIONS = [
'package-name',
'installer-tag',
'fetch-dir',
'stub-installer-tag',
'sfx-stub',
'wsx-stub',
]
transforms = TransformSequence()
transforms.add_validate(packaging_description_schema)
@transforms.add
def copy_in_useful_magic(config, jobs):
"""Copy attributes from upstream task to be used for keyed configuration."""
for job in jobs:
dep = job['primary-dependency']
job['build-platform'] = dep.attributes.get("build_platform")
yield job
@transforms.add
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by platform, etc."""
from six import text_type
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.task import task_description_schema
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.scriptworker import (
generate_beetmover_artifact_map,
generate_beetmover_upstream_artifacts,
get_beetmover_bucket_scope,
get_beetmover_action_scope,
)
from taskgraph.util.treeherder import replace_group
transforms = TransformSequence()
beetmover_description_schema = schema.extend(
{
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional("label"): text_type,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional("treeherder"): task_description_schema["treeherder"],
# locale is passed only for l10n beetmoving
Optional("locale"): text_type,
Required("shipping-phase"): task_description_schema["shipping-phase"],
Optional("shipping-product"): task_description_schema["shipping-product"],
Optional("attributes"): task_description_schema["attributes"],
}
"""
Transform the release-sign-and-push task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from six import text_type
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
from taskgraph.util.treeherder import inherit_treeherder_from_dep
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required
transforms = TransformSequence()
langpack_sign_push_description_schema = schema.extend(
{
Required("label"): text_type,
Required("description"): text_type,
Required("worker-type"): optionally_keyed_by("release-level", text_type),
Required("worker"): {
Required("implementation"): "push-addons",
Required("channel"): optionally_keyed_by(
"project", "platform", Any("listed", "unlisted")
),
Required("upstream-artifacts"): None, # Processed here below
},
Required("run-on-projects"): [],
Required("scopes"): optionally_keyed_by("release-level", [text_type]),
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.task import _run_task_suffix
from .. import GECKO
from taskgraph.util.docker import (
generate_context_hash, )
from taskgraph.util.taskcluster import get_root_url
from taskgraph.util.schema import (
Schema, )
from voluptuous import (
Optional,
Required,
)
DIGEST_RE = re.compile('^[0-9a-f]{64}$')
transforms = TransformSequence()
docker_image_schema = Schema({
# Name of the docker image.
Required('name'):
basestring,
# Name of the parent docker image.
Optional('parent'):
basestring,
# Treeherder symbol.
Required('symbol'):
basestring,
# relative path (from config.path) to the file the docker image was defined
# Additional paths to look for mozharness configs in. These should be
# relative to the base of the source checkout
Optional('config-paths'): [text_type],
# if true, perform a checkout of a comm-central based branch inside the
# gecko checkout
Required('comm-checkout', default=False):
bool,
},
# Override the default priority for the project
Optional('priority'):
task_description_schema['priority'],
})
transforms = TransformSequence()
transforms.add(check_if_partners_enabled)
transforms.add_validate(packaging_description_schema)
@transforms.add
def copy_in_useful_magic(config, jobs):
"""Copy attributes from upstream task to be used for keyed configuration."""
for job in jobs:
dep = job['primary-dependency']
job['build-platform'] = dep.attributes.get("build_platform")
yield job
@transforms.add
def handle_keyed_by(config, jobs):
optionally_keyed_by("release-level", text_type),
Required("partner-public-path"):
Any(None, text_type),
Required("partner-private-path"):
Any(None, text_type),
Optional("extra"):
object,
Required("shipping-phase"):
task_description_schema["shipping-phase"],
Optional("shipping-product"):
task_description_schema["shipping-product"],
Optional("priority"):
task_description_schema["priority"],
})
transforms = TransformSequence()
transforms.add_validate(beetmover_description_schema)
transforms.add(apply_partner_priority)
@transforms.add
def resolve_keys(config, jobs):
for job in jobs:
resolve_keyed_by(job,
"partner-bucket-scope",
item_name=job["label"],
**{"release-level": config.params.release_level()})
yield job
@transforms.add
Optional('workdir'): basestring,
# Any remaining content is verified against that job implementation's
# own schema.
Extra: object,
},
Required('worker-type'):
task_description_schema['worker-type'],
# This object will be passed through to the task description, with additions
# provided by the job's run-using function
Optional('worker'):
dict,
})
transforms = TransformSequence()
transforms.add_validate(job_description_schema)
@transforms.add
def rewrite_when_to_optimization(config, jobs):
for job in jobs:
when = job.pop('when', {})
if not when:
yield job
continue
files_changed = when.get('files-changed')
# implicitly add task config directory.
files_changed.append('{}/**'.format(config.path))
optionally_keyed_by('release-level', text_type),
Required('partner-public-path'):
Any(None, text_type),
Required('partner-private-path'):
Any(None, text_type),
Optional('extra'):
object,
Required('shipping-phase'):
task_description_schema['shipping-phase'],
Optional('shipping-product'):
task_description_schema['shipping-product'],
Optional('priority'):
task_description_schema['priority'],
})
transforms = TransformSequence()
transforms.add(check_if_partners_enabled)
transforms.add_validate(beetmover_description_schema)
@transforms.add
def resolve_keys(config, jobs):
for job in jobs:
resolve_keyed_by(job,
'partner-bucket-scope',
item_name=job['label'],
**{'release-level': config.params.release_level()})
yield job
@transforms.add
from datetime import date, timedelta
import json
from six import ensure_text, text_type
from voluptuous import (
Any,
Optional,
Extra,
)
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
from taskgraph.util.treeherder import split_symbol, join_symbol
transforms = TransformSequence()
perftest_description_schema = Schema({
# The test names and the symbols to use for them: [test-symbol, test-path]
Optional('perftest'): [[text_type]],
# Metrics to gather for the test. These will be merged
# with options specified through perftest-perfherder-global
Optional('perftest-metrics'):
optionally_keyed_by(
'perftest',
Any([text_type], {
text_type:
Any(None, {text_type: Any(None, text_type, [text_type])})
})),
"""
from __future__ import absolute_import, print_function, unicode_literals
from six import text_type
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
generate_beetmover_upstream_artifacts,
get_beetmover_bucket_scope,
get_beetmover_action_scope)
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
transforms = TransformSequence()
release_generate_checksums_beetmover_schema = schema.extend({
# depname is used in taskref's to identify the taskID of the unsigned things
Required('depname', default='build'):
text_type,
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional('label'):
text_type,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional('treeherder'):
task_description_schema['treeherder'],
Transform the partner repack task into an actual task description.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import resolve_keyed_by
from taskgraph.util.scriptworker import get_release_config
from taskgraph.util.partners import (
check_if_partners_enabled,
get_partner_url_config,
apply_partner_priority,
)
transforms = TransformSequence()
transforms.add(apply_partner_priority)
@transforms.add
def populate_repack_manifests_url(config, tasks):
for task in tasks:
partner_url_config = get_partner_url_config(config.params, config.graph_config)
for k in partner_url_config:
if config.kind.startswith(k):
task['worker'].setdefault('env', {})['REPACK_MANIFESTS_URL'] = \
partner_url_config[k]
break
else:
raise Exception("Can't find partner REPACK_MANIFESTS_URL")
task_description_schema['attributes'],
Required('treeherder'):
task_description_schema['treeherder'],
Required('run-on-projects'):
task_description_schema['run-on-projects'],
Required('shipping-phase'):
task_description_schema['shipping-phase'],
Required('shipping-product'):
task_description_schema['shipping-product'],
Required('worker-type'):
task_description_schema['worker-type'],
Required('worker'):
object,
})
transforms = TransformSequence()
transforms.add_validate(google_play_description_schema)
@transforms.add
def set_label(config, jobs):
for job in jobs:
job['label'] = job['name']
yield job
@transforms.add
def set_worker_data(config, jobs):
for job in jobs:
worker = job['worker']
