from gecko_taskgraph.util.scriptworker import ( get_beetmover_bucket_scope, get_beetmover_action_scope, generate_beetmover_upstream_artifacts, generate_beetmover_artifact_map, ) from gecko_taskgraph.util.treeherder import inherit_treeherder_from_dep from gecko_taskgraph.transforms.task import task_description_schema from voluptuous import Required, Optional import logging import copy logger = logging.getLogger(__name__) transforms = TransformSequence() beetmover_description_schema = schema.extend({ # attributes is used for enabling artifact-map by declarative artifacts Required("attributes"): { str: object }, # unique label to describe this beetmover task, defaults to {dep.label}-beetmover Optional("label"): str, # treeherder is allowed here to override any defaults we use for beetmover. See # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the # below transforms for defaults of various values. Optional("treeherder"): task_description_schema["treeherder"], Required("description"):
from collections import defaultdict import json import logging from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.partners import ( apply_partner_priority, check_if_partners_enabled, get_partner_config_by_kind, generate_attribution_code, ) log = logging.getLogger(__name__) transforms = TransformSequence() transforms.add(check_if_partners_enabled) transforms.add(apply_partner_priority) @transforms.add def add_command_arguments(config, tasks): enabled_partners = config.params.get("release_partners") dependencies = {} fetches = defaultdict(set) attributions = [] release_artifacts = [] attribution_config = get_partner_config_by_kind(config, config.kind) for partner_config in attribution_config.get("configs", []): # we might only be interested in a subset of all partners, eg for a respin
Optional("dependencies"): { str: taskref_or_string }, Optional("index"): { str: str }, Optional("routes"): [str], Required("shipping-phase"): task_description_schema["shipping-phase"], Required("shipping-product"): task_description_schema["shipping-product"], Optional("extra"): task_description_schema["extra"], }) transforms = TransformSequence() transforms.add_validate(beetmover_push_to_release_description_schema) @transforms.add def make_beetmover_push_to_release_description(config, jobs): for job in jobs: treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "Rel(BM-C)") treeherder.setdefault("tier", 1) treeherder.setdefault("kind", "build") treeherder.setdefault("platform", job["treeherder-platform"]) label = job["name"] description = "Beetmover push to release for '{product}'".format( product=job["product"])
from gecko_taskgraph.util.taskcluster import get_artifact_path from gecko_taskgraph.transforms.task import task_description_schema from voluptuous import Optional release_generate_checksums_signing_schema = schema.extend({ Optional("label"): str, Optional("treeherder"): task_description_schema["treeherder"], Optional("shipping-product"): task_description_schema["shipping-product"], Optional("shipping-phase"): task_description_schema["shipping-phase"], }) transforms = TransformSequence() transforms.add_validate(release_generate_checksums_signing_schema) @transforms.add def make_release_generate_checksums_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "SGenChcks") dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) treeherder.setdefault("platform", f"{dep_th_platform}/opt") treeherder.setdefault("tier", 1)
Required, ) from .task import task_description_schema logger = logging.getLogger(__name__) CONTEXTS_DIR = "docker-contexts" DIGEST_RE = re.compile("^[0-9a-f]{64}$") IMAGE_BUILDER_IMAGE = ( "mozillareleases/image_builder:5.0.0" "@sha256:" "e510a9a9b80385f71c112d61b2f2053da625aff2b6d430411ac42e424c58953f") transforms = TransformSequence() docker_image_schema = Schema({ # Name of the docker image. Required("name"): str, # Name of the parent docker image. Optional("parent"): str, # Treeherder symbol. Required("symbol"): str, # relative path (from config.path) to the file the docker image was defined # in. Optional("job-from"): str,
optionally_keyed_by("release-type", bool), # treeherder is allowed here to override any defaults we use for beetmover. See # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the # below transforms for defaults of various values. Optional("treeherder"): task_description_schema["treeherder"], Optional("attributes"): task_description_schema["attributes"], # Shipping product / phase Optional("shipping-product"): task_description_schema["shipping-product"], Optional("shipping-phase"): task_description_schema["shipping-phase"], }) transforms = TransformSequence() transforms.add_validate(balrog_description_schema) @transforms.add def handle_keyed_by(config, jobs): """Resolve fields that can be keyed by platform, etc.""" fields = [ "update-no-wnp", ] for job in jobs: label = job.get("dependent-task", object).__dict__.get("label", "?no-label?") for field in fields: resolve_keyed_by( item=job,
# file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Chunk the partner repack tasks by subpartner and locale """ import copy from mozbuild.chunkify import chunkify from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.partners import ( get_repack_ids_by_platform, apply_partner_priority, ) transforms = TransformSequence() transforms.add(apply_partner_priority) @transforms.add def chunk_partners(config, jobs): for job in jobs: dep_job = job["primary-dependency"] build_platform = dep_job.attributes["build_platform"] repack_id = dep_job.task.get("extra", {}).get("repack_id") repack_ids = dep_job.task.get("extra", {}).get("repack_ids") copy_repack_ids = job.pop("copy-repack-ids", False) if copy_repack_ids: assert repack_ids, "dep_job {} doesn't have repack_ids!".format( dep_job.label
Required("job-from"): task_description_schema["job-from"], Required("dependencies"): task_description_schema["dependencies"], Required("description"): task_description_schema["description"], Required("treeherder"): task_description_schema["treeherder"], Required("run-on-projects"): task_description_schema["run-on-projects"], Required("worker-type"): optionally_keyed_by("release-level", str), Required("worker"): object, Optional("scopes"): [str], Required("shipping-phase"): task_description_schema["shipping-phase"], Required("shipping-product"): task_description_schema["shipping-product"], Optional("extra"): task_description_schema["extra"], Optional("attributes"): task_description_schema["attributes"], } ) transforms = TransformSequence() transforms.add_validate(push_flatpak_description_schema) @transforms.add def make_task_description(config, jobs): for job in jobs: if len(job["dependencies"]) != 1: raise Exception("Exactly 1 dependency is required") job["worker"]["upstream-artifacts"] = generate_upstream_artifacts( job["dependencies"] ) resolve_keyed_by( job,
from gecko_taskgraph.util.scriptworker import get_signing_cert_scope from gecko_taskgraph.transforms.task import task_description_schema from voluptuous import Optional checksums_signing_description_schema = schema.extend({ Optional("label"): str, Optional("treeherder"): task_description_schema["treeherder"], Optional("shipping-product"): task_description_schema["shipping-product"], Optional("shipping-phase"): task_description_schema["shipping-phase"], }) transforms = TransformSequence() transforms.add_validate(checksums_signing_description_schema) @transforms.add def make_checksums_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = dep_job.attributes treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "css(N)") dep_th_platform = (dep_job.task.get("extra", {}).get( "treeherder", {}).get("machine", {}).get("platform", "")) treeherder.setdefault("platform", f"{dep_th_platform}/opt") treeherder.setdefault("tier", 1)
# file, You can obtain one at http://mozilla.org/MPL/2.0/. from copy import deepcopy from voluptuous import ( Optional, Required, Extra, ) from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.transforms.tests import test_description_schema from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema from gecko_taskgraph.util.treeherder import split_symbol, join_symbol transforms = TransformSequence() raptor_description_schema = Schema({ # Raptor specific configs. Optional("apps"): optionally_keyed_by("test-platform", "subtest", [str]), Optional("raptor-test"): str, Optional("raptor-subtests"): optionally_keyed_by("app", "test-platform", list), Optional("activity"): optionally_keyed_by("app", str), Optional("binary-path"): optionally_keyed_by("app", str), # Configs defined in the 'test_description_schema'. Optional("max-run-time"):
# License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """ Transform the release-sign-and-push task into an actual task description. """ from gecko_taskgraph.loader.single_dep import schema from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job from gecko_taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by from gecko_taskgraph.util.treeherder import inherit_treeherder_from_dep from gecko_taskgraph.transforms.task import task_description_schema from voluptuous import Any, Required transforms = TransformSequence() langpack_sign_push_description_schema = schema.extend( { Required("label"): str, Required("description"): str, Required("worker-type"): optionally_keyed_by("release-level", str), Required("worker"): { Required("implementation"): "push-addons", Required("channel"): optionally_keyed_by( "project", "platform", Any("listed", "unlisted") ), Required("upstream-artifacts"): None, # Processed here below }, Required("run-on-projects"): [], Required("scopes"): optionally_keyed_by("release-level", [str]),
optionally_keyed_by("release-level", str), Required("partner-public-path"): Any(None, str), Required("partner-private-path"): Any(None, str), Optional("extra"): object, Required("shipping-phase"): task_description_schema["shipping-phase"], Optional("shipping-product"): task_description_schema["shipping-product"], Optional("priority"): task_description_schema["priority"], }) transforms = TransformSequence() transforms.add_validate(beetmover_description_schema) transforms.add(apply_partner_priority) @transforms.add def resolve_keys(config, jobs): for job in jobs: resolve_keyed_by( job, "partner-bucket-scope", item_name=job["label"], **{"release-level": config.params.release_level()}, ) yield job
"file_names": CONFIG_PER_BOUNCER_PRODUCT_VANILLA["installer"]["file_names"], }, "stub-installer": { "name_postfix": "-{partner}-{sub_config}-stub", # We currently have a sole win32 stub installer that is to be used # in all windows platforms to toggle between full installers "path_template": RELEASES_PARTNERS_PATH_TEMPLATE.replace("{ftp_platform}", "win32"), "file_names": CONFIG_PER_BOUNCER_PRODUCT_VANILLA["stub-installer"]["file_names"], }, } transforms = TransformSequence() transforms.add(check_if_partners_enabled) @transforms.add def make_task_worker(config, jobs): for job in jobs: resolve_keyed_by(job, "worker-type", item_name=job["name"], **{"release-level": config.params.release_level()}) resolve_keyed_by(job, "scopes", item_name=job["name"], **{"release-level": config.params.release_level()}) resolve_keyed_by(job,
Optional("treeherder"): task_description_schema["treeherder"], Optional("shipping-product"): task_description_schema["shipping-product"], Optional("shipping-phase"): task_description_schema["shipping-phase"], }) SIGNING_FORMATS = { "target.installer.exe": ["autograph_authenticode_stub"], "target.stub-installer.exe": ["autograph_authenticode_stub"], "target.installer.msi": ["autograph_authenticode"], "target.installer.msix": ["autograph_authenticode_sha2"], } transforms = TransformSequence() transforms.add_validate(repackage_signing_description_schema) @transforms.add def make_repackage_signing_description(config, jobs): for job in jobs: dep_job = job["primary-dependency"] attributes = copy_attributes_from_dependent_job(dep_job) locale = attributes.get("locale", dep_job.attributes.get("locale")) attributes["repackage_type"] = "repackage-signing" treeherder = job.get("treeherder", {}) treeherder.setdefault("symbol", "rs(B)") dep_th_platform = dep_job.task.get("extra", {}).get("treeherder-platform")
Required("worker"): optionally_keyed_by("platform", job_description_schema["worker"]), Optional("python-version"): [int], Optional("dependencies"): { k: optionally_keyed_by("platform", v) for k, v in job_description_schema["dependencies"].items() }, # A list of artifacts to install from 'fetch' tasks. Optional("fetches"): { str: optionally_keyed_by("platform", job_description_schema["fetches"][str]), }, }) transforms = TransformSequence() transforms.add_validate(source_test_description_schema) @transforms.add def set_job_name(config, jobs): for job in jobs: if "job-from" in job and job["job-from"] != "kind.yml": from_name = os.path.splitext(job["job-from"])[0] job["name"] = "{}-{}".format(from_name, job["name"]) yield job @transforms.add def expand_platforms(config, jobs):
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import json from pipes import quote as shell_quote from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.scriptworker import get_release_config from gecko_taskgraph.util.schema import resolve_keyed_by import logging logger = logging.getLogger(__name__) transforms = TransformSequence() @transforms.add def add_command(config, jobs): for job in jobs: command = [ "python", "testing/mozharness/scripts/release/bouncer_check.py", ] job["run"].update({ "using": "mach", "mach": command, }) yield job
""" from gecko_taskgraph.loader.single_dep import schema from gecko_taskgraph.transforms.base import TransformSequence from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job from gecko_taskgraph.util.scriptworker import ( generate_beetmover_artifact_map, generate_beetmover_upstream_artifacts, get_beetmover_bucket_scope, get_beetmover_action_scope, ) from gecko_taskgraph.transforms.beetmover import craft_release_properties from gecko_taskgraph.transforms.task import task_description_schema from voluptuous import Optional transforms = TransformSequence() release_generate_checksums_beetmover_schema = schema.extend({ # unique label to describe this beetmover task, defaults to {dep.label}-beetmover Optional("label"): str, # treeherder is allowed here to override any defaults we use for beetmover. See # taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the # below transforms for defaults of various values. Optional("treeherder"): task_description_schema["treeherder"], Optional("shipping-phase"): task_description_schema["shipping-phase"], Optional("shipping-product"): task_description_schema["shipping-product"], Optional("attributes"):