from gecko_taskgraph.util.scriptworker import (
get_beetmover_bucket_scope,
get_beetmover_action_scope,
generate_beetmover_upstream_artifacts,
generate_beetmover_artifact_map,
)
from gecko_taskgraph.util.treeherder import inherit_treeherder_from_dep
from gecko_taskgraph.transforms.task import task_description_schema
from voluptuous import Required, Optional
import logging
import copy
logger = logging.getLogger(__name__)
transforms = TransformSequence()
beetmover_description_schema = schema.extend({
# attributes is used for enabling artifact-map by declarative artifacts
Required("attributes"): {
str: object
},
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional("label"):
str,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional("treeherder"):
task_description_schema["treeherder"],
Required("description"):
from collections import defaultdict
import json
import logging
from gecko_taskgraph.transforms.base import TransformSequence
from gecko_taskgraph.util.partners import (
apply_partner_priority,
check_if_partners_enabled,
get_partner_config_by_kind,
generate_attribution_code,
)
log = logging.getLogger(__name__)
transforms = TransformSequence()
transforms.add(check_if_partners_enabled)
transforms.add(apply_partner_priority)
@transforms.add
def add_command_arguments(config, tasks):
enabled_partners = config.params.get("release_partners")
dependencies = {}
fetches = defaultdict(set)
attributions = []
release_artifacts = []
attribution_config = get_partner_config_by_kind(config, config.kind)
for partner_config in attribution_config.get("configs", []):
# we might only be interested in a subset of all partners, eg for a respin
Optional("dependencies"): {
str: taskref_or_string
},
Optional("index"): {
str: str
},
Optional("routes"): [str],
Required("shipping-phase"):
task_description_schema["shipping-phase"],
Required("shipping-product"):
task_description_schema["shipping-product"],
Optional("extra"):
task_description_schema["extra"],
})
transforms = TransformSequence()
transforms.add_validate(beetmover_push_to_release_description_schema)
@transforms.add
def make_beetmover_push_to_release_description(config, jobs):
for job in jobs:
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "Rel(BM-C)")
treeherder.setdefault("tier", 1)
treeherder.setdefault("kind", "build")
treeherder.setdefault("platform", job["treeherder-platform"])
label = job["name"]
description = "Beetmover push to release for '{product}'".format(
product=job["product"])
from gecko_taskgraph.util.taskcluster import get_artifact_path
from gecko_taskgraph.transforms.task import task_description_schema
from voluptuous import Optional
release_generate_checksums_signing_schema = schema.extend({
Optional("label"):
str,
Optional("treeherder"):
task_description_schema["treeherder"],
Optional("shipping-product"):
task_description_schema["shipping-product"],
Optional("shipping-phase"):
task_description_schema["shipping-phase"],
})
transforms = TransformSequence()
transforms.add_validate(release_generate_checksums_signing_schema)
@transforms.add
def make_release_generate_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "SGenChcks")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault("tier", 1)
Required,
)
from .task import task_description_schema
logger = logging.getLogger(__name__)
CONTEXTS_DIR = "docker-contexts"
DIGEST_RE = re.compile("^[0-9a-f]{64}$")
IMAGE_BUILDER_IMAGE = (
"mozillareleases/image_builder:5.0.0"
"@sha256:"
"e510a9a9b80385f71c112d61b2f2053da625aff2b6d430411ac42e424c58953f")
transforms = TransformSequence()
docker_image_schema = Schema({
# Name of the docker image.
Required("name"):
str,
# Name of the parent docker image.
Optional("parent"):
str,
# Treeherder symbol.
Required("symbol"):
str,
# relative path (from config.path) to the file the docker image was defined
# in.
Optional("job-from"):
str,
optionally_keyed_by("release-type", bool),
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional("treeherder"):
task_description_schema["treeherder"],
Optional("attributes"):
task_description_schema["attributes"],
# Shipping product / phase
Optional("shipping-product"):
task_description_schema["shipping-product"],
Optional("shipping-phase"):
task_description_schema["shipping-phase"],
})
transforms = TransformSequence()
transforms.add_validate(balrog_description_schema)
@transforms.add
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by platform, etc."""
fields = [
"update-no-wnp",
]
for job in jobs:
label = job.get("dependent-task",
object).__dict__.get("label", "?no-label?")
for field in fields:
resolve_keyed_by(
item=job,
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Chunk the partner repack tasks by subpartner and locale
"""
import copy
from mozbuild.chunkify import chunkify
from gecko_taskgraph.transforms.base import TransformSequence
from gecko_taskgraph.util.partners import (
get_repack_ids_by_platform,
apply_partner_priority,
)
transforms = TransformSequence()
transforms.add(apply_partner_priority)
@transforms.add
def chunk_partners(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
build_platform = dep_job.attributes["build_platform"]
repack_id = dep_job.task.get("extra", {}).get("repack_id")
repack_ids = dep_job.task.get("extra", {}).get("repack_ids")
copy_repack_ids = job.pop("copy-repack-ids", False)
if copy_repack_ids:
assert repack_ids, "dep_job {} doesn't have repack_ids!".format(
dep_job.label
Required("job-from"): task_description_schema["job-from"],
Required("dependencies"): task_description_schema["dependencies"],
Required("description"): task_description_schema["description"],
Required("treeherder"): task_description_schema["treeherder"],
Required("run-on-projects"): task_description_schema["run-on-projects"],
Required("worker-type"): optionally_keyed_by("release-level", str),
Required("worker"): object,
Optional("scopes"): [str],
Required("shipping-phase"): task_description_schema["shipping-phase"],
Required("shipping-product"): task_description_schema["shipping-product"],
Optional("extra"): task_description_schema["extra"],
Optional("attributes"): task_description_schema["attributes"],
}
)
transforms = TransformSequence()
transforms.add_validate(push_flatpak_description_schema)
@transforms.add
def make_task_description(config, jobs):
for job in jobs:
if len(job["dependencies"]) != 1:
raise Exception("Exactly 1 dependency is required")
job["worker"]["upstream-artifacts"] = generate_upstream_artifacts(
job["dependencies"]
)
resolve_keyed_by(
job,
from gecko_taskgraph.util.scriptworker import get_signing_cert_scope
from gecko_taskgraph.transforms.task import task_description_schema
from voluptuous import Optional
checksums_signing_description_schema = schema.extend({
Optional("label"):
str,
Optional("treeherder"):
task_description_schema["treeherder"],
Optional("shipping-product"):
task_description_schema["shipping-product"],
Optional("shipping-phase"):
task_description_schema["shipping-phase"],
})
transforms = TransformSequence()
transforms.add_validate(checksums_signing_description_schema)
@transforms.add
def make_checksums_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = dep_job.attributes
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "css(N)")
dep_th_platform = (dep_job.task.get("extra", {}).get(
"treeherder", {}).get("machine", {}).get("platform", ""))
treeherder.setdefault("platform", f"{dep_th_platform}/opt")
treeherder.setdefault("tier", 1)
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from copy import deepcopy
from voluptuous import (
Optional,
Required,
Extra,
)
from gecko_taskgraph.transforms.base import TransformSequence
from gecko_taskgraph.transforms.tests import test_description_schema
from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
from gecko_taskgraph.util.treeherder import split_symbol, join_symbol
transforms = TransformSequence()
raptor_description_schema = Schema({
# Raptor specific configs.
Optional("apps"):
optionally_keyed_by("test-platform", "subtest", [str]),
Optional("raptor-test"):
str,
Optional("raptor-subtests"):
optionally_keyed_by("app", "test-platform", list),
Optional("activity"):
optionally_keyed_by("app", str),
Optional("binary-path"):
optionally_keyed_by("app", str),
# Configs defined in the 'test_description_schema'.
Optional("max-run-time"):
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the release-sign-and-push task into an actual task description.
"""
from gecko_taskgraph.loader.single_dep import schema
from gecko_taskgraph.transforms.base import TransformSequence
from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job
from gecko_taskgraph.util.schema import resolve_keyed_by, optionally_keyed_by
from gecko_taskgraph.util.treeherder import inherit_treeherder_from_dep
from gecko_taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required
transforms = TransformSequence()
langpack_sign_push_description_schema = schema.extend(
{
Required("label"): str,
Required("description"): str,
Required("worker-type"): optionally_keyed_by("release-level", str),
Required("worker"): {
Required("implementation"): "push-addons",
Required("channel"): optionally_keyed_by(
"project", "platform", Any("listed", "unlisted")
),
Required("upstream-artifacts"): None, # Processed here below
},
Required("run-on-projects"): [],
Required("scopes"): optionally_keyed_by("release-level", [str]),
optionally_keyed_by("release-level", str),
Required("partner-public-path"):
Any(None, str),
Required("partner-private-path"):
Any(None, str),
Optional("extra"):
object,
Required("shipping-phase"):
task_description_schema["shipping-phase"],
Optional("shipping-product"):
task_description_schema["shipping-product"],
Optional("priority"):
task_description_schema["priority"],
})
transforms = TransformSequence()
transforms.add_validate(beetmover_description_schema)
transforms.add(apply_partner_priority)
@transforms.add
def resolve_keys(config, jobs):
for job in jobs:
resolve_keyed_by(
job,
"partner-bucket-scope",
item_name=job["label"],
**{"release-level": config.params.release_level()},
)
yield job
"file_names":
CONFIG_PER_BOUNCER_PRODUCT_VANILLA["installer"]["file_names"],
},
"stub-installer": {
"name_postfix":
"-{partner}-{sub_config}-stub",
# We currently have a sole win32 stub installer that is to be used
# in all windows platforms to toggle between full installers
"path_template":
RELEASES_PARTNERS_PATH_TEMPLATE.replace("{ftp_platform}", "win32"),
"file_names":
CONFIG_PER_BOUNCER_PRODUCT_VANILLA["stub-installer"]["file_names"],
},
}
transforms = TransformSequence()
transforms.add(check_if_partners_enabled)
@transforms.add
def make_task_worker(config, jobs):
for job in jobs:
resolve_keyed_by(job,
"worker-type",
item_name=job["name"],
**{"release-level": config.params.release_level()})
resolve_keyed_by(job,
"scopes",
item_name=job["name"],
**{"release-level": config.params.release_level()})
resolve_keyed_by(job,
Optional("treeherder"):
task_description_schema["treeherder"],
Optional("shipping-product"):
task_description_schema["shipping-product"],
Optional("shipping-phase"):
task_description_schema["shipping-phase"],
})
SIGNING_FORMATS = {
"target.installer.exe": ["autograph_authenticode_stub"],
"target.stub-installer.exe": ["autograph_authenticode_stub"],
"target.installer.msi": ["autograph_authenticode"],
"target.installer.msix": ["autograph_authenticode_sha2"],
}
transforms = TransformSequence()
transforms.add_validate(repackage_signing_description_schema)
@transforms.add
def make_repackage_signing_description(config, jobs):
for job in jobs:
dep_job = job["primary-dependency"]
attributes = copy_attributes_from_dependent_job(dep_job)
locale = attributes.get("locale", dep_job.attributes.get("locale"))
attributes["repackage_type"] = "repackage-signing"
treeherder = job.get("treeherder", {})
treeherder.setdefault("symbol", "rs(B)")
dep_th_platform = dep_job.task.get("extra",
{}).get("treeherder-platform")
Required("worker"):
optionally_keyed_by("platform", job_description_schema["worker"]),
Optional("python-version"): [int],
Optional("dependencies"): {
k: optionally_keyed_by("platform", v)
for k, v in job_description_schema["dependencies"].items()
},
# A list of artifacts to install from 'fetch' tasks.
Optional("fetches"): {
str:
optionally_keyed_by("platform",
job_description_schema["fetches"][str]),
},
})
transforms = TransformSequence()
transforms.add_validate(source_test_description_schema)
@transforms.add
def set_job_name(config, jobs):
for job in jobs:
if "job-from" in job and job["job-from"] != "kind.yml":
from_name = os.path.splitext(job["job-from"])[0]
job["name"] = "{}-{}".format(from_name, job["name"])
yield job
@transforms.add
def expand_platforms(config, jobs):
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import json
from pipes import quote as shell_quote
from gecko_taskgraph.transforms.base import TransformSequence
from gecko_taskgraph.util.scriptworker import get_release_config
from gecko_taskgraph.util.schema import resolve_keyed_by
import logging
logger = logging.getLogger(__name__)
transforms = TransformSequence()
@transforms.add
def add_command(config, jobs):
for job in jobs:
command = [
"python",
"testing/mozharness/scripts/release/bouncer_check.py",
]
job["run"].update({
"using": "mach",
"mach": command,
})
yield job
"""
from gecko_taskgraph.loader.single_dep import schema
from gecko_taskgraph.transforms.base import TransformSequence
from gecko_taskgraph.util.attributes import copy_attributes_from_dependent_job
from gecko_taskgraph.util.scriptworker import (
generate_beetmover_artifact_map,
generate_beetmover_upstream_artifacts,
get_beetmover_bucket_scope,
get_beetmover_action_scope,
)
from gecko_taskgraph.transforms.beetmover import craft_release_properties
from gecko_taskgraph.transforms.task import task_description_schema
from voluptuous import Optional
transforms = TransformSequence()
release_generate_checksums_beetmover_schema = schema.extend({
# unique label to describe this beetmover task, defaults to {dep.label}-beetmover
Optional("label"):
str,
# treeherder is allowed here to override any defaults we use for beetmover. See
# taskcluster/gecko_taskgraph/transforms/task.py for the schema details, and the
# below transforms for defaults of various values.
Optional("treeherder"):
task_description_schema["treeherder"],
Optional("shipping-phase"):
task_description_schema["shipping-phase"],
Optional("shipping-product"):
task_description_schema["shipping-product"],
Optional("attributes"):