def craft_bouncer_entries(config, job):
release_config = get_release_config(config)
product = job["shipping-product"]
current_version = release_config["version"]
bouncer_products = job["bouncer-products"]
partners = get_partners_to_be_published(config)
entries = {}
for partner, sub_config_name, platforms in partners:
platforms = [PARTNER_PLATFORMS_TO_BOUNCER[p] for p in platforms]
entries.update({
craft_partner_bouncer_product_name(product, bouncer_product,
current_version, partner,
sub_config_name):
{
"options": {
"add_locales":
False, # partners may use different sets of locales
"ssl_only": craft_ssl_only(bouncer_product),
},
"paths_per_bouncer_platform":
craft_paths_per_bouncer_platform(
product,
bouncer_product,
platforms,
current_version,
partner,
sub_config_name,
),
}
for bouncer_product in bouncer_products
})
return entries
def format(config, tasks):
"""Apply format substitution to worker.env and worker.command."""
format_params = {
"release_config": get_release_config(config),
"config_params": config.params,
}
for task in tasks:
format_params["task"] = task
command = task.get("worker", {}).get("command", [])
task["worker"]["command"] = [x.format(**format_params) for x in command]
env = task.get("worker", {}).get("env", {})
for k in env.keys():
resolve_keyed_by(
env,
k,
"flatpak envs",
**{
"release-level": config.params.release_level(),
"project": config.params["project"],
}
)
task["worker"]["env"][k] = env[k].format(**format_params)
yield task
def interpolate(config, jobs):
release_config = get_release_config(config)
for job in jobs:
mh_options = list(job["run"]["options"])
job["run"]["options"] = [
option.format(
version=release_config["version"],
build_number=release_config["build_number"],
) for option in mh_options
]
yield job
def craft_bouncer_entries(config, job):
release_config = get_release_config(config)
product = job["shipping-product"]
bouncer_platforms = job["bouncer-platforms"]
current_version = release_config["version"]
current_build_number = release_config["build_number"]
bouncer_products = job["bouncer-products"]
previous_versions_string = release_config.get("partial_versions", None)
if previous_versions_string:
previous_versions = previous_versions_string.split(", ")
else:
logger.warn(
'No partials defined! Bouncer submission task won\'t send any \
partial-related entry for "{}"'.format(
job["name"]
)
)
bouncer_products = [
bouncer_product
for bouncer_product in bouncer_products
if "partial" not in bouncer_product
]
previous_versions = [None]
project = config.params["project"]
return {
craft_bouncer_product_name(
product,
bouncer_product,
current_version,
current_build_number,
previous_version,
): {
"options": {
"add_locales": False if "msix" in bouncer_product else True,
"ssl_only": craft_ssl_only(bouncer_product, project),
},
"paths_per_bouncer_platform": craft_paths_per_bouncer_platform(
product,
bouncer_product,
bouncer_platforms,
current_version,
current_build_number,
previous_version,
),
}
for bouncer_product in bouncer_products
for previous_version in previous_versions
}
def add_previous_versions(config, jobs):
release_config = get_release_config(config)
if not release_config.get("partial_versions"):
for job in jobs:
yield job
else:
extra_params = []
for partial in release_config["partial_versions"].split(","):
extra_params.append("--previous-version={}".format(
partial.split("build")[0].strip()))
for job in jobs:
job["run"]["mach"].extend(extra_params)
yield job
def handle_keyed_by(config, jobs):
"""Resolve fields that can be keyed by project, etc."""
fields = [
"run.config",
"run.product-field",
"run.extra-config",
]
release_config = get_release_config(config)
version = release_config["version"]
for job in jobs:
for field in fields:
resolve_keyed_by(
item=job,
field=field,
item_name=job["name"],
**{
"project": config.params["project"],
"release-level": config.params.release_level(),
"release-type": config.params["release_type"],
},
)
for cfg in job["run"]["config"]:
job["run"]["mach"].extend(["--config", cfg])
if config.kind == "cron-bouncer-check":
job["run"]["mach"].extend([
"--product-field={}".format(job["run"]["product-field"]),
"--products-url={}".format(job["run"]["products-url"]),
])
del job["run"]["product-field"]
del job["run"]["products-url"]
elif config.kind == "release-bouncer-check":
job["run"]["mach"].append(f"--version={version}")
del job["run"]["config"]
if "extra-config" in job["run"]:
env = job["worker"].setdefault("env", {})
env["EXTRA_MOZHARNESS_CONFIG"] = json.dumps(
job["run"]["extra-config"], sort_keys=True)
del job["run"]["extra-config"]
yield job
def add_command_arguments(config, tasks):
release_config = get_release_config(config)
# staging releases - pass reduced set of locales to the repacking script
all_locales = set()
partner_config = get_partner_config_by_kind(config, config.kind)
for partner in partner_config.values():
for sub_partner in partner.values():
all_locales.update(sub_partner.get("locales", []))
for task in tasks:
# add the MOZHARNESS_OPTIONS, eg version=61.0, build-number=1, platform=win64
if not task["attributes"]["build_platform"].endswith("-shippable"):
raise Exception(
"Unexpected partner repack platform: {}".format(
task["attributes"]["build_platform"], ), )
platform = task["attributes"]["build_platform"].partition(
"-shippable")[0]
task["run"]["options"] = [
"version={}".format(release_config["version"]),
"build-number={}".format(release_config["build_number"]),
f"platform={platform}",
]
if task["extra"]["limit-locales"]:
for locale in all_locales:
task["run"]["options"].append(f"limit-locale={locale}")
if "partner" in config.kind and config.params["release_partners"]:
for partner in config.params["release_partners"]:
task["run"]["options"].append(f"partner={partner}")
# The upstream taskIds are stored a special environment variable, because we want to use
# task-reference's to resolve dependencies, but the string handling of MOZHARNESS_OPTIONS
# blocks that. It's space-separated string of ids in the end.
task["worker"]["env"]["UPSTREAM_TASKIDS"] = {
"task-reference":
" ".join([f"<{dep}>" for dep in task["dependencies"]])
}
# Forward the release type for bouncer product construction
task["worker"]["env"]["RELEASE_TYPE"] = config.params["release_type"]
yield task
def add_notifications(config, jobs):
release_config = get_release_config(config)
for job in jobs:
label = "{}-{}".format(config.kind, job["name"])
notifications = job.pop("notifications", None)
if notifications:
resolve_keyed_by(notifications,
"emails",
label,
project=config.params["project"])
emails = notifications["emails"]
format_kwargs = dict(
task=job,
config=config.__dict__,
release_config=release_config,
)
subject = titleformatter.format(notifications["subject"],
**format_kwargs)
message = titleformatter.format(notifications["message"],
**format_kwargs)
emails = [email.format(**format_kwargs) for email in emails]
# By default, we only send mail on success to avoid messages like 'blah is in the
# candidates dir' when cancelling graphs, dummy job failure, etc
status_types = notifications.get("status-types", ["on-completed"])
for s in status_types:
job.setdefault("routes", []).extend(
[f"notify.email.{email}.{s}" for email in emails])
# Customize the email subject to include release name and build number
job.setdefault("extra", {}).update(
{"notify": {
"email": {
"subject": subject,
}
}})
if message:
job["extra"]["notify"]["email"]["content"] = message
yield job
def make_task_description(config, jobs):
release_config = get_release_config(config)
for job in jobs:
resolve_keyed_by(job,
"worker-type",
item_name=job["name"],
**{"release-level": config.params.release_level()})
resolve_keyed_by(job,
"scopes",
item_name=job["name"],
**{"release-level": config.params.release_level()})
job["worker"][
"release-name"] = "{product}-{version}-build{build_number}".format(
product=job["shipping-product"].capitalize(),
version=release_config["version"],
build_number=release_config["build_number"],
)
yield job
def craft_bouncer_entries(config, job):
release_config = get_release_config(config)
product = job["shipping-product"]
current_version = release_config["version"]
bouncer_products_per_alias = job["bouncer-products-per-alias"]
entries = {
bouncer_alias: craft_bouncer_product_name(
product,
bouncer_product,
current_version,
)
for bouncer_alias, bouncer_product in bouncer_products_per_alias.items()
}
partner_bouncer_products_per_alias = job.get("partner-bouncer-products-per-alias")
if partner_bouncer_products_per_alias:
partners = get_partners_to_be_published(config)
for partner, sub_config_name, _ in partners:
entries.update(
{
bouncer_alias.replace(
"PARTNER", f"{partner}-{sub_config_name}"
): craft_partner_bouncer_product_name(
product,
bouncer_product,
current_version,
partner,
sub_config_name,
)
for bouncer_alias, bouncer_product in partner_bouncer_products_per_alias.items() # NOQA: E501
}
)
return entries
def generate_update_line(config, jobs):
"""Resolve fields that can be keyed by platform, etc."""
release_config = get_release_config(config)
for job in jobs:
config_file = job.pop("whats-new-config")
update_config = load_yaml(config_file)
product = job["shipping-product"]
if product == "devedition":
product = "firefox"
job["worker"]["update-line"] = {}
for blob_type, suffix in [("wnp", ""), ("no-wnp", "-No-WNP")]:
context = {
"release-type": config.params["release_type"],
"product": product,
"version": GeckoVersion.parse(release_config["appVersion"]),
"blob-type": blob_type,
"build-id": config.params["moz_build_date"],
}
job["worker"]["update-line"][suffix] = generate_update_properties(
context, update_config
)
yield job
def add_command(config, tasks):
keyed_by_args = [
"channel",
"archive-prefix",
"previous-archive-prefix",
"aus-server",
"override-certs",
"include-version",
"mar-channel-id-override",
"last-watershed",
]
optional_args = [
"updater-platform",
]
release_config = get_release_config(config)
for task in tasks:
task["description"] = "generate update verify config for {}".format(
task["attributes"]["build_platform"]
)
command = [
"python",
"testing/mozharness/scripts/release/update-verify-config-creator.py",
"--product",
task["extra"]["product"],
"--stage-product",
task["shipping-product"],
"--app-name",
task["extra"]["app-name"],
"--branch-prefix",
task["extra"]["branch-prefix"],
"--platform",
task["extra"]["platform"],
"--to-version",
release_config["version"],
"--to-app-version",
release_config["appVersion"],
"--to-build-number",
str(release_config["build_number"]),
"--to-buildid",
config.params["moz_build_date"],
"--to-revision",
get_branch_rev(config),
"--output-file",
"update-verify.cfg",
]
repo_path = urlsplit(get_branch_repo(config)).path.lstrip("/")
command.extend(["--repo-path", repo_path])
if release_config.get("partial_versions"):
for partial in release_config["partial_versions"].split(","):
command.extend(["--partial-version", partial.split("build")[0]])
for arg in optional_args:
if task["extra"].get(arg):
command.append(f"--{arg}")
command.append(task["extra"][arg])
for arg in keyed_by_args:
thing = f"extra.{arg}"
resolve_keyed_by(
task,
thing,
item_name=task["name"],
platform=task["attributes"]["build_platform"],
**{
"release-type": config.params["release_type"],
"release-level": config.params.release_level(),
},
)
# ignore things that resolved to null
if not task["extra"].get(arg):
continue
if arg == "include-version":
task["extra"][arg] = INCLUDE_VERSION_REGEXES[task["extra"][arg]]
if arg == "mar-channel-id-override":
task["extra"][arg] = MAR_CHANNEL_ID_OVERRIDE_REGEXES[task["extra"][arg]]
command.append(f"--{arg}")
command.append(task["extra"][arg])
task["run"].update(
{
"using": "mach",
"mach": " ".join(command),
}
)
yield task
