async def build_release_image(self, release_name: str, arch: str, previous_list: List[str], metadata: Optional[Dict], dest_image_pullspec: str, reference_release: Optional[str]):
go_arch_suffix = util.go_suffix_for_arch(arch, is_private=False)
cmd = [
"oc",
"adm",
"release",
"new",
"-n",
f"ocp{go_arch_suffix}",
f"--name={release_name}",
f"--to-image={dest_image_pullspec}",
]
if self.runtime.dry_run:
cmd.append("--dry-run")
if reference_release:
cmd.append(f"--from-release=registry.ci.openshift.org/ocp{go_arch_suffix}/release{go_arch_suffix}:{reference_release}")
else:
major, minor = util.isolate_major_minor_in_group(self.group)
src_image_stream = f"{major}.{minor}-art-assembly-{self.assembly}{go_arch_suffix}"
cmd.extend(["--reference-mode=source", f"--from-image-stream={src_image_stream}"])
if previous_list:
cmd.append(f"--previous={','.join(previous_list)}")
if metadata:
cmd.append("--metadata")
cmd.append(json.dumps(metadata))
env = os.environ.copy()
env["GOTRACEBACK"] = "all"
await exectools.cmd_assert_async(cmd, env=env, stdout=sys.stderr)
async def run(self):
logger = self.runtime.logger
# Load group config and releases.yml
logger.info("Loading build data...")
group_config = await util.load_group_config(self.group, self.assembly, env=self._doozer_env_vars)
releases_config = await util.load_releases_config(self._doozer_working_dir / "ocp-build-data")
if releases_config.get("releases", {}).get(self.assembly) is None:
raise ValueError(f"To promote this release, assembly {self.assembly} must be explictly defined in releases.yml.")
permits = util.get_assembly_promotion_permits(releases_config, self.assembly)
# Get release name
assembly_type = util.get_assembly_type(releases_config, self.assembly)
release_name = util.get_release_name(assembly_type, self.group, self.assembly, self.release_offset)
# Ensure release name is valid
if not VersionInfo.isvalid(release_name):
raise ValueError(f"Release name `{release_name}` is not a valid semver.")
logger.info("Release name: %s", release_name)
self._slack_client.bind_channel(release_name)
slack_response = await self._slack_client.say(f"Promoting release `{release_name}` @release-artists")
slack_thread = slack_response["message"]["ts"]
justifications = []
try:
# Get arches
arches = self.arches or group_config.get("arches", [])
arches = list(set(map(brew_arch_for_go_arch, arches)))
if not arches:
raise ValueError("No arches specified.")
# Get previous list
upgrades_str: Optional[str] = group_config.get("upgrades")
if upgrades_str is None and assembly_type != assembly.AssemblyTypes.CUSTOM:
raise ValueError(f"Group config for assembly {self.assembly} is missing the required `upgrades` field. If no upgrade edges are expected, please explicitly set the `upgrades` field to empty string.")
previous_list = list(map(lambda s: s.strip(), upgrades_str.split(","))) if upgrades_str else []
# Ensure all versions in previous list are valid semvers.
if any(map(lambda version: not VersionInfo.isvalid(version), previous_list)):
raise ValueError("Previous list (`upgrades` field in group config) has an invalid semver.")
# Check for blocker bugs
if self.skip_blocker_bug_check or assembly_type in [assembly.AssemblyTypes.CANDIDATE, assembly.AssemblyTypes.CUSTOM]:
logger.info("Blocker Bug check is skipped.")
else:
logger.info("Checking for blocker bugs...")
# TODO: Needs an option in releases.yml to skip this check
try:
await self.check_blocker_bugs()
except VerificationError as err:
logger.warn("Blocker bugs found for release: %s", err)
justification = self._reraise_if_not_permitted(err, "BLOCKER_BUGS", permits)
justifications.append(justification)
logger.info("No blocker bugs found.")
# If there are CVEs, convert RHBAs to RHSAs and attach CVE flaw bugs
impetus_advisories = group_config.get("advisories", {})
futures = []
for impetus, advisory in impetus_advisories.items():
if not advisory:
continue
if advisory < 0: # placeholder advisory id is still in group config?
raise ValueError("Found invalid %s advisory %s", impetus, advisory)
logger.info("Attaching CVE flaws for %s advisory %s...", impetus, advisory)
futures.append(self.attach_cve_flaws(advisory))
try:
await asyncio.gather(*futures)
except ChildProcessError as err:
logger.warn("Error attaching CVE flaw bugs: %s", err)
justification = self._reraise_if_not_permitted(err, "CVE_FLAWS", permits)
justifications.append(justification)
# Attempt to move all advisories to QE
futures = []
for impetus, advisory in impetus_advisories.items():
if not advisory:
continue
logger.info("Moving advisory %s to QE...", advisory)
futures.append(self.change_advisory_state(advisory, "QE"))
try:
await asyncio.gather(*futures)
except ChildProcessError as err:
logger.warn("Error moving advisory %s to QE: %s", advisory, err)
# Ensure the image advisory is in QE (or later) state.
image_advisory = impetus_advisories.get("image", 0)
errata_url = ""
if assembly_type == assembly.AssemblyTypes.STANDARD:
if image_advisory <= 0:
err = VerificationError(f"No associated image advisory for {self.assembly} is defined.")
justification = self._reraise_if_not_permitted(err, "NO_ERRATA", permits)
justifications.append(justification)
else:
logger.info("Verifying associated image advisory %s...", image_advisory)
image_advisory_info = await self.get_advisory_info(image_advisory)
try:
self.verify_image_advisory(image_advisory_info)
live_id = self.get_live_id(image_advisory_info)
assert live_id
errata_url = f"https://access.redhat.com/errata/{live_id}" # don't quote
except VerificationError as err:
logger.warn("%s", err)
justification = self._reraise_if_not_permitted(err, "INVALID_ERRATA_STATUS", permits)
justifications.append(justification)
# Verify attached bugs
advisories = list(filter(lambda ad: ad > 0, impetus_advisories.values()))
if advisories:
if self.skip_attached_bug_check:
logger.info("Skip checking attached bugs.")
else:
logger.info("Verifying attached bugs...")
try:
await self.verify_attached_bugs(advisories)
except ChildProcessError as err:
logger.warn("Error verifying attached bugs: %s", err)
justification = self._reraise_if_not_permitted(err, "ATTACHED_BUGS", permits)
justifications.append(justification)
# Promote release images
futures = []
metadata = {}
description = group_config.get("description")
if description:
logger.warning("The following description message will be included in the metadata of release image: %s", description)
metadata["description"] = str(description)
if errata_url:
metadata["url"] = errata_url
reference_releases = util.get_assmebly_basis(releases_config, self.assembly).get("reference_releases", {})
tag_stable = assembly_type in [assembly.AssemblyTypes.STANDARD, assembly.AssemblyTypes.CANDIDATE]
release_infos = await self.promote_all_arches(release_name, arches, previous_list, metadata, reference_releases, tag_stable)
self._logger.info("All release images for %s have been promoted.", release_name)
# Wait for release controllers
pullspecs = list(map(lambda r: r["image"], release_infos))
if not tag_stable:
self._logger.warning("This release will not appear on release controllers. Pullspecs: %s", release_name, ", ".join(pullspecs))
await self._slack_client.say(f"Release {release_name} is ready. It will not appear on the release controllers. Please tell the user to manually pull the release images: {', '.join(pullspecs)}", slack_thread)
else: # Wait for release images to be accepted by the release controllers
self._logger.info("All release images for %s have been successfully promoted. Pullspecs: %s", release_name, ", ".join(pullspecs))
# check if release is already accepted (in case we timeout and run the job again)
accepted = []
for arch in arches:
go_arch_suffix = util.go_suffix_for_arch(arch)
release_stream = f"4-stable{go_arch_suffix}"
accepted.append(await self.is_accepted(release_name, arch, release_stream))
if not all(accepted):
self._logger.info("Determining upgrade tests...")
test_commands = self._get_upgrade_tests_commands(release_name, previous_list)
message = f"""A new release `{release_name}` is ready and needs some upgrade tests to be triggered.
Please open a chat with @cluster-bot and issue each of these lines individually:
{os.linesep.join(test_commands)}
"""
await self._slack_client.say(message, slack_thread)
self._logger.info("Waiting for release images for %s to be accepted by the release controller...", release_name)
futures = []
for arch in arches:
go_arch_suffix = util.go_suffix_for_arch(arch)
release_stream = f"4-stable{go_arch_suffix}"
futures.append(self.wait_for_stable(release_name, arch, release_stream))
try:
await asyncio.gather(*futures)
except RetryError as err:
message = f"Timeout waiting for release to be accepted by the release controllers: {err}"
self._logger.error(message)
self._logger.exception(err)
raise TimeoutError(message)
self._logger.info("All release images for %s have been accepted by the release controllers.", release_name)
message = f"Release `{release_name}` has been accepted by the release controllers."
await self._slack_client.say(message, slack_thread)
# Send image list
if not image_advisory:
self._logger.warning("No need to send an advisory image list because this release doesn't have an image advisory.")
elif assembly_type == assembly.AssemblyTypes.CANDIDATE:
self._logger.warning("No need to send an advisory image list for a candidate release.")
elif self.skip_image_list:
self._logger.warning("Skip sending advisory image list")
else:
self._logger.info("Gathering and sending advisory image list...")
mail_dir = self._working_dir / "email"
await self.send_image_list_email(release_name, image_advisory, mail_dir)
self._logger.info("Advisory image list sent.")
except Exception as err:
self._logger.exception(err)
error_message = f"Error promoting release {release_name}: {err}\n {traceback.format_exc()}"
message = f"Promoting release {release_name} failed with: {error_message}"
await self._slack_client.say(message, slack_thread)
raise
# Print release infos to console
data = {
"group": self.group,
"assembly": self.assembly,
"type": assembly_type.value,
"name": release_name,
"content": {},
"justifications": justifications,
}
if image_advisory > 0:
data["advisory"] = image_advisory
if errata_url:
data["live_url"] = errata_url
for arch, release_info in zip(arches, release_infos):
data["content"][arch] = {
"pullspec": release_info["image"],
"digest": release_info["digest"],
}
from_release = release_info.get("references", {}).get("metadata", {}).get("annotations", {}).get("release.openshift.io/from-release")
if from_release:
data["content"][arch]["from_release"] = from_release
rhcos = next((t for t in release_info.get("references", {}).get("spec", {}).get("tags", []) if t["name"] == "machine-os-content"), None)
if rhcos:
rhcos_version = rhcos["annotations"]["io.openshift.build.versions"].split("=")[1] # machine-os=48.84.202112162302-0 => 48.84.202112162302-0
data["content"][arch]["rhcos_version"] = rhcos_version
json.dump(data, sys.stdout)
async def promote_arch(self, release_name: str, arch: str, previous_list: List[str], metadata: Optional[Dict], reference_release: Optional[str], tag_stable: bool):
brew_arch = util.brew_arch_for_go_arch(arch) # ensure we are using Brew arches (e.g. aarch64) instead of golang arches (e.g. arm64).
dest_image_tag = f"{release_name}-{brew_arch}"
dest_image_pullspec = f"{self.DEST_RELEASE_IMAGE_REPO}:{dest_image_tag}"
self._logger.info("Checking if release image %s for %s (%s) already exists...", release_name, arch, dest_image_pullspec)
dest_image_info = await self.get_release_image_info(dest_image_pullspec)
if dest_image_info: # this arch-specific release image is already promoted
self._logger.warning("Release image %s for %s (%s) already exists", release_name, arch, dest_image_info["image"])
# TODO: Check if the existing release image matches the assembly definition.
if not dest_image_info or self.permit_overwrite:
if dest_image_info:
self._logger.warning("The existing release image %s will be overwritten!", dest_image_pullspec)
self._logger.info("Building arch-specific release image %s for %s (%s)...", release_name, arch, dest_image_pullspec)
await self.build_release_image(release_name, brew_arch, previous_list, metadata, dest_image_pullspec, reference_release)
self._logger.info("Release image for %s %s has been built and pushed to %s", release_name, arch, dest_image_pullspec)
self._logger.info("Getting release image information for %s...", dest_image_pullspec)
if not self.runtime.dry_run:
dest_image_info = await self.get_release_image_info(dest_image_pullspec, raise_if_not_found=True)
else:
# populate fake data for dry run
dest_image_info = {
"image": f"example.com/fake-release:{release_name}{brew_suffix_for_arch(arch)}",
"digest": f"fake:deadbeef{brew_suffix_for_arch(arch)}",
"references": {
"spec": {
"tags": [
{
"name": "machine-os-content",
"annotations": {"io.openshift.build.versions": "machine-os=00.00.212301010000-0"}
}
]
}
}
}
if reference_release:
dest_image_info["references"]["metadata"] = {"annotations": {"release.openshift.io/from-release": reference_release}}
else:
major, minor = util.isolate_major_minor_in_group(self.group)
go_arch_suffix = util.go_suffix_for_arch(arch, is_private=False)
dest_image_info["references"]["metadata"] = {"annotations": {"release.openshift.io/from-image-stream": f"fake{go_arch_suffix}/{major}.{minor}-art-assembly-{self.assembly}{go_arch_suffix}"}}
if not tag_stable:
self._logger.info("Release image %s will not appear on the release controller.", dest_image_pullspec)
return dest_image_info
go_arch_suffix = util.go_suffix_for_arch(arch)
namespace = f"ocp{go_arch_suffix}"
image_stream_tag = f"release{go_arch_suffix}:{release_name}"
namespace_image_stream_tag = f"{namespace}/{image_stream_tag}"
self._logger.info("Checking if ImageStreamTag %s exists...", namespace_image_stream_tag)
ist = await self.get_image_stream_tag(namespace, image_stream_tag)
if ist:
ist_digest = ist["image"]["dockerImageReference"].split("@")[-1]
if ist_digest == dest_image_info["digest"]:
self._logger.info("ImageStreamTag %s already exists with digest %s matching release image %s.", namespace_image_stream_tag, ist_digest, dest_image_pullspec)
return dest_image_info
message = f"ImageStreamTag {namespace_image_stream_tag} already exists, but it has a different digest ({ist_digest}) from the expected release image {dest_image_pullspec} ({dest_image_info['digest']})."
if not self.permit_overwrite:
raise ValueError(message)
self._logger.warning(message)
else:
self._logger.info("ImageStreamTag %s doesn't exist.", namespace_image_stream_tag)
self._logger.info("Tagging release image %s into %s...", dest_image_pullspec, namespace_image_stream_tag)
await self.tag_release(dest_image_pullspec, namespace_image_stream_tag)
self._logger.info("Release image %s has been tagged into %s.", dest_image_pullspec, namespace_image_stream_tag)
return dest_image_info