def _create_original_to_rebuilt_nvrs_map(self):
"""
Creates mapping of original build NVRs to rebuilt NVRs in advisory.
Including NVRs of the builds from the blocking advisories
:rtype: dict
:return: map of the original NVRs as keys and rebuilt NVRs as values
"""
nvrs_mapping = {}
# Get builds from all blocking advisories
blocking_advisories_builds = \
Errata().get_blocking_advisories_builds(self.event.advisory.errata_id)
# Get builds NVRs from the advisory attached to the message/event and
# then get original NVR for every build
for product_info in self.event.advisory.builds.values():
for build in product_info['builds']:
# Search for the first build that triggered the chain of rebuilds
# for every shipped NVR to get original NVR from it
original_nvr = self.get_published_original_nvr(build['nvr'])
if original_nvr is None:
continue
nvrs_mapping[original_nvr] = build['nvr']
build_nvr = parse_nvr(build['nvr'])
# Check builds from blocking advisories and add to the mapping
# all of them, that have overlapping package names
for block_build in blocking_advisories_builds:
block_build_nvr = parse_nvr(block_build)
if block_build_nvr['name'] == build_nvr['name'] and \
block_build_nvr['version'] == build_nvr['version']:
nvrs_mapping[block_build] = build['nvr']
return nvrs_mapping
def get_brewroot_arch_base_path(config, nvr, signed):
"""
:param config: Base cli config object
:param nvr: Will return the base directory under which the arch directories should exist.
:param signed: If True, the base directory under which signed arch directories should exit.
An exception will be raised if the nvr cannot be found unsigned in the brewroot as this
indicates the nvr has not been built.
"""
parsed_nvr = parse_nvr(nvr)
package_name = parsed_nvr["name"]
package_version = parsed_nvr["version"]
package_release = parsed_nvr["release"]
unsigned_arch_base_path = '{brew_packages}/{package_name}/{package_version}/{package_release}'.format(
brew_packages=config.packages_path,
package_name=package_name,
package_version=package_version,
package_release=package_release,
)
if not os.path.isdir(unsigned_arch_base_path):
raise IOError(
f'Unable to find {nvr} in brewroot filesystem: {unsigned_arch_base_path}'
)
if not signed:
return unsigned_arch_base_path
else:
return '{unsigned_arch_path}/data/signed/{signing_key_id}'.format(
unsigned_arch_path=unsigned_arch_base_path,
signing_key_id=config.signing_key_id,
)
def image_has_auto_rebuild_tag(self, image):
""" Check if image has a tag enabled for auto rebuild.
:param dict image: Dict representation of an image entity in Pyxis.
:rtype: bool
:return: True if image has a tag enabled for auto rebuild in repository, otherwise False.
"""
for repo in image['repositories']:
# Skip unpublished repository
if not repo['published']:
continue
auto_rebuild_tags = self._pyxis.get_auto_rebuild_tags(
repo['registry'], repo['repository'])
tags = [t['name'] for t in repo.get('tags', [])]
if set(auto_rebuild_tags) & set(tags):
return True
# It'd be more efficient to do this check first, but the exceptions are edge cases
# (e.g. testing) and it's best to not use it unless absolutely necessary
nvr = image['brew']['build']
parsed_nvr = parse_nvr(nvr)
nv = f'{parsed_nvr["name"]}-{parsed_nvr["version"]}'
if nv in conf.bundle_autorebuild_tag_exceptions:
self.log_info(
'The bundle %r has an exception for being tagged with an auto-rebuild tag',
nvr)
return True
return False
def is_embargoed(an_nvr):
# .p1 or inclusion in the embargoed_tag_nvrs indicates this rpm is embargoed OR *was* embargoed.
# We can ignore it if it has already shipped.
parsed_test_nvr = parse_nvr(an_nvr)
if released_nvr is None or compare_nvr(
parsed_test_nvr,
released_nvr) > 0: # If this nvr hasn't shipped
if '.p1' in an_nvr or an_nvr in embargoed_tag_nvrs: # It's embargoed!
return True
return False
def _verify_advisory_rpms_in_container_build(self, errata_id,
container_build_id):
"""
verify container built on brew has the latest rpms from an advisory
"""
if self.dry_run:
return (True, '')
# Get rpms in advisory. There can be multiple versions of RPMs with
# the same name, so we group them by a name in `advisory_rpms_by_name`
# and use set of the nvrs as a value.
advisory_rpms_by_name = {}
e = Errata()
build_nvrs = e.get_builds(errata_id)
if build_nvrs:
with koji_service(conf.koji_profile,
log,
login=False,
dry_run=self.dry_run) as session:
for build_nvr in build_nvrs:
build_rpms = session.get_build_rpms(build_nvr)
for rpm in build_rpms:
if rpm['name'] not in advisory_rpms_by_name:
advisory_rpms_by_name[rpm['name']] = set()
advisory_rpms_by_name[rpm['name']].add(rpm['nvr'])
# get rpms in container
with koji_service(conf.koji_profile,
log,
login=False,
dry_run=self.dry_run) as session:
container_rpms = session.get_rpms_in_container(container_build_id)
container_rpms_by_name = {
rpmlib.parse_nvr(x)['name']: x
for x in container_rpms
}
# For each RPM name in advisory, check that the RPM exists in the
# built container and its version is the same as one RPM in the
# advisory.
unmatched_rpms = []
for rpm_name, nvrs in advisory_rpms_by_name.items():
if rpm_name not in container_rpms_by_name:
continue
container_rpm_nvr = container_rpms_by_name[rpm_name]
if container_rpm_nvr not in nvrs:
unmatched_rpms.append(rpm_name)
if unmatched_rpms:
msg = ("The following RPMs in container build (%s) do not match "
"with the latest RPMs in advisory (%s):\n%s" %
(container_build_id, errata_id, unmatched_rpms))
return (False, msg)
return (True, "")
def is_embargoed(an_nvre):
# .p1 or inclusion in the embargoed_tag_nvrs indicates this rpm is embargoed OR *was* embargoed.
# We can ignore it if it has already shipped.
test_nvre_obj = parse_nvr(an_nvre)
if released_nvre_obj is None or compare_nvr(
test_nvre_obj,
released_nvre_obj) > 0: # If this nvr hasn't shipped
if '.p1' in an_nvre or strip_epoch(
an_nvre) in embargoed_tag_nvrs: # It's embargoed!
return True
return False
def from_image_member_deps(
self, el_version: int, assembly: str, releases_config: Model,
image_meta: ImageMetadata,
rpm_map: Dict[str, RPMMetadata]) -> Dict[str, Dict]:
""" Returns RPM builds defined in image member dependencies
:param el_version: RHEL version
:param assembly: Assembly name to query. If None, this method will return true latest builds.
:param releases_config: a Model for releases.yaml
:param image_meta: An instance of ImageMetadata
:param rpm_map: Map of rpm_distgit_key -> RPMMetadata
:return: a dict; keys are component names, values are Brew build dicts
"""
component_builds: Dict[str, Dict] = {
} # rpms pinned to the runtime assembly; keys are rpm component names, values are brew build dicts
meta_config = assembly_metadata_config(releases_config, assembly,
'image', image_meta.distgit_key,
image_meta.config)
# honor image member dependencies
dep_nvrs = {
parse_nvr(dep[f"el{el_version}"])["name"]: dep[f"el{el_version}"]
for dep in meta_config.dependencies.rpms if dep[f"el{el_version}"]
} # rpms for this rhel version listed in member dependencies; keys are rpm component names, values are nvrs
if dep_nvrs:
dep_nvr_list = list(dep_nvrs.values())
self._logger.info(
"Found %s NVRs defined in image member '%s' dependencies. Fetching build infos from Brew...",
len(dep_nvr_list), image_meta.distgit_key)
dep_builds = self._get_builds(dep_nvr_list)
missing_nvrs = [
nvr for nvr, build in zip(dep_nvr_list, dep_builds)
if not build
]
if missing_nvrs:
raise IOError(
f"The following image member dependency NVRs don't exist: {missing_nvrs}"
)
# Make sure image member dependencies have no ART managed rpms.
art_rpms_in_deps = {dep_build["name"]
for dep_build in dep_builds} & {
meta.rpm_name
for meta in rpm_map.values()
}
if art_rpms_in_deps:
raise ValueError(
f"Unable to build plashet. Image member dependencies cannot have ART managed RPMs: {art_rpms_in_deps}"
)
for dep_build in dep_builds:
component_builds[dep_build["name"]] = dep_build
return component_builds
def find_non_latest_rpms(self) -> List[Tuple[str, str]]:
"""
If the packages installed in this image overlap packages in the candidate tag,
return NVRs of the latest candidate builds that are not also installed in this image.
This indicates that the image has not picked up the latest from candidate.
Note that this is completely normal for non-STREAM assemblies. In fact, it is
normal for any assembly other than the assembly used for nightlies.
Unfortunately, rhcos builds are not performed in sync with all other builds.
Thus, it is natural for them to lag behind when RPMs change. The should catch
up with the next RHCOS build.
:return: Returns a list of Tuple[INSTALLED_NVRs, NEWEST_NVR] where
newest is from the "latest" state of the specified candidate tag
if same the package installed into this archive is not the same NVR.
"""
# Find the default candidate tag appropriate for the RHEL version used by this RHCOS build.
candidate_brew_tag = self.runtime.get_default_candidate_brew_tag(
el_target=self.get_rhel_base_version())
# N.B. the "rpms" installed in an image archive are individual RPMs, not brew rpm package builds.
# we compare against the individual RPMs from latest candidate rpm package builds.
with self.runtime.shared_build_status_detector() as bs_detector:
candidate_rpms: Dict[str, Dict] = {
# the RPMs are collected by name mainly to de-duplicate (same RPM, multiple arches)
rpm['name']: rpm
for rpm in bs_detector.find_unshipped_candidate_rpms(
candidate_brew_tag, self.runtime.brew_event)
}
old_nvrs: List[Tuple[str, str]] = []
# Translate the package builds into a list of individual RPMs. Build dict[rpm_name] -> nvr for every NVR installed
# in this RHCOS build.
installed_nvr_map: Dict[str, str] = {
parse_nvr(installed_nvr)['name']: installed_nvr
for installed_nvr in self.get_rpm_nvrs()
}
# we expect only a few unshipped candidates most of the the time, so we'll just search for those.
for name, rpm in candidate_rpms.items():
rpm_nvr = rpm['nvr']
if name in installed_nvr_map:
installed_nvr = installed_nvr_map[name]
if rpm_nvr != installed_nvr:
old_nvrs.append((installed_nvr, rpm_nvr))
return old_nvrs
def get_nvrs_golang(nvrs, logger):
container_nvrs, rpm_nvrs = [], []
for n in nvrs:
parsed_nvr = parse_nvr(n)
nvr_tuple = (parsed_nvr['name'], parsed_nvr['version'],
parsed_nvr['release'])
if 'container' in parsed_nvr['name']:
container_nvrs.append(nvr_tuple)
else:
rpm_nvrs.append(nvr_tuple)
nvrs = {}
if rpm_nvrs:
nvrs.update(util.get_golang_rpm_nvrs(rpm_nvrs, logger))
if container_nvrs:
nvrs.update(util.get_golang_container_nvrs(container_nvrs, logger))
util.pretty_print_nvrs_go(nvrs)
def from_pinned_by_is(self, el_version: int, assembly: str,
releases_config: Model,
rpm_map: Dict[str, RPMMetadata]) -> Dict[str, Dict]:
""" Returns RPM builds pinned by "is" in assembly config
:param el_version: RHEL version
:param assembly: Assembly name to query. If None, this method will return true latest builds.
:param releases_config: a Model for releases.yaml
:param rpm_map: Map of rpm_distgit_key -> RPMMetadata
:return: a dict; keys are component names, values are Brew build dicts
"""
pinned_nvrs: Dict[str, str] = {
} # rpms pinned to the runtime assembly; keys are rpm component names, values are nvrs
component_builds: Dict[str, Dict] = {
} # rpms pinned to the runtime assembly; keys are rpm component names, values are brew build dicts
# Honor pinned rpm nvrs pinned by "is"
for distgit_key, rpm_meta in rpm_map.items():
meta_config = assembly_metadata_config(releases_config, assembly,
'rpm', distgit_key,
rpm_meta.config)
nvr = meta_config["is"][f"el{el_version}"]
if not nvr:
continue
nvre_obj = parse_nvr(str(nvr))
if nvre_obj["name"] != rpm_meta.rpm_name:
raise ValueError(
f"RPM {nvr} is pinned to assembly {assembly} for distgit key {distgit_key}, but its package name is not {rpm_meta.rpm_name}."
)
pinned_nvrs[nvre_obj["name"]] = nvr
if pinned_nvrs:
pinned_nvr_list = list(pinned_nvrs.values())
self._logger.info(
"Found %s NVRs pinned to the runtime assembly %s. Fetching build infos from Brew...",
len(pinned_nvr_list), assembly)
pinned_builds = self._get_builds(pinned_nvr_list)
missing_nvrs = [
nvr for nvr, build in zip(pinned_nvr_list, pinned_builds)
if not build
]
if missing_nvrs:
raise IOError(
f"The following NVRs pinned by 'is' don't exist: {missing_nvrs}"
)
for pinned_build in pinned_builds:
component_builds[pinned_build["name"]] = pinned_build
return component_builds
def from_group_deps(self, el_version: int, group_config: Model,
rpm_map: Dict[str, RPMMetadata]) -> Dict[str, Dict]:
""" Returns RPM builds defined in group config dependencies
:param el_version: RHEL version
:param group_config: a Model for group config
:param rpm_map: Map of rpm_distgit_key -> RPMMetadata
:return: a dict; keys are component names, values are Brew build dicts
"""
component_builds: Dict[str, Dict] = {
} # rpms pinned to the runtime assembly; keys are rpm component names, values are brew build dicts
# honor group dependencies
dep_nvrs = {
parse_nvr(dep[f"el{el_version}"])["name"]: dep[f"el{el_version}"]
for dep in group_config.dependencies.rpms if dep[f"el{el_version}"]
} # rpms for this rhel version listed in group dependencies; keys are rpm component names, values are nvrs
if dep_nvrs:
dep_nvr_list = list(dep_nvrs.values())
self._logger.info(
"Found %s NVRs defined in group dependencies. Fetching build infos from Brew...",
len(dep_nvr_list))
dep_builds = self._get_builds(dep_nvr_list)
missing_nvrs = [
nvr for nvr, build in zip(dep_nvr_list, dep_builds)
if not build
]
if missing_nvrs:
raise IOError(
f"The following group dependency NVRs don't exist: {missing_nvrs}"
)
# Make sure group dependencies have no ART managed rpms.
art_rpms_in_group_deps = {
dep_build["name"]
for dep_build in dep_builds
} & {meta.rpm_name
for meta in rpm_map.values()}
if art_rpms_in_group_deps:
raise ValueError(
f"Unable to build plashet. Group dependencies cannot have ART managed RPMs: {art_rpms_in_group_deps}"
)
for dep_build in dep_builds:
component_builds[dep_build["name"]] = dep_build
return component_builds
def find_rhcos_build_rpm_inconsistencies(
rhcos_builds: List[RHCOSBuildInspector]) -> Dict[str, List[str]]:
"""
Looks through a set of RHCOS builds and finds if any of those builds contains a package version that
is inconsistent with the same package in another RHCOS build.
:return: Returns Dict[inconsistent_rpm_name] -> [inconsistent_nvrs, ...]. The Dictionary will be empty
if there are no inconsistencies detected.
"""
rpm_uses: Dict[str, Set[str]] = {}
for rhcos_build in rhcos_builds:
for nvr in rhcos_build.get_rpm_nvrs():
rpm_name = parse_nvr(nvr)['name']
if rpm_name not in rpm_uses:
rpm_uses[rpm_name] = set()
rpm_uses[rpm_name].add(nvr)
# Report back rpm name keys which were associated with more than one NVR in the set of RHCOS builds.
return {
rpm_name: nvr_list
for rpm_name, nvr_list in rpm_uses.items() if len(nvr_list) > 1
}
def from_tags(config, brew_tag, embargoed_brew_tag, embargoed_nvr,
signing_advisory_id, signing_advisory_mode, poll_for,
include_previous_for, include_previous, event, include_embargoed,
inherit):
"""
The repositories are filled with RPMs derived from the list of
brew tags. If the RPMs are not signed and a repo should contain signed content,
the specified advisory will be used for signing the RPMs (requires
automatic sign on attach).
If you specify --embargoed-brew-tag, plashet will treat any nvr found in this tag as if it is
embargoed (unless has already shipped). This is useful since the .p1 convention cannot be used
on RPMs not built by ART.
\b
--brew-tag <tag> <product_version> example: --brew-tag rhaos-4.5-rhel-8-candidate OSE-4.5-RHEL-8 --brew-tag .. ..
"""
koji_proxy = KojiWrapper(
koji.ClientSession(config.brew_url,
opts={
'krbservice': 'brewhub',
'serverca': '/etc/pki/brew/legacy.crt'
}))
koji_proxy.gssapi_login()
errata_proxy = xmlrpclib.ServerProxy(config.errata_xmlrpc_url)
if event:
event = int(event)
event_info = koji_proxy.getEvent(event)
else:
# If none was specified, lock in an event so that there are no race conditions with
# packages changing while we run.
event_info = koji_proxy.getLastEvent()
event = event_info['id']
# Gather up all nvrs tagged in the embargoed brew tags into a set.
embargoed_tag_nvrs = set()
embargoed_tag_nvrs.update(embargoed_nvr)
for ebt in embargoed_brew_tag:
for build in koji_proxy.listTagged(ebt,
latest=False,
inherit=False,
event=event,
type='rpm'):
embargoed_tag_nvrs.add(to_nvre(build))
logger.info(
'Will treat the following nvrs as potentially embargoed: {}'.format(
embargoed_tag_nvrs))
actual_embargoed_nvres = list() # A list of nvres detected as embargoed
desired_nvres = set()
historical_nvres = set()
nvr_product_version = {}
for tag, product_version in brew_tag:
released_package_nvre_obj = {
} # maps released package names to the most recently released package nvr object (e.g { 'name': ..., }
if tag.endswith('-candidate'):
"""
So here's the thing. If you ship a version of a package 1.16.6 via errata tool,
it will prevent you from shipping an older version of that package (e.g. 1.16.2) or even
attaching it to an errata. This prevents us from auto-signing the older package. Since it
is just invalid, we need to find the latest version of packages which have shipped
and make sure plashet filters out anything that is older before signing/building.
Without this filtering, the error from errata tool looks like:
b'{"error":"Unable to add build \'cri-o-1.16.6-2.rhaos4.3.git4936f44.el7\' which is older than cri-o-1.16.6-16.dev.rhaos4.3.git4936f44.el7"}'
"""
released_tag = tag[:tag.index('-candidate')]
for build in koji_proxy.listTagged(released_tag,
latest=True,
inherit=True,
event=event,
type='rpm'):
package_name = build['package_name']
released_package_nvre_obj[package_name] = parse_nvr(
to_nvre(build))
for build in koji_proxy.listTagged(tag,
latest=True,
inherit=inherit,
event=event,
type='rpm'):
package_name = build['package_name']
nvre = to_nvre(build)
nvre_obj = parse_nvr(nvre)
released_nvre_obj = None # if the package has shipped before, the parsed nvr of the most recently shipped
if package_name in released_package_nvre_obj:
released_nvre_obj = released_package_nvre_obj[package_name]
def is_embargoed(an_nvre):
# .p1 or inclusion in the embargoed_tag_nvrs indicates this rpm is embargoed OR *was* embargoed.
# We can ignore it if it has already shipped.
test_nvre_obj = parse_nvr(an_nvre)
if released_nvre_obj is None or compare_nvr(
test_nvre_obj,
released_nvre_obj) > 0: # If this nvr hasn't shipped
if '.p1' in an_nvre or strip_epoch(
an_nvre) in embargoed_tag_nvrs: # It's embargoed!
return True
return False
if package_name in config.exclude_package:
logger.info(
f'Skipping tagged but command line excluded package: {nvre}'
)
continue
if config.include_package and package_name not in config.include_package:
logger.info(
f'Skipping tagged but not command line included package: {nvre}'
)
continue
if is_embargoed(nvre):
# An embargoed build has not been shipped.
actual_embargoed_nvres.append(
nvre
) # Record that at the time of build, this was considered embargoed
if not include_embargoed:
# We are being asked to build a plashet without embargoed RPMs. We need to find a stand-in.
# Search through the tag's package history to find the last build that was NOT embargoed.
unembargoed_nvre = None
for build in koji_proxy.listTagged(tag,
package=package_name,
inherit=True,
event=event,
type='rpm'):
test_nvre = to_nvre(build)
if is_embargoed(test_nvre):
continue
unembargoed_nvre = test_nvre
break
if unembargoed_nvre is None:
raise IOError(
f'Unable to build unembargoed plashet. Lastest build of {package_name} ({nvre}) is embargoed but unable to find unembargoed version in history'
)
plashet_concerns.append(
f'Swapping embargoed nvr {nvre} for unembargoed nvr {unembargoed_nvre}.'
)
logger.info(plashet_concerns[-1])
nvre = unembargoed_nvre
if released_nvre_obj:
if compare_nvr(
nvre_obj, released_nvre_obj
) < 0: # if the current nvr is less than the released NVR
msg = f'Skipping tagged {nvre} because it is older than a released version: {released_nvre_obj}'
plashet_concerns.append(msg)
logger.error(msg)
continue
logger.info(f'{tag} contains package: {nvre}')
desired_nvres.add(nvre)
nvr_product_version[strip_epoch(nvre)] = product_version
if package_name.startswith(
tuple(include_previous_for)) or include_previous:
# The user has asked for non-latest entry for this package to be included in the plashet.
# we can try to find this by looking at the packages full history in this tag. Listing is
# newest -> oldest tagging event for this tag/package combination.
tag_history = koji_proxy.listTagged(tag,
package=package_name,
inherit=True,
event=event,
type='rpm')
tracking = False # There may have been embargo shenanigans above; so we can't assume [0] is our target nvr
for htag in tag_history:
history_nvre = to_nvre(htag)
if history_nvre == nvre:
# We've found the target NVR in the list. Everything that follows can be considered for history.
tracking = True
continue
if not tracking:
# Haven't found our target NVR yet; so we can't consider this entry for history.
continue
history_nvre_obj = parse_nvr(history_nvre)
if compare_nvr(history_nvre_obj, nvre_obj) > 0:
# Is our historical nvr > target for inclusion in plashet? If it is, a user of the plashet would
# pull in the historical nvr with a yum install. We can't allow that. Just give up -- this is
# not in line with the use case of history.
plashet_concerns.append(
f'Unable to include previous for {package_name} because history {history_nvre} is newer than latest tagged {nvre}'
)
break
if include_embargoed is False and is_embargoed(
history_nvre):
# smh.. history is still under embargo. What you are guys doing?!
plashet_concerns.append(
f'Unable to include previous for {package_name} because history {history_nvre} is under embargo'
)
break
historical_nvres.add(history_nvre)
nvr_product_version[strip_epoch(
history_nvre)] = product_version
break
if config.include_package and len(
config.include_package) != len(desired_nvres):
raise IOError(
f'Did not find all command line included packages {config.include_package}; only found {desired_nvres}'
)
# Did any of the archs require signed content?
possible_signing_needed = signed_desired(config)
if possible_signing_needed:
logger.info(f'At least one architecture requires signed nvres')
# Each set must be attached separately because you cannot attach two nvres of the same
# package to an errata at the same time.
for set_name, nvre_set in {
'latest_tagged': desired_nvres,
'previous_tagged': historical_nvres
}.items():
if not nvre_set:
logger.info(f'NVRE set {set_name} is empty; nothing to sign')
continue
if signing_advisory_id:
# Remove all builds attached to advisory before attempting signing
update_advisory_builds(config, errata_proxy,
signing_advisory_id, [],
nvr_product_version)
nvres_for_advisory = []
for nvre in nvre_set:
if not is_signed(config, nvre):
logger.info(
f'Found an unsigned nvr in nvre set {set_name} (will attempt to sign): {nvre}'
)
nvres_for_advisory.append(nvre)
logger.info(
f'Updating advisory to get nvre set {set_name} signed: {signing_advisory_id}'
)
update_advisory_builds(config, errata_proxy,
signing_advisory_id, nvres_for_advisory,
nvr_product_version)
else:
logger.warning(
f'No signing advisory specified; will simply poll and hope for nvre set {set_name}'
)
# Whether we've attached to advisory or no, wait until signing require is met
# or throw exception on timeout.
logger.info(
f'Waiting for all nvres in set {set_name} to be signed..')
for nvre in desired_nvres:
poll_for -= assert_signed(config, nvre)
if signing_advisory_id and signing_advisory_mode == 'clean':
# Seems that everything is signed; remove builds from the advisory.
update_advisory_builds(config, errata_proxy, signing_advisory_id, [],
nvr_product_version)
extra_embargo_info = { # Data related to embargoes that will be written into the plashet.yml
'embargoed_permitted': include_embargoed, # Whether we included or excluded these nvrs in the plashet
'detected_as_embargoed': actual_embargoed_nvres,
}
extra_data = { # Data that will be included in the plashet.yml after assembly.
'embargo_info': extra_embargo_info,
'included_previous_nvrs': list(historical_nvres),
}
all_nvres = set()
all_nvres.update(desired_nvres)
all_nvres.update(historical_nvres)
assemble_repo(config, all_nvres, event_info, extra_data=extra_data)
def test_valid_nvr(self):
self.assertEqual(parse_nvr("net-snmp-5.3.2.2-5.el5"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch=""))
self.assertEqual(parse_nvr("1:net-snmp-5.3.2.2-5.el5"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1"))
self.assertEqual(parse_nvr("net-snmp-1:5.3.2.2-5.el5"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1"))
self.assertEqual(parse_nvr("net-snmp-5.3.2.2-5.el5:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1"))
self.assertEqual(parse_nvr("/net-snmp-5.3.2.2-5.el5:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1"))
self.assertEqual(parse_nvr("/1:net-snmp-5.3.2.2-5.el5"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1"))
self.assertEqual(parse_nvr("foo/net-snmp-5.3.2.2-5.el5:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1"))
self.assertEqual(parse_nvr("foo/1:net-snmp-5.3.2.2-5.el5"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1"))
self.assertEqual(parse_nvr("/foo/bar/net-snmp-5.3.2.2-5.el5:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1"))
self.assertEqual(parse_nvr("/foo/bar/1:net-snmp-5.3.2.2-5.el5"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1"))
# test for name which contains the version number and a dash
self.assertEqual(parse_nvr("openmpi-1.10-1.10.2-2.el6"), dict(name="openmpi-1.10", version="1.10.2", release="2.el6", epoch=""))
def from_tags(config, brew_tag, embargoed_brew_tag, embargoed_nvr,
signing_advisory_id, signing_advisory_mode, poll_for, event,
include_embargoed):
"""
The repositories are filled with RPMs derived from the list of
brew tags. If the RPMs are not signed and a repo should contain signed content,
the specified advisory will be used for signing the RPMs (requires
automatic sign on attach).
If you specify --embargoed-brew-tag, plashet will treat any nvr found in this tag as if it is
embargoed (unless has already shipped). This is useful since the .p1 convention cannot be used
on RPMs not built by ART.
\b
--brew-tag <tag> <product_version> example: --brew-tag rhaos-4.5-rhel-8-candidate OSE-4.5-RHEL-8 --brew-tag .. ..
"""
koji_proxy = KojiWrapper(
koji.ClientSession(config.brew_url,
opts={
'krbservice': 'brewhub',
'serverca': '/etc/pki/brew/legacy.crt'
}))
koji_proxy.gssapi_login()
errata_proxy = xmlrpclib.ServerProxy(config.errata_xmlrpc_url)
if event:
event = int(event)
event_info = koji_proxy.getEvent(event)
else:
# If none was specified, lock in an event so that there are no race conditions with
# packages changing while we run.
event_info = koji_proxy.getLastEvent()
event = event_info['id']
# Gather up all nvrs tagged in the embargoed brew tags into a set.
embargoed_tag_nvrs = set()
embargoed_tag_nvrs.update(embargoed_nvr)
for ebt in embargoed_brew_tag:
for build in koji_proxy.listTagged(ebt,
latest=False,
inherit=False,
event=event,
type='rpm'):
embargoed_tag_nvrs.add(build['nvr'])
logger.info(
'Will treat the following nvrs as potentially embargoed: {}'.format(
embargoed_tag_nvrs))
actual_embargoed_nvrs = list() # A list of nvrs detected as embargoed
desired_nvrs = set()
nvr_product_version = {}
for tag, product_version in brew_tag:
released_package_nvrs = {
} # maps released package names to the most recently released package (parsed) nvr
if tag.endswith('-candidate'):
"""
So here's the thing. If you ship a version of a package 1.16.6 via errata tool,
it will prevent you from shipping an older version of that package (e.g. 1.16.2) or even
attaching it to an errata. This prevents us from auto-signing the older package. Since it
is just invalid, we need to find the latest version of packages which have shipped
and make sure plashet filters out anything that is older before signing/building.
Without this filtering, the error from errata tool looks like:
b'{"error":"Unable to add build \'cri-o-1.16.6-2.rhaos4.3.git4936f44.el7\' which is older than cri-o-1.16.6-16.dev.rhaos4.3.git4936f44.el7"}'
"""
released_tag = tag[:tag.index('-candidate')]
for build in koji_proxy.listTagged(released_tag,
latest=True,
inherit=True,
event=event,
type='rpm'):
package_name = build['package_name']
released_package_nvrs[package_name] = parse_nvr(build['nvr'])
for build in koji_proxy.listTagged(tag,
latest=True,
inherit=False,
event=event,
type='rpm'):
package_name = build['package_name']
nvr = build['nvr']
parsed_nvr = parse_nvr(nvr)
released_nvr = None # if the package has shipped before, the parsed nvr of the most recently shipped
if package_name in released_package_nvrs:
released_nvr = released_package_nvrs[package_name]
if package_name in config.exclude_package:
logger.info(
f'Skipping tagged but command line excluded package: {nvr}'
)
continue
if config.include_package and package_name not in config.include_package:
logger.info(
f'Skipping tagged but not command line included package: {nvr}'
)
continue
if '.p1' in nvr or nvr in embargoed_tag_nvrs:
# p1 or inclusion in the embargoed_tag_nvrs indicates this rpm is embargoed OR *was* embargoed.
# We can ignore it if it has already shipped.
if released_nvr is None or compare_nvr(
parsed_nvr,
released_nvr) > 0: # Is our nvr > last shipped?
# Our embargoed build has not been shipped.
actual_embargoed_nvrs.append(
nvr
) # Record that at the time of build, this was considered embargoed
if not include_embargoed:
# We are being asked to build a plashet without embargoed RPMs. We need to find a stand-in.
# Search through the tag's package history to find the last build that was NOT embargoed.
unembargoed_nvr = None
for build in koji_proxy.listTagged(
tag,
package=package_name,
inherit=True,
event=event,
type='rpm'):
test_nvr = build['nvr']
parsed_test_nvr = parse_nvr(test_nvr)
if released_nvr is None or compare_nvr(
parsed_test_nvr, released_nvr
) > 0: # If this nvr hasn't shipped
if '.p1' in test_nvr or test_nvr in embargoed_tag_nvrs: # Looks like this one is embargoed too
continue
unembargoed_nvr = test_nvr
break
if unembargoed_nvr is None:
raise IOError(
f'Unable to build unembargoed plashet. Lastest build of {package_name} ({nvr}) is embargoed but unable to find unembargoed version in history'
)
plashet_concerns.append(
f'Swapping embargoed nvr {nvr} for unembargoed nvr {unembargoed_nvr}.'
)
logger.info(plashet_concerns[-1])
nvr = unembargoed_nvr
else:
logger.info(
f'NVR {nvr} was potentially embargoed, but has already shipped'
)
if released_nvr:
if compare_nvr(
parsed_nvr, released_nvr
) < 0: # if the current nvr is less than the released NVR
msg = f'Skipping tagged {nvr} because it is older than a released version: {released_nvr}'
plashet_concerns.append(msg)
logger.error(msg)
continue
logger.info(f'{tag} contains package: {nvr}')
desired_nvrs.add(nvr)
nvr_product_version[nvr] = product_version
if config.include_package and len(
config.include_package) != len(desired_nvrs):
raise IOError(
f'Did not find all command line included packages {config.include_package}; only found {desired_nvrs}'
)
if signing_advisory_id and signing_advisory_mode == 'clean':
# Remove all builds attached to advisory before attempting signing
update_advisory_builds(config, errata_proxy, signing_advisory_id, [],
nvr_product_version)
# Did any of the archs require signed content?
possible_signing_needed = signed_desired(config)
if possible_signing_needed:
logger.info(f'At least one architecture requires signed nvrs')
if signing_advisory_id:
nvrs_for_advisory = []
for nvr in desired_nvrs:
if not is_signed(config, nvr):
logger.info(
f'Found an unsigned nvr (will attempt to signed): {nvr}'
)
nvrs_for_advisory.append(nvr)
logger.info(
f'Updating advisory to get nvrs signed: {signing_advisory_id}')
update_advisory_builds(config, errata_proxy, signing_advisory_id,
nvrs_for_advisory, nvr_product_version)
else:
logger.warning(
'No signing advisory specified; will simply poll and hope')
# Whether we've attached to advisory or no, wait until signing require is met
# or throw exception on timeout.
logger.info('Waiting for all nvrs to be signed..')
for nvr in desired_nvrs:
poll_for -= assert_signed(config, nvr)
if signing_advisory_id and signing_advisory_mode == 'clean':
# Seems that everything is signed; remove builds from the advisory.
update_advisory_builds(config, errata_proxy, signing_advisory_id, [],
nvr_product_version)
extra_embargo_info = { # Data related to embargos that will be written into the plashet.yml
'embargoed_permitted': include_embargoed, # Whether we included or excluded these nvrs in the plashet
'detected_as_embargoed': actual_embargoed_nvrs,
}
extra_data = { # Data that will be included in the plashet.yml after assembly.
'embargo_info': extra_embargo_info
}
assemble_repo(config, desired_nvrs, event_info, extra_data=extra_data)
def update_advisory_builds(config, errata_proxy, advisory_id, nvrs,
nvr_product_version):
"""
Attempts to get a specific set of RPM nvrs attached to an advisory
:param errata_proxy: proxy
:param advisory_id: The advisory to modify (should be in NEW_FILES)
:param product_version: Product version to attach RPMs for (e.g. RHEL-7-OSE-4.5)
:param nvrs: A list of RPM nvrs
:param nvr_product_version: A map of nvr->product_version
:return: n/a
Exception thrown if there is an error.
"""
desired_nvrs = set(nvrs)
errata_nvrs = set()
for build in errata_proxy.getErrataBrewBuilds(advisory_id):
nvr = build["brew_build_nvr"]
errata_nvrs.add(nvr)
to_remove = errata_nvrs.difference(desired_nvrs)
to_add = desired_nvrs.difference(errata_nvrs)
logger.info(f'Found currently attached to advisory: {to_remove}')
logger.info(f'Found not attached to advisory: {to_add}')
auth = HTTPKerberosAuth()
for nvr in to_remove:
remove_nvr_payload = {
'nvr': nvr,
}
res = requests.post(
f'{ERRATA_API_URL}/erratum/{advisory_id}/remove_build',
verify=ssl.get_default_verify_paths().openssl_cafile,
auth=auth,
json=remove_nvr_payload,
)
if res.status_code not in (200, 201):
logger.error(f'Error remove build from advisory: {res.content}')
raise IOError(
f'Unable to remove nvr from advisory {advisory_id}: {nvr}')
add_builds_payload = []
for nvr in to_add:
parsed_nvr = parse_nvr(nvr)
package_name = parsed_nvr["name"]
if package_name in config.exclude_package:
logger.info(
f'Skipping advisory attach for excluded package: {nvr}')
continue
add_builds_payload.append({
"product_version": nvr_product_version[nvr],
"build": nvr,
"file_types": ['rpm']
})
if add_builds_payload:
res = requests.post(
f'{ERRATA_API_URL}/erratum/{advisory_id}/add_builds',
verify=ssl.get_default_verify_paths().openssl_cafile,
auth=HTTPKerberosAuth(),
json=add_builds_payload,
)
if res.status_code not in (201, 200):
logger.error(f'Error attaching builds to advisory')
logger.error(f'Request: {add_builds_payload}')
logger.error(f'Response {res.status_code}: {res.content}')
raise IOError(
f'Unable to add nvrs to advisory {advisory_id}: {to_add}')
def check_rhcos_issues(
self, rhcos_build: RHCOSBuildInspector) -> List[AssemblyIssue]:
"""
Analyzes an RHCOS build to check whether the installed packages are consistent with:
1. package NVRs defined at the group dependency level
2. package NVRs defiend at the rhcos dependency level
3. package NVRs of any RPMs built in this assembly/group
:param rhcos_build: The RHCOS build to analyze.
:return: Returns a (potentially empty) list of inconsistencies in the build.
"""
self.runtime.logger.info(
f'Checking RHCOS build for consistency: {str(rhcos_build)}...')
issues: List[AssemblyIssue] = []
required_packages: Dict[str, str] = dict(
) # Dict[package_name] -> nvr # Dependency specified in 'rhcos' in assembly definition
desired_packages: Dict[str, str] = dict(
) # Dict[package_name] -> nvr # Dependency specified at group level
el_tag = f'el{rhcos_build.get_rhel_base_version()}'
for package_entry in (self.runtime.get_group_config().dependencies
or []):
if el_tag in package_entry:
nvr = package_entry[el_tag]
package_name = parse_nvr(nvr)['name']
desired_packages[package_name] = nvr
for package_entry in (self.assembly_rhcos_config.dependencies or []):
if el_tag in package_entry:
nvr = package_entry[el_tag]
package_name = parse_nvr(nvr)['name']
required_packages[package_name] = nvr
desired_packages[
package_name] = nvr # Override if something else was at the group level
installed_packages = rhcos_build.get_package_build_objects()
for package_name, desired_nvr in desired_packages.items():
if package_name in required_packages and package_name not in installed_packages:
# If the dependency is specified in the 'rhcos' section of the assembly, we must find it or raise an issue.
# This is impermissible because it can simply be fixed in the assembly definition.
issues.append(
AssemblyIssue(
f'Expected assembly defined rhcos dependency {desired_nvr} to be installed in {rhcos_build.build_id} but that package was not installed',
component='rhcos'))
if package_name in installed_packages:
installed_build_dict = installed_packages[package_name]
installed_nvr = installed_build_dict['nvr']
if installed_nvr != desired_nvr:
# We could consider permitting this in AssemblyTypes.CUSTOM, but it means that the RHCOS build
# could not be effectively reproduced by the rebuild job.
issues.append(
AssemblyIssue(
f'Expected {desired_nvr} to be installed in RHCOS build {rhcos_build.build_id} but found {installed_nvr}',
component='rhcos',
code=AssemblyIssueCode.
CONFLICTING_INHERITED_DEPENDENCY))
"""
If the rhcos build has RPMs from this group installed, make sure they match the NVRs associated with this assembly.
"""
for dgk, assembly_rpm_build in self.get_group_rpm_build_dicts(
el_ver=rhcos_build.get_rhel_base_version()).items():
if not assembly_rpm_build:
continue
package_name = assembly_rpm_build['package_name']
assembly_nvr = assembly_rpm_build['nvr']
if package_name in installed_packages:
installed_nvr = installed_packages[package_name]['nvr']
if assembly_nvr != installed_nvr:
# We could consider permitting this in AssemblyTypes.CUSTOM, but it means that the RHCOS build
# could not be effectively reproduced by the rebuild job.
issues.append(
AssemblyIssue(
f'Expected {rhcos_build.build_id}/{rhcos_build.brew_arch} image to contain assembly selected RPM build {assembly_nvr} but found {installed_nvr} installed',
component='rhcos',
code=AssemblyIssueCode.
CONFLICTING_GROUP_RPM_INSTALLED))
return issues
def test_valid_nvr(self):
self.assertEqual(
parse_nvr("net-snmp-5.3.2.2-5.el5"),
dict(name="net-snmp", version="5.3.2.2", release="5.el5",
epoch=""))
self.assertEqual(
parse_nvr("1:net-snmp-5.3.2.2-5.el5"),
dict(name="net-snmp",
version="5.3.2.2",
release="5.el5",
epoch="1"))
self.assertEqual(
parse_nvr("net-snmp-1:5.3.2.2-5.el5"),
dict(name="net-snmp",
version="5.3.2.2",
release="5.el5",
epoch="1"))
self.assertEqual(
parse_nvr("net-snmp-5.3.2.2-5.el5:1"),
dict(name="net-snmp",
version="5.3.2.2",
release="5.el5",
epoch="1"))
self.assertEqual(
parse_nvr("/net-snmp-5.3.2.2-5.el5:1"),
dict(name="net-snmp",
version="5.3.2.2",
release="5.el5",
epoch="1"))
self.assertEqual(
parse_nvr("/1:net-snmp-5.3.2.2-5.el5"),
dict(name="net-snmp",
version="5.3.2.2",
release="5.el5",
epoch="1"))
self.assertEqual(
parse_nvr("foo/net-snmp-5.3.2.2-5.el5:1"),
dict(name="net-snmp",
version="5.3.2.2",
release="5.el5",
epoch="1"))
self.assertEqual(
parse_nvr("foo/1:net-snmp-5.3.2.2-5.el5"),
dict(name="net-snmp",
version="5.3.2.2",
release="5.el5",
epoch="1"))
self.assertEqual(
parse_nvr("/foo/bar/net-snmp-5.3.2.2-5.el5:1"),
dict(name="net-snmp",
version="5.3.2.2",
release="5.el5",
epoch="1"))
self.assertEqual(
parse_nvr("/foo/bar/1:net-snmp-5.3.2.2-5.el5"),
dict(name="net-snmp",
version="5.3.2.2",
release="5.el5",
epoch="1"))
# test for name which contains the version number and a dash
self.assertEqual(
parse_nvr("openmpi-1.10-1.10.2-2.el6"),
dict(name="openmpi-1.10",
version="1.10.2",
release="2.el6",
epoch=""))
def _assemble_repo(config, nvrs: List[str]):
"""
This method is intended to be wrapped by assemble_repo.
Assembles one or more architecture specific repos in the
dest_dir with the specified nvrs. It is expected by the time this method
is called that all RPMs are signed if any of those arches requires signing.
:param config: cli config
:param nvrs: a list of nvrs to include.
:return: n/a
An exception will be thrown if no RPMs can be found matching an nvr.
"""
for arch_name, signing_mode in config.arch:
# These directories shouldn't exist yet. They will be created during assemble.
dest_arch_path = os.path.join(config.dest_dir, arch_name)
if config.repo_subdir:
dest_arch_path += '/' + config.repo_subdir.strip(
'/') # strip / from start and end
links_dir = os.path.join(dest_arch_path, 'Packages')
rpm_list_path = os.path.join(dest_arch_path, 'rpm_list')
mkdirs(links_dir)
# Each arch will have its own yum repo & thus needs its own rpm_list
with open(rpm_list_path, mode='w+') as rl:
for nvr in nvrs:
matched_count = 0
parsed_nvr = parse_nvr(nvr)
package_name = parsed_nvr["name"]
if package_name in config.exclude_package:
logger.info(
f'Skipping repo addition for excluded package: {nvr}')
continue
signed = (signing_mode == 'signed')
br_arch_base_path = get_brewroot_arch_base_path(
config, nvr, signed)
# Include noarch in each arch specific repo.
include_arches = [arch_name, 'noarch']
for a in include_arches:
brewroot_arch_path = os.path.join(br_arch_base_path, a)
if not os.path.isdir(brewroot_arch_path):
logger.debug(f'No {a} arch directory for {nvr}')
continue
logger.info(
f'Found {"signed" if signed else "unsigned"} {a} arch directory for {nvr}'
)
link_name = '{nvr}__{arch}'.format(
nvr=nvr,
arch=a,
)
if signed:
link_name += f'__{config.signing_key_id}'
package_link_path = os.path.join(links_dir, link_name)
os.symlink(brewroot_arch_path, package_link_path)
rpms = os.listdir(package_link_path)
if not rpms:
raise IOError(
f'Did not find any rpms in {brewroot_arch_path}')
for r in rpms:
rpm_path = os.path.join('Packages', link_name, r)
rl.write(rpm_path + '\n')
matched_count += 1
if not matched_count:
logger.warning(
"Unable to find any {arch} rpms for {nvr} in {p} ; this may be ok if the package doesn't support the arch and it is not required for that arch"
.format(arch=arch_name,
nvr=nvr,
p=get_brewroot_arch_base_path(
config, nvr, signed)))
if os.system('cd {repo_dir} && createrepo_c -i rpm_list .'.format(
repo_dir=dest_arch_path)) != 0:
raise IOError('Error creating repo at: {repo_dir}'.format(
repo_dir=dest_arch_path))
print('Successfully created repo at: {repo_dir}'.format(
repo_dir=dest_arch_path))
