def _verify_advisory_rpms_in_container_build(self, errata_id,
container_build_id):
"""
verify container built on brew has the latest rpms from an advisory
"""
if self.dry_run:
return (True, '')
# Get rpms in advisory. There can be multiple versions of RPMs with
# the same name, so we group them by a name in `advisory_rpms_by_name`
# and use set of the nvrs as a value.
advisory_rpms_by_name = {}
e = Errata()
build_nvrs = e.get_builds(errata_id)
if build_nvrs:
with koji_service(conf.koji_profile,
log,
login=False,
dry_run=self.dry_run) as session:
for build_nvr in build_nvrs:
build_rpms = session.get_build_rpms(build_nvr)
for rpm in build_rpms:
if rpm['name'] not in advisory_rpms_by_name:
advisory_rpms_by_name[rpm['name']] = set()
advisory_rpms_by_name[rpm['name']].add(rpm['nvr'])
# get rpms in container
with koji_service(conf.koji_profile,
log,
login=False,
dry_run=self.dry_run) as session:
container_rpms = session.get_rpms_in_container(container_build_id)
container_rpms_by_name = {
rpmlib.parse_nvr(x)['name']: x
for x in container_rpms
}
# For each RPM name in advisory, check that the RPM exists in the
# built container and its version is the same as one RPM in the
# advisory.
unmatched_rpms = []
for rpm_name, nvrs in advisory_rpms_by_name.items():
if rpm_name not in container_rpms_by_name:
continue
container_rpm_nvr = container_rpms_by_name[rpm_name]
if container_rpm_nvr not in nvrs:
unmatched_rpms.append(rpm_name)
if unmatched_rpms:
msg = ("The following RPMs in container build (%s) do not match "
"with the latest RPMs in advisory (%s):\n%s" %
(container_build_id, errata_id, unmatched_rpms))
return (False, msg)
return (True, "")
def prepare_yum_repo(self, db_event):
"""
Request a compose from ODCS for builds included in Errata advisory
Run a compose in ODCS to contain required RPMs for rebuilding images
later.
:param Event db_event: current event being handled that contains errata
advisory to get builds containing updated RPMs.
:return: a mapping returned from ODCS that represents the request
compose.
:rtype: dict
"""
errata_id = int(db_event.search_key)
packages = []
errata = Errata()
builds = errata.get_builds(errata_id)
compose_source = None
for nvr in builds:
packages += self._get_packages_for_compose(nvr)
source = self._get_compose_source(nvr)
if compose_source and compose_source != source:
# TODO: Handle this by generating two ODCS composes
db_event.builds_transition(
ArtifactBuildState.FAILED.value, "Packages for errata "
"advisory %d found in multiple different tags."
% (errata_id))
return
else:
compose_source = source
if compose_source is None:
db_event.builds_transition(
ArtifactBuildState.FAILED.value, 'None of builds %s of '
'advisory %d is the latest build in its candidate tag.'
% (builds, errata_id))
return
self.handler.log_info(
'Generating new compose for rebuild: '
'source: %s, source type: %s, packages: %s',
compose_source, 'tag', packages)
if not self.handler.dry_run:
new_compose = create_odcs_client().new_compose(
compose_source, 'tag', packages=packages,
sigkeys=conf.odcs_sigkeys, flags=["no_deps"])
else:
new_compose = self._fake_odcs_new_compose(
compose_source, 'tag', packages=packages)
return new_compose
def _find_images_to_rebuild(self, errata_id):
"""
Finds docker rebuild images from each build added to specific Errata
advisory.
Found images are yielded in proper rebuild order from base images to
leaf images through the docker build dependency chain.
:param int errata_id: Errata ID.
"""
errata = Errata()
errata_id = int(errata_id)
# Use the errata_id to find out Pulp repository IDs from Errata Tool
# and furthermore get content_sets from Pulp where signed RPM will end
# up eventually when advisories are shipped.
pulp_repo_ids = list(set(errata.get_pulp_repository_ids(errata_id)))
pulp = Pulp(server_url=conf.pulp_server_url,
username=conf.pulp_username,
password=conf.pulp_password)
content_sets = pulp.get_content_set_by_repo_ids(pulp_repo_ids)
self.log_info(
'RPMs from advisory ends up in following content sets: '
'%s', content_sets)
# Query images from LightBlue by signed RPM's srpm name and found
# content sets
lb = LightBlue(server_url=conf.lightblue_server_url,
cert=conf.lightblue_certificate,
private_key=conf.lightblue_private_key,
event_id=self.event.msg_id)
# Check if we are allowed to rebuild unpublished images and clear
# published and release_categories if so.
if self.event.is_allowed(self, published=True):
published = True
release_categories = conf.lightblue_release_categories
else:
published = None
release_categories = None
# Limit the Lightblue query to particular leaf images if set in Event.
leaf_container_images = None
if isinstance(self.event, ManualRebuildWithAdvisoryEvent):
leaf_container_images = self.event.container_images
# For each SRPM NVR, find out all the containers which include
# this SRPM NVR.
srpm_nvrs = set(errata.get_builds(errata_id))
# affected_pkgs contains the list of actually affected pkgs from the CVE.
# We don't need to build images that really don't affect the CVE, even if they are
# listed in the RHSA. So let's just remove the ones that are not listed in here.
# In case this is empty we are just going to use the "old" behavior before this
# change was made, and rebuild everything.
affected_pkgs = set(
[pkg['pkg_name'] for pkg in self.event.advisory.affected_pkgs])
if affected_pkgs:
tmp_srpm_nvrs = set(srpm_nvrs)
srpm_nvrs = set()
for srpm_nvr in tmp_srpm_nvrs:
srpm_name = kobo.rpmlib.parse_nvr(srpm_nvr)["name"]
# In case the SRPM NVR is modular, the `affected_pkgs` might contain
# modules which are in the "module_name:module_stream/pkg_name" format.
# We need to respect this format and only try to match the package name, it
# means only the "/pkg_name" part of affected_pkg.
if is_pkg_modular(srpm_nvr):
if any(
affected_pkg.endswith(f"/{srpm_name}")
for affected_pkg in affected_pkgs):
srpm_nvrs.add(srpm_nvr)
elif srpm_name in affected_pkgs:
srpm_nvrs.add(srpm_nvr)
self.log_info((
"Not going to rebuild container images with RPMS from these SRPMs "
"because they're not affected: %r"),
tmp_srpm_nvrs.difference(srpm_nvrs))
self.log_info(
"Going to find all the container images to rebuild as "
"result of %r update.", srpm_nvrs)
batches = lb.find_images_to_rebuild(
srpm_nvrs,
content_sets,
filter_fnc=self._filter_out_not_allowed_builds,
published=published,
release_categories=release_categories,
leaf_container_images=leaf_container_images)
return batches
class TestErrata(helpers.FreshmakerTestCase):
def setUp(self):
super(TestErrata, self).setUp()
self.errata = Errata("https://localhost/")
self.patcher = helpers.Patcher('freshmaker.errata.SFM2API.')
self.patcher.patch("fetch_cve_metadata", return_value=["moderate", {}])
def tearDown(self):
super(TestErrata, self).tearDown()
self.patcher.unpatch_all()
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event(self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].errata_id, 28484)
self.assertEqual(advisories[0].name, "RHSA-2017:28484")
self.assertEqual(advisories[0].state, "QE")
self.assertEqual(advisories[0].content_types, ["rpm"])
self.assertEqual(advisories[0].security_impact, "Important")
self.assertEqual(advisories[0].product_short_name, "product")
self.assertEqual(advisories[0].cve_list,
["CVE-2015-3253", "CVE-2016-6814"])
self.assertEqual(advisories[0].highest_cve_severity, "moderate")
self.assertEqual(advisories[0].has_hightouch_bug, True)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_empty_cve(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
mocked_errata.advisory_rest_json["content"]["content"]["cve"] = ""
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].cve_list, [])
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_no_bugs(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
mocked_errata.bugs = []
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].has_hightouch_bug, False)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_empty_bug_flags(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
for bug in mocked_errata.bugs:
bug["flags"] = ""
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].has_hightouch_bug, False)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_missing_all_errata(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
del mocked_errata.builds["libntirpc-1.4.3-4.el7rhgs"]["all_errata"]
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 0)
def test_advisories_from_event_unsupported_event(self):
event = GitRPMSpecChangeEvent("msgid", "libntirpc", "master", "foo")
with self.assertRaises(ValueError):
self.errata.advisories_from_event(event)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_errata_state_change_event(
self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
event = ErrataAdvisoryStateChangedEvent(
"msgid", ErrataAdvisory(28484, "name", "SHIPPED_LIVE", ['rpm']))
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].errata_id, 28484)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_builds_signed_all_signed(self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
self.assertTrue(self.errata.builds_signed(28484))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_builds_signed_some_unsigned(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
mocked_errata.builds["libntirpc-1.4.3-4.el7rhgs"][
"rpms_signed"] = False
self.assertFalse(self.errata.builds_signed(28484))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_builds_signed_missing_data(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
mocked_errata.builds["libntirpc-1.4.3-4.el7rhgs"] = {}
self.assertFalse(self.errata.builds_signed(28484))
@patch('freshmaker.errata.requests.get')
def test_get_errata_repo_ids(self, get):
get.return_value.json.return_value = {
'rhel-6-server-eus-source-rpms__6_DOT_7__x86_64': [],
'rhel-6-server-eus-optional-debug-rpms__6_DOT_7__i386': [
'/path/to/package.rpm',
'/path/to/package1.rpm',
'/path/to/package2.rpm',
],
'rhel-6-server-eus-rpms__6_DOT_7__x86_64': [],
}
repo_ids = self.errata.get_pulp_repository_ids(25718)
self.assertEqual(
set([
'rhel-6-server-eus-source-rpms__6_DOT_7__x86_64',
'rhel-6-server-eus-optional-debug-rpms__6_DOT_7__i386',
'rhel-6-server-eus-rpms__6_DOT_7__x86_64'
]), set(repo_ids))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_rhel_release_from_product_version(self, errata_http_get,
errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
ret = self.errata._rhel_release_from_product_version(
28484, "PRODUCT1-3.2-NFS")
self.assertEqual(ret, "RHEL-6-foobar")
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_rhel_release_from_product_version_unknown_product_ver(
self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
with self.assertRaises(ValueError):
self.errata._rhel_release_from_product_version(
28484, "PRODUCT1-2.9-NFS")
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_get_builds(self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
ret = self.errata.get_builds(28484, "")
self.assertEqual(
ret, set(['libntirpc-1.4.3-4.el7rhgs',
'libntirpc-1.4.3-4.el6rhs']))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_get_builds_rhel_7(self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
ret = self.errata.get_builds(28484, "RHEL-7")
self.assertEqual(ret, set(['libntirpc-1.4.3-4.el7rhgs']))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_get_builds_no_srpm(self, errata_http_get, errata_rest_get):
api = MockedErrataAPI(errata_rest_get, errata_http_get)
api.builds_json = {
"PRODUCT1": [{
"libntirpc-1.4.3-4.el7rhgs": {
"PRODUCT2-3.2-NFS": {
"x86_64":
["libntirpc-devel-1.4.3-4.el7rhgs.x86_64.rpm"]
}
}
}]
}
ret = self.errata.get_builds(28484, "")
self.assertEqual(ret, set(['libntirpc-1.4.3-4.el7rhgs']))
def test_get_docker_repo_tags(self):
with patch.object(self.errata, "xmlrpc") as xmlrpc:
xmlrpc.get_advisory_cdn_docker_file_list.return_value = {
'foo-container-1-1': {
'docker': {
'target': {
'repos': {
'foo-526': {
'tags': ['5.26', 'latest']
}
}
}
}
}
}
repo_tags = self.errata.get_docker_repo_tags(28484)
expected = {'foo-container-1-1': {'foo-526': ['5.26', 'latest']}}
self.assertEqual(repo_tags, expected)
def test_get_docker_repo_tags_xmlrpc_exception(self):
with patch.object(self.errata, "xmlrpc") as xmlrpc:
xmlrpc.get_advisory_cdn_docker_file_list.side_effect = ValueError(
"Expected XMLRPC test exception")
repo_tags = self.errata.get_docker_repo_tags(28484)
self.assertEqual(repo_tags, None)
def test_get_docker_repo_tags_xmlrpc_non_returned(self):
with patch.object(self.errata, "xmlrpc") as xmlrpc:
xmlrpc.get_advisory_cdn_docker_file_list.return_value = None
repo_tags = self.errata.get_docker_repo_tags(28484)
self.assertEqual(repo_tags, None)
#!/usr/bin/python3
from __future__ import print_function
import os
import sys
from pprint import pprint
# Allow imports from parent directory.
sys.path.insert(1, os.path.join(sys.path[0], '..'))
# Set the FRESHMAKER_DEVELOPER_ENV variable.
os.environ["FRESHMAKER_DEVELOPER_ENV"] = "1"
from freshmaker.errata import Errata
from freshmaker import conf
if len(sys.argv) != 2:
print("Template for testing freshmaker.Errata class")
print("Usage: ./errata.py ERRATA_TOOL_SERVER_URL")
sys.exit(1)
conf.errata_tool_server_url = sys.argv[1]
errata = Errata()
# Example usage:
data = errata.get_builds(42515)
pprint(data)
