def _find_images_to_rebuild(self, errata_id):
"""
Finds docker rebuild images from each build added to specific Errata
advisory.
Found images are yielded in proper rebuild order from base images to
leaf images through the docker build dependency chain.
:param int errata_id: Errata ID.
"""
errata = Errata()
errata_id = int(errata_id)
# Use the errata_id to find out Pulp repository IDs from Errata Tool
# and furthermore get content_sets from Pulp where signed RPM will end
# up eventually when advisories are shipped.
pulp_repo_ids = list(set(errata.get_pulp_repository_ids(errata_id)))
pulp = Pulp(server_url=conf.pulp_server_url,
username=conf.pulp_username,
password=conf.pulp_password)
content_sets = pulp.get_content_set_by_repo_ids(pulp_repo_ids)
self.log_info(
'RPMs from advisory ends up in following content sets: '
'%s', content_sets)
# Query images from LightBlue by signed RPM's srpm name and found
# content sets
lb = LightBlue(server_url=conf.lightblue_server_url,
cert=conf.lightblue_certificate,
private_key=conf.lightblue_private_key,
event_id=self.event.msg_id)
# Check if we are allowed to rebuild unpublished images and clear
# published and release_categories if so.
if self.event.is_allowed(self, published=True):
published = True
release_categories = conf.lightblue_release_categories
else:
published = None
release_categories = None
# Limit the Lightblue query to particular leaf images if set in Event.
leaf_container_images = None
if isinstance(self.event, ManualRebuildWithAdvisoryEvent):
leaf_container_images = self.event.container_images
# For each SRPM NVR, find out all the containers which include
# this SRPM NVR.
srpm_nvrs = set(errata.get_builds(errata_id))
# affected_pkgs contains the list of actually affected pkgs from the CVE.
# We don't need to build images that really don't affect the CVE, even if they are
# listed in the RHSA. So let's just remove the ones that are not listed in here.
# In case this is empty we are just going to use the "old" behavior before this
# change was made, and rebuild everything.
affected_pkgs = set(
[pkg['pkg_name'] for pkg in self.event.advisory.affected_pkgs])
if affected_pkgs:
tmp_srpm_nvrs = set(srpm_nvrs)
srpm_nvrs = set()
for srpm_nvr in tmp_srpm_nvrs:
srpm_name = kobo.rpmlib.parse_nvr(srpm_nvr)["name"]
# In case the SRPM NVR is modular, the `affected_pkgs` might contain
# modules which are in the "module_name:module_stream/pkg_name" format.
# We need to respect this format and only try to match the package name, it
# means only the "/pkg_name" part of affected_pkg.
if is_pkg_modular(srpm_nvr):
if any(
affected_pkg.endswith(f"/{srpm_name}")
for affected_pkg in affected_pkgs):
srpm_nvrs.add(srpm_nvr)
elif srpm_name in affected_pkgs:
srpm_nvrs.add(srpm_nvr)
self.log_info((
"Not going to rebuild container images with RPMS from these SRPMs "
"because they're not affected: %r"),
tmp_srpm_nvrs.difference(srpm_nvrs))
self.log_info(
"Going to find all the container images to rebuild as "
"result of %r update.", srpm_nvrs)
batches = lb.find_images_to_rebuild(
srpm_nvrs,
content_sets,
filter_fnc=self._filter_out_not_allowed_builds,
published=published,
release_categories=release_categories,
leaf_container_images=leaf_container_images)
return batches
class TestErrata(helpers.FreshmakerTestCase):
def setUp(self):
super(TestErrata, self).setUp()
self.errata = Errata("https://localhost/")
def tearDown(self):
super(TestErrata, self).tearDown()
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event(self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].errata_id, 28484)
self.assertEqual(advisories[0].name, "RHSA-2017:28484")
self.assertEqual(advisories[0].state, "QE")
self.assertEqual(advisories[0].content_types, ["rpm"])
self.assertEqual(advisories[0].security_impact, "important")
self.assertEqual(advisories[0].product_short_name, "product")
self.assertEqual(advisories[0].cve_list,
["CVE-2015-3253", "CVE-2016-6814"])
self.assertEqual(advisories[0].has_hightouch_bug, True)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_empty_cve(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
mocked_errata.advisory_rest_json["content"]["content"]["cve"] = ""
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].cve_list, [])
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_no_bugs(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
mocked_errata.bugs = []
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].has_hightouch_bug, False)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_empty_bug_flags(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
for bug in mocked_errata.bugs:
bug["flags"] = ""
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].has_hightouch_bug, False)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_missing_all_errata(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
del mocked_errata.builds["libntirpc-1.4.3-4.el7rhgs"]["all_errata"]
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 0)
def test_advisories_from_event_unsupported_event(self):
event = GitRPMSpecChangeEvent("msgid", "libntirpc", "master", "foo")
with self.assertRaises(ValueError):
self.errata.advisories_from_event(event)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_errata_state_change_event(
self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
event = ErrataAdvisoryStateChangedEvent(
"msgid", ErrataAdvisory(28484, "name", "SHIPPED_LIVE", ['rpm']))
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].errata_id, 28484)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_builds_signed_all_signed(self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
self.assertTrue(self.errata.builds_signed(28484))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_builds_signed_some_unsigned(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
mocked_errata.builds["libntirpc-1.4.3-4.el7rhgs"][
"rpms_signed"] = False
self.assertFalse(self.errata.builds_signed(28484))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_builds_signed_missing_data(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
mocked_errata.builds["libntirpc-1.4.3-4.el7rhgs"] = {}
self.assertFalse(self.errata.builds_signed(28484))
@patch('freshmaker.errata.requests.get')
def test_get_errata_repo_ids(self, get):
get.return_value.json.return_value = {
'rhel-6-server-eus-source-rpms__6_DOT_7__x86_64': [],
'rhel-6-server-eus-optional-debug-rpms__6_DOT_7__i386': [
'/path/to/package.rpm',
'/path/to/package1.rpm',
'/path/to/package2.rpm',
],
'rhel-6-server-eus-rpms__6_DOT_7__x86_64': [],
}
repo_ids = self.errata.get_pulp_repository_ids(25718)
self.assertEqual(
set([
'rhel-6-server-eus-source-rpms__6_DOT_7__x86_64',
'rhel-6-server-eus-optional-debug-rpms__6_DOT_7__i386',
'rhel-6-server-eus-rpms__6_DOT_7__x86_64'
]), set(repo_ids))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_rhel_release_from_product_version(self, errata_http_get,
errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
ret = self.errata._rhel_release_from_product_version(
28484, "PRODUCT1-3.2-NFS")
self.assertEqual(ret, "RHEL-6-foobar")
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_rhel_release_from_product_version_unknown_product_ver(
self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
with self.assertRaises(ValueError):
self.errata._rhel_release_from_product_version(
28484, "PRODUCT1-2.9-NFS")
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_get_nvrs(self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
srpms = self.errata.get_srpm_nvrs(28484, "")
binary_rpms = self.errata.get_binary_rpm_nvrs(28484)
self.assertEqual(
set(srpms),
set(['libntirpc-1.4.3-4.el7rhgs', 'libntirpc-1.4.3-4.el6rhs']))
self.assertEqual(
set(binary_rpms),
set([
'libntirpc-devel-1.4.3-4.el6rhs',
'libntirpc-devel-1.4.3-4.el7rhgs'
]))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_get_binary_rpms_rhel_7(self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
ret = self.errata.get_binary_rpm_nvrs(28484, "RHEL-7")
self.assertEqual(ret, ['libntirpc-devel-1.4.3-4.el7rhgs'])
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_get_srpm_nvrs_empty(self, errata_http_get, errata_rest_get):
api = MockedErrataAPI(errata_rest_get, errata_http_get)
api.builds_json = {
"PRODUCT1": [{
"libntirpc-1.4.3-4.el7rhgs": {
"PRODUCT2-3.2-NFS": {
"x86_64":
["libntirpc-devel-1.4.3-4.el7rhgs.x86_64.rpm"]
}
}
}]
}
ret = self.errata.get_srpm_nvrs(28484, "")
self.assertEqual(ret, [])
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_get_binary_nvrs_empty(self, errata_http_get, errata_rest_get):
api = MockedErrataAPI(errata_rest_get, errata_http_get)
api.builds_json = {
"PRODUCT1": [{
"libntirpc-1.4.3-4.el7rhgs": {
"PRODUCT2-3.2-NFS": {
"SRPMS":
["libntirpc-devel-1.4.3-4.el7rhgs.x86_64.rpm"]
}
}
}]
}
ret = self.errata.get_binary_rpm_nvrs(28484, "")
self.assertEqual(ret, [])
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_errata_get_cve_affected_rpm_nvrs(self, errata_http_get,
errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
ret = self.errata.get_cve_affected_rpm_nvrs(28484)
self.assertEqual(ret, ['libntirpc-1.4.3-4.el6rhs'])
def test_get_docker_repo_tags(self):
with patch.object(self.errata, "xmlrpc") as xmlrpc:
xmlrpc.get_advisory_cdn_docker_file_list.return_value = {
'foo-container-1-1': {
'docker': {
'target': {
'repos': {
'foo-526': {
'tags': ['5.26', 'latest']
}
}
}
}
}
}
repo_tags = self.errata.get_docker_repo_tags(28484)
expected = {'foo-container-1-1': {'foo-526': ['5.26', 'latest']}}
self.assertEqual(repo_tags, expected)
def test_get_docker_repo_tags_xmlrpc_exception(self):
with patch.object(self.errata, "xmlrpc") as xmlrpc:
xmlrpc.get_advisory_cdn_docker_file_list.side_effect = ValueError(
"Expected XMLRPC test exception")
repo_tags = self.errata.get_docker_repo_tags(28484)
self.assertEqual(repo_tags, None)
def test_get_docker_repo_tags_xmlrpc_non_returned(self):
with patch.object(self.errata, "xmlrpc") as xmlrpc:
xmlrpc.get_advisory_cdn_docker_file_list.return_value = None
repo_tags = self.errata.get_docker_repo_tags(28484)
self.assertEqual(repo_tags, None)
def _find_images_to_rebuild(self, errata_id):
"""
Finds docker rebuild images from each build added to specific Errata
advisory.
Found images are yielded in proper rebuild order from base images to
leaf images through the docker build dependency chain.
:param int errata_id: Errata ID.
"""
errata = Errata()
errata_id = int(errata_id)
# Use the errata_id to find out Pulp repository IDs from Errata Tool
# and furthermore get content_sets from Pulp where signed RPM will end
# up eventually when advisories are shipped.
pulp_repo_ids = list(set(errata.get_pulp_repository_ids(errata_id)))
pulp = Pulp(server_url=conf.pulp_server_url,
username=conf.pulp_username,
password=conf.pulp_password)
content_sets = pulp.get_content_set_by_repo_ids(pulp_repo_ids)
# Some container builds declare Pulp repos directly instead of content
# sets, but they are stored in the same location as content sets so they
# can be treated the same
content_sets.extend(pulp_repo_ids)
self.log_info(
'RPMs from advisory ends up in following content sets: '
'%s', content_sets)
# Query images from LightBlue by signed RPM's srpm name and found
# content sets
lb = LightBlue(server_url=conf.lightblue_server_url,
cert=conf.lightblue_certificate,
private_key=conf.lightblue_private_key,
event_id=self.current_db_event_id)
# Check if we are allowed to rebuild unpublished images and clear
# published and release_categories if so.
if self.event.is_allowed(self, published=True):
published = True
release_categories = conf.lightblue_release_categories
else:
published = None
release_categories = None
# Limit the Lightblue query to particular leaf images if set in Event.
leaf_container_images = None
if isinstance(self.event, ManualRebuildWithAdvisoryEvent):
leaf_container_images = self.event.container_images
# Get binary rpm nvrs which are affected by the CVEs in this advisory
affected_nvrs = self.event.advisory.affected_rpm_nvrs
# If there is no CVE affected binary rpms, this can be non-RHSA advisory,
# just rebuild images that have the builds in this advisory installed
if not affected_nvrs:
affected_nvrs = errata.get_binary_rpm_nvrs(errata_id)
self.log_info(
"Going to find all the container images to rebuild as "
"result of %r update.", affected_nvrs)
batches = lb.find_images_to_rebuild(
affected_nvrs,
content_sets,
filter_fnc=self._filter_out_not_allowed_builds,
published=published,
release_categories=release_categories,
leaf_container_images=leaf_container_images)
return batches
