def rebuild_if_not_exists(self, event, errata_id):
"""
Initiates rebuild of artifacts based on Errata advisory with
`errata_id` id.
:rtype: List of ErrataAdvisoryRPMsSignedEvent instances.
:return: List of extra events generated to initiate the rebuild.
"""
db_event = db.session.query(Event).filter_by(
event_type_id=EVENT_TYPES[ErrataAdvisoryRPMsSignedEvent],
search_key=str(errata_id)).first()
if (db_event and db_event.state != EventState.FAILED.value and
not event.manual):
log.debug("Ignoring Errata advisory %d - it already exists in "
"Freshmaker db.", errata_id)
return []
# Get additional info from Errata to fill in the needed data.
errata = Errata()
advisories = errata.advisories_from_event(event)
if not advisories:
log.error("Unknown Errata advisory %d" % errata_id)
return []
log.info("Generating ErrataAdvisoryRPMsSignedEvent for Errata "
"advisory %d, because its state changed to %s.", errata_id,
event.advisory.state)
advisory = advisories[0]
new_event = ErrataAdvisoryRPMsSignedEvent(
event.msg_id + "." + str(advisory.name), advisory)
new_event.dry_run = event.dry_run
new_event.manual = event.manual
return [new_event]
def prepare_yum_repo(self, db_event):
"""
Request a compose from ODCS for builds included in Errata advisory
Run a compose in ODCS to contain required RPMs for rebuilding images
later.
:param Event db_event: current event being handled that contains errata
advisory to get builds containing updated RPMs.
:return: a mapping returned from ODCS that represents the request
compose.
:rtype: dict
"""
errata_id = int(db_event.search_key)
packages = []
errata = Errata()
builds = errata.get_srpm_nvrs(errata_id)
compose_source = None
for nvr in builds:
packages += self._get_packages_for_compose(nvr)
source = self._get_compose_source(nvr)
if compose_source and compose_source != source:
# TODO: Handle this by generating two ODCS composes
db_event.builds_transition(
ArtifactBuildState.FAILED.value, "Packages for errata "
"advisory %d found in multiple different tags." %
(errata_id))
return
else:
compose_source = source
if compose_source is None:
db_event.builds_transition(
ArtifactBuildState.FAILED.value, 'None of builds %s of '
'advisory %d is the latest build in its candidate tag.' %
(builds, errata_id))
return
self.handler.log_info(
'Generating new compose for rebuild: '
'source: %s, source type: %s, packages: %s', compose_source, 'tag',
packages)
if not self.handler.dry_run:
new_compose = create_odcs_client().new_compose(
compose_source,
'tag',
packages=packages,
sigkeys=conf.odcs_sigkeys,
flags=["no_deps"])
else:
new_compose = self._fake_odcs_new_compose(compose_source,
'tag',
packages=packages)
return new_compose
def setUp(self):
super(TestErrataAuthorizedGet, self).setUp()
self.errata = Errata("https://localhost/")
self.patcher = helpers.Patcher('freshmaker.errata.')
self.requests_get = self.patcher.patch("requests.get")
self.response = MagicMock()
self.response.json.return_value = {"foo": "bar"}
self.unlink = self.patcher.patch("os.unlink")
def _verify_advisory_rpms_in_container_build(self, errata_id,
container_build_id):
"""
verify container built on brew has the latest rpms from an advisory
"""
if self.dry_run:
return (True, '')
# Get rpms in advisory. There can be multiple versions of RPMs with
# the same name, so we group them by a name in `advisory_rpms_by_name`
# and use set of the nvrs as a value.
advisory_rpms_by_name = {}
e = Errata()
build_nvrs = e.get_builds(errata_id)
if build_nvrs:
with koji_service(conf.koji_profile,
log,
login=False,
dry_run=self.dry_run) as session:
for build_nvr in build_nvrs:
build_rpms = session.get_build_rpms(build_nvr)
for rpm in build_rpms:
if rpm['name'] not in advisory_rpms_by_name:
advisory_rpms_by_name[rpm['name']] = set()
advisory_rpms_by_name[rpm['name']].add(rpm['nvr'])
# get rpms in container
with koji_service(conf.koji_profile,
log,
login=False,
dry_run=self.dry_run) as session:
container_rpms = session.get_rpms_in_container(container_build_id)
container_rpms_by_name = {
rpmlib.parse_nvr(x)['name']: x
for x in container_rpms
}
# For each RPM name in advisory, check that the RPM exists in the
# built container and its version is the same as one RPM in the
# advisory.
unmatched_rpms = []
for rpm_name, nvrs in advisory_rpms_by_name.items():
if rpm_name not in container_rpms_by_name:
continue
container_rpm_nvr = container_rpms_by_name[rpm_name]
if container_rpm_nvr not in nvrs:
unmatched_rpms.append(rpm_name)
if unmatched_rpms:
msg = ("The following RPMs in container build (%s) do not match "
"with the latest RPMs in advisory (%s):\n%s" %
(container_build_id, errata_id, unmatched_rpms))
return (False, msg)
return (True, "")
def handle(self, event):
log.info("Finding out all advisories including %s", event.nvr)
# When get a signed RPM, first step is to find out advisories
# containing that RPM and ensure all builds are signed.
errata = Errata()
advisories = errata.advisories_from_event(event)
# Filter out advisories which are not allowed by configuration.
advisories = [
advisory for advisory in advisories if self.allow_build(
ArtifactType.IMAGE,
advisory_name=advisory.name,
advisory_security_impact=advisory.security_impact,
advisory_highest_cve_severity=advisory.highest_cve_severity,
advisory_state=advisory.state)
]
# Filter out advisories which are already in Freshmaker DB.
advisories = self._filter_out_existing_advisories(advisories)
if not advisories:
log.info("No advisories found suitable for rebuilding Docker "
"images")
return []
if not all((errata.builds_signed(advisory.errata_id)
for advisory in advisories)):
log.info(
'Not all builds in %s are signed. Do not rebuild any '
'docker image until signed.', advisories)
return []
# Now we know that all advisories with this signed RPM have also other
# RPMs signed. We can then proceed and generate
# ErrataAdvisoryRPMsSignedEvent.
new_events = []
for advisory in advisories:
new_event = ErrataAdvisoryRPMsSignedEvent(
event.msg_id + "." + str(advisory.name), advisory)
db_event = Event.create(db.session,
new_event.msg_id,
new_event.search_key,
new_event.__class__,
released=False)
db.session.add(db_event)
new_events.append(new_event)
db.session.commit()
return new_events
def parse_post_data(self, data):
"""
Method shared between Frontend and Backend to parse the POST data
of manual rebuild JSON and generate the BaseEvent representation
of the rebuild request.
:param dict data: Dict generated from JSON from HTTP POST or parsed
from the UMB message sent from Frontend to Backend.
"""
msg_id = data.get('msg_id', "manual_rebuild_%s" % (str(time.time())))
dry_run = data.get('dry_run', False)
if data.get('errata_id', None):
errata_id = data.get('errata_id')
errata = Errata()
advisory = ErrataAdvisory.from_advisory_id(errata, errata_id)
event = ManualRebuildWithAdvisoryEvent(
msg_id,
advisory,
data.get("container_images", []),
data.get("metadata", None),
freshmaker_event_id=data.get('freshmaker_event_id'),
manual=True,
dry_run=dry_run,
requester=data.get('requester', None))
else:
event = ManualBundleRebuild(msg_id,
data.get('container_images', []),
data.get('bundle_images'),
data.get('metadata', None),
dry_run=dry_run,
requester=data.get('requester', None))
return event
def _create_original_to_rebuilt_nvrs_map(self):
"""
Creates mapping of original build NVRs to rebuilt NVRs in advisory.
Including NVRs of the builds from the blocking advisories
:rtype: dict
:return: map of the original NVRs as keys and rebuilt NVRs as values
"""
nvrs_mapping = {}
# Get builds from all blocking advisories
blocking_advisories_builds = \
Errata().get_blocking_advisories_builds(self.event.advisory.errata_id)
# Get builds NVRs from the advisory attached to the message/event and
# then get original NVR for every build
for product_info in self.event.advisory.builds.values():
for build in product_info['builds']:
# Search for the first build that triggered the chain of rebuilds
# for every shipped NVR to get original NVR from it
original_nvr = self.get_published_original_nvr(build['nvr'])
if original_nvr is None:
continue
nvrs_mapping[original_nvr] = build['nvr']
build_nvr = parse_nvr(build['nvr'])
# Check builds from blocking advisories and add to the mapping
# all of them, that have overlapping package names
for block_build in blocking_advisories_builds:
block_build_nvr = parse_nvr(block_build)
if block_build_nvr['name'] == build_nvr['name'] and \
block_build_nvr['version'] == build_nvr['version']:
nvrs_mapping[block_build] = build['nvr']
return nvrs_mapping
def parse(self, topic, msg):
msg_id = msg.get('msg_id')
inner_msg = msg.get('msg')
errata_id = int(inner_msg.get('errata_id'))
errata = Errata()
return ErrataAdvisoryStateChangedEvent(
msg_id, ErrataAdvisory.from_advisory_id(errata, errata_id))
class TestErrataAuthorizedGet(helpers.FreshmakerTestCase):
def setUp(self):
super(TestErrataAuthorizedGet, self).setUp()
self.errata = Errata("https://localhost/")
self.patcher = helpers.Patcher('freshmaker.errata.')
self.requests_get = self.patcher.patch("requests.get")
self.response = MagicMock()
self.response.json.return_value = {"foo": "bar"}
self.unlink = self.patcher.patch("os.unlink")
def tearDown(self):
super(TestErrataAuthorizedGet, self).tearDown()
self.patcher.unpatch_all()
def test_errata_authorized_get(self):
self.requests_get.return_value = self.response
data = self.errata._errata_authorized_get("http://localhost/test")
self.assertEqual(data, {"foo": "bar"})
def test_errata_authorized_get_kerberos_exception(self):
# Test that MutualAuthenticationError is retried.
self.requests_get.side_effect = [
MutualAuthenticationError, self.response
]
data = self.errata._errata_authorized_get("http://localhost/test")
self.assertEqual(data, {"foo": "bar"})
self.assertEqual(len(self.requests_get.mock_calls), 2)
def test_errata_authorized_get_kerberos_exception_401(self):
# Test that 401 error response is retried with kerberos ccache file
# removed.
error_response = MagicMock()
error_response.status_code = 401
error_response.raise_for_status.side_effect = HTTPError(
"Expected exception", response=error_response)
self.requests_get.side_effect = [error_response, self.response]
data = self.errata._errata_authorized_get("http://localhost/test")
self.assertEqual(data, {"foo": "bar"})
self.assertEqual(len(self.requests_get.mock_calls), 2)
self.unlink.assert_called_once_with(
helpers.AnyStringWith("freshmaker_cc"))
def parse(self, topic, msg):
msg_id = msg.get('msg_id')
inner_msg = msg.get('msg')
errata_id = int(inner_msg.get('errata_id'))
errata = Errata()
advisory = ErrataAdvisory.from_advisory_id(errata, errata_id)
# If advisory created by BOTAS and it's shipped,
# then return BotasErrataShippedEvent event
if advisory.state == "SHIPPED_LIVE" and \
advisory.reporter.startswith('botas'):
event = BotasErrataShippedEvent(msg_id, advisory)
else:
event = ErrataAdvisoryStateChangedEvent(msg_id, advisory)
return event
def parse(self, topic, msg):
msg_id = msg.get('msg_id')
inner_msg = msg.get('msg')
errata_id = int(inner_msg.get('errata_id'))
new_state = inner_msg.get('to')
errata = Errata()
advisory = ErrataAdvisory.from_advisory_id(errata, errata_id)
# When there is message delay, state change messages can arrive after
# advisory has been changed to a different state other than the one
# in message, so we override advisory state with the state in message
advisory.state = new_state
# If advisory created by BOTAS and it's shipped,
# then return BotasErrataShippedEvent event
if advisory.state == "SHIPPED_LIVE" and advisory.reporter.startswith(
'botas'):
event = BotasErrataShippedEvent(msg_id, advisory)
else:
event = ErrataAdvisoryStateChangedEvent(msg_id, advisory)
return event
def rebuild_images_depending_on_advisory(self, db_event, errata_id):
"""
Submits requests to Bob to rebuild the images depending on the
images updated in the advisory with ID `errata_id`.
"""
# Get the list of CDN repository names for each build in the advisory
# as well as the name of tags used for the images.
errata = Errata()
repo_tags = errata.get_docker_repo_tags(errata_id)
if not repo_tags:
msg = "No CDN repo found for advisory %r" % errata_id
self.log_info(msg)
db_event.transition(EventState.FAILED, msg)
db.session.commit()
return
# Use the Pulp to get the Docker repository name from the CDN repository
# name and store it into `docker_repos` dict.
pulp = Pulp(conf.pulp_docker_server_url, conf.pulp_docker_username,
conf.pulp_docker_password)
# {docker_repository_name: [list, of, docker, tags], ...}
docker_repos = {}
for per_build_repo_tags in repo_tags.values():
for cdn_repo, docker_repo_tags in per_build_repo_tags.items():
docker_repo = pulp.get_docker_repository_name(cdn_repo)
if not docker_repo:
self.log_error("No Docker repo found for CDN repo %r",
cdn_repo)
continue
docker_repos[docker_repo] = docker_repo_tags
self.log_info(
"Found following Docker repositories updated by the advisory: %r",
docker_repos.keys())
# Count the number of impacted builds to show them in state reason
# when moving the Event to COMPLETE.
num_impacted = None
# Submit rebuild request to Bob :).
for repo_name in docker_repos.keys():
self.log_info("Requesting Bob rebuild of %s", repo_name)
parent_build = self.record_build(
db_event,
repo_name,
ArtifactType.IMAGE_REPOSITORY,
state=ArtifactBuildState.DONE.value)
bob_url = "%s/update_children/%s" % (
conf.bob_server_url.rstrip('/'), repo_name)
headers = {"Authorization": "Bearer %s" % conf.bob_auth_token}
if self.dry_run:
self.log_info("DRY RUN: Skipping request to Bob.")
continue
r = requests.get(bob_url, headers=headers)
r.raise_for_status()
resp = r.json()
self.log_info("Response: %r", resp)
if "impacted" in resp:
if num_impacted is None:
num_impacted = 0
num_impacted += len(resp["impacted"])
for external_repo_name in resp["impacted"]:
self.record_build(db_event,
external_repo_name,
ArtifactType.IMAGE_REPOSITORY,
state=ArtifactBuildState.DONE.value,
dep_on=parent_build)
msg = "Advisory %s: Informed Bob about update of %d image repositories." % (
db_event.search_key, len(docker_repos))
if num_impacted is not None:
msg += " Bob is rebuilding %d impacted external image repositories." % (
num_impacted)
db_event.transition(EventState.COMPLETE, msg)
db.session.commit()
#!/usr/bin/python3
from __future__ import print_function
import os
import sys
from pprint import pprint
# Allow imports from parent directory.
sys.path.insert(1, os.path.join(sys.path[0], '..'))
# Set the FRESHMAKER_DEVELOPER_ENV variable.
os.environ["FRESHMAKER_DEVELOPER_ENV"] = "1"
from freshmaker.errata import Errata
from freshmaker import conf
if len(sys.argv) != 2:
print("Template for testing freshmaker.Errata class")
print("Usage: ./errata.py ERRATA_TOOL_SERVER_URL")
sys.exit(1)
conf.errata_tool_server_url = sys.argv[1]
errata = Errata()
# Example usage:
data = errata.get_builds(42515)
pprint(data)
def setUp(self):
super(TestErrata, self).setUp()
self.errata = Errata("https://localhost/")
class TestErrata(helpers.FreshmakerTestCase):
def setUp(self):
super(TestErrata, self).setUp()
self.errata = Errata("https://localhost/")
def tearDown(self):
super(TestErrata, self).tearDown()
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event(self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].errata_id, 28484)
self.assertEqual(advisories[0].name, "RHSA-2017:28484")
self.assertEqual(advisories[0].state, "QE")
self.assertEqual(advisories[0].content_types, ["rpm"])
self.assertEqual(advisories[0].security_impact, "important")
self.assertEqual(advisories[0].product_short_name, "product")
self.assertEqual(advisories[0].cve_list,
["CVE-2015-3253", "CVE-2016-6814"])
self.assertEqual(advisories[0].has_hightouch_bug, True)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_empty_cve(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
mocked_errata.advisory_rest_json["content"]["content"]["cve"] = ""
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].cve_list, [])
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_no_bugs(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
mocked_errata.bugs = []
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].has_hightouch_bug, False)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_empty_bug_flags(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
for bug in mocked_errata.bugs:
bug["flags"] = ""
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].has_hightouch_bug, False)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_missing_all_errata(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
del mocked_errata.builds["libntirpc-1.4.3-4.el7rhgs"]["all_errata"]
event = BrewSignRPMEvent("msgid", "libntirpc-1.4.3-4.el7rhgs")
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 0)
def test_advisories_from_event_unsupported_event(self):
event = GitRPMSpecChangeEvent("msgid", "libntirpc", "master", "foo")
with self.assertRaises(ValueError):
self.errata.advisories_from_event(event)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_advisories_from_event_errata_state_change_event(
self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
event = ErrataAdvisoryStateChangedEvent(
"msgid", ErrataAdvisory(28484, "name", "SHIPPED_LIVE", ['rpm']))
advisories = self.errata.advisories_from_event(event)
self.assertEqual(len(advisories), 1)
self.assertEqual(advisories[0].errata_id, 28484)
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_builds_signed_all_signed(self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
self.assertTrue(self.errata.builds_signed(28484))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_builds_signed_some_unsigned(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
mocked_errata.builds["libntirpc-1.4.3-4.el7rhgs"][
"rpms_signed"] = False
self.assertFalse(self.errata.builds_signed(28484))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_builds_signed_missing_data(self, errata_http_get,
errata_rest_get):
mocked_errata = MockedErrataAPI(errata_rest_get, errata_http_get)
mocked_errata.builds["libntirpc-1.4.3-4.el7rhgs"] = {}
self.assertFalse(self.errata.builds_signed(28484))
@patch('freshmaker.errata.requests.get')
def test_get_errata_repo_ids(self, get):
get.return_value.json.return_value = {
'rhel-6-server-eus-source-rpms__6_DOT_7__x86_64': [],
'rhel-6-server-eus-optional-debug-rpms__6_DOT_7__i386': [
'/path/to/package.rpm',
'/path/to/package1.rpm',
'/path/to/package2.rpm',
],
'rhel-6-server-eus-rpms__6_DOT_7__x86_64': [],
}
repo_ids = self.errata.get_pulp_repository_ids(25718)
self.assertEqual(
set([
'rhel-6-server-eus-source-rpms__6_DOT_7__x86_64',
'rhel-6-server-eus-optional-debug-rpms__6_DOT_7__i386',
'rhel-6-server-eus-rpms__6_DOT_7__x86_64'
]), set(repo_ids))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_rhel_release_from_product_version(self, errata_http_get,
errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
ret = self.errata._rhel_release_from_product_version(
28484, "PRODUCT1-3.2-NFS")
self.assertEqual(ret, "RHEL-6-foobar")
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_rhel_release_from_product_version_unknown_product_ver(
self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
with self.assertRaises(ValueError):
self.errata._rhel_release_from_product_version(
28484, "PRODUCT1-2.9-NFS")
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_get_nvrs(self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
srpms = self.errata.get_srpm_nvrs(28484, "")
binary_rpms = self.errata.get_binary_rpm_nvrs(28484)
self.assertEqual(
set(srpms),
set(['libntirpc-1.4.3-4.el7rhgs', 'libntirpc-1.4.3-4.el6rhs']))
self.assertEqual(
set(binary_rpms),
set([
'libntirpc-devel-1.4.3-4.el6rhs',
'libntirpc-devel-1.4.3-4.el7rhgs'
]))
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_get_binary_rpms_rhel_7(self, errata_http_get, errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
ret = self.errata.get_binary_rpm_nvrs(28484, "RHEL-7")
self.assertEqual(ret, ['libntirpc-devel-1.4.3-4.el7rhgs'])
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_get_srpm_nvrs_empty(self, errata_http_get, errata_rest_get):
api = MockedErrataAPI(errata_rest_get, errata_http_get)
api.builds_json = {
"PRODUCT1": [{
"libntirpc-1.4.3-4.el7rhgs": {
"PRODUCT2-3.2-NFS": {
"x86_64":
["libntirpc-devel-1.4.3-4.el7rhgs.x86_64.rpm"]
}
}
}]
}
ret = self.errata.get_srpm_nvrs(28484, "")
self.assertEqual(ret, [])
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_get_binary_nvrs_empty(self, errata_http_get, errata_rest_get):
api = MockedErrataAPI(errata_rest_get, errata_http_get)
api.builds_json = {
"PRODUCT1": [{
"libntirpc-1.4.3-4.el7rhgs": {
"PRODUCT2-3.2-NFS": {
"SRPMS":
["libntirpc-devel-1.4.3-4.el7rhgs.x86_64.rpm"]
}
}
}]
}
ret = self.errata.get_binary_rpm_nvrs(28484, "")
self.assertEqual(ret, [])
@patch.object(Errata, "_errata_rest_get")
@patch.object(Errata, "_errata_http_get")
def test_errata_get_cve_affected_rpm_nvrs(self, errata_http_get,
errata_rest_get):
MockedErrataAPI(errata_rest_get, errata_http_get)
ret = self.errata.get_cve_affected_rpm_nvrs(28484)
self.assertEqual(ret, ['libntirpc-1.4.3-4.el6rhs'])
def test_get_docker_repo_tags(self):
with patch.object(self.errata, "xmlrpc") as xmlrpc:
xmlrpc.get_advisory_cdn_docker_file_list.return_value = {
'foo-container-1-1': {
'docker': {
'target': {
'repos': {
'foo-526': {
'tags': ['5.26', 'latest']
}
}
}
}
}
}
repo_tags = self.errata.get_docker_repo_tags(28484)
expected = {'foo-container-1-1': {'foo-526': ['5.26', 'latest']}}
self.assertEqual(repo_tags, expected)
def test_get_docker_repo_tags_xmlrpc_exception(self):
with patch.object(self.errata, "xmlrpc") as xmlrpc:
xmlrpc.get_advisory_cdn_docker_file_list.side_effect = ValueError(
"Expected XMLRPC test exception")
repo_tags = self.errata.get_docker_repo_tags(28484)
self.assertEqual(repo_tags, None)
def test_get_docker_repo_tags_xmlrpc_non_returned(self):
with patch.object(self.errata, "xmlrpc") as xmlrpc:
xmlrpc.get_advisory_cdn_docker_file_list.return_value = None
repo_tags = self.errata.get_docker_repo_tags(28484)
self.assertEqual(repo_tags, None)
def _find_images_to_rebuild(self, errata_id):
"""
Finds docker rebuild images from each build added to specific Errata
advisory.
Found images are yielded in proper rebuild order from base images to
leaf images through the docker build dependency chain.
:param int errata_id: Errata ID.
"""
errata = Errata()
errata_id = int(errata_id)
# Use the errata_id to find out Pulp repository IDs from Errata Tool
# and furthermore get content_sets from Pulp where signed RPM will end
# up eventually when advisories are shipped.
pulp_repo_ids = list(set(errata.get_pulp_repository_ids(errata_id)))
pulp = Pulp(server_url=conf.pulp_server_url,
username=conf.pulp_username,
password=conf.pulp_password)
content_sets = pulp.get_content_set_by_repo_ids(pulp_repo_ids)
self.log_info(
'RPMs from advisory ends up in following content sets: '
'%s', content_sets)
# Query images from LightBlue by signed RPM's srpm name and found
# content sets
lb = LightBlue(server_url=conf.lightblue_server_url,
cert=conf.lightblue_certificate,
private_key=conf.lightblue_private_key,
event_id=self.event.msg_id)
# Check if we are allowed to rebuild unpublished images and clear
# published and release_categories if so.
if self.event.is_allowed(self, published=True):
published = True
release_categories = conf.lightblue_release_categories
else:
published = None
release_categories = None
# Limit the Lightblue query to particular leaf images if set in Event.
leaf_container_images = None
if isinstance(self.event, ManualRebuildWithAdvisoryEvent):
leaf_container_images = self.event.container_images
# For each SRPM NVR, find out all the containers which include
# this SRPM NVR.
srpm_nvrs = set(errata.get_builds(errata_id))
# affected_pkgs contains the list of actually affected pkgs from the CVE.
# We don't need to build images that really don't affect the CVE, even if they are
# listed in the RHSA. So let's just remove the ones that are not listed in here.
# In case this is empty we are just going to use the "old" behavior before this
# change was made, and rebuild everything.
affected_pkgs = set(
[pkg['pkg_name'] for pkg in self.event.advisory.affected_pkgs])
if affected_pkgs:
tmp_srpm_nvrs = set(srpm_nvrs)
srpm_nvrs = set()
for srpm_nvr in tmp_srpm_nvrs:
srpm_name = kobo.rpmlib.parse_nvr(srpm_nvr)["name"]
# In case the SRPM NVR is modular, the `affected_pkgs` might contain
# modules which are in the "module_name:module_stream/pkg_name" format.
# We need to respect this format and only try to match the package name, it
# means only the "/pkg_name" part of affected_pkg.
if is_pkg_modular(srpm_nvr):
if any(
affected_pkg.endswith(f"/{srpm_name}")
for affected_pkg in affected_pkgs):
srpm_nvrs.add(srpm_nvr)
elif srpm_name in affected_pkgs:
srpm_nvrs.add(srpm_nvr)
self.log_info((
"Not going to rebuild container images with RPMS from these SRPMs "
"because they're not affected: %r"),
tmp_srpm_nvrs.difference(srpm_nvrs))
self.log_info(
"Going to find all the container images to rebuild as "
"result of %r update.", srpm_nvrs)
batches = lb.find_images_to_rebuild(
srpm_nvrs,
content_sets,
filter_fnc=self._filter_out_not_allowed_builds,
published=published,
release_categories=release_categories,
leaf_container_images=leaf_container_images)
return batches
def _find_images_to_rebuild(self, errata_id):
"""
Finds docker rebuild images from each build added to specific Errata
advisory.
Found images are yielded in proper rebuild order from base images to
leaf images through the docker build dependency chain.
:param int errata_id: Errata ID.
"""
errata = Errata()
errata_id = int(errata_id)
# Use the errata_id to find out Pulp repository IDs from Errata Tool
# and furthermore get content_sets from Pulp where signed RPM will end
# up eventually when advisories are shipped.
pulp_repo_ids = list(set(errata.get_pulp_repository_ids(errata_id)))
pulp = Pulp(server_url=conf.pulp_server_url,
username=conf.pulp_username,
password=conf.pulp_password)
content_sets = pulp.get_content_set_by_repo_ids(pulp_repo_ids)
# Some container builds declare Pulp repos directly instead of content
# sets, but they are stored in the same location as content sets so they
# can be treated the same
content_sets.extend(pulp_repo_ids)
self.log_info(
'RPMs from advisory ends up in following content sets: '
'%s', content_sets)
# Query images from LightBlue by signed RPM's srpm name and found
# content sets
lb = LightBlue(server_url=conf.lightblue_server_url,
cert=conf.lightblue_certificate,
private_key=conf.lightblue_private_key,
event_id=self.current_db_event_id)
# Check if we are allowed to rebuild unpublished images and clear
# published and release_categories if so.
if self.event.is_allowed(self, published=True):
published = True
release_categories = conf.lightblue_release_categories
else:
published = None
release_categories = None
# Limit the Lightblue query to particular leaf images if set in Event.
leaf_container_images = None
if isinstance(self.event, ManualRebuildWithAdvisoryEvent):
leaf_container_images = self.event.container_images
# Get binary rpm nvrs which are affected by the CVEs in this advisory
affected_nvrs = self.event.advisory.affected_rpm_nvrs
# If there is no CVE affected binary rpms, this can be non-RHSA advisory,
# just rebuild images that have the builds in this advisory installed
if not affected_nvrs:
affected_nvrs = errata.get_binary_rpm_nvrs(errata_id)
self.log_info(
"Going to find all the container images to rebuild as "
"result of %r update.", affected_nvrs)
batches = lb.find_images_to_rebuild(
affected_nvrs,
content_sets,
filter_fnc=self._filter_out_not_allowed_builds,
published=published,
release_categories=release_categories,
leaf_container_images=leaf_container_images)
return batches
dictConfig(fedmsg_config.get('logging', {'version': 1}))
if len(sys.argv) < 2:
print("Queries Lightblue to find out all the images Freshmaker rebuilds.")
print("Usage: ./lightblue.py ERRATA_ID [[CONTAINER_IMAGE], ...]")
sys.exit(1)
container_images = sys.argv[2:]
app_context = app.app_context()
app_context.__enter__()
db.drop_all()
db.create_all()
db.session.commit()
errata = Errata()
kwargs = {}
if container_images:
EventClass = ManualRebuildWithAdvisoryEvent
kwargs['container_images'] = container_images
else:
EventClass = ErrataAdvisoryStateChangedEvent
event = EventClass(
"fake_message", ErrataAdvisory.from_advisory_id(errata, sys.argv[1]),
dry_run=True, **kwargs)
handler = RebuildImagesOnRPMAdvisoryChange()
with patch("freshmaker.consumer.get_global_consumer"):
handler.handle(event)
def setUp(self):
super(TestErrata, self).setUp()
self.errata = Errata("https://localhost/")
self.patcher = helpers.Patcher('freshmaker.errata.SFM2API.')
self.patcher.patch("fetch_cve_metadata", return_value=["moderate", {}])
#!/usr/bin/python3
from __future__ import print_function
import os
import sys
from pprint import pprint
# Allow imports from parent directory.
sys.path.insert(1, os.path.join(sys.path[0], '..'))
# Set the FRESHMAKER_DEVELOPER_ENV variable.
os.environ["FRESHMAKER_DEVELOPER_ENV"] = "1"
from freshmaker.errata import Errata
from freshmaker import conf
if len(sys.argv) != 2:
print("Template for testing freshmaker.Errata class")
print("Usage: ./errata.py ERRATA_TOOL_SERVER_URL")
sys.exit(1)
conf.errata_tool_server_url = sys.argv[1]
errata = Errata()
# Example usage:
data = errata.get_srpm_nvrs(42515)
pprint(data)
