def test_get_advisory(self, _make_request: Mock, ClientSession: Mock):
api = AsyncErrataAPI("https://errata.example.com")
_make_request.return_value = {"result": "fake"}
actual = get_event_loop().run_until_complete(api.get_advisory(1))
_make_request.assert_awaited_once_with(ANY, "GET", "/api/v1/erratum/1")
self.assertEqual(actual, {"result": "fake"})
_make_request.reset_mock()
actual = get_event_loop().run_until_complete(
api.get_advisory("RHBA-2021:0001"))
_make_request.assert_awaited_once_with(
ANY, "GET", "/api/v1/erratum/RHBA-2021%3A0001")
self.assertEqual(actual, {"result": "fake"})
def test_get_cves(self, _make_request: Mock, ClientSession: Mock):
api = AsyncErrataAPI("https://errata.example.com")
api.get_advisory = AsyncMock(
return_value={"content": {
"content": {
"cve": "A B C"
}
}})
actual = get_event_loop().run_until_complete(api.get_cves(1))
api.get_advisory.assert_awaited_once_with(1)
self.assertEqual(actual, ["A", "B", "C"])
class BugValidator:
def __init__(self,
runtime: Runtime,
use_jira: bool = False,
output: str = 'text'):
self.runtime = runtime
self.use_jira = use_jira
bug_tracker_cls = JIRABugTracker if use_jira else BugzillaBugTracker
self.config = bug_tracker_cls.get_config(runtime)
self.bug_tracker = bug_tracker_cls(self.config)
self.target_releases: List[str] = self.config['target_release']
self.product: str = self.config['product']
self.et_data: Dict[str, Any] = runtime.gitdata.load_data(
key='erratatool').data
self.errata_api = AsyncErrataAPI(
self.et_data.get("server", constants.errata_url))
self.problems: List[str] = []
self.output = output
async def close(self):
await self.errata_api.close()
def validate(
self,
non_flaw_bugs: Iterable[Bug],
verify_bug_status: bool,
):
non_flaw_bugs = self.filter_bugs_by_release(non_flaw_bugs,
complain=True)
blocking_bugs_for = self._get_blocking_bugs_for(non_flaw_bugs)
self._verify_blocking_bugs(blocking_bugs_for)
if verify_bug_status:
self._verify_bug_status(non_flaw_bugs)
if self.problems:
if self.output != 'slack':
red_print(
"Some bug problems were listed above. Please investigate.")
exit(1)
green_print(
"All bugs were verified. This check doesn't cover CVE flaw bugs.")
async def verify_attached_flaws(self, advisory_bugs: Dict[int, List[Bug]]):
futures = []
for advisory_id, attached_bugs in advisory_bugs.items():
attached_trackers = [
b for b in attached_bugs if b.is_tracker_bug()
]
attached_flaws = [b for b in attached_bugs if b.is_flaw_bug()]
futures.append(
self._verify_attached_flaws_for(advisory_id, attached_trackers,
attached_flaws))
await asyncio.gather(*futures)
if self.problems:
red_print(
"Some bug problems were listed above. Please investigate.")
exit(1)
green_print("All CVE flaw bugs were verified.")
async def _verify_attached_flaws_for(self, advisory_id: int,
attached_trackers: Iterable[Bug],
attached_flaws: Iterable[Bug]):
# Retrieve flaw bugs in Bugzilla for attached_tracker_bugs
tracker_flaws, flaw_id_bugs = self.bug_tracker.get_corresponding_flaw_bugs(
attached_trackers)
# Find first-fix flaws
first_fix_flaw_ids = set()
if attached_trackers:
current_target_release = bzutil.Bug.get_target_release(
attached_trackers)
if current_target_release[-1] == 'z':
first_fix_flaw_ids = flaw_id_bugs.keys()
else:
first_fix_flaw_ids = {
flaw_bug.id
for flaw_bug in flaw_id_bugs.values()
if bzutil.is_first_fix_any(self.bug_tracker.client(
), flaw_bug, current_target_release)
}
# Check if attached flaws match attached trackers
attached_flaw_ids = {b.id for b in attached_flaws}
missing_flaw_ids = attached_flaw_ids - first_fix_flaw_ids
if missing_flaw_ids:
self._complain(
f"On advisory {advisory_id}, {len(missing_flaw_ids)} flaw bugs are not attached but they are referenced by attached tracker bugs: {', '.join(sorted(map(str, missing_flaw_ids)))}."
" You probably need to attach those flaw bugs or drop the corresponding tracker bugs."
)
extra_flaw_ids = first_fix_flaw_ids - attached_flaw_ids
if extra_flaw_ids:
self._complain(
f"On advisory {advisory_id}, {len(extra_flaw_ids)} flaw bugs are attached but there are no tracker bugs referencing them: {', '.join(sorted(map(str, extra_flaw_ids)))}."
" You probably need to drop those flaw bugs or attach the corresponding tracker bugs."
)
# Check if advisory is of the expected type
advisory_info = await self.errata_api.get_advisory(advisory_id)
advisory_type = next(iter(advisory_info["errata"].keys())).upper(
) # should be one of [RHBA, RHSA, RHEA]
if not first_fix_flaw_ids:
if advisory_type == "RHSA":
self._complain(
f"Advisory {advisory_id} is of type {advisory_type} but has no first-fix flaw bugs. It should be converted to RHBA or RHEA."
)
return # The remaining checks are not needed for a non-RHSA.
if advisory_type != "RHSA":
self._complain(
f"Advisory {advisory_id} is of type {advisory_type} but has first-fix flaw bugs {first_fix_flaw_ids}. It should be converted to RHSA."
)
# Check if flaw bugs are associated with specific builds
cve_components_mapping: Dict[str, Set[str]] = {}
for tracker in attached_trackers:
component_name = tracker.whiteboard_component
if not component_name:
raise ValueError(
f"Tracker bug {tracker.id} doesn't have a valid component name in its whiteboard field."
)
flaw_ids = tracker_flaws[tracker.id]
for flaw_id in flaw_ids:
if len(flaw_id_bugs[flaw_id].alias) != 1:
raise ValueError(
f"Flaw bug {flaw_id} should have exact 1 alias.")
cve = flaw_id_bugs[flaw_id].alias[0]
cve_components_mapping.setdefault(cve,
set()).add(component_name)
current_cve_package_exclusions = await AsyncErrataUtils.get_advisory_cve_package_exclusions(
self.errata_api, advisory_id)
attached_builds = await self.errata_api.get_builds_flattened(
advisory_id)
expected_cve_packages_exclusions = AsyncErrataUtils.compute_cve_package_exclusions(
attached_builds, cve_components_mapping)
extra_cve_package_exclusions, missing_cve_package_exclusions = AsyncErrataUtils.diff_cve_package_exclusions(
current_cve_package_exclusions, expected_cve_packages_exclusions)
for cve, cve_package_exclusions in extra_cve_package_exclusions.items(
):
if cve_package_exclusions:
self._complain(
f"On advisory {advisory_id}, {cve} is not associated with Brew components {', '.join(sorted(cve_package_exclusions))}."
" You may need to associate the CVE with the components in the CVE mapping or drop the tracker bugs."
)
for cve, cve_package_exclusions in missing_cve_package_exclusions.items(
):
if cve_package_exclusions:
self._complain(
f"On advisory {advisory_id}, {cve} is associated with Brew components {', '.join(sorted(cve_package_exclusions))} without a tracker bug."
" You may need to explictly exclude those Brew components from the CVE mapping or attach the corresponding tracker bugs."
)
# Check if flaw bugs match the CVE field of the advisory
advisory_cves = advisory_info["content"]["content"]["cve"].split()
extra_cves = cve_components_mapping.keys() - advisory_cves
if extra_cves:
self._complain(
f"On advisory {advisory_id}, bugs for the following CVEs are already attached but they are not listed in advisory's `CVE Names` field: {', '.join(sorted(extra_cves))}"
)
missing_cves = advisory_cves - cve_components_mapping.keys()
if missing_cves:
self._complain(
f"On advisory {advisory_id}, bugs for the following CVEs are not attached but listed in advisory's `CVE Names` field: {', '.join(sorted(missing_cves))}"
)
async def get_attached_bugs(
self, advisory_ids: Iterable[str]) -> Dict[int, Set[Bug]]:
""" Get bugs attached to specified advisories
:return: a dict with advisory id as key and set of bug objects as value
"""
green_print(f"Retrieving bugs for advisory {advisory_ids}")
if self.use_jira:
issue_keys = {
advisory_id: [
issue["key"] for issue in
errata.get_jira_issue_from_advisory(advisory_id)
]
for advisory_id in advisory_ids
}
bug_map = self.bug_tracker.get_bugs_map(
[key for keys in issue_keys.values() for key in keys])
result = {
advisory_id: {bug_map[key]
for key in issue_keys[advisory_id]}
for advisory_id in advisory_ids
}
else:
advisories = await asyncio.gather(*[
self.errata_api.get_advisory(advisory_id)
for advisory_id in advisory_ids
])
bug_map = self.bug_tracker.get_bugs_map(
list({
b["bug"]["id"]
for ad in advisories for b in ad["bugs"]["bugs"]
}))
result = {
ad["content"]["content"]["errata_id"]:
{bug_map[b["bug"]["id"]]
for b in ad["bugs"]["bugs"]}
for ad in advisories
}
return result
def filter_bugs_by_product(self, bugs):
# filter out bugs for different product (presumably security flaw bugs)
return [b for b in bugs if b.product == self.product]
def filter_bugs_by_release(self,
bugs: Iterable[Bug],
complain: bool = False) -> List[Bug]:
# filter out bugs with an invalid target release
filtered_bugs = []
for b in bugs:
# b.target release is a list of size 0 or 1
if any(target in self.target_releases
for target in b.target_release):
filtered_bugs.append(b)
elif complain:
self._complain(
f"bug {b.id} target release {b.target_release} is not in "
f"{self.target_releases}. Does it belong in this release?")
return filtered_bugs
def _get_blocking_bugs_for(self, bugs):
# get blocker bugs in the next version for all bugs we are examining
candidate_blockers = [b.depends_on for b in bugs if b.depends_on]
candidate_blockers = {b for deps in candidate_blockers for b in deps}
v = minor_version_tuple(self.target_releases[0])
next_version = (v[0], v[1] + 1)
pattern = re.compile(r'^[0-9]+\.[0-9]+\.(0|z)$')
# retrieve blockers and filter to those with correct product and target version
blocking_bugs = {
bug.id: bug
for bug in self.bug_tracker.get_bugs(list(candidate_blockers))
# b.target release is a list of size 0 or 1
if any(
minor_version_tuple(target) == next_version
for target in bug.target_release
if pattern.match(target)) and bug.product == self.product
}
return {
bug:
[blocking_bugs[b] for b in bug.depends_on if b in blocking_bugs]
for bug in bugs
}
def _verify_blocking_bugs(self, blocking_bugs_for):
# complain about blocker bugs that aren't verified or shipped
for bug, blockers in blocking_bugs_for.items():
for blocker in blockers:
message = str()
if blocker.status not in [
'VERIFIED', 'RELEASE_PENDING', 'CLOSED'
]:
if self.output == 'text':
message = f"Regression possible: {bug.status} bug {bug.id} is a backport of bug " \
f"{blocker.id} which has status {blocker.status}"
elif self.output == 'slack':
message = f"`{bug.status}` bug <{bug.weburl}|{bug.id}> is a backport of " \
f"`{blocker.status}` bug <{blocker.weburl}|{blocker.id}>"
self._complain(message)
if blocker.status == 'CLOSED' and blocker.resolution not in [
'CURRENTRELEASE', 'NEXTRELEASE', 'ERRATA', 'DUPLICATE',
'NOTABUG', 'WONTFIX'
]:
if self.output == 'text':
message = f"Regression possible: {bug.status} bug {bug.id} is a backport of bug " \
f"{blocker.id} which was CLOSED {blocker.resolution}"
elif self.output == 'slack':
message = f"`{bug.status}` bug <{bug.weburl}|{bug.id}> is a backport of bug " \
f"<{blocker.weburl}|{blocker.id}> which was CLOSED `{blocker.resolution}`"
self._complain(message)
def _verify_bug_status(self, bugs):
# complain about bugs that are not yet VERIFIED or more.
for bug in bugs:
if bug.is_flaw_bug():
continue
if bug.status in ["VERIFIED", "RELEASE_PENDING"]:
continue
if bug.status == "CLOSED" and bug.resolution == "ERRATA":
continue
status = f"{bug.status}"
if bug.status == 'CLOSED':
status = f"{bug.status}: {bug.resolution}"
self._complain(f"Bug {bug.id} has status {status}")
def _complain(self, problem: str):
red_print(problem)
self.problems.append(problem)