def to_advisories(self, nvd_data):
for cve_item in nvd_data["CVE_Items"]:
if self.is_outdated(cve_item):
continue
if self.related_to_hardware(cve_item):
continue
cve_id = cve_item["cve"]["CVE_data_meta"]["ID"]
ref_urls = self.extract_reference_urls(cve_item)
references = [Reference(url=url) for url in ref_urls]
severity_scores = self.extract_severity_scores(cve_item)
references.append(
Reference(
url=f"https://nvd.nist.gov/vuln/detail/{cve_id}",
reference_id=cve_id,
severities=severity_scores,
)
)
summary = self.extract_summary(cve_item)
yield Advisory(
vulnerability_id=cve_id,
summary=summary,
references=references,
)
def test_load_advisory(self):
md_path = os.path.join(TEST_DATA, "RUSTSEC-2021-0032.md")
data = self.data_src._load_advisory(md_path)
expected_data = Advisory(
summary="",
vulnerability_id="CVE-2021-28033",
affected_packages=[
AffectedPackage(
vulnerable_package=PackageURL(
type="cargo",
name="byte_struct",
version="0.6.0",
),
patched_package=PackageURL(
type="cargo",
name="byte_struct",
version="0.6.1",
),
)
],
references=[
Reference(
reference_id="",
url="https://github.com/wwylele/byte-struct-rs/issues/1",
severities=[],
),
Reference(
reference_id="RUSTSEC-2021-0032",
url="https://rustsec.org/advisories/RUSTSEC-2021-0032.html",
severities=[],
),
],
)
assert expected_data == data
def test_to_advisories(self):
expected_advisories = [
Advisory(
summary="",
references=[
Reference(url="https://usn.ubuntu.com/763-1/",
reference_id="USN-763-1")
],
vulnerability_id="CVE-2009-0698",
),
Advisory(
summary="",
references=[
Reference(url="https://usn.ubuntu.com/763-1/",
reference_id="USN-763-1")
],
vulnerability_id="CVE-2009-1274",
),
]
found_advisories = self.data_src.to_advisories(self.db)
found_advisories = list(map(Advisory.normalized, found_advisories))
expected_advisories = list(
map(Advisory.normalized, expected_advisories))
assert sorted(found_advisories) == sorted(expected_advisories)
def _load_advisory(self, path: str) -> Optional[Advisory]:
record = load_toml(path)
advisory = record.get("advisory", {})
crate_name = advisory["package"]
references = []
if advisory.get("url"):
references.append(Reference(url=advisory["url"]))
all_versions = self.crates_api.get(crate_name)
affected_ranges = {
RangeSpecifier(r)
for r in chain.from_iterable(
record.get("affected", {}).get("functions", {}).values())
}
unaffected_ranges = {
RangeSpecifier(r)
for r in record.get("versions", {}).get("unaffected", [])
}
resolved_ranges = {
RangeSpecifier(r)
for r in record.get("versions", {}).get("patched", [])
}
unaffected, affected = categorize_versions(all_versions,
unaffected_ranges,
affected_ranges,
resolved_ranges)
impacted_purls = {
PackageURL(type="cargo", name=crate_name, version=v)
for v in affected
}
resolved_purls = {
PackageURL(type="cargo", name=crate_name, version=v)
for v in unaffected
}
cve_id = None
if "aliases" in advisory:
for alias in advisory["aliases"]:
if alias.startswith("CVE-"):
cve_id = alias
break
references.append(
Reference(
reference_id=advisory["id"],
url="https://rustsec.org/advisories/{}.html".format(
advisory["id"]),
))
return Advisory(
summary=advisory.get("description", ""),
impacted_package_urls=impacted_purls,
resolved_package_urls=resolved_purls,
cve_id=cve_id,
vuln_references=references,
)
def to_advisory(advisory_data):
affected_purls = []
if advisory_data.get("affected_packages"):
for rpm in advisory_data["affected_packages"]:
if rpm_to_purl(rpm):
affected_purls.append(rpm_to_purl(rpm))
references = []
if advisory_data.get("bugzilla"):
bugzilla = advisory_data.get("bugzilla")
references.append(
Reference(
url="https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla),
reference_id=bugzilla,
)
)
for rhsa in advisory_data["advisories"]:
references.append(
Reference(
url="https://access.redhat.com/errata/{}".format(rhsa), reference_id=rhsa,
)
)
references.append(Reference(url=advisory_data["resource_url"]))
return Advisory(
summary=advisory_data["bugzilla_description"],
cve_id=advisory_data["CVE"],
impacted_package_urls=affected_purls,
vuln_references=references,
)
def _parse(self, record) -> List[Advisory]:
advisories = []
for cve_id in record["issues"]:
affected_packages = []
for name in record["packages"]:
impacted_purls, resolved_purls = [], []
impacted_purls.append(
PackageURL(
name=name,
type="pacman",
namespace="archlinux",
version=record["affected"],
)
)
if record["fixed"]:
resolved_purls.append(
PackageURL(
name=name,
type="pacman",
namespace="archlinux",
version=record["fixed"],
)
)
affected_packages.extend(nearest_patched_package(impacted_purls, resolved_purls))
references = []
references.append(
Reference(
reference_id=record["name"],
url="https://security.archlinux.org/{}".format(record["name"]),
severities=[
VulnerabilitySeverity(
system=scoring_systems["avgs"], value=record["severity"]
)
],
)
)
for ref in record["advisories"]:
references.append(
Reference(
reference_id=ref,
url="https://security.archlinux.org/{}".format(ref),
)
)
advisories.append(
Advisory(
vulnerability_id=cve_id,
summary="",
affected_packages=affected_packages,
references=references,
)
)
return advisories
def extract_references(reference_data):
references = []
for ref in reference_data:
url = ref["url"]
if "GHSA-" in url.upper():
reference = Reference(url=url, reference_id=url.split("/")[-1])
else:
reference = Reference(url=url)
references.append(reference)
return references
def _load_advisories(
pkg_infos: Mapping[str, Any],
) -> List[Advisory]:
advisories = []
for fixed_vulns in pkg_infos["secfixes"].values():
if fixed_vulns is None:
continue
for vuln_ids in fixed_vulns:
vuln_ids = vuln_ids.split()
references = []
for reference_id in vuln_ids[1:]:
if reference_id.startswith("XSA"):
xsa_id = reference_id.split("-")[-1]
references.append(
Reference(
reference_id=reference_id,
url="https://xenbits.xen.org/xsa/advisory-{}.html".format(xsa_id),
)
)
elif reference_id.startswith("ZBX"):
references.append(
Reference(
reference_id=reference_id,
url="https://support.zabbix.com/browse/{}".format(reference_id),
)
)
elif reference_id.startswith("wnpa-sec"):
references.append(
Reference(
reference_id=reference_id,
url="https://www.wireshark.org/security/{}.html".format(
reference_id
),
)
)
# TODO: Handle the CVE-????-????? case
advisories.append(
Advisory(
summary="",
references=references,
vulnerability_id=vuln_ids[0] if is_cve(vuln_ids[0]) else "",
)
)
return advisories
def test_to_advisory(self):
data_source = ApacheKafkaDataSource(batch_size=1)
data_source.version_api = GitHubTagsAPI(
cache={"apache/kafka": ["2.1.2", "0.10.2.2"]})
expected_advisories = [
Advisory(
summary=
"In Apache Kafka versions between 0.11.0.0 and 2.1.0, it is possible to "
"manually\n craft a Produce request which bypasses transaction/idempotent ACL "
"validation.\n Only authenticated clients with Write permission on the "
"respective topics are\n able to exploit this vulnerability. Users should "
"upgrade to 2.1.1 or later\n where this vulnerability has been fixed.",
impacted_package_urls=[
PackageURL(
type="apache",
namespace=None,
name="kafka",
version="0.10.2.2",
qualifiers={},
subpath=None,
)
],
resolved_package_urls=[
PackageURL(
type="apache",
namespace=None,
name="kafka",
version="2.1.2",
qualifiers={},
subpath=None,
)
],
references=[
Reference(url="https://kafka.apache.org/cve-list",
reference_id=""),
Reference(
url=
"https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-17196",
reference_id="CVE-2018-17196",
),
],
vulnerability_id="CVE-2018-17196",
)
]
with open(TEST_DATA) as f:
found_advisories = data_source.to_advisory(f)
found_advisories = list(map(Advisory.normalized, found_advisories))
expected_advisories = list(
map(Advisory.normalized, expected_advisories))
assert sorted(found_advisories) == sorted(expected_advisories)
def test_to_advisories(self):
expected_advisories = [
Advisory(
summary=(
"Multiple integer overflows in TCMalloc (tcmalloc.cc) in gperftools "
"before 0.4 make it easier for context-dependent attackers to perform memory-related " # nopep8
"attacks such as buffer overflows via a large size value, which causes less memory to " # nopep8
"be allocated than expected."
),
impacted_package_urls=[],
resolved_package_urls=[],
vuln_references=sorted(
[
Reference(
url="http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", # nopep8
),
Reference(
url="http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", # nopep8
),
Reference(
url="https://nvd.nist.gov/vuln/detail/CVE-2005-4895",
severities=[
VulnerabilitySeverity(
system=scoring_systems["cvssv2"],
value="5.0",
),
VulnerabilitySeverity(
system=scoring_systems["cvssv2_vector"],
value="AV:N/AC:L/Au:N/C:N/I:N/A:P",
),
],
reference_id="CVE-2005-4895",
),
],
key=lambda x: x.url,
),
vulnerability_id="CVE-2005-4895",
)
]
assert len(self.nvd_data["CVE_Items"]) == 2
found_advisories = list(self.data_src.to_advisories(self.nvd_data))
# Only 1 advisory because other advisory is hardware related
assert len(found_advisories) == 1
found_advisories[0].vuln_references = sorted(
found_advisories[0].vuln_references, key=lambda x: x.url
) # nopep8
assert expected_advisories == found_advisories
def to_advisory(self, advisory_page):
advisories = []
advisory_page = BeautifulSoup(advisory_page, features="lxml")
cve_section_beginnings = advisory_page.find_all("h2")
for cve_section_beginning in cve_section_beginnings:
cve_id = cve_section_beginning.text.split("\n")[0]
cve_description_paragraph = cve_section_beginning.find_next_sibling(
"p")
cve_data_table = cve_section_beginning.find_next_sibling("table")
cve_data_table_rows = cve_data_table.find_all("tr")
affected_versions_row = cve_data_table_rows[0]
fixed_versions_row = cve_data_table_rows[1]
affected_version_ranges = to_version_ranges(
affected_versions_row.find_all("td")[1].text)
fixed_version_ranges = to_version_ranges(
fixed_versions_row.find_all("td")[1].text)
fixed_packages = [
PackageURL(type="apache", name="kafka", version=version)
for version in self.version_api.get("apache/kafka") if any([
version in version_range
for version_range in fixed_version_ranges
])
]
affected_packages = [
PackageURL(type="apache", name="kafka", version=version)
for version in self.version_api.get("apache/kafka") if any([
version in version_range
for version_range in affected_version_ranges
])
]
advisories.append(
Advisory(
vulnerability_id=cve_id,
summary=cve_description_paragraph.text,
impacted_package_urls=affected_packages,
resolved_package_urls=fixed_packages,
vuln_references=[
Reference(url=ASF_PAGE_URL),
Reference(
url=
f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve_id}",
reference_id=cve_id,
),
],
))
return advisories
def _parse(self, record) -> List[Advisory]:
advisories = []
for cve_id in record["issues"]:
impacted_purls, resolved_purls = set(), set()
for name in record["packages"]:
impacted_purls.add(
PackageURL(
name=name,
type="pacman",
namespace="archlinux",
version=record["affected"],
))
if record["fixed"]:
resolved_purls.add(
PackageURL(
name=name,
type="pacman",
namespace="archlinux",
version=record["fixed"],
))
vuln_references = []
vuln_references.append(
Reference(
reference_id=record["name"],
url="https://security.archlinux.org/{}".format(
record["name"]),
))
for ref in record["advisories"]:
vuln_references.append(
Reference(
reference_id=ref,
url="https://security.archlinux.org/{}".format(ref),
))
advisories.append(
Advisory(
cve_id=cve_id,
summary="",
impacted_package_urls=impacted_purls,
resolved_package_urls=resolved_purls,
vuln_references=vuln_references,
))
return advisories
def process_file(self, file) -> List[Advisory]:
record = load_json(file)
advisories = []
package_name = record["module_name"].strip()
all_versions = self.versions.get(package_name)
aff_range = record.get("vulnerable_versions", "")
fixed_range = record.get("patched_versions", "")
impacted_versions, resolved_versions = categorize_versions(
all_versions, aff_range, fixed_range
)
impacted_purls = _versions_to_purls(package_name, impacted_versions)
resolved_purls = _versions_to_purls(package_name, resolved_versions)
vuln_reference = [
Reference(
url=NPM_URL.format(f'/-/npm/v1/advisories/{record["id"]}'),
reference_id=record["id"],
)
]
for cve_id in record.get("cves") or [""]:
advisories.append(
Advisory(
summary=record.get("overview", ""),
cve_id=cve_id,
impacted_package_urls=impacted_purls,
resolved_package_urls=resolved_purls,
vuln_references=vuln_reference,
)
)
return advisories
def process_file(self, path) -> List[Advisory]:
with open(path) as f:
json_doc = json.load(f)
if self.vuln_id_from_desc(json_doc["description"]):
vuln_id = self.vuln_id_from_desc(json_doc["description"])
else:
return
affected_packages = []
for pkg in json_doc["packages"]:
affected_packages.append(
AffectedPackage(
vulnerable_package=PackageURL(name=pkg["id"],
version=pkg["affected"],
type="nuget"),
patched_package=PackageURL(name=pkg["id"],
version=pkg["fix"],
type="nuget"),
))
vuln_reference = [Reference(url=json_doc["link"], )]
return Advisory(
vulnerability_id=vuln_id,
summary=json_doc["description"],
affected_packages=affected_packages,
references=vuln_reference,
)
def updated_advisories(self) -> Set[Advisory]:
advisories = []
for package_name in self._api_response:
all_package_versions = self.versions.get(package_name)
if len(all_package_versions) == 0:
# PyPi does not have data about this package, we skip these
continue
for advisory in self._api_response[package_name]:
impacted_purls, resolved_purls = categorize_versions(
package_name, all_package_versions, advisory["specs"])
cve_ids = advisory.get("cve") or [""]
# meaning if cve_ids is not [''] but either ['CVE-123'] or ['CVE-123, CVE-124']
if len(cve_ids[0]):
cve_ids = [s.strip() for s in cve_ids.split(",")]
reference = [Reference(reference_id=advisory["id"])]
for cve_id in cve_ids:
advisories.append(
Advisory(
cve_id=cve_id,
summary=advisory["advisory"],
vuln_references=reference,
impacted_package_urls=impacted_purls,
resolved_package_urls=resolved_purls,
))
return self.batch_advisories(advisories)
def yaml_file_to_advisory(yaml_path):
impacted_packages = []
resolved_packages = []
references = []
data = load_yaml(yaml_path)
vuln_id = data["vulnerability_id"]
summary = "\n".join([note["text"] for note in data["notes"]])
for entry in data.get("artifacts", []):
package = PackageURL.from_string(entry["id"])
if entry["affected"]:
impacted_packages.append(package)
else:
resolved_packages.append(package)
for fix in data.get("fixes", []):
for commit in fix["commits"]:
references.append(
Reference(url=f"{commit['repository']}/{commit['id']}"))
return Advisory(
vulnerability_id=vuln_id,
summary=summary,
impacted_package_urls=impacted_packages,
resolved_package_urls=resolved_packages,
references=references,
)
def test_process_file(self):
exp_data = [
Advisory(
summary=(
'A command injection vulnerability in '
'Subversion may allow remote\n '
'attackers to execute arbitrary code.\n '),
impacted_package_urls={
PackageURL(
type='ebuild',
namespace='dev-vcs',
name='subversion',
version='0.1.1',
qualifiers=OrderedDict(),
subpath=None)},
resolved_package_urls={
PackageURL(
type='ebuild',
namespace='dev-vcs',
name='subversion',
version='1.9.7',
qualifiers=OrderedDict(),
subpath=None)},
vuln_references=[
Reference(
url='https://security.gentoo.org/glsa/201709-09',
reference_id='GLSA-201709-09')],
cve_id='CVE-2017-9800')]
found_data = self.data_src.process_file(TEST_DATA)
assert exp_data == found_data
def process_file(self, path) -> List[Advisory]:
with open(path) as f:
json_doc = json.load(f)
if self.vuln_id_from_desc(json_doc["description"]):
vuln_id = self.vuln_id_from_desc(json_doc["description"])
else:
return
affected_purls = set()
fixed_purls = set()
for pkg in json_doc['packages']:
affected_purls.add(
PackageURL(name=pkg['id'],
version=pkg['affected'],
type='nuget'))
fixed_purls.add(
PackageURL(name=pkg['id'],
version=pkg['fix'],
type='nuget'))
vuln_reference = [Reference(url=json_doc['link'], )]
return Advisory(summary=json_doc['description'],
impacted_package_urls=affected_purls,
resolved_package_urls=fixed_purls,
cve_id=vuln_id,
vuln_references=vuln_reference)
def process_file(self, path) -> List[Advisory]:
with open(path) as f:
json_doc = json.load(f)
if self.vuln_id_from_desc(json_doc["description"]):
vuln_id = self.vuln_id_from_desc(json_doc["description"])
else:
return
affected_purls = set()
fixed_purls = set()
for pkg in json_doc["packages"]:
affected_purls.add(
PackageURL(name=pkg["id"], version=pkg["affected"], type="nuget")
)
fixed_purls.add(PackageURL(name=pkg["id"], version=pkg["fix"], type="nuget"))
vuln_reference = [
Reference(
url=json_doc["link"],
)
]
return Advisory(
summary=json_doc["description"],
impacted_package_urls=affected_purls,
resolved_package_urls=fixed_purls,
vulnerability_id=vuln_id,
references=vuln_reference,
)
def process_file(self, path):
yaml_file = load_yaml(path)
pkg_name = yaml_file["package"]
safe_pkg_versions = []
vuln_pkg_versions = []
if not yaml_file.get("patched_versions"):
yaml_file["patched_versions"] = []
if not yaml_file.get("unaffected_versions"):
yaml_file["unaffected_versions"] = []
safe_pkg_versions, vuln_pkg_versions = self.get_versions_for_pkg_from_range_list(
yaml_file["patched_versions"] + yaml_file["unaffected_versions"],
pkg_name,
)
if yaml_file.get("cve"):
cve_id = "CVE-" + yaml_file["cve"]
else:
cve_id = ""
safe_purls = []
vuln_purls = []
safe_purls = [
PackageURL(name=pkg_name, type="hex", version=version) for version in safe_pkg_versions
]
vuln_purls = [
PackageURL(name=pkg_name, type="hex", version=version) for version in vuln_pkg_versions
]
references = [
Reference(
reference_id=yaml_file["id"],
),
Reference(
url=yaml_file["link"],
),
]
return Advisory(
summary=yaml_file["description"],
affected_packages=nearest_patched_package(vuln_purls, safe_purls),
vulnerability_id=cve_id,
references=references,
)
def to_advisories(xml_response: str) -> Set[Advisory]:
advisories = []
pkg_name = "openssl"
pkg_type = "generic"
root = ET.fromstring(xml_response)
for element in root:
if element.tag == "issue":
cve_id = ""
summary = ""
safe_pkg_versions = []
vuln_pkg_versions = []
ref_urls = []
for info in element:
if info.tag == "cve":
if info.attrib.get("name"):
cve_id = "CVE-" + info.attrib.get("name")
else:
continue
if info.tag == "affects":
# Vulnerable package versions
vuln_pkg_versions.append(info.attrib.get("version"))
if info.tag == "fixed":
# Fixed package versions
safe_pkg_versions.append(info.attrib.get("version"))
if info:
commit_hash = info[0].attrib["hash"]
ref_urls.append(
Reference(
url=
"https://github.com/openssl/openssl/commit/"
+ commit_hash))
if info.tag == "description":
# Description
summary = re.sub(r"\s+", " ", info.text).strip()
safe_purls = [
PackageURL(name=pkg_name, type=pkg_type, version=version)
for version in safe_pkg_versions
]
vuln_purls = [
PackageURL(name=pkg_name, type=pkg_type, version=version)
for version in vuln_pkg_versions
]
advisory = Advisory(
vulnerability_id=cve_id,
summary=summary,
affected_packages=nearest_patched_package(
vuln_purls, safe_purls),
references=ref_urls,
)
advisories.append(advisory)
return advisories
def test_get_usn_references(self):
eg_usn = "435-1"
expected_references = Reference(reference_id="USN-435-1",
url="https://usn.ubuntu.com/435-1/")
found_references = ubuntu_usn.get_usn_references(eg_usn)
assert found_references == expected_references
def to_advisories(self, apache_tomcat_advisory_html):
advisories = []
page_soup = BeautifulSoup(apache_tomcat_advisory_html, features="lxml")
pageh3s = page_soup.find_all("h3")
vuln_headings = [
i for i in pageh3s if "Fixed in Apache Tomcat" in i.text
]
for data in vuln_headings:
fixed_version = data.text.split(
"Fixed in Apache Tomcat")[-1].strip()
details_div = data.find_next_sibling()
for anchor_tag in details_div.find_all("a"):
if "cve.mitre.org" not in anchor_tag["href"]:
continue
cve_id = re.search(r"CVE-\d*-\d*", anchor_tag.text).group()
references = []
affected_packages = []
paragraph = anchor_tag.find_parent()
while paragraph and "Affects:" not in paragraph.text:
for ref in paragraph.find_all("a"):
references.append(Reference(url=ref["href"]))
paragraph = paragraph.find_next_sibling()
if not paragraph:
# At the end of details_div
continue
for version_range in parse_version_ranges(paragraph.text):
affected_packages.extend([
PackageURL(type="maven",
namespace="apache",
name="tomcat",
version=version) for version in
self.version_api.get("org.apache.tomcat:tomcat")
if version in version_range
])
fixed_package = [
PackageURL(type="maven",
namespace="apache",
name="tomcat",
version=fixed_version)
]
advisories.append(
Advisory(
summary="",
impacted_package_urls=affected_packages,
resolved_package_urls=fixed_package,
vulnerability_id=cve_id,
vuln_references=references,
))
return advisories
def to_advisories(data):
advisories = []
soup = BeautifulSoup(data, features="lxml")
table = soup.select("table")[0]
for row in table.select("tbody tr"):
ref_col, affected_col, fixed_col, severity_score_col, desc_col = row.select("td")
summary = desc_col.text
pkg_qualifiers = {}
if "windows" in summary.lower():
pkg_qualifiers = {"os": "windows"}
affected_packages = [
PackageURL(
type="generic",
name="postgresql",
version=version.strip(),
qualifiers=pkg_qualifiers,
)
for version in affected_col.text.split(",")
]
fixed_packages = [
PackageURL(
type="generic",
name="postgresql",
version=version.strip(),
qualifiers=pkg_qualifiers,
)
for version in fixed_col.text.split(",")
]
try:
cve_id = ref_col.select("nobr")[0].text
# This is for the anomaly in https://www.postgresql.org/support/security/8.1/ 's
# last entry
except IndexError:
pass
references = []
for a_tag in ref_col.select("a"):
link = a_tag.attrs["href"]
if link.startswith("/about/news/"):
# Convert postgresql official announcements to absolute url.
link = urljoin(BASE_URL, link)
references.append(Reference(url=link))
advisories.append(
Advisory(
vulnerability_id=cve_id,
summary=summary,
references=references,
impacted_package_urls=affected_packages,
resolved_package_urls=fixed_packages,
)
)
return advisories
def _parse(self, pkg_name: str, records: Mapping[str,
Any]) -> List[Advisory]:
advisories = []
for cve_id, record in records.items():
impacted_purls, resolved_purls = set(), set()
if not cve_id.startswith("CVE"):
continue
# vulnerabilities starting with something else may not be public yet
# see for instance https://web.archive.org/web/20201215213725/https://security-tracker.debian.org/tracker/TEMP-0000000-A2EB44 # nopep8
# TODO: this would need to be revisited though to ensure we are not missing out on anything # nopep8
for release_name, release_record in record["releases"].items():
if not release_record.get("repositories",
{}).get(release_name):
continue
purl = PackageURL(
name=pkg_name,
type="deb",
namespace="debian",
version=release_record["repositories"][release_name],
qualifiers={"distro": release_name},
)
if release_record.get("status", "") == "resolved":
resolved_purls.add(purl)
else:
impacted_purls.add(purl)
if "fixed_version" in release_record:
resolved_purls.add(
PackageURL(
name=pkg_name,
type="deb",
namespace="debian",
version=release_record["fixed_version"],
qualifiers={"distro": release_name},
))
references = []
debianbug = record.get("debianbug")
if debianbug:
bug_url = f"https://bugs.debian.org/cgi-bin/bugreport.cgi?bug={debianbug}"
references.append(
Reference(url=bug_url, reference_id=debianbug))
advisories.append(
Advisory(
vulnerability_id=cve_id,
summary=record.get("description", ""),
impacted_package_urls=impacted_purls,
resolved_package_urls=resolved_purls,
references=references,
))
return advisories
def process_response(self) -> List[Advisory]:
adv_list = []
for ecosystem in self.advisories:
self.set_version_api(ecosystem)
pkg_type = ecosystem.lower()
for resp_page in self.advisories[ecosystem]:
for adv in resp_page["data"]["securityVulnerabilities"]["edges"]:
name = adv["node"]["package"]["name"]
if self.process_name(ecosystem, name):
ns, pkg_name = self.process_name(ecosystem, name)
else:
continue
aff_range = adv["node"]["vulnerableVersionRange"]
aff_vers, unaff_vers = self.categorize_versions(
aff_range, self.version_api.get(name)
)
affected_purls = {
PackageURL(name=pkg_name, namespace=ns,
version=version, type=pkg_type)
for version in aff_vers
}
unaffected_purls = {
PackageURL(name=pkg_name, namespace=ns,
version=version, type=pkg_type)
for version in unaff_vers
}
cve_ids = set()
vuln_references = []
vuln_desc = adv["node"]["advisory"]["summary"]
for vuln in adv["node"]["advisory"]["identifiers"]:
if vuln["type"] == "CVE":
cve_ids.add(vuln["value"])
elif vuln["type"] == "GHSA":
ghsa = vuln['value']
vuln_references.append(Reference(
reference_id=ghsa,
url="https://github.com/advisories/{}".format(
ghsa)
))
for cve_id in cve_ids:
adv_list.append(
Advisory(
cve_id=cve_id,
summary=vuln_desc,
impacted_package_urls=affected_purls,
resolved_package_urls=unaffected_purls,
vuln_references=vuln_references,
)
)
return adv_list
def process_file(self, path) -> List[Advisory]:
record = load_yaml(path)
package_name = record.get("gem")
if not package_name:
return
if "cve" in record:
cve_id = "CVE-{}".format(record["cve"])
else:
return
safe_version_ranges = record.get("patched_versions", [])
# this case happens when the advisory contain only 'patched_versions' field
# and it has value None(i.e it is empty :( ).
if not safe_version_ranges:
safe_version_ranges = []
safe_version_ranges += record.get("unaffected_versions", [])
safe_version_ranges = [i for i in safe_version_ranges if i]
if not getattr(self, "pkg_manager_api", None):
self.pkg_manager_api = RubyVersionAPI()
all_vers = self.pkg_manager_api.get(package_name)
safe_versions, affected_versions = self.categorize_versions(
all_vers, safe_version_ranges)
impacted_purls = {
PackageURL(
name=package_name,
type="gem",
version=version,
)
for version in affected_versions
}
resolved_purls = {
PackageURL(
name=package_name,
type="gem",
version=version,
)
for version in safe_versions
}
references = []
if record.get("url"):
references.append(Reference(url=record.get("url")))
return Advisory(
summary=record.get("description", ""),
impacted_package_urls=impacted_purls,
resolved_package_urls=resolved_purls,
references=references,
vulnerability_id=cve_id,
)
def process_file(self, path) -> List[Advisory]:
record = load_yaml(path)
package_name = record.get('gem')
if not package_name:
return
if 'cve' in record:
cve_id = 'CVE-{}'.format(record['cve'])
else:
return
safe_version_ranges = record.get('patched_versions', [])
# this case happens when the advisory contain only 'patched_versions' field
# and it has value None(i.e it is empty :( ).
if not safe_version_ranges:
safe_version_ranges = []
safe_version_ranges += record.get('unaffected_versions', [])
safe_version_ranges = [i for i in safe_version_ranges if i]
if not getattr(self, 'pkg_manager_api', None):
self.pkg_manager_api = RubyVersionAPI()
all_vers = self.pkg_manager_api.get(package_name)
safe_versions, affected_versions = self.categorize_versions(
all_vers, safe_version_ranges)
impacted_purls = {
PackageURL(
name=package_name,
type='gem',
version=version,
)
for version in affected_versions
}
resolved_purls = {
PackageURL(
name=package_name,
type='gem',
version=version,
)
for version in safe_versions
}
references = []
if record.get('url'):
references.append(Reference(url=record.get('url')))
return Advisory(summary=record.get('description', ''),
impacted_package_urls=impacted_purls,
resolved_package_urls=resolved_purls,
vuln_references=references,
cve_id=cve_id)
def to_advisories(self, data):
advisories = []
soup = BeautifulSoup(data, features="lxml")
vuln_list = soup.select("li p")
# Example value of `vuln_list` :
# ['Excessive CPU usage in HTTP/2 with small window updates',
# <br/>,
# 'Severity: medium',
# <br/>,
# <a href="http://mailman.nginx.org/pipermail/nginx-announce/2019/000249.html">Advisory</a>, # nopep8
# <br/>,
# <a href="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-9511">CVE-2019-9511</a>,
# <br/>,
# 'Not vulnerable: 1.17.3+, 1.16.1+',
# <br/>,
# 'Vulnerable: 1.9.5-1.17.2']
for vuln_info in vuln_list:
references = []
for index, child in enumerate(vuln_info.children):
if index == 0:
# type of this child is bs4.element.NavigableString.
# Hence cast it into standard string
summary = str(child)
continue
# hasattr(child, "attrs") == False for bs4.element.NavigableString
if hasattr(child, "attrs") and child.attrs.get("href"):
link = child.attrs["href"]
references.append(Reference(url=link))
if "cve.mitre.org" in link:
cve_id = child.text
continue
if "Not vulnerable" in child:
fixed_packages = self.extract_fixed_pkgs(child)
continue
if "Vulnerable" in child:
vulnerable_packages = self.extract_vuln_pkgs(child)
continue
advisories.append(
Advisory(
vulnerability_id=cve_id,
summary=summary,
affected_packages=nearest_patched_package(vulnerable_packages, fixed_packages),
)
)
return advisories
def test__process_link(self):
expected_advisories = [
Advisory(
summary="",
references=[],
vulnerability_id="CVE-2019-14904",
),
Advisory(
summary="",
references=[],
vulnerability_id="CVE-2019-14905",
),
Advisory(
summary="",
references=[],
vulnerability_id="CVE-2019-14846",
),
Advisory(
summary="",
references=[],
vulnerability_id="CVE-2019-14856",
),
Advisory(
summary="",
references=[],
vulnerability_id="CVE-2019-14858",
),
Advisory(
summary="",
references=[
Reference(
url="https://xenbits.xen.org/xsa/advisory-295.html",
reference_id="XSA-295")
],
vulnerability_id="",
),
]
mock_requests = MagicMock()
mock_content = MagicMock()
with open(os.path.join(TEST_DATA, "main.yaml")) as f:
mock_requests.get = lambda x: mock_content
mock_content.content = f
with patch("vulnerabilities.importers.alpine_linux.requests",
new=mock_requests):
found_advisories = self.data_source._process_link(
"does not matter")
found_advisories = list(
map(Advisory.normalized, found_advisories))
expected_advisories = list(
map(Advisory.normalized, expected_advisories))
assert sorted(found_advisories) == sorted(expected_advisories)
