def test_categorize_versions_without_affected_ranges():
all_versions = {"1.0", "1.1", "2.0", "2.1", "3.0", "3.1"}
unaffected_ranges = [
VersionSpecifier.from_scheme_version_spec_string("semver", "< 1.2")
]
affected_ranges = []
resolved_ranges = [
VersionSpecifier.from_scheme_version_spec_string("semver", ">= 3.0")
]
unaffected_versions, affected_versions = categorize_versions(
all_versions,
unaffected_ranges,
affected_ranges,
resolved_ranges,
)
assert len(unaffected_versions) == 4
assert "1.0" in unaffected_versions
assert "1.1" in unaffected_versions
assert "3.0" in unaffected_versions
assert "3.1" in unaffected_versions
assert len(affected_versions) == 2
assert "2.0" in affected_versions
assert "2.1" in affected_versions
def to_version_ranges(version_range_text):
version_ranges = []
range_expressions = version_range_text.split(",")
for range_expression in range_expressions:
if "to" in range_expression:
# eg range_expression == "3.2.0 to 3.2.1"
lower_bound, upper_bound = range_expression.split("to")
lower_bound = f">={lower_bound}"
upper_bound = f"<={upper_bound}"
version_ranges.append(
VersionSpecifier.from_scheme_version_spec_string(
"maven", f"{lower_bound},{upper_bound}"))
elif "and later" in range_expression:
# eg range_expression == "2.1.1 and later"
range_expression = range_expression.replace("and later", "")
version_ranges.append(
VersionSpecifier.from_scheme_version_spec_string(
"maven", f">={range_expression}"))
else:
# eg range_expression == "3.0.0"
version_ranges.append(
VersionSpecifier.from_scheme_version_spec_string(
"maven", range_expression))
return version_ranges
def test_categorize_versions():
flatbuffers_versions = MOCKED_CRATES_API_VERSIONS["flatbuffers"]
unaffected_ranges = [
VersionSpecifier.from_scheme_version_spec_string("semver", "< 0.4.0")
]
affected_ranges = [
VersionSpecifier.from_scheme_version_spec_string("semver", ">= 0.4.0"),
VersionSpecifier.from_scheme_version_spec_string("semver", "<= 0.6.0"),
]
resolved_ranges = [
VersionSpecifier.from_scheme_version_spec_string("semver", ">= 0.6.1")
]
unaffected_versions, affected_versions = categorize_versions(
set(flatbuffers_versions),
unaffected_ranges,
affected_ranges,
resolved_ranges,
)
assert len(unaffected_versions) == 2
assert "0.3.0" in unaffected_versions
assert "0.6.5" in unaffected_versions
assert len(affected_versions) == 1
assert "0.5.0" in affected_versions
def test_to_version_ranges(self):
data = [
{
"version_affected": "?=",
"version_value": "1.3.0",
},
{
"version_affected": "=",
"version_value": "1.3.1",
},
{
"version_affected": "<",
"version_value": "1.3.2",
},
]
fixed_version_ranges, affected_version_ranges = self.data_src.to_version_ranges(
data)
# Check fixed packages
assert [
VersionSpecifier.from_scheme_version_spec_string(
"maven", ">=1.3.2")
] == fixed_version_ranges
# Check vulnerable packages
assert [
VersionSpecifier.from_scheme_version_spec_string(
"maven", "==1.3.0"),
VersionSpecifier.from_scheme_version_spec_string(
"maven", "==1.3.1"),
] == affected_version_ranges
def categorize_versions(
package_name: str,
all_versions: Set[str],
version_specs: Iterable[str],
) -> Tuple[Set[PackageURL], Set[PackageURL]]:
"""
:return: impacted, resolved purls
"""
impacted_versions, impacted_purls = set(), []
vurl_specs = []
for version_spec in version_specs:
vurl_specs.append(VersionSpecifier.from_scheme_version_spec_string("pypi", version_spec))
for version in all_versions:
try:
version_object = PYPIVersion(version)
except:
continue
if any([version_object in vurl_spec for vurl_spec in vurl_specs]):
impacted_versions.add(version)
impacted_purls.append(
PackageURL(
name=package_name,
type="pypi",
version=version,
)
)
resolved_purls = []
for version in all_versions - impacted_versions:
resolved_purls.append(PackageURL(name=package_name, type="pypi", version=version))
return impacted_purls, resolved_purls
def parse_version_ranges(string):
"""
This method yields VersionSpecifier objects obtained by
parsing `string`.
>>> list(parse_version_ranges("Affects: 9.0.0.M1 to 9.0.0.M9")) == [
... VersionSpecifier.from_scheme_version_spec_string('maven','<=9.0.0.M9,>=9.0.0.M1')
... ]
True
>>> list(parse_version_ranges("Affects: 9.0.0.M1")) == [
... VersionSpecifier.from_scheme_version_spec_string('maven','>=9.0.0.M1,<=9.0.0.M1')
... ]
True
>>> list(parse_version_ranges("Affects: 9.0.0.M1 to 9.0.0.M9, 1.2.3 to 3.4.5")) == [
... VersionSpecifier.from_scheme_version_spec_string('maven','<=9.0.0.M9,>=9.0.0.M1'),
... VersionSpecifier.from_scheme_version_spec_string('maven','<=3.4.5,>=1.2.3')
... ]
True
"""
version_rng_txt = string.split("Affects:")[-1].strip()
version_ranges = version_rng_txt.split(",")
for version_range in version_ranges:
if "to" in version_range:
lower_bound, upper_bound = version_range.split("to")
elif "-" in version_range and not any(
[i.isalpha() for i in version_range]):
lower_bound, upper_bound = version_range.split("-")
else:
lower_bound = upper_bound = version_range
yield VersionSpecifier.from_scheme_version_spec_string(
"maven", f">={lower_bound},<={upper_bound}")
def categorize_versions(
all_versions: Set[str],
affected_version_range: str,
fixed_version_range: str,
) -> Tuple[Set[str], Set[str]]:
"""
Seperate list of affected versions and unaffected versions from all versions
using the ranges specified.
:return: impacted, resolved versions
"""
if not all_versions:
# NPM registry has no data regarding this package, we skip these
return set(), set()
aff_spec = []
fix_spec = []
if affected_version_range:
aff_specs = normalize_ranges(affected_version_range)
aff_spec = [
VersionSpecifier.from_scheme_version_spec_string("semver", spec)
for spec in aff_specs
if len(spec) >= 3
]
if fixed_version_range:
fix_specs = normalize_ranges(fixed_version_range)
fix_spec = [
VersionSpecifier.from_scheme_version_spec_string("semver", spec)
for spec in fix_specs
if len(spec) >= 3
]
aff_ver, fix_ver = set(), set()
# Unaffected version is that version which is in the fixed_version_range
# or which is absent in the affected_version_range
for ver in all_versions:
ver_obj = SemverVersion(ver)
if not any([ver_obj in spec for spec in aff_spec]) or any(
[ver_obj in spec for spec in fix_spec]
):
fix_ver.add(ver)
else:
aff_ver.add(ver)
return aff_ver, fix_ver
def to_version_ranges(self, versions_data):
fixed_version_ranges = []
affected_version_ranges = []
for version_data in versions_data:
version_value = version_data["version_value"]
range_expression = version_data["version_affected"]
if range_expression == "<":
fixed_version_ranges.append(
VersionSpecifier.from_scheme_version_spec_string(
"maven", ">={}".format(version_value)
)
)
elif range_expression == "=" or range_expression == "?=":
affected_version_ranges.append(
VersionSpecifier.from_scheme_version_spec_string(
"maven", "{}".format(version_value)
)
)
return (fixed_version_ranges, affected_version_ranges)
def extract_vuln_pkgs(self, vuln_info):
vuln_status, version_infos = vuln_info.split(": ")
if "none" in version_infos:
return {}
version_ranges = []
windows_only = False
for version_info in version_infos.split(", "):
if version_info == "all":
# This is misleading since eventually some version get fixed.
continue
if "-" not in version_info:
# These are discrete versions
version_ranges.append(
VersionSpecifier.from_scheme_version_spec_string("semver", version_info[0])
)
continue
windows_only = "nginx/Windows" in version_info
version_info = version_info.replace("nginx/Windows", "")
lower_bound, upper_bound = version_info.split("-")
version_ranges.append(
VersionSpecifier.from_scheme_version_spec_string(
"semver", f">={lower_bound},<={upper_bound}"
)
)
valid_versions = find_valid_versions(self.version_api.get("nginx/nginx"), version_ranges)
qualifiers = {}
if windows_only:
qualifiers["os"] = "windows"
return [
PackageURL(type="generic", name="nginx", version=version, qualifiers=qualifiers)
for version in valid_versions
]
def categorize_versions(
package_type: str, version_range: str,
all_versions: Set[str]) -> Tuple[List[str], List[str]]:
version_class = version_class_by_package_type[package_type]
version_scheme = version_class.scheme
version_range = VersionSpecifier.from_scheme_version_spec_string(
version_scheme, version_range)
affected_versions = []
unaffected_versions = []
for version in all_versions:
if version_class(version) in version_range:
affected_versions.append(version)
else:
unaffected_versions.append(version)
return (affected_versions, unaffected_versions)
def get_pkg_versions_from_ranges(self, version_range_list):
"""Takes a list of version ranges(affected) of a package
as parameter and returns a tuple of safe package versions and
vulnerable package versions"""
all_version = self.version_api.get("istio/istio")
safe_pkg_versions = []
vuln_pkg_versions = []
version_ranges = [
VersionSpecifier.from_scheme_version_spec_string("semver", r)
for r in version_range_list
]
for version in all_version:
version_obj = SemverVersion(version)
if any([version_obj in v for v in version_ranges]):
vuln_pkg_versions.append(version)
safe_pkg_versions = set(all_version) - set(vuln_pkg_versions)
return safe_pkg_versions, vuln_pkg_versions
def get_versions_for_pkg_from_range_list(self, version_range_list, pkg_name):
# Takes a list of version ranges(pathced and unaffected) of a package
# as parameter and returns a tuple of safe package versions and
# vulnerable package versions
safe_pkg_versions = []
vuln_pkg_versions = []
all_version_list = self.pkg_manager_api.get(pkg_name)
if not version_range_list:
return [], all_version_list
version_ranges = [
VersionSpecifier.from_scheme_version_spec_string("semver", r)
for r in version_range_list
]
for version in all_version_list:
version_obj = SemverVersion(version)
if any([version_obj in v for v in version_ranges]):
safe_pkg_versions.append(version)
vuln_pkg_versions = set(all_version_list) - set(safe_pkg_versions)
return safe_pkg_versions, vuln_pkg_versions
def categorize_versions(all_versions, unaffected_version_ranges):
for id, elem in enumerate(unaffected_version_ranges):
unaffected_version_ranges[
id] = VersionSpecifier.from_scheme_version_spec_string(
"semver", elem)
safe_versions = []
vulnerable_versions = []
for i in all_versions:
vobj = SemverVersion(i)
is_vulnerable = False
for ver_rng in unaffected_version_ranges:
if vobj in ver_rng:
safe_versions.append(i)
is_vulnerable = True
break
if not is_vulnerable:
vulnerable_versions.append(i)
return safe_versions, vulnerable_versions
def extract_fixed_pkgs(self, vuln_info):
vuln_status, version_info = vuln_info.split(": ")
if "none" in version_info:
return {}
raw_ranges = version_info.split(",")
version_ranges = []
for rng in raw_ranges:
# Eg. "1.7.3+" gets converted to VersionSpecifier.from_scheme_version_spec_string("semver","^1.7.3")
# The advisory in this case uses `+` in the sense that any version
# with greater or equal `minor` version satisfies the range.
# "1.7.4" satisifes "1.7.3+", but "1.8.4" does not. "1.7.3+" has same
# semantics as that of "^1.7.3"
version_ranges.append(
VersionSpecifier.from_scheme_version_spec_string("semver", "^" + rng[:-1])
)
valid_versions = find_valid_versions(self.version_api.get("nginx/nginx"), version_ranges)
return [
PackageURL(type="generic", name="nginx", version=version) for version in valid_versions
]
def build_report(package_vulnerability_index, detected_packages):
report = []
_package_index = {}
for index, package in enumerate(detected_packages):
if package not in _package_index:
report.append({
"package_name": package.name,
"package_version": package.version,
"vulnerabilities": []
})
_package_index[package.name] = index
else:
index = _package_index[package.name]
if package.name in package_vulnerability_index:
version_object = PYPIVersion(package.version)
for vulnerability_data in package_vulnerability_index[
package.name]:
if version_object in VersionSpecifier.from_scheme_version_spec_string(
"pypi", vulnerability_data["version_range"]):
report[index]["vulnerabilities"].extend(
vulnerability_data["vulnerabilities"])
return json.dumps(report, indent=4)
def test_to_version_ranges(self):
# Check single version
assert [
VersionSpecifier.from_scheme_version_spec_string("maven", "=3.2.2")
] == to_version_ranges("3.2.2")
# Check range with lower and upper bounds
assert [
VersionSpecifier.from_scheme_version_spec_string("maven", ">=3.2.2, <=3.2.3")
] == to_version_ranges("3.2.2 to 3.2.3")
# Check range with "and later"
assert [
VersionSpecifier.from_scheme_version_spec_string("maven", ">=3.2.2")
] == to_version_ranges("3.2.2 and later")
# Check combination of above cases
assert [
VersionSpecifier.from_scheme_version_spec_string("maven", ">=3.2.2"),
VersionSpecifier.from_scheme_version_spec_string("maven", ">=3.2.2, <=3.2.3"),
VersionSpecifier.from_scheme_version_spec_string("maven", "==3.2.2"),
] == to_version_ranges("3.2.2 and later, 3.2.2 to 3.2.3, 3.2.2")
def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> List[Advisory]:
"""
The orchestration method of the OvalDataSource. This method breaks an
OVAL xml ElementTree into a list of `Advisory`.
Note: pkg_metadata is a mapping of Package URL data that MUST INCLUDE
"type" key.
Example value of pkg_metadata:
{"type":"deb","qualifiers":{"distro":"buster"} }
"""
all_adv = []
oval_doc = OvalParser(self.translations, xml_doc)
raw_data = oval_doc.get_data()
all_pkgs = self._collect_pkgs(raw_data)
self.set_api(all_pkgs)
# convert definition_data to Advisory objects
for definition_data in raw_data:
# These fields are definition level, i.e common for all elements
# connected/linked to an OvalDefinition
vuln_id = definition_data["vuln_id"]
description = definition_data["description"]
references = [Reference(url=url) for url in definition_data["reference_urls"]]
affected_packages = []
for test_data in definition_data["test_data"]:
for package_name in test_data["package_list"]:
if package_name and len(package_name) >= 50:
continue
affected_version_range = test_data["version_ranges"] or set()
version_class = version_class_by_package_type[pkg_metadata["type"]]
version_scheme = version_class.scheme
affected_version_range = VersionSpecifier.from_scheme_version_spec_string(
version_scheme, affected_version_range
)
all_versions = self.pkg_manager_api.get(package_name)
# FIXME: what is this 50 DB limit? that's too small for versions
# FIXME: we should not drop data this way
# This filter is for filtering out long versions.
# 50 is limit because that's what db permits atm.
all_versions = [version for version in all_versions if len(version) < 50]
if not all_versions:
continue
affected_purls = []
safe_purls = []
for version in all_versions:
purl = self.create_purl(
pkg_name=package_name,
pkg_version=version,
pkg_data=pkg_metadata,
)
if version_class(version) in affected_version_range:
affected_purls.append(purl)
else:
safe_purls.append(purl)
affected_packages.extend(
nearest_patched_package(affected_purls, safe_purls),
)
all_adv.append(
Advisory(
summary=description,
affected_packages=affected_packages,
vulnerability_id=vuln_id,
references=references,
)
)
return all_adv
def _load_advisory(self, path: str) -> Optional[Advisory]:
record = get_advisory_data(path)
advisory = record.get("advisory", {})
crate_name = advisory["package"]
references = []
if advisory.get("url"):
references.append(Reference(url=advisory["url"]))
all_versions = self.crates_api.get(crate_name)
# FIXME: Avoid wildcard version ranges for now.
# See https://github.com/RustSec/advisory-db/discussions/831
affected_ranges = [
VersionSpecifier.from_scheme_version_spec_string("semver", r)
for r in chain.from_iterable(
record.get("affected", {}).get("functions", {}).values())
if r != "*"
]
unaffected_ranges = [
VersionSpecifier.from_scheme_version_spec_string("semver", r)
for r in record.get("versions", {}).get("unaffected", [])
if r != "*"
]
resolved_ranges = [
VersionSpecifier.from_scheme_version_spec_string("semver", r)
for r in record.get("versions", {}).get("patched", []) if r != "*"
]
unaffected, affected = categorize_versions(all_versions,
unaffected_ranges,
affected_ranges,
resolved_ranges)
impacted_purls = [
PackageURL(type="cargo", name=crate_name, version=v)
for v in affected
]
resolved_purls = [
PackageURL(type="cargo", name=crate_name, version=v)
for v in unaffected
]
cve_id = None
if "aliases" in advisory:
for alias in advisory["aliases"]:
if alias.startswith("CVE-"):
cve_id = alias
break
references.append(
Reference(
reference_id=advisory["id"],
url="https://rustsec.org/advisories/{}.html".format(
advisory["id"]),
))
return Advisory(
summary=advisory.get("description", ""),
affected_packages=nearest_patched_package(impacted_purls,
resolved_purls),
vulnerability_id=cve_id,
references=references,
)