def test_have_vulnerabilities_for(test_data_env):
_init_distro_mappings()
sync_feeds(test_data_env)
failz = DistroNamespace(name='somecrzy', version='8', like_distro='debian')
passes = DistroNamespace(name='debian', version='8', like_distro='debian')
assert not vulnerabilities.have_vulnerabilities_for(failz), 'Should not have vulns for ' + failz.namespace_name
assert vulnerabilities.have_vulnerabilities_for(passes), 'Should have vulns for ' + passes.namespace_name
def test_relation_mapping(self):
for d in self.valid_distros:
ns = DistroNamespace(name=d[0], version=d[1])
self.assertIsNotNone(ns)
self.assertEqual(ns.name, d[0])
self.assertEqual(ns.version, d[1])
for d in self.unmapped_distros:
ns = DistroNamespace(name=d[0], version=d[1])
self.assertIsNotNone(ns)
def test_relation_mapping(anchore_db):
for d in valid_distros:
ns = DistroNamespace(name=d[0], version=d[1])
assert ns is not None
assert ns.name == d[0]
assert ns.version == d[1]
for d in unmapped_distros:
ns = DistroNamespace(name=d[0], version=d[1])
assert ns is not None
def match_and_vulnerable(vuln_obj, package_obj):
"""
Given a VulnerableArtifact record, is the given package object a match indicating that the package is vulnerable.
:param vuln_obj:
:param package_obj:
:param has_fix: boolean indicating if there is a corresponding fix record
:return:
"""
if not isinstance(vuln_obj, VulnerableArtifact):
raise TypeError('Expected a VulnerableArtifact type, got: {}'.format(
type(vuln_obj)))
dist = DistroNamespace.for_obj(package_obj)
flavor = dist.flavor
# Double-check names
if vuln_obj.name != package_obj.name and vuln_obj.name != package_obj.normalized_src_pkg:
log.warn(
'Name mismatch in vulnerable check. This should not happen: Fix: {}, Package: {}, Package_Norm_Src: {}, Package_Src: {}'
.format(vuln_obj.name, package_obj.name,
package_obj.normalized_src_pkg, package_obj.src_pkg))
return False
# Is it a catch-all record? Explicit 'None' or 'all' versions indicate all versions of the named package are vulnerable.
if vuln_obj.epochless_version in ['all', 'None']:
return True
# Is the package older than the fix?
if package_obj.fullversion == vuln_obj.epochless_version or package_obj.version == vuln_obj.epochless_version:
return True
# Newer or the same
return False
def evaluate(self, image_obj, context):
# Map to a namespace
ns = DistroNamespace.for_obj(image_obj)
oldest_update = None
if ns:
vulnerability_feed = DataFeeds.instance().vulnerabilities
for namespace_name in ns.like_namespace_names:
# Check feed names
groups = vulnerability_feed.group_by_name(namespace_name)
if groups:
# No records yet, but we have the feed, so may just not have any data yet
oldest_update = groups[0].last_sync
break
if self.max_age.value() is not None:
try:
if oldest_update is not None:
oldest_update = calendar.timegm(oldest_update.timetuple())
mintime = time.time() - int(int(self.max_age.value()) * 86400)
if oldest_update < mintime:
self._fire(msg="The vulnerability feed for this image distro is older than MAXAGE ("+str(self.max_age.value())+") days")
else:
self._fire(
msg="The vulnerability feed for this image distro is older than MAXAGE (" + str(self.max_age.value()) + ") days")
except Exception as err:
self._fire(msg="Cannot perform data feed up-to-date check - message from server: " + str(err))
def candidates_for_package(package_obj, distro_namespace=None):
"""
Return all vulnerabilities for the named package with the specified distro. Will apply to any version
of the package. If version is used, will filter to only those for the specified version.
:param package_obj: the package to match against
:param distro_namespace: the DistroNamespace object to match against (typically computed
:return: List of Vulnerabilities
"""
db = get_thread_scoped_session()
if not distro_namespace:
namespace_name = DistroNamespace.for_obj(package_obj).namespace_name
else:
namespace_name = distro_namespace
# Match the namespace and package name or src pkg name
fix_candidates = db.query(FixedArtifact).filter(
FixedArtifact.namespace_name == namespace_name,
or_(FixedArtifact.name == package_obj.name,
FixedArtifact.name == package_obj.normalized_src_pkg)).all()
# Match the namespace and package name or src pkg name
vulnerable_candidates = db.query(VulnerableArtifact).filter(
VulnerableArtifact.namespace_name == namespace_name,
or_(VulnerableArtifact.name == package_obj.name,
VulnerableArtifact.name == package_obj.normalized_src_pkg)).all()
return fix_candidates, vulnerable_candidates
def test_cve_mapping(anchore_db):
for d in distros:
ns = DistroNamespace(name=d[0], version=d[1])
r = {'flavor': ns.flavor, 'namespace_names': ns.like_namespace_names}
print(ns.namespace_name + ' ->' + json.dumps(r, indent=2))
assert ns is not None
assert ns.name == d[0]
assert ns.version == d[1]
def namespace_has_no_feed(name, version):
"""
Returns true if the given namespace has no direct CVE feed and false if it does.
:return: boolean if name,version tuple does not have a feed of its own
"""
ns = DistroNamespace.as_namespace_name(name, version)
found = ThreadLocalFeedGroupNameCache.lookup(
ns) # Returns a tuple (name, enabled:bool)
return not found or not found[1]
def test_cve_mapping(self):
for d in self.distros:
ns = DistroNamespace(name=d[0], version=d[1])
r = {
'flavor': ns.flavor,
'namespace_names': ns.like_namespace_names
}
print(ns.namespace_name + ' ->' + json.dumps(r, indent=2))
self.assertIsNotNone(ns)
self.assertEqual(ns.name, d[0])
self.assertEqual(ns.version, d[1])
def match_but_not_fixed(fix_obj, package_obj):
"""
Does the FixedArtifact match the package as a vulnerability such that the fix indicates the package is *not* fixed and is
therefore vulnerable.
:param fix_obj: as FixedArtifact record
:param package_obj: an ImagePackage record
:return: True if the names match and the fix record indicates the package is vulnerable and not fixed. False if no match or fix is applied and no vulnerability match
"""
if not isinstance(fix_obj, FixedArtifact):
raise TypeError('Expected a FixedArtifact type, got: {}'.format(
type(fix_obj)))
dist = DistroNamespace.for_obj(package_obj)
flavor = dist.flavor
log.debug('Package: {}, Fix: {}, Flavor: {}'.format(
package_obj.name, fix_obj.name, flavor))
# Double-check names
if fix_obj.name != package_obj.name and fix_obj.name != package_obj.normalized_src_pkg:
log.warn(
'Name mismatch in fix check. This should not happen: Fix: {}, Package: {}, Package_Norm_Src: {}, Package_Src: {}'
.format(fix_obj.name, package_obj.name,
package_obj.normalized_src_pkg, package_obj.src_pkg))
return False
# Handle the case where there is no version, indicating no fix available, all versions are vulnerable.
# Is it a catch-all record? Explicit 'None' versions indicate all versions of the named package are vulnerable.
if fix_obj.version == 'None':
return True
# Is the package older than the fix?
if flavor == 'RHEL':
if rpm_compare_versions(package_obj.name, package_obj.fullversion,
fix_obj.name, fix_obj.epochless_version) < 0:
log.debug('rpm Compared: {} < {}: True'.format(
package_obj.fullversion, fix_obj.epochless_version))
return True
elif flavor == 'DEB':
if dpkg_compare_versions(package_obj.fullversion, 'lt',
fix_obj.epochless_version):
log.debug('dpkg Compared: {} < {}: True'.format(
package_obj.fullversion, fix_obj.epochless_version))
return True
elif flavor == 'ALPINE':
if apkg_compare_versions(package_obj.fullversion, 'lt',
fix_obj.epochless_version):
log.debug('apkg Compared: {} < {}: True'.format(
package_obj.fullversion, fix_obj.epochless_version))
return True
# Newer or the same
return False
def vulnerabilities_for_package(package_obj):
"""
Given an ImagePackage object, return the vulnerabilities that it matches.
:param package_obj:
:return: list of Vulnerability objects
"""
log.debug('Finding vulnerabilities for package: {} - {}'.format(
package_obj.name, package_obj.version))
matches = []
dist = DistroNamespace(package_obj.distro_name, package_obj.distro_version,
package_obj.like_distro)
db = get_thread_scoped_session()
namespace_name_to_use = dist.namespace_name
# All options are the same, no need to loop
if len(set(dist.like_namespace_names)) > 1:
# Look for exact match first
if not DataFeeds.instance().vulnerabilities.group_by_name(
dist.namespace_name):
# Check all options for distro/flavor mappings, stop at first with records present
for namespace_name in dist.like_namespace_names:
record_count = db.query(Vulnerability).filter(
Vulnerability.namespace_name == namespace_name).count()
if record_count > 0:
namespace_name_to_use = namespace_name
break
fix_candidates, vulnerable_candidates = candidates_for_package(
package_obj, namespace_name_to_use)
for candidate in fix_candidates:
# De-dup evaluations based on the underlying vulnerability_id. For packages where src has many binary builds, once we have a match we have a match.
if candidate.vulnerability_id not in map(
lambda x: x.vulnerability_id, matches) and match_but_not_fixed(
candidate, package_obj):
matches.append(candidate)
for candidate in vulnerable_candidates:
if candidate.vulnerability_id not in map(
lambda x: x.vulnerability_id,
matches) and match_and_vulnerable(candidate, package_obj):
matches.append(candidate)
return matches
def evaluate(self, image_obj, context):
# Map to a namespace
ns = DistroNamespace.for_obj(image_obj)
oldest_update = None
if ns:
for namespace_name in ns.like_namespace_names:
# Check feed names
for feed in feed_registry.registered_vulnerability_feed_names(
):
# First match, assume only one matches for the namespace
group = get_feed_group_detached(feed, namespace_name)
if group:
# No records yet, but we have the feed, so may just not have any data yet
oldest_update = group.last_sync
logger.debug(
"Found date for oldest update in feed %s group %s date = %s",
feed,
group.name,
oldest_update,
)
break
if self.max_age.value() is not None:
try:
if oldest_update is not None:
oldest_update = calendar.timegm(oldest_update.timetuple())
mintime = time.time() - int(
int(self.max_age.value()) * 86400)
if oldest_update < mintime:
self._fire(
msg=
"The vulnerability feed for this image distro is older than MAXAGE ("
+ str(self.max_age.value()) + ") days")
else:
self._fire(
msg=
"The vulnerability feed for this image distro is older than MAXAGE ("
+ str(self.max_age.value()) + ") days")
except Exception as err:
self._fire(
msg=
"Cannot perform data feed up-to-date check - message from server: "
+ str(err))
def test_distromappings(anchore_db):
_init_distro_mappings()
c7 = DistroNamespace(name='centos', version='7', like_distro='centos')
assert c7.mapped_names() == []
assert c7.like_namespace_names == ['rhel:7']
r7 = DistroNamespace(name='rhel', version='7', like_distro='centos')
assert set(r7.mapped_names()) == {'centos', 'fedora', 'rhel'}
assert r7.like_namespace_names == ['rhel:7']
assert sorted(DistroMapping.distros_mapped_to('rhel', '7')) == sorted([
DistroTuple('rhel', '7', 'RHEL'),
DistroTuple('centos', '7', 'RHEL'),
DistroTuple('fedora', '7', 'RHEL')
])
def find_vulnerable_image_packages(vulnerability_obj):
"""
Given a vulnerability object, find images that are affected via their package manifests.
Result may have duplicates based on match type, caller must de-dup if desired.
:param vulnerability_obj:
:return: list of ImagePackage objects
"""
db = get_thread_scoped_session()
distro, version = vulnerability_obj.namespace_name.split(":", 1)
dist = DistroNamespace(distro, version)
related_names = (
dist.mapped_names()
) # Returns list of names that map to this one, not including itself necessarily
# related_names = get_namespace_related_names(distro, version, mapped_names)
# TODO would like a better way to do the pkg_type <-> namespace_name mapping, with other side in ImagePackage.vulnerabilities_for_package
likematch = None
if (":maven" in vulnerability_obj.namespace_name
or "java" in vulnerability_obj.namespace_name):
likematch = "java"
elif (":ruby" in vulnerability_obj.namespace_name
or "gem" in vulnerability_obj.namespace_name):
likematch = "gem"
elif (":js" in vulnerability_obj.namespace_name
or "npm" in vulnerability_obj.namespace_name):
likematch = "npm"
elif "python" in vulnerability_obj.namespace_name:
likematch = "python"
try:
affected = []
if vulnerability_obj.fixed_in:
# Check the fixed_in records
for fix_rec in vulnerability_obj.fixed_in:
package_candidates = []
# Find packages of related distro names with compatible versions, this does not have to be precise, just an initial filter.
pkgs = (db.query(ImagePackage).filter(
ImagePackage.distro_name.in_(related_names),
ImagePackage.distro_version.like(dist.version + "%"),
or_(
ImagePackage.name == fix_rec.name,
ImagePackage.normalized_src_pkg == fix_rec.name,
),
).all())
package_candidates += pkgs
# add non distro candidates
if likematch:
pkgs = (db.query(ImagePackage).filter(
ImagePackage.pkg_type.in_(nonos_package_types),
ImagePackage.pkg_type.like(likematch),
or_(
ImagePackage.name == fix_rec.name,
ImagePackage.normalized_src_pkg == fix_rec.name,
),
).all())
package_candidates += pkgs
for candidate in package_candidates:
if fix_rec.match_but_not_fixed(candidate):
affected.append(candidate)
if vulnerability_obj.vulnerable_in:
# Check the vulnerable_in records
for vuln_rec in vulnerability_obj.vulnerable_in:
package_candidates = []
# Find packages of related distro names with compatible versions, this does not have to be precise, just an initial filter.
pkgs = (db.query(ImagePackage).filter(
ImagePackage.distro_name.in_(related_names),
ImagePackage.distro_version.like(dist.version + "%"),
or_(
ImagePackage.name == vuln_rec.name,
ImagePackage.normalized_src_pkg == vuln_rec.name,
),
).all())
package_candidates += pkgs
for candidate in package_candidates:
if vuln_rec.match_and_vulnerable(candidate):
affected.append(candidate)
return affected
except Exception as e:
logger.exception(
"Failed to query and find packages affected by vulnerability: {}".
format(vulnerability_obj))
raise
def evaluate(self, image_obj, context):
if not have_vulnerabilities_for(DistroNamespace.for_obj(image_obj)):
self._fire(
msg=
"Feed data unavailable, cannot perform CVE scan for distro: " +
str(image_obj.distro_namespace))
def find_vulnerable_image_packages(vulnerability_obj):
"""
Given a vulnerability object, find images that are affected via their package manifests.
Result may have duplicates based on match type, caller must de-dup if desired.
:param vulnerability_obj:
:return: list of ImagePackage objects
"""
db = get_thread_scoped_session()
distro, version = vulnerability_obj.namespace_name.split(':', 1)
dist = DistroNamespace(distro, version)
related_names = dist.mapped_names() # Returns list of names that map to this one, not including itself necessarily
# Filter related_names down based on the presence of actual feeds/cve data. If there is an actual feed for a name, remove it from the list.
# Only do this if the list of related names is not just the name itself. (e.g. alpine = [alpine]).
if related_names != [distro]:
# Ensure we don't include any names that actually have a feed (can happen when new feeds arrive before the mapped_names() source
# is updated to break the 'like' relation between the distros.
related_names = [x for x in related_names if namespace_has_no_feed(x, version)]
# This is a weird case because it basically means that this distro doesn't map to itself as far as mapped_names() is
# concerned, but since that could be code lagging data (e.g. new feed group added for a new distro), add the name itself
# back into the list.
if distro not in related_names and not namespace_has_no_feed(distro, version):
related_names.append(distro)
# TODO would like a better way to do the pkg_type <-> namespace_name mapping, with other side in ImagePackage.vulnerabilities_for_package
likematch = None
if ':maven' in vulnerability_obj.namespace_name or 'java' in vulnerability_obj.namespace_name:
likematch = 'java'
elif ':ruby' in vulnerability_obj.namespace_name or 'gem' in vulnerability_obj.namespace_name:
likematch = 'gem'
elif ':js' in vulnerability_obj.namespace_name or 'npm' in vulnerability_obj.namespace_name:
likematch = 'npm'
elif 'python' in vulnerability_obj.namespace_name:
likematch = 'python'
try:
affected = []
if vulnerability_obj.fixed_in:
# Check the fixed_in records
for fix_rec in vulnerability_obj.fixed_in:
package_candidates = []
# Find packages of related distro names with compatible versions, this does not have to be precise, just an initial filter.
pkgs = db.query(ImagePackage).filter(ImagePackage.distro_name.in_(related_names), ImagePackage.distro_version.like(dist.version + '%'), or_(ImagePackage.name == fix_rec.name, ImagePackage.normalized_src_pkg == fix_rec.name)).all()
package_candidates += pkgs
# add non distro candidates
if likematch:
pkgs = db.query(ImagePackage).filter(ImagePackage.pkg_type.in_(nonos_package_types), ImagePackage.pkg_type.like(likematch), or_(ImagePackage.name == fix_rec.name, ImagePackage.normalized_src_pkg == fix_rec.name)).all()
package_candidates += pkgs
for candidate in package_candidates:
if fix_rec.match_but_not_fixed(candidate):
affected.append(candidate)
# if vulnerability_obj.vulnerable_in:
# # Check the vulnerable_in records
# for vuln_rec in vulnerability_obj.vulnerable_in:
# package_candidates = []
# # Find packages of related distro names with compatible versions, this does not have to be precise, just an initial filter.
# pkgs = db.query(ImagePackage).filter(ImagePackage.distro_name.in_(related_names),
# ImagePackage.distro_version.like(dist.version + '%'),
# or_(ImagePackage.name == fix_rec.name,
# ImagePackage.normalized_src_pkg == fix_rec.name)).all()
# package_candidates += pkgs
# for candidate in package_candidates:
# if vuln_rec.match_and_vulnerable(candidate):
# affected.append(candidate)
return affected
except Exception as e:
log.exception('Failed to query and find packages affected by vulnerability: {}'.format(vulnerability_obj))
raise
def get_image_vulnerabilities(user_id, image_id, force_refresh=False):
"""
Return the vulnerability listing for the specified image and load from catalog if not found and specifically asked
to do so.
Example json output:
{
"multi" : {
"url_column_index" : 7,
"result" : {
"rows" : [],
"rowcount" : 0,
"colcount" : 8,
"header" : [
"CVE_ID",
"Severity",
"*Total_Affected",
"Vulnerable_Package",
"Fix_Available",
"Fix_Images",
"Rebuild_Images",
"URL"
]
},
"querycommand" : "/usr/lib/python2.7/site-packages/anchore/anchore-modules/multi-queries/cve-scan.py /ebs_data/anchore/querytmp/queryimages.7026386 /ebs_data/anchore/data /ebs_data/anchore/querytmp/query.59057288 all",
"queryparams" : "all",
"warns" : [
"0005b136f0fb (prom/prometheus:master) cannot perform CVE scan: no CVE data is currently available for the detected base distro type (busybox:unknown_version,busybox:v1.26.2)"
]
}
}
:param user_id: user id of image to evaluate
:param image_id: image id to evaluate
:param force_refresh: if true, flush and recompute vulnerabilities rather than returning current values
:return:
"""
# Has image?
db = get_session()
try:
img = db.query(Image).get((image_id, user_id))
vulns = []
if not img:
abort(404)
else:
if force_refresh:
log.info('Forcing refresh of vulnerabiltiies for {}/{}'.format(user_id, image_id))
try:
current_vulns = img.vulnerabilities()
log.info('Removing {} current vulnerabilities for {}/{} to rescan'.format(len(current_vulns), user_id, image_id))
for v in current_vulns:
db.delete(v)
db.flush()
vulns = vulnerabilities_for_image(img)
log.info('Adding {} vulnerabilities from rescan to {}/{}'.format(len(vulns), user_id, image_id))
for v in vulns:
db.add(v)
db.commit()
except Exception as e:
log.exception('Error refreshing cve matches for image {}/{}'.format(user_id, image_id))
db.rollback()
abort(Response('Error refreshing vulnerability listing for image.', 500))
db = get_session()
db.refresh(img)
else:
vulns = img.vulnerabilities()
# Has vulnerabilities?
warns = []
if not vulns:
vulns = []
ns = DistroNamespace.for_obj(img)
if not have_vulnerabilities_for(ns):
warns = ['No vulnerability data available for image distro: {}'.format(ns.namespace_name)]
rows = []
for vuln in vulns:
# if vuln.vulnerability.fixed_in:
# fixes_in = filter(lambda x: x.name == vuln.pkg_name or x.name == vuln.package.normalized_src_pkg,
# vuln.vulnerability.fixed_in)
# fix_available_in = fixes_in[0].version if fixes_in else 'None'
# else:
# fix_available_in = 'None'
rows.append([
vuln.vulnerability_id,
vuln.vulnerability.severity,
1,
vuln.pkg_name + '-' + vuln.package.fullversion,
str(vuln.fixed_in()),
vuln.pkg_image_id,
'None', # Always empty this for now
vuln.vulnerability.link
]
)
vuln_listing = {
'multi': {
'url_column_index': 7,
'result': {
'header': TABLE_STYLE_HEADER_LIST,
'rowcount': len(rows),
'colcount': len(TABLE_STYLE_HEADER_LIST),
'rows': rows
},
'warns': warns
}
}
report = LegacyVulnerabilityReport.from_dict(vuln_listing)
resp = ImageVulnerabilityListing(user_id=user_id, image_id=image_id, legacy_report=report)
return resp.to_dict()
except HTTPException:
db.rollback()
raise
except Exception as e:
log.exception('Error checking image {}, {} for vulnerabiltiies. Rolling back'.format(user_id, image_id))
db.rollback()
abort(500)
finally:
db.close()
def test_namespace_support(test_data_env):
_init_distro_mappings()
sync_feeds(test_data_env)
# Not exhaustive, only for the feeds directly in the test data set
expected = [
DistroNamespace(name='alpine', version='3.6', like_distro='alpine'),
DistroNamespace(name='centos', version='7', like_distro='centos'),
DistroNamespace(name='centos', version='7.1', like_distro='centos'),
DistroNamespace(name='centos', version='7.3', like_distro='centos'),
DistroNamespace(name='ol', version='7.3', like_distro='centos'),
DistroNamespace(name='rhel', version='7.1', like_distro='centos'),
DistroNamespace(name='debian', version='9', like_distro='debian'),
DistroNamespace(name='ubuntu', version='16.10', like_distro='ubuntu')
]
fail = [
DistroNamespace(name='alpine', version='3.1', like_distro='alpine'),
DistroNamespace(name='alpine', version='3.1.1', like_distro='alpine'),
DistroNamespace(name='busybox', version='3', like_distro='busybox'),
DistroNamespace(name='linuxmint', version='16', like_distro='debian'),
DistroNamespace(name='redhat', version='6', like_distro='centos'),
DistroNamespace(name='ubuntu', version='1.0', like_distro='ubuntu'),
DistroNamespace(name='centos', version='1.0', like_distro='ubuntu'),
DistroNamespace(name='debian', version='1.0', like_distro='ubuntu'),
DistroNamespace(name='rhel', version='1.0', like_distro='ubuntu'),
DistroNamespace(name='busybox', version='1.0', like_distro='busybox'),
DistroNamespace(name='alpine', version='11.0', like_distro='ubuntu'),
DistroNamespace(name='fedora', version='25', like_distro='fedora'),
DistroNamespace(name='mageia', version='5', like_distro='mandriva,fedora')
]
for i in expected:
assert vulnerabilities.have_vulnerabilities_for(i), 'Expected vulns for namespace {}'.format(i.namespace_name)
for i in fail:
assert not vulnerabilities.have_vulnerabilities_for(i), 'Did not expect vulns for namespace {}'.format(i.namespace_name)
def get_image_vulnerabilities(user_id,
image_id,
force_refresh=False,
vendor_only=True):
"""
Return the vulnerability listing for the specified image and load from catalog if not found and specifically asked
to do so.
Example json output:
{
"multi" : {
"url_column_index" : 7,
"result" : {
"rows" : [],
"rowcount" : 0,
"colcount" : 8,
"header" : [
"CVE_ID",
"Severity",
"*Total_Affected",
"Vulnerable_Package",
"Fix_Available",
"Fix_Images",
"Rebuild_Images",
"URL"
]
},
"querycommand" : "/usr/lib/python2.7/site-packages/anchore/anchore-modules/multi-queries/cve-scan.py /ebs_data/anchore/querytmp/queryimages.7026386 /ebs_data/anchore/data /ebs_data/anchore/querytmp/query.59057288 all",
"queryparams" : "all",
"warns" : [
"0005b136f0fb (prom/prometheus:master) cannot perform CVE scan: no CVE data is currently available for the detected base distro type (busybox:unknown_version,busybox:v1.26.2)"
]
}
}
:param user_id: user id of image to evaluate
:param image_id: image id to evaluate
:param force_refresh: if true, flush and recompute vulnerabilities rather than returning current values
:param vendor_only: if true, filter out the vulnerabilities that vendors will explicitly not address
:return:
"""
# Has image?
db = get_session()
try:
img = db.query(Image).get((image_id, user_id))
vulns = []
if not img:
abort(404)
else:
if force_refresh:
log.info('Forcing refresh of vulnerabiltiies for {}/{}'.format(
user_id, image_id))
try:
vulns = rescan_image(img, db_session=db)
db.commit()
except Exception as e:
log.exception(
'Error refreshing cve matches for image {}/{}'.format(
user_id, image_id))
db.rollback()
abort(
Response(
'Error refreshing vulnerability listing for image.',
500))
db = get_session()
db.refresh(img)
vulns = img.vulnerabilities()
# Has vulnerabilities?
warns = []
if not vulns:
vulns = []
ns = DistroNamespace.for_obj(img)
if not have_vulnerabilities_for(ns):
warns = [
'No vulnerability data available for image distro: {}'.
format(ns.namespace_name)
]
rows = []
for vuln in vulns:
# if vuln.vulnerability.fixed_in:
# fixes_in = filter(lambda x: x.name == vuln.pkg_name or x.name == vuln.package.normalized_src_pkg,
# vuln.vulnerability.fixed_in)
# fix_available_in = fixes_in[0].version if fixes_in else 'None'
# else:
# fix_available_in = 'None'
# Skip the vulnerability if the vendor_only flag is set to True and the issue won't be addressed by the vendor
if vendor_only and vuln.fix_has_no_advisory():
continue
rows.append([
vuln.vulnerability_id,
vuln.vulnerability.severity,
1,
vuln.pkg_name + '-' + vuln.package.fullversion,
str(vuln.fixed_in()),
vuln.pkg_image_id,
'None', # Always empty this for now
vuln.vulnerability.link,
vuln.pkg_type,
'vulnerabilities',
vuln.vulnerability.namespace_name,
vuln.pkg_name,
vuln.package.fullversion,
])
vuln_listing = {
'multi': {
'url_column_index': 7,
'result': {
'header': TABLE_STYLE_HEADER_LIST,
'rowcount': len(rows),
'colcount': len(TABLE_STYLE_HEADER_LIST),
'rows': rows
},
'warns': warns
}
}
cpe_vuln_listing = []
try:
all_cpe_matches = db.query(
ImageCpe,
CpeVulnerability).filter(ImageCpe.image_id == image_id).filter(
ImageCpe.name == CpeVulnerability.name).filter(
ImageCpe.version == CpeVulnerability.version)
if not all_cpe_matches:
all_cpe_matches = []
cpe_hashes = {}
for image_cpe, vulnerability_cpe in all_cpe_matches:
cpe_vuln_el = {
'vulnerability_id': vulnerability_cpe.vulnerability_id,
'severity': vulnerability_cpe.severity,
'link': vulnerability_cpe.link,
'pkg_type': image_cpe.pkg_type,
'pkg_path': image_cpe.pkg_path,
'name': image_cpe.name,
'version': image_cpe.version,
'cpe': image_cpe.get_cpestring(),
'feed_name': vulnerability_cpe.feed_name,
'feed_namespace': vulnerability_cpe.namespace_name,
}
cpe_hash = hashlib.sha256(
utils.ensure_bytes(json.dumps(cpe_vuln_el))).hexdigest()
if not cpe_hashes.get(cpe_hash, False):
cpe_vuln_listing.append(cpe_vuln_el)
cpe_hashes[cpe_hash] = True
except Exception as err:
log.warn("could not fetch CPE matches - exception: " + str(err))
report = LegacyVulnerabilityReport.from_dict(vuln_listing)
resp = ImageVulnerabilityListing(user_id=user_id,
image_id=image_id,
legacy_report=report,
cpe_report=cpe_vuln_listing)
return resp.to_dict()
except HTTPException:
db.rollback()
raise
except Exception as e:
log.exception(
'Error checking image {}, {} for vulnerabiltiies. Rolling back'.
format(user_id, image_id))
db.rollback()
abort(500)
finally:
db.close()
def evaluate(self, image_obj, context):
if not have_vulnerabilities_for(DistroNamespace.for_obj(image_obj)):
self._fire(msg="UNSUPPORTEDDISTRO cannot perform CVE scan: " +
str(image_obj.distro_namespace))
def namespace_support_test():
init_db_persisted()
expected = [
DistroNamespace(name='alpine', version='3.3', like_distro='alpine'),
DistroNamespace(name='alpine', version='3.4', like_distro='alpine'),
DistroNamespace(name='alpine', version='3.5', like_distro='alpine'),
DistroNamespace(name='alpine', version='3.6', like_distro='alpine'),
DistroNamespace(name='centos', version='5', like_distro='centos'),
DistroNamespace(name='centos', version='6', like_distro='centos'),
DistroNamespace(name='centos', version='7', like_distro='centos'),
DistroNamespace(name='centos', version='7.1', like_distro='centos'),
DistroNamespace(name='centos', version='7.3', like_distro='centos'),
DistroNamespace(name='ol', version='6', like_distro='centos'),
DistroNamespace(name='ol', version='6.5', like_distro='centos'),
DistroNamespace(name='ol', version='7.3', like_distro='centos'),
DistroNamespace(name='rhel', version='7.1', like_distro='centos'),
DistroNamespace(name='debian', version='7', like_distro='debian'),
DistroNamespace(name='debian', version='8', like_distro='debian'),
DistroNamespace(name='debian', version='9', like_distro='debian'),
DistroNamespace(name='debian',
version='unstable',
like_distro='debian'),
DistroNamespace(name='ubuntu', version='12.04', like_distro='ubuntu'),
DistroNamespace(name='ubuntu', version='13.04', like_distro='ubuntu'),
DistroNamespace(name='ubuntu', version='14.04', like_distro='ubuntu'),
DistroNamespace(name='ubuntu', version='14.10', like_distro='ubuntu'),
DistroNamespace(name='ubuntu', version='15.04', like_distro='ubuntu'),
DistroNamespace(name='ubuntu', version='15.10', like_distro='ubuntu'),
DistroNamespace(name='ubuntu', version='16.04', like_distro='ubuntu'),
DistroNamespace(name='ubuntu', version='16.10', like_distro='ubuntu'),
DistroNamespace(name='ubuntu', version='17.04', like_distro='ubuntu'),
]
fail = [
DistroNamespace(name='alpine', version='3.1', like_distro='alpine'),
DistroNamespace(name='alpine', version='3.1.1', like_distro='alpine'),
DistroNamespace(name='busybox', version='3', like_distro='busybox'),
DistroNamespace(name='linuxmint', version='16', like_distro='debian'),
DistroNamespace(name='redhat', version='6', like_distro='centos'),
DistroNamespace(name='ubuntu', version='1.0', like_distro='ubuntu'),
DistroNamespace(name='centos', version='1.0', like_distro='ubuntu'),
DistroNamespace(name='debian', version='1.0', like_distro='ubuntu'),
DistroNamespace(name='rhel', version='1.0', like_distro='ubuntu'),
DistroNamespace(name='busybox', version='1.0', like_distro='busybox'),
DistroNamespace(name='alpine', version='11.0', like_distro='ubuntu'),
DistroNamespace(name='fedora', version='25', like_distro='fedora'),
DistroNamespace(name='mageia',
version='5',
like_distro='mandriva,fedora')
]
for i in expected:
if not vulnerabilities.have_vulnerabilities_for(i):
raise Exception('Bad failure: {}'.format(i.namespace_name))
for i in fail:
if vulnerabilities.have_vulnerabilities_for(i):
raise Exception('Should not have data for {}'.format(
i.namespace_name))
def evaluate(self, image_obj, context):
if not have_vulnerabilities_for(DistroNamespace.for_obj(image_obj)):
self._fire(
msg=
"Distro-specific feed data not found for distro namespace: %s. Cannot perform CVE scan OS/distro packages"
% image_obj.distro_namespace)
def load_and_normalize_packages(self, package_analysis_json, image_obj):
"""
Loads and normalizes package data from all distros
:param image_obj:
:param package_analysis_json:
:return: list of Package objects that can be added to an image
"""
pkgs = []
img_distro = DistroNamespace.for_obj(image_obj)
# pkgs.allinfo handling
pkgs_all = package_analysis_json.get('pkgs.allinfo', {}).values()
if not pkgs_all:
return []
else:
pkgs_all = pkgs_all[0]
for pkg_name, metadata_str in pkgs_all.items():
metadata = json.loads(metadata_str)
p = ImagePackage()
p.distro_name = image_obj.distro_name
p.distro_version = image_obj.distro_version
p.like_distro = image_obj.like_distro
p.name = pkg_name
p.version = metadata.get('version')
p.origin = metadata.get('origin')
p.size = metadata.get('size')
p.arch = metadata.get('arch')
p.license = metadata.get('license') if metadata.get(
'license') else metadata.get('lics')
p.release = metadata.get('release', 'N/A')
p.pkg_type = metadata.get('type')
p.src_pkg = metadata.get('sourcepkg')
p.image_user_id = image_obj.user_id
p.image_id = image_obj.id
if 'files' in metadata:
# Handle file data
p.files = metadata.get('files')
if p.release != 'N/A':
p.fullversion = p.version + '-' + p.release
else:
p.fullversion = p.version
if img_distro.flavor == 'DEB':
cleanvers = re.sub(re.escape("+b") + "\d+.*", "", p.version)
spkg = re.sub(re.escape("-" + cleanvers), "", p.src_pkg)
else:
spkg = re.sub(re.escape("-" + p.version) + ".*", "", p.src_pkg)
p.normalized_src_pkg = spkg
pkgs.append(p)
if pkgs:
return pkgs
else:
log.warn('Pkg Allinfo not found, reverting to using pkgs.all')
all_pkgs = package_analysis_json['pkgs.all']['base']
all_pkgs_src = package_analysis_json['pkgs_plus_source.all']['base']
for pkg_name, version in all_pkgs.items():
p = ImagePackage()
p.image_user_id = image_obj.user_id
p.image_id = image_obj.id
p.name = pkg_name
p.version = version
p.fullversion = all_pkgs_src[pkg_name]
if img_distro.flavor == 'RHEL':
name, parsed_version, release, epoch, arch = split_rpm_filename(
pkg_name + '-' + version + '.tmparch.rpm')
p.version = parsed_version
p.release = release
p.pkg_type = 'RPM'
p.origin = 'N/A'
p.src_pkg = 'N/A'
p.license = 'N/A'
p.arch = 'N/A'
elif img_distro.flavor == 'DEB':
try:
p.version, p.release = version.split('-')
except:
p.version = version
p.release = None
return pkgs
def get_image_vulnerabilities(
self,
image: Image,
db_session,
vendor_only: bool = True,
force_refresh: bool = False,
cache: bool = True,
):
# select the nvd class once and be done
_nvd_cls, _cpe_cls = select_nvd_classes(db_session)
# initialize the scanner
scanner = self.__scanner__()
user_id = image.user_id
image_id = image.id
results = []
if force_refresh:
log.info("Forcing refresh of vulnerabilities for {}/{}".format(
user_id, image_id))
try:
scanner.flush_and_recompute_vulnerabilities(
image, db_session=db_session)
db_session.commit()
except Exception as e:
log.exception(
"Error refreshing cve matches for image {}/{}".format(
user_id, image_id))
db_session.rollback()
return make_response_error(
"Error refreshing vulnerability listing for image.",
in_httpcode=500,
)
db_session = get_session()
db_session.refresh(image)
with timer("Image vulnerability primary lookup", log_level="debug"):
vulns = scanner.get_vulnerabilities(image)
# Has vulnerabilities?
warns = []
if not vulns:
vulns = []
ns = DistroNamespace.for_obj(image)
if not have_vulnerabilities_for(ns):
warns = [
"No vulnerability data available for image distro: {}".
format(ns.namespace_name)
]
rows = []
with timer("Image vulnerability nvd metadata merge",
log_level="debug"):
vulns = merge_nvd_metadata_image_packages(db_session, vulns,
_nvd_cls, _cpe_cls)
with timer("Image vulnerability output formatting", log_level="debug"):
for vuln, nvd_records in vulns:
fixed_artifact = vuln.fixed_artifact()
# Skip the vulnerability if the vendor_only flag is set to True and the issue won't be addressed by the vendor
if vendor_only and vuln.fix_has_no_advisory(
fixed_in=fixed_artifact):
continue
nvd_scores = [
self._make_cvss_score(score) for nvd_record in nvd_records
for score in nvd_record.get_cvss_scores_nvd()
]
results.append(
VulnerabilityMatch(
vulnerability=VulnerabilityModel(
vulnerability_id=vuln.vulnerability_id,
description="NA",
severity=vuln.vulnerability.severity,
link=vuln.vulnerability.link,
feed="vulnerabilities",
feed_group=vuln.vulnerability.namespace_name,
cvss_scores_nvd=nvd_scores,
cvss_scores_vendor=[],
created_at=vuln.vulnerability.created_at,
last_modified=vuln.vulnerability.updated_at,
),
artifact=Artifact(
name=vuln.pkg_name,
version=vuln.package.fullversion,
pkg_type=vuln.pkg_type,
pkg_path=vuln.pkg_path,
cpe="None",
cpe23="None",
),
fixes=[
FixedArtifact(
version=str(
vuln.fixed_in(fixed_in=fixed_artifact)),
wont_fix=vuln.fix_has_no_advisory(
fixed_in=fixed_artifact),
observed_at=fixed_artifact.fix_observed_at
if fixed_artifact else None,
)
],
match=Match(detected_at=vuln.created_at),
))
# TODO move dedup here so api doesn't have to
# cpe_vuln_listing = []
try:
with timer("Image vulnerabilities cpe matches", log_level="debug"):
all_cpe_matches = scanner.get_cpe_vulnerabilities(
image, _nvd_cls, _cpe_cls)
if not all_cpe_matches:
all_cpe_matches = []
api_endpoint = self._get_api_endpoint()
for image_cpe, vulnerability_cpe in all_cpe_matches:
link = vulnerability_cpe.parent.link
if not link:
link = "{}/query/vulnerabilities?id={}".format(
api_endpoint, vulnerability_cpe.vulnerability_id)
nvd_scores = [
self._make_cvss_score(score) for score in
vulnerability_cpe.parent.get_cvss_scores_nvd()
]
vendor_scores = [
self._make_cvss_score(score) for score in
vulnerability_cpe.parent.get_cvss_scores_vendor()
]
results.append(
VulnerabilityMatch(
vulnerability=VulnerabilityModel(
vulnerability_id=vulnerability_cpe.parent.
normalized_id,
description="NA",
severity=vulnerability_cpe.parent.severity,
link=link,
feed=vulnerability_cpe.feed_name,
feed_group=vulnerability_cpe.namespace_name,
cvss_scores_nvd=nvd_scores,
cvss_scores_vendor=vendor_scores,
created_at=vulnerability_cpe.parent.created_at,
last_modified=vulnerability_cpe.parent.
updated_at,
),
artifact=Artifact(
name=image_cpe.name,
version=image_cpe.version,
pkg_type=image_cpe.pkg_type,
pkg_path=image_cpe.pkg_path,
cpe=image_cpe.get_cpestring(),
cpe23=image_cpe.get_cpe23string(),
),
fixes=[
FixedArtifact(
version=item,
wont_fix=False,
observed_at=vulnerability_cpe.created_at,
) for item in vulnerability_cpe.get_fixed_in()
],
# using vulnerability created_at to indicate the match timestamp for now
match=Match(
detected_at=vulnerability_cpe.created_at),
))
except Exception as err:
log.exception("could not fetch CPE matches")
import uuid
return ImageVulnerabilitiesReport(
account_id=image.user_id,
image_id=image_id,
results=results,
metadata=VulnerabilitiesReportMetadata(
generated_at=datetime.datetime.utcnow(),
uuid=str(uuid.uuid4()),
generated_by=self._get_provider_metadata(),
),
problems=[],
)
