def test_userblacklist(self):
db = get_thread_scoped_session()
image = db.query(Image).get(
(self.test_env.get_images_named('centos_latest')[0][0], '0'))
t, gate, test_context = self.get_initialized_trigger(
UsernameMatchTrigger.__trigger_name__,
USERNAMEBLACKLIST='mail,ftp,foobar')
db.refresh(self.test_image)
test_context = gate.prepare_context(image, test_context)
t.evaluate(self.test_image, test_context)
print('Fired: {}'.format(t.fired))
self.assertEqual(len(t.fired), 2)
db.rollback()
def test_filenotstored(self):
db = get_thread_scoped_session()
image = db.query(Image).get(
(self.test_env.get_images_named('centos7_verify')[0][0], '0'))
t, gate, test_context = self.get_initialized_trigger(
FileNotStoredTrigger.__trigger_name__)
db.refresh(self.test_image)
test_context = gate.prepare_context(image, test_context)
t.evaluate(self.test_image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(0, len(t.fired))
db.rollback()
db = get_thread_scoped_session()
t, gate, test_context = self.get_initialized_trigger(
FileNotStoredTrigger.__trigger_name__)
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(1, len(t.fired))
db.rollback()
def check_fix_version(test_env):
logger.info('Checking fix versions')
db = get_thread_scoped_session()
img = db.query(Image).get((test_env.get_images_named('ruby')[0][0], '0'))
vulns = img.vulnerabilities()
for vuln in vulns:
if vuln.vulnerability.fixed_in:
fixes_in = [x for x in vuln.vulnerability.fixed_in if x.name == vuln.pkg_name or x.name == vuln.package.normalized_src_pkg]
fix_available_in = fixes_in[0].version if fixes_in else 'None'
else:
fix_available_in = 'None'
logger.info('{} Fix version: {}'.format(vuln.vulnerability_id, fix_available_in))
def test_shellblacklist(self):
db = get_thread_scoped_session()
image = db.query(Image).get(
(self.test_env.get_images_named('centos_latest')[0][0], '0'))
t, gate, test_context = self.get_initialized_trigger(
ShellMatchTrigger.__trigger_name__,
SHELLBLACKLIST='/bin/bash,/bin/ksh')
db.refresh(self.test_image)
test_context = gate.prepare_context(image, test_context)
t.evaluate(self.test_image, test_context)
print('Fired: {}'.format(t.fired))
self.assertEqual(len(t.fired), 1)
db.rollback()
def test_pentryblacklist(self):
db = get_thread_scoped_session()
image = db.query(Image).get(
(self.test_env.get_images_named('centos7_verify')[0][0], '0'))
t, gate, test_context = self.get_initialized_trigger(
PEntryMatchTrigger.__trigger_name__,
pentryblacklist='mail:x:8:12:mail:/var/spool/mail:/sbin/nologin')
db.refresh(self.test_image)
test_context = gate.prepare_context(image, test_context)
t.evaluate(self.test_image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(1, len(t.fired))
db.rollback()
def test_basic_evaluation(test_data_env_with_images_loaded):
db = get_thread_scoped_session()
logger.info('Session state: {}'.format(db.__dict__))
logger.info('Building executable bundle from default bundle')
test_tag = 'docker.io/library/ruby:latest'
test_bundle = test_data_env_with_images_loaded.get_bundle('multitest')
built = build_bundle(test_bundle, for_tag=test_tag)
assert not built.init_errors
logger.info('Got: {}'.format(built))
img_obj = get_image_named(db, test_data_env_with_images_loaded, 'ruby')
assert img_obj is not None
assert img_obj is not None, 'Failed to get an image object to test'
evaluation = built.execute(img_obj,
tag=test_tag,
context=ExecutionContext(db_session=db,
configuration={}))
assert evaluation is not None, 'Got None eval'
logger.info('Native json: {}\n'.format(
json.dumps(evaluation.json(), indent=2)))
logger.info('Table json: {}\n'.format(
json.dumps(evaluation.as_table_json(), indent=2)))
# Diff old an new defaults
multi_bundle = test_data_env_with_images_loaded.get_bundle('multi_default')
multi_default = build_bundle(multi_bundle, for_tag=test_tag)
assert not built.init_errors
logger.info('Got: {}'.format(multi_default))
assert img_obj is not None, 'Failed to get an image object to test'
multi_default_evaluation = multi_default.execute(img_obj,
tag=test_tag,
context=ExecutionContext(
db_session=db,
configuration={}))
default_built = build_bundle(
test_data_env_with_images_loaded.get_bundle('default'),
for_tag=test_tag)
assert not built.init_errors
logger.info('Got: {}'.format(default_built))
assert img_obj is not None, 'Failed to get an image object to test'
default_evaluation = default_built.execute(img_obj,
tag=test_tag,
context=ExecutionContext(
db_session=db,
configuration={}))
assert multi_default_evaluation.as_table_json(
) == default_evaluation.as_table_json()
def test_basic_evaluation(test_data_env_with_images_loaded):
db = get_thread_scoped_session()
logger.info("Session state: {}".format(db.__dict__))
logger.info("Building executable bundle from default bundle")
test_tag = "docker.io/library/ruby:latest"
test_bundle = test_data_env_with_images_loaded.get_bundle("multitest")
built = build_bundle(test_bundle, for_tag=test_tag)
assert not built.init_errors
logger.info("Got: {}".format(built))
img_obj = get_image_named(db, test_data_env_with_images_loaded, "ruby")
assert img_obj is not None
assert img_obj is not None, "Failed to get an image object to test"
evaluation = built.execute(img_obj,
tag=test_tag,
context=ExecutionContext(db_session=db,
configuration={}))
assert evaluation is not None, "Got None eval"
logger.info("Native json: {}\n".format(
json.dumps(evaluation.json(), indent=2)))
logger.info("Table json: {}\n".format(
json.dumps(evaluation.as_table_json(), indent=2)))
# Diff old an new defaults
multi_bundle = test_data_env_with_images_loaded.get_bundle("multi_default")
multi_default = build_bundle(multi_bundle, for_tag=test_tag)
assert not built.init_errors
logger.info("Got: {}".format(multi_default))
assert img_obj is not None, "Failed to get an image object to test"
multi_default_evaluation = multi_default.execute(img_obj,
tag=test_tag,
context=ExecutionContext(
db_session=db,
configuration={}))
default_built = build_bundle(
test_data_env_with_images_loaded.get_bundle("default"),
for_tag=test_tag)
assert not built.init_errors
logger.info("Got: {}".format(default_built))
assert img_obj is not None, "Failed to get an image object to test"
default_evaluation = default_built.execute(img_obj,
tag=test_tag,
context=ExecutionContext(
db_session=db,
configuration={}))
assert (multi_default_evaluation.as_table_json() ==
default_evaluation.as_table_json())
def load_test_data(cls):
print('Loading test data')
with open('data/trimmed_export_alpine.json') as f:
img_json = json.load(f)
loader = ImageLoader(img_json)
img = loader.load()
cls.img_id = img.id
session = get_thread_scoped_session()
session.add(img)
session.commit()
print('Done loading data')
def check_fix_version():
log.info('Checking fix versions')
db = get_thread_scoped_session()
img = db.query(Image).get((test_image_ids['node'], '0'))
vulns = img.vulnerabilities()
for vuln in vulns:
if vuln.vulnerability.fixed_in:
fixes_in = filter(
lambda x: x.name == vuln.pkg_name or x.name == vuln.package.
normalized_src_pkg, vuln.vulnerability.fixed_in)
fix_available_in = fixes_in[0].version if fixes_in else 'None'
else:
fix_available_in = 'None'
log.info('{} Fix version: {}'.format(vuln.vulnerability_id,
fix_available_in))
def load_images(request):
for image_id, path in request.cls.test_env.image_exports():
logger.info("Ensuring loaded: image id: {} from file: {}".format(
image_id, path))
t = ImageLoadTask(image_id=image_id, user_id="0", url="file://" + path)
t.execute()
db = get_thread_scoped_session()
test_image = db.query(Image).get((
request.cls.test_env.get_images_named(
request.cls.__default_image__)[0][0],
"0",
))
request.cls.test_image = test_image
db.rollback()
def test_base_out_of_date(self):
t, gate, test_context = self.get_initialized_trigger(
BaseOutOfDateTrigger.__trigger_name__)
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
print('Fired: {}'.format(t.fired))
self.assertGreaterEqual(len(t.fired), 1)
img_with_tree = db.query(Image).get(
(self.test_env.get_images_named('node')[0][0], '0'))
test_context = gate.prepare_context(img_with_tree, test_context)
t.evaluate(img_with_tree, test_context)
print('Fired: {}'.format(t.fired))
self.assertGreaterEqual(len(t.fired), 0)
def test_contentmatch(self):
t, gate, test_context = self.get_initialized_trigger(
ContentMatchTrigger.__trigger_name__, regex_name='.*password.*')
db = get_thread_scoped_session()
content_test_image = db.query(Image).get(
(self.test_env.get_images_named('alpine_3.5')[0][0], '0'))
test_context = gate.prepare_context(content_test_image, test_context)
t.evaluate(content_test_image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(0, len(t.fired))
t.reset()
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(0, len(t.fired))
def test_contentmatch(self):
t, gate, test_context = self.get_initialized_trigger(
ContentMatchTrigger.__trigger_name__, regex_name=".*password.*")
db = get_thread_scoped_session()
content_test_image = db.query(Image).get(
(self.test_env.get_images_named("alpine")[0][0], "0"))
test_context = gate.prepare_context(content_test_image, test_context)
t.evaluate(content_test_image, test_context)
logger.info(("Fired: {}".format(t.fired)))
self.assertEqual(0, len(t.fired))
t.reset()
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
logger.info(("Fired: {}".format(t.fired)))
self.assertEqual(0, len(t.fired))
def vulnerabilities_for_package(package_obj):
"""
Given an ImagePackage object, return the vulnerabilities that it matches.
:param package_obj:
:return: list of Vulnerability objects
"""
log.debug('Finding vulnerabilities for package: {} - {}'.format(
package_obj.name, package_obj.version))
matches = []
dist = DistroNamespace(package_obj.distro_name, package_obj.distro_version,
package_obj.like_distro)
db = get_thread_scoped_session()
namespace_name_to_use = dist.namespace_name
# All options are the same, no need to loop
if len(set(dist.like_namespace_names)) > 1:
# Look for exact match first
if not DataFeeds.instance().vulnerabilities.group_by_name(
dist.namespace_name):
# Check all options for distro/flavor mappings, stop at first with records present
for namespace_name in dist.like_namespace_names:
record_count = db.query(Vulnerability).filter(
Vulnerability.namespace_name == namespace_name).count()
if record_count > 0:
namespace_name_to_use = namespace_name
break
fix_candidates, vulnerable_candidates = candidates_for_package(
package_obj, namespace_name_to_use)
for candidate in fix_candidates:
# De-dup evaluations based on the underlying vulnerability_id. For packages where src has many binary builds, once we have a match we have a match.
if candidate.vulnerability_id not in map(
lambda x: x.vulnerability_id, matches) and match_but_not_fixed(
candidate, package_obj):
matches.append(candidate)
for candidate in vulnerable_candidates:
if candidate.vulnerability_id not in map(
lambda x: x.vulnerability_id,
matches) and match_and_vulnerable(candidate, package_obj):
matches.append(candidate)
return matches
def test_pkg_required(self):
db = get_thread_scoped_session()
try:
image = db.query(Image).get(
(self.test_env.get_images_named('node')[0][0], '0'))
# Image has '2.25-5+deb8u1'
t, gate, test_context = self.get_initialized_trigger(
RequiredPackageTrigger.__trigger_name__,
name='binutils',
version='2.25-6+deb8u1')
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(1, len(t.fired))
# Image has '2.25-5+deb8u1'
t, gate, test_context = self.get_initialized_trigger(
RequiredPackageTrigger.__trigger_name__,
name='binutils',
version='2.25-4+deb8u1',
version_match_type='minimum')
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(0, len(t.fired))
t, gate, test_context = self.get_initialized_trigger(
RequiredPackageTrigger.__trigger_name__, name='binutilityrepo')
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(1, len(t.fired))
t, gate, test_context = self.get_initialized_trigger(
RequiredPackageTrigger.__trigger_name__,
name='binutils',
version='2.25-6+deb8u1',
version_match_type='exact')
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(1, len(t.fired))
finally:
db.rollback()
def check_all_imgs_vuln():
db = get_thread_scoped_session()
try:
for img in db.query(Image).all():
logger.info('Checking vulnerabilities for image: {}'.format(img.id))
if not img:
logger.info('No image found with id: {}'.format(img.id))
raise Exception('Should have image')
vulns = vulnerabilities.vulnerabilities_for_image(img)
for v in vulns:
db.merge(v)
db.commit()
logger.info('Found: {}'.format(vulns))
except Exception as e:
logger.info('Error! {}'.format(e))
end_session()
def have_vulnerabilities_for(distro_namespace_obj):
"""
Does the system have any vulnerabilities for the given distro.
:param distro_namespace_obj:
:return: boolean
"""
# All options are the same, no need to loop
# Check all options for distro/flavor mappings
db = get_thread_scoped_session()
for namespace_name in distro_namespace_obj.like_namespace_names:
feed = get_feed_json(db_session=db, feed_name='vulnerabilities')
if feed and namespace_name in [x['name'] for x in feed.get('groups', [])]:
# No records yet, but we have the feed, so may just not have any data yet
return True
else:
return False
def test_pkgnotpresentdiff(self):
db = get_thread_scoped_session()
try:
image = db.query(Image).get(
(self.test_env.get_images_named('node')[0][0], '0'))
t, gate, test_context = self.get_initialized_trigger(
PkgNotPresentTrigger.__trigger_name__,
pkgfullmatch='binutils|2.25-5+deb8u1,libssl|123')
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
print('Fired: {}'.format(t.fired))
self.assertEqual(len(t.fired), 1)
t, gate, test_context = self.get_initialized_trigger(
PkgNotPresentTrigger.__trigger_name__,
pkgnamematch='binutilityrepo,binutils')
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
print('Fired: {}'.format(t.fired))
self.assertEqual(len(t.fired), 1)
t, gate, test_context = self.get_initialized_trigger(
PkgNotPresentTrigger.__trigger_name__,
pkgversmatch=
'binutils|2.25-5+deb8u1,randopackage|123,binutils|3.25-5+deb8u1'
)
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
print('Fired: {}'.format(t.fired))
self.assertEqual(len(t.fired), 2)
t, gate, test_context = self.get_initialized_trigger(
PkgNotPresentTrigger.__trigger_name__,
pkgfullmatch='binutils|2.25-5+deb8u1,libssl|123',
pkgnamematch='binutils,foobar',
pkgversmatch='binutils|2.25-5+deb8u1,libssl|10.2,blamo|123.123'
)
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
print('Fired: {}'.format(t.fired))
self.assertEqual(len(t.fired), 4)
finally:
db.rollback()
def cls_no_feeds_test_env(cls_test_data_env2, request):
"""
Same as fully_loaded_test_env but does not sync feeds
:param cls_test_data_env:
:param request:
:return:
"""
_init_distro_mappings()
for image_id, path in request.cls.test_env.image_exports():
logger.info(('Ensuring loaded: image id: {} from file: {}'.format(image_id, path)))
t = ImageLoadTask(image_id=image_id, user_id='0', url='file://' + path)
t.execute()
db = get_thread_scoped_session()
test_image = db.query(Image).get((request.cls.test_env.get_images_named(request.cls.__default_image__)[0][0], '0'))
request.cls.test_image = test_image
db.rollback()
def test_basic_legacy_evaluation(test_data_env_with_images_loaded):
db = get_thread_scoped_session()
logger.info('Building executable bundle from default bundle')
test_tag = 'docker.io/library/ruby:latest'
built = build_bundle(
test_data_env_with_images_loaded.get_bundle('default'),
for_tag=test_tag)
assert not built.init_errors
logger.info('Got: {}'.format(built))
img_obj = get_image_named(db, test_data_env_with_images_loaded, 'ruby')
assert img_obj is not None
assert img_obj is not None, 'Failed to get an image object to test'
evaluation = built.execute(img_obj,
tag=test_tag,
context=ExecutionContext(db_session=db,
configuration={}))
assert evaluation is not None, 'Got None eval'
logger.info(json.dumps(evaluation.json(), indent=2))
logger.info(json.dumps(evaluation.as_table_json(), indent=2))
logger.info('Building executable bundle from old default bundle')
test_tag = 'docker.io/library/ruby:latest'
built = build_bundle(
test_data_env_with_images_loaded.get_bundle('old_default'),
for_tag=test_tag)
assert not built.init_errors
logger.info('Got: {}'.format(built))
img_obj = get_image_named(db, test_data_env_with_images_loaded, 'ruby')
assert img_obj is not None
assert img_obj is not None, 'Failed to get an image object to test'
evaluation = built.execute(img_obj,
tag=test_tag,
context=ExecutionContext(db_session=db,
configuration={}))
assert evaluation is not None, 'Got None eval'
logger.info(json.dumps(evaluation.json(), indent=2))
logger.info(json.dumps(evaluation.as_table_json(), indent=2))
def have_vulnerabilities_for(distro_namespace_obj):
"""
Does the system have any vulnerabilities for the given distro.
:param distro_namespace_obj:
:return: boolean
"""
db = get_thread_scoped_session()
# All options are the same, no need to loop
# Check all options for distro/flavor mappings
vulnerability_feed = DataFeeds.instance().vulnerabilities
for namespace_name in distro_namespace_obj.like_namespace_names:
# Check feed names
if vulnerability_feed.group_by_name(namespace_name):
# No records yet, but we have the feed, so may just not have any data yet
return True
else:
return False
def cls_fully_loaded_test_env(cls_test_data_env2, request):
"""
Load the test env, including a feed sync and image analysis. Places the env in the class's test_env and test_image vars
:param cls_test_data_env:
:param request:
:return:
"""
_init_distro_mappings()
from anchore_engine.services.policy_engine.engine.tasks import FeedsUpdateTask
t = FeedsUpdateTask()
t.execute()
for image_id, path in request.cls.test_env.image_exports():
logger.info(('Ensuring loaded: image id: {} from file: {}'.format(image_id, path)))
t = ImageLoadTask(image_id=image_id, user_id='0', url='file://' + path)
t.execute()
db = get_thread_scoped_session()
test_image = db.query(Image).get((request.cls.test_env.get_images_named(request.cls.__default_image__)[0][0], '0'))
request.cls.test_image = test_image
db.rollback()
def test_vuln_image_updates(test_data_env):
sync_feeds(test_data_env, up_to=datetime.datetime(2017, 6, 1))
_load_images(test_data_env)
# Get the first set
initial_vulns = _img_vulns(test_data_env.get_images_named('ruby')[0][0])
# Rollback the sync time to get another sync with data
db = get_thread_scoped_session()
try:
f = reset_feed_sync_time(db, datetime.datetime(2017, 6, 1), feed_name='vulnerabilities')
db.add(f)
db.commit()
except:
logger.exception('Exception commiting update of feed sync timestamps')
db.rollback()
# Sync again to get new merged data
sync_feeds(test_data_env, up_to=datetime.datetime.utcnow())
check_fix_version(test_data_env)
rescan_img_id = list(test_data_env.image_map.keys())[0]
updated_vulns = _img_vulns(test_data_env.get_images_named('ruby')[0][0])
logger.info(json.dumps(updated_vulns, indent=2))
def setUp(self):
db = get_thread_scoped_session()
self.test_image = db.query(Image).get(
(self.test_env.get_images_named('ruby')[0][0], '0'))
def find_vulnerable_image_packages(vulnerability_obj):
"""
Given a vulnerability object, find images that are affected via their package manifests.
Result may have duplicates based on match type, caller must de-dup if desired.
:param vulnerability_obj:
:return: list of ImagePackage objects
"""
db = get_thread_scoped_session()
distro, version = vulnerability_obj.namespace_name.split(":", 1)
dist = DistroNamespace(distro, version)
related_names = (
dist.mapped_names()
) # Returns list of names that map to this one, not including itself necessarily
# related_names = get_namespace_related_names(distro, version, mapped_names)
# TODO would like a better way to do the pkg_type <-> namespace_name mapping, with other side in ImagePackage.vulnerabilities_for_package
likematch = None
if (":maven" in vulnerability_obj.namespace_name
or "java" in vulnerability_obj.namespace_name):
likematch = "java"
elif (":ruby" in vulnerability_obj.namespace_name
or "gem" in vulnerability_obj.namespace_name):
likematch = "gem"
elif (":js" in vulnerability_obj.namespace_name
or "npm" in vulnerability_obj.namespace_name):
likematch = "npm"
elif "python" in vulnerability_obj.namespace_name:
likematch = "python"
try:
affected = []
if vulnerability_obj.fixed_in:
# Check the fixed_in records
for fix_rec in vulnerability_obj.fixed_in:
package_candidates = []
# Find packages of related distro names with compatible versions, this does not have to be precise, just an initial filter.
pkgs = (db.query(ImagePackage).filter(
ImagePackage.distro_name.in_(related_names),
ImagePackage.distro_version.like(dist.version + "%"),
or_(
ImagePackage.name == fix_rec.name,
ImagePackage.normalized_src_pkg == fix_rec.name,
),
).all())
package_candidates += pkgs
# add non distro candidates
if likematch:
pkgs = (db.query(ImagePackage).filter(
ImagePackage.pkg_type.in_(nonos_package_types),
ImagePackage.pkg_type.like(likematch),
or_(
ImagePackage.name == fix_rec.name,
ImagePackage.normalized_src_pkg == fix_rec.name,
),
).all())
package_candidates += pkgs
for candidate in package_candidates:
if fix_rec.match_but_not_fixed(candidate):
affected.append(candidate)
if vulnerability_obj.vulnerable_in:
# Check the vulnerable_in records
for vuln_rec in vulnerability_obj.vulnerable_in:
package_candidates = []
# Find packages of related distro names with compatible versions, this does not have to be precise, just an initial filter.
pkgs = (db.query(ImagePackage).filter(
ImagePackage.distro_name.in_(related_names),
ImagePackage.distro_version.like(dist.version + "%"),
or_(
ImagePackage.name == vuln_rec.name,
ImagePackage.normalized_src_pkg == vuln_rec.name,
),
).all())
package_candidates += pkgs
for candidate in package_candidates:
if vuln_rec.match_and_vulnerable(candidate):
affected.append(candidate)
return affected
except Exception as e:
logger.exception(
"Failed to query and find packages affected by vulnerability: {}".
format(vulnerability_obj))
raise
def test_pkg_required(self):
db = get_thread_scoped_session()
try:
image = self.test_image
# Image has libc6 - 2.24-11+deb9u4
# Positive tests... should not result in match
# Require the version in image, no match expected.
t, gate, test_context = self.get_initialized_trigger(
RequiredPackageTrigger.__trigger_name__,
name="libc6",
version="2.24-11+deb9u4",
version_match_type="exact",
)
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
logger.info(("Fired: {}".format(t.fired)))
self.assertEqual(0, len(t.fired))
# Require a min version < than version installed, expect 0 match
t, gate, test_context = self.get_initialized_trigger(
RequiredPackageTrigger.__trigger_name__,
name="libc6",
version="2.24-10+deb9u4",
version_match_type="minimum",
)
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
logger.info(("Fired: {}".format(t.fired)))
self.assertEqual(0, len(t.fired))
# Requirement not met
# Require an exact version not present, expect 1 match
t, gate, test_context = self.get_initialized_trigger(
RequiredPackageTrigger.__trigger_name__,
name="libc6",
version="2.24-10+deb9u4",
)
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
logger.info(("Fired: {}".format(t.fired)))
self.assertEqual(1, len(t.fired))
# Require exact match that doesn't match version, expect 1 match
t, gate, test_context = self.get_initialized_trigger(
RequiredPackageTrigger.__trigger_name__,
name="libc6",
version="2.24-10+deb9u4",
version_match_type="exact",
)
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
logger.info(("Fired: {}".format(t.fired)))
self.assertEqual(1, len(t.fired))
# Require min version > installed version, expect 1 match
t, gate, test_context = self.get_initialized_trigger(
RequiredPackageTrigger.__trigger_name__,
name="libc6",
version="2.24-15+deb9u4",
version_match_type="minimum",
)
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
logger.info(("Fired: {}".format(t.fired)))
self.assertEqual(1, len(t.fired))
# Require a package not installed, expect 1 match
t, gate, test_context = self.get_initialized_trigger(
RequiredPackageTrigger.__trigger_name__,
name="libc-not-installed")
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
logger.info(("Fired: {}".format(t.fired)))
self.assertEqual(1, len(t.fired))
# Require a package not installed even with version check, expect 1 match
t, gate, test_context = self.get_initialized_trigger(
RequiredPackageTrigger.__trigger_name__,
name="libc-not-installed",
version="1.1.0",
version_match_type="exact",
)
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
logger.info(("Fired: {}".format(t.fired)))
self.assertEqual(1, len(t.fired))
# Require a package not installed even with min version chec, expect 1 match
t, gate, test_context = self.get_initialized_trigger(
RequiredPackageTrigger.__trigger_name__,
name="libc-not-installed",
version="1.1.0",
version_match_type="minimum",
)
test_context = gate.prepare_context(image, test_context)
t.evaluate(image, test_context)
logger.info(("Fired: {}".format(t.fired)))
self.assertEqual(1, len(t.fired))
finally:
db.rollback()
def test_verifytrigger(self):
"""
Test package verification gate
Image for testing: debian:9-slim (sha256:d4f7ac076cf641652722c33b026fccd52933bb5c26aa703d3cef2dd5b022422a)
Since this is a slim image, it is missing a lot of docs pages, so dpkg -V returns 1889 results.
For this test, specifically focus on data modified
/usr/share/doc/tzdata/copyright file modified in result report with wrong digests and mode
/usr/share/doc/tzdata/README.Debian file remove in analysis but present in the pkg db
Image misses: /usr/share/doc/tzdata/changelog.gz and changelog.Debian.gz without any test-modification, as pulled from dockerhub.
:return:
"""
logger.info("Default params check")
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__)
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
self.assertGreater(len(
t.fired), 0) # 1890 is the exact number for this specific image
self.assertTrue(len([x for x in t.fired if "missing" in x.msg]) > 0)
self.assertTrue(len([x for x in t.fired if "changed" in x.msg]) > 0)
# self.assertTrue(('missing' in t.fired[0].msg and 'changed' in t.fired[1].msg) or ('missing' in t.fired[1].msg and 'changed' in t.fired[0].msg))
logger.info("Specific dirs and check only changed")
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__,
only_directories="/usr/share/doc/tzdata/",
check="changed",
)
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
self.assertEqual(len(t.fired), 1)
logger.debug("Fired: {}".format(t.fired))
self.assertTrue("changed" in t.fired[0].msg)
logger.info("Check only missing")
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__, check="missing")
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
logger.debug("Fired: {}".format(t.fired))
self.assertGreaterEqual(len(t.fired), 1)
logger.info("Specific pkg, with issues")
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__, only_packages="tzdata")
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
logger.debug("Fired: {}".format(t.fired))
self.assertEqual(len(t.fired), 4)
logger.info("Specific pkg, with issues")
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__, only_packages="zlib1g")
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
logger.debug("Fired: {}".format(t.fired))
self.assertEqual(len(t.fired), 2)
logger.info("Specific pkg, with issues")
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__, only_packages="util-linux")
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
logger.debug("Fired: {}".format(t.fired))
self.assertGreaterEqual(len(t.fired), 1)
logger.info("Specific pkg, no issues")
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__, only_packages="findutil-notfound")
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
logger.debug("Fired: {}".format(t.fired))
self.assertEqual(len(t.fired), 0)
logger.info("Trying default params on all loaded images")
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__)
for img_id, meta in list(self.test_env.image_map.items()):
if img_id == self.test_image.id:
continue
t.reset()
img_obj = db.query(Image).get((img_id, "0"))
logger.info("Default params check on img: {}".format(img_id))
test_context = gate.prepare_context(img_obj, test_context)
t.evaluate(img_obj, test_context)
logger.info(
"Image name: {}, id: {}, Fired count: {}\nFired: {}".format(
meta.get("name"), img_id, len(t.fired), t.fired))
def find_vulnerable_image_packages(vulnerability_obj):
"""
Given a vulnerability object, find images that are affected via their package manifests.
Result may have duplicates based on match type, caller must de-dup if desired.
:param vulnerability_obj:
:return: list of ImagePackage objects
"""
db = get_thread_scoped_session()
distro, version = vulnerability_obj.namespace_name.split(':', 1)
dist = DistroNamespace(distro, version)
related_names = dist.mapped_names() # Returns list of names that map to this one, not including itself necessarily
# Filter related_names down based on the presence of actual feeds/cve data. If there is an actual feed for a name, remove it from the list.
# Only do this if the list of related names is not just the name itself. (e.g. alpine = [alpine]).
if related_names != [distro]:
# Ensure we don't include any names that actually have a feed (can happen when new feeds arrive before the mapped_names() source
# is updated to break the 'like' relation between the distros.
related_names = [x for x in related_names if namespace_has_no_feed(x, version)]
# This is a weird case because it basically means that this distro doesn't map to itself as far as mapped_names() is
# concerned, but since that could be code lagging data (e.g. new feed group added for a new distro), add the name itself
# back into the list.
if distro not in related_names and not namespace_has_no_feed(distro, version):
related_names.append(distro)
# TODO would like a better way to do the pkg_type <-> namespace_name mapping, with other side in ImagePackage.vulnerabilities_for_package
likematch = None
if ':maven' in vulnerability_obj.namespace_name or 'java' in vulnerability_obj.namespace_name:
likematch = 'java'
elif ':ruby' in vulnerability_obj.namespace_name or 'gem' in vulnerability_obj.namespace_name:
likematch = 'gem'
elif ':js' in vulnerability_obj.namespace_name or 'npm' in vulnerability_obj.namespace_name:
likematch = 'npm'
elif 'python' in vulnerability_obj.namespace_name:
likematch = 'python'
try:
affected = []
if vulnerability_obj.fixed_in:
# Check the fixed_in records
for fix_rec in vulnerability_obj.fixed_in:
package_candidates = []
# Find packages of related distro names with compatible versions, this does not have to be precise, just an initial filter.
pkgs = db.query(ImagePackage).filter(ImagePackage.distro_name.in_(related_names), ImagePackage.distro_version.like(dist.version + '%'), or_(ImagePackage.name == fix_rec.name, ImagePackage.normalized_src_pkg == fix_rec.name)).all()
package_candidates += pkgs
# add non distro candidates
if likematch:
pkgs = db.query(ImagePackage).filter(ImagePackage.pkg_type.in_(nonos_package_types), ImagePackage.pkg_type.like(likematch), or_(ImagePackage.name == fix_rec.name, ImagePackage.normalized_src_pkg == fix_rec.name)).all()
package_candidates += pkgs
for candidate in package_candidates:
if fix_rec.match_but_not_fixed(candidate):
affected.append(candidate)
# if vulnerability_obj.vulnerable_in:
# # Check the vulnerable_in records
# for vuln_rec in vulnerability_obj.vulnerable_in:
# package_candidates = []
# # Find packages of related distro names with compatible versions, this does not have to be precise, just an initial filter.
# pkgs = db.query(ImagePackage).filter(ImagePackage.distro_name.in_(related_names),
# ImagePackage.distro_version.like(dist.version + '%'),
# or_(ImagePackage.name == fix_rec.name,
# ImagePackage.normalized_src_pkg == fix_rec.name)).all()
# package_candidates += pkgs
# for candidate in package_candidates:
# if vuln_rec.match_and_vulnerable(candidate):
# affected.append(candidate)
return affected
except Exception as e:
log.exception('Failed to query and find packages affected by vulnerability: {}'.format(vulnerability_obj))
raise
def test_duplicate_rule_evaluation(test_data_env_with_images_loaded):
logger.info('Building executable bundle from default bundle')
test_tag = 'docker.io/library/ruby:latest'
multi_gate_bundle = {
'id':
'multigate1',
'name':
'Multigate test1',
'version':
'1_0',
'policies': [{
'id':
'policy1',
'name':
'Test policy1',
'version':
'1_0',
'rules': [{
'gate': 'always',
'trigger': 'always',
'params': [],
'action': 'GO'
}, {
'gate': 'always',
'trigger': 'always',
'params': [],
'action': 'STOP'
}, {
'action':
'stop',
'gate':
'dockerfile',
'trigger':
'instruction',
'params': [{
'name': 'instruction',
'value': 'RUN'
}, {
'name': 'check',
'value': 'exists'
}]
}, {
'action':
'STOP',
'gate':
'dockerfile',
'trigger':
'instruction',
'params': [{
'name': 'instruction',
'value': 'USER'
}, {
'name': 'CHECK',
'value': 'not_exists'
}]
}, {
'action':
'STOP',
'gate':
'dockerfile',
'trigger':
'instruction',
'params': [{
'name': 'instruction',
'value': 'RUN'
}, {
'name': 'CHECK',
'value': '=',
'check_value': 'yum update -y'
}]
}]
}],
'whitelists': [],
'mappings': [{
'registry': '*',
'repository': '*',
'image': {
'type': 'tag',
'value': '*'
},
'policy_id': 'policy1',
'whitelist_ids': []
}]
}
built = build_bundle(multi_gate_bundle, for_tag=test_tag)
assert not built.init_errors
logger.info('Got: {}'.format(built))
db = get_thread_scoped_session()
img_obj = get_image_named(db, test_data_env_with_images_loaded, 'ruby')
assert img_obj is not None
assert img_obj is not None, 'Failed to get an image object to test'
evaluation = built.execute(img_obj,
tag=test_tag,
context=ExecutionContext(db_session=db,
configuration={}))
assert evaluation is not None, 'Got None eval'
logger.info(json.dumps(evaluation.json(), indent=2))
logger.info(json.dumps(evaluation.as_table_json(), indent=2))
def test_verifytrigger(self):
"""
Expects the default image to have exactly 1 verify file changed and 1 missing. For the debian test used that is in:
/usr/share/locale/ for the missing entry and changed file (first entries in the verify analyzer output for the latest debian image at the test time
:return:
"""
print('Default params check')
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__)
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(len(t.fired), 2)
self.assertTrue(
('missing' in t.fired[0].msg and 'changed' in t.fired[1].msg)
or ('missing' in t.fired[1].msg and 'changed' in t.fired[0].msg))
print('Specific dirs and check only changed')
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__,
only_directories='/bin,/usr/bin,/usr/local/bin,/usr/share/locale',
check='changed')
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(len(t.fired), 1)
self.assertTrue('changed' in t.fired[0].msg)
print('Check only missing')
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__, check='missing')
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(len(t.fired), 1)
self.assertTrue('missing' in t.fired[0].msg)
print('Specific pkg, with issues')
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__,
only_packages='perl-base,libapt-pkg5.0,tzdata')
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(len(t.fired), 2)
self.assertTrue(
('missing' in t.fired[0].msg and 'changed' in t.fired[1].msg)
or ('missing' in t.fired[1].msg and 'changed' in t.fired[0].msg))
print('Specific pkg, with issues')
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__, only_packages='perl-base,tzdata')
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(len(t.fired), 1)
self.assertTrue('missing' in t.fired[0].msg)
print('Specific pkg, with issues')
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__,
only_packages='libapt-pkg5.0,tzdata')
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(len(t.fired), 1)
self.assertTrue('changed' in t.fired[0].msg)
print('Specific pkg, no issues')
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__,
only_packages='tzdata,openssl-client')
db = get_thread_scoped_session()
db.refresh(self.test_image)
test_context = gate.prepare_context(self.test_image, test_context)
t.evaluate(self.test_image, test_context)
print(('Fired: {}'.format(t.fired)))
self.assertEqual(len(t.fired), 0)
print('Trying default params on all loaded images')
t, gate, test_context = self.get_initialized_trigger(
VerifyTrigger.__trigger_name__)
for img_id, meta in list(self.test_env.image_map.items()):
if img_id == self.test_image.id:
continue
t.reset()
img_obj = db.query(Image).get((img_id, '0'))
print(('Default params check on img: {}'.format(img_id)))
test_context = gate.prepare_context(img_obj, test_context)
t.evaluate(img_obj, test_context)
print(('Image name: {}, id: {}, Fired count: {}\nFired: {}'.format(
meta.get('name'), img_id, len(t.fired), t.fired)))