def load_and_normalize_packages(self, package_analysis_json, image_obj):
"""
Loads and normalizes package data from all distros
:param image_obj:
:param package_analysis_json:
:return: list of Package objects that can be added to an image
"""
pkgs = []
img_distro = DistroNamespace.for_obj(image_obj)
# pkgs.allinfo handling
pkgs_all = package_analysis_json.get('pkgs.allinfo', {}).values()
if not pkgs_all:
return []
else:
pkgs_all = pkgs_all[0]
for pkg_name, metadata_str in pkgs_all.items():
metadata = json.loads(metadata_str)
p = ImagePackage()
p.distro_name = image_obj.distro_name
p.distro_version = image_obj.distro_version
p.like_distro = image_obj.like_distro
p.name = pkg_name
p.version = metadata.get('version')
p.origin = metadata.get('origin')
p.size = metadata.get('size')
p.arch = metadata.get('arch')
p.license = metadata.get('license') if metadata.get(
'license') else metadata.get('lics')
p.release = metadata.get('release', 'N/A')
p.pkg_type = metadata.get('type')
p.src_pkg = metadata.get('sourcepkg')
p.image_user_id = image_obj.user_id
p.image_id = image_obj.id
if 'files' in metadata:
# Handle file data
p.files = metadata.get('files')
if p.release != 'N/A':
p.fullversion = p.version + '-' + p.release
else:
p.fullversion = p.version
if img_distro.flavor == 'DEB':
cleanvers = re.sub(re.escape("+b") + "\d+.*", "", p.version)
spkg = re.sub(re.escape("-" + cleanvers), "", p.src_pkg)
else:
spkg = re.sub(re.escape("-" + p.version) + ".*", "", p.src_pkg)
p.normalized_src_pkg = spkg
pkgs.append(p)
if pkgs:
return pkgs
else:
log.warn('Pkg Allinfo not found, reverting to using pkgs.all')
all_pkgs = package_analysis_json['pkgs.all']['base']
all_pkgs_src = package_analysis_json['pkgs_plus_source.all']['base']
for pkg_name, version in all_pkgs.items():
p = ImagePackage()
p.image_user_id = image_obj.user_id
p.image_id = image_obj.id
p.name = pkg_name
p.version = version
p.fullversion = all_pkgs_src[pkg_name]
if img_distro.flavor == 'RHEL':
name, parsed_version, release, epoch, arch = split_rpm_filename(
pkg_name + '-' + version + '.tmparch.rpm')
p.version = parsed_version
p.release = release
p.pkg_type = 'RPM'
p.origin = 'N/A'
p.src_pkg = 'N/A'
p.license = 'N/A'
p.arch = 'N/A'
elif img_distro.flavor == 'DEB':
try:
p.version, p.release = version.split('-')
except:
p.version = version
p.release = None
return pkgs
test_image.dockerfile_contents = 'FROM BLAH'
test_package = ImagePackage(image_user_id=test_user_id,
image_id=test_img_id,
name='testpackage',
version='1.0',
pkg_type='RPM')
test_package.src_pkg = 'testpackage'
test_package.distro_name = 'centos'
test_package.distro_version = '7'
test_package.like_distro = 'centos'
test_package.license = 'apache2'
test_package.fullversion = '1.0'
test_package.normalized_src_pkg = '1.0'
test_package.release = ''
test_package.size = 1000
test_package.origin = 'upstream'
test_package.arch = 'x86_64'
test_package.image = test_image
test_cve = Vulnerability(id='CVE123', namespace_name='centos:7')
test_cve.severity = 'High'
test_cve.description = 'some test cve'
test_cve.cvss2_score = '1.0'
test_cve.metadata_json = {}
test_cve.cvss2_vectors = ''
test_cve.link = 'http://mitre.com/cve123'
test_fixedin = FixedArtifact(vulnerability_id=test_cve.id)
test_fixedin.name = 'testpackage'
test_fixedin.version = '1.1'
def test_cve_updates(test_data_env):
test_env = test_data_env
test_env.init_feeds()
test_user_id = 'test1'
test_img_id = 'img1'
test_image = Image(user_id=test_user_id, id=test_img_id, distro_name='centos', distro_version='7')
test_image.familytree_json = [test_img_id]
test_image.layers_json = [test_img_id]
test_image.layer_info_json = ['somelayer_here']
test_image.like_distro = 'centos'
test_image.state = 'analyzed'
test_image.digest = 'digest1'
test_image.anchore_type = 'undefined'
test_image.dockerfile_mode = 'Guessed'
test_image.docker_history_json = ['line1', 'line2']
test_image.docker_data_json = {'Config': {}, 'ContainerConfig': {}}
test_image.dockerfile_contents = 'FROM BLAH'
test_package = ImagePackage(image_user_id=test_user_id, image_id=test_img_id, name='testpackage', version='1.0', pkg_type='RPM')
test_package.src_pkg = 'testpackage'
test_package.distro_name = 'centos'
test_package.distro_version = '7'
test_package.like_distro = 'centos'
test_package.license = 'apache2'
test_package.fullversion = '1.0'
test_package.normalized_src_pkg = '1.0'
test_package.release = ''
test_package.size = 1000
test_package.origin = 'upstream'
test_package.arch = 'x86_64'
test_package.image = test_image
test_cve = Vulnerability(id='CVE123', namespace_name='centos:7')
test_cve.severity = 'High'
test_cve.description = 'some test cve'
test_cve.cvss2_score = '1.0'
test_cve.metadata_json = {}
test_cve.cvss2_vectors = ''
test_cve.link = 'http://mitre.com/cve123'
test_fixedin = FixedArtifact(vulnerability_id=test_cve.id)
test_fixedin.name = 'testpackage'
test_fixedin.version = '1.1'
test_fixedin.version_format = 'rpm'
test_fixedin.epochless_version = '1.1'
test_fixedin.include_later_versions = True
test_fixedin.parent = test_cve
test_cve.fixed_in = [test_fixedin]
test_vulnin = VulnerableArtifact(vulnerability_id=test_cve.id)
test_vulnin.name = 'testpackage'
test_vulnin.version = '0.9'
test_vulnin.epochless_version = '0.9'
test_vulnin.namespace_name = 'centos:7'
test_vulnin.version_format = 'rpm'
test_vulnin.include_previous_versions = False
test_vulnin.parent = test_cve
test_cve.vulnerable_in = [test_vulnin]
db = get_session()
try:
db.add(test_image)
db.add(test_package)
db.commit()
except sqlalchemy.exc.IntegrityError:
db.rollback()
except Exception:
logger.exception('Unexpected failure')
raise
db = get_session()
try:
db.add(test_cve)
feeds.process_updated_vulnerability(db, test_cve)
db.commit()
except sqlalchemy.exc.IntegrityError:
logger.exception('Failed!')
db.rollback()
finally:
db = get_session()
i = db.query(Image).get((test_img_id, test_user_id))
print(('Vulns: {}'.format(i.vulnerabilities())))
db.commit()
test_cve2 = Vulnerability(id='CVE123', namespace_name='centos:7')
test_cve2.severity = 'Medium'
test_cve2.description = 'some test cve'
test_cve2.cvss2_score = '1.0'
test_cve2.metadata_json = {}
test_cve2.cvss2_vectors = ''
test_cve2.link = 'http://mitre.com/cve123'
fix2 = FixedArtifact(name='pkg2', version='1.2', epochless_version='1.2')
fix2.namespace_name = 'centos:7'
fix2.vulnerability_id = test_cve2.id
test_cve2.fixed_in = [fix2]
db = get_session()
try:
t2 = db.merge(test_cve2)
db.add(t2)
feeds.process_updated_vulnerability(db, t2)
db.commit()
except sqlalchemy.exc.IntegrityError:
logger.exception('Failed!')
db.rollback()
finally:
db = get_session()
i = db.query(Image).get((test_img_id, test_user_id))
print(('Vulns: {}'.format(i.vulnerabilities())))
db.commit()
def test_github_advisory_fixed_in(test_data_env):
test_env = test_data_env
test_env.init_feeds()
test_user_id = 'test1'
test_img_id = 'img1'
test_image = Image(
user_id=test_user_id, id=test_img_id,
distro_name='centos', distro_version='7'
)
test_image.familytree_json = [test_img_id]
test_image.layers_json = [test_img_id]
test_image.layer_info_json = ['somelayer_here']
test_image.like_distro = 'centos'
test_image.state = 'analyzed'
test_image.digest = 'digest1'
test_image.anchore_type = 'undefined'
test_image.dockerfile_mode = 'Guessed'
test_image.docker_history_json = ['line1', 'line2']
test_image.docker_data_json = {'Config': {}, 'ContainerConfig': {}}
test_image.dockerfile_contents = 'FROM BLAH'
test_package = ImagePackage(
image_user_id=test_user_id, image_id=test_img_id,
name='testpackage', version='1.0', pkg_type='python'
)
test_package.src_pkg = 'testpackage'
test_package.distro_name = 'centos'
test_package.distro_version = '7'
test_package.like_distro = 'centos'
test_package.license = 'apache2'
test_package.fullversion = '1.0'
test_package.normalized_src_pkg = '1.0'
test_package.release = ''
test_package.size = 1000
test_package.origin = 'upstream'
test_package.arch = 'x86_64'
test_package.image = test_image
test_cve = Vulnerability(id='GHSA-rpch-cqj9-h65r', namespace_name='github:python')
test_cve.severity = 'High'
test_cve.description = 'some advisory ghsa'
test_cve.link = 'http://mitre.com/cve123'
test_fixedin = FixedArtifact(vulnerability_id=test_cve.id)
test_fixedin.name = 'testpackage'
test_fixedin.version = 'None'
test_fixedin.fix_metadata = {'first_patched_version': '1.2'}
test_fixedin.version_format = 'semver'
test_fixedin.parent = test_cve
test_cve.fixed_in = [test_fixedin]
db = get_session()
try:
db.add(test_image)
db.add(test_package)
db.commit()
except sqlalchemy.exc.IntegrityError:
db.rollback()
except Exception:
logger.exception('Unexpected failure')
raise
db = get_session()
# XXX This needs to be a fixture
try:
db.add(test_cve)
feeds.process_updated_vulnerability(db, test_cve)
db.commit()
except sqlalchemy.exc.IntegrityError:
logger.exception('Failed!')
db.rollback()
db = get_session()
image_vuln = db.query(Image).get((test_img_id, test_user_id))
# should be one vulnerability
vulnerabilities = image_vuln.vulnerabilities()
assert len(vulnerabilities) == 1
img_pkg_vuln = vulnerabilities[0]
assert img_pkg_vuln.fixed_in() == '1.2'
def test_cve_updates(test_data_env):
test_env = test_data_env
test_env.init_feeds()
test_user_id = "test1"
test_img_id = "img1"
test_image = Image(user_id=test_user_id,
id=test_img_id,
distro_name="centos",
distro_version="7")
test_image.familytree_json = [test_img_id]
test_image.layers_json = [test_img_id]
test_image.layer_info_json = ["somelayer_here"]
test_image.like_distro = "centos"
test_image.state = "analyzed"
test_image.digest = "digest1"
test_image.anchore_type = "undefined"
test_image.dockerfile_mode = "Guessed"
test_image.docker_history_json = ["line1", "line2"]
test_image.docker_data_json = {"Config": {}, "ContainerConfig": {}}
test_image.dockerfile_contents = "FROM BLAH"
test_package = ImagePackage(
image_user_id=test_user_id,
image_id=test_img_id,
name="testpackage",
version="1.0",
pkg_type="RPM",
)
test_package.src_pkg = "testpackage"
test_package.distro_name = "centos"
test_package.distro_version = "7"
test_package.like_distro = "centos"
test_package.license = "apache2"
test_package.fullversion = "1.0"
test_package.normalized_src_pkg = "1.0"
test_package.release = ""
test_package.size = 1000
test_package.origin = "upstream"
test_package.arch = "x86_64"
test_package.image = test_image
test_cve = Vulnerability(id="CVE123", namespace_name="centos:7")
test_cve.severity = "High"
test_cve.description = "some test cve"
test_cve.cvss2_score = "1.0"
test_cve.metadata_json = {}
test_cve.cvss2_vectors = ""
test_cve.link = "http://mitre.com/cve123"
test_fixedin = FixedArtifact(vulnerability_id=test_cve.id)
test_fixedin.name = "testpackage"
test_fixedin.version = "1.1"
test_fixedin.version_format = "rpm"
test_fixedin.epochless_version = "1.1"
test_fixedin.include_later_versions = True
test_fixedin.parent = test_cve
test_cve.fixed_in = [test_fixedin]
test_vulnin = VulnerableArtifact(vulnerability_id=test_cve.id)
test_vulnin.name = "testpackage"
test_vulnin.version = "0.9"
test_vulnin.epochless_version = "0.9"
test_vulnin.namespace_name = "centos:7"
test_vulnin.version_format = "rpm"
test_vulnin.include_previous_versions = False
test_vulnin.parent = test_cve
test_cve.vulnerable_in = [test_vulnin]
db = get_session()
try:
db.add(test_image)
db.add(test_package)
db.commit()
except sqlalchemy.exc.IntegrityError:
db.rollback()
except Exception:
logger.exception("Unexpected failure")
raise
db = get_session()
try:
db.add(test_cve)
feeds.process_updated_vulnerability(db, test_cve)
db.commit()
except sqlalchemy.exc.IntegrityError:
logger.exception("Failed!")
db.rollback()
finally:
db = get_session()
i = db.query(Image).get((test_img_id, test_user_id))
print(("Vulns: {}".format(i.vulnerabilities())))
db.commit()
test_cve2 = Vulnerability(id="CVE123", namespace_name="centos:7")
test_cve2.severity = "Medium"
test_cve2.description = "some test cve"
test_cve2.cvss2_score = "1.0"
test_cve2.metadata_json = {}
test_cve2.cvss2_vectors = ""
test_cve2.link = "http://mitre.com/cve123"
fix2 = FixedArtifact(name="pkg2", version="1.2", epochless_version="1.2")
fix2.namespace_name = "centos:7"
fix2.vulnerability_id = test_cve2.id
test_cve2.fixed_in = [fix2]
db = get_session()
try:
t2 = db.merge(test_cve2)
db.add(t2)
feeds.process_updated_vulnerability(db, t2)
db.commit()
except sqlalchemy.exc.IntegrityError:
logger.exception("Failed!")
db.rollback()
finally:
db = get_session()
i = db.query(Image).get((test_img_id, test_user_id))
print(("Vulns: {}".format(i.vulnerabilities())))
db.commit()
def test_github_advisory_fixed_in(test_data_env):
test_env = test_data_env
test_env.init_feeds()
test_user_id = "test1"
test_img_id = "img1"
test_image = Image(user_id=test_user_id,
id=test_img_id,
distro_name="centos",
distro_version="7")
test_image.familytree_json = [test_img_id]
test_image.layers_json = [test_img_id]
test_image.layer_info_json = ["somelayer_here"]
test_image.like_distro = "centos"
test_image.state = "analyzed"
test_image.digest = "digest1"
test_image.anchore_type = "undefined"
test_image.dockerfile_mode = "Guessed"
test_image.docker_history_json = ["line1", "line2"]
test_image.docker_data_json = {"Config": {}, "ContainerConfig": {}}
test_image.dockerfile_contents = "FROM BLAH"
test_package = ImagePackage(
image_user_id=test_user_id,
image_id=test_img_id,
name="testpackage",
version="1.0",
pkg_type="python",
)
test_package.src_pkg = "testpackage"
test_package.distro_name = "centos"
test_package.distro_version = "7"
test_package.like_distro = "centos"
test_package.license = "apache2"
test_package.fullversion = "1.0"
test_package.normalized_src_pkg = "1.0"
test_package.release = ""
test_package.size = 1000
test_package.origin = "upstream"
test_package.arch = "x86_64"
test_package.image = test_image
test_cve = Vulnerability(id="GHSA-rpch-cqj9-h65r",
namespace_name="github:python")
test_cve.severity = "High"
test_cve.description = "some advisory ghsa"
test_cve.link = "http://mitre.com/cve123"
test_fixedin = FixedArtifact(vulnerability_id=test_cve.id)
test_fixedin.name = "testpackage"
test_fixedin.version = "None"
test_fixedin.fix_metadata = {"first_patched_version": "1.2"}
test_fixedin.version_format = "semver"
test_fixedin.parent = test_cve
test_cve.fixed_in = [test_fixedin]
db = get_session()
try:
db.add(test_image)
db.add(test_package)
db.commit()
except sqlalchemy.exc.IntegrityError:
db.rollback()
except Exception:
logger.exception("Unexpected failure")
raise
db = get_session()
# XXX This needs to be a fixture
try:
db.add(test_cve)
feeds.process_updated_vulnerability(db, test_cve)
db.commit()
except sqlalchemy.exc.IntegrityError:
logger.exception("Failed!")
db.rollback()
db = get_session()
image_vuln = db.query(Image).get((test_img_id, test_user_id))
# should be one vulnerability
vulnerabilities = image_vuln.vulnerabilities()
assert len(vulnerabilities) == 1
img_pkg_vuln = vulnerabilities[0]
assert img_pkg_vuln.fixed_in() == "1.2"