def test_have_vulnerabilities_for(test_data_env): _init_distro_mappings() sync_feeds(test_data_env) failz = DistroNamespace(name='somecrzy', version='8', like_distro='debian') passes = DistroNamespace(name='debian', version='8', like_distro='debian') assert not vulnerabilities.have_vulnerabilities_for(failz), 'Should not have vulns for ' + failz.namespace_name assert vulnerabilities.have_vulnerabilities_for(passes), 'Should have vulns for ' + passes.namespace_name
def test_relation_mapping(self): for d in self.valid_distros: ns = DistroNamespace(name=d[0], version=d[1]) self.assertIsNotNone(ns) self.assertEqual(ns.name, d[0]) self.assertEqual(ns.version, d[1]) for d in self.unmapped_distros: ns = DistroNamespace(name=d[0], version=d[1]) self.assertIsNotNone(ns)
def test_relation_mapping(anchore_db): for d in valid_distros: ns = DistroNamespace(name=d[0], version=d[1]) assert ns is not None assert ns.name == d[0] assert ns.version == d[1] for d in unmapped_distros: ns = DistroNamespace(name=d[0], version=d[1]) assert ns is not None
def match_and_vulnerable(vuln_obj, package_obj): """ Given a VulnerableArtifact record, is the given package object a match indicating that the package is vulnerable. :param vuln_obj: :param package_obj: :param has_fix: boolean indicating if there is a corresponding fix record :return: """ if not isinstance(vuln_obj, VulnerableArtifact): raise TypeError('Expected a VulnerableArtifact type, got: {}'.format( type(vuln_obj))) dist = DistroNamespace.for_obj(package_obj) flavor = dist.flavor # Double-check names if vuln_obj.name != package_obj.name and vuln_obj.name != package_obj.normalized_src_pkg: log.warn( 'Name mismatch in vulnerable check. This should not happen: Fix: {}, Package: {}, Package_Norm_Src: {}, Package_Src: {}' .format(vuln_obj.name, package_obj.name, package_obj.normalized_src_pkg, package_obj.src_pkg)) return False # Is it a catch-all record? Explicit 'None' or 'all' versions indicate all versions of the named package are vulnerable. if vuln_obj.epochless_version in ['all', 'None']: return True # Is the package older than the fix? if package_obj.fullversion == vuln_obj.epochless_version or package_obj.version == vuln_obj.epochless_version: return True # Newer or the same return False
def evaluate(self, image_obj, context): # Map to a namespace ns = DistroNamespace.for_obj(image_obj) oldest_update = None if ns: vulnerability_feed = DataFeeds.instance().vulnerabilities for namespace_name in ns.like_namespace_names: # Check feed names groups = vulnerability_feed.group_by_name(namespace_name) if groups: # No records yet, but we have the feed, so may just not have any data yet oldest_update = groups[0].last_sync break if self.max_age.value() is not None: try: if oldest_update is not None: oldest_update = calendar.timegm(oldest_update.timetuple()) mintime = time.time() - int(int(self.max_age.value()) * 86400) if oldest_update < mintime: self._fire(msg="The vulnerability feed for this image distro is older than MAXAGE ("+str(self.max_age.value())+") days") else: self._fire( msg="The vulnerability feed for this image distro is older than MAXAGE (" + str(self.max_age.value()) + ") days") except Exception as err: self._fire(msg="Cannot perform data feed up-to-date check - message from server: " + str(err))
def candidates_for_package(package_obj, distro_namespace=None): """ Return all vulnerabilities for the named package with the specified distro. Will apply to any version of the package. If version is used, will filter to only those for the specified version. :param package_obj: the package to match against :param distro_namespace: the DistroNamespace object to match against (typically computed :return: List of Vulnerabilities """ db = get_thread_scoped_session() if not distro_namespace: namespace_name = DistroNamespace.for_obj(package_obj).namespace_name else: namespace_name = distro_namespace # Match the namespace and package name or src pkg name fix_candidates = db.query(FixedArtifact).filter( FixedArtifact.namespace_name == namespace_name, or_(FixedArtifact.name == package_obj.name, FixedArtifact.name == package_obj.normalized_src_pkg)).all() # Match the namespace and package name or src pkg name vulnerable_candidates = db.query(VulnerableArtifact).filter( VulnerableArtifact.namespace_name == namespace_name, or_(VulnerableArtifact.name == package_obj.name, VulnerableArtifact.name == package_obj.normalized_src_pkg)).all() return fix_candidates, vulnerable_candidates
def test_cve_mapping(anchore_db): for d in distros: ns = DistroNamespace(name=d[0], version=d[1]) r = {'flavor': ns.flavor, 'namespace_names': ns.like_namespace_names} print(ns.namespace_name + ' ->' + json.dumps(r, indent=2)) assert ns is not None assert ns.name == d[0] assert ns.version == d[1]
def namespace_has_no_feed(name, version): """ Returns true if the given namespace has no direct CVE feed and false if it does. :return: boolean if name,version tuple does not have a feed of its own """ ns = DistroNamespace.as_namespace_name(name, version) found = ThreadLocalFeedGroupNameCache.lookup( ns) # Returns a tuple (name, enabled:bool) return not found or not found[1]
def test_cve_mapping(self): for d in self.distros: ns = DistroNamespace(name=d[0], version=d[1]) r = { 'flavor': ns.flavor, 'namespace_names': ns.like_namespace_names } print(ns.namespace_name + ' ->' + json.dumps(r, indent=2)) self.assertIsNotNone(ns) self.assertEqual(ns.name, d[0]) self.assertEqual(ns.version, d[1])
def match_but_not_fixed(fix_obj, package_obj): """ Does the FixedArtifact match the package as a vulnerability such that the fix indicates the package is *not* fixed and is therefore vulnerable. :param fix_obj: as FixedArtifact record :param package_obj: an ImagePackage record :return: True if the names match and the fix record indicates the package is vulnerable and not fixed. False if no match or fix is applied and no vulnerability match """ if not isinstance(fix_obj, FixedArtifact): raise TypeError('Expected a FixedArtifact type, got: {}'.format( type(fix_obj))) dist = DistroNamespace.for_obj(package_obj) flavor = dist.flavor log.debug('Package: {}, Fix: {}, Flavor: {}'.format( package_obj.name, fix_obj.name, flavor)) # Double-check names if fix_obj.name != package_obj.name and fix_obj.name != package_obj.normalized_src_pkg: log.warn( 'Name mismatch in fix check. This should not happen: Fix: {}, Package: {}, Package_Norm_Src: {}, Package_Src: {}' .format(fix_obj.name, package_obj.name, package_obj.normalized_src_pkg, package_obj.src_pkg)) return False # Handle the case where there is no version, indicating no fix available, all versions are vulnerable. # Is it a catch-all record? Explicit 'None' versions indicate all versions of the named package are vulnerable. if fix_obj.version == 'None': return True # Is the package older than the fix? if flavor == 'RHEL': if rpm_compare_versions(package_obj.name, package_obj.fullversion, fix_obj.name, fix_obj.epochless_version) < 0: log.debug('rpm Compared: {} < {}: True'.format( package_obj.fullversion, fix_obj.epochless_version)) return True elif flavor == 'DEB': if dpkg_compare_versions(package_obj.fullversion, 'lt', fix_obj.epochless_version): log.debug('dpkg Compared: {} < {}: True'.format( package_obj.fullversion, fix_obj.epochless_version)) return True elif flavor == 'ALPINE': if apkg_compare_versions(package_obj.fullversion, 'lt', fix_obj.epochless_version): log.debug('apkg Compared: {} < {}: True'.format( package_obj.fullversion, fix_obj.epochless_version)) return True # Newer or the same return False
def vulnerabilities_for_package(package_obj): """ Given an ImagePackage object, return the vulnerabilities that it matches. :param package_obj: :return: list of Vulnerability objects """ log.debug('Finding vulnerabilities for package: {} - {}'.format( package_obj.name, package_obj.version)) matches = [] dist = DistroNamespace(package_obj.distro_name, package_obj.distro_version, package_obj.like_distro) db = get_thread_scoped_session() namespace_name_to_use = dist.namespace_name # All options are the same, no need to loop if len(set(dist.like_namespace_names)) > 1: # Look for exact match first if not DataFeeds.instance().vulnerabilities.group_by_name( dist.namespace_name): # Check all options for distro/flavor mappings, stop at first with records present for namespace_name in dist.like_namespace_names: record_count = db.query(Vulnerability).filter( Vulnerability.namespace_name == namespace_name).count() if record_count > 0: namespace_name_to_use = namespace_name break fix_candidates, vulnerable_candidates = candidates_for_package( package_obj, namespace_name_to_use) for candidate in fix_candidates: # De-dup evaluations based on the underlying vulnerability_id. For packages where src has many binary builds, once we have a match we have a match. if candidate.vulnerability_id not in map( lambda x: x.vulnerability_id, matches) and match_but_not_fixed( candidate, package_obj): matches.append(candidate) for candidate in vulnerable_candidates: if candidate.vulnerability_id not in map( lambda x: x.vulnerability_id, matches) and match_and_vulnerable(candidate, package_obj): matches.append(candidate) return matches
def evaluate(self, image_obj, context): # Map to a namespace ns = DistroNamespace.for_obj(image_obj) oldest_update = None if ns: for namespace_name in ns.like_namespace_names: # Check feed names for feed in feed_registry.registered_vulnerability_feed_names( ): # First match, assume only one matches for the namespace group = get_feed_group_detached(feed, namespace_name) if group: # No records yet, but we have the feed, so may just not have any data yet oldest_update = group.last_sync logger.debug( "Found date for oldest update in feed %s group %s date = %s", feed, group.name, oldest_update, ) break if self.max_age.value() is not None: try: if oldest_update is not None: oldest_update = calendar.timegm(oldest_update.timetuple()) mintime = time.time() - int( int(self.max_age.value()) * 86400) if oldest_update < mintime: self._fire( msg= "The vulnerability feed for this image distro is older than MAXAGE (" + str(self.max_age.value()) + ") days") else: self._fire( msg= "The vulnerability feed for this image distro is older than MAXAGE (" + str(self.max_age.value()) + ") days") except Exception as err: self._fire( msg= "Cannot perform data feed up-to-date check - message from server: " + str(err))
def test_distromappings(anchore_db): _init_distro_mappings() c7 = DistroNamespace(name='centos', version='7', like_distro='centos') assert c7.mapped_names() == [] assert c7.like_namespace_names == ['rhel:7'] r7 = DistroNamespace(name='rhel', version='7', like_distro='centos') assert set(r7.mapped_names()) == {'centos', 'fedora', 'rhel'} assert r7.like_namespace_names == ['rhel:7'] assert sorted(DistroMapping.distros_mapped_to('rhel', '7')) == sorted([ DistroTuple('rhel', '7', 'RHEL'), DistroTuple('centos', '7', 'RHEL'), DistroTuple('fedora', '7', 'RHEL') ])
def find_vulnerable_image_packages(vulnerability_obj): """ Given a vulnerability object, find images that are affected via their package manifests. Result may have duplicates based on match type, caller must de-dup if desired. :param vulnerability_obj: :return: list of ImagePackage objects """ db = get_thread_scoped_session() distro, version = vulnerability_obj.namespace_name.split(":", 1) dist = DistroNamespace(distro, version) related_names = ( dist.mapped_names() ) # Returns list of names that map to this one, not including itself necessarily # related_names = get_namespace_related_names(distro, version, mapped_names) # TODO would like a better way to do the pkg_type <-> namespace_name mapping, with other side in ImagePackage.vulnerabilities_for_package likematch = None if (":maven" in vulnerability_obj.namespace_name or "java" in vulnerability_obj.namespace_name): likematch = "java" elif (":ruby" in vulnerability_obj.namespace_name or "gem" in vulnerability_obj.namespace_name): likematch = "gem" elif (":js" in vulnerability_obj.namespace_name or "npm" in vulnerability_obj.namespace_name): likematch = "npm" elif "python" in vulnerability_obj.namespace_name: likematch = "python" try: affected = [] if vulnerability_obj.fixed_in: # Check the fixed_in records for fix_rec in vulnerability_obj.fixed_in: package_candidates = [] # Find packages of related distro names with compatible versions, this does not have to be precise, just an initial filter. pkgs = (db.query(ImagePackage).filter( ImagePackage.distro_name.in_(related_names), ImagePackage.distro_version.like(dist.version + "%"), or_( ImagePackage.name == fix_rec.name, ImagePackage.normalized_src_pkg == fix_rec.name, ), ).all()) package_candidates += pkgs # add non distro candidates if likematch: pkgs = (db.query(ImagePackage).filter( ImagePackage.pkg_type.in_(nonos_package_types), ImagePackage.pkg_type.like(likematch), or_( ImagePackage.name == fix_rec.name, ImagePackage.normalized_src_pkg == fix_rec.name, ), ).all()) package_candidates += pkgs for candidate in package_candidates: if fix_rec.match_but_not_fixed(candidate): affected.append(candidate) if vulnerability_obj.vulnerable_in: # Check the vulnerable_in records for vuln_rec in vulnerability_obj.vulnerable_in: package_candidates = [] # Find packages of related distro names with compatible versions, this does not have to be precise, just an initial filter. pkgs = (db.query(ImagePackage).filter( ImagePackage.distro_name.in_(related_names), ImagePackage.distro_version.like(dist.version + "%"), or_( ImagePackage.name == vuln_rec.name, ImagePackage.normalized_src_pkg == vuln_rec.name, ), ).all()) package_candidates += pkgs for candidate in package_candidates: if vuln_rec.match_and_vulnerable(candidate): affected.append(candidate) return affected except Exception as e: logger.exception( "Failed to query and find packages affected by vulnerability: {}". format(vulnerability_obj)) raise
def evaluate(self, image_obj, context): if not have_vulnerabilities_for(DistroNamespace.for_obj(image_obj)): self._fire( msg= "Feed data unavailable, cannot perform CVE scan for distro: " + str(image_obj.distro_namespace))
def find_vulnerable_image_packages(vulnerability_obj): """ Given a vulnerability object, find images that are affected via their package manifests. Result may have duplicates based on match type, caller must de-dup if desired. :param vulnerability_obj: :return: list of ImagePackage objects """ db = get_thread_scoped_session() distro, version = vulnerability_obj.namespace_name.split(':', 1) dist = DistroNamespace(distro, version) related_names = dist.mapped_names() # Returns list of names that map to this one, not including itself necessarily # Filter related_names down based on the presence of actual feeds/cve data. If there is an actual feed for a name, remove it from the list. # Only do this if the list of related names is not just the name itself. (e.g. alpine = [alpine]). if related_names != [distro]: # Ensure we don't include any names that actually have a feed (can happen when new feeds arrive before the mapped_names() source # is updated to break the 'like' relation between the distros. related_names = [x for x in related_names if namespace_has_no_feed(x, version)] # This is a weird case because it basically means that this distro doesn't map to itself as far as mapped_names() is # concerned, but since that could be code lagging data (e.g. new feed group added for a new distro), add the name itself # back into the list. if distro not in related_names and not namespace_has_no_feed(distro, version): related_names.append(distro) # TODO would like a better way to do the pkg_type <-> namespace_name mapping, with other side in ImagePackage.vulnerabilities_for_package likematch = None if ':maven' in vulnerability_obj.namespace_name or 'java' in vulnerability_obj.namespace_name: likematch = 'java' elif ':ruby' in vulnerability_obj.namespace_name or 'gem' in vulnerability_obj.namespace_name: likematch = 'gem' elif ':js' in vulnerability_obj.namespace_name or 'npm' in vulnerability_obj.namespace_name: likematch = 'npm' elif 'python' in vulnerability_obj.namespace_name: likematch = 'python' try: affected = [] if vulnerability_obj.fixed_in: # Check the fixed_in records for fix_rec in vulnerability_obj.fixed_in: package_candidates = [] # Find packages of related distro names with compatible versions, this does not have to be precise, just an initial filter. pkgs = db.query(ImagePackage).filter(ImagePackage.distro_name.in_(related_names), ImagePackage.distro_version.like(dist.version + '%'), or_(ImagePackage.name == fix_rec.name, ImagePackage.normalized_src_pkg == fix_rec.name)).all() package_candidates += pkgs # add non distro candidates if likematch: pkgs = db.query(ImagePackage).filter(ImagePackage.pkg_type.in_(nonos_package_types), ImagePackage.pkg_type.like(likematch), or_(ImagePackage.name == fix_rec.name, ImagePackage.normalized_src_pkg == fix_rec.name)).all() package_candidates += pkgs for candidate in package_candidates: if fix_rec.match_but_not_fixed(candidate): affected.append(candidate) # if vulnerability_obj.vulnerable_in: # # Check the vulnerable_in records # for vuln_rec in vulnerability_obj.vulnerable_in: # package_candidates = [] # # Find packages of related distro names with compatible versions, this does not have to be precise, just an initial filter. # pkgs = db.query(ImagePackage).filter(ImagePackage.distro_name.in_(related_names), # ImagePackage.distro_version.like(dist.version + '%'), # or_(ImagePackage.name == fix_rec.name, # ImagePackage.normalized_src_pkg == fix_rec.name)).all() # package_candidates += pkgs # for candidate in package_candidates: # if vuln_rec.match_and_vulnerable(candidate): # affected.append(candidate) return affected except Exception as e: log.exception('Failed to query and find packages affected by vulnerability: {}'.format(vulnerability_obj)) raise
def get_image_vulnerabilities(user_id, image_id, force_refresh=False): """ Return the vulnerability listing for the specified image and load from catalog if not found and specifically asked to do so. Example json output: { "multi" : { "url_column_index" : 7, "result" : { "rows" : [], "rowcount" : 0, "colcount" : 8, "header" : [ "CVE_ID", "Severity", "*Total_Affected", "Vulnerable_Package", "Fix_Available", "Fix_Images", "Rebuild_Images", "URL" ] }, "querycommand" : "/usr/lib/python2.7/site-packages/anchore/anchore-modules/multi-queries/cve-scan.py /ebs_data/anchore/querytmp/queryimages.7026386 /ebs_data/anchore/data /ebs_data/anchore/querytmp/query.59057288 all", "queryparams" : "all", "warns" : [ "0005b136f0fb (prom/prometheus:master) cannot perform CVE scan: no CVE data is currently available for the detected base distro type (busybox:unknown_version,busybox:v1.26.2)" ] } } :param user_id: user id of image to evaluate :param image_id: image id to evaluate :param force_refresh: if true, flush and recompute vulnerabilities rather than returning current values :return: """ # Has image? db = get_session() try: img = db.query(Image).get((image_id, user_id)) vulns = [] if not img: abort(404) else: if force_refresh: log.info('Forcing refresh of vulnerabiltiies for {}/{}'.format(user_id, image_id)) try: current_vulns = img.vulnerabilities() log.info('Removing {} current vulnerabilities for {}/{} to rescan'.format(len(current_vulns), user_id, image_id)) for v in current_vulns: db.delete(v) db.flush() vulns = vulnerabilities_for_image(img) log.info('Adding {} vulnerabilities from rescan to {}/{}'.format(len(vulns), user_id, image_id)) for v in vulns: db.add(v) db.commit() except Exception as e: log.exception('Error refreshing cve matches for image {}/{}'.format(user_id, image_id)) db.rollback() abort(Response('Error refreshing vulnerability listing for image.', 500)) db = get_session() db.refresh(img) else: vulns = img.vulnerabilities() # Has vulnerabilities? warns = [] if not vulns: vulns = [] ns = DistroNamespace.for_obj(img) if not have_vulnerabilities_for(ns): warns = ['No vulnerability data available for image distro: {}'.format(ns.namespace_name)] rows = [] for vuln in vulns: # if vuln.vulnerability.fixed_in: # fixes_in = filter(lambda x: x.name == vuln.pkg_name or x.name == vuln.package.normalized_src_pkg, # vuln.vulnerability.fixed_in) # fix_available_in = fixes_in[0].version if fixes_in else 'None' # else: # fix_available_in = 'None' rows.append([ vuln.vulnerability_id, vuln.vulnerability.severity, 1, vuln.pkg_name + '-' + vuln.package.fullversion, str(vuln.fixed_in()), vuln.pkg_image_id, 'None', # Always empty this for now vuln.vulnerability.link ] ) vuln_listing = { 'multi': { 'url_column_index': 7, 'result': { 'header': TABLE_STYLE_HEADER_LIST, 'rowcount': len(rows), 'colcount': len(TABLE_STYLE_HEADER_LIST), 'rows': rows }, 'warns': warns } } report = LegacyVulnerabilityReport.from_dict(vuln_listing) resp = ImageVulnerabilityListing(user_id=user_id, image_id=image_id, legacy_report=report) return resp.to_dict() except HTTPException: db.rollback() raise except Exception as e: log.exception('Error checking image {}, {} for vulnerabiltiies. Rolling back'.format(user_id, image_id)) db.rollback() abort(500) finally: db.close()
def test_namespace_support(test_data_env): _init_distro_mappings() sync_feeds(test_data_env) # Not exhaustive, only for the feeds directly in the test data set expected = [ DistroNamespace(name='alpine', version='3.6', like_distro='alpine'), DistroNamespace(name='centos', version='7', like_distro='centos'), DistroNamespace(name='centos', version='7.1', like_distro='centos'), DistroNamespace(name='centos', version='7.3', like_distro='centos'), DistroNamespace(name='ol', version='7.3', like_distro='centos'), DistroNamespace(name='rhel', version='7.1', like_distro='centos'), DistroNamespace(name='debian', version='9', like_distro='debian'), DistroNamespace(name='ubuntu', version='16.10', like_distro='ubuntu') ] fail = [ DistroNamespace(name='alpine', version='3.1', like_distro='alpine'), DistroNamespace(name='alpine', version='3.1.1', like_distro='alpine'), DistroNamespace(name='busybox', version='3', like_distro='busybox'), DistroNamespace(name='linuxmint', version='16', like_distro='debian'), DistroNamespace(name='redhat', version='6', like_distro='centos'), DistroNamespace(name='ubuntu', version='1.0', like_distro='ubuntu'), DistroNamespace(name='centos', version='1.0', like_distro='ubuntu'), DistroNamespace(name='debian', version='1.0', like_distro='ubuntu'), DistroNamespace(name='rhel', version='1.0', like_distro='ubuntu'), DistroNamespace(name='busybox', version='1.0', like_distro='busybox'), DistroNamespace(name='alpine', version='11.0', like_distro='ubuntu'), DistroNamespace(name='fedora', version='25', like_distro='fedora'), DistroNamespace(name='mageia', version='5', like_distro='mandriva,fedora') ] for i in expected: assert vulnerabilities.have_vulnerabilities_for(i), 'Expected vulns for namespace {}'.format(i.namespace_name) for i in fail: assert not vulnerabilities.have_vulnerabilities_for(i), 'Did not expect vulns for namespace {}'.format(i.namespace_name)
def get_image_vulnerabilities(user_id, image_id, force_refresh=False, vendor_only=True): """ Return the vulnerability listing for the specified image and load from catalog if not found and specifically asked to do so. Example json output: { "multi" : { "url_column_index" : 7, "result" : { "rows" : [], "rowcount" : 0, "colcount" : 8, "header" : [ "CVE_ID", "Severity", "*Total_Affected", "Vulnerable_Package", "Fix_Available", "Fix_Images", "Rebuild_Images", "URL" ] }, "querycommand" : "/usr/lib/python2.7/site-packages/anchore/anchore-modules/multi-queries/cve-scan.py /ebs_data/anchore/querytmp/queryimages.7026386 /ebs_data/anchore/data /ebs_data/anchore/querytmp/query.59057288 all", "queryparams" : "all", "warns" : [ "0005b136f0fb (prom/prometheus:master) cannot perform CVE scan: no CVE data is currently available for the detected base distro type (busybox:unknown_version,busybox:v1.26.2)" ] } } :param user_id: user id of image to evaluate :param image_id: image id to evaluate :param force_refresh: if true, flush and recompute vulnerabilities rather than returning current values :param vendor_only: if true, filter out the vulnerabilities that vendors will explicitly not address :return: """ # Has image? db = get_session() try: img = db.query(Image).get((image_id, user_id)) vulns = [] if not img: abort(404) else: if force_refresh: log.info('Forcing refresh of vulnerabiltiies for {}/{}'.format( user_id, image_id)) try: vulns = rescan_image(img, db_session=db) db.commit() except Exception as e: log.exception( 'Error refreshing cve matches for image {}/{}'.format( user_id, image_id)) db.rollback() abort( Response( 'Error refreshing vulnerability listing for image.', 500)) db = get_session() db.refresh(img) vulns = img.vulnerabilities() # Has vulnerabilities? warns = [] if not vulns: vulns = [] ns = DistroNamespace.for_obj(img) if not have_vulnerabilities_for(ns): warns = [ 'No vulnerability data available for image distro: {}'. format(ns.namespace_name) ] rows = [] for vuln in vulns: # if vuln.vulnerability.fixed_in: # fixes_in = filter(lambda x: x.name == vuln.pkg_name or x.name == vuln.package.normalized_src_pkg, # vuln.vulnerability.fixed_in) # fix_available_in = fixes_in[0].version if fixes_in else 'None' # else: # fix_available_in = 'None' # Skip the vulnerability if the vendor_only flag is set to True and the issue won't be addressed by the vendor if vendor_only and vuln.fix_has_no_advisory(): continue rows.append([ vuln.vulnerability_id, vuln.vulnerability.severity, 1, vuln.pkg_name + '-' + vuln.package.fullversion, str(vuln.fixed_in()), vuln.pkg_image_id, 'None', # Always empty this for now vuln.vulnerability.link, vuln.pkg_type, 'vulnerabilities', vuln.vulnerability.namespace_name, vuln.pkg_name, vuln.package.fullversion, ]) vuln_listing = { 'multi': { 'url_column_index': 7, 'result': { 'header': TABLE_STYLE_HEADER_LIST, 'rowcount': len(rows), 'colcount': len(TABLE_STYLE_HEADER_LIST), 'rows': rows }, 'warns': warns } } cpe_vuln_listing = [] try: all_cpe_matches = db.query( ImageCpe, CpeVulnerability).filter(ImageCpe.image_id == image_id).filter( ImageCpe.name == CpeVulnerability.name).filter( ImageCpe.version == CpeVulnerability.version) if not all_cpe_matches: all_cpe_matches = [] cpe_hashes = {} for image_cpe, vulnerability_cpe in all_cpe_matches: cpe_vuln_el = { 'vulnerability_id': vulnerability_cpe.vulnerability_id, 'severity': vulnerability_cpe.severity, 'link': vulnerability_cpe.link, 'pkg_type': image_cpe.pkg_type, 'pkg_path': image_cpe.pkg_path, 'name': image_cpe.name, 'version': image_cpe.version, 'cpe': image_cpe.get_cpestring(), 'feed_name': vulnerability_cpe.feed_name, 'feed_namespace': vulnerability_cpe.namespace_name, } cpe_hash = hashlib.sha256( utils.ensure_bytes(json.dumps(cpe_vuln_el))).hexdigest() if not cpe_hashes.get(cpe_hash, False): cpe_vuln_listing.append(cpe_vuln_el) cpe_hashes[cpe_hash] = True except Exception as err: log.warn("could not fetch CPE matches - exception: " + str(err)) report = LegacyVulnerabilityReport.from_dict(vuln_listing) resp = ImageVulnerabilityListing(user_id=user_id, image_id=image_id, legacy_report=report, cpe_report=cpe_vuln_listing) return resp.to_dict() except HTTPException: db.rollback() raise except Exception as e: log.exception( 'Error checking image {}, {} for vulnerabiltiies. Rolling back'. format(user_id, image_id)) db.rollback() abort(500) finally: db.close()
def evaluate(self, image_obj, context): if not have_vulnerabilities_for(DistroNamespace.for_obj(image_obj)): self._fire(msg="UNSUPPORTEDDISTRO cannot perform CVE scan: " + str(image_obj.distro_namespace))
def namespace_support_test(): init_db_persisted() expected = [ DistroNamespace(name='alpine', version='3.3', like_distro='alpine'), DistroNamespace(name='alpine', version='3.4', like_distro='alpine'), DistroNamespace(name='alpine', version='3.5', like_distro='alpine'), DistroNamespace(name='alpine', version='3.6', like_distro='alpine'), DistroNamespace(name='centos', version='5', like_distro='centos'), DistroNamespace(name='centos', version='6', like_distro='centos'), DistroNamespace(name='centos', version='7', like_distro='centos'), DistroNamespace(name='centos', version='7.1', like_distro='centos'), DistroNamespace(name='centos', version='7.3', like_distro='centos'), DistroNamespace(name='ol', version='6', like_distro='centos'), DistroNamespace(name='ol', version='6.5', like_distro='centos'), DistroNamespace(name='ol', version='7.3', like_distro='centos'), DistroNamespace(name='rhel', version='7.1', like_distro='centos'), DistroNamespace(name='debian', version='7', like_distro='debian'), DistroNamespace(name='debian', version='8', like_distro='debian'), DistroNamespace(name='debian', version='9', like_distro='debian'), DistroNamespace(name='debian', version='unstable', like_distro='debian'), DistroNamespace(name='ubuntu', version='12.04', like_distro='ubuntu'), DistroNamespace(name='ubuntu', version='13.04', like_distro='ubuntu'), DistroNamespace(name='ubuntu', version='14.04', like_distro='ubuntu'), DistroNamespace(name='ubuntu', version='14.10', like_distro='ubuntu'), DistroNamespace(name='ubuntu', version='15.04', like_distro='ubuntu'), DistroNamespace(name='ubuntu', version='15.10', like_distro='ubuntu'), DistroNamespace(name='ubuntu', version='16.04', like_distro='ubuntu'), DistroNamespace(name='ubuntu', version='16.10', like_distro='ubuntu'), DistroNamespace(name='ubuntu', version='17.04', like_distro='ubuntu'), ] fail = [ DistroNamespace(name='alpine', version='3.1', like_distro='alpine'), DistroNamespace(name='alpine', version='3.1.1', like_distro='alpine'), DistroNamespace(name='busybox', version='3', like_distro='busybox'), DistroNamespace(name='linuxmint', version='16', like_distro='debian'), DistroNamespace(name='redhat', version='6', like_distro='centos'), DistroNamespace(name='ubuntu', version='1.0', like_distro='ubuntu'), DistroNamespace(name='centos', version='1.0', like_distro='ubuntu'), DistroNamespace(name='debian', version='1.0', like_distro='ubuntu'), DistroNamespace(name='rhel', version='1.0', like_distro='ubuntu'), DistroNamespace(name='busybox', version='1.0', like_distro='busybox'), DistroNamespace(name='alpine', version='11.0', like_distro='ubuntu'), DistroNamespace(name='fedora', version='25', like_distro='fedora'), DistroNamespace(name='mageia', version='5', like_distro='mandriva,fedora') ] for i in expected: if not vulnerabilities.have_vulnerabilities_for(i): raise Exception('Bad failure: {}'.format(i.namespace_name)) for i in fail: if vulnerabilities.have_vulnerabilities_for(i): raise Exception('Should not have data for {}'.format( i.namespace_name))
def evaluate(self, image_obj, context): if not have_vulnerabilities_for(DistroNamespace.for_obj(image_obj)): self._fire( msg= "Distro-specific feed data not found for distro namespace: %s. Cannot perform CVE scan OS/distro packages" % image_obj.distro_namespace)
def load_and_normalize_packages(self, package_analysis_json, image_obj): """ Loads and normalizes package data from all distros :param image_obj: :param package_analysis_json: :return: list of Package objects that can be added to an image """ pkgs = [] img_distro = DistroNamespace.for_obj(image_obj) # pkgs.allinfo handling pkgs_all = package_analysis_json.get('pkgs.allinfo', {}).values() if not pkgs_all: return [] else: pkgs_all = pkgs_all[0] for pkg_name, metadata_str in pkgs_all.items(): metadata = json.loads(metadata_str) p = ImagePackage() p.distro_name = image_obj.distro_name p.distro_version = image_obj.distro_version p.like_distro = image_obj.like_distro p.name = pkg_name p.version = metadata.get('version') p.origin = metadata.get('origin') p.size = metadata.get('size') p.arch = metadata.get('arch') p.license = metadata.get('license') if metadata.get( 'license') else metadata.get('lics') p.release = metadata.get('release', 'N/A') p.pkg_type = metadata.get('type') p.src_pkg = metadata.get('sourcepkg') p.image_user_id = image_obj.user_id p.image_id = image_obj.id if 'files' in metadata: # Handle file data p.files = metadata.get('files') if p.release != 'N/A': p.fullversion = p.version + '-' + p.release else: p.fullversion = p.version if img_distro.flavor == 'DEB': cleanvers = re.sub(re.escape("+b") + "\d+.*", "", p.version) spkg = re.sub(re.escape("-" + cleanvers), "", p.src_pkg) else: spkg = re.sub(re.escape("-" + p.version) + ".*", "", p.src_pkg) p.normalized_src_pkg = spkg pkgs.append(p) if pkgs: return pkgs else: log.warn('Pkg Allinfo not found, reverting to using pkgs.all') all_pkgs = package_analysis_json['pkgs.all']['base'] all_pkgs_src = package_analysis_json['pkgs_plus_source.all']['base'] for pkg_name, version in all_pkgs.items(): p = ImagePackage() p.image_user_id = image_obj.user_id p.image_id = image_obj.id p.name = pkg_name p.version = version p.fullversion = all_pkgs_src[pkg_name] if img_distro.flavor == 'RHEL': name, parsed_version, release, epoch, arch = split_rpm_filename( pkg_name + '-' + version + '.tmparch.rpm') p.version = parsed_version p.release = release p.pkg_type = 'RPM' p.origin = 'N/A' p.src_pkg = 'N/A' p.license = 'N/A' p.arch = 'N/A' elif img_distro.flavor == 'DEB': try: p.version, p.release = version.split('-') except: p.version = version p.release = None return pkgs
def get_image_vulnerabilities( self, image: Image, db_session, vendor_only: bool = True, force_refresh: bool = False, cache: bool = True, ): # select the nvd class once and be done _nvd_cls, _cpe_cls = select_nvd_classes(db_session) # initialize the scanner scanner = self.__scanner__() user_id = image.user_id image_id = image.id results = [] if force_refresh: log.info("Forcing refresh of vulnerabilities for {}/{}".format( user_id, image_id)) try: scanner.flush_and_recompute_vulnerabilities( image, db_session=db_session) db_session.commit() except Exception as e: log.exception( "Error refreshing cve matches for image {}/{}".format( user_id, image_id)) db_session.rollback() return make_response_error( "Error refreshing vulnerability listing for image.", in_httpcode=500, ) db_session = get_session() db_session.refresh(image) with timer("Image vulnerability primary lookup", log_level="debug"): vulns = scanner.get_vulnerabilities(image) # Has vulnerabilities? warns = [] if not vulns: vulns = [] ns = DistroNamespace.for_obj(image) if not have_vulnerabilities_for(ns): warns = [ "No vulnerability data available for image distro: {}". format(ns.namespace_name) ] rows = [] with timer("Image vulnerability nvd metadata merge", log_level="debug"): vulns = merge_nvd_metadata_image_packages(db_session, vulns, _nvd_cls, _cpe_cls) with timer("Image vulnerability output formatting", log_level="debug"): for vuln, nvd_records in vulns: fixed_artifact = vuln.fixed_artifact() # Skip the vulnerability if the vendor_only flag is set to True and the issue won't be addressed by the vendor if vendor_only and vuln.fix_has_no_advisory( fixed_in=fixed_artifact): continue nvd_scores = [ self._make_cvss_score(score) for nvd_record in nvd_records for score in nvd_record.get_cvss_scores_nvd() ] results.append( VulnerabilityMatch( vulnerability=VulnerabilityModel( vulnerability_id=vuln.vulnerability_id, description="NA", severity=vuln.vulnerability.severity, link=vuln.vulnerability.link, feed="vulnerabilities", feed_group=vuln.vulnerability.namespace_name, cvss_scores_nvd=nvd_scores, cvss_scores_vendor=[], created_at=vuln.vulnerability.created_at, last_modified=vuln.vulnerability.updated_at, ), artifact=Artifact( name=vuln.pkg_name, version=vuln.package.fullversion, pkg_type=vuln.pkg_type, pkg_path=vuln.pkg_path, cpe="None", cpe23="None", ), fixes=[ FixedArtifact( version=str( vuln.fixed_in(fixed_in=fixed_artifact)), wont_fix=vuln.fix_has_no_advisory( fixed_in=fixed_artifact), observed_at=fixed_artifact.fix_observed_at if fixed_artifact else None, ) ], match=Match(detected_at=vuln.created_at), )) # TODO move dedup here so api doesn't have to # cpe_vuln_listing = [] try: with timer("Image vulnerabilities cpe matches", log_level="debug"): all_cpe_matches = scanner.get_cpe_vulnerabilities( image, _nvd_cls, _cpe_cls) if not all_cpe_matches: all_cpe_matches = [] api_endpoint = self._get_api_endpoint() for image_cpe, vulnerability_cpe in all_cpe_matches: link = vulnerability_cpe.parent.link if not link: link = "{}/query/vulnerabilities?id={}".format( api_endpoint, vulnerability_cpe.vulnerability_id) nvd_scores = [ self._make_cvss_score(score) for score in vulnerability_cpe.parent.get_cvss_scores_nvd() ] vendor_scores = [ self._make_cvss_score(score) for score in vulnerability_cpe.parent.get_cvss_scores_vendor() ] results.append( VulnerabilityMatch( vulnerability=VulnerabilityModel( vulnerability_id=vulnerability_cpe.parent. normalized_id, description="NA", severity=vulnerability_cpe.parent.severity, link=link, feed=vulnerability_cpe.feed_name, feed_group=vulnerability_cpe.namespace_name, cvss_scores_nvd=nvd_scores, cvss_scores_vendor=vendor_scores, created_at=vulnerability_cpe.parent.created_at, last_modified=vulnerability_cpe.parent. updated_at, ), artifact=Artifact( name=image_cpe.name, version=image_cpe.version, pkg_type=image_cpe.pkg_type, pkg_path=image_cpe.pkg_path, cpe=image_cpe.get_cpestring(), cpe23=image_cpe.get_cpe23string(), ), fixes=[ FixedArtifact( version=item, wont_fix=False, observed_at=vulnerability_cpe.created_at, ) for item in vulnerability_cpe.get_fixed_in() ], # using vulnerability created_at to indicate the match timestamp for now match=Match( detected_at=vulnerability_cpe.created_at), )) except Exception as err: log.exception("could not fetch CPE matches") import uuid return ImageVulnerabilitiesReport( account_id=image.user_id, image_id=image_id, results=results, metadata=VulnerabilitiesReportMetadata( generated_at=datetime.datetime.utcnow(), uuid=str(uuid.uuid4()), generated_by=self._get_provider_metadata(), ), problems=[], )