def test_cve_updates(test_data_env): test_env = test_data_env test_env.init_feeds() test_user_id = "test1" test_img_id = "img1" test_image = Image(user_id=test_user_id, id=test_img_id, distro_name="centos", distro_version="7") test_image.familytree_json = [test_img_id] test_image.layers_json = [test_img_id] test_image.layer_info_json = ["somelayer_here"] test_image.like_distro = "centos" test_image.state = "analyzed" test_image.digest = "digest1" test_image.anchore_type = "undefined" test_image.dockerfile_mode = "Guessed" test_image.docker_history_json = ["line1", "line2"] test_image.docker_data_json = {"Config": {}, "ContainerConfig": {}} test_image.dockerfile_contents = "FROM BLAH" test_package = ImagePackage( image_user_id=test_user_id, image_id=test_img_id, name="testpackage", version="1.0", pkg_type="RPM", ) test_package.src_pkg = "testpackage" test_package.distro_name = "centos" test_package.distro_version = "7" test_package.like_distro = "centos" test_package.license = "apache2" test_package.fullversion = "1.0" test_package.normalized_src_pkg = "1.0" test_package.release = "" test_package.size = 1000 test_package.origin = "upstream" test_package.arch = "x86_64" test_package.image = test_image test_cve = Vulnerability(id="CVE123", namespace_name="centos:7") test_cve.severity = "High" test_cve.description = "some test cve" test_cve.cvss2_score = "1.0" test_cve.metadata_json = {} test_cve.cvss2_vectors = "" test_cve.link = "http://mitre.com/cve123" test_fixedin = FixedArtifact(vulnerability_id=test_cve.id) test_fixedin.name = "testpackage" test_fixedin.version = "1.1" test_fixedin.version_format = "rpm" test_fixedin.epochless_version = "1.1" test_fixedin.include_later_versions = True test_fixedin.parent = test_cve test_cve.fixed_in = [test_fixedin] test_vulnin = VulnerableArtifact(vulnerability_id=test_cve.id) test_vulnin.name = "testpackage" test_vulnin.version = "0.9" test_vulnin.epochless_version = "0.9" test_vulnin.namespace_name = "centos:7" test_vulnin.version_format = "rpm" test_vulnin.include_previous_versions = False test_vulnin.parent = test_cve test_cve.vulnerable_in = [test_vulnin] db = get_session() try: db.add(test_image) db.add(test_package) db.commit() except sqlalchemy.exc.IntegrityError: db.rollback() except Exception: logger.exception("Unexpected failure") raise db = get_session() try: db.add(test_cve) feeds.process_updated_vulnerability(db, test_cve) db.commit() except sqlalchemy.exc.IntegrityError: logger.exception("Failed!") db.rollback() finally: db = get_session() i = db.query(Image).get((test_img_id, test_user_id)) print(("Vulns: {}".format(i.vulnerabilities()))) db.commit() test_cve2 = Vulnerability(id="CVE123", namespace_name="centos:7") test_cve2.severity = "Medium" test_cve2.description = "some test cve" test_cve2.cvss2_score = "1.0" test_cve2.metadata_json = {} test_cve2.cvss2_vectors = "" test_cve2.link = "http://mitre.com/cve123" fix2 = FixedArtifact(name="pkg2", version="1.2", epochless_version="1.2") fix2.namespace_name = "centos:7" fix2.vulnerability_id = test_cve2.id test_cve2.fixed_in = [fix2] db = get_session() try: t2 = db.merge(test_cve2) db.add(t2) feeds.process_updated_vulnerability(db, t2) db.commit() except sqlalchemy.exc.IntegrityError: logger.exception("Failed!") db.rollback() finally: db = get_session() i = db.query(Image).get((test_img_id, test_user_id)) print(("Vulns: {}".format(i.vulnerabilities()))) db.commit()
def sync_metadata(feed_client: IFeedSource, to_sync: list = None, operation_id=None) -> tuple: """ Get metadata from source and sync db metadata records to that (e.g. add any new groups or feeds) Executes as a unit-of-work for db, so will commit result and returns the records found on upstream source. If a record exists in db but was not found upstream, it is not returned :param feed_client: :param to_sync: list of string feed names to sync metadata on :return: tuple, first element: dict of names mapped to db records post-sync only including records successfully updated by upstream, second element is a list of tuples where each tuple is (failed_feed_name, error_obj) """ if not to_sync: return {}, [] db = get_session() try: log.info('Syncing feed and group metadata from upstream source (operation_id={})'.format(operation_id)) source_resp = feed_client.list_feeds() if to_sync: feeds = filter(lambda x: x.name in to_sync, source_resp.feeds) else: feeds = [] failed = [] source_feeds = {x.name: {'meta': x, 'groups': feed_client.list_feed_groups(x.name).groups} for x in feeds} log.debug('Upstream feeds available: {}'.format(source_feeds)) db_feeds = DataFeeds._pivot_and_filter_feeds_by_config(to_sync, list(source_feeds.keys()), get_all_feeds(db)) for feed_name, feed_api_record in source_feeds.items(): try: log.info('Syncing metadata for feed: {} (operation_id={})'.format(feed_name, operation_id)) api_feed = feed_api_record['meta'] db_feed = db_feeds.get(api_feed.name) # Do this instead of a db.merge() to ensure no timestamps are reset or overwritten if not db_feed: log.debug('Adding new feed metadata record to db: {} (operation_id={})'.format(api_feed.name, operation_id)) db_feed = FeedMetadata(name=api_feed.name, description=api_feed.description, access_tier=api_feed.access_tier) db.add(db_feed) db.flush() else: log.debug('Feed metadata already in db: {} (operation_id={})'.format(api_feed.name, operation_id)) # Check for any update db_feed.description = api_feed.description db_feed.access_tier = api_feed.access_tier db_groups = {x.name: x for x in db_feed.groups} for api_group in feed_api_record.get('groups', []): db_group = db_groups.get(api_group.name) # Do this instead of a db.merge() to ensure no timestamps are reset or overwritten if not db_group: log.debug('Adding new feed metadata record to db: {} (operation_id={})'.format(api_group.name, operation_id)) db_group = FeedGroupMetadata(name=api_group.name, description=api_group.description, access_tier=api_group.access_tier, feed=db_feed) db_group.last_sync = None db.add(db_group) else: log.debug('Feed group metadata already in db: {} (operation_id={})'.format(api_group.name, operation_id)) db_group.access_tier = api_group.access_tier db_group.description = api_group.description except Exception as e: log.exception('Error syncing feed {}'.format(feed_name)) log.warn('Could not sync metadata for feed: {} (operation_id={})'.format(feed_name, operation_id)) failed.append((feed_name, e)) finally: db.flush() # Reload db_feeds = DataFeeds._pivot_and_filter_feeds_by_config(to_sync, list(source_feeds.keys()), get_all_feeds(db)) db.commit() log.info('Metadata sync from feeds upstream source complete (operation_id={})'.format(operation_id)) return db_feeds, failed except Exception as e: log.error('Rolling back feed metadata update due to error: {} (operation_id={})'.format(e, operation_id)) db.rollback() raise
def get_image_vulnerabilities(user_id, image_id, force_refresh=False, vendor_only=True): """ Return the vulnerability listing for the specified image and load from catalog if not found and specifically asked to do so. Example json output: { "multi" : { "url_column_index" : 7, "result" : { "rows" : [], "rowcount" : 0, "colcount" : 8, "header" : [ "CVE_ID", "Severity", "*Total_Affected", "Vulnerable_Package", "Fix_Available", "Fix_Images", "Rebuild_Images", "URL" ] }, "querycommand" : "/usr/lib/python2.7/site-packages/anchore/anchore-modules/multi-queries/cve-scan.py /ebs_data/anchore/querytmp/queryimages.7026386 /ebs_data/anchore/data /ebs_data/anchore/querytmp/query.59057288 all", "queryparams" : "all", "warns" : [ "0005b136f0fb (prom/prometheus:master) cannot perform CVE scan: no CVE data is currently available for the detected base distro type (busybox:unknown_version,busybox:v1.26.2)" ] } } :param user_id: user id of image to evaluate :param image_id: image id to evaluate :param force_refresh: if true, flush and recompute vulnerabilities rather than returning current values :param vendor_only: if true, filter out the vulnerabilities that vendors will explicitly not address :return: """ # Has image? db = get_session() try: img = db.query(Image).get((image_id, user_id)) vulns = [] if not img: abort(404) else: if force_refresh: log.info('Forcing refresh of vulnerabiltiies for {}/{}'.format( user_id, image_id)) try: vulns = rescan_image(img, db_session=db) db.commit() except Exception as e: log.exception( 'Error refreshing cve matches for image {}/{}'.format( user_id, image_id)) db.rollback() abort( Response( 'Error refreshing vulnerability listing for image.', 500)) db = get_session() db.refresh(img) vulns = img.vulnerabilities() # Has vulnerabilities? warns = [] if not vulns: vulns = [] ns = DistroNamespace.for_obj(img) if not have_vulnerabilities_for(ns): warns = [ 'No vulnerability data available for image distro: {}'. format(ns.namespace_name) ] rows = [] for vuln in vulns: # if vuln.vulnerability.fixed_in: # fixes_in = filter(lambda x: x.name == vuln.pkg_name or x.name == vuln.package.normalized_src_pkg, # vuln.vulnerability.fixed_in) # fix_available_in = fixes_in[0].version if fixes_in else 'None' # else: # fix_available_in = 'None' # Skip the vulnerability if the vendor_only flag is set to True and the issue won't be addressed by the vendor if vendor_only and vuln.fix_has_no_advisory(): continue rows.append([ vuln.vulnerability_id, vuln.vulnerability.severity, 1, vuln.pkg_name + '-' + vuln.package.fullversion, str(vuln.fixed_in()), vuln.pkg_image_id, 'None', # Always empty this for now vuln.vulnerability.link, vuln.pkg_type, 'vulnerabilities', vuln.vulnerability.namespace_name, vuln.pkg_name, vuln.package.fullversion, ]) vuln_listing = { 'multi': { 'url_column_index': 7, 'result': { 'header': TABLE_STYLE_HEADER_LIST, 'rowcount': len(rows), 'colcount': len(TABLE_STYLE_HEADER_LIST), 'rows': rows }, 'warns': warns } } cpe_vuln_listing = [] try: all_cpe_matches = db.query( ImageCpe, CpeVulnerability).filter(ImageCpe.image_id == image_id).filter( ImageCpe.name == CpeVulnerability.name).filter( ImageCpe.version == CpeVulnerability.version) if not all_cpe_matches: all_cpe_matches = [] for image_cpe, vulnerability_cpe in all_cpe_matches: cpe_vuln_el = { 'vulnerability_id': vulnerability_cpe.vulnerability_id, 'severity': vulnerability_cpe.severity, 'link': vulnerability_cpe.link, 'pkg_type': image_cpe.pkg_type, 'pkg_path': image_cpe.pkg_path, 'name': image_cpe.name, 'version': image_cpe.version, 'cpe': image_cpe.get_cpestring(), 'feed_name': vulnerability_cpe.feed_name, 'feed_namespace': vulnerability_cpe.namespace_name, } cpe_vuln_listing.append(cpe_vuln_el) except Exception as err: log.warn("could not fetch CPE matches - exception: " + str(err)) report = LegacyVulnerabilityReport.from_dict(vuln_listing) resp = ImageVulnerabilityListing(user_id=user_id, image_id=image_id, legacy_report=report, cpe_report=cpe_vuln_listing) return resp.to_dict() except HTTPException: db.rollback() raise except Exception as e: log.exception( 'Error checking image {}, {} for vulnerabiltiies. Rolling back'. format(user_id, image_id)) db.rollback() abort(500) finally: db.close()
def sync_metadata( source_feeds: Dict[ str, Dict[str, Union[FeedAPIRecord, List[FeedAPIGroupRecord]]] ], to_sync: List[str] = None, operation_id: Optional[str] = None, groups: bool = True, ) -> Tuple[Dict[str, FeedMetadata], List[Tuple[str, Union[str, BaseException]]]]: """ Get metadata from source and sync db metadata records to that (e.g. add any new groups or feeds) Executes as a unit-of-work for db, so will commit result and returns the records found on upstream source. If a record exists in db but was not found upstream, it is not returned :param source_feeds: mapping containing FeedAPIRecord and FeedAPIGroupRecord :type source_feeds: Dict[str, Dict[str, Union[FeedAPIRecord, List[FeedAPIGroupRecord]]]] :param to_sync: list of string feed names to sync metadata on :type to_sync: List[str] :param operation_id: UUID4 hexadecimal string :type operation_id: Optional[str] :param groups: whether or not to sync group metadata (defaults to True, which will sync group metadata) :type groups: bool :return: tuple, first element: dict of names mapped to db records post-sync only including records successfully updated by upstream, second element is a list of tuples where each tuple is (failed_feed_name, error_obj) :rtype: Tuple[Dict[str, FeedMetadata], List[Tuple[str, Union[str, BaseException]]] """ if not to_sync: return {}, [] db = get_session() try: logger.info( "Syncing feed and group metadata from upstream source (operation_id={})".format( operation_id ) ) failed = [] db_feeds = MetadataSyncUtils._pivot_and_filter_feeds_by_config( to_sync, list(source_feeds.keys()), get_all_feeds(db) ) for feed_name, feed_api_record in source_feeds.items(): try: logger.info( "Syncing metadata for feed: {} (operation_id={})".format( feed_name, operation_id ) ) feed_metadata_map = MetadataSyncUtils._sync_feed_metadata( db, feed_api_record, db_feeds, operation_id ) if groups: MetadataSyncUtils._sync_feed_group_metadata( db, feed_api_record, feed_metadata_map, operation_id ) except Exception as e: logger.exception("Error syncing feed {}".format(feed_name)) logger.warn( "Could not sync metadata for feed: {} (operation_id={})".format( feed_name, operation_id ) ) failed.append((feed_name, e)) finally: db.flush() # Reload db_feeds = MetadataSyncUtils._pivot_and_filter_feeds_by_config( to_sync, list(source_feeds.keys()), get_all_feeds(db) ) db.commit() logger.info( "Metadata sync from feeds upstream source complete (operation_id={})".format( operation_id ) ) return db_feeds, failed except Exception as e: logger.error( "Rolling back feed metadata update due to error: {} (operation_id={})".format( e, operation_id ) ) db.rollback() raise
def __init__(self, metadata=None, src=None): if not metadata: db = get_session() metadata = db.query(FeedMetadata).get(self.__feed_name__) super(AnchoreServiceFeed, self).__init__(metadata=metadata, src=src)
def check_user_image_inline(user_id, image_id, tag, bundle): """ Execute a policy evaluation using the info in the request body including the bundle content :param user_id: :param image_id: :param tag: :param bundle: :return: """ db = get_session() try: # Input validation try: img_obj = db.query(Image).get((image_id, user_id)) except: abort(Response(response='Image not found', status=404)) if not img_obj: log.info( 'Request for evaluation of image that cannot be found: user_id = {}, image_id = {}' .format(user_id, image_id)) abort(Response(response='Image not found', status=404)) # Build bundle exec. problems = [] executable_bundle = None try: executable_bundle = build_bundle(bundle, for_tag=tag) if executable_bundle.init_errors: problems = executable_bundle.init_errors except InitializationError as e: log.exception( 'Bundle construction and initialization returned errors') problems = e.causes if not problems: # Execute bundle try: eval_result = executable_bundle.execute( img_obj, tag, ExecutionContext(db_session=db, configuration={})) except Exception as e: log.exception( 'Error executing policy bundle {} against image {} w/tag {}: {}' .format(bundle['id'], image_id, tag, e.message)) abort( Response( response= 'Cannot execute given policy against the image due to errors executing the policy bundle: {}' .format(e.message), status=500)) else: # Construct a failure eval with details on the errors and mappings to send to client eval_result = build_empty_error_execution(img_obj, tag, executable_bundle, errors=problems, warnings=[]) if executable_bundle and executable_bundle.mapping and len( executable_bundle.mapping.mapping_rules) == 1: eval_result.executed_mapping = executable_bundle.mapping.mapping_rules[ 0] resp = PolicyEvaluation() resp.user_id = user_id resp.image_id = image_id resp.tag = tag resp.bundle = bundle resp.matched_mapping_rule = eval_result.executed_mapping.json( ) if eval_result.executed_mapping else {} resp.last_modified = int(time.time()) resp.final_action = eval_result.policy_decision.final_decision resp.result = eval_result.as_table_json() resp.created_at = int(time.time()) resp.evaluation_problems = [ problem_from_exception(i) for i in eval_result.errors ] resp.evaluation_problems += [ problem_from_exception(i) for i in eval_result.warnings ] if resp.evaluation_problems: for i in resp.evaluation_problems: log.warn( 'Returning evaluation response for image {}/{} w/tag {} and bundle {} that contains error: {}' .format(user_id, image_id, tag, bundle['id'], json.dumps(i.to_dict()))) return resp.to_dict() except HTTPException as e: db.rollback() log.exception('Caught exception in execution: {}'.format(e)) raise except Exception as e: db.rollback() log.exception('Failed processing bundle evaluation: {}'.format(e)) abort(Response('Unexpected internal error', 500)) finally: db.close()
def test_github_advisory_fixed_in(test_data_env): test_env = test_data_env test_env.init_feeds() test_user_id = 'test1' test_img_id = 'img1' test_image = Image( user_id=test_user_id, id=test_img_id, distro_name='centos', distro_version='7' ) test_image.familytree_json = [test_img_id] test_image.layers_json = [test_img_id] test_image.layer_info_json = ['somelayer_here'] test_image.like_distro = 'centos' test_image.state = 'analyzed' test_image.digest = 'digest1' test_image.anchore_type = 'undefined' test_image.dockerfile_mode = 'Guessed' test_image.docker_history_json = ['line1', 'line2'] test_image.docker_data_json = {'Config': {}, 'ContainerConfig': {}} test_image.dockerfile_contents = 'FROM BLAH' test_package = ImagePackage( image_user_id=test_user_id, image_id=test_img_id, name='testpackage', version='1.0', pkg_type='python' ) test_package.src_pkg = 'testpackage' test_package.distro_name = 'centos' test_package.distro_version = '7' test_package.like_distro = 'centos' test_package.license = 'apache2' test_package.fullversion = '1.0' test_package.normalized_src_pkg = '1.0' test_package.release = '' test_package.size = 1000 test_package.origin = 'upstream' test_package.arch = 'x86_64' test_package.image = test_image test_cve = Vulnerability(id='GHSA-rpch-cqj9-h65r', namespace_name='github:python') test_cve.severity = 'High' test_cve.description = 'some advisory ghsa' test_cve.link = 'http://mitre.com/cve123' test_fixedin = FixedArtifact(vulnerability_id=test_cve.id) test_fixedin.name = 'testpackage' test_fixedin.version = 'None' test_fixedin.fix_metadata = {'first_patched_version': '1.2'} test_fixedin.version_format = 'semver' test_fixedin.parent = test_cve test_cve.fixed_in = [test_fixedin] db = get_session() try: db.add(test_image) db.add(test_package) db.commit() except sqlalchemy.exc.IntegrityError: db.rollback() except Exception: logger.exception('Unexpected failure') raise db = get_session() # XXX This needs to be a fixture try: db.add(test_cve) feeds.process_updated_vulnerability(db, test_cve) db.commit() except sqlalchemy.exc.IntegrityError: logger.exception('Failed!') db.rollback() db = get_session() image_vuln = db.query(Image).get((test_img_id, test_user_id)) # should be one vulnerability vulnerabilities = image_vuln.vulnerabilities() assert len(vulnerabilities) == 1 img_pkg_vuln = vulnerabilities[0] assert img_pkg_vuln.fixed_in() == '1.2'
def test_regexes(large_whitelist, test_data_env_with_images_loaded): """ Test regular expressions in the trigger_id part of the WL rule :return: """ logger.info('Building executable bundle from default bundle') test_tag = 'docker.io/library/node:latest' bundle = copy.deepcopy(default_bundle) node_whitelist = [ x for x in bundle['whitelists'] if x['id'] == 'wl_jessie' ][0] node_whitelist['items'] = [ x for x in node_whitelist['items'] if 'binutils' in x['trigger_id'] ] node_whitelist['items'].append({ 'gate': 'vulnerabilities', 'trigger_id': 'CVE-2016-6515+openssh-client', 'id': 'testinserted3' }) node_whitelist['items'].append({ 'gate': 'vulnerabilities', 'trigger_id': 'CVE-2016-6515+*', 'id': 'test-cve-2016-6515' }) node_whitelist['items'].append({ 'gate': 'vulnerabilities', 'trigger_id': 'CVE-2017*', 'id': 'testinserted2' }) node_whitelist['items'].append({ 'gate': 'vulnerabilities', 'trigger_id': '*binutils*', 'id': 'testinserted1' }) db = get_session() img_obj = db.query(Image).get( (test_data_env_with_images_loaded.get_images_named('node')[0][0], '0')) assert img_obj is not None ExecutableWhitelist._use_indexes = True built = build_bundle(bundle, for_tag=test_tag) assert not built.init_errors logger.info('Executing with indexes') t = time.time() evaluation = built.execute(img_obj, tag=test_tag, context=ExecutionContext(db_session=db, configuration={})) t1 = time.time() - t logger.info(('Took: {}'.format(t1))) assert evaluation is not None ExecutableWhitelist._use_indexes = False non_index_built = build_bundle(bundle, for_tag=test_tag) assert not non_index_built.init_errors logger.info('Executing without indexes') t2 = time.time() evaluation2 = non_index_built.execute(img_obj, tag=test_tag, context=ExecutionContext( db_session=db, configuration={})) t2 = time.time() - t2 logger.info(('Took: {}'.format(t2))) assert evaluation2 is not None ExecutableWhitelist._use_indexes = True assert evaluation.json()['bundle_decision']['policy_decisions'][0][ 'decisions'] == evaluation2.json( )['bundle_decision']['policy_decisions'][0]['decisions'] logger.info( ('Evaluation: {}'.format(json.dumps(evaluation.json(), indent=2)))) open_ssl_wl_match = { "action": "go", "rule": { "action": "stop", "gate": "vulnerabilities", "trigger": "package", "params": {} }, "match": { "message": "HIGH Vulnerability found in package - openssh-client (CVE-2016-6515 - https://security-tracker.debian.org/tracker/CVE-2016-6515)", "trigger": "package", "whitelisted": { "whitelist_id": "wl_jessie", "matched_rule_id": "testinserted3", "whitelist_name": "CVE whitelist for jessie - 12092017" }, "trigger_id": "CVE-2016-6515+openssh-client" } } assert open_ssl_wl_match in evaluation.json( )['bundle_decision']['policy_decisions'][0]['decisions'] assert len([ x for x in evaluation.json()['bundle_decision']['policy_decisions'][0] ['decisions'] if x['match'].get('whitelisted', {}).get('matched_rule_id', '') in ['testinserted1', 'testinserted2', 'testinserted3'] ]) >= 1
def rescan_images_created_between(self, from_time, to_time): """ If this was a vulnerability update (e.g. timestamps vuln feeds lies in that interval), then look for any images that were loaded in that interval and re-scan the cves for those to ensure that no ordering of transactions caused cves to be missed for an image. This is an alternative to a blocking approach by which image loading is blocked during feed syncs. :param from_time: :param to_time: :return: count of updated images """ if from_time is None or to_time is None: raise ValueError('Cannot process None timestamp') log.info('Rescanning images loaded between {} and {}'.format( from_time.isoformat(), to_time.isoformat())) count = 0 db = get_session() try: # it is critical that these tuples are in proper index order for the primary key of the Images object so that subsequent get() operation works imgs = [(x.id, x.user_id) for x in db.query(Image).filter( Image.created_at >= from_time, Image.created_at <= to_time)] log.info('Detected images: {} for rescan'.format( ' ,'.join([str(x) for x in imgs]) if imgs else '[]')) finally: db.rollback() retry_max = 3 for img in imgs: for i in range(retry_max): try: # New transaction for each image to get incremental progress db = get_session() try: # If the type or ordering of 'img' tuple changes, this needs to be updated as it relies on symmetry of that tuple and the identity key of the Image entity image_obj = db.query(Image).get(img) if image_obj: log.info( 'Rescanning image {} post-vuln sync'.format( img)) vulns = rescan_image(image_obj, db_session=db) count += 1 else: log.warn( 'Failed to lookup image with tuple: {}'.format( str(img))) db.commit() finally: db.rollback() break except Exception as e: log.exception( 'Caught exception updating vulnerability scan results for image {}. Waiting and retrying' .format(img)) time.sleep(5) return count
def get_image_vulnerabilities(user_id, image_id, force_refresh=False, vendor_only=True): """ Return the vulnerability listing for the specified image and load from catalog if not found and specifically asked to do so. Example json output: { "multi" : { "url_column_index" : 7, "result" : { "rows" : [], "rowcount" : 0, "colcount" : 8, "header" : [ "CVE_ID", "Severity", "*Total_Affected", "Vulnerable_Package", "Fix_Available", "Fix_Images", "Rebuild_Images", "URL" ] }, "querycommand" : "/usr/lib/python2.7/site-packages/anchore/anchore-modules/multi-queries/cve-scan.py /ebs_data/anchore/querytmp/queryimages.7026386 /ebs_data/anchore/data /ebs_data/anchore/querytmp/query.59057288 all", "queryparams" : "all", "warns" : [ "0005b136f0fb (prom/prometheus:master) cannot perform CVE scan: no CVE data is currently available for the detected base distro type (busybox:unknown_version,busybox:v1.26.2)" ] } } :param user_id: user id of image to evaluate :param image_id: image id to evaluate :param force_refresh: if true, flush and recompute vulnerabilities rather than returning current values :param vendor_only: if true, filter out the vulnerabilities that vendors will explicitly not address :return: """ # Has image? db = get_session() try: img = db.query(Image).get((image_id, user_id)) if not img: return make_response_error("Image not found", in_httpcode=404), 404 provider = get_vulnerabilities_provider() report = provider.get_image_vulnerabilities_json( image=img, vendor_only=vendor_only, db_session=db, force_refresh=force_refresh, use_store=True, ) db.commit() return report, 200 except HTTPException: db.rollback() raise except Exception as e: log.exception( "Error checking image {}, {} for vulnerabiltiies. Rolling back". format(user_id, image_id)) db.rollback() return make_response_error(e, in_httpcode=500), 500 finally: db.close()
test_fixedin.epochless_version = '1.1' test_fixedin.include_later_versions = True test_fixedin.parent = test_cve test_cve.fixed_in = [test_fixedin] test_vulnin = VulnerableArtifact(vulnerability_id=test_cve.id) test_vulnin.name = 'testpackage' test_vulnin.version = '0.9' test_vulnin.epochless_version = '0.9' test_vulnin.namespace_name = 'centos:7' test_vulnin.version_format = 'rpm' test_vulnin.include_previous_versions = False test_vulnin.parent = test_cve test_cve.vulnerable_in = [test_vulnin] db = get_session() try: db.add(test_image) db.add(test_package) db.commit() except sqlalchemy.exc.IntegrityError as e: db.rollback() except Exception as e: log.exception('Unexpected failure') raise db = get_session() try: db.add(test_cve) FeedsUpdateTask.process_updated_vulnerability(db, test_cve) db.commit()
def check_user_image_inline(user_id, image_id, tag, bundle): """ Execute a policy evaluation using the info in the request body including the bundle content :param user_id: :param image_id: :param tag: :param bundle: :return: """ timer = time.time() db = get_session() cache_mgr = None try: # Input validation if tag is None: # set tag value to a value that only matches wildcards tag = "*/*:*" try: img_obj = db.query(Image).get((image_id, user_id)) except: return make_response_error("Image not found", in_httpcode=404), 404 if not img_obj: log.info( "Request for evaluation of image that cannot be found: user_id = {}, image_id = {}" .format(user_id, image_id)) return make_response_error("Image not found", in_httpcode=404), 404 if evaluation_cache_enabled: timer2 = time.time() try: try: conn_timeout = ( ApiRequestContextProxy.get_service().configuration.get( "catalog_client_conn_timeout", DEFAULT_CACHE_CONN_TIMEOUT)) read_timeout = ( ApiRequestContextProxy.get_service().configuration.get( "catalog_client_read_timeout", DEFAULT_CACHE_READ_TIMEOUT)) cache_mgr = EvaluationCacheManager(img_obj, tag, bundle, conn_timeout, read_timeout) except ValueError as err: log.warn( "Could not leverage cache due to error in bundle data: {}" .format(err)) cache_mgr = None if cache_mgr is None: log.info( "Could not initialize cache manager for policy evaluation, skipping cache usage" ) else: cached_result = cache_mgr.refresh() if cached_result: metrics.counter_inc( name="anchore_policy_evaluation_cache_hits") metrics.histogram_observe( "anchore_policy_evaluation_cache_access_latency", time.time() - timer2, status="hit", ) log.info( "Returning cached result of policy evaluation for {}/{}, with tag {} and bundle {} with digest {}. Last evaluation: {}" .format( user_id, image_id, tag, cache_mgr.bundle_id, cache_mgr.bundle_digest, cached_result.get("last_modified"), )) return cached_result else: metrics.counter_inc( name="anchore_policy_evaluation_cache_misses") metrics.histogram_observe( "anchore_policy_evaluation_cache_access_latency", time.time() - timer2, status="miss", ) log.info( "Policy evaluation not cached, or invalid, executing evaluation for {}/{} with tag {} and bundle {} with digest {}" .format( user_id, image_id, tag, cache_mgr.bundle_id, cache_mgr.bundle_digest, )) except Exception as ex: log.exception( "Unexpected error operating on policy evaluation cache. Skipping use of cache." ) else: log.info("Policy evaluation cache disabled. Executing evaluation") # Build bundle exec. problems = [] executable_bundle = None try: # Allow deprecated gates here to support upgrade cases from old policy bundles. executable_bundle = build_bundle(bundle, for_tag=tag, allow_deprecated=True) if executable_bundle.init_errors: problems = executable_bundle.init_errors except InitializationError as e: log.exception( "Bundle construction and initialization returned errors") problems = e.causes eval_result = None if not problems: # Execute bundle try: eval_result = executable_bundle.execute( img_obj, tag, ExecutionContext(db_session=db, configuration={})) except Exception as e: log.exception( "Error executing policy bundle {} against image {} w/tag {}: {}" .format(bundle["id"], image_id, tag, e)) return ( make_response_error( "Internal bundle evaluation error", details={ "message": "Cannot execute given policy against the image due to errors executing the policy bundle: {}" .format(e) }, in_httpcode=500, ), 500, ) else: # Construct a failure eval with details on the errors and mappings to send to client eval_result = build_empty_error_execution(img_obj, tag, executable_bundle, errors=problems, warnings=[]) if (executable_bundle and executable_bundle.mapping and len(executable_bundle.mapping.mapping_rules) == 1): eval_result.executed_mapping = executable_bundle.mapping.mapping_rules[ 0] resp = PolicyEvaluation() resp.user_id = user_id resp.image_id = image_id resp.tag = tag resp.bundle = bundle resp.matched_mapping_rule = (eval_result.executed_mapping.json() if eval_result.executed_mapping else False) resp.last_modified = int(time.time()) resp.final_action = eval_result.bundle_decision.final_decision.name resp.final_action_reason = eval_result.bundle_decision.reason resp.matched_whitelisted_images_rule = ( eval_result.bundle_decision.whitelisted_image.json() if eval_result.bundle_decision.whitelisted_image else False) resp.matched_blacklisted_images_rule = ( eval_result.bundle_decision.blacklisted_image.json() if eval_result.bundle_decision.blacklisted_image else False) resp.result = eval_result.as_table_json() resp.created_at = int(time.time()) resp.evaluation_problems = [ problem_from_exception(i) for i in eval_result.errors ] resp.evaluation_problems += [ problem_from_exception(i) for i in eval_result.warnings ] if resp.evaluation_problems: for i in resp.evaluation_problems: log.warn( "Returning evaluation response for image {}/{} w/tag {} and bundle {} that contains error: {}" .format(user_id, image_id, tag, bundle["id"], json.dumps(i.to_json()))) metrics.histogram_observe( "anchore_policy_evaluation_time_seconds", time.time() - timer, status="fail", ) else: metrics.histogram_observe( "anchore_policy_evaluation_time_seconds", time.time() - timer, status="success", ) result = resp.to_json() # Never let the cache block returning results try: if evaluation_cache_enabled and cache_mgr is not None: cache_mgr.save(result) except Exception as ex: log.exception( "Failed saving policy result in cache. Skipping and continuing." ) db.commit() return result except HTTPException as e: db.rollback() log.exception("Caught exception in execution: {}".format(e)) raise except Exception as e: db.rollback() log.exception("Failed processing bundle evaluation: {}".format(e)) return ( make_response_error( "Unexpected internal error", details={"message": str(e)}, in_httpcode=500, ), 500, ) finally: db.close()
def execute(self): """ Execute a load. Fetch from the catalog and send to loader. :return: the ImageLoad result object including the image object and its vulnerabilities or None if image already found """ self.start_time = datetime.datetime.utcnow() try: db = get_session() img = db.query(Image).get((self.image_id, self.user_id)) if img is not None: if not self.force_reload: logger.info( "Image {}/{} already found in the system. Will not re-load." .format(self.user_id, self.image_id)) db.close() return None else: logger.info( "Deleting image {}/{} and all associated resources for reload" .format(self.user_id, self.image_id)) # for pkg_vuln in img.vulnerabilities(): # db.delete(pkg_vuln) db.delete(img) # Close the session during the data fetch. # db.close() image_obj = self._load_image_analysis() if not image_obj: logger.error("Could not load image analysis") raise ImageLoadError( "Failed to load image: user_id = {}, image_id = {}, fetch_url = {}" .format(self.user_id, self.image_id, self.fetch_url)) db = get_session() try: logger.info("Adding image to db") db.add(image_obj) with timer("Generating vulnerability matches", log_level="info"): get_vulnerabilities_provider().load_image( image=image_obj, db_session=db, use_store=True, # save results to cache ) db.commit() except: logger.exception("Error adding image to db") db.rollback() raise return ImageLoadResult(image_obj) except Exception as e: logger.exception("Error loading and scanning image: {}".format( self.image_id)) raise finally: self.stop_time = datetime.datetime.utcnow()
def get_image_vulnerabilities( self, image: Image, db_session, vendor_only: bool = True, force_refresh: bool = False, cache: bool = True, ): # select the nvd class once and be done _nvd_cls, _cpe_cls = select_nvd_classes(db_session) # initialize the scanner scanner = self.__scanner__() user_id = image.user_id image_id = image.id results = [] if force_refresh: log.info("Forcing refresh of vulnerabilities for {}/{}".format( user_id, image_id)) try: scanner.flush_and_recompute_vulnerabilities( image, db_session=db_session) db_session.commit() except Exception as e: log.exception( "Error refreshing cve matches for image {}/{}".format( user_id, image_id)) db_session.rollback() return make_response_error( "Error refreshing vulnerability listing for image.", in_httpcode=500, ) db_session = get_session() db_session.refresh(image) with timer("Image vulnerability primary lookup", log_level="debug"): vulns = scanner.get_vulnerabilities(image) # Has vulnerabilities? warns = [] if not vulns: vulns = [] ns = DistroNamespace.for_obj(image) if not have_vulnerabilities_for(ns): warns = [ "No vulnerability data available for image distro: {}". format(ns.namespace_name) ] rows = [] with timer("Image vulnerability nvd metadata merge", log_level="debug"): vulns = merge_nvd_metadata_image_packages(db_session, vulns, _nvd_cls, _cpe_cls) with timer("Image vulnerability output formatting", log_level="debug"): for vuln, nvd_records in vulns: fixed_artifact = vuln.fixed_artifact() # Skip the vulnerability if the vendor_only flag is set to True and the issue won't be addressed by the vendor if vendor_only and vuln.fix_has_no_advisory( fixed_in=fixed_artifact): continue nvd_scores = [ self._make_cvss_score(score) for nvd_record in nvd_records for score in nvd_record.get_cvss_scores_nvd() ] results.append( VulnerabilityMatch( vulnerability=VulnerabilityModel( vulnerability_id=vuln.vulnerability_id, description="NA", severity=vuln.vulnerability.severity, link=vuln.vulnerability.link, feed="vulnerabilities", feed_group=vuln.vulnerability.namespace_name, cvss_scores_nvd=nvd_scores, cvss_scores_vendor=[], created_at=vuln.vulnerability.created_at, last_modified=vuln.vulnerability.updated_at, ), artifact=Artifact( name=vuln.pkg_name, version=vuln.package.fullversion, pkg_type=vuln.pkg_type, pkg_path=vuln.pkg_path, cpe="None", cpe23="None", ), fixes=[ FixedArtifact( version=str( vuln.fixed_in(fixed_in=fixed_artifact)), wont_fix=vuln.fix_has_no_advisory( fixed_in=fixed_artifact), observed_at=fixed_artifact.fix_observed_at if fixed_artifact else None, ) ], match=Match(detected_at=vuln.created_at), )) # TODO move dedup here so api doesn't have to # cpe_vuln_listing = [] try: with timer("Image vulnerabilities cpe matches", log_level="debug"): all_cpe_matches = scanner.get_cpe_vulnerabilities( image, _nvd_cls, _cpe_cls) if not all_cpe_matches: all_cpe_matches = [] api_endpoint = self._get_api_endpoint() for image_cpe, vulnerability_cpe in all_cpe_matches: link = vulnerability_cpe.parent.link if not link: link = "{}/query/vulnerabilities?id={}".format( api_endpoint, vulnerability_cpe.vulnerability_id) nvd_scores = [ self._make_cvss_score(score) for score in vulnerability_cpe.parent.get_cvss_scores_nvd() ] vendor_scores = [ self._make_cvss_score(score) for score in vulnerability_cpe.parent.get_cvss_scores_vendor() ] results.append( VulnerabilityMatch( vulnerability=VulnerabilityModel( vulnerability_id=vulnerability_cpe.parent. normalized_id, description="NA", severity=vulnerability_cpe.parent.severity, link=link, feed=vulnerability_cpe.feed_name, feed_group=vulnerability_cpe.namespace_name, cvss_scores_nvd=nvd_scores, cvss_scores_vendor=vendor_scores, created_at=vulnerability_cpe.parent.created_at, last_modified=vulnerability_cpe.parent. updated_at, ), artifact=Artifact( name=image_cpe.name, version=image_cpe.version, pkg_type=image_cpe.pkg_type, pkg_path=image_cpe.pkg_path, cpe=image_cpe.get_cpestring(), cpe23=image_cpe.get_cpe23string(), ), fixes=[ FixedArtifact( version=item, wont_fix=False, observed_at=vulnerability_cpe.created_at, ) for item in vulnerability_cpe.get_fixed_in() ], # using vulnerability created_at to indicate the match timestamp for now match=Match( detected_at=vulnerability_cpe.created_at), )) except Exception as err: log.exception("could not fetch CPE matches") import uuid return ImageVulnerabilitiesReport( account_id=image.user_id, image_id=image_id, results=results, metadata=VulnerabilitiesReportMetadata( generated_at=datetime.datetime.utcnow(), uuid=str(uuid.uuid4()), generated_by=self._get_provider_metadata(), ), problems=[], )
def _sync_group(self, group_download_result: GroupDownloadResult, full_flush=False, local_repo=None, operation_id=None): """ Sync data from a single group and return the data. This operation is scoped to a transaction on the db. :param group_obj: :return: """ total_updated_count = 0 result = build_group_sync_result() result['group'] = group_download_result.group sync_started = None db = get_session() group_db_obj = None if self.metadata: db.refresh(self.metadata) group_db_obj = self.group_by_name(group_download_result.group) if not group_db_obj: logger.error( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'Skipping sync for feed group {}, not found in db, record should have been synced already' )) return result download_started = group_download_result.started.replace( tzinfo=datetime.timezone.utc) sync_started = time.time() try: if full_flush: logger.info( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'Performing data flush prior to sync as requested')) self._flush_group(group_db_obj, operation_id=operation_id) mapper = self._load_mapper(group_db_obj) # Iterate thru the records and commit logger.info( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'Syncing {} total update records into db in sets of {}'. format(group_download_result.total_records, self.RECORDS_PER_CHUNK))) count = 0 for record in local_repo.read(group_download_result.feed, group_download_result.group, 0): mapped = mapper.map(record) merged = db.merge(mapped) total_updated_count += 1 count += 1 if count >= self.RECORDS_PER_CHUNK: # Commit group_db_obj.count = self.record_count( group_db_obj.name, db) db.commit() db = get_session() logger.info( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'DB Update Progress: {}/{}'.format( total_updated_count, group_download_result.total_records))) count = 0 else: group_db_obj.count = self.record_count(group_db_obj.name, db) db.commit() db = get_session() logger.info( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'DB Update Progress: {}/{}'.format( total_updated_count, group_download_result.total_records))) logger.debug( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'Updating last sync timestamp to {}'.format( download_started))) group_db_obj = self.group_by_name(group_download_result.group) # There is potential failures that could happen when downloading, # skipping updating the `last_sync` allows the system to retry if group_download_result.status == 'complete': group_db_obj.last_sync = download_started group_db_obj.count = self.record_count(group_db_obj.name, db) db.add(group_db_obj) db.commit() except Exception as e: logger.exception( log_msg_ctx(operation_id, group_download_result.feed, group_download_result.group, 'Error syncing')) db.rollback() raise e finally: sync_time = time.time() - sync_started total_group_time = time.time() - download_started.timestamp() logger.info( log_msg_ctx(operation_id, group_download_result.feed, group_download_result.group, 'Sync to db duration: {} sec'.format(sync_time))) logger.info( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'Total sync, including download, duration: {} sec'.format( total_group_time))) result['updated_record_count'] = total_updated_count result['status'] = 'success' result['total_time_seconds'] = total_group_time result['updated_image_count'] = 0 return result
def execute(self): """ Execute a load. Fetch from the catalog and send to loader. :return: the ImageLoad result object including the image object and its vulnerabilities or None if image already found """ self.start_time = datetime.datetime.utcnow() try: db = get_session() img = db.query(Image).get((self.image_id, self.user_id)) if img is not None: if not self.force_reload: log.info( 'Image {}/{} already found in the system. Will not re-load.' .format(self.user_id, self.image_id)) db.close() return None else: log.info( 'Deleting image {}/{} and all associated resources for reload' .format(self.user_id, self.image_id)) for pkg_vuln in img.vulnerabilities(): db.delete(pkg_vuln) db.delete(img) # Close the session during the data fetch. #db.close() image_obj = self._load_image_analysis() if not image_obj: log.error('Could not load image analysis') raise ImageLoadError( 'Failed to load image: user_id = {}, image_id = {}, fetch_url = {}' .format(self.user_id, self.image_id, self.fetch_url)) db = get_session() try: log.info("Adding image to db") db.add(image_obj) ts = time.time() log.info("Adding image package vulnerabilities to db") vulns = vulnerabilities_for_image(image_obj) for vuln in vulns: db.add(vuln) db.commit() #log.debug("TIMER TASKS: {}".format(time.time() - ts)) except: log.exception('Error adding image to db') db.rollback() raise return ImageLoadResult(image_obj, vulns) except Exception as e: log.exception('Error loading and scanning image: {}'.format( self.image_id)) raise finally: self.stop_time = datetime.datetime.utcnow()
def _sync_group(self, group_download_result: GroupDownloadResult, full_flush=False, local_repo=None, operation_id=None): """ Sync data from a single group and return the data. This operation is scoped to a transaction on the db. :param group_download_result :return: """ total_updated_count = 0 result = build_group_sync_result() result['group'] = group_download_result.group sync_started = None db = get_session() db.refresh(self.metadata) group_db_obj = self.group_by_name(group_download_result.group) if not group_db_obj: logger.error( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'Skipping group sync. Record not found in db, should have been synced already' )) return result sync_started = time.time() download_started = group_download_result.started.replace( tzinfo=datetime.timezone.utc) try: updated_images = set( ) # To get unique set of all images updated by this sync if full_flush: logger.info( log_msg_ctx(operation_id, group_download_result.feed, group_download_result.group, 'Performing group data flush prior to sync')) self._flush_group(group_db_obj, operation_id=operation_id) mapper = self._load_mapper(group_db_obj) # Iterate thru the records and commit count = 0 for record in local_repo.read(group_download_result.feed, group_download_result.group, 0): mapped = mapper.map(record) updated_image_ids = self.update_vulnerability( db, mapped, vulnerability_processing_fn=VulnerabilityFeed. __vuln_processing_fn__) updated_images = updated_images.union( set(updated_image_ids )) # Record after commit to ensure in-sync. merged = db.merge(mapped) total_updated_count += 1 count += 1 if len(updated_image_ids) > 0: db.flush( ) # Flush after every one so that mem footprint stays small if lots of images are updated if count >= self.RECORDS_PER_CHUNK: # Commit group_db_obj.count = self.record_count( group_db_obj.name, db) db.commit() logger.info( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'DB Update Progress: {}/{}'.format( total_updated_count, group_download_result.total_records))) db = get_session() count = 0 else: group_db_obj.count = self.record_count(group_db_obj.name, db) db.commit() logger.info( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'DB Update Progress: {}/{}'.format( total_updated_count, group_download_result.total_records))) db = get_session() logger.debug( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'Updating last sync timestamp to {}'.format( download_started))) group_db_obj = self.group_by_name(group_download_result.group) group_db_obj.last_sync = download_started group_db_obj.count = self.record_count(group_db_obj.name, db) db.add(group_db_obj) db.commit() except Exception as e: logger.exception( log_msg_ctx(operation_id, group_download_result.feed, group_download_result.group, 'Error syncing group')) db.rollback() raise e finally: total_group_time = time.time() - download_started.timestamp() sync_time = time.time() - sync_started logger.info( log_msg_ctx(operation_id, group_download_result.feed, group_download_result.group, 'Sync to db duration: {} sec'.format(sync_time))) logger.info( log_msg_ctx( operation_id, group_download_result.feed, group_download_result.group, 'Total sync, including download, duration: {} sec'.format( total_group_time))) result['updated_record_count'] = total_updated_count result['status'] = 'success' result['total_time_seconds'] = total_group_time result['updated_image_count'] = 0 return result
def test_image_blacklist(self): bundle = { 'id': 'multigate1', 'name': 'Multigate test1', 'version': '1_0', 'policies': [{ 'id': 'policy1', 'name': 'Test policy1', 'version': '1_0', 'rules': [{ 'gate': 'always', 'trigger': 'always', 'params': [], 'action': 'STOP' }] }], 'whitelists': [], 'mappings': [{ 'registry': '*', 'repository': '*', 'image': { 'type': 'tag', 'value': '*' }, 'policy_id': 'policy1', 'whitelist_ids': [] }], 'blacklisted_images': [{ 'registry': '*', 'repository': '*', 'image': { 'type': 'tag', 'value': 'latest' } }], 'whitelisted_images': [] } db = get_session() img_obj = db.query(Image).get((self.test_image_ids['ruby'], '0')) if not img_obj: self.load_images() self.assertIsNotNone(img_obj, 'Failed to get an image object to test') test_tag = 'docker.io/library/ruby:alpine' built = build_bundle(bundle, for_tag=test_tag) evaluation = built.execute(img_obj, tag=test_tag, context=ExecutionContext(db_session=db, configuration={})) self.assertIsNotNone(evaluation) self.assertEqual(GateAction.stop, evaluation.bundle_decision.final_decision) self.assertEqual('policy_evaluation', evaluation.bundle_decision.reason) self.assertIsNotNone(img_obj, 'Failed to get an image object to test') test_tag = 'docker.io/library/ruby:latest' built = build_bundle(bundle, for_tag=test_tag) evaluation = built.execute(img_obj, tag=test_tag, context=ExecutionContext(db_session=db, configuration={})) self.assertIsNotNone(evaluation) self.assertEqual(GateAction.stop, evaluation.bundle_decision.final_decision) self.assertEqual('blacklisted', evaluation.bundle_decision.reason) bundle = { 'id': 'emptytest1', 'name': 'Empty mapping test1', 'version': '1_0', 'policies': [], 'whitelists': [], 'mappings': [], 'blacklisted_images': [{ 'registry': '*', 'repository': '*', 'image': { 'type': 'tag', 'value': '*' } }], 'whitelisted_images': [] } built = build_bundle(bundle, for_tag=test_tag) evaluation = built.execute(img_obj, tag=test_tag, context=ExecutionContext(db_session=db, configuration={})) self.assertIsNotNone(evaluation) self.assertEqual(GateAction.stop, evaluation.bundle_decision.final_decision) self.assertEqual('blacklisted', evaluation.bundle_decision.reason) bundle = { 'id': 'emptytest1', 'name': 'Empty mapping test1', 'version': '1_0', 'policies': [], 'whitelists': [], 'mappings': [], 'whitelisted_images': [{ 'registry': '*', 'repository': '*', 'image': { 'type': 'tag', 'value': '*' } }], 'blacklisted_images': [] } built = build_bundle(bundle, for_tag=test_tag) evaluation = built.execute(img_obj, tag=test_tag, context=ExecutionContext(db_session=db, configuration={})) self.assertIsNotNone(evaluation) self.assertEqual(GateAction.go, evaluation.bundle_decision.final_decision) self.assertEqual('whitelisted', evaluation.bundle_decision.reason)
def get_image_vulnerabilities(user_id, image_id, force_refresh=False): """ Return the vulnerability listing for the specified image and load from catalog if not found and specifically asked to do so. Example json output: { "multi" : { "url_column_index" : 7, "result" : { "rows" : [], "rowcount" : 0, "colcount" : 8, "header" : [ "CVE_ID", "Severity", "*Total_Affected", "Vulnerable_Package", "Fix_Available", "Fix_Images", "Rebuild_Images", "URL" ] }, "querycommand" : "/usr/lib/python2.7/site-packages/anchore/anchore-modules/multi-queries/cve-scan.py /ebs_data/anchore/querytmp/queryimages.7026386 /ebs_data/anchore/data /ebs_data/anchore/querytmp/query.59057288 all", "queryparams" : "all", "warns" : [ "0005b136f0fb (prom/prometheus:master) cannot perform CVE scan: no CVE data is currently available for the detected base distro type (busybox:unknown_version,busybox:v1.26.2)" ] } } :param user_id: user id of image to evaluate :param image_id: image id to evaluate :param force_refresh: if true, flush and recompute vulnerabilities rather than returning current values :return: """ # Has image? db = get_session() try: img = db.query(Image).get((image_id, user_id)) vulns = [] if not img: abort(404) else: if force_refresh: log.info('Forcing refresh of vulnerabiltiies for {}/{}'.format( user_id, image_id)) try: current_vulns = img.vulnerabilities() log.info( 'Removing {} current vulnerabilities for {}/{} to rescan' .format(len(current_vulns), user_id, image_id)) for v in current_vulns: db.delete(v) db.flush() vulns = vulnerabilities_for_image(img) log.info('Adding {} vulnerabilities from rescan to {}/{}'. format(len(vulns), user_id, image_id)) for v in vulns: db.add(v) db.commit() except Exception as e: log.exception( 'Error refreshing cve matches for image {}/{}'.format( user_id, image_id)) db.rollback() abort( Response( 'Error refreshing vulnerability listing for image.', 500)) db = get_session() db.refresh(img) else: vulns = img.vulnerabilities() # Has vulnerabilities? warns = [] if not vulns: vulns = [] ns = DistroNamespace.for_obj(img) if not have_vulnerabilities_for(ns): warns = [ 'No vulnerability data available for image distro: {}'. format(ns.namespace_name) ] rows = [] for vuln in vulns: # if vuln.vulnerability.fixed_in: # fixes_in = filter(lambda x: x.name == vuln.pkg_name or x.name == vuln.package.normalized_src_pkg, # vuln.vulnerability.fixed_in) # fix_available_in = fixes_in[0].version if fixes_in else 'None' # else: # fix_available_in = 'None' rows.append([ vuln.vulnerability_id, vuln.vulnerability.severity, 1, vuln.pkg_name + '-' + vuln.package.fullversion, str(vuln.fixed_in()), vuln.pkg_image_id, 'None', # Always empty this for now vuln.vulnerability.link ]) vuln_listing = { 'multi': { 'url_column_index': 7, 'result': { 'header': TABLE_STYLE_HEADER_LIST, 'rowcount': len(rows), 'colcount': len(TABLE_STYLE_HEADER_LIST), 'rows': rows }, 'warns': warns } } report = LegacyVulnerabilityReport.from_dict(vuln_listing) resp = ImageVulnerabilityListing(user_id=user_id, image_id=image_id, legacy_report=report) return resp.to_dict() except HTTPException: db.rollback() raise except Exception as e: log.exception( 'Error checking image {}, {} for vulnerabiltiies. Rolling back'. format(user_id, image_id)) db.rollback() abort(500) finally: db.close()
def testWhitelists(self): print('Building executable bundle from default bundle') test_tag = 'docker.io/library/ruby:latest' built = build_bundle(self.default_bundle, for_tag=test_tag) self.assertFalse(built.init_errors) print('Got: {}'.format(built)) db = get_session() img_obj = db.query(Image).get((self.test_image_ids['ruby'], '0')) if not img_obj: self.load_images() self.assertIsNotNone(img_obj, 'Failed to get an image object to test') evaluation = built.execute(img_obj, tag=test_tag, context=ExecutionContext(db_session=db, configuration={})) self.assertIsNotNone(evaluation, 'Got None eval') print(json.dumps(evaluation.json(), indent=2)) print(json.dumps(evaluation.as_table_json(), indent=2)) to_whitelist = evaluation.bundle_decision.policy_decision.decisions[0] whitelist_bundle = copy.deepcopy(self.default_bundle) whitelist_bundle['whitelists'].append({ 'id': 'generated_whitelist1', 'name': 'test_whitelist', 'version': '1_0', 'items': [{ 'gate': to_whitelist.match.trigger.gate_cls.__gate_name__, 'trigger_id': to_whitelist.match.id, 'id': 'test_whitelistitem' }] }) whitelist_bundle['mappings'][0]['whitelist_ids'] = [ 'generated_whitelist1' ] built = build_bundle(whitelist_bundle, for_tag=test_tag) print('Got updated: {}'.format(built)) db = get_session() img_obj = db.query(Image).get((self.test_image_ids['ruby'], '0')) if not img_obj: self.load_images() self.assertIsNotNone(img_obj, 'Failed to get an image object to test') evaluation = built.execute(img_obj, tag=test_tag, context=ExecutionContext(db_session=db, configuration={})) self.assertIsNotNone(evaluation, 'Got None eval') #print(json.dumps(evaluation.json(), indent=2)) #print(json.dumps(evaluation.as_table_json(), indent=2)) self.assertNotIn( to_whitelist.match.id, map( lambda x: x.match.id if not (hasattr(x.match, 'is_whitelisted') and x.match. is_whitelisted) else None, evaluation.bundle_decision.policy_decision.decisions))
def test_cve_updates(test_data_env): test_env = test_data_env test_env.init_feeds() test_user_id = 'test1' test_img_id = 'img1' test_image = Image(user_id=test_user_id, id=test_img_id, distro_name='centos', distro_version='7') test_image.familytree_json = [test_img_id] test_image.layers_json = [test_img_id] test_image.layer_info_json = ['somelayer_here'] test_image.like_distro = 'centos' test_image.state = 'analyzed' test_image.digest = 'digest1' test_image.anchore_type = 'undefined' test_image.dockerfile_mode = 'Guessed' test_image.docker_history_json = ['line1', 'line2'] test_image.docker_data_json = {'Config': {}, 'ContainerConfig': {}} test_image.dockerfile_contents = 'FROM BLAH' test_package = ImagePackage(image_user_id=test_user_id, image_id=test_img_id, name='testpackage', version='1.0', pkg_type='RPM') test_package.src_pkg = 'testpackage' test_package.distro_name = 'centos' test_package.distro_version = '7' test_package.like_distro = 'centos' test_package.license = 'apache2' test_package.fullversion = '1.0' test_package.normalized_src_pkg = '1.0' test_package.release = '' test_package.size = 1000 test_package.origin = 'upstream' test_package.arch = 'x86_64' test_package.image = test_image test_cve = Vulnerability(id='CVE123', namespace_name='centos:7') test_cve.severity = 'High' test_cve.description = 'some test cve' test_cve.cvss2_score = '1.0' test_cve.metadata_json = {} test_cve.cvss2_vectors = '' test_cve.link = 'http://mitre.com/cve123' test_fixedin = FixedArtifact(vulnerability_id=test_cve.id) test_fixedin.name = 'testpackage' test_fixedin.version = '1.1' test_fixedin.version_format = 'rpm' test_fixedin.epochless_version = '1.1' test_fixedin.include_later_versions = True test_fixedin.parent = test_cve test_cve.fixed_in = [test_fixedin] test_vulnin = VulnerableArtifact(vulnerability_id=test_cve.id) test_vulnin.name = 'testpackage' test_vulnin.version = '0.9' test_vulnin.epochless_version = '0.9' test_vulnin.namespace_name = 'centos:7' test_vulnin.version_format = 'rpm' test_vulnin.include_previous_versions = False test_vulnin.parent = test_cve test_cve.vulnerable_in = [test_vulnin] db = get_session() try: db.add(test_image) db.add(test_package) db.commit() except sqlalchemy.exc.IntegrityError: db.rollback() except Exception: logger.exception('Unexpected failure') raise db = get_session() try: db.add(test_cve) feeds.process_updated_vulnerability(db, test_cve) db.commit() except sqlalchemy.exc.IntegrityError: logger.exception('Failed!') db.rollback() finally: db = get_session() i = db.query(Image).get((test_img_id, test_user_id)) print(('Vulns: {}'.format(i.vulnerabilities()))) db.commit() test_cve2 = Vulnerability(id='CVE123', namespace_name='centos:7') test_cve2.severity = 'Medium' test_cve2.description = 'some test cve' test_cve2.cvss2_score = '1.0' test_cve2.metadata_json = {} test_cve2.cvss2_vectors = '' test_cve2.link = 'http://mitre.com/cve123' fix2 = FixedArtifact(name='pkg2', version='1.2', epochless_version='1.2') fix2.namespace_name = 'centos:7' fix2.vulnerability_id = test_cve2.id test_cve2.fixed_in = [fix2] db = get_session() try: t2 = db.merge(test_cve2) db.add(t2) feeds.process_updated_vulnerability(db, t2) db.commit() except sqlalchemy.exc.IntegrityError: logger.exception('Failed!') db.rollback() finally: db = get_session() i = db.query(Image).get((test_img_id, test_user_id)) print(('Vulns: {}'.format(i.vulnerabilities()))) db.commit()
def testDeprecatedGateEvaluation(self): bundle = { 'id': 'someid', 'version': '1_0', 'whitelists': [], 'policies': [{ 'id': 'abc', 'name': 'Deprecated Policy', 'version': '1_0', 'rules': [{ 'gate': 'PKGDIFF', 'trigger': 'pkgadd', 'params': [], 'action': 'stop' }, { 'gate': 'always', 'trigger': 'always', 'action': 'go', 'params': [] }, { 'gate': 'ANCHORESEC', 'trigger': 'VULNLOW', 'action': 'warn', 'params': [] }] }], 'mappings': [{ 'registry': '*', 'repository': '*', 'image': { 'type': 'tag', 'value': '*' }, 'name': 'Default', 'policy_id': 'abc', 'whitelist_ids': [] }] } print('Building executable bundle from default bundle') test_tag = 'docker.io/library/ruby:latest' with self.assertRaises(InitializationError) as ex: built = build_bundle(bundle, for_tag=test_tag, allow_deprecated=False) print('Got: {}'.format(built)) db = get_session() img_obj = db.query(Image).get((self.test_image_ids['ruby'], '0')) if not img_obj: self.load_images() self.assertIsNotNone(img_obj, 'Failed to get an image object to test') evaluation = built.execute(img_obj, tag=test_tag, context=ExecutionContext( db_session=db, configuration={})) built = build_bundle(bundle, for_tag=test_tag, allow_deprecated=True) print('Got: {}'.format(built)) db = get_session() img_obj = db.query(Image).get((self.test_image_ids['ruby'], '0')) if not img_obj: self.load_images() self.assertIsNotNone(img_obj, 'Failed to get an image object to test') evaluation = built.execute(img_obj, tag=test_tag, context=ExecutionContext(db_session=db, configuration={})) self.assertIsNotNone(evaluation, 'Got None eval') print('Result: {}'.format(json.dumps(evaluation.json(), indent=2))) self.assertIsNotNone(evaluation.warnings)
def query_client(client_id): db = get_session() c = db.query(OAuth2Client).filter_by(client_id=client_id).first() return c
def testPolicyInitError(self): db = get_session() img_obj = db.query(Image).get((self.test_image_ids['ruby'], '0')) ruby_tag = 'dockerhub/library/ruby:latest' with self.assertRaises(UnsupportedVersionError) as f: built = build_bundle({ 'id': 'someid', 'version': 'invalid_version', 'name': 'invalid_version', 'whitelists': [], 'policies': [], 'mappings': [] }) built.execute(image_object=img_obj, context=None, tag=ruby_tag) with self.assertRaises(InitializationError) as f: built = build_bundle( { 'id': 'someid', 'version': '1_0', 'name': 'invalid_version', 'whitelists': [{ 'id': 'whitelist1', 'version': 'invalid_version', 'name': 'bad whitelist', 'rules': [] }], 'policies': [{ 'id': 'ok_policy', 'version': 'v1.0', 'name': 'bad policy', 'rules': [] }], 'mappings': [{ 'registry': '*', 'repository': '*', 'image': { 'type': 'tag', 'value': '*' }, 'policy_id': 'ok_policy', 'whitelist_ids': ['whitelist1'] }] }, for_tag='dockerhub/library/centos:latest') built.execute(image_object=img_obj, context=None, tag='dockerhub/library/centos:latest') self.assertEqual(type(f.exception.causes[0]), UnsupportedVersionError) with self.assertRaises(InitializationError) as f: built = build_bundle( { 'id': 'someid', 'version': '1_0', 'name': 'invalid_version', 'whitelists': [{ 'id': 'whitelist1', 'version': '1_0', 'name': 'okwhitelist', 'items': [] }], 'policies': [{ 'id': 'invalid_policy', 'version': 'invalid_version', 'name': 'bad policy', 'rules': [] }], 'mappings': [{ 'registry': '*', 'repository': '*', 'image': { 'type': 'tag', 'value': '*' }, 'policy_id': 'invalid_policy', 'whitelist_ids': ['whitelist1'] }] }, for_tag='dockerhub/library/centos:latest') built.execute(image_object=img_obj, context=None, tag='dockerhub/library/centos:latest') self.assertEqual(type(f.exception.causes[0]), UnsupportedVersionError) with self.assertRaises(InitializationError) as f: built = build_bundle({ 'id': 'someid', 'version': '1_0', 'name': 'invalid_version', 'whitelists': [{ 'id': 'whitelist1', 'version': '1_0', 'name': 'ok whitelist', 'items': [] }], 'policies': [{ 'id': 'okpolicy', 'version': '2_0', 'name': 'ok policy', 'rules': [] }], 'mappings': [{ 'id': 'invalid_mapping', 'policy_id': 'okpolicy', 'whitelist_ids': ['whitelist1'], 'registry': '*', 'repository': '*', 'image': { 'type': 'tag', 'value': '*' } }] }) built.execute(image_object=img_obj, context=None, tag=ruby_tag) self.assertEqual(type(f.exception.causes[0]), UnsupportedVersionError)
def _sync_group(self, group_obj, vulnerability_processing_fn=None): """ Sync data from a single group and return the data. The vulnerability_processing_fn callback is invoked for each item within the transaction scope. :param group_obj: the group object to sync :param bulk_load: should the load be done in bulk fashion, typically this is only for first run as it bypasses all per-item processing :return: """ sync_time = time.time() updated_images = [] db = get_session() try: next_token = '' while next_token is not None: if next_token == '': next_token = None fetch_time = time.time() new_data_deduped, next_token = self._get_deduped_data( group_obj, since=group_obj.last_sync, next_token=next_token, max_pages=self.MAX_FEED_SYNC_PAGES) fetch_time = time.time() - fetch_time log.debug('Group data fetch took {} sec'.format(fetch_time)) log.debug('Merging {} records from group {}'.format( len(new_data_deduped), group_obj.name)) db_time = time.time() for rec in new_data_deduped: # Make any updates and changes within this single transaction scope updated_image_ids = self.update_vulnerability( db, rec, vulnerability_processing_fn=vulnerability_processing_fn ) updated_images += updated_image_ids # Record after commit to ensure in-sync. db.flush() log.debug('Db merge took {} sec'.format(time.time() - db_time)) group_obj.last_sync = datetime.datetime.utcnow() db.add(group_obj) db.commit() except Exception as e: log.exception('Error syncing group: {}'.format(group_obj)) db.rollback() raise finally: sync_time = time.time() - sync_time log.info('Syncing group took {} sec'.format(sync_time)) # # # new_data_deduped, next_token = self._get_deduped_data(group_obj=group_obj, since=group_obj.last_sync) # updated_images = [] # db = get_session() # try: # log.info('Merging {} vulnerabilities into db for group {}'.format(len(new_data_deduped), group_obj.name)) # for rec in new_data_deduped: # # Make any updates and changes within this single transaction scope # updated_image_ids = self.update_vulnerability(db, rec, vulnerability_processing_fn=vulnerability_processing_fn) # updated_images += updated_image_ids # Record after commit to ensure in-sync. # db.flush() # # group_obj.last_sync = datetime.datetime.utcnow() # db.add(group_obj) # db.commit() # except Exception as e: # log.exception('Error during processing of vulnerabilities. Rolling back transaction and aborting') # db.rollback() # raise return updated_images
def testInvalidActions(self): db = get_session() img_obj = db.query(Image).get((self.test_image_ids['ruby'], '0')) with self.assertRaises(InitializationError) as f: built = build_bundle(self.test_env.get_bundle('bad_bundle1')) built.execute(image_object=img_obj, context=None, tag='dockerhub/library/ruby:latest') built.execute(image_object=img_obj, context=None, tag='test') with self.assertRaises(InitializationError) as f: built = build_bundle({ 'id': 'someid', 'version': '1_0', 'name': 'invalid_actions', 'whitelists': [{ 'id': 'whitelist1', 'version': '1_0', 'name': 'ok whitelist', 'items': [] }], 'policies': [{ 'id': 'okpolicy', 'version': '1_0', 'name': 'ok policy', 'rules': [{ 'gate': 'ANCHORESEC', 'trigger': 'UNSUPPPORTEDDISTRO', 'action': 'HELLO', 'params': [] }] }], 'mappings': [{ 'policy_id': 'okpolicy', 'whitelist_ids': ['whitelist1'], 'registry': '*', 'repository': '*', 'image': { 'type': 'tag', 'value': '*' } }] }) built.execute(image_object=img_obj, context=None, tag=None) with self.assertRaises(InitializationError) as f: bad_param1 = build_bundle({ 'id': 'someid', 'version': '1_0', 'name': 'invalid_params', 'whitelists': [{ 'id': 'whitelist1', 'version': '1_0', 'name': 'ok whitelist', 'items': [] }], 'policies': [{ 'id': 'okpolicy', 'version': '1_0', 'name': 'ok policy', 'rules': [{ 'gate': 'ANCHORESEC', 'trigger': 'FEEDOUTOFDATE', 'action': 'GO', 'params': [{ 'name': 'MAXAGE', 'value': 0.1 }] }] }], 'mappings': [{ 'policy_id': 'okpolicy', 'whitelist_ids': ['whitelist1'], 'registry': '*', 'repository': '*', 'image': { 'type': 'tag', 'value': '*' } }] }) built.execute(image_object=img_obj, context=None, tag=None) with self.assertRaises(InitializationError) as f: bad_param2 = build_bundle({ 'id': 'someid', 'version': '1_0', 'name': 'invalid_params', 'whitelists': [{ 'id': 'whitelist1', 'version': '1_0', 'name': 'ok whitelist', 'items': [] }], 'policies': [{ 'id': 'okpolicy', 'version': '1_0', 'name': 'ok policy', 'rules': [{ 'gate': 'ANCHORESEC', 'trigger': 'FEEDOUTOFDATE', 'action': 'GO', 'params': [{ 'name': 'MAXIMUS_AGIMUS', 'value': 10 }] }] }], 'mappings': [{ 'policy_id': 'okpolicy', 'whitelist_ids': ['whitelist1'], 'registry': '*', 'repository': '*', 'image': { 'type': 'tag', 'value': '*' } }] }) built.execute(image_object=img_obj, context=None, tag=None)
def check_user_image_inline(user_id, image_id, tag, bundle): """ Execute a policy evaluation using the info in the request body including the bundle content :param user_id: :param image_id: :param tag: :param bundle: :return: """ timer = time.time() db = get_session() try: # Input validation if tag is None: # set tag value to a value that only matches wildcards tag = '*/*:*' try: img_obj = db.query(Image).get((image_id, user_id)) except: abort(Response(response='Image not found', status=404)) if not img_obj: log.info( 'Request for evaluation of image that cannot be found: user_id = {}, image_id = {}' .format(user_id, image_id)) abort(Response(response='Image not found', status=404)) # Build bundle exec. problems = [] executable_bundle = None try: # Allow deprecated gates here to support upgrade cases from old policy bundles. executable_bundle = build_bundle(bundle, for_tag=tag, allow_deprecated=True) if executable_bundle.init_errors: problems = executable_bundle.init_errors except InitializationError as e: log.exception( 'Bundle construction and initialization returned errors') problems = e.causes eval_result = None if not problems: # Execute bundle try: eval_result = executable_bundle.execute( img_obj, tag, ExecutionContext(db_session=db, configuration={})) except Exception as e: log.exception( 'Error executing policy bundle {} against image {} w/tag {}: {}' .format(bundle['id'], image_id, tag, e.message)) abort( Response( response= 'Cannot execute given policy against the image due to errors executing the policy bundle: {}' .format(e.message), status=500)) else: # Construct a failure eval with details on the errors and mappings to send to client eval_result = build_empty_error_execution(img_obj, tag, executable_bundle, errors=problems, warnings=[]) if executable_bundle and executable_bundle.mapping and len( executable_bundle.mapping.mapping_rules) == 1: eval_result.executed_mapping = executable_bundle.mapping.mapping_rules[ 0] resp = PolicyEvaluation() resp.user_id = user_id resp.image_id = image_id resp.tag = tag resp.bundle = bundle resp.matched_mapping_rule = eval_result.executed_mapping.json( ) if eval_result.executed_mapping else False resp.last_modified = int(time.time()) resp.final_action = eval_result.bundle_decision.final_decision.name resp.final_action_reason = eval_result.bundle_decision.reason resp.matched_whitelisted_images_rule = eval_result.bundle_decision.whitelisted_image.json( ) if eval_result.bundle_decision.whitelisted_image else False resp.matched_blacklisted_images_rule = eval_result.bundle_decision.blacklisted_image.json( ) if eval_result.bundle_decision.blacklisted_image else False resp.result = eval_result.as_table_json() resp.created_at = int(time.time()) resp.evaluation_problems = [ problem_from_exception(i) for i in eval_result.errors ] resp.evaluation_problems += [ problem_from_exception(i) for i in eval_result.warnings ] if resp.evaluation_problems: for i in resp.evaluation_problems: log.warn( 'Returning evaluation response for image {}/{} w/tag {} and bundle {} that contains error: {}' .format(user_id, image_id, tag, bundle['id'], json.dumps(i.to_dict()))) anchore_engine.subsys.metrics.histogram_observe( 'anchore_policy_evaluation_time_seconds', time.time() - timer, status="fail") else: anchore_engine.subsys.metrics.histogram_observe( 'anchore_policy_evaluation_time_seconds', time.time() - timer, status="success") return resp.to_dict() except HTTPException as e: db.rollback() log.exception('Caught exception in execution: {}'.format(e)) raise except Exception as e: db.rollback() log.exception('Failed processing bundle evaluation: {}'.format(e)) abort(Response('Unexpected internal error', 500)) finally: db.close()
def testDuplicateRuleEvaluation(self): print('Building executable bundle from default bundle') test_tag = 'docker.io/library/ruby:latest' multi_gate_bundle = { 'id': 'multigate1', 'name': 'Multigate test1', 'version': '1_0', 'policies': [{ 'id': 'policy1', 'name': 'Test policy1', 'version': '1_0', 'rules': [{ 'gate': 'DOCKERFILECHECK', 'trigger': 'FROMSCRATCH', 'params': [], 'action': 'GO' }, { 'gate': 'DOCKERFILECHECK', 'trigger': 'FROMSCRATCH', 'params': [], 'action': 'STOP' }, { 'action': 'stop', 'gate': 'DOCKERFILECHECK', 'trigger': 'DIRECTIVECHECK', 'params': [{ 'name': 'DIRECTIVES', 'value': 'RUN' }, { 'name': 'CHECK', 'value': 'exists' }] }, { 'action': 'STOP', 'gate': 'DOCKERFILECHECK', 'trigger': 'DIRECTIVECHECK', 'params': [{ 'name': 'DIRECTIVES', 'value': 'USER' }, { 'name': 'CHECK', 'value': 'not_exists' }] }, { 'action': 'STOP', 'gate': 'DOCKERFILECHECK', 'trigger': 'DIRECTIVECHECK', 'params': [{ 'name': 'DIRECTIVES', 'value': 'RUN' }, { 'name': 'CHECK', 'value': '=', 'check_value': 'yum update -y' }] }] }], 'whitelists': [], 'mappings': [{ 'registry': '*', 'repository': '*', 'image': { 'type': 'tag', 'value': '*' }, 'policy_id': 'policy1', 'whitelist_ids': [] }] } built = build_bundle(multi_gate_bundle, for_tag=test_tag) self.assertFalse(built.init_errors) print('Got: {}'.format(built)) db = get_session() img_obj = db.query(Image).get((self.test_image_ids['ruby'], '0')) if not img_obj: self.load_images() self.assertIsNotNone(img_obj, 'Failed to get an image object to test') evaluation = built.execute(img_obj, tag=test_tag, context=ExecutionContext(db_session=db, configuration={})) self.assertIsNotNone(evaluation, 'Got None eval') print(json.dumps(evaluation.json(), indent=2)) print(json.dumps(evaluation.as_table_json(), indent=2))
def testRegexes(self): """ Test regular expressions in the trigger_id part of the WL rule :return: """ print('Building executable bundle from default bundle') test_tag = 'docker.io/library/node:latest' bundle = copy.deepcopy(self.default_bundle) node_whitelist = [ x for x in bundle['whitelists'] if x['id'] == 'wl_jessie' ][0] node_whitelist['items'] = [ x for x in node_whitelist['items'] if 'binutils' in x['trigger_id'] ] node_whitelist['items'].append({ 'gate': 'ANCHORESEC', 'trigger_id': 'CVE-2016-6515+openssh-client', 'id': 'testinserted3' }) node_whitelist['items'].append({ 'gate': 'ANCHORESEC', 'trigger_id': 'CVE-2016-6515+*', 'id': 'test-cve-2016-6515' }) node_whitelist['items'].append({ 'gate': 'ANCHORESEC', 'trigger_id': 'CVE-2017*', 'id': 'testinserted2' }) node_whitelist['items'].append({ 'gate': 'ANCHORESEC', 'trigger_id': '*binutils*', 'id': 'testinserted1' }) db = get_session() img_obj = db.query(Image).get( (self.test_env.get_images_named('node')[0][0], '0')) if not img_obj: self.load_images() self.assertIsNotNone(img_obj, 'Failed to get an image object to test') ExecutableWhitelist._use_indexes = True built = build_bundle(bundle, for_tag=test_tag) self.assertFalse(built.init_errors) print('Executing with indexes') t = time.time() evaluation = built.execute(img_obj, tag=test_tag, context=ExecutionContext(db_session=db, configuration={})) t1 = time.time() - t print(('Took: {}'.format(t1))) self.assertIsNotNone(evaluation, 'Got None eval') ExecutableWhitelist._use_indexes = False non_index_built = build_bundle(bundle, for_tag=test_tag) self.assertFalse(non_index_built.init_errors) print('Executing without indexes') t2 = time.time() evaluation2 = non_index_built.execute(img_obj, tag=test_tag, context=ExecutionContext( db_session=db, configuration={})) t2 = time.time() - t2 print(('Took: {}'.format(t2))) self.assertIsNotNone(evaluation2, 'Got None eval') ExecutableWhitelist._use_indexes = True self.assertListEqual( evaluation.json()['bundle_decision']['policy_decisions'][0] ['decisions'], evaluation2.json()['bundle_decision']['policy_decisions'][0] ['decisions']) print(('Evaluation: {}'.format(json.dumps(evaluation.json(), indent=2)))) open_ssl_wl_match = { "action": "go", "rule": { "action": "stop", "gate": "ANCHORESEC", "trigger": "VULNHIGH", "params": {} }, "match": { "message": "HIGH Vulnerability found in package - openssh-client (CVE-2016-6515 - https://security-tracker.debian.org/tracker/CVE-2016-6515)", "trigger": "VULNHIGH", "whitelisted": { "whitelist_id": "wl_jessie", "matched_rule_id": "testinserted3", "whitelist_name": "CVE whitelist for jessie - 12092017" }, "trigger_id": "CVE-2016-6515+openssh-client" } } self.assertIn( open_ssl_wl_match, evaluation.json()['bundle_decision']['policy_decisions'][0] ['decisions']) self.assertGreaterEqual( len([ x for x in evaluation.json()['bundle_decision'] ['policy_decisions'][0]['decisions'] if x['match'].get('whitelisted', {}).get( 'matched_rule_id', '') in ['testinserted1', 'testinserted2', 'testinserted3'] ]), 1)
def test_github_advisory_fixed_in(test_data_env): test_env = test_data_env test_env.init_feeds() test_user_id = "test1" test_img_id = "img1" test_image = Image(user_id=test_user_id, id=test_img_id, distro_name="centos", distro_version="7") test_image.familytree_json = [test_img_id] test_image.layers_json = [test_img_id] test_image.layer_info_json = ["somelayer_here"] test_image.like_distro = "centos" test_image.state = "analyzed" test_image.digest = "digest1" test_image.anchore_type = "undefined" test_image.dockerfile_mode = "Guessed" test_image.docker_history_json = ["line1", "line2"] test_image.docker_data_json = {"Config": {}, "ContainerConfig": {}} test_image.dockerfile_contents = "FROM BLAH" test_package = ImagePackage( image_user_id=test_user_id, image_id=test_img_id, name="testpackage", version="1.0", pkg_type="python", ) test_package.src_pkg = "testpackage" test_package.distro_name = "centos" test_package.distro_version = "7" test_package.like_distro = "centos" test_package.license = "apache2" test_package.fullversion = "1.0" test_package.normalized_src_pkg = "1.0" test_package.release = "" test_package.size = 1000 test_package.origin = "upstream" test_package.arch = "x86_64" test_package.image = test_image test_cve = Vulnerability(id="GHSA-rpch-cqj9-h65r", namespace_name="github:python") test_cve.severity = "High" test_cve.description = "some advisory ghsa" test_cve.link = "http://mitre.com/cve123" test_fixedin = FixedArtifact(vulnerability_id=test_cve.id) test_fixedin.name = "testpackage" test_fixedin.version = "None" test_fixedin.fix_metadata = {"first_patched_version": "1.2"} test_fixedin.version_format = "semver" test_fixedin.parent = test_cve test_cve.fixed_in = [test_fixedin] db = get_session() try: db.add(test_image) db.add(test_package) db.commit() except sqlalchemy.exc.IntegrityError: db.rollback() except Exception: logger.exception("Unexpected failure") raise db = get_session() # XXX This needs to be a fixture try: db.add(test_cve) feeds.process_updated_vulnerability(db, test_cve) db.commit() except sqlalchemy.exc.IntegrityError: logger.exception("Failed!") db.rollback() db = get_session() image_vuln = db.query(Image).get((test_img_id, test_user_id)) # should be one vulnerability vulnerabilities = image_vuln.vulnerabilities() assert len(vulnerabilities) == 1 img_pkg_vuln = vulnerabilities[0] assert img_pkg_vuln.fixed_in() == "1.2"