def sync_feeds(test_env, up_to=None):
if up_to:
test_env.set_max_feed_time(up_to)
logger.info('Syncing vuln and packages')
DataFeeds.__scratch_dir__ = '/tmp'
DataFeeds.sync(['vulnerabilities', 'packages'], feed_client=test_env.feed_client)
logger.info('Sync complete')
def sync_feeds(test_env, up_to=None):
if up_to:
test_env.set_max_feed_time(up_to)
logger.info("Syncing vuln and packages")
DataFeeds.__scratch_dir__ = "/tmp"
DataFeeds.sync(["vulnerabilities", "packages"],
feed_client=test_env.feed_client)
logger.info("Sync complete")
def test_sync_fail(test_data_env):
DataFeeds.__scratch_dir__ = '/tmp'
# No such feed
result = DataFeeds.sync(to_sync=['nvd'], feed_client=test_data_env.feed_client)
assert len(result) == 1
assert result[0]['status'] == 'failure'
DataFeeds.__scratch_dir__ = '/tmp'
result = DataFeeds.sync(to_sync=['vulnerabilities', 'packages', 'nvdv2', 'vulndb'], feed_client=test_data_env.feed_client)
assert len(result) == 4
assert not any(filter(lambda x: x['status'] == 'failure', result))
def test_sync_repo(test_data_env):
repo = LocalFeedDataRepo.from_disk('test/data/feeds_repo')
assert repo.has_metadata(), 'Repo should have metadata'
assert repo.has_root(), 'Repo should have root dir already'
with pytest.raises(Exception):
DataFeeds.sync_from_fetched(repo, catalog_client=None)
assert DataFeeds.sync_metadata(
feed_client=test_data_env.feed_client) == empty_metadata_sync_result
assert DataFeeds.sync_metadata(
feed_client=test_data_env.feed_client,
to_sync=['vulnerabilities'])[0].get('vulnerabilities') is not None
assert DataFeeds.sync_from_fetched(repo, catalog_client=None)
def cls_fully_loaded_test_env(cls_test_data_env2, request):
"""
Load the test env, including a feed sync and image analysis. Places the env in the class's test_env and test_image vars
:param cls_test_data_env:
:param request:
:return:
"""
_init_distro_mappings()
DataFeeds.__scratch_dir__ = '/tmp'
DataFeeds.sync(to_sync=['vulnerabilities', 'packages', 'nvdv2', 'vulndb'],
feed_client=request.cls.test_env.feed_client)
load_images(request)
def test_sync_repo(test_data_env, test_data_path):
feeds_repo_path = os.path.join(test_data_path, "feeds_repo")
repo = LocalFeedDataRepo.from_disk(feeds_repo_path)
assert repo.has_metadata(), "Repo should have metadata"
assert repo.has_root(), "Repo should have root dir already"
with pytest.raises(Exception):
DataFeeds.sync_from_fetched(repo, catalog_client=None)
assert (DataFeeds.sync_metadata(
feed_client=test_data_env.feed_client) == empty_metadata_sync_result)
assert (DataFeeds.sync_metadata(feed_client=test_data_env.feed_client,
to_sync=["vulnerabilities"
])[0].get("vulnerabilities")
is not None)
assert DataFeeds.sync_from_fetched(repo, catalog_client=None)
def test_sync_fail(test_data_env):
DataFeeds.__scratch_dir__ = "/tmp"
# No such feed
result = DataFeeds.sync(to_sync=["nvd"],
feed_client=test_data_env.feed_client)
assert len(result) == 1
assert result[0]["status"] == "failure"
DataFeeds.__scratch_dir__ = "/tmp"
result = DataFeeds.sync(
to_sync=["vulnerabilities", "packages", "nvdv2", "vulndb"],
feed_client=test_data_env.feed_client,
)
assert len(result) == 4
assert not any(filter(lambda x: x["status"] == "failure", result))
def test_metadata_sync(test_data_env):
r = DataFeeds.sync_metadata(feed_client=test_data_env.feed_client)
assert r == empty_metadata_sync_result, 'Expected empty dict result from metadata sync with no to_sync directive'
r = DataFeeds.sync_metadata(
feed_client=test_data_env.feed_client,
to_sync=['vulnerabilities', 'packages', 'vulndb', 'nvdv2'])
assert r is not None, 'Expected dict result from metadata sync'
assert type(r) == tuple and type(r[0]) == dict and type(
r[1]
) == list, 'Expected tuple with element 1 = dict result from metadata sync'
assert len(r[0]) == 4, 'Expected dict result from metadata sync'
assert r[0].get('vulnerabilities')
assert r[0].get('packages')
assert r[0].get('vulndb')
assert r[0].get('nvdv2')
def test_group_lookups(test_data_env):
r = DataFeeds.sync_metadata(feed_client=test_data_env.feed_client)
assert r == empty_metadata_sync_result, 'No metadata should be returned from sync with empty to_sync input'
r = DataFeeds.sync_metadata(feed_client=test_data_env.feed_client,
to_sync=['vulnerabilities'])
assert r and len(
r[0]
) == 1, 'Metadata should be returned from sync with non-empty to_sync list'
df = feed_instance_by_name('vulnerabilities')
assert df is not None, 'vulnerabilities feed instance not loaded'
assert df.metadata, 'No vuln metadata found'
logger.info('Vuln feed metadata {}'.format(df.metadata.to_json()))
assert not df.group_by_name('not_a_real_Group'), 'Found non-existent group'
assert df.group_by_name('alpine:3.6'), 'Should have found group alpine:3.6'
def run_legacy_sync(
test_env: LocalTestDataEnvironment, to_sync: List[str]
) -> List[FeedSyncResult]:
DataFeeds.__scratch_dir__ = "/tmp"
feed_url = os.getenv("ANCHORE_GRYPE_DB_URL", "https://ancho.re/v1/service/feeds")
data_clause = {}
for feed_name in to_sync:
data_clause[feed_name] = {"enabled": True, "url": feed_url}
config = {
"provider": "legacy",
"sync": {
"enabled": os.getenv("ANCHORE_FEEDS_ENABLED", True),
"ssl_verify": os.getenv("ANCHORE_FEEDS_SSL_VERIFY", True),
"connection_timeout_seconds": 3,
"read_timeout_seconds": 60,
"data": data_clause,
},
}
vulnerabilities_provider = LegacyProvider()
default_sync_config = vulnerabilities_provider.get_default_sync_config()
sync_configs = compute_selected_configs_to_sync(
provider="legacy",
vulnerabilities_config=config,
default_provider_sync_config=default_sync_config,
)
sync_utils = vulnerabilities_provider.get_sync_utils(sync_configs)
sync_utils.get_client = MagicMock(return_value=test_env.feed_client)
return DataFeeds.sync(sync_util_provider=sync_utils)
def test_vuln_sync(test_data_env):
with session_scope() as db:
vcount = db.query(Vulnerability).count()
logger.info('Starting with {} vuln records'.format(vcount))
assert vcount == 0, 'Not starting with empty table'
logger.info('Syncing vulnerabilities')
t = time.time()
DataFeeds.__scratch_dir__ = '/tmp'
DataFeeds.sync(to_sync=['vulnerabilities'],
feed_client=test_data_env.feed_client)
t = time.time() - t
logger.info('Done with vulnerabilities. Took: {} sec'.format(t))
with session_scope() as db:
logger.info('Has {} vuln records'.format(
db.query(Vulnerability).count()))
def test_group_lookups(test_data_env):
r = DataFeeds.sync_metadata(feed_client=test_data_env.feed_client)
assert (
r == empty_metadata_sync_result
), "No metadata should be returned from sync with empty to_sync input"
r = DataFeeds.sync_metadata(feed_client=test_data_env.feed_client,
to_sync=["vulnerabilities"])
assert (
r and len(r[0]) == 1
), "Metadata should be returned from sync with non-empty to_sync list"
df = feed_instance_by_name("vulnerabilities")
assert df is not None, "vulnerabilities feed instance not loaded"
assert df.metadata, "No vuln metadata found"
logger.info("Vuln feed metadata {}".format(df.metadata.to_json()))
assert not df.group_by_name("not_a_real_Group"), "Found non-existent group"
assert df.group_by_name("alpine:3.6"), "Should have found group alpine:3.6"
def test_vuln_sync(test_data_env):
with session_scope() as db:
vcount = db.query(Vulnerability).count()
logger.info("Starting with {} vuln records".format(vcount))
assert vcount == 0, "Not starting with empty table"
logger.info("Syncing vulnerabilities")
t = time.time()
DataFeeds.__scratch_dir__ = "/tmp"
DataFeeds.sync(to_sync=["vulnerabilities"],
feed_client=test_data_env.feed_client)
t = time.time() - t
logger.info("Done with vulnerabilities. Took: {} sec".format(t))
with session_scope() as db:
logger.info("Has {} vuln records".format(
db.query(Vulnerability).count()))
def test_metadata_sync(test_data_env):
r = DataFeeds.sync_metadata(feed_client=test_data_env.feed_client)
assert (
r == empty_metadata_sync_result
), "Expected empty dict result from metadata sync with no to_sync directive"
r = DataFeeds.sync_metadata(
feed_client=test_data_env.feed_client,
to_sync=["vulnerabilities", "packages", "vulndb", "nvdv2"],
)
assert r is not None, "Expected dict result from metadata sync"
assert (type(r) == tuple and type(r[0]) == dict and type(r[1]) == list
), "Expected tuple with element 1 = dict result from metadata sync"
assert len(r[0]) == 4, "Expected dict result from metadata sync"
assert r[0].get("vulnerabilities")
assert r[0].get("packages")
assert r[0].get("vulndb")
assert r[0].get("nvdv2")
def test_package_sync(test_data_env):
with session_scope() as db:
ncount = db.query(NpmMetadata).count()
gcount = db.query(GemMetadata).count()
assert ncount == 0, 'Not starting with empty table'
assert gcount == 0, 'Not starting with empty table'
logger.info('Syncing packages')
t = time.time()
DataFeeds.__scratch_dir__ = '/tmp'
DataFeeds.sync(to_sync=['packages'], feed_client=test_data_env.feed_client)
t = time.time() - t
logger.info('Done with packages. Took: {} sec'.format(t))
with session_scope() as db:
ncount = db.query(NpmMetadata).count()
gcount = db.query(GemMetadata).count()
logger.info('Has {} npm records'.format(ncount))
logger.info('Has {} gem records'.format(gcount))
def test_package_sync(test_data_env):
with session_scope() as db:
ncount = db.query(NpmMetadata).count()
gcount = db.query(GemMetadata).count()
assert ncount == 0, "Not starting with empty table"
assert gcount == 0, "Not starting with empty table"
logger.info("Syncing packages")
t = time.time()
DataFeeds.__scratch_dir__ = "/tmp"
DataFeeds.sync(to_sync=["packages"], feed_client=test_data_env.feed_client)
t = time.time() - t
logger.info("Done with packages. Took: {} sec".format(t))
with session_scope() as db:
ncount = db.query(NpmMetadata).count()
gcount = db.query(GemMetadata).count()
logger.info("Has {} npm records".format(ncount))
logger.info("Has {} gem records".format(gcount))
def test_metadata_sync(test_data_env):
source_feeds = DataFeeds.get_feed_group_information(
test_data_env.feed_client)
r = MetadataSyncUtils.sync_metadata(source_feeds=source_feeds)
assert (
r == empty_metadata_sync_result
), "Expected empty dict result from metadata sync with no to_sync directive"
to_sync = ["vulnerabilities", "packages", "vulndb", "nvdv2"]
source_feeds = DataFeeds.get_feed_group_information(
test_data_env.feed_client, to_sync=to_sync)
r = MetadataSyncUtils.sync_metadata(
source_feeds=source_feeds,
to_sync=to_sync,
)
assert r is not None, "Expected dict result from metadata sync"
assert (type(r) == tuple and type(r[0]) == dict and type(r[1]) == list
), "Expected tuple with element 1 = dict result from metadata sync"
assert len(r[0]) == 4, "Expected dict result from metadata sync"
assert r[0].get("vulnerabilities")
assert r[0].get("packages")
assert r[0].get("vulndb")
assert r[0].get("nvdv2")
def test_pivot_and_filter_feeds_by_config(feed_db_records):
v = [VulnerabilityFeed.__feed_name__]
v_db = [feed_db_records[VulnerabilityFeed.__feed_name__]]
v_n = [VulnerabilityFeed.__feed_name__, NvdV2Feed.__feed_name__]
v_n_db = [feed_db_records[VulnerabilityFeed.__feed_name__], feed_db_records[NvdV2Feed.__feed_name__]]
n = [NvdV2Feed.__feed_name__]
n_db = [feed_db_records[NvdV2Feed.__feed_name__]]
p = [PackagesFeed.__feed_name__]
p_db = [feed_db_records[PackagesFeed.__feed_name__]]
matrix = [
{'to_sync': [], 'source_found': [], 'db_found': [], 'expected_result': {}},
{
'to_sync': v,
'source_found': v,
'db_found': v_db,
'expected_result': {VulnerabilityFeed.__feed_name__: feed_db_records[VulnerabilityFeed.__feed_name__]}
},
{
'to_sync': v,
'source_found': v,
'db_found': v_n_db,
'expected_result': {VulnerabilityFeed.__feed_name__: feed_db_records[VulnerabilityFeed.__feed_name__]}
},
{
'to_sync': v,
'source_found': v_n,
'db_found': v_n_db,
'expected_result': {VulnerabilityFeed.__feed_name__: feed_db_records[VulnerabilityFeed.__feed_name__]}
},
{
'to_sync': v_n,
'source_found': v_n,
'db_found': v_n_db,
'expected_result': {
VulnerabilityFeed.__feed_name__: feed_db_records[VulnerabilityFeed.__feed_name__],
NvdV2Feed.__feed_name__: feed_db_records[NvdV2Feed.__feed_name__]
}
},
{
'to_sync': n,
'source_found': v,
'db_found': v_db,
'expected_result': {}
},
{
'to_sync': v,
'source_found': n,
'db_found': v_db,
'expected_result': {}
},
{
'to_sync': v,
'source_found': n,
'db_found': p_db,
'expected_result': {}
},
{
'to_sync': v_n,
'source_found': v_n,
'db_found': v_db,
'expected_result': {
VulnerabilityFeed.__feed_name__: feed_db_records[VulnerabilityFeed.__feed_name__]
}
},
{
'to_sync': v_n,
'source_found': v,
'db_found': v_n_db,
'expected_result': {
VulnerabilityFeed.__feed_name__: feed_db_records[VulnerabilityFeed.__feed_name__]
}
},
]
for input in matrix:
assert DataFeeds._pivot_and_filter_feeds_by_config(input['to_sync'], input['source_found'], input['db_found']) == input['expected_result']
def execute(self):
logger.info('Starting feed sync. (operation_id={})'.format(self.uuid))
# Feed syncs will update the images with any new cves that are pulled in for a the sync. As such, any images that are loaded while the sync itself is in progress need to be
# re-scanned for cves since the transaction ordering can result in the images being loaded with data prior to sync but not included in the sync process itself.
# Create feed task begin event
error = None
with session_scope() as session:
mgr = identities.manager_factory.for_session(session)
catalog_client = internal_client_for(CatalogClient, userId=None)
try:
notify_event(
FeedSyncTaskStarted(
groups=self.feeds if self.feeds else 'all'),
catalog_client, self.uuid)
except:
logger.exception(
'Ignoring event generation error before feed sync. (operation_id={})'
.format(self.uuid))
start_time = datetime.datetime.utcnow()
try:
start_time = datetime.datetime.utcnow()
updated_dict = DataFeeds.sync(to_sync=self.feeds,
full_flush=self.full_flush,
catalog_client=catalog_client,
operation_id=self.uuid)
logger.info('Feed sync complete (operation_id={})'.format(
self.uuid))
return updated_dict
except Exception as e:
error = e
logger.exception(
'Failure refreshing and syncing feeds. (operation_id={})'.
format(self.uuid))
raise
finally:
end_time = datetime.datetime.utcnow()
# log feed sync event
try:
if error:
notify_event(
FeedSyncTaskFailed(
groups=self.feeds if self.feeds else 'all',
error=error), catalog_client, self.uuid)
else:
notify_event(
FeedSyncTaskCompleted(
groups=self.feeds if self.feeds else 'all'),
catalog_client, self.uuid)
except:
logger.exception(
'Ignoring event generation error after feed sync (operation_id={})'
.format(self.uuid))
try:
self.rescan_images_created_between(from_time=start_time,
to_time=end_time)
except:
logger.exception(
'Unexpected exception rescanning vulns for images added during the feed sync. (operation_id={})'
.format(self.uuid))
raise
finally:
end_session()
def test_pivot_and_filter_feeds_by_config(feed_db_records):
v = [VulnerabilityFeed.__feed_name__]
v_db = [feed_db_records[VulnerabilityFeed.__feed_name__]]
v_n = [VulnerabilityFeed.__feed_name__, NvdV2Feed.__feed_name__]
v_n_db = [
feed_db_records[VulnerabilityFeed.__feed_name__],
feed_db_records[NvdV2Feed.__feed_name__],
]
n = [NvdV2Feed.__feed_name__]
n_db = [feed_db_records[NvdV2Feed.__feed_name__]]
p = [PackagesFeed.__feed_name__]
p_db = [feed_db_records[PackagesFeed.__feed_name__]]
matrix = [
{
"to_sync": [],
"source_found": [],
"db_found": [],
"expected_result": {}
},
{
"to_sync": v,
"source_found": v,
"db_found": v_db,
"expected_result": {
VulnerabilityFeed.__feed_name__:
feed_db_records[VulnerabilityFeed.__feed_name__]
},
},
{
"to_sync": v,
"source_found": v,
"db_found": v_n_db,
"expected_result": {
VulnerabilityFeed.__feed_name__:
feed_db_records[VulnerabilityFeed.__feed_name__]
},
},
{
"to_sync": v,
"source_found": v_n,
"db_found": v_n_db,
"expected_result": {
VulnerabilityFeed.__feed_name__:
feed_db_records[VulnerabilityFeed.__feed_name__]
},
},
{
"to_sync": v_n,
"source_found": v_n,
"db_found": v_n_db,
"expected_result": {
VulnerabilityFeed.__feed_name__:
feed_db_records[VulnerabilityFeed.__feed_name__],
NvdV2Feed.__feed_name__:
feed_db_records[NvdV2Feed.__feed_name__],
},
},
{
"to_sync": n,
"source_found": v,
"db_found": v_db,
"expected_result": {}
},
{
"to_sync": v,
"source_found": n,
"db_found": v_db,
"expected_result": {}
},
{
"to_sync": v,
"source_found": n,
"db_found": p_db,
"expected_result": {}
},
{
"to_sync": v_n,
"source_found": v_n,
"db_found": v_db,
"expected_result": {
VulnerabilityFeed.__feed_name__:
feed_db_records[VulnerabilityFeed.__feed_name__]
},
},
{
"to_sync": v_n,
"source_found": v,
"db_found": v_n_db,
"expected_result": {
VulnerabilityFeed.__feed_name__:
feed_db_records[VulnerabilityFeed.__feed_name__]
},
},
]
for input in matrix:
assert (DataFeeds._pivot_and_filter_feeds_by_config(
input["to_sync"], input["source_found"],
input["db_found"]) == input["expected_result"])
def execute(self) -> List[FeedSyncResult]:
logger.info("Starting feed sync. (operation_id={})".format(self.uuid))
# Feed syncs will update the images with any new cves that are pulled in for a the sync. As such, any images that are loaded while the sync itself is in progress need to be
# re-scanned for cves since the transaction ordering can result in the images being loaded with data prior to sync but not included in the sync process itself.
# Create feed task begin event
error = None
with session_scope() as session:
mgr = identities.manager_factory.for_session(session)
catalog_client = internal_client_for(CatalogClient, userId=None)
try:
notify_event(
FeedSyncTaskStarted(groups=list(self.sync_configs.keys(
)) if self.sync_configs else "all"),
catalog_client,
self.uuid,
)
except:
logger.exception(
"Ignoring event generation error before feed sync. (operation_id={})"
.format(self.uuid))
start_time = datetime.datetime.utcnow()
try:
start_time = datetime.datetime.utcnow()
updated_data_feeds = list()
updated_data_feeds.extend(
DataFeeds.sync(
sync_util_provider=GrypeProvider().get_sync_utils(
self.sync_configs),
full_flush=self.full_flush,
catalog_client=catalog_client,
operation_id=self.uuid,
))
updated_data_feeds.extend(
DataFeeds.sync(
sync_util_provider=LegacyProvider().get_sync_utils(
self.sync_configs),
full_flush=self.full_flush,
catalog_client=catalog_client,
operation_id=self.uuid,
))
logger.info("Feed sync complete (operation_id={})".format(
self.uuid))
return updated_data_feeds
except Exception as e:
error = e
logger.exception(
"Failure refreshing and syncing feeds. (operation_id={})".
format(self.uuid))
raise
finally:
end_time = datetime.datetime.utcnow()
# log feed sync event
try:
if error:
notify_event(
FeedSyncTaskFailed(
groups=list(self.sync_configs.keys())
if self.sync_configs else "all",
error=error,
),
catalog_client,
self.uuid,
)
else:
notify_event(
FeedSyncTaskCompleted(
groups=list(self.sync_configs.keys()
) if self.sync_configs else "all"),
catalog_client,
self.uuid,
)
except:
logger.exception(
"Ignoring event generation error after feed sync (operation_id={})"
.format(self.uuid))
try:
get_vulnerabilities_provider(
).rescan_images_loaded_during_feed_sync(self.uuid,
from_time=start_time,
to_time=end_time)
except:
logger.exception(
"Unexpected exception rescanning vulns for images added during the feed sync. (operation_id={})"
.format(self.uuid))
raise
finally:
end_session()
