def evaluate(self, image_obj, context):
feeds = DataFeeds.instance()
# Map to a namespace
ns = DistroNamespace.for_obj(image_obj)
oldest_update = None
if ns:
vulnerability_feed = DataFeeds.instance().vulnerabilities
for namespace_name in ns.like_namespace_names:
# Check feed names
groups = vulnerability_feed.group_by_name(namespace_name)
if groups:
# No records yet, but we have the feed, so may just not have any data yet
oldest_update = groups[0].last_sync
break
maxage = self.eval_params.get('MAXAGE')
if maxage:
try:
if oldest_update is not None:
oldest_update = calendar.timegm(oldest_update.timetuple())
mintime = time.time() - int(int(maxage) * 86400)
if oldest_update < mintime:
self._fire(msg="FEEDOUTOFDATE The vulnerability feed for this image distro is older than MAXAGE ("+str(maxage)+") days")
else:
self._fire(
msg="FEEDOUTOFDATE The vulnerability feed for this image distro is older than MAXAGE (" + str(
maxage) + ") days")
except Exception as err:
self._fire(msg="FEEDOUTOFDATE Cannot perform data feed up-to-date check - message from server: " + str(err))
def execute(self):
log.info('Starting feed update')
try:
f = DataFeeds.instance()
updated = []
vuln_updates = {}
# try:
# # Update the vulnerabilities with updates to the affected images done within the same transaction scope
# vuln_updates = df.vulnerabilities.sync(item_processing_fn=FeedsUpdateTask.process_updated_vulnerability)
# group_counts = [len(grp_updates) for grp_updates in vuln_updates.values()]
# total_count = reduce(lambda x, y: x + y, group_counts, 0)
# log.info('Processed {} vulnerability updates in {} groups'.format(total_count, len(group_counts)))
#
updated_dict = f.sync(to_sync=self.feeds)
for g in updated_dict:
updated += updated_dict[g]
log.info('Feed sync complete')
return updated
except:
log.exception('Failure refreshing and syncing feeds')
raise
finally:
end_session()
def test_gems_full_data(test_data_env):
count = 1
c = DataFeeds.instance().vulnerabilities.source
for v in c.get_feed_group_data('packages', 'gem'):
r = gem_mapper.map(v)
_gem_validator(r)
count += 1
def test_full_data(self):
c = DataFeeds.instance().vulnerabilities.source
for g in c.list_feed_groups('vulnerabilities'):
print('Group: {}'.format(g.name))
for v in c.get_feed_group_data('vulnerabilities', g.name):
r = self.mapper.map(v)
self.assertTrue(TestVulnerabilityMapping._vuln_validator(r), 'Failed validation on: {}'.format(v))
def execute(self):
log.info('Starting feed update')
# Feed syncs will update the images with any new cves that are pulled in for a the sync. As such, any images that are loaded while the sync itself is in progress need to be
# re-scanned for cves since the transaction ordering can result in the images being loaded with data prior to sync but not included in the sync process itself.
start_time = datetime.datetime.utcnow()
try:
f = DataFeeds.instance()
start_time = datetime.datetime.utcnow()
f.vuln_fn = FeedsUpdateTask.process_updated_vulnerability
f.vuln_flush_fn = FeedsUpdateTask.flush_vulnerability_matches
updated_dict = f.sync(to_sync=self.feeds,
full_flush=self.full_flush)
log.info('Feed sync complete. Results = {}'.format(updated_dict))
return updated_dict
except:
log.exception('Failure refreshing and syncing feeds')
raise
finally:
end_time = datetime.datetime.utcnow()
try:
self.rescan_images_created_between(from_time=start_time,
to_time=end_time)
except:
log.exception(
'Unexpected exception rescanning vulns for images added during the feed sync'
)
raise
finally:
end_session()
def test_vuln_full_data(test_data_env):
c = DataFeeds.instance().vulnerabilities.source
for g in c.list_feed_groups('vulnerabilities'):
print(('Group: {}'.format(g.name)))
for v in c.get_feed_group_data('vulnerabilities', g.name):
r = vuln_mapper.map(v)
assert vuln_validator(r), 'Failed validation on: {}'.format(v)
def execute(self):
log.info('Starting feed update')
# Feed syncs will update the images with any new cves that are pulled in for a the sync. As such, any images that are loaded while the sync itself is in progress need to be
# re-scanned for cves since the transaction ordering can result in the images being loaded with data prior to sync but not included in the sync process itself.
start_time = datetime.datetime.utcnow()
try:
f = DataFeeds.instance()
updated = []
start_time = datetime.datetime.utcnow()
f.vuln_fn = FeedsUpdateTask.process_updated_vulnerability
f.vuln_flush_fn = FeedsUpdateTask.flush_vulnerability_matches
updated_dict = f.sync(to_sync=self.feeds, full_flush=self.full_flush)
# Response is dict with feed name and dict for each group mapped to list of images updated
log.info('Updated: {}'.format(updated_dict))
for feed in updated_dict:
for updated_imgs in list(updated_dict[feed].values()):
updated += updated_imgs
log.info('Feed sync complete')
return updated
except:
log.exception('Failure refreshing and syncing feeds')
raise
finally:
end_time = datetime.datetime.utcnow()
self.rescan_images_created_between(from_time=start_time, to_time=end_time)
end_session()
def test_full_data(self):
count = 1
c = DataFeeds.instance().vulnerabilities.source
for v in c.get_feed_group_data('packages', 'gem'):
r = self.mapper.map(v)
self.assertTrue(TestGemMapping._gem_validator(r), 'Failed validation on #{}: {}'.format(count, v))
count += 1
def test_group_lookups(self):
df = DataFeeds.instance()
df.vulnerabilities.refresh_groups()
bad = df.vulnerabilities.group_by_name('not_a_real_Group')
self.assertFalse(bad, 'Found non-existent group')
self.assertIsNotNone(df.vulnerabilities.group_by_name('alpine:3.3'),
'Should have found group alpine:3.3')
def evaluate(self, image_obj, context):
try:
feed_meta = DataFeeds.instance().packages.group_by_name(FEED_KEY)
if feed_meta and feed_meta[0].last_sync:
return
except Exception as e:
log.exception('Error determining feed presence for npms. Defaulting to firing trigger')
self._fire()
return
def sync_feeds(test_env, up_to=None):
df = DataFeeds.instance()
if up_to:
test_env.set_max_feed_time(up_to)
logger.info('Syncing vuln')
df.vulnerabilities.sync(item_processing_fn=FeedsUpdateTask.process_updated_vulnerability)
logger.info('Syncing packages')
df.packages.sync()
logger.info('Sync complete')
def execute(self):
db = get_session()
try:
count = db.query(ImagePackageVulnerability).delete()
log.info('Deleted {} vulnerability match records in flush'.format(count))
f = DataFeeds.instance()
f.flush()
db.commit()
except:
log.exception('Error executing feeds flush task')
raise
def test_bulk_vuln_sync(self):
with session_scope() as db:
vcount = db.query(Vulnerability).count()
log.info('Starting with {} vuln records'.format(vcount))
self.assertEqual(vcount, 0, 'Not starting with empty table')
df = DataFeeds.instance()
t = time.time()
df.vulnerabilities.bulk_sync()
t = time.time() - t
log.info('Done with vulnerabilities. Took: {} sec'.format(t))
log.info('Has {} vuln records'.format(db.query(Vulnerability).count()))
def test_vuln_sync(self):
print('Test0')
with session_scope() as db:
vcount = db.query(Vulnerability).count()
log.info('Starting with {} vuln records'.format(vcount))
self.assertEqual(vcount, 0, 'Not starting with empty table')
df = DataFeeds.instance()
log.info('Syncing vulnerabilities')
t = time.time()
df.vulnerabilities.sync(group='alpine:3.3')
t = time.time() - t
log.info('Done with vulnerabilities. Took: {} sec'.format(t))
with session_scope() as db:
log.info('Has {} vuln records'.format(db.query(Vulnerability).count()))
def execute(self):
log.info('Starting initial feed sync')
f = DataFeeds.instance()
try:
updated = []
updated_dict = f.bulk_sync(to_sync=self.feed_list)
for g in updated_dict:
updated += updated_dict[g]
log.info('Initial feed sync complete')
return updated
except:
log.exception('Failure refreshing and syncing feeds')
raise
finally:
end_session()
def test_bulk_package_sync(self):
with session_scope() as db:
ncount = db.query(NpmMetadata).count()
gcount = db.query(GemMetadata).count()
self.assertEqual(ncount, 0, 'Not starting with empty table')
self.assertEqual(gcount, 0, 'Not starting with empty table')
df = DataFeeds.instance()
t = time.time()
df.packages.bulk_sync()
t = time.time() - t
log.info('Done with bulk package sync. Took: {} sec'.format(t))
with session_scope() as db:
ncount = db.query(NpmMetadata).count()
gcount = db.query(GemMetadata).count()
log.info('Has {} npm records'.format(ncount))
log.info('Has {} gem records'.format(gcount))
def test_package_sync(test_data_env):
with session_scope() as db:
ncount = db.query(NpmMetadata).count()
gcount = db.query(GemMetadata).count()
assert ncount == 0, 'Not starting with empty table'
assert gcount == 0, 'Not starting with empty table'
df = DataFeeds.instance()
logger.info('Syncing packages')
t = time.time()
df.packages.sync()
t = time.time() - t
logger.info('Done with packages. Took: {} sec'.format(t))
with session_scope() as db:
ncount = db.query(NpmMetadata).count()
gcount = db.query(GemMetadata).count()
logger.info('Has {} npm records'.format(ncount))
logger.info('Has {} gem records'.format(gcount))
def list_feeds(include_counts=False):
"""
GET /feeds
:return:
"""
f = DataFeeds.instance()
meta = f.list_metadata()
response = []
for feed in meta:
i = FeedMetadata()
i.name = feed.name
i.last_full_sync = feed.last_full_sync.isoformat(
) if feed.last_full_sync else None
i.created_at = feed.created_at.isoformat() if feed.created_at else None
i.updated_at = feed.last_update.isoformat(
) if feed.last_update else None
i.groups = []
for group in feed.groups:
g = FeedGroupMetadata()
g.name = group.name
g.last_sync = group.last_sync.isoformat(
) if group.last_sync else None
g.created_at = group.created_at.isoformat(
) if group.created_at else None
if include_counts:
# Compute count (this is slow)
g.record_count = f.records_for(i.name, g.name)
else:
g.record_count = None
i.groups.append(g.to_dict())
response.append(i.to_dict())
return jsonify(response)
def execute(self):
log.info('Starting feed update')
# Feed syncs will update the images with any new cves that are pulled in for a the sync. As such, any images that are loaded while the sync itself is in progress need to be
# re-scanned for cves since the transaction ordering can result in the images being loaded with data prior to sync but not included in the sync process itself.
# Create feed task begin event
error = None
with session_scope() as session:
mgr = identities.manager_factory.for_session(session)
catalog_client = internal_client_for(CatalogClient, userId=None)
try:
catalog_client.add_event(
FeedSyncTaskStarted(
groups=self.feeds if self.feeds else 'all'))
except:
log.exception('Ignoring event generation error before feed sync')
start_time = datetime.datetime.utcnow()
try:
f = DataFeeds.instance()
start_time = datetime.datetime.utcnow()
f.vuln_fn = FeedsUpdateTask.process_updated_vulnerability
f.vuln_flush_fn = FeedsUpdateTask.flush_vulnerability_matches
updated_dict = f.sync(to_sync=self.feeds,
full_flush=self.full_flush,
catalog_client=catalog_client)
log.info('Feed sync complete. Results = {}'.format(updated_dict))
return updated_dict
except Exception as e:
error = e
log.exception('Failure refreshing and syncing feeds')
raise
finally:
end_time = datetime.datetime.utcnow()
# log feed sync event
try:
if error:
catalog_client.add_event(
FeedSyncTaskFailed(
groups=self.feeds if self.feeds else 'all',
error=error))
else:
catalog_client.add_event(
FeedSyncTaskCompleted(
groups=self.feeds if self.feeds else 'all'))
except:
log.exception(
'Ignoring event generation error after feed sync')
try:
self.rescan_images_created_between(from_time=start_time,
to_time=end_time)
except:
log.exception(
'Unexpected exception rescanning vulns for images added during the feed sync'
)
raise
finally:
end_session()
def test_group_lookups(test_data_env):
df = DataFeeds.instance()
df.vulnerabilities.refresh_groups()
bad = df.vulnerabilities.group_by_name('not_a_real_Group')
assert not bad, 'Found non-existent group'
assert df.vulnerabilities.group_by_name('alpine:3.3') is not None, 'Should have found group alpine:3.3'