def run_feeds_update(cls, json_obj=None, force_flush=False):
"""
Creates a task and runs it, optionally with a thread if locking is enabled.
:return:
"""
error = None
feeds = None
with session_scope() as session:
mgr = identities.manager_factory.for_session(session)
system_user = mgr.get_system_credentials()
catalog_client = CatalogClient(user=system_user[0], password=system_user[1])
try:
feeds = get_selected_feeds_to_sync(localconfig.get_config())
if json_obj:
task = cls.from_json(json_obj)
if not task:
return None
task.feeds = feeds
else:
task = FeedsUpdateTask(feeds_to_sync=feeds, flush=force_flush)
# Create feed task begin event
try:
catalog_client.add_event(FeedSyncStart(groups=feeds if feeds else 'all'))
except:
log.exception('Ignoring event generation error before feed sync')
result = []
if cls.locking_enabled:
# system_user = get_system_user_auth()
run_target_with_lease(user_auth=system_user, lease_id='feed_sync', ttl=90, target=lambda: result.append(task.execute()))
# A bit of work-around for the lambda def to get result from thread execution
if result:
result = result[0]
else:
result = task.execute()
return result
except LeaseAcquisitionFailedError as ex:
error = ex
log.exception('Could not acquire lock on feed sync, likely another sync already in progress')
raise Exception('Cannot execute feed sync, lock is held by another feed sync in progress')
except Exception as e:
error = e
log.exception('Error executing feeds update')
raise e
finally:
# log feed sync event
try:
if error:
catalog_client.add_event(FeedSyncFail(groups=feeds if feeds else 'all', error=error))
else:
catalog_client.add_event(FeedSyncComplete(groups=feeds if feeds else 'all'))
except:
log.exception('Ignoring event generation error after feed sync')
def run_feeds_update(cls, json_obj=None, force_flush=False):
"""
Creates a task and runs it, optionally with a thread if locking is enabled.
:return:
"""
error = None
feeds = None
with session_scope() as session:
mgr = identities.manager_factory.for_session(session)
catalog_client = internal_client_for(CatalogClient, userId=None)
try:
feeds = get_selected_feeds_to_sync(localconfig.get_config())
if json_obj:
task = cls.from_json(json_obj)
if not task:
return None
task.feeds = feeds
else:
task = FeedsUpdateTask(feeds_to_sync=feeds, flush=force_flush)
# Create feed task begin event
try:
catalog_client.add_event(
FeedSyncStart(groups=feeds if feeds else 'all'))
except:
log.exception(
'Ignoring event generation error before feed sync')
result = []
if cls.locking_enabled:
run_target_with_lease(
account=None,
lease_id='feed_sync',
ttl=90,
target=lambda: result.append(task.execute()))
# A bit of work-around for the lambda def to get result from thread execution
if result:
result = result[0]
else:
result = task.execute()
return result
except Exception as e:
error = e
log.exception('Error executing feeds update')
raise e
finally:
# log feed sync event
try:
if error:
catalog_client.add_event(
FeedSyncFail(groups=feeds if feeds else 'all',
error=error))
else:
catalog_client.add_event(
FeedSyncComplete(groups=feeds if feeds else 'all'))
except:
log.exception(
'Ignoring event generation error after feed sync')
def execute(self):
log.info('Starting feed update')
# Feed syncs will update the images with any new cves that are pulled in for a the sync. As such, any images that are loaded while the sync itself is in progress need to be
# re-scanned for cves since the transaction ordering can result in the images being loaded with data prior to sync but not included in the sync process itself.
# Create feed task begin event
error = None
with session_scope() as session:
mgr = identities.manager_factory.for_session(session)
catalog_client = internal_client_for(CatalogClient, userId=None)
try:
catalog_client.add_event(
FeedSyncStart(groups=self.feeds if self.feeds else 'all'))
except:
log.exception('Ignoring event generation error before feed sync')
start_time = datetime.datetime.utcnow()
try:
f = DataFeeds.instance()
start_time = datetime.datetime.utcnow()
f.vuln_fn = FeedsUpdateTask.process_updated_vulnerability
f.vuln_flush_fn = FeedsUpdateTask.flush_vulnerability_matches
updated_dict = f.sync(to_sync=self.feeds,
full_flush=self.full_flush)
log.info('Feed sync complete. Results = {}'.format(updated_dict))
return updated_dict
except Exception as e:
error = e
log.exception('Failure refreshing and syncing feeds')
raise
finally:
end_time = datetime.datetime.utcnow()
# log feed sync event
try:
if error:
catalog_client.add_event(
FeedSyncFail(
groups=self.feeds if self.feeds else 'all',
error=error))
else:
catalog_client.add_event(
FeedSyncComplete(
groups=self.feeds if self.feeds else 'all'))
except:
log.exception(
'Ignoring event generation error after feed sync')
try:
self.rescan_images_created_between(from_time=start_time,
to_time=end_time)
except:
log.exception(
'Unexpected exception rescanning vulns for images added during the feed sync'
)
raise
finally:
end_session()