def run_feeds_update(cls, json_obj=None, force_flush=False):
"""
Creates a task and runs it, optionally with a thread if locking is enabled.
:return:
"""
error = None
feeds = None
with session_scope() as session:
mgr = identities.manager_factory.for_session(session)
system_user = mgr.get_system_credentials()
catalog_client = CatalogClient(user=system_user[0], password=system_user[1])
try:
feeds = get_selected_feeds_to_sync(localconfig.get_config())
if json_obj:
task = cls.from_json(json_obj)
if not task:
return None
task.feeds = feeds
else:
task = FeedsUpdateTask(feeds_to_sync=feeds, flush=force_flush)
# Create feed task begin event
try:
catalog_client.add_event(FeedSyncStart(groups=feeds if feeds else 'all'))
except:
log.exception('Ignoring event generation error before feed sync')
result = []
if cls.locking_enabled:
# system_user = get_system_user_auth()
run_target_with_lease(user_auth=system_user, lease_id='feed_sync', ttl=90, target=lambda: result.append(task.execute()))
# A bit of work-around for the lambda def to get result from thread execution
if result:
result = result[0]
else:
result = task.execute()
return result
except LeaseAcquisitionFailedError as ex:
error = ex
log.exception('Could not acquire lock on feed sync, likely another sync already in progress')
raise Exception('Cannot execute feed sync, lock is held by another feed sync in progress')
except Exception as e:
error = e
log.exception('Error executing feeds update')
raise e
finally:
# log feed sync event
try:
if error:
catalog_client.add_event(FeedSyncFail(groups=feeds if feeds else 'all', error=error))
else:
catalog_client.add_event(FeedSyncComplete(groups=feeds if feeds else 'all'))
except:
log.exception('Ignoring event generation error after feed sync')
def do_feed_sync(msg):
if 'FeedsUpdateTask' not in locals():
from anchore_engine.services.policy_engine.engine.tasks import FeedsUpdateTask
if 'get_selected_feeds_to_sync' not in locals():
from anchore_engine.services.policy_engine.engine.feeds import get_selected_feeds_to_sync
handler_success = False
timer = time.time()
logger.info("FIRING: feed syncer")
try:
feeds = get_selected_feeds_to_sync(localconfig.get_config())
logger.info('Syncing configured feeds: {}'.format(feeds))
result = FeedsUpdateTask.run_feeds_update(json_obj=msg.get('data'))
if result is not None:
handler_success = True
else:
logger.warn('Feed sync task marked as disabled, so skipping')
except ValueError as e:
logger.warn('Received msg of wrong type')
except Exception as err:
logger.warn("failure in feed sync handler - exception: " + str(err))
if handler_success:
anchore_engine.subsys.metrics.summary_observe('anchore_monitor_runtime_seconds', time.time() - timer, function='do_feed_sync', status="success")
else:
anchore_engine.subsys.metrics.summary_observe('anchore_monitor_runtime_seconds', time.time() - timer, function='do_feed_sync', status="fail")
def create_feed_update(notification):
"""
Creates a feed data update notification.
:param notification:
:return:
"""
if not connexion.request.is_json:
abort(400)
notification = FeedUpdateNotification.from_dict(notification)
result = []
try:
feeds = get_selected_feeds_to_sync(localconfig.get_config())
task = FeedsUpdateTask(feeds_to_sync=feeds)
result = task.execute()
except HTTPException:
raise
except Exception as e:
log.exception('Error executing feed update task')
abort(
Response(status=500,
response=json.dumps({
'error':
'feed sync failure',
'details':
'Failure syncing feed: {}'.format(e.message)
}),
mimetype='application/json'))
return jsonify(['{}/{}'.format(x[0], x[1]) for x in result]), 200
def _init_feeds():
"""
Perform an initial feed sync using a bulk-sync if no sync has been done yet.
:return:
"""
image_count_bulk_sync_threshold = 0 # More than this many images will result in the system doing a regular sync instead of a bulk sync.
logger.info('Initializing feeds if necessary')
from anchore_engine.services.policy_engine.engine import vulnerabilities, feeds
from anchore_engine.services.policy_engine.engine.tasks import FeedsUpdateTask, InitialFeedSyncTask
feeds = feeds.get_selected_feeds_to_sync(localconfig.get_config())
task = InitialFeedSyncTask(feeds_to_sync=feeds)
task.execute()
return True
def run_feeds_update(cls, json_obj=None, force_flush=False):
"""
Creates a task and runs it, optionally with a thread if locking is enabled.
:return:
"""
try:
feeds = get_selected_feeds_to_sync(localconfig.get_config())
if json_obj:
task = cls.from_json(json_obj)
if not task:
return None
task.feeds = feeds
else:
task = FeedsUpdateTask(feeds_to_sync=feeds, flush=force_flush)
result = []
if cls.locking_enabled:
system_user = get_system_user_auth()
run_target_with_lease(
user_auth=system_user,
lease_id='feed_sync',
ttl=90,
target=lambda: result.append(task.execute()))
# A bit of work-around for the lambda def to get result from thread execution
if result:
result = result[0]
else:
result = task.execute()
return result
except LeaseAcquisitionFailedError as ex:
log.exception(
'Could not acquire lock on feed sync, likely another sync already in progress'
)
raise Exception(
'Cannot execute feed sync, lock is held by another feed sync in progress'
)
except Exception as e:
log.exception('Error executing feeds update')
raise e
def run_feeds_update(cls, json_obj=None, force_flush=False):
"""
Creates a task and runs it, optionally with a thread if locking is enabled.
:return:
"""
feeds = None
try:
feeds = get_selected_feeds_to_sync(localconfig.get_config())
if json_obj:
task = cls.from_json(json_obj)
if not task:
return None
task.feeds = feeds
else:
task = FeedsUpdateTask(feeds_to_sync=feeds, flush=force_flush)
result = []
if cls.locking_enabled:
run_target_with_lease(
account=None,
lease_id='feed_sync',
ttl=90,
target=lambda: result.append(task.execute()))
# A bit of work-around for the lambda def to get result from thread execution
if result:
result = result[0]
else:
result = task.execute()
return result
except Exception:
log.exception('Error executing feeds update')
raise
def run_feeds_update(cls, json_obj=None, force_flush=False):
"""
Creates a task and runs it, optionally with a thread if locking is enabled.
:return:
"""
error = None
feeds = None
with session_scope() as session:
mgr = identities.manager_factory.for_session(session)
catalog_client = internal_client_for(CatalogClient, userId=None)
try:
feeds = get_selected_feeds_to_sync(localconfig.get_config())
if json_obj:
task = cls.from_json(json_obj)
if not task:
return None
task.feeds = feeds
else:
task = FeedsUpdateTask(feeds_to_sync=feeds, flush=force_flush)
# Create feed task begin event
try:
catalog_client.add_event(
FeedSyncStart(groups=feeds if feeds else 'all'))
except:
log.exception(
'Ignoring event generation error before feed sync')
result = []
if cls.locking_enabled:
run_target_with_lease(
account=None,
lease_id='feed_sync',
ttl=90,
target=lambda: result.append(task.execute()))
# A bit of work-around for the lambda def to get result from thread execution
if result:
result = result[0]
else:
result = task.execute()
return result
except Exception as e:
error = e
log.exception('Error executing feeds update')
raise e
finally:
# log feed sync event
try:
if error:
catalog_client.add_event(
FeedSyncFail(groups=feeds if feeds else 'all',
error=error))
else:
catalog_client.add_event(
FeedSyncComplete(groups=feeds if feeds else 'all'))
except:
log.exception(
'Ignoring event generation error after feed sync')
