def run_feeds_update(cls, json_obj=None, force_flush=False): """ Creates a task and runs it, optionally with a thread if locking is enabled. :return: """ error = None feeds = None with session_scope() as session: mgr = identities.manager_factory.for_session(session) system_user = mgr.get_system_credentials() catalog_client = CatalogClient(user=system_user[0], password=system_user[1]) try: feeds = get_selected_feeds_to_sync(localconfig.get_config()) if json_obj: task = cls.from_json(json_obj) if not task: return None task.feeds = feeds else: task = FeedsUpdateTask(feeds_to_sync=feeds, flush=force_flush) # Create feed task begin event try: catalog_client.add_event(FeedSyncStart(groups=feeds if feeds else 'all')) except: log.exception('Ignoring event generation error before feed sync') result = [] if cls.locking_enabled: # system_user = get_system_user_auth() run_target_with_lease(user_auth=system_user, lease_id='feed_sync', ttl=90, target=lambda: result.append(task.execute())) # A bit of work-around for the lambda def to get result from thread execution if result: result = result[0] else: result = task.execute() return result except LeaseAcquisitionFailedError as ex: error = ex log.exception('Could not acquire lock on feed sync, likely another sync already in progress') raise Exception('Cannot execute feed sync, lock is held by another feed sync in progress') except Exception as e: error = e log.exception('Error executing feeds update') raise e finally: # log feed sync event try: if error: catalog_client.add_event(FeedSyncFail(groups=feeds if feeds else 'all', error=error)) else: catalog_client.add_event(FeedSyncComplete(groups=feeds if feeds else 'all')) except: log.exception('Ignoring event generation error after feed sync')
def do_feed_sync(msg): if 'FeedsUpdateTask' not in locals(): from anchore_engine.services.policy_engine.engine.tasks import FeedsUpdateTask if 'get_selected_feeds_to_sync' not in locals(): from anchore_engine.services.policy_engine.engine.feeds import get_selected_feeds_to_sync handler_success = False timer = time.time() logger.info("FIRING: feed syncer") try: feeds = get_selected_feeds_to_sync(localconfig.get_config()) logger.info('Syncing configured feeds: {}'.format(feeds)) result = FeedsUpdateTask.run_feeds_update(json_obj=msg.get('data')) if result is not None: handler_success = True else: logger.warn('Feed sync task marked as disabled, so skipping') except ValueError as e: logger.warn('Received msg of wrong type') except Exception as err: logger.warn("failure in feed sync handler - exception: " + str(err)) if handler_success: anchore_engine.subsys.metrics.summary_observe('anchore_monitor_runtime_seconds', time.time() - timer, function='do_feed_sync', status="success") else: anchore_engine.subsys.metrics.summary_observe('anchore_monitor_runtime_seconds', time.time() - timer, function='do_feed_sync', status="fail")
def create_feed_update(notification): """ Creates a feed data update notification. :param notification: :return: """ if not connexion.request.is_json: abort(400) notification = FeedUpdateNotification.from_dict(notification) result = [] try: feeds = get_selected_feeds_to_sync(localconfig.get_config()) task = FeedsUpdateTask(feeds_to_sync=feeds) result = task.execute() except HTTPException: raise except Exception as e: log.exception('Error executing feed update task') abort( Response(status=500, response=json.dumps({ 'error': 'feed sync failure', 'details': 'Failure syncing feed: {}'.format(e.message) }), mimetype='application/json')) return jsonify(['{}/{}'.format(x[0], x[1]) for x in result]), 200
def _init_feeds(): """ Perform an initial feed sync using a bulk-sync if no sync has been done yet. :return: """ image_count_bulk_sync_threshold = 0 # More than this many images will result in the system doing a regular sync instead of a bulk sync. logger.info('Initializing feeds if necessary') from anchore_engine.services.policy_engine.engine import vulnerabilities, feeds from anchore_engine.services.policy_engine.engine.tasks import FeedsUpdateTask, InitialFeedSyncTask feeds = feeds.get_selected_feeds_to_sync(localconfig.get_config()) task = InitialFeedSyncTask(feeds_to_sync=feeds) task.execute() return True
def run_feeds_update(cls, json_obj=None, force_flush=False): """ Creates a task and runs it, optionally with a thread if locking is enabled. :return: """ try: feeds = get_selected_feeds_to_sync(localconfig.get_config()) if json_obj: task = cls.from_json(json_obj) if not task: return None task.feeds = feeds else: task = FeedsUpdateTask(feeds_to_sync=feeds, flush=force_flush) result = [] if cls.locking_enabled: system_user = get_system_user_auth() run_target_with_lease( user_auth=system_user, lease_id='feed_sync', ttl=90, target=lambda: result.append(task.execute())) # A bit of work-around for the lambda def to get result from thread execution if result: result = result[0] else: result = task.execute() return result except LeaseAcquisitionFailedError as ex: log.exception( 'Could not acquire lock on feed sync, likely another sync already in progress' ) raise Exception( 'Cannot execute feed sync, lock is held by another feed sync in progress' ) except Exception as e: log.exception('Error executing feeds update') raise e
def run_feeds_update(cls, json_obj=None, force_flush=False): """ Creates a task and runs it, optionally with a thread if locking is enabled. :return: """ feeds = None try: feeds = get_selected_feeds_to_sync(localconfig.get_config()) if json_obj: task = cls.from_json(json_obj) if not task: return None task.feeds = feeds else: task = FeedsUpdateTask(feeds_to_sync=feeds, flush=force_flush) result = [] if cls.locking_enabled: run_target_with_lease( account=None, lease_id='feed_sync', ttl=90, target=lambda: result.append(task.execute())) # A bit of work-around for the lambda def to get result from thread execution if result: result = result[0] else: result = task.execute() return result except Exception: log.exception('Error executing feeds update') raise
def run_feeds_update(cls, json_obj=None, force_flush=False): """ Creates a task and runs it, optionally with a thread if locking is enabled. :return: """ error = None feeds = None with session_scope() as session: mgr = identities.manager_factory.for_session(session) catalog_client = internal_client_for(CatalogClient, userId=None) try: feeds = get_selected_feeds_to_sync(localconfig.get_config()) if json_obj: task = cls.from_json(json_obj) if not task: return None task.feeds = feeds else: task = FeedsUpdateTask(feeds_to_sync=feeds, flush=force_flush) # Create feed task begin event try: catalog_client.add_event( FeedSyncStart(groups=feeds if feeds else 'all')) except: log.exception( 'Ignoring event generation error before feed sync') result = [] if cls.locking_enabled: run_target_with_lease( account=None, lease_id='feed_sync', ttl=90, target=lambda: result.append(task.execute())) # A bit of work-around for the lambda def to get result from thread execution if result: result = result[0] else: result = task.execute() return result except Exception as e: error = e log.exception('Error executing feeds update') raise e finally: # log feed sync event try: if error: catalog_client.add_event( FeedSyncFail(groups=feeds if feeds else 'all', error=error)) else: catalog_client.add_event( FeedSyncComplete(groups=feeds if feeds else 'all')) except: log.exception( 'Ignoring event generation error after feed sync')