def _handle_garbage_collection(self): """ Periodically look for executions which have "handling" set to "True" and haven't been updated for a while (this likely indicates that an execution as picked up by a scheduler process which died before finishing the processing or similar) and reset handling to False so other scheduler can pick it up. """ query = { 'scheduled_start_timestamp__lte': date.append_milliseconds_to_time( date.get_datetime_utc_now(), -EXECUTION_SCHEDUELING_TIMEOUT_THRESHOLD_MS ), 'handling': True } execution_queue_item_dbs = ActionExecutionSchedulingQueue.query(**query) or [] for execution_queue_item_db in execution_queue_item_dbs: execution_queue_item_db.handling = False try: ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False) LOG.info('Removing lock for orphaned execution queue item: %s', execution_queue_item_db.id) except db_exc.StackStormDBObjectWriteConflictError: LOG.info( 'Execution queue item updated before rescheduling: %s', execution_queue_item_db.id )
def _handle_garbage_collection(self): """ Periodically look for executions which have "handling" set to "True" and haven't been updated for a while (this likely indicates that an execution as picked up by a scheduler process which died before finishing the processing or similar) and reset handling to False so other scheduler can pick it up. """ query = { 'scheduled_start_timestamp__lte': date.append_milliseconds_to_time( date.get_datetime_utc_now(), -EXECUTION_SCHEDUELING_TIMEOUT_THRESHOLD_MS), 'handling': True } execution_queue_item_dbs = ActionExecutionSchedulingQueue.query( **query) or [] for execution_queue_item_db in execution_queue_item_dbs: execution_queue_item_db.handling = False try: ActionExecutionSchedulingQueue.add_or_update( execution_queue_item_db, publish=False) LOG.info('Removing lock for orphaned execution queue item: %s', execution_queue_item_db.id) except db_exc.StackStormDBObjectWriteConflictError: LOG.info( 'Execution queue item updated before rescheduling: %s', execution_queue_item_db.id)
def _get_next_execution(self): """ Sort execution requests by FIFO and priority and get the latest, highest priority item from the queue and pop it off. """ query = { 'scheduled_start_timestamp__lte': date.get_datetime_utc_now(), 'handling': False, 'limit': 1, 'order_by': [ '+scheduled_start_timestamp', ] } execution_queue_item_db = ActionExecutionSchedulingQueue.query(**query).first() if not execution_queue_item_db: return None # Mark that this scheduler process is currently handling (processing) that request # NOTE: This operation is atomic (CAS) execution_queue_item_db.handling = True try: ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False) return execution_queue_item_db except db_exc.StackStormDBObjectWriteConflictError: LOG.info('Execution queue item handled by another scheduler: %s', execution_queue_item_db.id) return None
def _reset_handling_flag(self): """ Periodically look for executions which have "handling" set to "True" and haven't been updated for a while (this likely indicates that an execution as picked up by a scheduler process which died before finishing the processing or similar) and reset handling to False so other scheduler can pick it up. """ query = { 'scheduled_start_timestamp__lte': date.append_milliseconds_to_time( date.get_datetime_utc_now(), -self._execution_scheduling_timeout_threshold_min ), 'handling': True } execution_queue_item_dbs = ActionExecutionSchedulingQueue.query(**query) or [] for execution_queue_item_db in execution_queue_item_dbs: execution_queue_item_db.handling = False try: ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False) LOG.info( '[%s] Removing lock for orphaned execution queue item "%s".', execution_queue_item_db.action_execution_id, str(execution_queue_item_db.id) ) except db_exc.StackStormDBObjectWriteConflictError: LOG.info( '[%s] Execution queue item "%s" updated during garbage collection.', execution_queue_item_db.action_execution_id, str(execution_queue_item_db.id) )
def _get_next_execution(self): """ Sort execution requests by FIFO and priority and get the latest, highest priority item from the queue and pop it off. """ query = { 'scheduled_start_timestamp__lte': date.get_datetime_utc_now(), 'handling': False, 'limit': 1, 'order_by': [ '+scheduled_start_timestamp', ] } execution_queue_item_db = ActionExecutionSchedulingQueue.query( **query).first() if not execution_queue_item_db: return None # Mark that this scheduler process is currently handling (processing) that request # NOTE: This operation is atomic (CAS) execution_queue_item_db.handling = True try: ActionExecutionSchedulingQueue.add_or_update( execution_queue_item_db, publish=False) return execution_queue_item_db except db_exc.StackStormDBObjectWriteConflictError: LOG.info('Execution queue item handled by another scheduler: %s', execution_queue_item_db.id) return None
def _cleanup_policy_delayed(self): """ Clean up any action execution in the deprecated policy-delayed status. Associated entries in the scheduling queue will be removed and the action execution will be moved back into requested status. """ policy_delayed_liveaction_dbs = LiveAction.query(status="policy-delayed") or [] for liveaction_db in policy_delayed_liveaction_dbs: ex_que_qry = {"liveaction_id": str(liveaction_db.id), "handling": False} execution_queue_item_dbs = ( ActionExecutionSchedulingQueue.query(**ex_que_qry) or [] ) for execution_queue_item_db in execution_queue_item_dbs: # Mark the entry in the scheduling queue for handling. try: execution_queue_item_db.handling = True execution_queue_item_db = ( ActionExecutionSchedulingQueue.add_or_update( execution_queue_item_db, publish=False ) ) except db_exc.StackStormDBObjectWriteConflictError: msg = ( '[%s] Item "%s" is currently being processed by another scheduler.' % ( execution_queue_item_db.action_execution_id, str(execution_queue_item_db.id), ) ) LOG.error(msg) raise Exception(msg) # Delete the entry from the scheduling queue. LOG.info( '[%s] Removing policy-delayed entry "%s" from the scheduling queue.', execution_queue_item_db.action_execution_id, str(execution_queue_item_db.id), ) ActionExecutionSchedulingQueue.delete(execution_queue_item_db) # Update the status of the liveaction and execution to requested. LOG.info( '[%s] Removing policy-delayed entry "%s" from the scheduling queue.', execution_queue_item_db.action_execution_id, str(execution_queue_item_db.id), ) liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_REQUESTED ) execution_service.update_execution(liveaction_db)
def _fix_missing_action_execution_id(self): """ Auto-populate the action_execution_id in ActionExecutionSchedulingQueue if empty. """ for entry in ActionExecutionSchedulingQueue.query(action_execution_id__in=['', None]): execution_db = ActionExecution.get(liveaction__id=entry.liveaction_id) if not execution_db: continue msg = '[%s] Populating action_execution_id for item "%s".' LOG.info(msg, str(execution_db.id), str(entry.id)) entry.action_execution_id = str(execution_db.id) ActionExecutionSchedulingQueue.add_or_update(entry, publish=False)
def process(self, request): """ Adds execution into execution_scheduling database for scheduling :param request: Action execution request. :type request: ``st2common.models.db.liveaction.LiveActionDB`` """ if request.status != action_constants.LIVEACTION_STATUS_REQUESTED: LOG.info( '%s is ignoring %s (id=%s) with "%s" status.', self.__class__.__name__, type(request), request.id, request.status, ) return try: liveaction_db = action_utils.get_liveaction_by_id(str(request.id)) except StackStormDBObjectNotFoundError: LOG.exception("Failed to find liveaction %s in the database.", str(request.id)) raise query = { "liveaction_id": str(liveaction_db.id), } queued_requests = ActionExecutionSchedulingQueue.query(**query) if len(queued_requests) > 0: # Particular execution is already being scheduled return queued_requests[0] if liveaction_db.delay and liveaction_db.delay > 0: liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False) execution_queue_item_db = self._create_execution_queue_item_db_from_liveaction( liveaction_db, delay=liveaction_db.delay) ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False) return execution_queue_item_db
def _get_next_execution(self): """ Sort execution requests by FIFO and priority and get the latest, highest priority item from the queue and pop it off. NOTE: FIFO order is not guaranteed anymore for executions which are re-scheduled and delayed due to a policy. """ query = { "scheduled_start_timestamp__lte": date.get_datetime_utc_now(), "handling": False, "limit": 1, "order_by": ["+scheduled_start_timestamp", "+original_start_timestamp"], } execution_queue_item_db = ActionExecutionSchedulingQueue.query( **query).first() if not execution_queue_item_db: return None # Mark that this scheduler process is currently handling (processing) that request # NOTE: This operation is atomic (CAS) msg = '[%s] Retrieved item "%s" from scheduling queue.' LOG.info(msg, execution_queue_item_db.action_execution_id, execution_queue_item_db.id) execution_queue_item_db.handling = True try: ActionExecutionSchedulingQueue.add_or_update( execution_queue_item_db, publish=False) return execution_queue_item_db except db_exc.StackStormDBObjectWriteConflictError: LOG.info( '[%s] Item "%s" is already handled by another scheduler.', execution_queue_item_db.action_execution_id, str(execution_queue_item_db.id), ) return None
def process(self, request): """ Adds execution into execution_scheduling database for scheduling :param request: Action execution request. :type request: ``st2common.models.db.liveaction.LiveActionDB`` """ if request.status != action_constants.LIVEACTION_STATUS_REQUESTED: LOG.info('%s is ignoring %s (id=%s) with "%s" status.', self.__class__.__name__, type(request), request.id, request.status) return try: liveaction_db = action_utils.get_liveaction_by_id(str(request.id)) except StackStormDBObjectNotFoundError: LOG.exception('Failed to find liveaction %s in the database.', str(request.id)) raise query = { 'liveaction_id': str(liveaction_db.id), } queued_requests = ActionExecutionSchedulingQueue.query(**query) if len(queued_requests) > 0: # Particular execution is already being scheduled return queued_requests[0] if liveaction_db.delay and liveaction_db.delay > 0: liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False ) execution_queue_item_db = self._create_execution_queue_item_db_from_liveaction( liveaction_db, delay=liveaction_db.delay ) ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False) return execution_queue_item_db