Example #1
0
    def _delay(self, liveaction_db, execution_queue_item_db):
        action_execution_id = str(execution_queue_item_db.action_execution_id)
        liveaction_id = str(execution_queue_item_db.liveaction_id)
        queue_item_id = str(execution_queue_item_db.id)
        extra = {'queue_item_id': queue_item_id}

        LOG.info(
            '[%s] Liveaction "%s" is delayed and scheduling queue is updated.',
            action_execution_id, liveaction_id, extra=extra
        )

        liveaction_db = action_service.update_status(
            liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False
        )

        execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
            date.get_datetime_utc_now(),
            POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS
        )

        try:
            execution_queue_item_db.handling = False
            ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
        except db_exc.StackStormDBObjectWriteConflictError:
            LOG.warning(
                '[%s] Database write conflict on updating scheduling queue.',
                action_execution_id, extra=extra
            )
Example #2
0
    def _handle_garbage_collection(self):
        """
        Periodically look for executions which have "handling" set to "True" and haven't been
        updated for a while (this likely indicates that an execution as picked up by a scheduler
        process which died before finishing the processing or similar) and reset handling to
        False so other scheduler can pick it up.
        """
        query = {
            'scheduled_start_timestamp__lte':
            date.append_milliseconds_to_time(
                date.get_datetime_utc_now(),
                -EXECUTION_SCHEDUELING_TIMEOUT_THRESHOLD_MS),
            'handling':
            True
        }

        execution_queue_item_dbs = ActionExecutionSchedulingQueue.query(
            **query) or []

        for execution_queue_item_db in execution_queue_item_dbs:
            execution_queue_item_db.handling = False

            try:
                ActionExecutionSchedulingQueue.add_or_update(
                    execution_queue_item_db, publish=False)
                LOG.info('Removing lock for orphaned execution queue item: %s',
                         execution_queue_item_db.id)
            except db_exc.StackStormDBObjectWriteConflictError:
                LOG.info(
                    'Execution queue item updated before rescheduling: %s',
                    execution_queue_item_db.id)
Example #3
0
    def _reset_handling_flag(self):
        """
        Periodically look for executions which have "handling" set to "True" and haven't been
        updated for a while (this likely indicates that an execution as picked up by a scheduler
        process which died before finishing the processing or similar) and reset handling to
        False so other scheduler can pick it up.
        """
        query = {
            'scheduled_start_timestamp__lte': date.append_milliseconds_to_time(
                date.get_datetime_utc_now(),
                -self._execution_scheduling_timeout_threshold_min
            ),
            'handling': True
        }

        execution_queue_item_dbs = ActionExecutionSchedulingQueue.query(**query) or []

        for execution_queue_item_db in execution_queue_item_dbs:
            execution_queue_item_db.handling = False

            try:
                ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
                LOG.info(
                    '[%s] Removing lock for orphaned execution queue item "%s".',
                    execution_queue_item_db.action_execution_id,
                    str(execution_queue_item_db.id)
                )
            except db_exc.StackStormDBObjectWriteConflictError:
                LOG.info(
                    '[%s] Execution queue item "%s" updated during garbage collection.',
                    execution_queue_item_db.action_execution_id,
                    str(execution_queue_item_db.id)
                )
Example #4
0
    def _get_next_execution(self):
        """
        Sort execution requests by FIFO and priority and get the latest, highest priority item from
        the queue and pop it off.
        """
        query = {
            'scheduled_start_timestamp__lte': date.get_datetime_utc_now(),
            'handling': False,
            'limit': 1,
            'order_by': [
                '+scheduled_start_timestamp',
            ]
        }

        execution_queue_item_db = ActionExecutionSchedulingQueue.query(
            **query).first()

        if not execution_queue_item_db:
            return None

        # Mark that this scheduler process is currently handling (processing) that request
        # NOTE: This operation is atomic (CAS)
        execution_queue_item_db.handling = True

        try:
            ActionExecutionSchedulingQueue.add_or_update(
                execution_queue_item_db, publish=False)
            return execution_queue_item_db
        except db_exc.StackStormDBObjectWriteConflictError:
            LOG.info('Execution queue item handled by another scheduler: %s',
                     execution_queue_item_db.id)

        return None
Example #5
0
File: handler.py Project: mahak/st2
    def _get_next_execution(self):
        """
        Sort execution requests by FIFO and priority and get the latest, highest priority item from
        the queue and pop it off.
        """
        query = {
            'scheduled_start_timestamp__lte': date.get_datetime_utc_now(),
            'handling': False,
            'limit': 1,
            'order_by': [
                '+scheduled_start_timestamp',
            ]
        }

        execution_queue_item_db = ActionExecutionSchedulingQueue.query(**query).first()

        if not execution_queue_item_db:
            return None

        # Mark that this scheduler process is currently handling (processing) that request
        # NOTE: This operation is atomic (CAS)
        execution_queue_item_db.handling = True

        try:
            ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
            return execution_queue_item_db
        except db_exc.StackStormDBObjectWriteConflictError:
            LOG.info('Execution queue item handled by another scheduler: %s',
                     execution_queue_item_db.id)

        return None
Example #6
0
File: handler.py Project: mahak/st2
    def _handle_garbage_collection(self):
        """
        Periodically look for executions which have "handling" set to "True" and haven't been
        updated for a while (this likely indicates that an execution as picked up by a scheduler
        process which died before finishing the processing or similar) and reset handling to
        False so other scheduler can pick it up.
        """
        query = {
            'scheduled_start_timestamp__lte': date.append_milliseconds_to_time(
                date.get_datetime_utc_now(),
                -EXECUTION_SCHEDUELING_TIMEOUT_THRESHOLD_MS
            ),
            'handling': True
        }

        execution_queue_item_dbs = ActionExecutionSchedulingQueue.query(**query) or []

        for execution_queue_item_db in execution_queue_item_dbs:
            execution_queue_item_db.handling = False

            try:
                ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
                LOG.info('Removing lock for orphaned execution queue item: %s',
                         execution_queue_item_db.id)
            except db_exc.StackStormDBObjectWriteConflictError:
                LOG.info(
                    'Execution queue item updated before rescheduling: %s',
                    execution_queue_item_db.id
                )
Example #7
0
    def _regulate_and_schedule(self, liveaction_db, execution_queue_item_db):
        action_execution_id = str(execution_queue_item_db.action_execution_id)
        liveaction_id = str(execution_queue_item_db.liveaction_id)
        queue_item_id = str(execution_queue_item_db.id)
        extra = {'queue_item_id': queue_item_id}

        LOG.info(
            '[%s] Liveaction "%s" has status "%s" before applying policies.',
            action_execution_id,
            liveaction_id,
            liveaction_db.status,
            extra=extra)

        # Apply policies defined for the action.
        liveaction_db = policy_service.apply_pre_run_policies(liveaction_db)

        LOG.info(
            '[%s] Liveaction "%s" has status "%s" after applying policies.',
            action_execution_id,
            liveaction_id,
            liveaction_db.status,
            extra=extra)

        if liveaction_db.status == action_constants.LIVEACTION_STATUS_DELAYED:
            LOG.info(
                '[%s] Liveaction "%s" is delayed and scheduling queue is updated.',
                action_execution_id,
                liveaction_id,
                extra=extra)

            liveaction_db = action_service.update_status(
                liveaction_db,
                action_constants.LIVEACTION_STATUS_DELAYED,
                publish=False)

            execution_queue_item_db.handling = False
            execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
                date.get_datetime_utc_now(),
                POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS)

            try:
                ActionExecutionSchedulingQueue.add_or_update(
                    execution_queue_item_db, publish=False)
            except db_exc.StackStormDBObjectWriteConflictError:
                LOG.warning(
                    '[%s] Database write conflict on updating scheduling queue.',
                    action_execution_id,
                    extra=extra)

            return

        if (liveaction_db.status
                in action_constants.LIVEACTION_COMPLETED_STATES
                or liveaction_db.status
                in action_constants.LIVEACTION_CANCEL_STATES):
            ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
            return

        self._schedule(liveaction_db, execution_queue_item_db)
Example #8
0
    def _fix_missing_action_execution_id(self):
        """
        Auto-populate the action_execution_id in ActionExecutionSchedulingQueue if empty.
        """
        for entry in ActionExecutionSchedulingQueue.query(action_execution_id__in=['', None]):
            execution_db = ActionExecution.get(liveaction__id=entry.liveaction_id)

            if not execution_db:
                continue

            msg = '[%s] Populating action_execution_id for item "%s".'
            LOG.info(msg, str(execution_db.id), str(entry.id))
            entry.action_execution_id = str(execution_db.id)
            ActionExecutionSchedulingQueue.add_or_update(entry, publish=False)
Example #9
0
    def _regulate_and_schedule(self, liveaction_db, execution_queue_item_db):
        # Apply policies defined for the action.
        liveaction_db = policy_service.apply_pre_run_policies(liveaction_db)

        liveaction_id = str(liveaction_db.id)
        queue_item_id = str(execution_queue_item_db.id)

        extra = {
            'liveaction_id': liveaction_id,
            'liveaction_status': liveaction_db.status,
            'queue_item_id': queue_item_id
        }

        LOG.info('Liveaction (%s) Status Pre-Run: %s (%s)',
                 liveaction_id,
                 liveaction_db.status,
                 queue_item_id,
                 extra=extra)

        if liveaction_db.status is action_constants.LIVEACTION_STATUS_POLICY_DELAYED:
            liveaction_db = action_service.update_status(
                liveaction_db,
                action_constants.LIVEACTION_STATUS_DELAYED,
                publish=False)

            execution_queue_item_db.handling = False
            execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
                date.get_datetime_utc_now(),
                POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS)

            try:
                ActionExecutionSchedulingQueue.add_or_update(
                    execution_queue_item_db, publish=False)
            except db_exc.StackStormDBObjectWriteConflictError:
                LOG.warning(
                    'Execution queue item update conflict during scheduling: %s',
                    execution_queue_item_db.id)

            return

        if (liveaction_db.status
                in action_constants.LIVEACTION_COMPLETED_STATES
                or liveaction_db.status
                in action_constants.LIVEACTION_CANCEL_STATES):
            ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
            return

        self._schedule(liveaction_db, execution_queue_item_db)
Example #10
0
    def test_failed_next_item(self, mocked_logger):
        self.reset()

        liveaction_db = self._create_liveaction_db()

        schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
            liveaction_db,
        )

        schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(schedule_q_db)

        with mock.patch(
            'st2actions.scheduler.handler.ActionExecutionSchedulingQueue.add_or_update',
            side_effect=db_exc.StackStormDBObjectWriteConflictError(schedule_q_db)
        ):
            schedule_q_db = self.scheduling_queue._get_next_execution()
            self.assertIsNone(schedule_q_db)

        self.assertEqual(mocked_logger.info.call_count, 2)
        call_args = mocked_logger.info.call_args_list[1][0]
        self.assertEqual(r'[%s] Item "%s" is already handled by another scheduler.', call_args[0])

        schedule_q_db = self.scheduling_queue._get_next_execution()
        self.assertIsNotNone(schedule_q_db)
        ActionExecutionSchedulingQueue.delete(schedule_q_db)
Example #11
0
    def test_populate_action_execution_id(self):
        self.reset()

        liveaction_db = self._create_liveaction_db()

        schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
            liveaction_db)

        # Manually unset the action_execution_id ot mock DB model of previous version.
        schedule_q_db.action_execution_id = None
        schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(
            schedule_q_db)
        schedule_q_db = ActionExecutionSchedulingQueue.get_by_id(
            str(schedule_q_db.id))
        self.assertIsNone(schedule_q_db.action_execution_id)

        # Run the clean up logic.
        self.scheduling_queue._fix_missing_action_execution_id()

        # Check that the action_execution_id is populated.
        schedule_q_db = ActionExecutionSchedulingQueue.get_by_id(
            str(schedule_q_db.id))
        execution_db = execution_service.update_execution(liveaction_db)
        self.assertEqual(schedule_q_db.action_execution_id,
                         str(execution_db.id))
Example #12
0
    def test_failed_next_item(self, mocked_logger):
        self.reset()

        liveaction_db = self._create_liveaction_db()

        schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
            liveaction_db, )

        schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(
            schedule_q_db)

        with mock.patch(
                "st2actions.scheduler.handler.ActionExecutionSchedulingQueue.add_or_update",
                side_effect=db_exc.StackStormDBObjectWriteConflictError(
                    schedule_q_db),
        ):
            schedule_q_db = self.scheduling_queue._get_next_execution()
            self.assertIsNone(schedule_q_db)

        self.assertEqual(mocked_logger.info.call_count, 2)
        call_args = mocked_logger.info.call_args_list[1][0]
        self.assertEqual(
            r'[%s] Item "%s" is already handled by another scheduler.',
            call_args[0])

        schedule_q_db = self.scheduling_queue._get_next_execution()
        self.assertIsNotNone(schedule_q_db)
        ActionExecutionSchedulingQueue.delete(schedule_q_db)
Example #13
0
    def test_failed_next_item(self, mocked_logger):
        self.reset()

        liveaction_db = self._create_liveaction_db()

        schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
            liveaction_db, )

        schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(
            schedule_q_db)

        with mock.patch(
                'st2actions.scheduler.handler.ActionExecutionSchedulingQueue.add_or_update',
                side_effect=db_exc.StackStormDBObjectWriteConflictError(
                    schedule_q_db)):
            schedule_q_db = self.scheduling_queue._get_next_execution()
            self.assertIsNone(schedule_q_db)

        mocked_logger.info.assert_called_once()
        call_args = mocked_logger.info.call_args_list[0][0]
        self.assertEqual(
            r'Execution queue item handled by another scheduler: %s',
            call_args[0])

        schedule_q_db = self.scheduling_queue._get_next_execution()
        self.assertIsNotNone(schedule_q_db)
        ActionExecutionSchedulingQueue.delete(schedule_q_db)
Example #14
0
    def process(self, request):
        """
        Adds execution into execution_scheduling database for scheduling

        :param request: Action execution request.
        :type request: ``st2common.models.db.liveaction.LiveActionDB``
        """
        if request.status != action_constants.LIVEACTION_STATUS_REQUESTED:
            LOG.info(
                '%s is ignoring %s (id=%s) with "%s" status.',
                self.__class__.__name__,
                type(request),
                request.id,
                request.status,
            )
            return

        try:
            liveaction_db = action_utils.get_liveaction_by_id(str(request.id))
        except StackStormDBObjectNotFoundError:
            LOG.exception("Failed to find liveaction %s in the database.",
                          str(request.id))
            raise

        query = {
            "liveaction_id": str(liveaction_db.id),
        }

        queued_requests = ActionExecutionSchedulingQueue.query(**query)

        if len(queued_requests) > 0:
            # Particular execution is already being scheduled
            return queued_requests[0]

        if liveaction_db.delay and liveaction_db.delay > 0:
            liveaction_db = action_service.update_status(
                liveaction_db,
                action_constants.LIVEACTION_STATUS_DELAYED,
                publish=False)

        execution_queue_item_db = self._create_execution_queue_item_db_from_liveaction(
            liveaction_db, delay=liveaction_db.delay)

        ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db,
                                                     publish=False)

        return execution_queue_item_db
Example #15
0
    def test_next_execution(self):
        self.reset()

        schedule_q_dbs = []
        delays = [2000, 5000, 4000]
        expected_order = [0, 2, 1]
        test_cases = []

        for delay in delays:
            liveaction_db = self._create_liveaction_db()
            delayed_start = date.append_milliseconds_to_time(
                liveaction_db.start_timestamp, delay)

            test_case = {
                'liveaction': liveaction_db,
                'delay': delay,
                'delayed_start': delayed_start
            }

            test_cases.append(test_case)

        for test_case in test_cases:
            schedule_q_dbs.append(
                ActionExecutionSchedulingQueue.add_or_update(
                    self.scheduler.
                    _create_execution_queue_item_db_from_liveaction(
                        test_case['liveaction'],
                        test_case['delay'],
                    )))

        # Wait maximum delay seconds so the query works as expected
        eventlet.sleep(3.2)

        for index in expected_order:
            test_case = test_cases[index]

            date_mock = mock.MagicMock()
            date_mock.get_datetime_utc_now.return_value = test_case[
                'delayed_start']
            date_mock.append_milliseconds_to_time = date.append_milliseconds_to_time

            with mock.patch('st2actions.scheduler.handler.date', date_mock):
                schedule_q_db = self.scheduling_queue._get_next_execution()
                ActionExecutionSchedulingQueue.delete(schedule_q_db)

            self.assertIsInstance(schedule_q_db,
                                  ActionExecutionSchedulingQueueItemDB)
            self.assertEqual(schedule_q_db.delay, test_case['delay'])
            self.assertEqual(schedule_q_db.liveaction_id,
                             str(test_case['liveaction'].id))

            # NOTE: We can't directly assert on the timestamp due to the delays on the code and
            # timing variance
            scheduled_start_timestamp = schedule_q_db.scheduled_start_timestamp
            test_case_start_timestamp = test_case['delayed_start']
            start_timestamp_diff = (scheduled_start_timestamp -
                                    test_case_start_timestamp)
            self.assertTrue(
                start_timestamp_diff <= datetime.timedelta(seconds=1))
Example #16
0
File: handler.py Project: mahak/st2
    def _delay(self, liveaction_db, execution_queue_item_db):
        liveaction_db = action_service.update_status(
            liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False
        )

        execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
            date.get_datetime_utc_now(),
            POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS
        )

        try:
            execution_queue_item_db.handling = False
            ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
        except db_exc.StackStormDBObjectWriteConflictError:
            LOG.warning(
                'Execution queue item update conflict during scheduling: %s',
                execution_queue_item_db.id
            )
Example #17
0
    def _delay(self, liveaction_db, execution_queue_item_db):
        liveaction_db = action_service.update_status(
            liveaction_db,
            action_constants.LIVEACTION_STATUS_DELAYED,
            publish=False)

        execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
            date.get_datetime_utc_now(),
            POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS)

        try:
            execution_queue_item_db.handling = False
            ActionExecutionSchedulingQueue.add_or_update(
                execution_queue_item_db, publish=False)
        except db_exc.StackStormDBObjectWriteConflictError:
            LOG.warning(
                'Execution queue item update conflict during scheduling: %s',
                execution_queue_item_db.id)
Example #18
0
    def _get_next_execution(self):
        """
        Sort execution requests by FIFO and priority and get the latest, highest priority item from
        the queue and pop it off.

        NOTE: FIFO order is not guaranteed anymore for executions which are re-scheduled and delayed
        due to a policy.
        """
        query = {
            "scheduled_start_timestamp__lte":
            date.get_datetime_utc_now(),
            "handling":
            False,
            "limit":
            1,
            "order_by":
            ["+scheduled_start_timestamp", "+original_start_timestamp"],
        }

        execution_queue_item_db = ActionExecutionSchedulingQueue.query(
            **query).first()

        if not execution_queue_item_db:
            return None

        # Mark that this scheduler process is currently handling (processing) that request
        # NOTE: This operation is atomic (CAS)
        msg = '[%s] Retrieved item "%s" from scheduling queue.'
        LOG.info(msg, execution_queue_item_db.action_execution_id,
                 execution_queue_item_db.id)
        execution_queue_item_db.handling = True

        try:
            ActionExecutionSchedulingQueue.add_or_update(
                execution_queue_item_db, publish=False)
            return execution_queue_item_db
        except db_exc.StackStormDBObjectWriteConflictError:
            LOG.info(
                '[%s] Item "%s" is already handled by another scheduler.',
                execution_queue_item_db.action_execution_id,
                str(execution_queue_item_db.id),
            )

        return None
Example #19
0
    def _cleanup_policy_delayed(self):
        """
        Clean up any action execution in the deprecated policy-delayed status. Associated
        entries in the scheduling queue will be removed and the action execution will be
        moved back into requested status.
        """

        policy_delayed_liveaction_dbs = LiveAction.query(status="policy-delayed") or []

        for liveaction_db in policy_delayed_liveaction_dbs:
            ex_que_qry = {"liveaction_id": str(liveaction_db.id), "handling": False}
            execution_queue_item_dbs = (
                ActionExecutionSchedulingQueue.query(**ex_que_qry) or []
            )

            for execution_queue_item_db in execution_queue_item_dbs:
                # Mark the entry in the scheduling queue for handling.
                try:
                    execution_queue_item_db.handling = True
                    execution_queue_item_db = (
                        ActionExecutionSchedulingQueue.add_or_update(
                            execution_queue_item_db, publish=False
                        )
                    )
                except db_exc.StackStormDBObjectWriteConflictError:
                    msg = (
                        '[%s] Item "%s" is currently being processed by another scheduler.'
                        % (
                            execution_queue_item_db.action_execution_id,
                            str(execution_queue_item_db.id),
                        )
                    )
                    LOG.error(msg)
                    raise Exception(msg)

                # Delete the entry from the scheduling queue.
                LOG.info(
                    '[%s] Removing policy-delayed entry "%s" from the scheduling queue.',
                    execution_queue_item_db.action_execution_id,
                    str(execution_queue_item_db.id),
                )

                ActionExecutionSchedulingQueue.delete(execution_queue_item_db)

                # Update the status of the liveaction and execution to requested.
                LOG.info(
                    '[%s] Removing policy-delayed entry "%s" from the scheduling queue.',
                    execution_queue_item_db.action_execution_id,
                    str(execution_queue_item_db.id),
                )

                liveaction_db = action_service.update_status(
                    liveaction_db, action_constants.LIVEACTION_STATUS_REQUESTED
                )

                execution_service.update_execution(liveaction_db)
Example #20
0
File: handler.py Project: mahak/st2
    def _regulate_and_schedule(self, liveaction_db, execution_queue_item_db):
        # Apply policies defined for the action.
        liveaction_db = policy_service.apply_pre_run_policies(liveaction_db)

        liveaction_id = str(liveaction_db.id)
        queue_item_id = str(execution_queue_item_db.id)

        extra = {
            'liveaction_id': liveaction_id,
            'liveaction_status': liveaction_db.status,
            'queue_item_id': queue_item_id
        }

        LOG.info('Liveaction (%s) Status Pre-Run: %s (%s)', liveaction_id, liveaction_db.status,
                 queue_item_id, extra=extra)

        if liveaction_db.status is action_constants.LIVEACTION_STATUS_POLICY_DELAYED:
            liveaction_db = action_service.update_status(
                liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False
            )

            execution_queue_item_db.handling = False
            execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time(
                date.get_datetime_utc_now(),
                POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS
            )

            try:
                ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
            except db_exc.StackStormDBObjectWriteConflictError:
                LOG.warning(
                    'Execution queue item update conflict during scheduling: %s',
                    execution_queue_item_db.id
                )

            return

        if (liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES or
                liveaction_db.status in action_constants.LIVEACTION_CANCEL_STATES):
            ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
            return

        self._schedule(liveaction_db, execution_queue_item_db)
Example #21
0
    def process(self, request):
        """
        Adds execution into execution_scheduling database for scheduling

        :param request: Action execution request.
        :type request: ``st2common.models.db.liveaction.LiveActionDB``
        """
        if request.status != action_constants.LIVEACTION_STATUS_REQUESTED:
            LOG.info('%s is ignoring %s (id=%s) with "%s" status.',
                     self.__class__.__name__, type(request), request.id, request.status)
            return

        try:
            liveaction_db = action_utils.get_liveaction_by_id(str(request.id))
        except StackStormDBObjectNotFoundError:
            LOG.exception('Failed to find liveaction %s in the database.', str(request.id))
            raise

        query = {
            'liveaction_id': str(liveaction_db.id),
        }

        queued_requests = ActionExecutionSchedulingQueue.query(**query)

        if len(queued_requests) > 0:
            # Particular execution is already being scheduled
            return queued_requests[0]

        if liveaction_db.delay and liveaction_db.delay > 0:
            liveaction_db = action_service.update_status(
                liveaction_db,
                action_constants.LIVEACTION_STATUS_DELAYED,
                publish=False
            )

        execution_queue_item_db = self._create_execution_queue_item_db_from_liveaction(
            liveaction_db,
            delay=liveaction_db.delay
        )

        ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)

        return execution_queue_item_db
Example #22
0
    def test_next_execution(self):
        self.reset()

        schedule_q_dbs = []
        delays = [2000, 5000, 4000]
        expected_order = [0, 2, 1]
        test_cases = []

        for delay in delays:
            liveaction_db = self._create_liveaction_db()
            delayed_start = date.append_milliseconds_to_time(liveaction_db.start_timestamp, delay)

            test_case = {
                'liveaction': liveaction_db,
                'delay': delay,
                'delayed_start': delayed_start
            }

            test_cases.append(test_case)

        for test_case in test_cases:
            schedule_q_dbs.append(
                ActionExecutionSchedulingQueue.add_or_update(
                    self.scheduler._create_execution_queue_item_db_from_liveaction(
                        test_case['liveaction'],
                        test_case['delay'],
                    )
                )
            )

        # Wait maximum delay seconds so the query works as expected
        eventlet.sleep(3.2)

        for index in expected_order:
            test_case = test_cases[index]

            date_mock = mock.MagicMock()
            date_mock.get_datetime_utc_now.return_value = test_case['delayed_start']
            date_mock.append_milliseconds_to_time = date.append_milliseconds_to_time

            with mock.patch('st2actions.scheduler.handler.date', date_mock):
                schedule_q_db = self.scheduling_queue._get_next_execution()
                ActionExecutionSchedulingQueue.delete(schedule_q_db)

            self.assertIsInstance(schedule_q_db, ActionExecutionSchedulingQueueItemDB)
            self.assertEqual(schedule_q_db.delay, test_case['delay'])
            self.assertEqual(schedule_q_db.liveaction_id, str(test_case['liveaction'].id))

            # NOTE: We can't directly assert on the timestamp due to the delays on the code and
            # timing variance
            scheduled_start_timestamp = schedule_q_db.scheduled_start_timestamp
            test_case_start_timestamp = test_case['delayed_start']
            start_timestamp_diff = (scheduled_start_timestamp - test_case_start_timestamp)
            self.assertTrue(start_timestamp_diff <= datetime.timedelta(seconds=1))
Example #23
0
    def test_next_execution(self):
        self.reset()

        schedule_q_dbs = []
        delays = [100, 5000, 1000]
        expected_order = [0, 2, 1]
        test_cases = []

        for delay in delays:
            liveaction_db = self._create_liveaction_db()
            delayed_start = date.append_milliseconds_to_time(
                liveaction_db.start_timestamp, delay)

            test_case = {
                'liveaction': liveaction_db,
                'delay': delay,
                'delayed_start': delayed_start
            }

            test_cases.append(test_case)

        for test_case in test_cases:
            schedule_q_dbs.append(
                ActionExecutionSchedulingQueue.add_or_update(
                    self.scheduler.
                    _create_execution_queue_item_db_from_liveaction(
                        test_case['liveaction'],
                        test_case['delay'],
                    )))

        for index in expected_order:
            test_case = test_cases[index]

            date_mock = mock.MagicMock()
            date_mock.get_datetime_utc_now.return_value = test_case[
                'delayed_start']
            date_mock.append_milliseconds_to_time = date.append_milliseconds_to_time

            with mock.patch('st2actions.scheduler.handler.date', date_mock):
                schedule_q_db = self.scheduling_queue._get_next_execution()
                ActionExecutionSchedulingQueue.delete(schedule_q_db)

            self.assertIsInstance(schedule_q_db,
                                  ActionExecutionSchedulingQueueItemDB)
            self.assertEqual(schedule_q_db.scheduled_start_timestamp,
                             test_case['delayed_start'])
            self.assertEqual(schedule_q_db.delay, test_case['delay'])
            self.assertEqual(schedule_q_db.liveaction_id,
                             str(test_case['liveaction'].id))
Example #24
0
    def test_cleanup_policy_delayed(self):
        self.reset()

        liveaction_db = self._create_liveaction_db()

        schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
            liveaction_db
        )

        schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(schedule_q_db)

        # Manually update the liveaction to policy-delayed status.
        # Using action_service.update_status will throw an exception on the
        # deprecated action_constants.LIVEACTION_STATUS_POLICY_DELAYED.
        liveaction_db.status = "policy-delayed"
        liveaction_db = LiveAction.add_or_update(liveaction_db)
        execution_db = execution_service.update_execution(liveaction_db)

        # Check that the execution status is set to policy-delayed.
        liveaction_db = LiveAction.get_by_id(str(liveaction_db.id))
        self.assertEqual(liveaction_db.status, "policy-delayed")

        execution_db = ActionExecution.get_by_id(str(execution_db.id))
        self.assertEqual(execution_db.status, "policy-delayed")

        # Run the clean up logic.
        self.scheduling_queue._cleanup_policy_delayed()

        # Check that the execution status is reset to requested.
        liveaction_db = LiveAction.get_by_id(str(liveaction_db.id))
        self.assertEqual(
            liveaction_db.status, action_constants.LIVEACTION_STATUS_REQUESTED
        )

        execution_db = ActionExecution.get_by_id(str(execution_db.id))
        self.assertEqual(
            execution_db.status, action_constants.LIVEACTION_STATUS_REQUESTED
        )

        # The old entry should have been deleted. Since the execution is
        # reset to requested, there should be a new scheduling entry.
        new_schedule_q_db = self.scheduling_queue._get_next_execution()
        self.assertIsNotNone(new_schedule_q_db)
        self.assertNotEqual(str(schedule_q_db.id), str(new_schedule_q_db.id))
        self.assertEqual(
            schedule_q_db.action_execution_id, new_schedule_q_db.action_execution_id
        )
        self.assertEqual(schedule_q_db.liveaction_id, new_schedule_q_db.liveaction_id)
Example #25
0
    def test_cleanup_policy_delayed(self):
        self.reset()

        liveaction_db = self._create_liveaction_db()

        schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
            liveaction_db
        )

        schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(schedule_q_db)

        # Manually update the liveaction to policy-delayed status.
        # Using action_service.update_status will throw an exception on the
        # deprecated action_constants.LIVEACTION_STATUS_POLICY_DELAYED.
        liveaction_db.status = 'policy-delayed'
        liveaction_db = LiveAction.add_or_update(liveaction_db)
        execution_db = execution_service.update_execution(liveaction_db)

        # Check that the execution status is set to policy-delayed.
        liveaction_db = LiveAction.get_by_id(str(liveaction_db.id))
        self.assertEqual(liveaction_db.status, 'policy-delayed')

        execution_db = ActionExecution.get_by_id(str(execution_db.id))
        self.assertEqual(execution_db.status, 'policy-delayed')

        # Run the clean up logic.
        self.scheduling_queue._cleanup_policy_delayed()

        # Check that the execution status is reset to requested.
        liveaction_db = LiveAction.get_by_id(str(liveaction_db.id))
        self.assertEqual(liveaction_db.status, action_constants.LIVEACTION_STATUS_REQUESTED)

        execution_db = ActionExecution.get_by_id(str(execution_db.id))
        self.assertEqual(execution_db.status, action_constants.LIVEACTION_STATUS_REQUESTED)

        # The old entry should have been deleted. Since the execution is
        # reset to requested, there should be a new scheduling entry.
        new_schedule_q_db = self.scheduling_queue._get_next_execution()
        self.assertIsNotNone(new_schedule_q_db)
        self.assertNotEqual(str(schedule_q_db.id), str(new_schedule_q_db.id))
        self.assertEqual(schedule_q_db.action_execution_id, new_schedule_q_db.action_execution_id)
        self.assertEqual(schedule_q_db.liveaction_id, new_schedule_q_db.liveaction_id)
Example #26
0
    def test_garbage_collection(self):
        self.reset()

        liveaction_db = self._create_liveaction_db()

        schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
            liveaction_db,
            -70000,
        )

        schedule_q_db.handling = True
        schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(schedule_q_db)

        schedule_q_db = self.scheduling_queue._get_next_execution()
        self.assertIsNone(schedule_q_db)

        self.scheduling_queue._handle_garbage_collection()

        schedule_q_db = self.scheduling_queue._get_next_execution()
        self.assertIsNotNone(schedule_q_db)
Example #27
0
    def test_garbage_collection(self):
        self.reset()

        liveaction_db = self._create_liveaction_db()

        schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
            liveaction_db,
            -70000,
        )

        schedule_q_db.handling = True
        schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(schedule_q_db)

        schedule_q_db = self.scheduling_queue._get_next_execution()
        self.assertIsNone(schedule_q_db)

        self.scheduling_queue._handle_garbage_collection()

        schedule_q_db = self.scheduling_queue._get_next_execution()
        self.assertIsNotNone(schedule_q_db)
Example #28
0
    def test_populate_action_execution_id(self):
        self.reset()

        liveaction_db = self._create_liveaction_db()

        schedule_q_db = self.scheduler._create_execution_queue_item_db_from_liveaction(
            liveaction_db
        )

        # Manually unset the action_execution_id ot mock DB model of previous version.
        schedule_q_db.action_execution_id = None
        schedule_q_db = ActionExecutionSchedulingQueue.add_or_update(schedule_q_db)
        schedule_q_db = ActionExecutionSchedulingQueue.get_by_id(str(schedule_q_db.id))
        self.assertIsNone(schedule_q_db.action_execution_id)

        # Run the clean up logic.
        self.scheduling_queue._fix_missing_action_execution_id()

        # Check that the action_execution_id is populated.
        schedule_q_db = ActionExecutionSchedulingQueue.get_by_id(str(schedule_q_db.id))
        execution_db = execution_service.update_execution(liveaction_db)
        self.assertEqual(schedule_q_db.action_execution_id, str(execution_db.id))