def test_resume(self): # Launch the workflow execution. liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS) liveaction, execution = action_service.request(liveaction) liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING) mistral_context = liveaction.context.get('mistral', None) self.assertIsNotNone(mistral_context) self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id')) self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name')) # Pause the workflow execution. requester = cfg.CONF.system_user.user liveaction, execution = action_service.request_pause(liveaction, requester) executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'PAUSED') liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSING) # Manually update the liveaction from pausing to paused. The paused state # is usually updated by the mistral querier. action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED) liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED) # Resume the workflow execution. liveaction, execution = action_service.request_resume(liveaction, requester) executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'RUNNING') liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
def test_over_threshold(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = LiveAction.get_all() self.assertEqual(len(scheduled), policy_db.parameters['threshold']) for liveaction in scheduled: self.assertIn(liveaction.status, SCHEDULED_STATES) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def recover_delayed_executions(): coordinator = coordination.get_coordinator() dt_now = date_utils.get_datetime_utc_now() dt_delta = datetime.timedelta(seconds=cfg.CONF.scheduler.delayed_execution_recovery) dt_timeout = dt_now - dt_delta with coordinator.get_lock('st2-rescheduling-delayed-executions'): liveactions = LiveAction.query(status=action_constants.LIVEACTION_STATUS_DELAYED, start_timestamp__lte=dt_timeout, order_by=['start_timestamp']) if not liveactions: return LOG.info('There are %d liveactions that have been delayed for longer than %d seconds.', len(liveactions), cfg.CONF.scheduler.delayed_execution_recovery) # Update status to requested and publish status for each liveactions. rescheduled = 0 for instance in liveactions: try: action_service.update_status(instance, action_constants.LIVEACTION_STATUS_REQUESTED, publish=True) rescheduled += 1 except: LOG.exception('Unable to reschedule liveaction. <LiveAction.id=%s>', instance.id) LOG.info('Rescheduled %d out of %d delayed liveactions.', len(liveactions), rescheduled)
def test_over_threshold(self): policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency.attr") self.assertGreater(policy_db.parameters["threshold"], 0) self.assertIn("actionstr", policy_db.parameters["attributes"]) for i in range(0, policy_db.parameters["threshold"]): liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"}) action_service.request(liveaction) scheduled = LiveAction.get_all() self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) for liveaction in scheduled: self.assertIn(liveaction.status, SCHEDULED_STATES) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"}) liveaction, _ = action_service.request(liveaction) delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "bar"}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) # Mark one of the execution as completed. action_service.update_status(scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_retry_policy_applied_on_workflow_failure(self): wf_name = 'sequential' wf_ac_ref = TEST_PACK + '.' + wf_name wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, wf_name + '.yaml') lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name']) lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db) lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result) # Ensure there is only one execution recorded. self.assertEqual(len(lv_db_access.LiveAction.query(action=wf_ac_ref)), 1) # Identify the records for the workflow and task. wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0] t1_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))[0] t1_lv_ac_db = lv_db_access.LiveAction.query(task_execution=str(t1_ex_db.id))[0] t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))[0] # Manually set the status to fail. ac_svc.update_status(t1_lv_ac_db, ac_const.LIVEACTION_STATUS_FAILED) t1_lv_ac_db = lv_db_access.LiveAction.query(task_execution=str(t1_ex_db.id))[0] t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))[0] self.assertEqual(t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED) notifier.get_notifier().process(t1_ac_ex_db) workflows.get_engine().process(t1_ac_ex_db) # Assert the main workflow is completed. ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id)) self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED) notifier.get_notifier().process(ac_ex_db) # Ensure execution is retried. self.assertEqual(len(lv_db_access.LiveAction.query(action=wf_ac_ref)), 2)
def _apply_after(self, target): # Schedule the oldest delayed executions. filters = self._get_filters(target) filters["status"] = action_constants.LIVEACTION_STATUS_DELAYED requests = action_access.LiveAction.query(order_by=["start_timestamp"], limit=1, **filters) if requests: action_service.update_status(requests[0], action_constants.LIVEACTION_STATUS_REQUESTED, publish=True)
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. # Assert the action is delayed. delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running states. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally succeeded state. # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def _apply_after(self, target): # Schedule the oldest delayed executions. requests = action_access.LiveAction.query(action=target.action, status=action_constants.LIVEACTION_STATUS_DELAYED, order_by=['start_timestamp'], limit=1) if requests: action_service.update_status( requests[0], action_constants.LIVEACTION_STATUS_REQUESTED, publish=True)
def test_request_cancellation_uncancelable_state(self): request, execution = self._submit_request() self.assertIsNotNone(execution) self.assertEqual(execution.id, request.id) self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_REQUESTED) # Update execution status to FAILED. action_service.update_status(execution, action_constants.LIVEACTION_STATUS_FAILED, False) execution = action_db.get_liveaction_by_id(execution.id) self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_FAILED) # Request cancellation. self.assertRaises(Exception, action_service.request_cancellation, execution)
def test_request_cancellation(self): request, execution = self._submit_request() self.assertIsNotNone(execution) self.assertEqual(execution.id, request.id) self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_REQUESTED) # Update execution status to RUNNING. action_service.update_status(execution, action_constants.LIVEACTION_STATUS_RUNNING, False) execution = action_db.get_liveaction_by_id(execution.id) self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_RUNNING) # Request cancellation. execution = self._submit_cancellation(execution) self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_CANCELING)
def _apply_before(self, target): # Get the count of scheduled instances of the action. scheduled = action_access.LiveAction.count( action=target.action, status=action_constants.LIVEACTION_STATUS_SCHEDULED) # Get the count of running instances of the action. running = action_access.LiveAction.count( action=target.action, status=action_constants.LIVEACTION_STATUS_RUNNING) count = scheduled + running # Mark the execution as scheduled if threshold is not reached or delayed otherwise. if count < self.threshold: LOG.debug('There are %s instances of %s in scheduled or running status. ' 'Threshold of %s is not reached. Action execution will be scheduled.', count, target.action, self._policy_ref) status = action_constants.LIVEACTION_STATUS_SCHEDULED else: action = 'delayed' if self.policy_action == 'delay' else 'canceled' LOG.debug('There are %s instances of %s in scheduled or running status. ' 'Threshold of %s is reached. Action execution will be %s.', count, target.action, self._policy_ref, action) status = self._get_status_for_policy_action(action=self.policy_action) # Update the status in the database. Publish status for cancellation so the # appropriate runner can cancel the execution. Other statuses are not published # because they will be picked up by the worker(s) to be processed again, # leading to duplicate action executions. publish = (status == action_constants.LIVEACTION_STATUS_CANCELING) target = action_service.update_status(target, status, publish=publish) return target
def _apply_before(self, target): # Get the count of scheduled and running instances of the action. filters = self._get_filters(target) # Get the count of scheduled instances of the action. filters['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED scheduled = action_access.LiveAction.count(**filters) # Get the count of running instances of the action. filters['status'] = action_constants.LIVEACTION_STATUS_RUNNING running = action_access.LiveAction.count(**filters) count = scheduled + running # Mark the execution as scheduled if threshold is not reached or delayed otherwise. if count < self.threshold: LOG.debug('There are %s instances of %s in scheduled or running status. ' 'Threshold of %s is not reached. Action execution will be scheduled.', count, target.action, self._policy_ref) status = action_constants.LIVEACTION_STATUS_SCHEDULED else: action = 'delayed' if self.policy_action == 'delay' else 'canceled' LOG.debug('There are %s instances of %s in scheduled or running status. ' 'Threshold of %s is reached. Action execution will be %s.', count, target.action, self._policy_ref, action) status = self._get_status_for_policy_action(action=self.policy_action) # Update the status in the database but do not publish. target = action_service.update_status(target, status, publish=False) return target
def _update_to_scheduled(liveaction_db, execution_queue_item_db): liveaction_id = str(liveaction_db.id) queue_item_id = str(execution_queue_item_db.id) extra = { 'liveaction_id': liveaction_id, 'liveaction_status': liveaction_db.status, 'queue_item_id': queue_item_id } # Update liveaction status to "scheduled". LOG.info('Liveaction (%s) Status Update to Scheduled 1: %s (%s)', liveaction_id, liveaction_db.status, queue_item_id, extra=extra) if liveaction_db.status in [action_constants.LIVEACTION_STATUS_REQUESTED, action_constants.LIVEACTION_STATUS_DELAYED]: liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_SCHEDULED, publish=False) # Publish the "scheduled" status here manually. Otherwise, there could be a # race condition with the update of the action_execution_db if the execution # of the liveaction completes first. LiveAction.publish_status(liveaction_db) extra['liveaction_status'] = liveaction_db.status # Delete execution queue entry only after status is published. ActionExecutionSchedulingQueue.delete(execution_queue_item_db) LOG.info('Liveaction (%s) Status Update to Scheduled 2: %s (%s)', liveaction_id, liveaction_db.status, queue_item_id)
def _apply_before(self, target): # Get the count of scheduled instances of the action. scheduled = action_access.LiveAction.count( action=target.action, status=action_constants.LIVEACTION_STATUS_SCHEDULED) # Get the count of running instances of the action. running = action_access.LiveAction.count( action=target.action, status=action_constants.LIVEACTION_STATUS_RUNNING) count = scheduled + running # Mark the execution as scheduled if threshold is not reached or delayed otherwise. if count < self.threshold: LOG.debug('There are %s instances of %s in scheduled or running status. ' 'Threshold of %s is not reached. Action execution will be scheduled.', count, target.action, self._policy_ref) status = action_constants.LIVEACTION_STATUS_SCHEDULED else: LOG.debug('There are %s instances of %s in scheduled or running status. ' 'Threshold of %s is reached. Action execution will be delayed.', count, target.action, self._policy_ref) status = action_constants.LIVEACTION_STATUS_DELAYED # Update the status in the database but do not publish. target = action_service.update_status(target, status, publish=False) return target
def resume(self): # Restore runner and action parameters since they are not provided on resume. runner_parameters, action_parameters = param_utils.render_final_params( self.runner_type.runner_parameters, self.action.parameters, self.liveaction.parameters, self.liveaction.context ) # Assign runner parameters needed for pre-run. if runner_parameters: self.runner_parameters = runner_parameters # Restore chain holder if it is not initialized. if not self.chain_holder: self.pre_run() # Change the status of the liveaction from resuming to running. self.liveaction = action_service.update_status( self.liveaction, action_constants.LIVEACTION_STATUS_RUNNING, publish=False ) # Run the action chain. return self._run_chain(action_parameters, resuming=True)
def test_cancel_delayed_execution(self): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED) # Manually update the liveaction from requested to delayed to mock concurrency policy. action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) # Cancel is only called when liveaction is still in running state. # Otherwise, the cancellation is only a state change. self.assertFalse(runners.ActionRunner.cancel.called) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)
def _invoke_action(self, action_db, runnertype_db, params, context=None, additional_contexts=None): """ Schedule an action execution. :type action_exec_spec: :class:`ActionExecutionSpecDB` :param params: Partially rendered parameters to execute the action with. :type params: ``dict`` :rtype: :class:`LiveActionDB` on successful scheduling, None otherwise. """ action_ref = action_db.ref runnertype_db = action_utils.get_runnertype_by_name(action_db.runner_type['name']) liveaction_db = LiveActionDB(action=action_ref, context=context, parameters=params) try: liveaction_db.parameters = self.get_resolved_parameters( runnertype_db=runnertype_db, action_db=action_db, params=liveaction_db.parameters, context=liveaction_db.context, additional_contexts=additional_contexts) except param_exc.ParamException as e: # We still need to create a request, so liveaction_db is assigned an ID liveaction_db, execution_db = action_service.create_request(liveaction_db) # By this point the execution is already in the DB therefore need to mark it failed. _, e, tb = sys.exc_info() action_service.update_status( liveaction=liveaction_db, new_status=action_constants.LIVEACTION_STATUS_FAILED, result={'error': six.text_type(e), 'traceback': ''.join(traceback.format_tb(tb, 20))}) # Might be a good idea to return the actual ActionExecution rather than bubble up # the exception. raise validation_exc.ValueValidationException(six.text_type(e)) liveaction_db, execution_db = action_service.request(liveaction_db) return execution_db
def _schedule_execution(self, liveaction, user=None): # Initialize execution context if it does not exist. if not hasattr(liveaction, 'context'): liveaction.context = dict() liveaction.context['user'] = user LOG.debug('User is: %s' % liveaction.context['user']) # Retrieve other st2 context from request header. if 'st2-context' in pecan.request.headers and pecan.request.headers['st2-context']: context = jsonify.try_loads(pecan.request.headers['st2-context']) if not isinstance(context, dict): raise ValueError('Unable to convert st2-context from the headers into JSON.') liveaction.context.update(context) # Schedule the action execution. liveaction_db = LiveActionAPI.to_model(liveaction) liveaction_db, actionexecution_db = action_service.create_request(liveaction_db) action_db = action_utils.get_action_by_ref(liveaction_db.action) runnertype_db = action_utils.get_runnertype_by_name(action_db.runner_type['name']) try: liveaction_db.parameters = param_utils.render_live_params( runnertype_db.runner_parameters, action_db.parameters, liveaction_db.parameters, liveaction_db.context) except ParamException: # By this point the execution is already in the DB therefore need to mark it failed. _, e, tb = sys.exc_info() action_service.update_status( liveaction=liveaction_db, new_status=LIVEACTION_STATUS_FAILED, result={'error': str(e), 'traceback': ''.join(traceback.format_tb(tb, 20))}) # Might be a good idea to return the actual ActionExecution rather than bubble up # the execption. raise ValueValidationException(str(e)) liveaction_db = LiveAction.add_or_update(liveaction_db, publish=False) _, actionexecution_db = action_service.publish_request(liveaction_db, actionexecution_db) from_model_kwargs = self._get_from_model_kwargs_for_request(request=pecan.request) return ActionExecutionAPI.from_model(actionexecution_db, from_model_kwargs)
def test_no_retry_policy_applied_on_task_failure(self): wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'subworkflow.yaml') lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name']) lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db) lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result) # Identify the records for the main workflow. wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0] tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id)) self.assertEqual(len(tk_ex_dbs), 1) # Identify the records for the tasks. t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_dbs[0].id))[0] t1_wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(t1_ac_ex_db.id))[0] self.assertEqual(t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_RUNNING) self.assertEqual(t1_wf_ex_db.status, wf_statuses.RUNNING) # Ensure there is only one execution for the task. tk_ac_ref = TEST_PACK + '.' + 'sequential' self.assertEqual(len(lv_db_access.LiveAction.query(action=tk_ac_ref)), 1) # Fail the subtask of the subworkflow. t1_t1_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(t1_wf_ex_db.id))[0] t1_t1_lv_ac_db = lv_db_access.LiveAction.query(task_execution=str(t1_t1_ex_db.id))[0] ac_svc.update_status(t1_t1_lv_ac_db, ac_const.LIVEACTION_STATUS_FAILED) t1_t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t1_t1_ex_db.id))[0] self.assertEqual(t1_t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED) notifier.get_notifier().process(t1_t1_ac_ex_db) workflows.get_engine().process(t1_t1_ac_ex_db) # Ensure the task execution is not retried. self.assertEqual(len(lv_db_access.LiveAction.query(action=tk_ac_ref)), 1) # Process the failure of the subworkflow. t1_ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(t1_ac_ex_db.id)) self.assertEqual(t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED) workflows.get_engine().process(t1_ac_ex_db) # Assert the main workflow is completed. ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id)) self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
def process(self, request): """Schedules the LiveAction and publishes the request to the appropriate action runner(s). LiveAction in statuses other than "requested" are ignored. :param request: Action execution request. :type request: ``st2common.models.db.liveaction.LiveActionDB`` """ if request.status != action_constants.LIVEACTION_STATUS_REQUESTED: LOG.info( '%s is ignoring %s (id=%s) with "%s" status.', self.__class__.__name__, type(request), request.id, request.status, ) return try: liveaction_db = action_utils.get_liveaction_by_id(request.id) except StackStormDBObjectNotFoundError: LOG.exception("Failed to find liveaction %s in the database.", request.id) raise # Apply policies defined for the action. liveaction_db = self._apply_pre_run_policies(liveaction_db=liveaction_db) # Exit if the status of the request is no longer runnable. # The status could have be changed by one of the policies. if liveaction_db.status not in [ action_constants.LIVEACTION_STATUS_REQUESTED, action_constants.LIVEACTION_STATUS_SCHEDULED, ]: LOG.info( '%s is ignoring %s (id=%s) with "%s" status after policies are applied.', self.__class__.__name__, type(request), request.id, liveaction_db.status, ) return # Update liveaction status to "scheduled". if liveaction_db.status == action_constants.LIVEACTION_STATUS_REQUESTED: liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_SCHEDULED, publish=False ) # Publish the "scheduled" status here manually. Otherwise, there could be a # race condition with the update of the action_execution_db if the execution # of the liveaction completes first. LiveAction.publish_status(liveaction_db)
def test_put_resume_with_result(self): # Add the runner type to the list of runners that support pause and resume. action_constants.WORKFLOW_RUNNER_TYPES.append(ACTION_1['runner_type']) try: post_resp = self._do_post(LIVE_ACTION_1) self.assertEqual(post_resp.status_int, 201) execution_id = self._get_actionexecution_id(post_resp) updates = {'status': 'running'} put_resp = self._do_put(execution_id, updates) self.assertEqual(put_resp.status_int, 200) self.assertEqual(put_resp.json['status'], 'running') updates = {'status': 'pausing'} put_resp = self._do_put(execution_id, updates) self.assertEqual(put_resp.status_int, 200) self.assertEqual(put_resp.json['status'], 'pausing') self.assertIsNone(put_resp.json.get('result')) # Manually change the status to paused because only the runner pause method should # set the paused status directly to the liveaction and execution database objects. liveaction_id = self._get_liveaction_id(post_resp) liveaction = action_db_util.get_liveaction_by_id(liveaction_id) action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED) get_resp = self._do_get_one(execution_id) self.assertEqual(get_resp.status_int, 200) self.assertEqual(get_resp.json['status'], 'paused') self.assertIsNone(get_resp.json.get('result')) updates = {'status': 'resuming', 'result': {'stdout': 'foobar'}} put_resp = self._do_put(execution_id, updates, expect_errors=True) self.assertEqual(put_resp.status_int, 400) self.assertIn('result is not applicable', put_resp.json['faultstring']) finally: action_constants.WORKFLOW_RUNNER_TYPES.remove(ACTION_1['runner_type'])
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency") self.assertGreater(policy_db.parameters["threshold"], 0) for i in range(0, policy_db.parameters["threshold"]): liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo"}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the execution as completed. action_service.update_status(scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def test_cancel_delayed_execution_with_parent(self): liveaction = LiveActionDB( action='wolfpack.action-1', parameters={'actionstr': 'foo'}, context={'parent': {'execution_id': uuid.uuid4().hex}} ) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED) # Manually update the liveaction from requested to delayed to mock concurrency policy. action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Cancel execution. action_service.request_cancellation(liveaction, cfg.CONF.system_user.user) # Cancel is only called when liveaction is still in running state. # Otherwise, the cancellation is only a state change. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELING)
def process(self, request): """Schedules the LiveAction and publishes the request to the appropriate action runner(s). LiveAction in statuses other than "requested" are ignored. :param request: Action execution request. :type request: ``st2common.models.db.liveaction.LiveActionDB`` """ if request.status != action_constants.LIVEACTION_STATUS_REQUESTED: LOG.info('%s is ignoring %s (id=%s) with "%s" status.', self.__class__.__name__, type(request), request.id, request.status) return try: liveaction_db = action_utils.get_liveaction_by_id(request.id) except StackStormDBObjectNotFoundError: LOG.exception('Failed to find liveaction %s in the database.', request.id) raise # Apply policies defined for the action. liveaction_db = self._apply_pre_run_policies( liveaction_db=liveaction_db) # Exit if the status of the request is no longer runnable. # The status could have be changed by one of the policies. if liveaction_db.status not in [ action_constants.LIVEACTION_STATUS_REQUESTED, action_constants.LIVEACTION_STATUS_SCHEDULED ]: LOG.info( '%s is ignoring %s (id=%s) with "%s" status after policies are applied.', self.__class__.__name__, type(request), request.id, liveaction_db.status) return # Update liveaction status to "scheduled". if liveaction_db.status == action_constants.LIVEACTION_STATUS_REQUESTED: liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_SCHEDULED, publish=False) # Publish the "scheduled" status here manually. Otherwise, there could be a # race condition with the update of the action_execution_db if the execution # of the liveaction completes first. LiveAction.publish_status(liveaction_db)
def test_req_resume(self): # Add the runner type to the list of runners that support pause and resume. action_constants.WORKFLOW_RUNNER_TYPES.append(ACTION['runner_type']) try: req, ex = self._submit_request() self.assertIsNotNone(ex) self.assertEqual(ex.id, req.id) self.assertEqual(ex.status, action_constants.LIVEACTION_STATUS_REQUESTED) # Update ex status to RUNNING. action_service.update_status( ex, action_constants.LIVEACTION_STATUS_RUNNING, False) ex = action_db.get_liveaction_by_id(ex.id) self.assertEqual(ex.status, action_constants.LIVEACTION_STATUS_RUNNING) # Request pause. ex = self._submit_pause(ex) self.assertEqual(ex.status, action_constants.LIVEACTION_STATUS_PAUSING) # Update ex status to PAUSED. action_service.update_status( ex, action_constants.LIVEACTION_STATUS_PAUSED, False) ex = action_db.get_liveaction_by_id(ex.id) self.assertEqual(ex.status, action_constants.LIVEACTION_STATUS_PAUSED) # Request resume. ex = self._submit_resume(ex) self.assertEqual(ex.status, action_constants.LIVEACTION_STATUS_RESUMING) finally: action_constants.WORKFLOW_RUNNER_TYPES.remove( ACTION['runner_type'])
def test_resume(self): # Launch the workflow execution. liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS) liveaction, execution = action_service.request(liveaction) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING) mistral_context = liveaction.context.get('mistral', None) self.assertIsNotNone(mistral_context) self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id')) self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name')) # Pause the workflow execution. requester = cfg.CONF.system_user.user liveaction, execution = action_service.request_pause( liveaction, requester) executions.ExecutionManager.update.assert_called_with( WF1_EXEC.get('id'), 'PAUSED') liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_PAUSING) # Manually update the liveaction from pausing to paused. The paused state # is usually updated by the mistral querier. action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED) liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_PAUSED) # Resume the workflow execution. liveaction, execution = action_service.request_resume( liveaction, requester) executions.ExecutionManager.update.assert_called_with( WF1_EXEC.get('id'), 'RUNNING') liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_RUNNING)
def test_over_threshold(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES)
def recover_delayed_executions(): coordinator = coordination.get_coordinator() dt_now = date_utils.get_datetime_utc_now() dt_delta = datetime.timedelta( seconds=cfg.CONF.scheduler.delayed_execution_recovery) dt_timeout = dt_now - dt_delta with coordinator.get_lock('st2-rescheduling-delayed-executions'): liveactions = LiveAction.query( status=action_constants.LIVEACTION_STATUS_DELAYED, start_timestamp__lte=dt_timeout, order_by=['start_timestamp']) if not liveactions: return LOG.info( 'There are %d liveactions that have been delayed for longer than %d seconds.', len(liveactions), cfg.CONF.scheduler.delayed_execution_recovery) # Update status to requested and publish status for each liveactions. rescheduled = 0 for instance in liveactions: try: action_service.update_status( instance, action_constants.LIVEACTION_STATUS_REQUESTED, publish=True) rescheduled += 1 except: LOG.exception( 'Unable to reschedule liveaction. <LiveAction.id=%s>', instance.id) LOG.info('Rescheduled %d out of %d delayed liveactions.', len(liveactions), rescheduled)
def _cleanup_policy_delayed(self): """ Clean up any action execution in the deprecated policy-delayed status. Associated entries in the scheduling queue will be removed and the action execution will be moved back into requested status. """ policy_delayed_liveaction_dbs = LiveAction.query(status='policy-delayed') or [] for liveaction_db in policy_delayed_liveaction_dbs: ex_que_qry = {'liveaction_id': str(liveaction_db.id), 'handling': False} execution_queue_item_dbs = ActionExecutionSchedulingQueue.query(**ex_que_qry) or [] for execution_queue_item_db in execution_queue_item_dbs: # Mark the entry in the scheduling queue for handling. try: execution_queue_item_db.handling = True execution_queue_item_db = ActionExecutionSchedulingQueue.add_or_update( execution_queue_item_db, publish=False) except db_exc.StackStormDBObjectWriteConflictError: msg = ( '[%s] Item "%s" is currently being processed by another scheduler.' % (execution_queue_item_db.action_execution_id, str(execution_queue_item_db.id)) ) LOG.error(msg) raise Exception(msg) # Delete the entry from the scheduling queue. LOG.info( '[%s] Removing policy-delayed entry "%s" from the scheduling queue.', execution_queue_item_db.action_execution_id, str(execution_queue_item_db.id) ) ActionExecutionSchedulingQueue.delete(execution_queue_item_db) # Update the status of the liveaction and execution to requested. LOG.info( '[%s] Removing policy-delayed entry "%s" from the scheduling queue.', execution_queue_item_db.action_execution_id, str(execution_queue_item_db.id) ) liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_REQUESTED) execution_service.update_execution(liveaction_db)
def _regulate_and_schedule(self, liveaction_db, execution_queue_item_db): # Apply policies defined for the action. liveaction_db = policy_service.apply_pre_run_policies(liveaction_db) liveaction_id = str(liveaction_db.id) queue_item_id = str(execution_queue_item_db.id) extra = { 'liveaction_id': liveaction_id, 'liveaction_status': liveaction_db.status, 'queue_item_id': queue_item_id } LOG.info('Liveaction (%s) Status Pre-Run: %s (%s)', liveaction_id, liveaction_db.status, queue_item_id, extra=extra) if liveaction_db.status is action_constants.LIVEACTION_STATUS_POLICY_DELAYED: liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False) execution_queue_item_db.handling = False execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time( date.get_datetime_utc_now(), POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS) try: ActionExecutionSchedulingQueue.add_or_update( execution_queue_item_db, publish=False) except db_exc.StackStormDBObjectWriteConflictError: LOG.warning( 'Execution queue item update conflict during scheduling: %s', execution_queue_item_db.id) return if (liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES or liveaction_db.status in action_constants.LIVEACTION_CANCEL_STATES): ActionExecutionSchedulingQueue.delete(execution_queue_item_db) return self._schedule(liveaction_db, execution_queue_item_db)
def process(self, request): """ Adds execution into execution_scheduling database for scheduling :param request: Action execution request. :type request: ``st2common.models.db.liveaction.LiveActionDB`` """ if request.status != action_constants.LIVEACTION_STATUS_REQUESTED: LOG.info( '%s is ignoring %s (id=%s) with "%s" status.', self.__class__.__name__, type(request), request.id, request.status, ) return try: liveaction_db = action_utils.get_liveaction_by_id(str(request.id)) except StackStormDBObjectNotFoundError: LOG.exception("Failed to find liveaction %s in the database.", str(request.id)) raise query = { "liveaction_id": str(liveaction_db.id), } queued_requests = ActionExecutionSchedulingQueue.query(**query) if len(queued_requests) > 0: # Particular execution is already being scheduled return queued_requests[0] if liveaction_db.delay and liveaction_db.delay > 0: liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False) execution_queue_item_db = self._create_execution_queue_item_db_from_liveaction( liveaction_db, delay=liveaction_db.delay) ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False) return execution_queue_item_db
def _apply_before(self, target): # Get the count of scheduled and running instances of the action. filters = self._get_filters(target) # Get the count of scheduled instances of the action. filters["status"] = action_constants.LIVEACTION_STATUS_SCHEDULED scheduled = action_access.LiveAction.count(**filters) # Get the count of running instances of the action. filters["status"] = action_constants.LIVEACTION_STATUS_RUNNING running = action_access.LiveAction.count(**filters) count = scheduled + running # Mark the execution as scheduled if threshold is not reached or delayed otherwise. if count < self.threshold: LOG.debug( "There are %s instances of %s in scheduled or running status. " "Threshold of %s is not reached. Action execution will be scheduled.", count, target.action, self._policy_ref, ) status = action_constants.LIVEACTION_STATUS_REQUESTED else: action = "delayed" if self.policy_action == "delay" else "canceled" LOG.debug( "There are %s instances of %s in scheduled or running status. " "Threshold of %s is reached. Action execution will be %s.", count, target.action, self._policy_ref, action, ) status = self._get_status_for_policy_action( action=self.policy_action) # Update the status in the database. Publish status for cancellation so the # appropriate runner can cancel the execution. Other statuses are not published # because they will be picked up by the worker(s) to be processed again, # leading to duplicate action executions. publish = status == action_constants.LIVEACTION_STATUS_CANCELING target = action_service.update_status(target, status, publish=publish) return target
def _delay(self, liveaction_db, execution_queue_item_db): liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False) execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time( date.get_datetime_utc_now(), POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS) try: execution_queue_item_db.handling = False ActionExecutionSchedulingQueue.add_or_update( execution_queue_item_db, publish=False) except db_exc.StackStormDBObjectWriteConflictError: LOG.warning( 'Execution queue item update conflict during scheduling: %s', execution_queue_item_db.id)
def _delay(self, liveaction_db, execution_queue_item_db): liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False ) execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time( date.get_datetime_utc_now(), POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS ) try: execution_queue_item_db.handling = False ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False) except db_exc.StackStormDBObjectWriteConflictError: LOG.warning( 'Execution queue item update conflict during scheduling: %s', execution_queue_item_db.id )
def process(self, request): """ Adds execution into execution_scheduling database for scheduling :param request: Action execution request. :type request: ``st2common.models.db.liveaction.LiveActionDB`` """ if request.status != action_constants.LIVEACTION_STATUS_REQUESTED: LOG.info('%s is ignoring %s (id=%s) with "%s" status.', self.__class__.__name__, type(request), request.id, request.status) return try: liveaction_db = action_utils.get_liveaction_by_id(str(request.id)) except StackStormDBObjectNotFoundError: LOG.exception('Failed to find liveaction %s in the database.', str(request.id)) raise query = { 'liveaction_id': str(liveaction_db.id), } queued_requests = ActionExecutionSchedulingQueue.query(**query) if len(queued_requests) > 0: # Particular execution is already being scheduled return queued_requests[0] if liveaction_db.delay and liveaction_db.delay > 0: liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False ) execution_queue_item_db = self._create_execution_queue_item_db_from_liveaction( liveaction_db, delay=liveaction_db.delay ) ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False) return execution_queue_item_db
def _regulate_and_schedule(self, liveaction_db, execution_queue_item_db): # Apply policies defined for the action. liveaction_db = policy_service.apply_pre_run_policies(liveaction_db) liveaction_id = str(liveaction_db.id) queue_item_id = str(execution_queue_item_db.id) extra = { 'liveaction_id': liveaction_id, 'liveaction_status': liveaction_db.status, 'queue_item_id': queue_item_id } LOG.info('Liveaction (%s) Status Pre-Run: %s (%s)', liveaction_id, liveaction_db.status, queue_item_id, extra=extra) if liveaction_db.status is action_constants.LIVEACTION_STATUS_POLICY_DELAYED: liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False ) execution_queue_item_db.handling = False execution_queue_item_db.scheduled_start_timestamp = date.append_milliseconds_to_time( date.get_datetime_utc_now(), POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS ) try: ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False) except db_exc.StackStormDBObjectWriteConflictError: LOG.warning( 'Execution queue item update conflict during scheduling: %s', execution_queue_item_db.id ) return if (liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES or liveaction_db.status in action_constants.LIVEACTION_CANCEL_STATES): ActionExecutionSchedulingQueue.delete(execution_queue_item_db) return self._schedule(liveaction_db, execution_queue_item_db)
def resume(self): # Restore runner and action parameters since they are not provided on resume. runner_parameters, action_parameters = param_utils.render_final_params( self.runner_type.runner_parameters, self.action.parameters, self.liveaction.parameters, self.liveaction.context) # Assign runner parameters needed for pre-run. if runner_parameters: self.runner_parameters = runner_parameters # Restore chain holder if it is not initialized. if not self.chain_holder: self.pre_run() # Change the status of the liveaction from resuming to running. self.liveaction = action_service.update_status( self.liveaction, action_constants.LIVEACTION_STATUS_RUNNING, publish=False) # Run the action chain. return self._run_chain(action_parameters, resuming=True)
def _update_to_scheduled(liveaction_db, execution_queue_item_db): liveaction_id = str(liveaction_db.id) queue_item_id = str(execution_queue_item_db.id) extra = { 'liveaction_id': liveaction_id, 'liveaction_status': liveaction_db.status, 'queue_item_id': queue_item_id } # Update liveaction status to "scheduled". LOG.info('Liveaction (%s) Status Update to Scheduled 1: %s (%s)', liveaction_id, liveaction_db.status, queue_item_id, extra=extra) if liveaction_db.status in [ action_constants.LIVEACTION_STATUS_REQUESTED, action_constants.LIVEACTION_STATUS_DELAYED ]: liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_SCHEDULED, publish=False) # Publish the "scheduled" status here manually. Otherwise, there could be a # race condition with the update of the action_execution_db if the execution # of the liveaction completes first. LiveAction.publish_status(liveaction_db) extra['liveaction_status'] = liveaction_db.status # Delete execution queue entry only after status is published. ActionExecutionSchedulingQueue.delete(execution_queue_item_db) LOG.info('Liveaction (%s) Status Update to Scheduled 2: %s (%s)', liveaction_id, liveaction_db.status, queue_item_id)
def _update_to_scheduled(liveaction_db, execution_queue_item_db): action_execution_id = str(execution_queue_item_db.action_execution_id) liveaction_id = str(execution_queue_item_db.liveaction_id) queue_item_id = str(execution_queue_item_db.id) extra = {'queue_item_id': queue_item_id} # Update liveaction status to "scheduled". LOG.info( '[%s] Liveaction "%s" with status "%s" is updated to status "scheduled."', action_execution_id, liveaction_id, liveaction_db.status, extra=extra ) if liveaction_db.status in [action_constants.LIVEACTION_STATUS_REQUESTED, action_constants.LIVEACTION_STATUS_DELAYED]: liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_SCHEDULED, publish=False) # Publish the "scheduled" status here manually. Otherwise, there could be a # race condition with the update of the action_execution_db if the execution # of the liveaction completes first. LiveAction.publish_status(liveaction_db) # Delete execution queue entry only after status is published. ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
def test_include_result_to_error_log(self): username = "******" wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") wf_input = {"who": "Thanos"} lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"], parameters=wf_input) lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db) # Assert action execution is running. lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result) wf_ex_dbs = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id)) wf_ex_db = wf_ex_dbs[0] # Assert task1 is already completed. query_filters = { "workflow_execution": str(wf_ex_db.id), "task_id": "task1" } tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id))[0] tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id( tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.context.get("user"), username) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # Manually override and fail the action execution and write some result. # Action execution result can contain dotted notation so ensure this is tested. result = {"127.0.0.1": {"hostname": "foobar"}} ac_svc.update_status( tk1_lv_ac_db, ac_const.LIVEACTION_STATUS_FAILED, result=result, publish=False, ) tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id))[0] tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id( tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_FAILED) self.assertDictEqual(tk1_lv_ac_db.result, result) # Manually handle action execution completion. wf_svc.handle_action_execution_completion(tk1_ac_ex_db) # Assert task and workflow failed. tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) self.assertEqual(tk1_ex_db.status, wf_statuses.FAILED) wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) self.assertEqual(wf_ex_db.status, wf_statuses.FAILED) # Assert result is included in the error log. expected_errors = [{ "message": "Execution failed. See result for details.", "type": "error", "task_id": "task1", "result": { "127.0.0.1": { "hostname": "foobar" } }, }] self.assertListEqual(wf_ex_db.errors, expected_errors)
def tearDown(self): for liveaction in LiveAction.get_all(): action_service.update_status( liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
def test_over_threshold_delay_executions(self): # Ensure the concurrency policy is accurate. policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters['threshold']): parameters = {'actionstr': 'foo-' + str(i)} liveaction = LiveActionDB(action='wolfpack.action-1', parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo-last'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 1 # Tally requested state. # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to go into delayed state. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) # Mark one of the scheduled/running execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally requested state. # Once capacity freed up, the delayed execution is published as requested again. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to be scheduled. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def _regulate_and_schedule(self, liveaction_db, execution_queue_item_db): action_execution_id = str(execution_queue_item_db.action_execution_id) liveaction_id = str(execution_queue_item_db.liveaction_id) queue_item_id = str(execution_queue_item_db.id) extra = {"queue_item_id": queue_item_id} LOG.info( '[%s] Liveaction "%s" has status "%s" before applying policies.', action_execution_id, liveaction_id, liveaction_db.status, extra=extra, ) # Apply policies defined for the action. liveaction_db = policy_service.apply_pre_run_policies(liveaction_db) LOG.info( '[%s] Liveaction "%s" has status "%s" after applying policies.', action_execution_id, liveaction_id, liveaction_db.status, extra=extra, ) if liveaction_db.status == action_constants.LIVEACTION_STATUS_DELAYED: LOG.info( '[%s] Liveaction "%s" is delayed and scheduling queue is updated.', action_execution_id, liveaction_id, extra=extra, ) liveaction_db = action_service.update_status( liveaction_db, action_constants.LIVEACTION_STATUS_DELAYED, publish=False ) execution_queue_item_db.handling = False execution_queue_item_db.scheduled_start_timestamp = ( date.append_milliseconds_to_time( date.get_datetime_utc_now(), POLICY_DELAYED_EXECUTION_RESCHEDULE_TIME_MS, ) ) try: ActionExecutionSchedulingQueue.add_or_update( execution_queue_item_db, publish=False ) except db_exc.StackStormDBObjectWriteConflictError: LOG.warning( "[%s] Database write conflict on updating scheduling queue.", action_execution_id, extra=extra, ) return if ( liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES or liveaction_db.status in action_constants.LIVEACTION_CANCEL_STATES ): ActionExecutionSchedulingQueue.delete(execution_queue_item_db) return self._schedule(liveaction_db, execution_queue_item_db)
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into delayed state. liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True ) expected_num_pubs += 1 # Tally succeeded state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as requested again. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_over_threshold_delay_executions(self): # Ensure the concurrency policy is accurate. policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency') self.assertGreater(policy_db.parameters['threshold'], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters['threshold']): parameters = {'actionstr': 'foo-' + str(i)} liveaction = LiveActionDB(action='wolfpack.action-1', parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo-last'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to go into delayed state. liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the scheduled/running execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True ) expected_num_pubs += 1 # Tally succeeded state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as scheduled. expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. # Since states are being processed async, wait for the liveaction to be scheduled. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Check the status changes. execution = ActionExecution.get(liveaction__id=str(liveaction.id)) expected_status_changes = ['requested', 'delayed', 'requested', 'scheduled', 'running'] actual_status_changes = [entry['status'] for entry in execution.log] self.assertListEqual(actual_status_changes, expected_status_changes)
def test_resume_missing_subworkflow_action(self): requester = cfg.CONF.system_user.user liveaction1 = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS) liveaction1, execution1 = action_service.request(liveaction1) liveaction1 = self._wait_on_status(liveaction1, action_constants.LIVEACTION_STATUS_RUNNING) liveaction2 = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS) liveaction2, execution2 = action_service.request(liveaction2) liveaction2 = self._wait_on_status(liveaction2, action_constants.LIVEACTION_STATUS_RUNNING) # Mock the children of the parent execution to make this # test case has subworkflow execution. with mock.patch.object( ActionExecutionDB, 'children', new_callable=mock.PropertyMock) as action_ex_children_mock: action_ex_children_mock.return_value = [execution2.id] mistral_context = liveaction1.context.get('mistral', None) self.assertIsNotNone(mistral_context) self.assertEqual(mistral_context['execution_id'], WF2_EXEC.get('id')) self.assertEqual(mistral_context['workflow_name'], WF2_EXEC.get('workflow_name')) # Pause the parent liveaction and check that the request is cascaded down. liveaction1, execution1 = action_service.request_pause(liveaction1, requester) self.assertTrue(executions.ExecutionManager.update.called) self.assertEqual(executions.ExecutionManager.update.call_count, 2) calls = [ mock.call(WF2_EXEC.get('id'), 'PAUSED'), mock.call(WF1_EXEC.get('id'), 'PAUSED') ] executions.ExecutionManager.update.assert_has_calls(calls, any_order=False) liveaction1 = LiveAction.get_by_id(str(liveaction1.id)) self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_PAUSING) liveaction2 = LiveAction.get_by_id(str(liveaction2.id)) self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_PAUSING) # Manually set the liveaction status to PAUSED. action_service.update_status(liveaction2, action_constants.LIVEACTION_STATUS_PAUSED) action_service.update_status(liveaction1, action_constants.LIVEACTION_STATUS_PAUSED) liveaction1 = LiveAction.get_by_id(str(liveaction1.id)) self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_PAUSED) liveaction2 = LiveAction.get_by_id(str(liveaction2.id)) self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_PAUSED) # Mock the children of the parent execution to make this # test case has subworkflow execution. with mock.patch.object( ActionExecutionDB, 'children', new_callable=mock.PropertyMock) as action_ex_children_mock: action_ex_children_mock.return_value = [uuid.uuid4().hex] # Resume the parent liveaction and check that the request is cascaded down. liveaction1, execution1 = action_service.request_resume(liveaction1, requester) # Includes the previous calls. self.assertTrue(executions.ExecutionManager.update.called) self.assertEqual(executions.ExecutionManager.update.call_count, 3) calls = [ mock.call(WF2_EXEC.get('id'), 'PAUSED'), mock.call(WF1_EXEC.get('id'), 'PAUSED'), mock.call(WF2_EXEC.get('id'), 'RUNNING'), ] executions.ExecutionManager.update.assert_has_calls(calls, any_order=False) # The workflow execution will fail because the liveaction for the subworkflow # should not be missing and we do not know what state it is in. liveaction1 = LiveAction.get_by_id(str(liveaction1.id)) self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_FAILED) self.assertIn('not a valid ObjectId', liveaction1.result.get('error', ''))
def test_over_threshold_delay_executions(self): # Ensure the concurrency policy is accurate. policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency") self.assertGreater(policy_db.parameters["threshold"], 0) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters["threshold"]): parameters = {"actionstr": "foo-" + str(i)} liveaction = LiveActionDB(action="wolfpack.action-1", parameters=parameters) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "foo-last"}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed async, wait for the liveaction to go into delayed state. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the scheduled/running execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally succeeded state. self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as scheduled. expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. # Since states are being processed async, wait for the liveaction to be scheduled. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Check the status changes. execution = ActionExecution.get(liveaction__id=str(liveaction.id)) expected_status_changes = [ "requested", "delayed", "requested", "scheduled", "running", ] actual_status_changes = [entry["status"] for entry in execution.log] self.assertListEqual(actual_status_changes, expected_status_changes)
def put(self, id, liveaction_api, requester_user, show_secrets=False): """ Updates a single execution. Handles requests: PUT /executions/<id> """ if not requester_user: requester_user = UserDB(cfg.CONF.system_user.user) from_model_kwargs = { 'mask_secrets': self._get_mask_secrets(requester_user, show_secrets=show_secrets) } execution_api = self._get_one_by_id( id=id, requester_user=requester_user, from_model_kwargs=from_model_kwargs, permission_type=PermissionType.EXECUTION_STOP) if not execution_api: abort(http_client.NOT_FOUND, 'Execution with id %s not found.' % id) liveaction_id = execution_api.liveaction['id'] if not liveaction_id: abort( http_client.INTERNAL_SERVER_ERROR, 'Execution object missing link to liveaction %s.' % liveaction_id) try: liveaction_db = LiveAction.get_by_id(liveaction_id) except: abort( http_client.INTERNAL_SERVER_ERROR, 'Execution object missing link to liveaction %s.' % liveaction_id) if liveaction_db.status in LIVEACTION_COMPLETED_STATES: abort(http_client.BAD_REQUEST, 'Execution is already in completed state.') try: liveaction_db = action_service.update_status(liveaction_db, liveaction_api.status, result=getattr( liveaction_api, 'result', None)) except Exception as e: LOG.exception('Failed updating liveaction %s. %s', liveaction_db.id, str(e)) abort(http_client.INTERNAL_SERVER_ERROR, 'Failed updating execution.') execution_api = self._get_one_by_id( id=id, requester_user=requester_user, from_model_kwargs=from_model_kwargs, permission_type=PermissionType.EXECUTION_STOP) return execution_api
def update_status(liveaction_api, liveaction_db): status = liveaction_api.status result = getattr(liveaction_api, 'result', None) liveaction_db = action_service.update_status(liveaction_db, status, result) actionexecution_db = ActionExecution.get(liveaction__id=str(liveaction_db.id)) return (liveaction_db, actionexecution_db)
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr') self.assertGreater(policy_db.parameters['threshold'], 0) self.assertIn('actionstr', policy_db.parameters['attributes']) for i in range(0, policy_db.parameters['threshold']): liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) action_service.request(liveaction) # Since states are being processed asynchronously, wait for the # liveactions to go into scheduled states. for i in range(0, 100): eventlet.sleep(1) scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] if len(scheduled) == policy_db.parameters['threshold']: break scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES] self.assertEqual(len(scheduled), policy_db.parameters['threshold']) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'}) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. # Since states are being processed asynchronously, wait for the # liveaction to go into delayed state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status == action_constants.LIVEACTION_STATUS_DELAYED: break # Assert the action is delayed. delayed = LiveAction.get_by_id(str(liveaction.id)) self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'}) liveaction, _ = action_service.request(liveaction) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running states. # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status in SCHEDULED_STATES: break liveaction = LiveAction.get_by_id(str(liveaction.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True) expected_num_pubs += 1 # Tally succeeded state. # Once capacity freed up, the delayed execution is published as requested again. expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. for i in range(0, 100): eventlet.sleep(1) liveaction = LiveAction.get_by_id(str(liveaction.id)) if liveaction.status in SCHEDULED_STATES: break # Execution is expected to be rescheduled. liveaction = LiveAction.get_by_id(str(delayed.id)) self.assertIn(liveaction.status, SCHEDULED_STATES) self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def test_over_threshold_delay_executions(self): policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency.attr") self.assertGreater(policy_db.parameters["threshold"], 0) self.assertIn("actionstr", policy_db.parameters["attributes"]) # Launch action executions until the expected threshold is reached. for i in range(0, policy_db.parameters["threshold"]): liveaction = LiveActionDB( action="wolfpack.action-1", parameters={"actionstr": "foo"} ) action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Check the number of action executions in scheduled state. scheduled = [ item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES ] self.assertEqual(len(scheduled), policy_db.parameters["threshold"]) # Assert the correct number of published states and action executions. This is to avoid # duplicate executions caused by accidental publishing of state in the concurrency policies. # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running']) expected_num_exec = len(scheduled) expected_num_pubs = expected_num_exec * 3 self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be delayed since concurrency threshold is reached. liveaction = LiveActionDB( action="wolfpack.action-1", parameters={"actionstr": "foo"} ) liveaction, _ = action_service.request(liveaction) expected_num_pubs += 1 # Tally requested state. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into delayed state. liveaction = self._wait_on_status( liveaction, action_constants.LIVEACTION_STATUS_DELAYED ) expected_num_exec += 0 # This request will not be scheduled for execution. expected_num_pubs += 0 # The delayed status change should not be published. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Execution is expected to be scheduled since concurrency threshold is not reached. # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not. liveaction = LiveActionDB( action="wolfpack.action-1", parameters={"actionstr": "bar"} ) liveaction, _ = action_service.request(liveaction) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Since states are being processed asynchronously, wait for the # liveaction to go into scheduled state. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) expected_num_exec += 1 # This request is expected to be executed. expected_num_pubs += 3 # Tally requested, scheduled, and running state. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count) # Mark one of the execution as completed. action_service.update_status( scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True ) expected_num_pubs += 1 # Tally succeeded state. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) # Run the scheduler to schedule action executions. self._process_scheduling_queue() # Once capacity freed up, the delayed execution is published as requested again. liveaction = self._wait_on_statuses(liveaction, SCHEDULED_STATES) expected_num_exec += 1 # The delayed request is expected to be executed. expected_num_pubs += 2 # Tally scheduled and running state. self.assertEqual( expected_num_pubs, LiveActionPublisher.publish_state.call_count ) self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)
def _schedule_execution(self, liveaction, requester_user, user=None, context_string=None, show_secrets=False): # Initialize execution context if it does not exist. if not hasattr(liveaction, 'context'): liveaction.context = dict() liveaction.context['user'] = user LOG.debug('User is: %s' % liveaction.context['user']) # Retrieve other st2 context from request header. if context_string: context = try_loads(context_string) if not isinstance(context, dict): raise ValueError( 'Unable to convert st2-context from the headers into JSON.' ) liveaction.context.update(context) # Schedule the action execution. liveaction_db = LiveActionAPI.to_model(liveaction) liveaction_db, actionexecution_db = action_service.create_request( liveaction_db) action_db = action_utils.get_action_by_ref(liveaction_db.action) runnertype_db = action_utils.get_runnertype_by_name( action_db.runner_type['name']) try: liveaction_db.parameters = param_utils.render_live_params( runnertype_db.runner_parameters, action_db.parameters, liveaction_db.parameters, liveaction_db.context) except ParamException: # By this point the execution is already in the DB therefore need to mark it failed. _, e, tb = sys.exc_info() action_service.update_status(liveaction=liveaction_db, new_status=LIVEACTION_STATUS_FAILED, result={ 'error': str(e), 'traceback': ''.join( traceback.format_tb(tb, 20)) }) # Might be a good idea to return the actual ActionExecution rather than bubble up # the execption. raise ValueValidationException(str(e)) liveaction_db = LiveAction.add_or_update(liveaction_db, publish=False) _, actionexecution_db = action_service.publish_request( liveaction_db, actionexecution_db) mask_secrets = self._get_mask_secrets(requester_user, show_secrets=show_secrets) execution_api = ActionExecutionAPI.from_model( actionexecution_db, mask_secrets=mask_secrets) return Response(json=execution_api, status=http_client.CREATED)
def test_resume_missing_subworkflow_action(self): requester = cfg.CONF.system_user.user liveaction1 = LiveActionDB(action=WF2_NAME, parameters=ACTION_PARAMS) liveaction1, execution1 = action_service.request(liveaction1) liveaction1 = LiveAction.get_by_id(str(liveaction1.id)) self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_RUNNING) liveaction2 = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS) liveaction2, execution2 = action_service.request(liveaction2) liveaction2 = LiveAction.get_by_id(str(liveaction2.id)) self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_RUNNING) # Mock the children of the parent execution to make this # test case has subworkflow execution. with mock.patch.object( ActionExecutionDB, 'children', new_callable=mock.PropertyMock) as action_ex_children_mock: action_ex_children_mock.return_value = [execution2.id] mistral_context = liveaction1.context.get('mistral', None) self.assertIsNotNone(mistral_context) self.assertEqual(mistral_context['execution_id'], WF2_EXEC.get('id')) self.assertEqual(mistral_context['workflow_name'], WF2_EXEC.get('workflow_name')) # Pause the parent liveaction and check that the request is cascaded down. liveaction1, execution1 = action_service.request_pause(liveaction1, requester) self.assertTrue(executions.ExecutionManager.update.called) self.assertEqual(executions.ExecutionManager.update.call_count, 2) calls = [ mock.call(WF2_EXEC.get('id'), 'PAUSED'), mock.call(WF1_EXEC.get('id'), 'PAUSED') ] executions.ExecutionManager.update.assert_has_calls(calls, any_order=False) liveaction1 = LiveAction.get_by_id(str(liveaction1.id)) self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_PAUSING) liveaction2 = LiveAction.get_by_id(str(liveaction2.id)) self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_PAUSING) # Manually set the liveaction status to PAUSED. action_service.update_status(liveaction2, action_constants.LIVEACTION_STATUS_PAUSED) action_service.update_status(liveaction1, action_constants.LIVEACTION_STATUS_PAUSED) liveaction1 = LiveAction.get_by_id(str(liveaction1.id)) self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_PAUSED) liveaction2 = LiveAction.get_by_id(str(liveaction2.id)) self.assertEqual(liveaction2.status, action_constants.LIVEACTION_STATUS_PAUSED) # Mock the children of the parent execution to make this # test case has subworkflow execution. with mock.patch.object( ActionExecutionDB, 'children', new_callable=mock.PropertyMock) as action_ex_children_mock: action_ex_children_mock.return_value = [uuid.uuid4().hex] # Resume the parent liveaction and check that the request is cascaded down. liveaction1, execution1 = action_service.request_resume(liveaction1, requester) # Includes the previous calls. self.assertTrue(executions.ExecutionManager.update.called) self.assertEqual(executions.ExecutionManager.update.call_count, 3) calls = [ mock.call(WF2_EXEC.get('id'), 'PAUSED'), mock.call(WF1_EXEC.get('id'), 'PAUSED'), mock.call(WF2_EXEC.get('id'), 'RUNNING'), ] executions.ExecutionManager.update.assert_has_calls(calls, any_order=False) # The workflow execution will fail because the liveaction for the subworkflow # should not be missing and we do not know what state it is in. liveaction1 = LiveAction.get_by_id(str(liveaction1.id)) self.assertEqual(liveaction1.status, action_constants.LIVEACTION_STATUS_FAILED) self.assertIn('not a valid ObjectId', liveaction1.result.get('error', ''))
def tearDown(self): # Ensure all liveactions are canceled at end of each test. for liveaction in LiveAction.get_all(): action_service.update_status( liveaction, action_constants.LIVEACTION_STATUS_CANCELED)
def _schedule_execution(self, liveaction, requester_user, user=None, context_string=None, show_secrets=False, pack=None): # Initialize execution context if it does not exist. if not hasattr(liveaction, 'context'): liveaction.context = dict() liveaction.context['user'] = user liveaction.context['pack'] = pack LOG.debug('User is: %s' % liveaction.context['user']) # Retrieve other st2 context from request header. if context_string: context = try_loads(context_string) if not isinstance(context, dict): raise ValueError( 'Unable to convert st2-context from the headers into JSON.' ) liveaction.context.update(context) # Include RBAC context (if RBAC is available and enabled) if cfg.CONF.rbac.enable: user_db = UserDB(name=user) role_dbs = rbac_service.get_roles_for_user(user_db=user_db, include_remote=True) roles = [role_db.name for role_db in role_dbs] liveaction.context['rbac'] = {'user': user, 'roles': roles} # Schedule the action execution. liveaction_db = LiveActionAPI.to_model(liveaction) action_db = action_utils.get_action_by_ref(liveaction_db.action) runnertype_db = action_utils.get_runnertype_by_name( action_db.runner_type['name']) try: liveaction_db.parameters = param_utils.render_live_params( runnertype_db.runner_parameters, action_db.parameters, liveaction_db.parameters, liveaction_db.context) except param_exc.ParamException: # We still need to create a request, so liveaction_db is assigned an ID liveaction_db, actionexecution_db = action_service.create_request( liveaction_db) # By this point the execution is already in the DB therefore need to mark it failed. _, e, tb = sys.exc_info() action_service.update_status( liveaction=liveaction_db, new_status=action_constants.LIVEACTION_STATUS_FAILED, result={ 'error': str(e), 'traceback': ''.join(traceback.format_tb(tb, 20)) }) # Might be a good idea to return the actual ActionExecution rather than bubble up # the exception. raise validation_exc.ValueValidationException(str(e)) # The request should be created after the above call to render_live_params # so any templates in live parameters have a chance to render. liveaction_db, actionexecution_db = action_service.create_request( liveaction_db) liveaction_db = LiveAction.add_or_update(liveaction_db, publish=False) _, actionexecution_db = action_service.publish_request( liveaction_db, actionexecution_db) mask_secrets = self._get_mask_secrets(requester_user, show_secrets=show_secrets) execution_api = ActionExecutionAPI.from_model( actionexecution_db, mask_secrets=mask_secrets) return Response(json=execution_api, status=http_client.CREATED)
def put(self, id, liveaction_api, requester_user, show_secrets=False): """ Updates a single execution. Handles requests: PUT /executions/<id> """ if not requester_user: requester_user = UserDB(cfg.CONF.system_user.user) from_model_kwargs = { 'mask_secrets': self._get_mask_secrets(requester_user, show_secrets=show_secrets) } execution_api = self._get_one_by_id( id=id, requester_user=requester_user, from_model_kwargs=from_model_kwargs, permission_type=PermissionType.EXECUTION_STOP) if not execution_api: abort(http_client.NOT_FOUND, 'Execution with id %s not found.' % id) liveaction_id = execution_api.liveaction['id'] if not liveaction_id: abort( http_client.INTERNAL_SERVER_ERROR, 'Execution object missing link to liveaction %s.' % liveaction_id) try: liveaction_db = LiveAction.get_by_id(liveaction_id) except: abort( http_client.INTERNAL_SERVER_ERROR, 'Execution object missing link to liveaction %s.' % liveaction_id) if liveaction_db.status in action_constants.LIVEACTION_COMPLETED_STATES: abort(http_client.BAD_REQUEST, 'Execution is already in completed state.') if (getattr(liveaction_api, 'result', None) is not None and liveaction_api.status in [ action_constants.LIVEACTION_STATUS_PAUSING, action_constants.LIVEACTION_STATUS_PAUSED, action_constants.LIVEACTION_STATUS_RESUMING ]): abort( http_client.BAD_REQUEST, 'The result is not applicable for pausing and resuming execution.' ) try: if (liveaction_api.status == action_constants.LIVEACTION_STATUS_PAUSING or liveaction_api.status == action_constants.LIVEACTION_STATUS_PAUSED): liveaction_db, actionexecution_db = action_service.request_pause( liveaction_db, requester_user.name or cfg.CONF.system_user.user) elif liveaction_api.status == action_constants.LIVEACTION_STATUS_RESUMING: liveaction_db, actionexecution_db = action_service.request_resume( liveaction_db, requester_user.name or cfg.CONF.system_user.user) else: liveaction_db = action_service.update_status( liveaction_db, liveaction_api.status, result=getattr(liveaction_api, 'result', None)) actionexecution_db = ActionExecution.get( liveaction__id=str(liveaction_db.id)) except runner_exc.InvalidActionRunnerOperationError as e: LOG.exception('Failed updating liveaction %s. %s', liveaction_db.id, str(e)) abort(http_client.BAD_REQUEST, 'Failed updating execution. %s' % str(e)) except runner_exc.UnexpectedActionExecutionStatusError as e: LOG.exception('Failed updating liveaction %s. %s', liveaction_db.id, str(e)) abort(http_client.BAD_REQUEST, 'Failed updating execution. %s' % str(e)) except Exception as e: LOG.exception('Failed updating liveaction %s. %s', liveaction_db.id, str(e)) abort(http_client.INTERNAL_SERVER_ERROR, 'Failed updating execution due to unexpected error.') mask_secrets = self._get_mask_secrets(requester_user, show_secrets=show_secrets) execution_api = ActionExecutionAPI.from_model( actionexecution_db, mask_secrets=mask_secrets) return execution_api