def _schedule_run_workflow(task_ex, task_spec, wf_input, index,
parent_wf_spec):
parent_wf_ex = task_ex.workflow_execution
wf_spec_name = task_spec.get_workflow_name()
wf_def = e_utils.resolve_workflow_definition(parent_wf_ex.workflow_name,
parent_wf_spec.get_name(),
wf_spec_name)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
wf_params = {'task_execution_id': task_ex.id, 'with_items_index': index}
if 'env' in parent_wf_ex.params:
wf_params['env'] = parent_wf_ex.params['env']
for k, v in list(wf_input.items()):
if k not in wf_spec.get_input():
wf_params[k] = v
del wf_input[k]
wf_ex_id, _ = wf_ex_service.create_workflow_execution(
wf_def.name, wf_input, "sub-workflow execution", wf_params, wf_spec)
scheduler.schedule_call(None,
'mistral.engine.task_handler.resume_workflow',
0,
wf_ex_id=wf_ex_id,
env=None)
def _schedule_refresh_task_state(task_ex, delay=0):
"""Schedules task preconditions check.
This method provides transactional decoupling of task preconditions
check from events that can potentially satisfy those preconditions.
It's needed in non-locking model in order to avoid 'phantom read'
phenomena when reading state of multiple tasks to see if a task that
depends on them can start. Just starting a separate transaction
without using scheduler is not safe due to concurrency window that
we'll have in this case (time between transactions) whereas scheduler
is a special component that is designed to be resistant to failures.
:param task_ex: Task execution.
:param delay: Delay.
"""
key = 'th_c_t_s_a-%s' % task_ex.id
scheduler.schedule_call(
None,
_REFRESH_TASK_STATE_PATH,
delay,
key=key,
task_ex_id=task_ex.id
)
def _schedule_run_workflow(task_ex, task_spec, wf_input, index):
parent_wf_ex = task_ex.workflow_execution
parent_wf_spec = spec_parser.get_workflow_spec(parent_wf_ex.spec)
wf_spec_name = task_spec.get_workflow_name()
wf_def = e_utils.resolve_workflow_definition(parent_wf_ex.workflow_name,
parent_wf_spec.get_name(),
wf_spec_name)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
wf_params = {'task_execution_id': task_ex.id, 'with_items_index': index}
if 'env' in parent_wf_ex.params:
wf_params['env'] = parent_wf_ex.params['env']
for k, v in wf_input.items():
if k not in wf_spec.get_input():
wf_params[k] = v
del wf_input[k]
scheduler.schedule_call(None,
'mistral.engine.task_handler.run_workflow',
0,
wf_name=wf_def.name,
wf_input=wf_input,
wf_params=wf_params)
def test_scheduler_with_factory(self, factory):
target_method_name = 'run_something'
factory.return_value = type(
'something',
(object,),
{
target_method_name:
mock.MagicMock(side_effect=self.target_method)
}
)
scheduler.schedule_call(
TARGET_METHOD_PATH,
target_method_name,
DELAY,
**{'name': 'task', 'id': '123'}
)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
call = self._assert_single_item(
calls,
target_method_name=target_method_name
)
self.assertIn('name', call['method_arguments'])
self.queue.get()
factory().run_something.assert_called_once_with(name='task', id='123')
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def test_scheduler_call_target_method_with_correct_auth(self, method):
default_context = base.get_context(default=True)
auth_context.set_ctx(default_context)
default_project_id = (
default_context._BaseContext__values['project_id'])
method_args1 = {'expected_project_id': default_project_id}
scheduler.schedule_call(None, CHECK_CONTEXT_METHOD_PATH, DELAY,
**method_args1)
second_context = base.get_context(default=False)
auth_context.set_ctx(second_context)
second_project_id = (second_context._BaseContext__values['project_id'])
method_args2 = {'expected_project_id': second_project_id}
scheduler.schedule_call(None, CHECK_CONTEXT_METHOD_PATH, DELAY,
**method_args2)
eventlet.sleep(WAIT)
method.assert_any_call(default_project_id, default_project_id)
method.assert_any_call(second_project_id, second_project_id)
self.assertNotEqual(default_project_id, second_project_id)
def test_scheduler_call_target_method_with_correct_auth(self, method):
method.side_effect = self.target_check_context_method
default_context = base.get_context(default=True)
auth_context.set_ctx(default_context)
default_project_id = (
default_context.project_id
)
scheduler.schedule_call(
None,
TARGET_METHOD_PATH,
DELAY,
**{'expected_project_id': default_project_id}
)
second_context = base.get_context(default=False)
auth_context.set_ctx(second_context)
second_project_id = (
second_context.project_id
)
scheduler.schedule_call(
None,
TARGET_METHOD_PATH,
DELAY,
**{'expected_project_id': second_project_id}
)
self.assertNotEqual(default_project_id, second_project_id)
for _ in range(2):
self.assertTrue(self.queue.get())
def test_scheduler_doesnt_handel_calls_the_failed_on_update(self):
def stop_thread_groups():
[tg.stop() for tg in self.tgs]
self.tgs = [scheduler.setup(), scheduler.setup()]
self.addCleanup(stop_thread_groups)
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
TARGET_METHOD_NAME,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
eventlet.sleep(WAIT)
# If the scheduler does handel calls that failed on update
# NotFoundException will raise.
db_api.get_delayed_call(calls[0].id)
db_api.delete_delayed_call(calls[0].id)
def test_scheduler_call_target_method_with_correct_auth(self, method):
method.side_effect = self.target_check_context_method
default_context = base.get_context(default=True)
auth_context.set_ctx(default_context)
default_project_id = (
default_context.project_id
)
scheduler.schedule_call(
None,
TARGET_METHOD_PATH,
DELAY,
**{'expected_project_id': default_project_id}
)
second_context = base.get_context(default=False)
auth_context.set_ctx(second_context)
second_project_id = (
second_context.project_id
)
scheduler.schedule_call(
None,
TARGET_METHOD_PATH,
DELAY,
**{'expected_project_id': second_project_id}
)
self.assertNotEqual(default_project_id, second_project_id)
for _ in range(2):
self.assertTrue(self.queue.get())
def _schedule_run_workflow(task_ex, task_spec, wf_input, index):
parent_wf_ex = task_ex.workflow_execution
parent_wf_spec = spec_parser.get_workflow_spec(parent_wf_ex.spec)
wf_spec_name = task_spec.get_workflow_name()
wf_def = e_utils.resolve_workflow_definition(
parent_wf_ex.workflow_name,
parent_wf_spec.get_name(),
wf_spec_name
)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
wf_params = {
'task_execution_id': task_ex.id,
'with_items_index': index
}
if 'env' in parent_wf_ex.params:
wf_params['env'] = parent_wf_ex.params['env']
for k, v in wf_input.items():
if k not in wf_spec.get_input():
wf_params[k] = v
del wf_input[k]
scheduler.schedule_call(
None,
'mistral.engine.task_handler.run_workflow',
0,
wf_name=wf_def.name,
wf_input=wf_input,
wf_params=wf_params
)
def _schedule_run_action(task_ex, task_spec, action_input, index):
wf_ex = task_ex.workflow_execution
wf_spec = spec_parser.get_workflow_spec(wf_ex.spec)
action_spec_name = task_spec.get_action_name()
action_def = action_handler.resolve_definition(
action_spec_name,
task_ex,
wf_spec
)
action_ex = action_handler.create_action_execution(
action_def, action_input, task_ex, index
)
target = expr.evaluate_recursively(
task_spec.get_target(),
utils.merge_dicts(
copy.deepcopy(action_input),
copy.copy(task_ex.in_context)
)
)
scheduler.schedule_call(
None,
'mistral.engine.action_handler.run_existing_action',
0,
action_ex_id=action_ex.id,
target=target
)
def test_scheduler_with_factory(self, factory):
target_method_name = 'run_something'
factory.return_value = type(
'something',
(object,),
{
target_method_name:
mock.MagicMock(side_effect=self.target_method)
}
)
scheduler.schedule_call(
TARGET_METHOD_PATH,
target_method_name,
DELAY,
**{'name': 'task', 'id': '123'}
)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
call = self._assert_single_item(
calls,
target_method_name=target_method_name
)
self.assertIn('name', call['method_arguments'])
self.queue.get()
factory().run_something.assert_called_once_with(name='task', id='123')
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def _schedule_send_result_to_parent_workflow(self):
scheduler.schedule_call(
None,
_SEND_RESULT_TO_PARENT_WORKFLOW_PATH,
0,
wf_ex_id=self.wf_ex.id
)
def test_scheduler_with_custom_batch_size(self):
self.scheduler.stop()
number_delayed_calls = 5
processed_calls_at_time = []
real_delete_calls_method = scheduler.Scheduler.delete_calls
@staticmethod
def delete_calls_counter(delayed_calls):
real_delete_calls_method(delayed_calls)
for _ in range(len(delayed_calls)):
self.queue.put("item")
processed_calls_at_time.append(len(delayed_calls))
scheduler.Scheduler.delete_calls = delete_calls_counter
# Create 5 delayed calls
for i in range(number_delayed_calls):
scheduler.schedule_call(
None,
TARGET_METHOD_PATH,
0,
**{'name': 'task', 'id': i}
)
# Start scheduler which process 2 calls at a time
self.scheduler = scheduler.Scheduler(0, 1, 2)
self.scheduler.start()
# Wait when all of calls will be processed
for _ in range(number_delayed_calls):
self.queue.get()
self.assertListEqual([1, 2, 2], sorted(processed_calls_at_time))
def test_scheduler_multi_instance(self, method):
def stop_thread_groups():
[tg.stop() for tg in self.tgs]
self.tgs = [scheduler.setup(), scheduler.setup()]
self.addCleanup(stop_thread_groups)
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
TARGET_METHOD_PATH,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
self._assert_single_item(calls, target_method_name=TARGET_METHOD_PATH)
eventlet.sleep(WAIT)
method.assert_called_once_with(name='task', id='321')
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
def after_task_complete(self, task_ex, task_spec):
super(WaitAfterPolicy, self).after_task_complete(task_ex, task_spec)
context_key = 'wait_after_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Skip, already processed.
return
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
state = task_ex.state
# Set task state to 'DELAYED'.
task_ex.state = states.RUNNING_DELAYED
# Schedule to change task state to RUNNING again.
scheduler.schedule_call(
_ENGINE_CLIENT_PATH,
'on_task_state_change',
self.delay,
state=state,
task_ex_id=task_ex.id,
)
def _schedule_run_action(task_ex, task_spec, action_input, index, wf_spec):
action_spec_name = task_spec.get_action_name()
action_def = action_handler.resolve_definition(
action_spec_name,
task_ex,
wf_spec
)
action_ex = action_handler.create_action_execution(
action_def, action_input, task_ex, index
)
target = expr.evaluate_recursively(
task_spec.get_target(),
utils.merge_dicts(
copy.deepcopy(action_input),
copy.deepcopy(task_ex.in_context)
)
)
scheduler.schedule_call(
None,
'mistral.engine.action_handler.run_existing_action',
0,
action_ex_id=action_ex.id,
target=target
)
def test_scheduler_with_factory(self, factory):
target_method = 'run_something'
method_args = {'name': 'task', 'id': '123'}
scheduler.schedule_call(
FACTORY_METHOD_PATH,
target_method,
DELAY,
**method_args
)
calls = db_api.get_delayed_calls_to_start(
datetime.datetime.now() + datetime.timedelta(seconds=2)
)
call = self._assert_single_item(
calls,
target_method_name=target_method
)
self.assertIn('name', call['method_arguments'])
eventlet.sleep(WAIT)
factory().run_something.assert_called_once_with(name='task', id='123')
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
def schedule(self, input_dict, target, index=0, desc='', safe_rerun=False):
assert not self.action_ex
# Assign the action execution ID here to minimize database calls.
# Otherwise, the input property of the action execution DB object needs
# to be updated with the action execution ID after the action execution
# DB object is created.
action_ex_id = utils.generate_unicode_uuid()
self._insert_action_context(action_ex_id, input_dict)
self._create_action_execution(
self._prepare_input(input_dict),
self._prepare_runtime_context(index, safe_rerun),
desc=desc,
action_ex_id=action_ex_id
)
scheduler.schedule_call(
None,
_RUN_EXISTING_ACTION_PATH,
0,
action_ex_id=self.action_ex.id,
target=target
)
def test_scheduler_multi_instance(self, method):
scheds = [scheduler.start(), scheduler.start()]
def stop_schedulers():
[scheduler.stop_scheduler(s, True) for s in scheds]
self.addCleanup(stop_schedulers)
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(None, TARGET_METHOD_PATH, DELAY, **method_args)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
self._assert_single_item(calls, target_method_name=TARGET_METHOD_PATH)
eventlet.sleep(WAIT)
method.assert_called_once_with(name='task', id='321')
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
def test_scheduler_with_factory(self, factory):
target_method = 'run_something'
method_args = {'name': 'task', 'id': '123'}
scheduler.schedule_call(
FACTORY_METHOD_PATH,
target_method,
DELAY,
**method_args
)
calls = db_api.get_delayed_calls_to_start(
datetime.datetime.now() + datetime.timedelta(seconds=2)
)
call = self._assert_single_item(
calls,
target_method_name=target_method
)
self.assertIn('name', call['method_arguments'])
eventlet.sleep(WAIT)
factory().run_something.assert_called_once_with(name='task', id='123')
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
def test_scheduler_delete_calls(self, method):
def stop_thread_groups():
[tg.stop() for tg in self.tgs]
self.tgs = [scheduler.setup(), scheduler.setup()]
self.addCleanup(stop_thread_groups)
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
TARGET_METHOD_NAME,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
self._assert_single_item(calls, target_method_name=TARGET_METHOD_NAME)
eventlet.sleep(WAIT)
self.assertRaises(exc.NotFoundException,
db_api.get_delayed_call,
calls[0].id
)
def before_task_start(self, task_ex, task_spec):
super(WaitBeforePolicy, self).before_task_start(task_ex, task_spec)
context_key = "wait_before_policy"
runtime_context = _ensure_context_has_key(task_ex.runtime_context, context_key)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get("skip"):
# Unset state 'DELAYED'.
wf_trace.info(task_ex, "Task '%s' [%s -> %s]" % (task_ex.name, states.DELAYED, states.RUNNING))
task_ex.state = states.RUNNING
return
if task_ex.state != states.IDLE:
policy_context.update({"skip": True})
_log_task_delay(task_ex, self.delay)
task_ex.state = states.DELAYED
scheduler.schedule_call(None, _RUN_EXISTING_TASK_PATH, self.delay, task_ex_id=task_ex.id)
def test_scheduler_doesnt_handle_calls_the_failed_on_update(
self,
update_delayed_call):
def update_call_failed(id, values, query_filter):
self.queue.put("item")
return None, 0
update_delayed_call.side_effect = update_call_failed
scheduler.schedule_call(
None,
TARGET_METHOD_PATH,
DELAY,
**{'name': 'task', 'id': '321'}
)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.queue.get()
eventlet.sleep(1)
update_delayed_call.assert_called_with(
id=calls[0].id,
values=mock.ANY,
query_filter=mock.ANY
)
# If the scheduler does handel calls that failed on update
# DBEntityNotFoundException will raise.
db_api.get_delayed_call(calls[0].id)
db_api.delete_delayed_call(calls[0].id)
def after_task_complete(self, task_ex, task_spec):
super(WaitAfterPolicy, self).after_task_complete(task_ex, task_spec)
context_key = 'wait_after_policy'
runtime_context = _ensure_context_has_key(task_ex.runtime_context,
context_key)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Skip, already processed.
return
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
state = task_ex.state
# Set task state to 'DELAYED'.
task_ex.state = states.DELAYED
# Schedule to change task state to RUNNING again.
scheduler.schedule_call(
_ENGINE_CLIENT_PATH,
'on_task_state_change',
self.delay,
state=state,
task_ex_id=task_ex.id,
)
def schedule_on_action_update(action_ex, delay=0):
"""Schedules task update check.
This method provides transactional decoupling of action update from
task update check. It's needed in non-locking model in order to
avoid 'phantom read' phenomena when reading state of multiple actions
to see if a task is updated. Just starting a separate transaction
without using scheduler is not safe due to concurrency window that we'll
have in this case (time between transactions) whereas scheduler is a
special component that is designed to be resistant to failures.
:param action_ex: Action execution.
:param delay: Minimum amount of time before task update check
should be made.
"""
# Optimization to avoid opening a new transaction if it's not needed.
if not action_ex.task_execution.spec.get('with-items'):
_on_action_update(action_ex)
return
key = 'th_on_a_u-%s' % action_ex.task_execution_id
scheduler.schedule_call(None,
_SCHEDULED_ON_ACTION_UPDATE_PATH,
delay,
key=key,
action_ex_id=action_ex.id,
wf_action=isinstance(action_ex,
models.WorkflowExecution))
def _schedule_send_result_to_parent_workflow(wf_ex):
scheduler.schedule_call(
None,
'mistral.engine.workflow_handler.send_result_to_parent_workflow',
0,
wf_ex_id=wf_ex.id
)
def _schedule_send_result_to_parent_workflow(self):
scheduler.schedule_call(
None,
_SEND_RESULT_TO_PARENT_WORKFLOW_PATH,
0,
wf_ex_id=self.wf_ex.id
)
def schedule(self, input_dict, target, index=0, desc=''):
assert not self.action_ex
# Assign the action execution ID here to minimize database calls.
# Otherwise, the input property of the action execution DB object needs
# to be updated with the action execution ID after the action execution
# DB object is created.
action_ex_id = utils.generate_unicode_uuid()
self._insert_action_context(action_ex_id, input_dict)
self._create_action_execution(
self._prepare_input(input_dict),
self._prepare_runtime_context(index),
desc=desc,
action_ex_id=action_ex_id
)
scheduler.schedule_call(
None,
_RUN_EXISTING_ACTION_PATH,
0,
action_ex_id=self.action_ex.id,
target=target
)
def _schedule_refresh_task_state(task_ex, delay=0):
"""Schedules task preconditions check.
This method provides transactional decoupling of task preconditions
check from events that can potentially satisfy those preconditions.
It's needed in non-locking model in order to avoid 'phantom read'
phenomena when reading state of multiple tasks to see if a task that
depends on them can start. Just starting a separate transaction
without using scheduler is not safe due to concurrency window that
we'll have in this case (time between transactions) whereas scheduler
is a special component that is designed to be resistant to failures.
:param task_ex: Task execution.
:param delay: Delay.
"""
key = 'th_c_t_s_a-%s' % task_ex.id
scheduler.schedule_call(
None,
_REFRESH_TASK_STATE_PATH,
delay,
key=key,
task_ex_id=task_ex.id
)
def schedule_on_action_complete(action_ex, delay=0):
"""Schedules task completion check.
This method provides transactional decoupling of action completion from
task completion check. It's needed in non-locking model in order to
avoid 'phantom read' phenomena when reading state of multiple actions
to see if a task is completed. Just starting a separate transaction
without using scheduler is not safe due to concurrency window that we'll
have in this case (time between transactions) whereas scheduler is a
special component that is designed to be resistant to failures.
:param action_ex: Action execution.
:param delay: Minimum amount of time before task completion check
should be made.
"""
# Optimization to avoid opening a new transaction if it's not needed.
if not action_ex.task_execution.spec.get('with-items'):
_on_action_complete(action_ex)
return
key = 'th_on_a_c-%s' % action_ex.task_execution_id
scheduler.schedule_call(
None,
_SCHEDULED_ON_ACTION_COMPLETE_PATH,
delay,
key=key,
action_ex_id=action_ex.id,
wf_action=isinstance(action_ex, models.WorkflowExecution)
)
def test_scheduler_without_factory(self, method):
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
FACTORY_METHOD_PATH,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
call = self._assert_single_item(
calls,
target_method_name=FACTORY_METHOD_PATH
)
self.assertIn('name', call['method_arguments'])
eventlet.sleep(WAIT)
method.assert_called_once_with(name='task', id='321')
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
def test_scheduler_with_custom_batch_size(self):
self.scheduler.stop()
number_delayed_calls = 5
processed_calls_at_time = []
real_delete_calls_method = scheduler.Scheduler.delete_calls
@staticmethod
def delete_calls_counter(delayed_calls):
real_delete_calls_method(delayed_calls)
for _ in range(len(delayed_calls)):
self.queue.put("item")
processed_calls_at_time.append(len(delayed_calls))
scheduler.Scheduler.delete_calls = delete_calls_counter
# Create 5 delayed calls
for i in range(number_delayed_calls):
scheduler.schedule_call(None, TARGET_METHOD_PATH, 0, **{
'name': 'task',
'id': i
})
# Start scheduler which process 2 calls at a time
self.scheduler = scheduler.Scheduler(0, 1, 2)
self.scheduler.start()
# Wait when all of calls will be processed
for _ in range(number_delayed_calls):
self.queue.get()
self.assertEqual([2, 2, 1], processed_calls_at_time)
def before_task_start(self, task_ex, task_spec):
super(WaitBeforePolicy, self).before_task_start(task_ex, task_spec)
context_key = 'wait_before_policy'
runtime_context = _ensure_context_has_key(task_ex.runtime_context,
context_key)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Unset state 'DELAYED'.
wf_trace.info(
task_ex, "Task '%s' [%s -> %s]" %
(task_ex.name, states.DELAYED, states.RUNNING))
task_ex.state = states.RUNNING
return
if task_ex.state != states.IDLE:
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
task_ex.state = states.DELAYED
scheduler.schedule_call(
None,
_RUN_EXISTING_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def test_scheduler_without_factory(self, method):
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
FACTORY_METHOD_PATH,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
call = self._assert_single_item(
calls,
target_method_name=FACTORY_METHOD_PATH
)
self.assertIn('name', call['method_arguments'])
eventlet.sleep(WAIT)
method.assert_called_once_with(name='task', id='321')
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
def test_scheduler_doesnt_handle_calls_the_failed_on_update(
self, update_delayed_call):
def update_call_failed(id, values, query_filter):
self.queue.put("item")
return None, 0
update_delayed_call.side_effect = update_call_failed
scheduler.schedule_call(None, TARGET_METHOD_PATH, DELAY, **{
'name': 'task',
'id': '321'
})
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.queue.get()
eventlet.sleep(1)
update_delayed_call.assert_called_with(id=calls[0].id,
values=mock.ANY,
query_filter=mock.ANY)
# If the scheduler does handel calls that failed on update
# DBEntityNotFoundException will raise.
db_api.get_delayed_call(calls[0].id)
db_api.delete_delayed_call(calls[0].id)
def test_scheduler_with_serializer(self, factory):
target_method_name = 'run_something'
factory.return_value = type('something', (object, ), {
target_method_name:
mock.MagicMock(side_effect=self.target_method)
})
task_result = ml_actions.Result('data', 'error')
method_args = {'name': 'task', 'id': '123', 'result': task_result}
serializers = {'result': 'mistral.workflow.utils.ResultSerializer'}
scheduler.schedule_call(TARGET_METHOD_PATH,
target_method_name,
DELAY,
serializers=serializers,
**method_args)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
call = self._assert_single_item(calls,
target_method_name=target_method_name)
self.assertIn('name', call['method_arguments'])
self.queue.get()
result = factory().run_something.call_args[1].get('result')
self.assertIsInstance(result, ml_actions.Result)
self.assertEqual('data', result.data)
self.assertEqual('error', result.error)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
context_key = "retry_task_policy"
runtime_context = _ensure_context_has_key(task_ex.runtime_context, context_key)
continue_on_evaluation = expressions.evaluate(
self._continue_on_clause, data_flow.evaluate_task_outbound_context(task_ex)
)
task_ex.runtime_context = runtime_context
state = task_ex.state
if not states.is_completed(state):
return
policy_context = runtime_context[context_key]
retry_no = 0
if "retry_no" in policy_context:
retry_no = policy_context["retry_no"]
del policy_context["retry_no"]
retries_remain = retry_no + 1 < self.count
stop_continue_flag = task_ex.state == states.SUCCESS and not self._continue_on_clause
stop_continue_flag = stop_continue_flag or (self._continue_on_clause and not continue_on_evaluation)
break_triggered = task_ex.state == states.ERROR and self.break_on
if not retries_remain or break_triggered or stop_continue_flag:
return
_log_task_delay(task_ex, self.delay)
data_flow.invalidate_task_execution_result(task_ex)
task_ex.state = states.DELAYED
policy_context["retry_no"] = retry_no + 1
runtime_context[context_key] = policy_context
scheduler.schedule_call(None, _RUN_EXISTING_TASK_PATH, self.delay, task_ex_id=task_ex.id)
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
No need to move to next iteration.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
task_ex.runtime_context = runtime_context
state = task_ex.state
if state != states.ERROR:
return
wf_trace.info(
task_ex,
"Task '%s' [%s -> ERROR]"
% (task_ex.name, task_ex.state)
)
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no + 1 < self.count
if not retries_remain or self.break_on:
return
_log_task_delay(task_ex, self.delay)
task_ex.state = states.DELAYED
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
scheduler.schedule_call(
None,
_RUN_EXISTING_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def before_task_start(self, task_ex, task_spec):
super(TimeoutPolicy, self).before_task_start(task_ex, task_spec)
scheduler.schedule_call(
None,
"mistral.engine.policies.fail_task_if_incomplete",
self.delay,
task_ex_id=task_ex.id,
timeout=self.delay,
)
wf_trace.info(task_ex, "Timeout check scheduled [task=%s, timeout(s)=%s]." % (task_ex.id, self.delay))
def schedule(self, input_dict, target, index=0, desc=''):
assert not self.action_ex
self._create_action_execution(self._prepare_input(input_dict),
self._prepare_runtime_context(index),
desc=desc)
scheduler.schedule_call(None,
_RUN_EXISTING_ACTION_PATH,
0,
action_ex_id=self.action_ex.id,
target=target)
def before_task_start(self, task_ex, task_spec):
super(TimeoutPolicy, self).before_task_start(task_ex, task_spec)
scheduler.schedule_call(
None,
'mistral.engine.policies.fail_task_if_incomplete',
self.delay,
task_ex_id=task_ex.id,
timeout=self.delay)
wf_trace.info(
task_ex, "Timeout check scheduled [task=%s, timeout(s)=%s]." %
(task_ex.id, self.delay))
def test_scheduler_delete_calls(self, method):
method.side_effect = self.target_method
scheduler.schedule_call(None, TARGET_METHOD_PATH, DELAY, **{
'name': 'task',
'id': '321'
})
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self._assert_single_item(calls, target_method_name=TARGET_METHOD_PATH)
self.queue.get()
self.assertRaises(exc.DBEntityNotFoundError, db_api.get_delayed_call,
calls[0].id)
def test_scheduler_with_serializer(self, factory):
target_method = 'run_something'
task_result = wf_utils.Result('data', 'error')
method_args = {
'name': 'task',
'id': '123',
'result': task_result
}
serializers = {
'result': 'mistral.workflow.utils.ResultSerializer'
}
delay = 1.5
scheduler.schedule_call(
FACTORY_METHOD_NAME,
target_method,
delay,
serializers=serializers,
**method_args
)
calls = db_api.get_delayed_calls_to_start(
datetime.datetime.now() + datetime.timedelta(seconds=2)
)
call = self._assert_single_item(
calls,
target_method_name=target_method
)
self.assertIn('name', call['method_arguments'])
eventlet.sleep(delay)
result = factory().run_something.call_args[1].get('result')
self.assertIsInstance(result, wf_utils.Result)
self.assertEqual('data', result.data)
self.assertEqual('error', result.error)
calls = db_api.get_delayed_calls_to_start(
datetime.datetime.now() + datetime.timedelta(seconds=1)
)
self.assertEqual(0, len(calls))
def test_scheduler_delete_calls(self, method):
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(None, FACTORY_METHOD_PATH, DELAY,
**method_args)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
self._assert_single_item(calls, target_method_name=FACTORY_METHOD_PATH)
eventlet.sleep(WAIT)
self.assertRaises(exc.DBEntityNotFoundError, db_api.get_delayed_call,
calls[0].id)
def test_scheduler_with_serializer(self, factory):
target_method_name = 'run_something'
factory.return_value = type(
'something',
(object,),
{
target_method_name:
mock.MagicMock(side_effect=self.target_method)
}
)
task_result = ml_actions.Result('data', 'error')
method_args = {
'name': 'task',
'id': '123',
'result': task_result
}
serializers = {
'result': 'mistral.workflow.utils.ResultSerializer'
}
scheduler.schedule_call(
TARGET_METHOD_PATH,
target_method_name,
DELAY,
serializers=serializers,
**method_args
)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
call = self._assert_single_item(
calls,
target_method_name=target_method_name
)
self.assertIn('name', call['method_arguments'])
self.queue.get()
result = factory().run_something.call_args[1].get('result')
self.assertIsInstance(result, ml_actions.Result)
self.assertEqual('data', result.data)
self.assertEqual('error', result.error)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def test_scheduler_doesnt_handle_calls_the_failed_on_update(self):
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(None, FACTORY_METHOD_PATH, DELAY,
**method_args)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
eventlet.sleep(WAIT)
# If the scheduler does handel calls that failed on update
# DBEntityNotFoundException will raise.
db_api.get_delayed_call(calls[0].id)
db_api.delete_delayed_call(calls[0].id)
def schedule(self, input_dict, target, index=0, desc=''):
assert not self.action_ex
self._create_action_execution(
self._prepare_input(input_dict),
self._prepare_runtime_context(index),
desc=desc
)
scheduler.schedule_call(
None,
_RUN_EXISTING_ACTION_PATH,
0,
action_ex_id=self.action_ex.id,
target=target
)
def before_task_start(self, task_ex, task_spec):
super(TimeoutPolicy, self).before_task_start(task_ex, task_spec)
# No timeout if delay is 0
if self.delay == 0:
return
scheduler.schedule_call(None,
_FAIL_IF_INCOMPLETE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
timeout=self.delay)
wf_trace.info(
task_ex, "Timeout check scheduled [task=%s, timeout(s)=%s]." %
(task_ex.id, self.delay))
def test_scheduler_with_serializer(self, factory):
target_method = 'run_something'
task_result = wf_utils.Result('data', 'error')
method_args = {
'name': 'task',
'id': '123',
'result': task_result
}
serializers = {
'result': 'mistral.workflow.utils.ResultSerializer'
}
scheduler.schedule_call(
FACTORY_METHOD_PATH,
target_method,
DELAY,
serializers=serializers,
**method_args
)
calls = db_api.get_delayed_calls_to_start(
datetime.datetime.now() + datetime.timedelta(seconds=WAIT)
)
call = self._assert_single_item(
calls,
target_method_name=target_method
)
self.assertIn('name', call['method_arguments'])
eventlet.sleep(WAIT)
result = factory().run_something.call_args[1].get('result')
self.assertIsInstance(result, wf_utils.Result)
self.assertEqual('data', result.data)
self.assertEqual('error', result.error)
calls = db_api.get_delayed_calls_to_start(
datetime.datetime.now() + datetime.timedelta(seconds=1)
)
self.assertEqual(0, len(calls))
def after_task_complete(self, task_ex, task_spec):
super(WaitAfterPolicy, self).after_task_complete(task_ex, task_spec)
# No need to postpone a task if delay is 0
if self.delay == 0:
return
context_key = 'wait_after_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Skip, already processed.
return
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
end_state = task_ex.state
end_state_info = task_ex.state_info
# TODO(rakhmerov): Policies probably need to have tasks.Task
# interface in order to manage task state safely.
# Set task state to 'RUNNING_DELAYED'.
task_ex.state = states.RUNNING_DELAYED
task_ex.state_info = (
'Suspended by wait-after policy for %s seconds' % self.delay
)
# Schedule to change task state to RUNNING again.
scheduler.schedule_call(
None,
_COMPLETE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
state=end_state,
state_info=end_state_info
)
def after_task_complete(self, task_ex, task_spec):
super(WaitAfterPolicy, self).after_task_complete(task_ex, task_spec)
# No need to postpone a task if delay is 0
if self.delay == 0:
return
context_key = 'wait_after_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Skip, already processed.
return
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
end_state = task_ex.state
end_state_info = task_ex.state_info
# TODO(rakhmerov): Policies probably need to have tasks.Task
# interface in order to manage task state safely.
# Set task state to 'RUNNING_DELAYED'.
task_ex.state = states.RUNNING_DELAYED
task_ex.state_info = (
'Suspended by wait-after policy for %s seconds' % self.delay
)
# Schedule to change task state to RUNNING again.
scheduler.schedule_call(
None,
_COMPLETE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
state=end_state,
state_info=end_state_info
)
def _schedule_noop_action(task_ex, task_spec, wf_spec):
wf_ex = task_ex.workflow_execution
action_def = action_handler.resolve_action_definition(
'std.noop', wf_ex.workflow_name, wf_spec.get_name())
action_ex = action_handler.create_action_execution(action_def, {}, task_ex)
target = expr.evaluate_recursively(task_spec.get_target(),
task_ex.in_context)
scheduler.schedule_call(
None,
'mistral.engine.action_handler.run_existing_action',
0,
action_ex_id=action_ex.id,
target=target)
def _schedule_check_and_fix_integrity(wf_ex, delay=0):
"""Schedules workflow integrity check.
:param wf_ex: Workflow execution.
:param delay: Minimum amount of time before the check should be made.
"""
if CONF.engine.execution_integrity_check_delay < 0:
# Never check integrity if it's a negative value.
return
key = _get_integrity_check_key(wf_ex)
scheduler.schedule_call(None,
_CHECK_AND_FIX_INTEGRITY_PATH,
delay,
key=key,
wf_ex_id=wf_ex.id)
def test_scheduler_without_factory(self, method):
method.side_effect = self.target_method
scheduler.schedule_call(None, TARGET_METHOD_PATH, DELAY, **{
'name': 'task',
'id': '321'
})
calls = db_api.get_delayed_calls_to_start(get_time_delay())
call = self._assert_single_item(calls,
target_method_name=TARGET_METHOD_PATH)
self.assertIn('name', call['method_arguments'])
self.queue.get()
method.assert_called_once_with(name='task', id='321')
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def schedule(self, input_dict, target, index=0, desc=''):
parent_wf_ex = self.task_ex.workflow_execution
parent_wf_spec = spec_parser.get_workflow_spec(parent_wf_ex.spec)
task_spec = spec_parser.get_task_spec(self.task_ex.spec)
wf_spec_name = task_spec.get_workflow_name()
wf_def = e_utils.resolve_workflow_definition(
parent_wf_ex.workflow_name,
parent_wf_spec.get_name(),
wf_spec_name
)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
wf_params = {
'task_execution_id': self.task_ex.id,
'index': index
}
if 'env' in parent_wf_ex.params:
wf_params['env'] = parent_wf_ex.params['env']
for k, v in list(input_dict.items()):
if k not in wf_spec.get_input():
wf_params[k] = v
del input_dict[k]
wf_ex, _ = wf_ex_service.create_workflow_execution(
wf_def.name,
input_dict,
"sub-workflow execution",
wf_params,
wf_spec
)
scheduler.schedule_call(
None,
_RESUME_WORKFLOW_PATH,
0,
wf_ex_id=wf_ex.id,
env=None
)
def _schedule_run_action(task_ex, task_spec, action_input, index):
wf_ex = task_ex.workflow_execution
wf_spec = spec_parser.get_workflow_spec(wf_ex.spec)
action_spec_name = task_spec.get_action_name()
# TODO(rakhmerov): Refactor ad-hoc actions and isolate them.
action_def = e_utils.resolve_action_definition(
wf_ex.workflow_name,
wf_spec.get_name(),
action_spec_name
)
if action_def.spec:
# Ad-hoc action.
action_spec = spec_parser.get_action_spec(action_def.spec)
base_name = action_spec.get_base()
action_def = e_utils.resolve_action_definition(
task_ex.workflow_name,
wf_spec.get_name(),
base_name
)
action_ex = _create_action_execution(
task_ex, action_def, action_input, index
)
target = expr.evaluate_recursively(
task_spec.get_target(),
utils.merge_dicts(
copy.deepcopy(action_input),
copy.copy(task_ex.in_context)
)
)
scheduler.schedule_call(
None,
'mistral.engine.task_handler.run_action',
0,
action_ex_id=action_ex.id,
target=target
)
def before_task_start(self, task_ex, task_spec):
super(WaitBeforePolicy, self).before_task_start(task_ex, task_spec)
# No need to wait for a task if delay is 0
if self.delay == 0:
return
context_key = 'wait_before_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Unset state 'RUNNING_DELAYED'.
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s]"
% (task_ex.name, states.RUNNING_DELAYED, states.RUNNING)
)
task_ex.state = states.RUNNING
return
if task_ex.state != states.IDLE:
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
task_ex.state = states.RUNNING_DELAYED
scheduler.schedule_call(
None,
_CONTINUE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def _schedule_run_workflow(task_ex, task_spec, wf_input, index,
parent_wf_spec):
parent_wf_ex = task_ex.workflow_execution
wf_spec_name = task_spec.get_workflow_name()
wf_def = e_utils.resolve_workflow_definition(
parent_wf_ex.workflow_name,
parent_wf_spec.get_name(),
wf_spec_name
)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
wf_params = {
'task_execution_id': task_ex.id,
'with_items_index': index
}
if 'env' in parent_wf_ex.params:
wf_params['env'] = parent_wf_ex.params['env']
for k, v in list(wf_input.items()):
if k not in wf_spec.get_input():
wf_params[k] = v
del wf_input[k]
wf_ex_id, _ = wf_ex_service.create_workflow_execution(
wf_def.name,
wf_input,
"sub-workflow execution",
wf_params,
wf_spec
)
scheduler.schedule_call(
None,
'mistral.engine.task_handler.resume_workflow',
0,
wf_ex_id=wf_ex_id,
env=None
)
def test_scheduler_call_target_method_with_correct_auth(self, method):
default_context = base.get_context(default=True)
auth_context.set_ctx(default_context)
default_project_id = (
default_context._BaseContext__values['project_id']
)
method_args1 = {'expected_project_id': default_project_id}
scheduler.schedule_call(
None,
CHECK_CONTEXT_METHOD_PATH,
DELAY,
**method_args1
)
second_context = base.get_context(default=False)
auth_context.set_ctx(second_context)
second_project_id = (
second_context._BaseContext__values['project_id']
)
method_args2 = {'expected_project_id': second_project_id}
scheduler.schedule_call(
None,
CHECK_CONTEXT_METHOD_PATH,
DELAY,
**method_args2
)
eventlet.sleep(WAIT)
method.assert_any_call(
default_project_id,
default_project_id
)
method.assert_any_call(
second_project_id,
second_project_id
)
self.assertNotEqual(default_project_id, second_project_id)
def test_scheduler_doesnt_handle_calls_the_failed_on_update(self):
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
FACTORY_METHOD_PATH,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
eventlet.sleep(WAIT)
# If the scheduler does handel calls that failed on update
# DBEntityNotFoundException will raise.
db_api.get_delayed_call(calls[0].id)
db_api.delete_delayed_call(calls[0].id)