def test_touch_conductor_deadlock(self, mock_update, mock_sleep):
mock_sleep.return_value = None
mock_update.side_effect = [db_exc.DBDeadlock(), None]
c = self._create_test_cdr()
self.dbapi.touch_conductor(c.hostname)
self.assertEqual(2, mock_update.call_count)
self.assertEqual(2, mock_sleep.call_count)
def test_multi_exception_raised_on_exceed(self):
# limit retries so this doesn't take 40 seconds
mock.patch.object(db_api._retry_db_errors, 'max_retries',
new=2).start()
e = exceptions.MultipleExceptions([ValueError(), db_exc.DBDeadlock()])
with testtools.ExpectedException(exceptions.MultipleExceptions):
self._decorated_function(db_api.MAX_RETRIES + 1, e)
def _test_allocate_multiple_ips_with_exception(self,
exc_on_deallocate=False):
mocks = self._prepare_ipam()
fail_ip = '192.168.43.15'
auto_ip = '172.23.128.94'
subnet_id = self._gen_subnet_id()
data = {'': ['172.23.128.0/17', subnet_id],
fail_ip: ['192.168.43.0/24', self._gen_subnet_id()],
'8.8.8.8': ['8.0.0.0/8', self._gen_subnet_id()]}
ips = self._convert_to_ips(data)
mocks['subnets'].allocate.side_effect = self._get_allocate_mock(
subnet_id, auto_ip=auto_ip, fail_ip=fail_ip,
exception=db_exc.DBDeadlock())
# Exception should be raised on attempt to allocate second ip.
# Revert action should be performed for the already allocated ips,
# In this test case only one ip should be deallocated
# and original error should be reraised
self.assertRaises(db_exc.DBDeadlock,
mocks['ipam']._ipam_allocate_ips,
mock.ANY,
mocks['driver'],
mocks['port'],
ips)
# get_subnet should be called only for the first two networks
get_calls = [mock.call([data[ip][1]]) for ip in ['', fail_ip]]
mocks['driver'].get_allocator.assert_has_calls(
get_calls, any_order=True)
# Allocate should be called for the first two ips only
self._validate_allocate_calls(ips[:-1], mocks)
# Deallocate should be called for the first ip only
mocks['subnet'].deallocate.assert_called_once_with(auto_ip)
def test_multi_exception_raised_on_exceed(self):
# limit retries so this doesn't take 40 seconds
retry_fixture = fixture.DBRetryErrorsFixture(max_retries=2)
retry_fixture.setUp()
e = exceptions.MultipleExceptions([ValueError(), db_exc.DBDeadlock()])
with testtools.ExpectedException(exceptions.MultipleExceptions):
self._decorated_function(db_api.MAX_RETRIES + 1, e)
retry_fixture.cleanUp()
def concurrent_increment(s):
db_api.sqla_remove(se.Session, 'before_commit',
concurrent_increment)
# slip in a concurrent update that will bump the revision
plugin = directory.get_plugin()
plugin.update_port(nctx.get_admin_context(),
port['port']['id'], new)
raise db_exc.DBDeadlock()
def test_retry_if_session_inactive_no_retry_in_active_session(self):
context = mock.Mock()
context.session.is_active = True
with testtools.ExpectedException(db_exc.DBDeadlock):
# retry decorator should have no effect in an active session
self._context_function(context, [], {1: 2},
fail_count=1,
exc_to_raise=db_exc.DBDeadlock())
def test_retry_if_session_inactive_args_not_mutated_after_retries(self):
context = mock.Mock()
context.session.is_active = False
list_arg = [1, 2, 3, 4]
dict_arg = {1: 'a', 2: 'b'}
l, d = self._context_function(context, list_arg, dict_arg,
5, db_exc.DBDeadlock())
# even though we had 5 failures the list and dict should only
# be mutated once
self.assertEqual(5, len(l))
self.assertEqual(3, len(d))
def _check_precommit(self, resource, operation):
meth_name = "%s_%s_precommit" % (operation, resource)
method = getattr(self._manager, meth_name)
fake_ctxt = mock.Mock()
fake_ctxt.current = {}
with mock.patch.object(mechanism_test.TestMechanismDriver, meth_name,
side_effect=db_exc.DBDeadlock()):
self.assertRaises(db_exc.DBDeadlock, method, fake_ctxt)
with mock.patch.object(mechanism_test.TestMechanismDriver, meth_name,
side_effect=RuntimeError()):
self.assertRaises(ml2_exc.MechanismDriverError, method, fake_ctxt)
def test_deallocate_multiple_ips_with_exception(self):
mocks = self._prepare_ipam()
fail_ip = '192.168.43.15'
data = {
fail_ip: ['192.168.43.0/24',
self._gen_subnet_id()],
'0.10.8.8': ['0.10.0.0/8', self._gen_subnet_id()]
}
ips = self._convert_to_ips(data)
mocks['subnet'].deallocate.side_effect = self._get_deallocate_mock(
fail_ip=fail_ip, exception=db_exc.DBDeadlock())
mocks['subnet'].allocate.side_effect = ValueError('Some-error')
# Validate that exception from deallocate (DBDeadlock) is not replaced
# by exception from allocate (ValueError) in rollback block,
# so original exception is not changed
self.assertRaises(db_exc.DBDeadlock,
mocks['ipam']._ipam_deallocate_ips, mock.ANY,
mocks['driver'], mock.ANY, ips)
mocks['subnets'].allocate.assert_called_once_with(mock.ANY)
def _deadlock_error(operational_error, match, engine_name, is_disconnect):
"""Filter for MySQL or Postgresql deadlock error.
NOTE(comstud): In current versions of DB backends, Deadlock violation
messages follow the structure:
mysql+mysqldb::
(OperationalError) (1213, 'Deadlock found when trying to get lock; '
'try restarting transaction') <query_str> <query_args>
mysql+mysqlconnector::
(InternalError) 1213 (40001): Deadlock found when trying to get lock;
try restarting transaction
postgresql::
(TransactionRollbackError) deadlock detected <deadlock_details>
"""
raise exception.DBDeadlock(operational_error)
def test_multi_nested_exception_contains_deadlock(self):
i = exceptions.MultipleExceptions([ValueError(), db_exc.DBDeadlock()])
e = exceptions.MultipleExceptions([ValueError(), i])
self.assertIsNone(self._decorated_function(1, e))
def random_deadlock(self, session, flush_context, instances):
if random.randrange(0, 51) > 49: # 1/50 probability
raise db_exc.DBDeadlock()
def callback_raise_retriable(*args, **kwargs):
raise db_exc.DBDeadlock()
class ErrorHandlingEngineTest(base.EngineTestCase):
def test_invalid_workflow_input(self):
# Check that in case of invalid input workflow objects aren't even
# created.
wf_text = """
version: '2.0'
wf:
input:
- param1
- param2
tasks:
task1:
action: std.noop
"""
wf_service.create_workflows(wf_text)
self.assertRaises(
exc.InputException,
self.engine.start_workflow,
'wf',
'',
{'wrong_param': 'some_value'}
)
self.assertEqual(0, len(db_api.get_workflow_executions()))
self.assertEqual(0, len(db_api.get_task_executions()))
self.assertEqual(0, len(db_api.get_action_executions()))
def test_first_task_error(self):
# Check that in case of an error in first task workflow objects are
# still persisted properly.
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.fail
on-success: task2
task2:
action: std.noop
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsNotNone(db_api.get_workflow_execution(wf_ex.id))
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(1, len(task_execs))
self._assert_single_item(task_execs, name='task1', state=states.ERROR)
def test_action_error(self):
# Check that state of all workflow objects (workflow executions,
# task executions, action executions) is properly persisted in case
# of action error.
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.fail
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(1, len(task_execs))
self._assert_single_item(task_execs, name='task1', state=states.ERROR)
def test_task_error(self):
# Check that state of all workflow objects (workflow executions,
# task executions, action executions) is properly persisted in case
# of an error at task level.
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.noop
publish:
my_var: <% invalid_yaql_function() %>
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
# Now we need to make sure that task is in ERROR state but action
# is in SUCCESS because error occurred in 'publish' clause which
# must not affect action state.
task_execs = wf_ex.task_executions
self.assertEqual(1, len(task_execs))
task_ex = self._assert_single_item(
task_execs,
name='task1',
state=states.ERROR
)
action_execs = task_ex.executions
self.assertEqual(1, len(action_execs))
self._assert_single_item(
action_execs,
name='std.noop',
state=states.SUCCESS
)
def test_task_error_with_on_handlers(self):
# Check that state of all workflow objects (workflow executions,
# task executions, action executions) is properly persisted in case
# of an error at task level and this task has on-XXX handlers.
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.noop
publish:
my_var: <% invalid_yaql_function() %>
on-success:
- task2
on-error:
- task3
task2:
description: This task must never run.
action: std.noop
task3:
action: std.noop
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
# Now we need to make sure that task is in ERROR state but action
# is in SUCCESS because error occurred in 'publish' clause which
# must not affect action state.
task_execs = wf_ex.task_executions
# NOTE: task3 must not run because on-error handler triggers
# only on error outcome of an action (or workflow) associated
# with a task.
self.assertEqual(1, len(task_execs))
task_ex = self._assert_single_item(
task_execs,
name='task1',
state=states.ERROR
)
action_execs = task_ex.executions
self.assertEqual(1, len(action_execs))
self._assert_single_item(
action_execs,
name='std.noop',
state=states.SUCCESS
)
def test_workflow_error(self):
# Check that state of all workflow objects (workflow executions,
# task executions, action executions) is properly persisted in case
# of an error at task level.
wf_text = """
version: '2.0'
wf:
output:
my_output: <% $.invalid_yaql_variable %>
tasks:
task1:
action: std.noop
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
# Now we need to make sure that task and action are in SUCCESS
# state because mistake at workflow level (output evaluation)
# must not affect them.
task_execs = wf_ex.task_executions
self.assertEqual(1, len(task_execs))
task_ex = self._assert_single_item(
task_execs,
name='task1',
state=states.SUCCESS
)
action_execs = task_ex.executions
self.assertEqual(1, len(action_execs))
self._assert_single_item(
action_execs,
name='std.noop',
state=states.SUCCESS
)
def test_action_error_with_wait_before_policy(self):
# Check that state of all workflow objects (workflow executions,
# task executions, action executions) is properly persisted in case
# of action error and task has 'wait-before' policy. It is an
# implicit test for task continuation because 'wait-before' inserts
# a delay between preparing task execution object and scheduling
# actions. If an error happens during scheduling actions (e.g.
# invalid YAQL in action parameters) then we also need to handle
# this properly, meaning that task and workflow state should go
# into ERROR state.
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.echo output=<% invalid_yaql_function() %>
wait-before: 1
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(1, len(task_execs))
task_ex = self._assert_single_item(
task_execs,
name='task1',
state=states.ERROR
)
action_execs = task_ex.executions
self.assertEqual(0, len(action_execs))
def test_action_error_with_wait_after_policy(self):
# Check that state of all workflow objects (workflow executions,
# task executions, action executions) is properly persisted in case
# of action error and task has 'wait-after' policy. It is an
# implicit test for task completion because 'wait-after' inserts
# a delay between actual task completion and logic that calculates
# next workflow commands. If an error happens while calculating
# next commands (e.g. invalid YAQL in on-XXX clauses) then we also
# need to handle this properly, meaning that task and workflow state
# should go into ERROR state.
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.noop
wait-after: 1
on-success:
- task2: <% invalid_yaql_function() %>
task2:
action: std.noop
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(1, len(task_execs))
task_ex = self._assert_single_item(
task_execs,
name='task1',
state=states.ERROR
)
action_execs = task_ex.executions
self.assertEqual(1, len(action_execs))
self._assert_single_item(
action_execs,
name='std.noop',
state=states.SUCCESS
)
def test_error_message_format_key_error(self):
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.noop
on-success:
- succeed: <% $.invalid_yaql %>
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
state_info = task_ex.state_info
self.assertIsNotNone(state_info)
self.assertLess(state_info.find('error'), state_info.find('data'))
def test_error_message_format_unknown_function(self):
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.noop
publish:
my_var: <% invalid_yaql_function() %>
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
state_info = task_ex.state_info
self.assertIsNotNone(state_info)
self.assertGreater(state_info.find('error='), 0)
self.assertLess(state_info.find('error='), state_info.find('data='))
def test_error_message_format_invalid_on_task_run(self):
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.echo output={{ _.invalid_var }}
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
state_info = task_ex.state_info
self.assertIsNotNone(state_info)
self.assertGreater(state_info.find('error='), 0)
self.assertLess(state_info.find('error='), state_info.find('wf='))
def test_error_message_format_on_task_continue(self):
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.echo output={{ _.invalid_var }}
wait-before: 1
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
state_info = task_ex.state_info
self.assertIsNotNone(state_info)
self.assertGreater(state_info.find('error='), 0)
self.assertLess(state_info.find('error='), state_info.find('wf='))
def test_error_message_format_on_action_complete(self):
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.noop
publish:
my_var: <% invalid_yaql_function() %>
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
state_info = task_ex.state_info
print(state_info)
self.assertIsNotNone(state_info)
self.assertGreater(state_info.find('error='), 0)
self.assertLess(state_info.find('error='), state_info.find('wf='))
def test_error_message_format_complete_task(self):
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.noop
wait-after: 1
on-success:
- task2: <% invalid_yaql_function() %>
task2:
action: std.noop
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
state_info = task_ex.state_info
self.assertIsNotNone(state_info)
self.assertGreater(state_info.find('error='), 0)
self.assertLess(state_info.find('error='), state_info.find('wf='))
def test_error_message_format_on_adhoc_action_error(self):
wb_text = """
version: '2.0'
name: wb
actions:
my_action:
input:
- output
output: <% invalid_yaql_function() %>
base: std.echo
base-input:
output: <% $.output %>
workflows:
wf:
tasks:
task1:
action: my_action output="test"
"""
wb_service.create_workbook_v2(wb_text)
wf_ex = self.engine.start_workflow('wb.wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
state_info = task_ex.state_info
self.assertIsNotNone(state_info)
self.assertGreater(state_info.find('error='), 0)
self.assertLess(state_info.find('error='), state_info.find('action='))
def test_publish_bad_yaql(self):
wf_text = """---
version: '2.0'
wf:
type: direct
input:
- my_dict:
- id: 1
value: 11
tasks:
task1:
action: std.noop
publish:
problem_var: <% $.my_dict.where($.value = 13).id.first() %>
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
action_ex = task_ex.action_executions[0]
self.assertEqual(states.SUCCESS, action_ex.state)
self.assertEqual(states.ERROR, task_ex.state)
self.assertIsNotNone(task_ex.state_info)
self.assertEqual(states.ERROR, wf_ex.state)
def test_publish_bad_jinja(self):
wf_text = """---
version: '2.0'
wf:
type: direct
input:
- my_dict:
- id: 1
value: 11
tasks:
task1:
action: std.noop
publish:
problem_var: '{{ (_.my_dict|some_invalid_filter).id }}'
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
action_ex = task_ex.action_executions[0]
self.assertEqual(states.SUCCESS, action_ex.state)
self.assertEqual(states.ERROR, task_ex.state)
self.assertIsNotNone(task_ex.state_info)
self.assertEqual(states.ERROR, wf_ex.state)
def test_invalid_task_input(self):
wf_text = """---
version: '2.0'
wf:
tasks:
task1:
action: std.noop
on-success: task2
task2:
action: std.echo output=<% $.non_existing_function_AAA() %>
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
tasks = wf_ex.task_executions
self.assertEqual(2, len(tasks))
self._assert_single_item(tasks, name='task1', state=states.SUCCESS)
t2 = self._assert_single_item(tasks, name='task2', state=states.ERROR)
self.assertIsNotNone(t2.state_info)
self.assertIn('Can not evaluate YAQL expression', t2.state_info)
self.assertIsNotNone(wf_ex.state_info)
self.assertIn('Can not evaluate YAQL expression', wf_ex.state_info)
def test_invalid_action_result(self):
self.register_action_class(
'test.invalid_unicode_action',
InvalidUnicodeAction
)
wf_text = """---
version: '2.0'
wf:
tasks:
task1:
action: test.invalid_unicode_action
on-success: task2
task2:
action: std.noop
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(1, len(wf_ex.task_executions))
task_ex = wf_ex.task_executions[0]
self.assertIn("UnicodeDecodeError: utf", wf_ex.state_info)
self.assertIn("UnicodeDecodeError: utf", task_ex.state_info)
@mock.patch(
'mistral.utils.expression_utils.get_yaql_context',
mock.MagicMock(
side_effect=[
db_exc.DBDeadlock(), # Emulating DB deadlock
expression_utils.get_yaql_context({}) # Successful run
]
)
)
def test_db_error_in_yaql_expression(self):
# This test just checks that the workflow completes successfully
# even if a DB deadlock occurs during YAQL expression evaluation.
# The engine in this case should should just retry the transactional
# method.
wf_text = """---
version: '2.0'
wf:
tasks:
task1:
action: std.echo output="Hello"
publish:
my_var: <% 1 + 1 %>
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(1, len(wf_ex.task_executions))
task_ex = wf_ex.task_executions[0]
self.assertDictEqual({'my_var': 2}, task_ex.published)
@mock.patch(
'mistral.utils.expression_utils.get_jinja_context',
mock.MagicMock(
side_effect=[
db_exc.DBDeadlock(), # Emulating DB deadlock
expression_utils.get_jinja_context({}) # Successful run
]
)
)
def test_db_error_in_jinja_expression(self):
# This test just checks that the workflow completes successfully
# even if a DB deadlock occurs during Jinja expression evaluation.
# The engine in this case should should just retry the transactional
# method.
wf_text = """---
version: '2.0'
wf:
tasks:
task1:
action: std.echo output="Hello"
publish:
my_var: "{{ 1 + 1 }}"
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(1, len(wf_ex.task_executions))
task_ex = wf_ex.task_executions[0]
self.assertDictEqual({'my_var': 2}, task_ex.published)
def some_method_deadlock():
raise exception.DBDeadlock('test')
def test_multi_exception_raised_on_exceed(self):
e = exceptions.MultipleExceptions([ValueError(), db_exc.DBDeadlock()])
with testtools.ExpectedException(exceptions.MultipleExceptions):
self._decorated_function(db_api.MAX_RETRIES + 1, e)
def concurrent_increment(s):
db_api.sqla_remove(se.Session, 'before_commit',
concurrent_increment)
# slip in a concurrent update that will bump the revision
self._update('ports', port['port']['id'], new)
raise db_exc.DBDeadlock()
def _raise_exceptions():
self.attempts -= 1
if self.attempts <= 0:
raise TestException("Exit")
raise db_exc.DBDeadlock("Fake Exception")