def testTaskRuntimeExceeded(self):
"""Task task failing from timeout."""
task_inst = test_utils.new_mock_task(task_class=test_utils.FailTask)
sample_values = [
(i, test_utils.FailTask._get_delay_sec_for_retry(i)) for i in
range(task_inst.max_retries)]
payload = {
'args': [],
'kwargs': {},
'app_data': {}}
for retry, delay_sec in sample_values:
with mock.patch(
'kale.publisher.Publisher.publish') as publish_func:
message = test_utils.MockMessage(task_inst, retry_num=retry)
retried = test_utils.FailTask.handle_failure(
message, exceptions.TaskException('Exception'))
self.assertTrue(retried)
publish_func.assert_called_once_with(
test_utils.FailTask, message.task_id, payload,
current_retry_num=(retry + 1), delay_sec=delay_sec)
retry = retry + 1
with mock.patch(
'kale.task.Task._report_permanent_failure') as fail_func:
exc = exceptions.TaskException('Exception')
message = test_utils.MockMessage(task_inst, retry_num=retry)
retried = test_utils.FailTask.handle_failure(message, exc)
self.assertFalse(retried)
fail_func.assert_called_once_with(
message, exc, task.PERMANENT_FAILURE_RETRIES_EXCEEDED, False)
def testTaskRetryDelayWithFailure(self):
"""Task task retrying with delay with failure"""
task_inst = test_utils.new_mock_task(task_class=test_utils.FailTask)
sample_values = [(i, test_utils.FailTask._get_delay_sec_for_retry(i))
for i in range(task_inst.max_retries)]
payload = {'args': [], 'kwargs': {}, 'app_data': {}}
for failure, delay_sec in sample_values:
with mock.patch(
'kale.publisher.Publisher.publish') as publish_func:
message = test_utils.MockMessage(task_inst,
failure_num=failure,
retry_num=failure)
retried = test_utils.FailTask.handle_failure(
message,
exceptions.TaskException('Exception'),
increment_failure_num=True)
self.assertTrue(retried)
publish_func.assert_called_once_with(
test_utils.FailTask,
message.task_id,
payload,
current_failure_num=(failure + 1),
current_retry_num=(failure + 1),
delay_sec=delay_sec)
def testTaskRuntimeExceeded(self):
"""Task task failing from timeout."""
task_inst = test_utils.new_mock_task(task_class=test_utils.FailTask)
sample_values = [(i, test_utils.FailTask._get_delay_sec_for_retry(i))
for i in range(task_inst.max_retries)]
payload = {'args': [], 'kwargs': {}, 'app_data': {}}
for retry, delay_sec in sample_values:
with mock.patch(
'kale.publisher.Publisher.publish') as publish_func:
message = test_utils.MockMessage(task_inst, retry_num=retry)
retried = test_utils.FailTask.handle_failure(
message, exceptions.TaskException('Exception'))
self.assertTrue(retried)
publish_func.assert_called_once_with(test_utils.FailTask,
message.task_id,
payload,
current_retry_num=(retry +
1),
delay_sec=delay_sec)
retry = retry + 1
with mock.patch(
'kale.task.Task._report_permanent_failure') as fail_func:
exc = exceptions.TaskException('Exception')
message = test_utils.MockMessage(task_inst, retry_num=retry)
retried = test_utils.FailTask.handle_failure(message, exc)
self.assertFalse(retried)
fail_func.assert_called_once_with(
message, exc, task.PERMANENT_FAILURE_RETRIES_EXCEEDED, False)
def testRunWorkerFailTask(self):
"""Test running a task."""
setup_env = self._create_patch(
'kale.task.Task._setup_task_environment')
pre_run = self._create_patch('kale.task.Task._pre_run')
post_run = self._create_patch('kale.task.Task._post_run')
clean_env = self._create_patch(
'kale.task.Task._clean_task_environment')
settings.RUN_TASK_AS_CHILD = False
task_inst = test_utils.new_mock_task(task_class=test_utils.FailTask)
task_inst._start_time = 1
task_args = [1, 'a']
with self.assertRaises(exceptions.TaskException) as exc_ctxt_mngr:
task_inst.run(*task_args)
setup_env.assert_called_once_with()
pre_run.assert_called_once_with(*task_args)
assert not post_run.called, '_post_run should not have been called.'
clean_env.assert_called_once_with(task_id='fail_task',
task_name='kale.test_utils.FailTask',
exc=exc_ctxt_mngr.exception)
self.assertTrue(task_inst._end_time > 0)
self.assertTrue(task_inst._task_latency_sec > 0)
def testExceptionStacktraceChildProcess(self):
"""Test that the exception stacktrace is perserved when running as a child process."""
settings.RUN_TASK_AS_CHILD = True
task_inst = test_utils.new_mock_task(
task_class=test_utils.MultiFunctionTask)
task_inst._start_time = 1
expected_stack_trace_lines = [
'task_inst.run()', 'self._run_task_as_child(*args, **kwargs)',
'exc.reraise()', 'utils.raise_(*self.exc_info)',
'task_inst.run_task(*args, **kwargs)', 'self._a()', 'self._b()',
'self._c()', "raise exceptions.TaskException('Task failed.')"
]
# Run in try/except instead of assertRaises to use sys.exc_info.
try:
task_inst.run()
self.fail('Should have raised TaskException')
except exceptions.TaskException:
stack_trace = ''.join(
traceback.format_exception(*sys.exc_info())).split('\n')
# Remove all lines that aren't the actual function calls.
# Note: exc.with_traceback is the function call added by future.utils.raise_ if py3.
stack_trace_lines = [
line.strip() for line in stack_trace
if line and 'File' not in line and 'Traceback' not in line
and 'TaskException: ' not in line
and 'exc.with_traceback' not in line
]
self.assertEqual(stack_trace_lines, expected_stack_trace_lines)
self.assertTrue(task_inst._end_time > 0)
self.assertTrue(task_inst._task_latency_sec > 0)
def testBlacklistedTaskFails(self):
"""Test that a blacklisted task raises an exception."""
setup_env = self._create_patch(
'kale.task.Task._setup_task_environment')
pre_run = self._create_patch('kale.task.Task._pre_run')
run_task = self._create_patch('kale.task.Task.run_task')
clean_env = self._create_patch(
'kale.task.Task._clean_task_environment')
check_blacklist = self._create_patch('kale.task.Task._check_blacklist')
raised_exc = exceptions.BlacklistedException()
check_blacklist.side_effect = raised_exc
task_inst = test_utils.new_mock_task(task_class=test_utils.MockTask)
task_inst._start_time = 1
task_args = [1, 'a']
with self.assertRaises(exceptions.BlacklistedException):
task_inst.run(*task_args)
setup_env.assert_called_once_with()
pre_run.assert_called_once_with(*task_args)
self.assertFalse(run_task.called)
clean_env.assert_called_once_with(
task_id='mock_task', task_name='kale.test_utils.MockTask',
exc=raised_exc)
def testBlacklistedTaskFails(self):
"""Test that a blacklisted task raises an exception."""
setup_env = self._create_patch(
'kale.task.Task._setup_task_environment')
pre_run = self._create_patch('kale.task.Task._pre_run')
run_task = self._create_patch('kale.task.Task.run_task')
clean_env = self._create_patch(
'kale.task.Task._clean_task_environment')
check_blacklist = self._create_patch('kale.task.Task._check_blacklist')
raised_exc = exceptions.BlacklistedException()
check_blacklist.side_effect = raised_exc
task_inst = test_utils.new_mock_task(task_class=test_utils.MockTask)
task_inst._start_time = 1
task_args = [1, 'a']
with self.assertRaises(exceptions.BlacklistedException):
task_inst.run(*task_args)
setup_env.assert_called_once_with()
pre_run.assert_called_once_with(*task_args)
self.assertFalse(run_task.called)
clean_env.assert_called_once_with(task_id='mock_task',
task_name='kale.test_utils.MockTask',
exc=raised_exc)
def testRunWorkerChildProcess(self):
"""Test running a task in the child process.."""
settings.RUN_TASK_AS_CHILD = True
task_inst = test_utils.new_mock_task(task_class=test_utils.MockTask)
task_args = [1, 'a']
task_inst.run(*task_args)
def testTargetRuntimeExceeded(self):
"""Task task target runtime exceeded."""
task_inst = test_utils.new_mock_task(
task_class=test_utils.SlowButNotTooSlowTask)
with mock.patch(
'kale.task.Task._alert_runtime_exceeded') as time_exceeded:
task_inst.run()
self.assertTrue(time_exceeded.called)
def testTargetRuntimeExceeded(self):
"""Task task target runtime exceeded."""
task_inst = test_utils.new_mock_task(
task_class=test_utils.SlowButNotTooSlowTask)
with mock.patch(
'kale.task.Task._alert_runtime_exceeded') as time_exceeded:
task_inst.run()
self.assertTrue(time_exceeded.called)
def testRunWorkerFailTaskChildProcess(self):
"""Test running a task in the child process.."""
settings.RUN_TASK_AS_CHILD = True
task_inst = test_utils.new_mock_task(task_class=test_utils.FailTask)
task_inst._start_time = 1
task_args = [1, 'a']
with self.assertRaises(exceptions.TaskException):
task_inst.run(*task_args)
self.assertTrue(task_inst._end_time > 0)
self.assertTrue(task_inst._task_latency_sec > 0)
def testTaskUnrecoverableException(self):
"""Task task failing with unrecoverable exception."""
task_inst = test_utils.new_mock_task(task_class=test_utils.FailTask)
message = test_utils.MockMessage(task_inst)
with mock.patch(
'kale.task.Task._report_permanent_failure') as fail_func:
exc = SyntaxError('Unrecoverable Error')
retried = test_utils.FailTask.handle_failure(message, exc)
self.assertFalse(retried)
fail_func.assert_called_once_with(
message, exc, task.PERMANENT_FAILURE_UNRECOVERABLE, True)
def testTaskUnrecoverableException(self):
"""Task task failing with unrecoverable exception."""
task_inst = test_utils.new_mock_task(task_class=test_utils.FailTask)
message = test_utils.MockMessage(task_inst)
with mock.patch(
'kale.task.Task._report_permanent_failure') as fail_func:
exc = SyntaxError('Unrecoverable Error')
retried = test_utils.FailTask.handle_failure(message, exc)
self.assertFalse(retried)
fail_func.assert_called_once_with(
message, exc, task.PERMANENT_FAILURE_UNRECOVERABLE, True)
def testTaskRuntimeExceeded(self):
"""Task task failing from timeout."""
task_inst = test_utils.new_mock_task(task_class=test_utils.FailTask)
with mock.patch(
'kale.task.Task._report_permanent_failure') as fail_func:
exc = exceptions.TaskException('Exception')
message = test_utils.MockMessage(
task_inst, retry_num=0, failure_num=task_inst.max_retries + 1)
retried = test_utils.FailTask.handle_failure(message, exc)
self.assertFalse(retried)
fail_func.assert_called_once_with(
message, exc, task.PERMANENT_FAILURE_RETRIES_EXCEEDED, False)
def testTaskRetriesExceeded(self):
"""Task task failing with retries exceeded."""
task_inst = test_utils.new_mock_task(task_class=test_utils.FailTask)
message = test_utils.MockMessage(
task_inst, retry_num=test_utils.FailTask.max_retries)
with mock.patch(
'kale.task.Task._report_permanent_failure') as fail_func:
exc = exceptions.TaskException('Exception')
retried = test_utils.FailTask.handle_failure(message, exc)
self.assertFalse(retried)
fail_func.assert_called_once_with(
message, exc, task.PERMANENT_FAILURE_RETRIES_EXCEEDED, False)
def testTaskNoRetries(self):
"""Task task failing with retries disabled."""
task_inst = test_utils.new_mock_task(
task_class=test_utils.FailTaskNoRetries)
message = test_utils.MockMessage(task_inst)
with mock.patch(
'kale.task.Task._report_permanent_failure') as fail_func:
exc = exceptions.TaskException('Exception')
retried = test_utils.FailTaskNoRetries.handle_failure(message, exc)
self.assertFalse(retried)
fail_func.assert_called_once_with(
message, exc, task.PERMANENT_FAILURE_NO_RETRY, True)
def testBlacklistedTaskFailsChildProcess(self):
"""Test that a blacklisted task raises an exception in the child process.."""
check_blacklist = self._create_patch('kale.task.Task._check_blacklist')
raised_exc = exceptions.BlacklistedException()
check_blacklist.side_effect = raised_exc
settings.RUN_TASK_AS_CHILD = True
task_inst = test_utils.new_mock_task(task_class=test_utils.MockTask)
task_inst._start_time = 1
task_args = [1, 'a']
with self.assertRaises(exceptions.BlacklistedException):
task_inst.run(*task_args)
def testTaskNoRetries(self):
"""Task task failing with retries disabled."""
task_inst = test_utils.new_mock_task(
task_class=test_utils.FailTaskNoRetries)
message = test_utils.MockMessage(task_inst)
with mock.patch(
'kale.task.Task._report_permanent_failure') as fail_func:
exc = exceptions.TaskException('Exception')
retried = test_utils.FailTaskNoRetries.handle_failure(message, exc)
self.assertFalse(retried)
fail_func.assert_called_once_with(
message, exc, task.PERMANENT_FAILURE_NO_RETRY, True)
def testRunWorker(self):
"""Test running a task."""
setup_env = self._create_patch(
'kale.task.Task._setup_task_environment')
pre_run = self._create_patch('kale.task.Task._pre_run')
post_run = self._create_patch('kale.task.Task._post_run')
clean_env = self._create_patch(
'kale.task.Task._clean_task_environment')
task_inst = test_utils.new_mock_task(task_class=test_utils.MockTask)
task_args = [1, 'a']
task_inst.run(*task_args)
setup_env.assert_called_once_with()
pre_run.assert_called_once_with(*task_args)
post_run.assert_called_once_with(*task_args)
clean_env.assert_called_once_with(
task_id='mock_task', task_name='kale.test_utils.MockTask')
def testRunWorker(self):
"""Test running a task."""
setup_env = self._create_patch(
'kale.task.Task._setup_task_environment')
pre_run = self._create_patch('kale.task.Task._pre_run')
post_run = self._create_patch('kale.task.Task._post_run')
clean_env = self._create_patch(
'kale.task.Task._clean_task_environment')
task_inst = test_utils.new_mock_task(task_class=test_utils.MockTask)
task_args = [1, 'a']
task_inst.run(*task_args)
setup_env.assert_called_once_with()
pre_run.assert_called_once_with(*task_args)
post_run.assert_called_once_with(*task_args)
clean_env.assert_called_once_with(task_id='mock_task',
task_name='kale.test_utils.MockTask')
def testDelayedPublish(self):
task_inst = test_utils.new_mock_task(task_class=test_utils.MockTask)
delay_sec = 60
random_arg = 99
random_kwarg = 100
payload = {
'args': (random_arg, ),
'kwargs': {
'random_kwarg': random_kwarg,
'delay_sec': delay_sec
},
'app_data': {}
}
with mock.patch('kale.publisher.Publisher.publish') as publish_func:
task_inst.publish({},
random_arg,
delay_sec=delay_sec,
random_kwarg=random_kwarg)
message = test_utils.MockMessage(task_inst)
publish_func.assert_called_once_with(test_utils.MockTask,
message.task_id,
payload,
delay_sec=delay_sec)
def testRunWorkerFailTask(self):
"""Test running a task."""
setup_env = self._create_patch(
'kale.task.Task._setup_task_environment')
pre_run = self._create_patch('kale.task.Task._pre_run')
post_run = self._create_patch('kale.task.Task._post_run')
clean_env = self._create_patch(
'kale.task.Task._clean_task_environment')
task_inst = test_utils.new_mock_task(task_class=test_utils.FailTask)
task_inst._start_time = 1
task_args = [1, 'a']
with self.assertRaises(exceptions.TaskException) as exc_ctxt_mngr:
task_inst.run(*task_args)
setup_env.assert_called_once_with()
pre_run.assert_called_once_with(*task_args)
assert not post_run.called, '_post_run should not have been called.'
clean_env.assert_called_once_with(
task_id='fail_task', task_name='kale.test_utils.FailTask',
exc=exc_ctxt_mngr.exception)
self.assertTrue(task_inst._end_time > 0)
self.assertTrue(task_inst._task_latency_sec > 0)
