def test_pre_task_launch_error(self):
"""Tests running through a job execution where a pre-task fails to launch"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
job_exe = JobExecution.objects.get_job_exe_with_job_and_job_type(
self._job_exe_id)
running_job_exe = RunningJobExecution(job_exe)
# Start pre-task
task = running_job_exe.start_next_task()
pre_task_id = task.id
# Pre-task fails to launch
pre_task_results = TaskResults(pre_task_id)
pre_task_results.exit_code = 1
pre_task_results.when = now()
running_job_exe.task_fail(pre_task_results)
# Check results
job_exe = JobExecution.objects.select_related().get(
id=self._job_exe_id)
self.assertEqual(job_exe.status, 'FAILED')
self.assertEqual(job_exe.error.name, 'docker-task-launch')
def test_docker_terminated_error(self):
"""Tests running through a job execution where a Docker container terminates"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
job_exe = JobExecution.objects.get_job_exe_with_job_and_job_type(self._job_exe_id)
running_job_exe = RunningJobExecution(job_exe)
# Start pre-task
task = running_job_exe.start_next_task()
pre_task_id = task.id
# Pre-task running
pre_task_started = now()
update = job_test_utils.create_task_status_update(pre_task_id, 'agent', TaskStatusUpdate.RUNNING,
pre_task_started)
running_job_exe.task_update(update)
# Pre-task Docker container terminates
update = job_test_utils.create_task_status_update(pre_task_id, 'agent', TaskStatusUpdate.FAILED, now(),
reason='REASON_EXECUTOR_TERMINATED')
running_job_exe.task_update(update)
# Check results
job_exe = JobExecution.objects.select_related().get(id=self._job_exe_id)
self.assertEqual(job_exe.status, 'FAILED')
self.assertEqual(job_exe.error.name, 'docker-terminated')
def test_post_task_launch_error(self):
"""Tests running through a job execution where a post-task fails to launch"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
job_exe = JobExecution.objects.get_job_exe_with_job_and_job_type(
self._job_exe_id)
running_job_exe = RunningJobExecution(job_exe)
# Start pre-task
task = running_job_exe.start_next_task()
pre_task_id = task.id
# Pre-task running
pre_task_started = now()
running_job_exe.task_start(pre_task_id, pre_task_started)
# Complete pre-task
pre_task_completed = pre_task_started + timedelta(seconds=1)
pre_task_results = TaskResults(pre_task_id)
pre_task_results.exit_code = 0
pre_task_results.when = pre_task_completed
running_job_exe.task_complete(pre_task_results)
# Start job-task
task = running_job_exe.start_next_task()
job_task_id = task.id
# Job-task running
job_task_started = now()
running_job_exe.task_start(job_task_id, job_task_started)
# Complete job-task
job_task_completed = job_task_started + timedelta(seconds=1)
job_task_results = TaskResults(job_task_id)
job_task_results.exit_code = 0
job_task_results.when = job_task_completed
running_job_exe.task_complete(job_task_results)
# Start post-task
task = running_job_exe.start_next_task()
post_task_id = task.id
# Post-task fails to launch
post_task_results = TaskResults(post_task_id)
post_task_results.exit_code = 1
post_task_results.when = now()
running_job_exe.task_fail(post_task_results)
# Check results
job_exe = JobExecution.objects.select_related().get(
id=self._job_exe_id)
self.assertEqual(job_exe.status, 'FAILED')
self.assertEqual(job_exe.error.name, 'docker-task-launch')
def test_post_task_launch_error(self):
"""Tests running through a job execution where a post-task fails to launch"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
job_exe = JobExecution.objects.get_job_exe_with_job_and_job_type(self._job_exe_id)
running_job_exe = RunningJobExecution(job_exe)
# Start pre-task
task = running_job_exe.start_next_task()
pre_task_id = task.id
# Pre-task running
pre_task_started = now()
running_job_exe.task_start(pre_task_id, pre_task_started)
# Complete pre-task
pre_task_completed = pre_task_started + timedelta(seconds=1)
pre_task_results = TaskResults(pre_task_id)
pre_task_results.exit_code = 0
pre_task_results.when = pre_task_completed
running_job_exe.task_complete(pre_task_results)
# Start job-task
task = running_job_exe.start_next_task()
job_task_id = task.id
# Job-task running
job_task_started = now()
running_job_exe.task_start(job_task_id, job_task_started)
# Complete job-task
job_task_completed = job_task_started + timedelta(seconds=1)
job_task_results = TaskResults(job_task_id)
job_task_results.exit_code = 0
job_task_results.when = job_task_completed
running_job_exe.task_complete(job_task_results)
# Start post-task
task = running_job_exe.start_next_task()
post_task_id = task.id
# Post-task fails to launch
post_task_results = TaskResults(post_task_id)
post_task_results.exit_code = 1
post_task_results.when = now()
running_job_exe.task_fail(post_task_results)
# Check results
job_exe = JobExecution.objects.select_related().get(id=self._job_exe_id)
self.assertEqual(job_exe.status, 'FAILED')
self.assertEqual(job_exe.error.name, 'docker-task-launch')
def test_post_task_launch_error(self):
"""Tests running through a job execution where a post-task fails to launch"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
job_exe = JobExecution.objects.get_job_exe_with_job_and_job_type(self._job_exe_id)
running_job_exe = RunningJobExecution(job_exe)
# Start pre-task
task = running_job_exe.start_next_task()
pre_task_id = task.id
# Pre-task running
pre_task_started = now()
update = job_test_utils.create_task_status_update(pre_task_id, 'agent', TaskStatusUpdate.RUNNING,
pre_task_started)
running_job_exe.task_update(update)
# Complete pre-task
pre_task_completed = pre_task_started + timedelta(seconds=1)
update = job_test_utils.create_task_status_update(pre_task_id, 'agent', TaskStatusUpdate.FINISHED,
pre_task_completed)
running_job_exe.task_update(update)
# Start job-task
task = running_job_exe.start_next_task()
job_task_id = task.id
# Job-task running
job_task_started = now()
update = job_test_utils.create_task_status_update(job_task_id, 'agent', TaskStatusUpdate.RUNNING,
job_task_started)
running_job_exe.task_update(update)
# Complete job-task
job_task_completed = job_task_started + timedelta(seconds=1)
update = job_test_utils.create_task_status_update(job_task_id, 'agent', TaskStatusUpdate.FINISHED,
job_task_completed)
running_job_exe.task_update(update)
# Start post-task
task = running_job_exe.start_next_task()
post_task_id = task.id
# Post-task fails to launch
update = job_test_utils.create_task_status_update(post_task_id, 'agent', TaskStatusUpdate.FAILED, now())
running_job_exe.task_update(update)
# Check results
job_exe = JobExecution.objects.select_related().get(id=self._job_exe_id)
self.assertEqual(job_exe.status, 'FAILED')
self.assertEqual(job_exe.error.name, 'docker-task-launch')
def setUp(self):
django.setup()
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
self.node_model_1 = node_test_utils.create_node()
self.job_exe_model_1 = job_test_utils.create_job_exe(
status='RUNNING', node=self.node_model_1)
self.job_exe_1 = RunningJobExecution(self.job_exe_model_1)
self.node_model_2 = node_test_utils.create_node()
self.job_exe_model_2 = job_test_utils.create_job_exe(
status='RUNNING', node=self.node_model_2)
self.job_exe_2 = RunningJobExecution(self.job_exe_model_2)
self.job_exe_mgr = JobExecutionManager()
def test_general_algorithm_error(self):
"""Tests running through a job execution where the job-task has a general algorithm error (non-zero exit code)
"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
job_exe = JobExecution.objects.get_job_exe_with_job_and_job_type(
self._job_exe_id)
running_job_exe = RunningJobExecution(job_exe)
# Start pre-task
task = running_job_exe.start_next_task()
pre_task_id = task.id
# Pre-task running
pre_task_started = now()
running_job_exe.task_start(pre_task_id, pre_task_started)
# Complete pre-task
pre_task_completed = pre_task_started + timedelta(seconds=1)
pre_task_results = TaskResults(pre_task_id)
pre_task_results.exit_code = 0
pre_task_results.when = pre_task_completed
running_job_exe.task_complete(pre_task_results)
# Start job-task
task = running_job_exe.start_next_task()
job_task_id = task.id
# Job-task running
job_task_started = now()
running_job_exe.task_start(job_task_id, job_task_started)
# Fail job-task
job_task_failed = job_task_started + timedelta(seconds=1)
job_task_results = TaskResults(job_task_id)
job_task_results.exit_code = 1
job_task_results.when = job_task_failed
running_job_exe.task_fail(job_task_results)
# Check results
job_exe = JobExecution.objects.select_related().get(
id=self._job_exe_id)
self.assertEqual(job_exe.status, 'FAILED')
self.assertEqual(job_exe.error.name, 'algorithm-unknown')
def test_general_algorithm_error(self):
"""Tests running through a job execution where the job-task has a general algorithm error (non-zero exit code)
"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
job_exe = JobExecution.objects.get_job_exe_with_job_and_job_type(self._job_exe_id)
running_job_exe = RunningJobExecution(job_exe)
# Start pre-task
task = running_job_exe.start_next_task()
pre_task_id = task.id
# Pre-task running
pre_task_started = now()
update = job_test_utils.create_task_status_update(pre_task_id, 'agent', TaskStatusUpdate.RUNNING,
pre_task_started)
running_job_exe.task_update(update)
# Complete pre-task
pre_task_completed = pre_task_started + timedelta(seconds=1)
update = job_test_utils.create_task_status_update(pre_task_id, 'agent', TaskStatusUpdate.FINISHED,
pre_task_completed)
running_job_exe.task_update(update)
# Start job-task
task = running_job_exe.start_next_task()
job_task_id = task.id
# Job-task running
job_task_started = now()
update = job_test_utils.create_task_status_update(job_task_id, 'agent', TaskStatusUpdate.RUNNING,
job_task_started)
running_job_exe.task_update(update)
# Fail job-task
job_task_failed = job_task_started + timedelta(seconds=1)
update = job_test_utils.create_task_status_update(job_task_id, 'agent', TaskStatusUpdate.FAILED,
job_task_failed, exit_code=1)
running_job_exe.task_update(update)
# Check results
job_exe = JobExecution.objects.select_related().get(id=self._job_exe_id)
self.assertEqual(job_exe.status, 'FAILED')
self.assertEqual(job_exe.error.name, 'algorithm-unknown')
def test_get_error_missing(self):
"""Tests that general algorithm error is returned when a non-registered name is found in the mapping"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4)
self.assertIsNotNone(error)
self.assertEqual(error.name, 'algorithm-unknown')
def test_get_error_missing(self):
"""Tests that general algorithm error is returned when a non-registered name is found in the mapping"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4)
self.assertIsNotNone(error)
self.assertEqual(error.name, 'algorithm-unknown')
def test_get_error_missing_default(self):
"""Tests that custom error is returned when a non-registered name is found in the mapping"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
default_error = error_test_utils.create_error()
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4, default_error.name)
self.assertIsNotNone(error)
self.assertEqual(error.name, default_error.name)
def test_get_error_missing_default(self):
"""Tests that custom error is returned when a non-registered name is found in the mapping"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
error_interface_dict = {
'version': '1.0',
'exit_codes': {
'1': self.error_1.name,
'2': self.error_2.name,
'3': self.error_3.name,
},
}
default_error = error_test_utils.create_error()
error_interface = ErrorInterface(error_interface_dict)
error = error_interface.get_error(4, default_error.name)
self.assertIsNotNone(error)
self.assertEqual(error.name, default_error.name)
def test_pre_task_launch_error(self):
"""Tests running through a job execution where a pre-task fails to launch"""
# Clear error cache so test works correctly
CACHED_BUILTIN_ERRORS.clear()
job_exe = JobExecution.objects.get_job_exe_with_job_and_job_type(self._job_exe_id)
running_job_exe = RunningJobExecution(job_exe)
# Start pre-task
task = running_job_exe.start_next_task()
pre_task_id = task.id
# Pre-task fails to launch
pre_task_results = TaskResults(pre_task_id)
pre_task_results.when = now()
running_job_exe.task_fail(pre_task_results)
# Check results
job_exe = JobExecution.objects.select_related().get(id=self._job_exe_id)
self.assertEqual(job_exe.status, 'FAILED')
self.assertEqual(job_exe.error.name, 'task-launch')
def setUp(self):
django.setup()
CACHED_BUILTIN_ERRORS.clear() # Clear error cache since the error models keep getting rolled back
