def test_status_slurm_submit_running(self, status_mock, path_exists_mock, get_job_by_id_mock, update_job_mock): path_exists_mock.side_effect = Mock(return_value=False) db_job = { 'submit_id': 4321, 'working_directory': "a/test/working/directory", 'submit_directory': "a/submit/directory/" } get_job_by_id_mock.side_effect = Mock(return_value=db_job) details = { 'scheduler_id': 1234 } from core.status import status status_mock.side_effect = Mock(return_value=(JobStatus.SUBMITTED, JobStatus.display_name(JobStatus.SUBMITTED))) result = status(details) # Job has only one step that is submitted, and job is not completed self.assertEqual(result['status'], [{'what': 'submit', 'status': 30, 'info': 'Submitted'}]) self.assertEqual(result['complete'], False) status_mock.side_effect = Mock(return_value=(JobStatus.RUNNING, JobStatus.display_name(JobStatus.RUNNING))) result = status(details) # Job has only one step that is running, and job is not completed self.assertEqual(result['status'], [{'what': 'submit', 'status': 50, 'info': 'Running'}]) self.assertEqual(result['complete'], False) # Update job (To remove the submit_id) should never have been called self.assertEqual(update_job_mock.call_count, 0)
def test_status_slurm_submit_error(self, status_mock, path_exists_mock, get_job_by_id_mock, update_job_mock, delete_job_mock): path_exists_mock.side_effect = Mock(return_value=False) db_job = { 'submit_id': 4321, 'working_directory': "a/test/working/directory", 'submit_directory': "a/submit/directory/" } get_job_by_id_mock.side_effect = Mock(return_value=db_job) details = { 'scheduler_id': 1234 } from core.status import status status_mock.side_effect = Mock(return_value=(JobStatus.ERROR, JobStatus.display_name(JobStatus.ERROR))) result = status(details) # Job has only one step that is error, and job is completed self.assertEqual(result['status'], [{'what': 'submit', 'status': 400, 'info': 'Error'}]) self.assertEqual(result['complete'], True) # Delete should have been called self.assertEqual(delete_job_mock.call_count, 1) self.assertEqual(update_job_mock.call_count, 0)
def test_status_slurm_submit_success(self, status_mock, path_exists_mock, get_job_by_id_mock, update_job_mock): path_exists_mock.side_effect = Mock(return_value=False) db_job = { 'submit_id': 4321, 'working_directory': "a/test/working/directory", 'submit_directory': "a/submit/directory/" } get_job_by_id_mock.side_effect = Mock(return_value=db_job) details = { 'scheduler_id': 1234 } from core.status import status status_mock.side_effect = Mock(return_value=(JobStatus.COMPLETED, JobStatus.display_name(JobStatus.COMPLETED))) result = status(details) # Job has only one step that is error, and job is completed self.assertEqual(result['status'], [{'what': 'submit', 'status': 500, 'info': 'Completed'}]) self.assertEqual(result['complete'], False) # Delete should have been called and the submit_id key should no longer exist in the job record self.assertEqual(update_job_mock.call_count, 1) self.assertDictEqual(update_job_mock.call_args[0][0], { 'working_directory': "a/test/working/directory", 'submit_directory': "a/submit/directory/" })
def test_status_job_doesnt_exist(self, get_job_by_id_mock): get_job_by_id_mock.side_effect = Mock(return_value=None) from core.status import status result = status({'scheduler_id': 1234}) self.assertEqual( result['status'], [ { 'what': 'system', 'status': 400, 'info': 'Job does not exist. Perhaps it failed to start?' } ] ) self.assertEqual(result['complete'], True)
def test_status_condor(self, status_mock, get_job_by_id_mock, delete_job_mock): db_job = { 'submit_id': 1234, 'working_directory': "a/working/directory", 'submit_directory': "submit" } get_job_by_id_mock.side_effect = Mock(return_value=db_job) details = { 'scheduler_id': 1234 } from core.status import status job_status_result = None def job_status_mock(*args, **kwargs): return job_status_result, JobStatus.display_name(job_status_result) status_mock.side_effect = job_status_mock job_status_result = JobStatus.QUEUED result = status(details) self.assertEqual( result['status'], [ {'status': 40, 'what': 'submit', 'info': 'Queued'}, ] ) self.assertEqual(result['complete'], False) self.assertEqual(delete_job_mock.call_count, 0) job_status_result = JobStatus.RUNNING result = status(details) self.assertEqual( result['status'], [ {'status': 50, 'what': 'submit', 'info': 'Running'}, ] ) self.assertEqual(result['complete'], False) self.assertEqual(delete_job_mock.call_count, 0) job_status_result = JobStatus.COMPLETED result = status(details) self.assertEqual( result['status'], [ {'status': 500, 'what': 'submit', 'info': 'Completed'}, ] ) self.assertEqual(result['complete'], True) self.assertEqual(delete_job_mock.call_count, 1) job_status_result = JobStatus.ERROR result = status(details) self.assertEqual( result['status'], [ {'status': 400, 'what': 'submit', 'info': 'Error'}, ] ) self.assertEqual(result['complete'], True) self.assertEqual(delete_job_mock.call_count, 2) job_status_result = JobStatus.OUT_OF_MEMORY result = status(details) self.assertEqual( result['status'], [ {'status': 402, 'what': 'submit', 'info': 'Out of Memory'}, ] ) self.assertEqual(result['complete'], True) self.assertEqual(delete_job_mock.call_count, 3)
def test_status_slurm(self, status_mock, path_exists_mock, get_job_by_id_mock, delete_job_mock): path_exists_mock.side_effect = Mock(return_value=True) with TemporaryDirectory() as tmpdir: db_job = { 'working_directory': str(tmpdir), 'submit_directory': "" } get_job_by_id_mock.side_effect = Mock(return_value=db_job) details = { 'scheduler_id': 1234 } with open(os.path.join(tmpdir, 'slurm_ids'), "w") as f: f.writelines(['jid0 12345\n', 'jid1 54321\n']) f.flush() from core.status import status job_status_values = [] job_status_count = 0 def job_status_mock(*args, **kwargs): nonlocal job_status_count job_status_count += 1 return job_status_values[job_status_count - 1], JobStatus.display_name( job_status_values[job_status_count - 1]) status_mock.side_effect = job_status_mock job_status_values = [JobStatus.QUEUED, JobStatus.QUEUED] result = status(details) self.assertEqual( result['status'], [ {'status': 500, 'what': 'submit', 'info': 'Completed'}, {'status': 40, 'what': 'jid0', 'info': 'Queued'}, {'status': 40, 'what': 'jid1', 'info': 'Queued'} ] ) self.assertEqual(result['complete'], False) self.assertEqual(delete_job_mock.call_count, 0) job_status_count = 0 job_status_values = [JobStatus.RUNNING, JobStatus.QUEUED] result = status(details) self.assertEqual( result['status'], [ {'status': 500, 'what': 'submit', 'info': 'Completed'}, {'status': 50, 'what': 'jid0', 'info': 'Running'}, {'status': 40, 'what': 'jid1', 'info': 'Queued'} ] ) self.assertEqual(result['complete'], False) self.assertEqual(delete_job_mock.call_count, 0) job_status_count = 0 job_status_values = [JobStatus.COMPLETED, JobStatus.QUEUED] result = status(details) self.assertEqual( result['status'], [ {'status': 500, 'what': 'submit', 'info': 'Completed'}, {'status': 500, 'what': 'jid0', 'info': 'Completed'}, {'status': 40, 'what': 'jid1', 'info': 'Queued'} ] ) self.assertEqual(result['complete'], False) self.assertEqual(delete_job_mock.call_count, 0) job_status_count = 0 job_status_values = [JobStatus.COMPLETED, JobStatus.RUNNING] result = status(details) self.assertEqual( result['status'], [ {'status': 500, 'what': 'submit', 'info': 'Completed'}, {'status': 500, 'what': 'jid0', 'info': 'Completed'}, {'status': 50, 'what': 'jid1', 'info': 'Running'} ] ) self.assertEqual(result['complete'], False) self.assertEqual(delete_job_mock.call_count, 0) job_status_count = 0 job_status_values = [JobStatus.COMPLETED, JobStatus.COMPLETED] result = status(details) self.assertEqual( result['status'], [ {'status': 500, 'what': 'submit', 'info': 'Completed'}, {'status': 500, 'what': 'jid0', 'info': 'Completed'}, {'status': 500, 'what': 'jid1', 'info': 'Completed'} ] ) self.assertEqual(result['complete'], True) self.assertEqual(delete_job_mock.call_count, 1) job_status_count = 0 job_status_values = [JobStatus.WALL_TIME_EXCEEDED, JobStatus.CANCELLED] result = status(details) self.assertEqual( result['status'], [ {'status': 500, 'what': 'submit', 'info': 'Completed'}, {'status': 401, 'what': 'jid0', 'info': 'Wall Time Exceeded'}, {'status': 70, 'what': 'jid1', 'info': 'Cancelled'} ] ) self.assertEqual(result['complete'], True) self.assertEqual(delete_job_mock.call_count, 2) job_status_count = 0 job_status_values = [JobStatus.COMPLETED, JobStatus.CANCELLED] result = status(details) self.assertEqual( result['status'], [ {'status': 500, 'what': 'submit', 'info': 'Completed'}, {'status': 500, 'what': 'jid0', 'info': 'Completed'}, {'status': 70, 'what': 'jid1', 'info': 'Cancelled'} ] ) self.assertEqual(result['complete'], True) self.assertEqual(delete_job_mock.call_count, 3) job_status_count = 0 job_status_values = [JobStatus.COMPLETED, JobStatus.ERROR] result = status(details) self.assertEqual( result['status'], [ {'status': 500, 'what': 'submit', 'info': 'Completed'}, {'status': 500, 'what': 'jid0', 'info': 'Completed'}, {'status': 400, 'what': 'jid1', 'info': 'Error'} ] ) self.assertEqual(result['complete'], True) self.assertEqual(delete_job_mock.call_count, 4)