def test_get_status_bjobs_fails_once_says_EXIT_in_the_fourth_job_status_is_failed(
        self, run_process_mock
    ):
        self.count_fail_three_times_and_then_return_FAIL = 0

        def fail_one_time_and_then_return_FAIL(cmd):
            self.count_fail_three_times_and_then_return_FAIL += 1
            if self.count_fail_three_times_and_then_return_FAIL == 1:
                raise BjobsError
            elif self.count_fail_three_times_and_then_return_FAIL == 2:
                return "EXIT", ""
            else:
                assert False

        run_process_mock.side_effect = fail_one_time_and_then_return_FAIL

        lsf_status_checker = StatusChecker(
            123, "dummy", wait_between_tries=0.001, max_status_checks=4
        )
        actual = lsf_status_checker.get_status()
        expected = "failed"
        self.assertEqual(actual, expected)
        assert_called_n_times_with_same_args(
            run_process_mock, 2, "bjobs -o 'stat' -noheader 123"
        )
    def test_get_status_bjobs_fails_three_times_PEND_fourth_time_job_status_running(
        self, run_process_mock
    ):
        self.count_fail_three_times_and_then_return_PEND = 0

        def fail_three_times_and_then_return_PEND(cmd):
            self.count_fail_three_times_and_then_return_PEND += 1
            if self.count_fail_three_times_and_then_return_PEND == 1:
                raise BjobsError
            elif self.count_fail_three_times_and_then_return_PEND == 2:
                raise KeyError
            elif self.count_fail_three_times_and_then_return_PEND == 3:
                raise CalledProcessError(1, "bjobs")
            elif self.count_fail_three_times_and_then_return_PEND == 4:
                return "PEND", ""
            else:
                assert False

        run_process_mock.side_effect = fail_three_times_and_then_return_PEND

        lsf_status_checker = StatusChecker(
            123, "dummy", wait_between_tries=0.001, max_status_checks=4
        )
        actual = lsf_status_checker.get_status()
        expected = "running"
        self.assertEqual(actual, expected)
        assert_called_n_times_with_same_args(
            run_process_mock, 4, "bjobs -o 'stat' -noheader 123"
        )
 def test___get_status___status_UNKWN_and_wait_unknown___job_status_is_running(
     self, run_process_mock
 ):
     lsf_status_checker = StatusChecker(123, "dummy", kill_unknown=False)
     actual = lsf_status_checker.get_status()
     expected = lsf_status_checker.RUNNING
     self.assertEqual(actual, expected)
     run_process_mock.assert_called_once_with("bjobs -o 'stat' -noheader 123")
 def test___get_status___bjobs_says_process_is_WAIT___job_status_is_running(
     self, run_process_mock
 ):
     lsf_status_checker = StatusChecker(123, "dummy")
     actual = lsf_status_checker.get_status()
     expected = "running"
     self.assertEqual(actual, expected)
     run_process_mock.assert_called_once_with("bjobs -o 'stat' -noheader 123")
 def test___get_status___status_ZOMBI_and_ignore_zombie___job_status_is_failed(
     self, run_process_mock
 ):
     jobid = 123
     lsf_status_checker = StatusChecker(jobid, "dummy", kill_zombie=False)
     actual = lsf_status_checker.get_status()
     expected = lsf_status_checker.FAILED
     self.assertEqual(actual, expected)
     run_process_mock.assert_called_once_with(
         "bjobs -o 'stat' -noheader {}".format(jobid)
     )
 def test___get_status___status_UNKWN_and_kill_unknown___job_status_is_running(
     self, run_process_mock
 ):
     jobid = 123
     lsf_status_checker = StatusChecker(jobid, "dummy", kill_unknown=True)
     actual = lsf_status_checker.get_status()
     expected = lsf_status_checker.RUNNING
     self.assertEqual(actual, expected)
     calls = [
         call("bjobs -o 'stat' -noheader {}".format(jobid)),
         call("bkill -r {}".format(jobid)),
     ]
     run_process_mock.assert_has_calls(calls, any_order=False)
 def test_get_status_bjobs_fails_resource_line_does_not_exist_job_status_is_running(
     self, get_lines_of_log_file_mock, run_process_mock
 ):
     lsf_status_checker = StatusChecker(
         123, "dummy", wait_between_tries=0.001, max_status_checks=4
     )
     actual = lsf_status_checker.get_status()
     expected = lsf_status_checker.RUNNING
     self.assertEqual(actual, expected)
     assert_called_n_times_with_same_args(
         run_process_mock, 4, "bjobs -o 'stat' -noheader 123"
     )
     get_lines_of_log_file_mock.assert_called_once_with()
 def test_get_status_checking_log_raises_tail_error_status_is_failed(
     self, get_lines_of_log_file_mock, run_process_mock
 ):
     lsf_status_checker = StatusChecker(
         123, "dummy", wait_between_tries=0.001, max_status_checks=4
     )
     actual = lsf_status_checker.get_status()
     expected = lsf_status_checker.FAILED
     self.assertEqual(actual, expected)
     assert_called_n_times_with_same_args(
         run_process_mock, 4, "bjobs -o 'stat' -noheader 123"
     )
     get_lines_of_log_file_mock.assert_called_once_with()
 def test_get_status_bjobs_fails_query_status_using_log_job_status_is_success(
     self, get_lines_of_log_file_mock, run_process_mock
 ):
     lsf_status_checker = StatusChecker(
         123, "dummy", wait_between_tries=0.001, max_status_checks=4
     )
     actual = lsf_status_checker.get_status()
     expected = "success"
     self.assertEqual(actual, expected)
     assert_called_n_times_with_same_args(
         run_process_mock, 4, "bjobs -o 'stat' -noheader 123"
     )
     get_lines_of_log_file_mock.assert_called_once_with()
 def test___get_status___status_ZOMBI_and_kill_zombie___job_status_is_failed(
     self, run_process_mock
 ):
     jobid = 123
     lsf_status_checker = StatusChecker(jobid, "dummy", kill_zombie=True)
     actual = lsf_status_checker.get_status()
     expected = lsf_status_checker.FAILED
     self.assertEqual(actual, expected)
     calls = [
         call("bjobs -o 'stat' -noheader {}".format(jobid)),
         call("bkill -r {}".format(jobid)),
     ]
     run_process_mock.assert_has_calls(calls, any_order=False)