def test_kbd_interrupt_in_wait_with_limited_concurrency( async_runner, make_cases, make_async_exec_ctx): # The general idea for this test is to allow enough time for all the # four checks to be submitted and at the same time we need the # KeyboardInterruptCheck to finish first (the corresponding wait should # trigger the failure), so as to make the framework kill the remaining # three. ctx = make_async_exec_ctx(2) next(ctx) runner, _ = async_runner with pytest.raises(KeyboardInterrupt): runner.runall( make_cases([ KeyboardInterruptCheck(), SleepCheck(10), SleepCheck(10), SleepCheck(10) ])) # FIXME: Dump everything in case Github #1369 appears print(util.repr(runner)) print(runner.stats.failure_report()) print(util.repr(rt.runtime().site_config)) assert_interrupted_run(runner)
def test_kbd_interrupt_in_setup_with_limited_concurrency(self): checks = [ SleepCheck(1), SleepCheck(1), SleepCheck(1), KeyboardInterruptCheck(phase='setup') ] self._run_checks(checks, 2)
def test_kbd_interrupt_in_wait_with_concurrency(self): checks = [ KeyboardInterruptCheck(), SleepCheck(10), SleepCheck(10), SleepCheck(10) ] self._run_checks(checks, 4)
def test_kbd_interrupt_in_wait_with_limited_concurrency(self): # The general idea for this test is to allow enough time for all the # four checks to be submitted and at the same time we need the # KeyboardInterruptCheck to finish first (the corresponding wait should # trigger the failure), so as to make the framework kill the remaining # three. checks = [KeyboardInterruptCheck(), SleepCheck(10), SleepCheck(10), SleepCheck(10)] self._run_checks(checks, 2)
def test_kbd_interrupt_in_setup_with_limited_concurrency( async_runner, make_cases, make_async_exec_ctx): ctx = make_async_exec_ctx(2) next(ctx) runner, _ = async_runner with pytest.raises(KeyboardInterrupt): runner.runall(make_cases([ SleepCheck(1), SleepCheck(1), SleepCheck(1), KeyboardInterruptCheck(phase='setup') ])) assert_interrupted_run(runner)
def test_kbd_interrupt_in_wait_with_concurrency(async_runner, make_cases, make_async_exec_ctx): ctx = make_async_exec_ctx(4) next(ctx) runner, _ = async_runner with pytest.raises(KeyboardInterrupt): runner.runall(make_cases([ KeyboardInterruptCheck(), SleepCheck(10), SleepCheck(10), SleepCheck(10) ])) assert_interrupted_run(runner)
def test_kbd_interrupt_in_setup_with_concurrency(async_runner, make_cases, make_exec_ctx): make_exec_ctx(options=max_jobs_opts(4)) runner, _ = async_runner with pytest.raises(KeyboardInterrupt): runner.runall( make_cases([ SleepCheck(1), SleepCheck(1), SleepCheck(1), KeyboardInterruptCheck(phase='setup') ])) assert_interrupted_run(runner)
def test_concurrency_unlimited(async_runner, make_cases, make_async_exec_ctx): num_checks = 3 # Trigger evaluation of the execution context ctx = make_async_exec_ctx(num_checks) next(ctx) runner, monitor = async_runner runner.runall(make_cases([SleepCheck(.5) for i in range(num_checks)])) # Ensure that all tests were run and without failures. assert num_checks == runner.stats.num_cases() assert_runall(runner) assert 0 == len(runner.stats.failed()) # Ensure that maximum concurrency was reached as fast as possible assert num_checks == max(monitor.num_tasks) assert num_checks == monitor.num_tasks[num_checks] begin_stamps, end_stamps = _read_timestamps(monitor.tasks) # Warn if not all tests were run in parallel; the corresponding strict # check would be: # # assert begin_stamps[-1] <= end_stamps[0] # if begin_stamps[-1] > end_stamps[0]: pytest.skip('the system seems too much loaded.')
def test_kbd_interrupt_in_wait_with_limited_concurrency( async_runner, make_cases, make_async_exec_ctx): # The general idea for this test is to allow enough time for all the # four checks to be submitted and at the same time we need the # KeyboardInterruptCheck to finish first (the corresponding wait should # trigger the failure), so as to make the framework kill the remaining # three. ctx = make_async_exec_ctx(2) next(ctx) runner, _ = async_runner with pytest.raises(KeyboardInterrupt): runner.runall(make_cases([ KeyboardInterruptCheck(), SleepCheck(10), SleepCheck(10), SleepCheck(10) ])) assert_interrupted_run(runner)
def test_run_complete_fails_main_loop(async_runner, make_cases, make_exec_ctx): make_exec_ctx(options=max_jobs_opts(1)) runner, _ = async_runner num_checks = 3 runner.runall( make_cases( [SleepCheckPollFail(10), SleepCheck(0.1), SleepCheckPollFail(10)])) assert_runall(runner) stats = runner.stats assert stats.num_cases() == num_checks assert len(stats.failed()) == 2 # Verify that the succeeded test is the SleepCheck for t in stats.tasks(): if not t.failed: assert isinstance(t.check, SleepCheck)
def test_concurrency_limited(async_runner, make_cases, make_async_exec_ctx): # The number of checks must be <= 2*max_jobs. num_checks, max_jobs = 5, 3 ctx = make_async_exec_ctx(max_jobs) next(ctx) runner, monitor = async_runner runner.runall(make_cases([SleepCheck(.5) for i in range(num_checks)])) # Ensure that all tests were run and without failures. assert num_checks == runner.stats.num_cases() assert_runall(runner) assert 0 == len(runner.stats.failed()) # Ensure that maximum concurrency was reached as fast as possible assert max_jobs == max(monitor.num_tasks) assert max_jobs == monitor.num_tasks[max_jobs] begin_stamps, end_stamps = _read_timestamps(monitor.tasks) # Ensure that the jobs after the first #max_jobs were each run after # one of the previous #max_jobs jobs had finished # (e.g. begin[max_jobs] > end[0]). # Note: we may ensure this strictly as we may ensure serial behaviour. begin_after_end = ( b > e for b, e in zip(begin_stamps[max_jobs:], end_stamps[:-max_jobs])) assert all(begin_after_end) # NOTE: to ensure that these remaining jobs were also run # in parallel one could do the command hereafter; however, it would # require to substantially increase the sleep time (in SleepCheck), # because of the delays in rescheduling (1s, 2s, 3s, 1s, 2s,...). # We currently prefer not to do this last concurrency test to avoid an # important prolongation of the unit test execution time. # self.assertTrue(self.begin_stamps[-1] < self.end_stamps[max_jobs]) # Warn if the first #max_jobs jobs were not run in parallel; the # corresponding strict check would be: # self.assertTrue(self.begin_stamps[max_jobs-1] <= self.end_stamps[0]) if begin_stamps[max_jobs - 1] > end_stamps[0]: pytest.skip('the system seems too loaded.')
def test_concurrency_none(self): checks = [SleepCheck(0.5) for i in range(3)] num_checks = len(checks) self.set_max_jobs(1) self.runner.runall(checks) # Ensure that all tests were run and without failures. self.assertEqual(len(checks), self.runner.stats.num_cases()) self.assertEqual(0, self.runner.stats.num_failures()) # Ensure that a single task was running all the time self.assertEqual(1, max(self.monitor.num_tasks)) # Read the timestamps sorted to permit simple concurrency tests. self.read_timestamps(self.monitor.tasks) # Ensure that the jobs were run after the previous job had finished # (e.g. begin[1] > end[0]). begin_after_end = ( b > e for b, e in zip(self.begin_stamps[1:], self.end_stamps[:-1])) self.assertTrue(all(begin_after_end))
def test_concurrency_unlimited(self): checks = [SleepCheck(0.5) for i in range(3)] self.set_max_jobs(len(checks)) self.runner.runall(checks) # Ensure that all tests were run and without failures. self.assertEqual(len(checks), self.runner.stats.num_cases()) self.assertEqual(0, self.runner.stats.num_failures()) # Ensure that maximum concurrency was reached as fast as possible self.assertEqual(len(checks), max(self.monitor.num_tasks)) self.assertEqual(len(checks), self.monitor.num_tasks[len(checks)]) self.read_timestamps(self.monitor.tasks) # Warn if not all tests were run in parallel; the corresponding strict # check would be: # # self.assertTrue(self.begin_stamps[-1] <= self.end_stamps[0]) # if self.begin_stamps[-1] > self.end_stamps[0]: self.skipTest('the system seems too much loaded.')
def test_run_complete_fails_busy_loop(async_runner, make_cases, make_async_exec_ctx): ctx = make_async_exec_ctx(1) next(ctx) runner, _ = async_runner num_checks = 3 runner.runall( make_cases([ SleepCheckPollFailLate(1), SleepCheck(0.1), SleepCheckPollFailLate(0.5) ])) assert_runall(runner) stats = runner.stats assert stats.num_cases() == num_checks assert len(stats.failed()) == 2 # Verify that the succeeded test is the SleepCheck for t in stats.tasks(): if not t.failed: assert isinstance(t.check, SleepCheck)
def test_concurrency_limited(self): # The number of checks must be <= 2*max_jobs. checks = [SleepCheck(0.5) for i in range(5)] max_jobs = len(checks) - 2 self.set_max_jobs(max_jobs) self.runner.runall(checks) # Ensure that all tests were run and without failures. self.assertEqual(len(checks), self.runner.stats.num_cases()) self.assertEqual(0, self.runner.stats.num_failures()) # Ensure that maximum concurrency was reached as fast as possible self.assertEqual(max_jobs, max(self.monitor.num_tasks)) self.assertEqual(max_jobs, self.monitor.num_tasks[max_jobs]) self.read_timestamps(self.monitor.tasks) # Ensure that the jobs after the first #max_jobs were each run after # one of the previous #max_jobs jobs had finished # (e.g. begin[max_jobs] > end[0]). # Note: we may ensure this strictly as we may ensure serial behaviour. begin_after_end = (b > e for b, e in zip(self.begin_stamps[max_jobs:], self.end_stamps[:-max_jobs])) self.assertTrue(all(begin_after_end)) # NOTE: to ensure that these remaining jobs were also run # in parallel one could do the command hereafter; however, it would # require to substantially increase the sleep time (in SleepCheck), # because of the delays in rescheduling (1s, 2s, 3s, 1s, 2s,...). # We currently prefer not to do this last concurrency test to avoid an # important prolongation of the unit test execution time. # self.assertTrue(self.begin_stamps[-1] < self.end_stamps[max_jobs]) # Warn if the first #max_jobs jobs were not run in parallel; the # corresponding strict check would be: # self.assertTrue(self.begin_stamps[max_jobs-1] <= self.end_stamps[0]) if self.begin_stamps[max_jobs - 1] > self.end_stamps[0]: self.skipTest('the system seems too loaded.')
def test_concurrency_none(async_runner, make_cases, make_exec_ctx): num_checks = 3 make_exec_ctx(options=max_jobs_opts(1)) runner, monitor = async_runner runner.runall(make_cases([SleepCheck(.5) for i in range(num_checks)])) # Ensure that all tests were run and without failures. assert num_checks == runner.stats.num_cases() assert_runall(runner) assert 0 == len(runner.stats.failed()) # Ensure that a single task was running all the time assert 1 == max(monitor.num_tasks) # Read the timestamps sorted to permit simple concurrency tests. begin_stamps, end_stamps = _read_timestamps(monitor.tasks) # Ensure that the jobs were run after the previous job had finished # (e.g. begin[1] > end[0]). begin_after_end = (b > e for b, e in zip(begin_stamps[1:], end_stamps[:-1])) assert all(begin_after_end)
def test_sigterm(self): # Wrapper of self.runall which is used from a child process and # passes any exception, number of cases and failures to the parent # process def _runall(checks, ns): exc = None try: self.runall(checks) except BaseException as e: exc = e finally: ns.exc = exc ns.num_cases = self.runner.stats.num_cases() ns.num_failures = len(self.runner.stats.failures()) with multiprocessing.Manager() as manager: ns = manager.Namespace() p = multiprocessing.Process(target=_runall, args=([SleepCheck(20)], ns)) p.start() # Allow some time so that the SleepCheck is submitted. # The sleep time of the submitted test is much larger to # ensure that it does not finish before the termination signal time.sleep(0.2) p.terminate() p.join() # Either the test is submitted and it fails due to the termination # or it is not yet submitted when the termination signal is sent assert (ns.num_cases, ns.num_failures) in {(1, 1), (0, 0)} with pytest.raises(ReframeForceExitError, match='received TERM signal'): if ns.exc: raise ns.exc