def test_kbd_interrupt_in_wait_with_limited_concurrency(
async_runner, make_cases, make_async_exec_ctx):
# The general idea for this test is to allow enough time for all the
# four checks to be submitted and at the same time we need the
# KeyboardInterruptCheck to finish first (the corresponding wait should
# trigger the failure), so as to make the framework kill the remaining
# three.
ctx = make_async_exec_ctx(2)
next(ctx)
runner, _ = async_runner
with pytest.raises(KeyboardInterrupt):
runner.runall(
make_cases([
KeyboardInterruptCheck(),
SleepCheck(10),
SleepCheck(10),
SleepCheck(10)
]))
# FIXME: Dump everything in case Github #1369 appears
print(util.repr(runner))
print(runner.stats.failure_report())
print(util.repr(rt.runtime().site_config))
assert_interrupted_run(runner)
def test_kbd_interrupt_in_setup_with_limited_concurrency(self):
checks = [
SleepCheck(1),
SleepCheck(1),
SleepCheck(1),
KeyboardInterruptCheck(phase='setup')
]
self._run_checks(checks, 2)
def test_kbd_interrupt_in_wait_with_concurrency(self):
checks = [
KeyboardInterruptCheck(),
SleepCheck(10),
SleepCheck(10),
SleepCheck(10)
]
self._run_checks(checks, 4)
def test_kbd_interrupt_in_wait_with_limited_concurrency(self):
# The general idea for this test is to allow enough time for all the
# four checks to be submitted and at the same time we need the
# KeyboardInterruptCheck to finish first (the corresponding wait should
# trigger the failure), so as to make the framework kill the remaining
# three.
checks = [KeyboardInterruptCheck(),
SleepCheck(10), SleepCheck(10), SleepCheck(10)]
self._run_checks(checks, 2)
def test_kbd_interrupt_in_setup_with_limited_concurrency(
async_runner, make_cases, make_async_exec_ctx):
ctx = make_async_exec_ctx(2)
next(ctx)
runner, _ = async_runner
with pytest.raises(KeyboardInterrupt):
runner.runall(make_cases([
SleepCheck(1), SleepCheck(1), SleepCheck(1),
KeyboardInterruptCheck(phase='setup')
]))
assert_interrupted_run(runner)
def test_kbd_interrupt_in_wait_with_concurrency(async_runner, make_cases,
make_async_exec_ctx):
ctx = make_async_exec_ctx(4)
next(ctx)
runner, _ = async_runner
with pytest.raises(KeyboardInterrupt):
runner.runall(make_cases([
KeyboardInterruptCheck(), SleepCheck(10),
SleepCheck(10), SleepCheck(10)
]))
assert_interrupted_run(runner)
def test_kbd_interrupt_in_setup_with_concurrency(async_runner, make_cases,
make_exec_ctx):
make_exec_ctx(options=max_jobs_opts(4))
runner, _ = async_runner
with pytest.raises(KeyboardInterrupt):
runner.runall(
make_cases([
SleepCheck(1),
SleepCheck(1),
SleepCheck(1),
KeyboardInterruptCheck(phase='setup')
]))
assert_interrupted_run(runner)
def test_concurrency_unlimited(async_runner, make_cases, make_async_exec_ctx):
num_checks = 3
# Trigger evaluation of the execution context
ctx = make_async_exec_ctx(num_checks)
next(ctx)
runner, monitor = async_runner
runner.runall(make_cases([SleepCheck(.5) for i in range(num_checks)]))
# Ensure that all tests were run and without failures.
assert num_checks == runner.stats.num_cases()
assert_runall(runner)
assert 0 == len(runner.stats.failed())
# Ensure that maximum concurrency was reached as fast as possible
assert num_checks == max(monitor.num_tasks)
assert num_checks == monitor.num_tasks[num_checks]
begin_stamps, end_stamps = _read_timestamps(monitor.tasks)
# Warn if not all tests were run in parallel; the corresponding strict
# check would be:
#
# assert begin_stamps[-1] <= end_stamps[0]
#
if begin_stamps[-1] > end_stamps[0]:
pytest.skip('the system seems too much loaded.')
def test_kbd_interrupt_in_wait_with_limited_concurrency(
async_runner, make_cases, make_async_exec_ctx):
# The general idea for this test is to allow enough time for all the
# four checks to be submitted and at the same time we need the
# KeyboardInterruptCheck to finish first (the corresponding wait should
# trigger the failure), so as to make the framework kill the remaining
# three.
ctx = make_async_exec_ctx(2)
next(ctx)
runner, _ = async_runner
with pytest.raises(KeyboardInterrupt):
runner.runall(make_cases([
KeyboardInterruptCheck(), SleepCheck(10),
SleepCheck(10), SleepCheck(10)
]))
assert_interrupted_run(runner)
def test_run_complete_fails_main_loop(async_runner, make_cases, make_exec_ctx):
make_exec_ctx(options=max_jobs_opts(1))
runner, _ = async_runner
num_checks = 3
runner.runall(
make_cases(
[SleepCheckPollFail(10),
SleepCheck(0.1),
SleepCheckPollFail(10)]))
assert_runall(runner)
stats = runner.stats
assert stats.num_cases() == num_checks
assert len(stats.failed()) == 2
# Verify that the succeeded test is the SleepCheck
for t in stats.tasks():
if not t.failed:
assert isinstance(t.check, SleepCheck)
def test_concurrency_limited(async_runner, make_cases, make_async_exec_ctx):
# The number of checks must be <= 2*max_jobs.
num_checks, max_jobs = 5, 3
ctx = make_async_exec_ctx(max_jobs)
next(ctx)
runner, monitor = async_runner
runner.runall(make_cases([SleepCheck(.5) for i in range(num_checks)]))
# Ensure that all tests were run and without failures.
assert num_checks == runner.stats.num_cases()
assert_runall(runner)
assert 0 == len(runner.stats.failed())
# Ensure that maximum concurrency was reached as fast as possible
assert max_jobs == max(monitor.num_tasks)
assert max_jobs == monitor.num_tasks[max_jobs]
begin_stamps, end_stamps = _read_timestamps(monitor.tasks)
# Ensure that the jobs after the first #max_jobs were each run after
# one of the previous #max_jobs jobs had finished
# (e.g. begin[max_jobs] > end[0]).
# Note: we may ensure this strictly as we may ensure serial behaviour.
begin_after_end = (
b > e for b, e in zip(begin_stamps[max_jobs:], end_stamps[:-max_jobs]))
assert all(begin_after_end)
# NOTE: to ensure that these remaining jobs were also run
# in parallel one could do the command hereafter; however, it would
# require to substantially increase the sleep time (in SleepCheck),
# because of the delays in rescheduling (1s, 2s, 3s, 1s, 2s,...).
# We currently prefer not to do this last concurrency test to avoid an
# important prolongation of the unit test execution time.
# self.assertTrue(self.begin_stamps[-1] < self.end_stamps[max_jobs])
# Warn if the first #max_jobs jobs were not run in parallel; the
# corresponding strict check would be:
# self.assertTrue(self.begin_stamps[max_jobs-1] <= self.end_stamps[0])
if begin_stamps[max_jobs - 1] > end_stamps[0]:
pytest.skip('the system seems too loaded.')
def test_concurrency_none(self):
checks = [SleepCheck(0.5) for i in range(3)]
num_checks = len(checks)
self.set_max_jobs(1)
self.runner.runall(checks)
# Ensure that all tests were run and without failures.
self.assertEqual(len(checks), self.runner.stats.num_cases())
self.assertEqual(0, self.runner.stats.num_failures())
# Ensure that a single task was running all the time
self.assertEqual(1, max(self.monitor.num_tasks))
# Read the timestamps sorted to permit simple concurrency tests.
self.read_timestamps(self.monitor.tasks)
# Ensure that the jobs were run after the previous job had finished
# (e.g. begin[1] > end[0]).
begin_after_end = (
b > e for b, e in zip(self.begin_stamps[1:], self.end_stamps[:-1]))
self.assertTrue(all(begin_after_end))
def test_concurrency_unlimited(self):
checks = [SleepCheck(0.5) for i in range(3)]
self.set_max_jobs(len(checks))
self.runner.runall(checks)
# Ensure that all tests were run and without failures.
self.assertEqual(len(checks), self.runner.stats.num_cases())
self.assertEqual(0, self.runner.stats.num_failures())
# Ensure that maximum concurrency was reached as fast as possible
self.assertEqual(len(checks), max(self.monitor.num_tasks))
self.assertEqual(len(checks), self.monitor.num_tasks[len(checks)])
self.read_timestamps(self.monitor.tasks)
# Warn if not all tests were run in parallel; the corresponding strict
# check would be:
#
# self.assertTrue(self.begin_stamps[-1] <= self.end_stamps[0])
#
if self.begin_stamps[-1] > self.end_stamps[0]:
self.skipTest('the system seems too much loaded.')
def test_run_complete_fails_busy_loop(async_runner, make_cases,
make_async_exec_ctx):
ctx = make_async_exec_ctx(1)
next(ctx)
runner, _ = async_runner
num_checks = 3
runner.runall(
make_cases([
SleepCheckPollFailLate(1),
SleepCheck(0.1),
SleepCheckPollFailLate(0.5)
]))
assert_runall(runner)
stats = runner.stats
assert stats.num_cases() == num_checks
assert len(stats.failed()) == 2
# Verify that the succeeded test is the SleepCheck
for t in stats.tasks():
if not t.failed:
assert isinstance(t.check, SleepCheck)
def test_concurrency_limited(self):
# The number of checks must be <= 2*max_jobs.
checks = [SleepCheck(0.5) for i in range(5)]
max_jobs = len(checks) - 2
self.set_max_jobs(max_jobs)
self.runner.runall(checks)
# Ensure that all tests were run and without failures.
self.assertEqual(len(checks), self.runner.stats.num_cases())
self.assertEqual(0, self.runner.stats.num_failures())
# Ensure that maximum concurrency was reached as fast as possible
self.assertEqual(max_jobs, max(self.monitor.num_tasks))
self.assertEqual(max_jobs, self.monitor.num_tasks[max_jobs])
self.read_timestamps(self.monitor.tasks)
# Ensure that the jobs after the first #max_jobs were each run after
# one of the previous #max_jobs jobs had finished
# (e.g. begin[max_jobs] > end[0]).
# Note: we may ensure this strictly as we may ensure serial behaviour.
begin_after_end = (b > e for b, e in zip(self.begin_stamps[max_jobs:],
self.end_stamps[:-max_jobs]))
self.assertTrue(all(begin_after_end))
# NOTE: to ensure that these remaining jobs were also run
# in parallel one could do the command hereafter; however, it would
# require to substantially increase the sleep time (in SleepCheck),
# because of the delays in rescheduling (1s, 2s, 3s, 1s, 2s,...).
# We currently prefer not to do this last concurrency test to avoid an
# important prolongation of the unit test execution time.
# self.assertTrue(self.begin_stamps[-1] < self.end_stamps[max_jobs])
# Warn if the first #max_jobs jobs were not run in parallel; the
# corresponding strict check would be:
# self.assertTrue(self.begin_stamps[max_jobs-1] <= self.end_stamps[0])
if self.begin_stamps[max_jobs - 1] > self.end_stamps[0]:
self.skipTest('the system seems too loaded.')
def test_concurrency_none(async_runner, make_cases, make_exec_ctx):
num_checks = 3
make_exec_ctx(options=max_jobs_opts(1))
runner, monitor = async_runner
runner.runall(make_cases([SleepCheck(.5) for i in range(num_checks)]))
# Ensure that all tests were run and without failures.
assert num_checks == runner.stats.num_cases()
assert_runall(runner)
assert 0 == len(runner.stats.failed())
# Ensure that a single task was running all the time
assert 1 == max(monitor.num_tasks)
# Read the timestamps sorted to permit simple concurrency tests.
begin_stamps, end_stamps = _read_timestamps(monitor.tasks)
# Ensure that the jobs were run after the previous job had finished
# (e.g. begin[1] > end[0]).
begin_after_end = (b > e
for b, e in zip(begin_stamps[1:], end_stamps[:-1]))
assert all(begin_after_end)
def test_sigterm(self):
# Wrapper of self.runall which is used from a child process and
# passes any exception, number of cases and failures to the parent
# process
def _runall(checks, ns):
exc = None
try:
self.runall(checks)
except BaseException as e:
exc = e
finally:
ns.exc = exc
ns.num_cases = self.runner.stats.num_cases()
ns.num_failures = len(self.runner.stats.failures())
with multiprocessing.Manager() as manager:
ns = manager.Namespace()
p = multiprocessing.Process(target=_runall,
args=([SleepCheck(20)], ns))
p.start()
# Allow some time so that the SleepCheck is submitted.
# The sleep time of the submitted test is much larger to
# ensure that it does not finish before the termination signal
time.sleep(0.2)
p.terminate()
p.join()
# Either the test is submitted and it fails due to the termination
# or it is not yet submitted when the termination signal is sent
assert (ns.num_cases, ns.num_failures) in {(1, 1), (0, 0)}
with pytest.raises(ReframeForceExitError,
match='received TERM signal'):
if ns.exc:
raise ns.exc