async def test_abort_many_jobs_returns_immediately(num_jobs, run):
todo = [TJob(f'foo #{i}', async_sleep=5) for i in range(num_jobs)]
with assert_elapsed_time(lambda t: t < 1):
done = await run(todo, abort_after=0.1)
assert verify_tasks(done,
{f'foo #{i}': Cancelled
for i in range(num_jobs)})
async def test_abort_job_with_two_non_terminating_kills_both(run, num_workers):
if num_workers < 2:
pytest.skip('need Scheduler with at least 2 workers')
argv1 = mock_argv('ignore:SIGTERM', 'ignore:SIGINT', 'FOO', 'sleep:5')
argv2 = mock_argv('ignore:SIGTERM', 'ignore:SIGINT', 'BAR', 'sleep:5')
async def coro(ctx):
with ctx.tjob.subprocess_xevents(argv1, result='kill'):
async with ctx.subprocess(argv1, stdout=PIPE,
kill_delay=0.1) as proc1:
assert b'FOO\n' == await proc1.stdout.readline()
with ctx.tjob.subprocess_xevents(argv2, result='kill'):
async with ctx.subprocess(argv2,
stdout=PIPE,
kill_delay=0.1) as proc2:
assert b'BAR\n' == await proc2.stdout.readline()
# Both subprocesses will now ignore SIGTERM and we can
# proceed with cancelling.
async with abort_in(0.1):
await proc2.wait()
await proc1.wait()
todo = [TJob('foo', coro=coro)]
with assert_elapsed_time(lambda t: t < 1):
done = await run(todo)
assert verify_tasks(done, {'foo': Cancelled})
async def test_abort_many_spawned_jobs_returns_immediately(num_jobs, run):
todo = [
TJob(
'foo',
spawn=[TJob(f'bar #{i}', async_sleep=5) for i in range(100)],
await_spawn=True,
)
]
with assert_elapsed_time(lambda t: t < 1):
done = await run(todo, abort_after=0.1)
expect = {f'bar #{i}': Cancelled for i in range(100)}
expect['foo'] = Cancelled
assert verify_tasks(done, expect)
async def test_non_terminating_subprocess_is_killed(run):
argv = mock_argv('ignore:SIGTERM', 'FOO', 'sleep:5')
async def coro(ctx):
with ctx.tjob.subprocess_xevents(argv, result='kill'):
async with ctx.subprocess(argv, stdout=PIPE,
kill_delay=0.1) as proc:
output = await proc.stdout.readline()
# MISSING await proc.wait() here to trigger termination
return output
todo = [TJob('foo', coro=coro)]
with assert_elapsed_time(lambda t: t < 0.5):
done = await run(todo)
assert verify_tasks(done, {'foo': b'FOO\n'})
async def test_decorated_output_from_aborted_processes(num_workers, run,
verify_output):
todo = [
TJob(name, mode='mock_argv', extras=['sleep:5'])
for name in ['foo', 'bar', 'baz']
]
with assert_elapsed_time(lambda t: t < 0.75):
await run(todo, abort_after=0.5)
# We have 3 jobs, but can only run as many concurrently as there are
# workers available. The rest will be cancelled before they start.
assert verify_output(
[job.xout() for job in todo][:num_workers],
[], # No stderr output as this happens _after_ the aborted sleep
)
async def test_abort_one_non_terminating_job_teminates_then_kills(run):
argv = mock_argv('ignore:SIGTERM', 'ignore:SIGINT', 'FOO', 'sleep:5')
async def coro(ctx):
with ctx.tjob.subprocess_xevents(argv, result='kill'):
async with ctx.subprocess(argv, stdout=PIPE,
kill_delay=0.1) as proc:
assert b'FOO\n' == await proc.stdout.readline()
# Subprocess will now ignore SIGTERM when we are cancelled
async with abort_in(0.1):
await proc.wait()
todo = [TJob('foo', coro=coro)]
with assert_elapsed_time(lambda t: t < 2):
done = await run(todo)
assert verify_tasks(done, {'foo': Cancelled})
async def test_abort_job_with_two_subprocs_terminates_both(run, num_workers):
if num_workers < 2:
pytest.skip('need Scheduler with at least 2 workers')
argv = mock_argv('sleep:5')
async def coro(ctx):
with ctx.tjob.subprocess_xevents(argv, result='terminate'):
async with ctx.subprocess(argv) as proc1:
with ctx.tjob.subprocess_xevents(argv, result='terminate'):
async with ctx.subprocess(argv) as proc2:
await proc2.wait()
await proc1.wait()
todo = [TJob('foo', coro=coro)]
with assert_elapsed_time(lambda t: t < 1):
done = await run(todo, abort_after=0.2)
assert verify_tasks(done, {'foo': Cancelled})
async def test_abort_many_jobs_in_threads_cannot_return_soon(num_jobs, run):
todo = [
TJob(f'foo #{i}', thread=lambda ctx: time.sleep(0.3))
for i in range(num_jobs)
]
with assert_elapsed_time(lambda t: t > 0.3): # must wait for all threads
done = await run(todo, abort_after=0.1)
# In some scenarios it seems a few jobs manage to successfully complete
# before we get around to aborting the rest.
def cancelled_xor_successful(task):
try:
if task.result() is None:
return True
except asyncio.CancelledError:
return True
except BaseException:
pass
return False
assert verify_tasks(
done, {f'foo #{i}': cancelled_xor_successful
for i in range(num_jobs)})
async def test_abort_one_spawned_job_returns_immediately(run):
todo = [TJob('foo', spawn=[TJob('bar', async_sleep=5)], await_spawn=True)]
with assert_elapsed_time(lambda t: t < 1):
done = await run(todo, abort_after=0.1)
assert verify_tasks(done, {'foo': Cancelled, 'bar': Cancelled})
async def test_abort_one_job_in_subproc_returns_immediately(run):
todo = [TJob('foo', argv=mock_argv('sleep:5'))]
with assert_elapsed_time(lambda t: t < 1):
done = await run(todo, abort_after=0.1)
assert verify_tasks(done, {'foo': Cancelled})
async def test_abort_one_job_in_thread_cannot_return_immediately(run):
todo = [TJob('foo', thread=lambda ctx: time.sleep(0.2))]
with assert_elapsed_time(lambda t: t > 0.2): # must wait for thread
done = await run(todo, abort_after=0.1)
assert verify_tasks(done, {'foo': Cancelled})
async def test_return_before_abort(run):
todo = [TJob('foo', async_sleep=0.1)]
with assert_elapsed_time(lambda t: t < 0.4):
done = await run(todo, abort_after=0.5)
assert verify_tasks(done, {'foo': 'foo done'})