def test_exception_catching(self):
"""Test that the thread exits cleanly when __run() raises."""
def new_run(*args):
raise TypeError("foobar")
error_event = threading.Event()
stop_event = threading.Event()
stop_event.set()
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
attach_futures = [
FluxExecutorFuture(threading.get_ident()) for _ in range(5)
]
running_futures = set(
[FluxExecutorFuture(threading.get_ident()) for _ in range(5)])
deque = collections.deque(
[_SubmitPackage((None, ), {}, fut) for fut in futures])
deque.extend([_AttachPackage(None, fut) for fut in attach_futures])
thread = _FluxExecutorThread(error_event, stop_event, deque, 0.01, (),
{})
thread._FluxExecutorThread__running_user_futures.update(
running_futures)
# replace the __run method
thread._FluxExecutorThread__run = types.MethodType(new_run, thread)
with self.assertRaisesRegex(TypeError, "foobar"):
thread.run()
self.assertTrue(error_event.is_set())
for fut in list(futures) + list(running_futures) + list(
attach_futures):
with self.assertRaisesRegex(TypeError, "foobar"):
fut.result()
def test_finish_completion(self):
thread = _FluxExecutorThread(
threading.Event(), collections.deque(), 0.01, (), {}
)
for exit_status in (0, 1, 15, 120, 255):
flag = threading.Event()
fut = FluxExecutorFuture(threading.get_ident()).add_done_callback(
lambda fut: flag.set()
)
thread._FluxExecutorThread__event_update(
ShamJobEventWatchFuture(
EventLogEvent(
{
"name": "finish",
"timestamp": 0,
"context": {"status": exit_status},
}
)
),
fut,
)
self.assertTrue(fut.done())
self.assertTrue(flag.is_set())
if os.WIFEXITED(exit_status):
self.assertEqual(fut.result(), os.WEXITSTATUS(exit_status))
elif os.WIFSIGNALED(exit_status):
self.assertEqual(fut.result(), -os.WTERMSIG(exit_status))
def test_exception_completion(self):
jobspec = JobspecV1.from_command(["false"])
thread = _FluxExecutorThread(threading.Event(), threading.Event(),
collections.deque(), 0.01, (), {})
fut = FluxExecutorFuture(threading.get_ident())
self.assertFalse(fut.done())
fut._set_event(EventLogEvent({"name": "start", "timestamp": 0}))
self.assertFalse(fut.done())
thread._FluxExecutorThread__event_update(
ShamJobEventWatchFuture(
EventLogEvent({
"name": "exception",
"timestamp": 0,
"context": {
"severity": 1,
"type": "foobar"
},
})),
fut,
)
self.assertFalse(fut.done())
thread._FluxExecutorThread__event_update(
ShamJobEventWatchFuture(
EventLogEvent({
"name": "exception",
"timestamp": 0,
"context": {
"severity": 0,
"type": "foobar"
},
})),
fut,
)
self.assertTrue(fut.done())
self.assertIsInstance(fut.exception(), JobException)
def test_exit_condition(self):
deq = collections.deque()
event = threading.Event()
thread = _FluxExecutorThread(event, deq, 0.01, (), {})
self.assertTrue(thread._FluxExecutorThread__work_remains())
event.set()
self.assertFalse(thread._FluxExecutorThread__work_remains())
deq.append(None)
self.assertTrue(thread._FluxExecutorThread__work_remains())
def test_bad_jobspecs(self):
deq = collections.deque()
event = threading.Event()
thread = _FluxExecutorThread(event, deq, 0.01, (), {})
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
deq.extend(((None,), {}, f) for f in futures) # send jobspec of None
event.set()
thread.run()
self.assertFalse(deq)
self.assertEqual(0, thread._FluxExecutorThread__remaining_flux_futures)
for fut in futures:
self.assertIsInstance(fut.exception(), OSError)
def test_bad_submit_arguments(self):
"""send bad arguments to ``flux.job.submit``"""
deq = collections.deque()
event = threading.Event()
thread = _FluxExecutorThread(event, deq, 0.01, (), {})
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
jobspec = JobspecV1.from_command(["false"])
deq.extend(((jobspec,), {"not_an_arg": 42}, f) for f in futures)
event.set()
thread.run()
self.assertFalse(deq)
self.assertEqual(0, thread._FluxExecutorThread__remaining_flux_futures)
for fut in futures:
self.assertIsInstance(fut.exception(), TypeError)
def test_cancel(self):
deq = collections.deque()
event = threading.Event()
jobspec = JobspecV1.from_command(["false"])
thread = _FluxExecutorThread(event, deq, 0.01, (), {})
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
for fut in futures:
deq.append(((jobspec,), {}, fut))
fut.cancel()
event.set()
thread.run()
for fut in futures:
with self.assertRaises(cf.CancelledError):
fut.result()
with self.assertRaises(cf.CancelledError):
fut.jobid()
def test_exception_catching_reactor_run(self):
"""Test that the thread exits cleanly when reactor_run returns < 0"""
def new_reactor_run(*args, **kwargs):
# returning < 0 causes the thread to raise
return -1
jobspec = JobspecV1.from_command(["true"])
error_event = threading.Event()
stop_event = threading.Event()
stop_event.set()
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
deque = collections.deque(
[_SubmitPackage((jobspec, ), {}, fut) for fut in futures])
thread = _FluxExecutorThread(error_event, stop_event, deque, 0.01, (),
{})
# replace the reactor_run method of the thread's flux handle
thread._FluxExecutorThread__flux_handle.reactor_run = types.MethodType(
new_reactor_run, thread._FluxExecutorThread__flux_handle)
with self.assertRaisesRegex(RuntimeError, "reactor start failed"):
thread.run()
self.assertTrue(error_event.is_set())
for fut in list(futures):
with self.assertRaisesRegex(RuntimeError, "reactor start failed"):
fut.result()
def test_bad_attach_arguments(self):
deq = collections.deque()
event = threading.Event()
thread = _FluxExecutorThread(threading.Event(), event, deq, 0.01, (),
{})
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
deq.extend(_AttachPackage(None, f)
for f in futures) # send jobid of None
event.set()
thread.run()
self.assertFalse(deq)
self.assertFalse(thread._FluxExecutorThread__running_user_futures)
for fut in futures:
self.assertIsInstance(fut.exception(), TypeError)
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
deq.extend(_AttachPackage(0, f) for f in futures) # send jobid of None
event.set()
thread.run()
self.assertFalse(deq)
self.assertFalse(thread._FluxExecutorThread__running_user_futures)
for fut in futures:
with self.assertRaisesRegex(ValueError,
r".*does not match any job.*"):
fut.result(timeout=0)