def test_exception_catching(self):
"""Test that the thread exits cleanly when __run() raises."""
def new_run(*args):
raise TypeError("foobar")
error_event = threading.Event()
stop_event = threading.Event()
stop_event.set()
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
attach_futures = [
FluxExecutorFuture(threading.get_ident()) for _ in range(5)
]
running_futures = set(
[FluxExecutorFuture(threading.get_ident()) for _ in range(5)])
deque = collections.deque(
[_SubmitPackage((None, ), {}, fut) for fut in futures])
deque.extend([_AttachPackage(None, fut) for fut in attach_futures])
thread = _FluxExecutorThread(error_event, stop_event, deque, 0.01, (),
{})
thread._FluxExecutorThread__running_user_futures.update(
running_futures)
# replace the __run method
thread._FluxExecutorThread__run = types.MethodType(new_run, thread)
with self.assertRaisesRegex(TypeError, "foobar"):
thread.run()
self.assertTrue(error_event.is_set())
for fut in list(futures) + list(running_futures) + list(
attach_futures):
with self.assertRaisesRegex(TypeError, "foobar"):
fut.result()
def test_finish_completion(self):
thread = _FluxExecutorThread(
threading.Event(), collections.deque(), 0.01, (), {}
)
for exit_status in (0, 1, 15, 120, 255):
flag = threading.Event()
fut = FluxExecutorFuture(threading.get_ident()).add_done_callback(
lambda fut: flag.set()
)
thread._FluxExecutorThread__event_update(
ShamJobEventWatchFuture(
EventLogEvent(
{
"name": "finish",
"timestamp": 0,
"context": {"status": exit_status},
}
)
),
fut,
)
self.assertTrue(fut.done())
self.assertTrue(flag.is_set())
if os.WIFEXITED(exit_status):
self.assertEqual(fut.result(), os.WEXITSTATUS(exit_status))
elif os.WIFSIGNALED(exit_status):
self.assertEqual(fut.result(), -os.WTERMSIG(exit_status))
def test_exception_completion(self):
jobspec = JobspecV1.from_command(["false"])
thread = _FluxExecutorThread(threading.Event(), threading.Event(),
collections.deque(), 0.01, (), {})
fut = FluxExecutorFuture(threading.get_ident())
self.assertFalse(fut.done())
fut._set_event(EventLogEvent({"name": "start", "timestamp": 0}))
self.assertFalse(fut.done())
thread._FluxExecutorThread__event_update(
ShamJobEventWatchFuture(
EventLogEvent({
"name": "exception",
"timestamp": 0,
"context": {
"severity": 1,
"type": "foobar"
},
})),
fut,
)
self.assertFalse(fut.done())
thread._FluxExecutorThread__event_update(
ShamJobEventWatchFuture(
EventLogEvent({
"name": "exception",
"timestamp": 0,
"context": {
"severity": 0,
"type": "foobar"
},
})),
fut,
)
self.assertTrue(fut.done())
self.assertIsInstance(fut.exception(), JobException)
def test_set_jobid(self):
for jobid in (21, 594240, None, -1, "foobar"):
fut = FluxExecutorFuture(threading.get_ident())
with self.assertRaises(cf.TimeoutError):
fut.jobid(timeout=0)
fut._set_jobid(jobid)
self.assertEqual(jobid, fut.jobid(timeout=0))
with self.assertRaises(RuntimeError):
fut._set_jobid(jobid)
def test_multiple_events(self):
fut = FluxExecutorFuture(threading.get_ident())
counter = itertools.count()
fut.add_event_callback("urgency", lambda fut, event: next(counter))
total_iterations = 5
for _ in range(5):
fut._set_event(EventLogEvent({"name": "urgency", "timestamp": 0}))
self.assertEqual(next(counter), total_iterations)
new_counter = itertools.count()
fut.add_event_callback("urgency", lambda fut, event: next(new_counter))
self.assertEqual(next(new_counter), total_iterations)
fut._set_event(EventLogEvent({"name": "urgency", "timestamp": 0}))
# invoking the callback increments the counter, as does checking the counter
self.assertEqual(next(counter), total_iterations + 2)
self.assertEqual(next(new_counter), total_iterations + 2)
def test_multiple_cancel(self):
fut = FluxExecutorFuture(threading.get_ident())
invocations = itertools.count()
fut.add_done_callback(lambda _: next(invocations))
self.assertFalse(fut.cancelled())
for _ in range(3): # test cancelling more than once
self.assertTrue(fut.cancel())
self.assertEqual(next(invocations), 1)
with self.assertRaises(cf.CancelledError):
fut.jobid()
def test_jobid_callback(self):
fut = FluxExecutorFuture(threading.get_ident())
flag = threading.Event()
fut.add_jobid_callback(lambda f: flag.set())
self.assertFalse(flag.is_set())
fut._set_jobid(5)
self.assertTrue(flag.is_set())
# now check that adding callbacks fires them immediately
flag.clear()
fut.add_jobid_callback(lambda f: flag.set())
self.assertTrue(flag.is_set())
def test_adding_callbacks_from_callbacks(self):
fut = FluxExecutorFuture(threading.get_ident())
flag = threading.Event()
nested_callback = lambda fut, event: flag.set()
fut.add_event_callback(
"start", lambda fut, event: fut.add_event_callback(
"start", nested_callback))
fut._set_event(EventLogEvent({"name": "start", "timestamp": 0}))
self.assertTrue(flag.is_set())
def test_bad_jobspecs(self):
deq = collections.deque()
event = threading.Event()
thread = _FluxExecutorThread(event, deq, 0.01, (), {})
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
deq.extend(((None,), {}, f) for f in futures) # send jobspec of None
event.set()
thread.run()
self.assertFalse(deq)
self.assertEqual(0, thread._FluxExecutorThread__remaining_flux_futures)
for fut in futures:
self.assertIsInstance(fut.exception(), OSError)
def test_bad_event(self):
fut = FluxExecutorFuture(threading.get_ident())
event = "not an event"
log_event = EventLogEvent({"name": event, "timestamp": 0})
with self.assertRaises(ValueError):
fut._set_event(log_event)
with self.assertRaises(ValueError):
fut.add_event_callback(event, None)
def test_bad_submit_arguments(self):
"""send bad arguments to ``flux.job.submit``"""
deq = collections.deque()
event = threading.Event()
thread = _FluxExecutorThread(event, deq, 0.01, (), {})
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
jobspec = JobspecV1.from_command(["false"])
deq.extend(((jobspec,), {"not_an_arg": 42}, f) for f in futures)
event.set()
thread.run()
self.assertFalse(deq)
self.assertEqual(0, thread._FluxExecutorThread__remaining_flux_futures)
for fut in futures:
self.assertIsInstance(fut.exception(), TypeError)
def test_event_callback(self):
fut = FluxExecutorFuture(threading.get_ident())
flag = threading.Event()
for state in ("clean", "start", "alloc"):
flag.clear()
fut.add_event_callback(
state, lambda f, log:
(flag.set(), self.assertEqual(log.name, state)))
log_event = EventLogEvent({"name": state, "timestamp": 0})
fut._set_event(log_event)
self.assertTrue(flag.is_set())
# now check that adding callbacks fires them immediately
flag.clear()
fut.add_event_callback(
state, lambda f, log:
(flag.set(), self.assertEqual(log.name, state)))
self.assertTrue(flag.is_set())
def test_bad_attach_arguments(self):
deq = collections.deque()
event = threading.Event()
thread = _FluxExecutorThread(threading.Event(), event, deq, 0.01, (),
{})
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
deq.extend(_AttachPackage(None, f)
for f in futures) # send jobid of None
event.set()
thread.run()
self.assertFalse(deq)
self.assertFalse(thread._FluxExecutorThread__running_user_futures)
for fut in futures:
self.assertIsInstance(fut.exception(), TypeError)
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
deq.extend(_AttachPackage(0, f) for f in futures) # send jobid of None
event.set()
thread.run()
self.assertFalse(deq)
self.assertFalse(thread._FluxExecutorThread__running_user_futures)
for fut in futures:
with self.assertRaisesRegex(ValueError,
r".*does not match any job.*"):
fut.result(timeout=0)
def test_cancel(self):
deq = collections.deque()
event = threading.Event()
jobspec = JobspecV1.from_command(["false"])
thread = _FluxExecutorThread(event, deq, 0.01, (), {})
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
for fut in futures:
deq.append(((jobspec,), {}, fut))
fut.cancel()
event.set()
thread.run()
for fut in futures:
with self.assertRaises(cf.CancelledError):
fut.result()
with self.assertRaises(cf.CancelledError):
fut.jobid()
def test_exception_catching_reactor_run(self):
"""Test that the thread exits cleanly when reactor_run returns < 0"""
def new_reactor_run(*args, **kwargs):
# returning < 0 causes the thread to raise
return -1
jobspec = JobspecV1.from_command(["true"])
error_event = threading.Event()
stop_event = threading.Event()
stop_event.set()
futures = [FluxExecutorFuture(threading.get_ident()) for _ in range(5)]
deque = collections.deque(
[_SubmitPackage((jobspec, ), {}, fut) for fut in futures])
thread = _FluxExecutorThread(error_event, stop_event, deque, 0.01, (),
{})
# replace the reactor_run method of the thread's flux handle
thread._FluxExecutorThread__flux_handle.reactor_run = types.MethodType(
new_reactor_run, thread._FluxExecutorThread__flux_handle)
with self.assertRaisesRegex(RuntimeError, "reactor start failed"):
thread.run()
self.assertTrue(error_event.is_set())
for fut in list(futures):
with self.assertRaisesRegex(RuntimeError, "reactor start failed"):
fut.result()
def test_callback_deadlock(self):
flag = threading.Event()
# try waiting for the future from the callback
fut = FluxExecutorFuture(threading.get_ident()).add_jobid_callback(
lambda fut: (fut.result(), flag.set()))
fut._set_jobid(5)
# check that flag wasn't set---because the callback raised
self.assertFalse(flag.is_set())
# try the same with an event callback
log_event = EventLogEvent({"name": "debug", "timestamp": 0})
fut.add_event_callback(log_event.name, lambda fut, event:
(fut.result(), flag.set()))
fut._set_event(log_event)
self.assertFalse(flag.is_set())
# now complete the future and try again
fut.set_result(21)
fut.add_jobid_callback(lambda fut: (fut.result(), flag.set()))
self.assertTrue(flag.is_set())
flag.clear()
fut.add_event_callback(log_event.name, lambda fut, event:
(fut.result(), flag.set()))
self.assertTrue(flag.is_set())
def test_jobid_unavailable(self):
"""Tests for cases where fetching the jobid should raise an exception."""
fut = FluxExecutorFuture(threading.get_ident())
fut._set_jobid(None, exc=OverflowError("foobar"))
with self.assertRaisesRegex(OverflowError, "foobar"):
fut.jobid(0)
with self.assertRaises(RuntimeError):
fut._set_jobid(1)
# test that jobid raises if future is cancelled
fut = FluxExecutorFuture(threading.get_ident())
fut.cancel()
with self.assertRaises(cf.CancelledError):
fut.jobid(0)
# test that jobid doesn't raise if future is cancelled but jobid already set
fut = FluxExecutorFuture(threading.get_ident())
jobid = 10
fut._set_jobid(jobid)
fut.cancel()
self.assertEqual(fut.jobid(0), jobid)
# test that jobid raises if an exception is set before jobid set
fut = FluxExecutorFuture(threading.get_ident())
fut.set_exception(ValueError("foobar"))
with self.assertRaisesRegex(RuntimeError,
r"job could not be submitted.*"):
fut.jobid()
