def test_transition_to_running(self, supervisor, worker_thread):
cmd = commands.ResumeCmd()
supervisor.send_command(cmd)
add_work = commands.SetMaxNumIterations(2).awaitable
supervisor.send_command(add_work)
add_work.wait()
assert supervisor.state == State.Running
def test_finished_training_should_transition_to_paused(self, supervisor, worker_thread, exemplum):
cmd = commands.ResumeCmd()
supervisor.send_command(cmd)
add_work = commands.SetMaxNumIterations(2).awaitable
supervisor.send_command(add_work)
add_work.wait()
assert supervisor.state == State.Running
time.sleep(0.1) # FIXME: Find a better way to wait for pause event with timeout
assert supervisor.state == State.Idle
def test_exception_during_train_should_transition_to_paused(
self, supervisor, worker_thread, exemplum):
train_called = threading.Event()
def _exc():
train_called.set()
raise Exception()
exemplum.train = _exc
cmd = commands.ResumeCmd()
supervisor.send_command(cmd)
add_work = commands.SetMaxNumIterations(2).awaitable
supervisor.send_command(add_work)
add_work.wait()
train_called.wait()
assert supervisor.state == State.Running
time.sleep(
0.2
) # FIXME: Find a better way to wait for pause event with timeout
assert supervisor.state == State.Paused
def test_exception_during_train_should_transition_to_paused(self, supervisor, worker_thread, exemplum):
train_called = threading.Event()
train_proceed = threading.Event()
def _exc():
train_called.set()
train_proceed.wait()
raise Exception()
exemplum.train = _exc
cmd = commands.ResumeCmd()
supervisor.send_command(cmd)
assert supervisor.state == State.Paused
add_work = commands.SetMaxNumIterations(2).awaitable
supervisor.send_command(add_work)
add_work.wait()
train_called.wait()
wait(lambda: supervisor.state == State.Running, max_wait=1)
train_proceed.set()
wait(lambda: supervisor.state == State.Paused, max_wait=1)
def set_max_num_iterations(self, num: int) -> None:
self._supervisor.send_command(commands.SetMaxNumIterations(num))