def test_resuming_transitions_to_idle_with_no_devices(
self, supervisor, worker_thread):
cmd = commands.ResumeCmd().awaitable
supervisor.send_command(cmd)
cmd.wait()
assert State.Idle == supervisor.state
def test_transition_to_running(self, supervisor, worker_thread):
cmd = commands.ResumeCmd()
supervisor.send_command(cmd)
add_work = commands.SetMaxNumIterations(2).awaitable
supervisor.send_command(add_work)
add_work.wait()
assert supervisor.state == State.Running
def test_stop_command_has_higher_priorityj(self):
cmd_queue = cmds.CommandPriorityQueue()
stop_cmd = cmds.StopCmd()
cmd_queue.put_nowait(cmds.ResumeCmd())
cmd_queue.put_nowait(stop_cmd)
cmd_queue.put_nowait(cmds.PauseCmd())
received_cmd = cmd_queue.get_nowait()
assert stop_cmd is received_cmd
def test_finished_training_should_transition_to_paused(self, supervisor, worker_thread, exemplum):
cmd = commands.ResumeCmd()
supervisor.send_command(cmd)
add_work = commands.SetMaxNumIterations(2).awaitable
supervisor.send_command(add_work)
add_work.wait()
assert supervisor.state == State.Running
time.sleep(0.1) # FIXME: Find a better way to wait for pause event with timeout
assert supervisor.state == State.Idle
def test_exception_during_train_should_transition_to_paused(
self, supervisor, worker_thread, exemplum):
train_called = threading.Event()
def _exc():
train_called.set()
raise Exception()
exemplum.train = _exc
cmd = commands.ResumeCmd()
supervisor.send_command(cmd)
add_work = commands.SetMaxNumIterations(2).awaitable
supervisor.send_command(add_work)
add_work.wait()
train_called.wait()
assert supervisor.state == State.Running
time.sleep(
0.2
) # FIXME: Find a better way to wait for pause event with timeout
assert supervisor.state == State.Paused
def test_exception_during_train_should_transition_to_paused(self, supervisor, worker_thread, exemplum):
train_called = threading.Event()
train_proceed = threading.Event()
def _exc():
train_called.set()
train_proceed.wait()
raise Exception()
exemplum.train = _exc
cmd = commands.ResumeCmd()
supervisor.send_command(cmd)
assert supervisor.state == State.Paused
add_work = commands.SetMaxNumIterations(2).awaitable
supervisor.send_command(add_work)
add_work.wait()
train_called.wait()
wait(lambda: supervisor.state == State.Running, max_wait=1)
train_proceed.set()
wait(lambda: supervisor.state == State.Paused, max_wait=1)
def resume_training(self) -> None:
resume_cmd = commands.ResumeCmd()
self._supervisor.send_command(resume_cmd.awaitable)
resume_cmd.awaitable.wait()