def stop(self) -> None:
"""
Stop background experiment.
"""
_logger.info('Stopping experiment, please wait...')
atexit.unregister(self.stop)
if self.id is not None:
nni.runtime.log.stop_experiment_log(self.id)
if self._proc is not None:
try:
# this if is to deal with the situation that
# nnimanager is cleaned up by ctrl+c first
if self._proc.poll() is None:
rest.delete(self.port, '/experiment')
except Exception as e:
_logger.exception(e)
_logger.warning(
'Cannot gracefully stop experiment, killing NNI process...'
)
kill_command(self._proc.pid)
if self._pipe is not None:
self._pipe.close()
if self._dispatcher_thread is not None:
self._dispatcher.stopping = True
self._dispatcher_thread.join(timeout=1)
self.id = None
self.port = None
self._proc = None
self._pipe = None
self._dispatcher = None
self._dispatcher_thread = None
_logger.info('Experiment stopped')
def test_detect_process(self):
if sys.platform == 'win32':
cmds = ['timeout', '360000']
else:
cmds = ['sleep', '360000']
process = Popen(cmds, stdout=PIPE, stderr=STDOUT)
self.assertTrue(detect_process(process.pid))
kill_command(process.pid)
def test_kill_process_slow_patiently():
process = subprocess.Popen([sys.executable, __file__, '--mode', 'kill_slow'])
time.sleep(1) # wait 1 second for the process to launch and register hooks
start_time = time.time()
kill_command(process.pid, timeout=3) # wait long enough
end_time = time.time()
assert end_time - start_time < 5
if sys.platform == 'linux':
assert end_time - start_time > 1 # I don't know why windows is super fast
def test_kill_process():
process = multiprocessing.Process(target=process_normal)
process.start()
time.sleep(0.5)
start_time = time.time()
kill_command(process.pid)
end_time = time.time()
assert not _check_pid_running(process.pid)
assert end_time - start_time < 2
def test_kill_process_slow_no_patience():
process = subprocess.Popen([sys.executable, __file__, '--mode', 'kill_slow'])
time.sleep(1) # wait 1 second for the process to launch and register hooks
start_time = time.time()
kill_command(process.pid, timeout=1) # didn't wait long enough
end_time = time.time()
if sys.platform == 'linux': # FIXME: on non-linux, seems that the time of termination can't be controlled
assert 0.5 < end_time - start_time < 2
assert process.poll() is None
assert _check_pid_running(process.pid)
else:
assert end_time - start_time < 2
# Wait more seconds and it will exit eventually
for _ in range(20):
time.sleep(1)
if not _check_pid_running(process.pid):
return
def stop_mock_experiment():
config = Config('config')
kill_command(config.get_config('restServerPid'))
nnictl_experiment_config = Experiments()
nnictl_experiment_config.remove_experiment('xOpEwA5w')
def stop_mock_experiment():
nnictl_experiment_config = Experiments()
experiments_dict = nnictl_experiment_config.get_all_experiments()
kill_command(experiments_dict['xOpEwA5w'].get('pid'))
nnictl_experiment_config = Experiments()
nnictl_experiment_config.remove_experiment('xOpEwA5w')
def process_patiently_kill():
process = subprocess.Popen([sys.executable, __file__, '--mode', 'kill_very_slow'])
time.sleep(1)
kill_command(process.pid) # wait long enough
def test_detect_process(self):
cmds = ['sleep', '360000']
process = Popen(cmds, stdout=PIPE, stderr=STDOUT)
self.assertTrue(detect_process(process.pid))
kill_command(process.pid)