def test_check_experiment_status_tuner_no_more_trial():
content = "TUNER_NO_MORE_TRIAL"
error = ""
with patch(
"requests.get", side_effect=lambda url: mocked_status_get(url, content, error)
):
check_experiment_status(wait=0.1, max_retries=1)
def test_check_experiment_status_done():
content = "DONE"
error = ""
with patch(
"requests.get", side_effect=lambda url: mocked_status_get(url, content, error)
):
check_experiment_status(wait=0.1, max_retries=1)
def test_check_experiment_status_no_more_trial():
content = "NO_MORE_TRIAL"
error = ""
with pytest.raises(TimeoutError) as excinfo:
with patch("requests.get",
side_effect=lambda url: mocked_status_get(
url, content, error)):
check_experiment_status(wait=0.1, max_retries=1)
assert "check_experiment_status() timed out" == str(excinfo.value)
def test_check_experiment_status_running():
content = "RUNNING"
error = ""
with pytest.raises(TimeoutError) as excinfo:
with patch(
"requests.get",
side_effect=lambda url: mocked_status_get(url, content, error),
):
check_experiment_status(wait=0.1, max_retries=1)
assert "check_experiment_status() timed out" == str(excinfo.value)
def test_check_experiment_status_failed():
content = "some_failed_status"
error = "NNI_ERROR"
with pytest.raises(RuntimeError) as excinfo:
with patch("requests.get",
side_effect=lambda url: mocked_status_get(
url, content, error)):
check_experiment_status(wait=0.1, max_retries=1)
assert "NNI experiment failed to complete with status some_failed_status - NNI_ERROR" == str(
excinfo.value)
### 4. Execute NNI Trials
# Make sure that there is no experiment running
stop_nni() #it couldn't find the specified file so i just removed it :)
config_path = os.path.join(TMP_DIR, 'config_gru.yml')
nni_env = os.environ.copy()
nni_env['PATH'] = ':' + nni_env['PATH']
proc = subprocess.run(['nnictl', 'create', '--config', config_path], env=nni_env)
if proc.returncode != 0:
raise RuntimeError("'nnictl create' failed with code %d" % proc.returncode)
with Timer() as time_tpe:
check_experiment_status(wait=WAITING_TIME, max_retries=MAX_RETRIES)
"""### 5. Show Results
The trial with the best metric and the corresponding metrics and hyperparameters can also be read from the Web UI
or from the JSON file created by the training script. Below, we do this programmatically using [nni_utils.py](../../reco_utils/nni/nni_utils.py)
"""
trials, best_metrics, best_params, best_trial_path = get_trials('maximize')
# best_metrics
# best_params
