def _submit_experiment(
config: Optional[Dict[str, Any]],
context_dir: str,
command: Optional[List[str]],
test: bool = False,
master_url: Optional[str] = None,
) -> int:
if context_dir == "":
raise errors.InvalidExperimentException("Cannot specify the context directory to be empty.")
context_path = pathlib.Path(context_dir)
config = {**constants.DEFAULT_EXP_CFG, **(config or {})}
config.setdefault("internal", {})
config["internal"]["native"] = {"command": _set_command_default(context_path, command)}
logging.info(f"Creating an experiment with config: {config}")
if master_url is None:
master_url = util.get_default_master_address()
exp_context = context.Context.from_local(context_path)
# When a requested_user isn't specified to initialize_session(), the
# authentication module will attempt to use the token store to grab the
# current logged-in user. If there is no logged in user found, it will
# default to constants.DEFAULT_DETERMINED_USER.
auth.initialize_session(master_url, requested_user=None, try_reauth=True)
if test:
return api.create_test_experiment_and_follow_logs(master_url, config, exp_context)
else:
return api.create_experiment_and_follow_logs(master_url, config, exp_context)
def test_streaming_metrics_api() -> None:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
pool = mp.pool.ThreadPool(processes=7)
experiment_id = exp.create_experiment(
conf.fixtures_path("mnist_pytorch/adaptive_short.yaml"),
conf.tutorials_path("mnist_pytorch"),
)
# To fully test the streaming APIs, the requests need to start running immediately after the
# experiment, and then stay open until the experiment is complete. To accomplish this with all
# of the API calls on a single experiment, we spawn them all in threads.
# The HP importance portion of this test is commented out until the feature is enabled by
# default
metric_names_thread = pool.apply_async(request_metric_names,
(experiment_id, ))
train_metric_batches_thread = pool.apply_async(
request_train_metric_batches, (experiment_id, ))
valid_metric_batches_thread = pool.apply_async(
request_valid_metric_batches, (experiment_id, ))
train_trials_snapshot_thread = pool.apply_async(
request_train_trials_snapshot, (experiment_id, ))
valid_trials_snapshot_thread = pool.apply_async(
request_valid_trials_snapshot, (experiment_id, ))
train_trials_sample_thread = pool.apply_async(request_train_trials_sample,
(experiment_id, ))
valid_trials_sample_thread = pool.apply_async(request_valid_trials_sample,
(experiment_id, ))
metric_names_results = metric_names_thread.get()
train_metric_batches_results = train_metric_batches_thread.get()
valid_metric_batches_results = valid_metric_batches_thread.get()
train_trials_snapshot_results = train_trials_snapshot_thread.get()
valid_trials_snapshot_results = valid_trials_snapshot_thread.get()
train_trials_sample_results = train_trials_sample_thread.get()
valid_trials_sample_results = valid_trials_sample_thread.get()
if metric_names_results is not None:
pytest.fail("metric-names: %s. Results: %s" % metric_names_results)
if train_metric_batches_results is not None:
pytest.fail("metric-batches (training): %s. Results: %s" %
train_metric_batches_results)
if valid_metric_batches_results is not None:
pytest.fail("metric-batches (validation): %s. Results: %s" %
valid_metric_batches_results)
if train_trials_snapshot_results is not None:
pytest.fail("trials-snapshot (training): %s. Results: %s" %
train_trials_snapshot_results)
if valid_trials_snapshot_results is not None:
pytest.fail("trials-snapshot (validation): %s. Results: %s" %
valid_trials_snapshot_results)
if train_trials_sample_results is not None:
pytest.fail("trials-sample (training): %s. Results: %s" %
train_trials_sample_results)
if valid_trials_sample_results is not None:
pytest.fail("trials-sample (validation): %s. Results: %s" %
valid_trials_sample_results)
def get_num_running_commands() -> int:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "commands")
assert r.status_code == requests.codes.ok, r.text
return len([
command for _id, command in r.json().items()
if command["state"] == "RUNNING"
])
def change_experiment_state(experiment_id: int, new_state: str) -> None:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.patch(
conf.make_master_url(),
"experiments/{}".format(experiment_id),
headers={"Content-Type": "application/merge-patch+json"},
body={"state": new_state},
)
assert r.status_code == requests.codes.no_content, r.text
def cluster_slots() -> Dict[str, Any]:
"""
cluster_slots returns a dict of slots that each agent has.
:return: Dict[AgentID, List[Slot]]
"""
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "agents")
assert r.status_code == requests.codes.ok, r.text
json = r.json() # type: Dict[str, Any]
return {agent["id"]: agent["slots"].values() for agent in json.values()}
def test_hp_importance_api() -> None:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
pool = mp.pool.ThreadPool(processes=1)
experiment_id = exp.create_experiment(
conf.fixtures_path("mnist_pytorch/random.yaml"),
conf.tutorials_path("mnist_pytorch"),
)
hp_importance_thread = pool.apply_async(request_hp_importance, (experiment_id,))
hp_importance_results = hp_importance_thread.get()
if hp_importance_results is not None:
pytest.fail("hyperparameter-importance: %s. Results: %s" % hp_importance_results)
def get_command(id: str) -> Dict[str, Any]:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "commands")
assert r.status_code == requests.codes.ok, r.text
return cast(Dict[str, Any], r.json()["/commands/" + id])
def __init__(self, master: Optional[str], user: Optional[str]):
self._master = master or util.get_default_master_address()
self._user = user
auth.initialize_session(self._master, self._user, try_reauth=True)
def trial_logs(trial_id: int) -> List[str]:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
return [tl["message"] for tl in api.trial_logs(conf.make_master_url(), trial_id)]
def trial_metrics(trial_id: int) -> Dict[str, Any]:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "trials/{}/metrics".format(trial_id))
assert r.status_code == requests.codes.ok, r.text
json = r.json() # type: Dict[str, Any]
return json
def num_experiments() -> int:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "experiments")
assert r.status_code == requests.codes.ok, r.text
return len(r.json())
def experiment_json(experiment_id: int) -> Dict[str, Any]:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "experiments/{}".format(experiment_id))
assert r.status_code == requests.codes.ok, r.text
json = r.json() # type: Dict[str, Any]
return json
def f(namespace: Namespace) -> Any:
v = vars(namespace)
auth.initialize_session(namespace.master,
v.get("user"),
try_reauth=True)
return func(namespace)
def get_command(id: str) -> Any:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "api/v1/commands/" + id)
assert r.status_code == requests.codes.ok, r.text
return r.json()["command"]
