def f(namespace: Namespace) -> Any:
v = vars(namespace)
try:
auth.initialize_session(namespace.master, v.get("user"), try_reauth=False)
except api.errors.UnauthenticatedException:
pass
return func(namespace)
def change_experiment_state(experiment_id: int, new_state: str) -> None:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.patch(
conf.make_master_url(),
"experiments/{}".format(experiment_id),
headers={"Content-Type": "application/merge-patch+json"},
body={"state": new_state},
)
assert r.status_code == requests.codes.no_content, r.text
def experiment_id_completer(prefix: str, parsed_args: Namespace,
**kwargs: Any) -> List[str]:
auth.initialize_session(parsed_args.master,
parsed_args.user,
try_reauth=True)
q = api.GraphQLQuery(parsed_args.master)
q.op.experiments().id()
resp = q.send()
return [str(e["id"]) for e in resp.experiments]
def get_num_running_commands() -> int:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "commands")
assert r.status_code == requests.codes.ok, r.text
return len([
command for _id, command in r.json().items()
if command["state"] == "RUNNING"
])
def test_streaming_metrics_api() -> None:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
pool = mp.pool.ThreadPool(processes=7)
experiment_id = exp.create_experiment(
conf.fixtures_path("mnist_pytorch/adaptive_short.yaml"),
conf.tutorials_path("mnist_pytorch"),
)
# To fully test the streaming APIs, the requests need to start running immediately after the
# experiment, and then stay open until the experiment is complete. To accomplish this with all
# of the API calls on a single experiment, we spawn them all in threads.
metric_names_thread = pool.apply_async(request_metric_names,
(experiment_id, ))
train_metric_batches_thread = pool.apply_async(
request_train_metric_batches, (experiment_id, ))
valid_metric_batches_thread = pool.apply_async(
request_valid_metric_batches, (experiment_id, ))
train_trials_snapshot_thread = pool.apply_async(
request_train_trials_snapshot, (experiment_id, ))
valid_trials_snapshot_thread = pool.apply_async(
request_valid_trials_snapshot, (experiment_id, ))
train_trials_sample_thread = pool.apply_async(request_train_trials_sample,
(experiment_id, ))
valid_trials_sample_thread = pool.apply_async(request_valid_trials_sample,
(experiment_id, ))
metric_names_results = metric_names_thread.get()
train_metric_batches_results = train_metric_batches_thread.get()
valid_metric_batches_results = valid_metric_batches_thread.get()
train_trials_snapshot_results = train_trials_snapshot_thread.get()
valid_trials_snapshot_results = valid_trials_snapshot_thread.get()
train_trials_sample_results = train_trials_sample_thread.get()
valid_trials_sample_results = valid_trials_sample_thread.get()
if metric_names_results is not None:
pytest.fail("metric-names: %s. Results: %s" % metric_names_results)
if train_metric_batches_results is not None:
pytest.fail("metric-batches (training): %s. Results: %s" %
train_metric_batches_results)
if valid_metric_batches_results is not None:
pytest.fail("metric-batches (validation): %s. Results: %s" %
valid_metric_batches_results)
if train_trials_snapshot_results is not None:
pytest.fail("trials-snapshot (training): %s. Results: %s" %
train_trials_snapshot_results)
if valid_trials_snapshot_results is not None:
pytest.fail("trials-snapshot (validation): %s. Results: %s" %
valid_trials_snapshot_results)
if train_trials_sample_results is not None:
pytest.fail("trials-sample (training): %s. Results: %s" %
train_trials_sample_results)
if valid_trials_sample_results is not None:
pytest.fail("trials-sample (validation): %s. Results: %s" %
valid_trials_sample_results)
def cluster_slots() -> Dict[str, Any]:
"""
cluster_slots returns a dict of slots that each agent has.
:return: Dict[AgentID, List[Slot]]
"""
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "agents")
assert r.status_code == requests.codes.ok, r.text
json = r.json() # type: Dict[str, Any]
return {agent["id"]: agent["slots"].values() for agent in json.values()}
def create_experiment(
config: Optional[Dict[str, Any]],
context_dir: str,
command: Optional[List[str]],
test_mode: bool = False,
master_url: Optional[str] = None,
) -> Optional[int]:
"""Submit an experiment to the Determined master.
Alternatively, use det.create() with a mode argument of "submit".
Args:
name (Optional[str]): The URL of the Determined master node. If None
(default), then the master address will be inferred from the
environment.
Returns:
The ID of the created experiment.
"""
if context_dir == "":
raise errors.InvalidExperimentException(
"Cannot specify the context directory to be empty.")
context_path = pathlib.Path(context_dir)
config = {**constants.DEFAULT_EXP_CFG, **(config or {})}
config.setdefault("internal", {})
config["internal"]["native"] = {
"command": set_command_default(context_path, command)
}
print("Creating an experiment with config: {}".format(config))
if master_url is None:
master_url = util.get_default_master_address()
exp_context = context.Context.from_local(context_path)
# When a requested_user isn't specified to initialize_session(), the
# authentication module will attempt to use the token store to grab the
# current logged-in user. If there is no logged in user found, it will
# default to constants.DEFAULT_DETERMINED_USER.
auth.initialize_session(master_url, requested_user=None, try_reauth=True)
if test_mode:
exp_id = api.create_test_experiment(master_url, config, exp_context)
else:
exp_id = api.create_experiment(master_url, config, exp_context)
print("Created experiment {}".format(exp_id))
return exp_id
def test_hp_importance_api() -> None:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
pool = mp.pool.ThreadPool(processes=1)
experiment_id = exp.create_experiment(
conf.fixtures_path("mnist_pytorch/random.yaml"),
conf.tutorials_path("mnist_pytorch"),
)
hp_importance_thread = pool.apply_async(request_hp_importance, (experiment_id,))
hp_importance_results = hp_importance_thread.get()
if hp_importance_results is not None:
pytest.fail("hyperparameter-importance: %s. Results: %s" % hp_importance_results)
def _submit_experiment(
config: Optional[Dict[str, Any]],
context_dir: str,
command: Optional[List[str]],
test: bool = False,
master_url: Optional[str] = None,
) -> Optional[int]:
if context_dir == "":
raise errors.InvalidExperimentException(
"Cannot specify the context directory to be empty.")
context_path = pathlib.Path(context_dir)
config = {**constants.DEFAULT_EXP_CFG, **(config or {})}
config.setdefault("internal", {})
config["internal"]["native"] = {
"command": _set_command_default(context_path, command)
}
logging.info(f"Creating an experiment with config: {config}")
if master_url is None:
master_url = util.get_default_master_address()
exp_context = context.Context.from_local(context_path)
# When a requested_user isn't specified to initialize_session(), the
# authentication module will attempt to use the token store to grab the
# current logged-in user. If there is no logged in user found, it will
# default to constants.DEFAULT_DETERMINED_USER.
auth.initialize_session(master_url, requested_user=None, try_reauth=True)
if test:
print(colored("Validating experiment configuration...", "yellow"),
end="\r")
api.create_experiment(master_url, config, exp_context, None, True)
print(
colored("Experiment configuration validation succeeded! 🎉",
"green"))
exp_id = api.create_test_experiment(master_url, config, exp_context)
print(colored("Test experiment ID: {}".format(exp_id), "green"))
api.follow_test_experiment_logs(master_url, exp_id)
else:
exp_id = api.create_experiment(master_url, config, exp_context)
logging.info(f"Created experiment {exp_id}")
api.follow_experiment_logs(master_url, exp_id)
return exp_id
def trial_logs(trial_id: int) -> List[str]:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "trials/{}/logs".format(trial_id))
assert r.status_code == requests.codes.ok, r.text
return [t["message"] for t in r.json()]
def trial_metrics(trial_id: int) -> Dict[str, Any]:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "trials/{}/metrics".format(trial_id))
assert r.status_code == requests.codes.ok, r.text
json = r.json() # type: Dict[str, Any]
return json
def num_experiments() -> int:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "experiments")
assert r.status_code == requests.codes.ok, r.text
return len(r.json())
def experiment_json(experiment_id: int) -> Dict[str, Any]:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "experiments/{}".format(experiment_id))
assert r.status_code == requests.codes.ok, r.text
json = r.json() # type: Dict[str, Any]
return json
def query() -> api.GraphQLQuery:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
return api.GraphQLQuery(conf.make_master_url())
def trial_logs(trial_id: int) -> List[str]:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
return [
tl["message"]
for tl in api.trial_logs(conf.make_master_url(), trial_id)
]
def f(namespace: Namespace) -> Any:
v = vars(namespace)
auth.initialize_session(namespace.master, v.get("user"), try_reauth=True)
return func(namespace)
def get_command(id: str) -> Dict[str, Any]:
auth.initialize_session(conf.make_master_url(), try_reauth=True)
r = api.get(conf.make_master_url(), "commands")
assert r.status_code == requests.codes.ok, r.text
return cast(Dict[str, Any], r.json()["/commands/" + id])
def __init__(self, master: Optional[str], user: Optional[str]):
self._master = master or util.get_default_master_address()
self._user = user
auth.initialize_session(self._master, self._user, try_reauth=True)
