def _submit_experiment( config: Optional[Dict[str, Any]], context_dir: str, command: Optional[List[str]], test: bool = False, master_url: Optional[str] = None, ) -> int: if context_dir == "": raise errors.InvalidExperimentException("Cannot specify the context directory to be empty.") context_path = pathlib.Path(context_dir) config = {**constants.DEFAULT_EXP_CFG, **(config or {})} config.setdefault("internal", {}) config["internal"]["native"] = {"command": _set_command_default(context_path, command)} logging.info(f"Creating an experiment with config: {config}") if master_url is None: master_url = util.get_default_master_address() exp_context = context.Context.from_local(context_path) # When a requested_user isn't specified to initialize_session(), the # authentication module will attempt to use the token store to grab the # current logged-in user. If there is no logged in user found, it will # default to constants.DEFAULT_DETERMINED_USER. auth.initialize_session(master_url, requested_user=None, try_reauth=True) if test: return api.create_test_experiment_and_follow_logs(master_url, config, exp_context) else: return api.create_experiment_and_follow_logs(master_url, config, exp_context)
def test_streaming_metrics_api() -> None: auth.initialize_session(conf.make_master_url(), try_reauth=True) pool = mp.pool.ThreadPool(processes=7) experiment_id = exp.create_experiment( conf.fixtures_path("mnist_pytorch/adaptive_short.yaml"), conf.tutorials_path("mnist_pytorch"), ) # To fully test the streaming APIs, the requests need to start running immediately after the # experiment, and then stay open until the experiment is complete. To accomplish this with all # of the API calls on a single experiment, we spawn them all in threads. # The HP importance portion of this test is commented out until the feature is enabled by # default metric_names_thread = pool.apply_async(request_metric_names, (experiment_id, )) train_metric_batches_thread = pool.apply_async( request_train_metric_batches, (experiment_id, )) valid_metric_batches_thread = pool.apply_async( request_valid_metric_batches, (experiment_id, )) train_trials_snapshot_thread = pool.apply_async( request_train_trials_snapshot, (experiment_id, )) valid_trials_snapshot_thread = pool.apply_async( request_valid_trials_snapshot, (experiment_id, )) train_trials_sample_thread = pool.apply_async(request_train_trials_sample, (experiment_id, )) valid_trials_sample_thread = pool.apply_async(request_valid_trials_sample, (experiment_id, )) metric_names_results = metric_names_thread.get() train_metric_batches_results = train_metric_batches_thread.get() valid_metric_batches_results = valid_metric_batches_thread.get() train_trials_snapshot_results = train_trials_snapshot_thread.get() valid_trials_snapshot_results = valid_trials_snapshot_thread.get() train_trials_sample_results = train_trials_sample_thread.get() valid_trials_sample_results = valid_trials_sample_thread.get() if metric_names_results is not None: pytest.fail("metric-names: %s. Results: %s" % metric_names_results) if train_metric_batches_results is not None: pytest.fail("metric-batches (training): %s. Results: %s" % train_metric_batches_results) if valid_metric_batches_results is not None: pytest.fail("metric-batches (validation): %s. Results: %s" % valid_metric_batches_results) if train_trials_snapshot_results is not None: pytest.fail("trials-snapshot (training): %s. Results: %s" % train_trials_snapshot_results) if valid_trials_snapshot_results is not None: pytest.fail("trials-snapshot (validation): %s. Results: %s" % valid_trials_snapshot_results) if train_trials_sample_results is not None: pytest.fail("trials-sample (training): %s. Results: %s" % train_trials_sample_results) if valid_trials_sample_results is not None: pytest.fail("trials-sample (validation): %s. Results: %s" % valid_trials_sample_results)
def get_num_running_commands() -> int: auth.initialize_session(conf.make_master_url(), try_reauth=True) r = api.get(conf.make_master_url(), "commands") assert r.status_code == requests.codes.ok, r.text return len([ command for _id, command in r.json().items() if command["state"] == "RUNNING" ])
def change_experiment_state(experiment_id: int, new_state: str) -> None: auth.initialize_session(conf.make_master_url(), try_reauth=True) r = api.patch( conf.make_master_url(), "experiments/{}".format(experiment_id), headers={"Content-Type": "application/merge-patch+json"}, body={"state": new_state}, ) assert r.status_code == requests.codes.no_content, r.text
def cluster_slots() -> Dict[str, Any]: """ cluster_slots returns a dict of slots that each agent has. :return: Dict[AgentID, List[Slot]] """ auth.initialize_session(conf.make_master_url(), try_reauth=True) r = api.get(conf.make_master_url(), "agents") assert r.status_code == requests.codes.ok, r.text json = r.json() # type: Dict[str, Any] return {agent["id"]: agent["slots"].values() for agent in json.values()}
def test_hp_importance_api() -> None: auth.initialize_session(conf.make_master_url(), try_reauth=True) pool = mp.pool.ThreadPool(processes=1) experiment_id = exp.create_experiment( conf.fixtures_path("mnist_pytorch/random.yaml"), conf.tutorials_path("mnist_pytorch"), ) hp_importance_thread = pool.apply_async(request_hp_importance, (experiment_id,)) hp_importance_results = hp_importance_thread.get() if hp_importance_results is not None: pytest.fail("hyperparameter-importance: %s. Results: %s" % hp_importance_results)
def get_command(id: str) -> Dict[str, Any]: auth.initialize_session(conf.make_master_url(), try_reauth=True) r = api.get(conf.make_master_url(), "commands") assert r.status_code == requests.codes.ok, r.text return cast(Dict[str, Any], r.json()["/commands/" + id])
def __init__(self, master: Optional[str], user: Optional[str]): self._master = master or util.get_default_master_address() self._user = user auth.initialize_session(self._master, self._user, try_reauth=True)
def trial_logs(trial_id: int) -> List[str]: auth.initialize_session(conf.make_master_url(), try_reauth=True) return [tl["message"] for tl in api.trial_logs(conf.make_master_url(), trial_id)]
def trial_metrics(trial_id: int) -> Dict[str, Any]: auth.initialize_session(conf.make_master_url(), try_reauth=True) r = api.get(conf.make_master_url(), "trials/{}/metrics".format(trial_id)) assert r.status_code == requests.codes.ok, r.text json = r.json() # type: Dict[str, Any] return json
def num_experiments() -> int: auth.initialize_session(conf.make_master_url(), try_reauth=True) r = api.get(conf.make_master_url(), "experiments") assert r.status_code == requests.codes.ok, r.text return len(r.json())
def experiment_json(experiment_id: int) -> Dict[str, Any]: auth.initialize_session(conf.make_master_url(), try_reauth=True) r = api.get(conf.make_master_url(), "experiments/{}".format(experiment_id)) assert r.status_code == requests.codes.ok, r.text json = r.json() # type: Dict[str, Any] return json
def f(namespace: Namespace) -> Any: v = vars(namespace) auth.initialize_session(namespace.master, v.get("user"), try_reauth=True) return func(namespace)
def get_command(id: str) -> Any: auth.initialize_session(conf.make_master_url(), try_reauth=True) r = api.get(conf.make_master_url(), "api/v1/commands/" + id) assert r.status_code == requests.codes.ok, r.text return r.json()["command"]