def __init__( self, master: Optional[str] = None, user: Optional[str] = None, password: Optional[str] = None, cert_path: Optional[str] = None, cert_name: Optional[str] = None, noverify: bool = False, ): master = master or util.get_default_master_address() cert = certs.default_load( master_url=master, explicit_path=cert_path, explicit_cert_name=cert_name, explicit_noverify=noverify, ) # TODO: This should probably be try_reauth=False, but it appears that would break the case # where the default credentials are available from the master and could be discovered by # a REST API call against the master. auth = authentication.Authentication(master, user, password, try_reauth=True, cert=cert) self._session = session.Session(master, user, auth, cert)
def create_test_session() -> session.Session: murl = conf.make_master_url() certs.cli_cert = certs.default_load(murl) authentication.cli_auth = authentication.Authentication(murl, try_reauth=True) return session.Session(murl, "determined", authentication.cli_auth, certs.cli_cert)
def follow_experiment_logs(master_url: str, exp_id: int) -> None: # Get the ID of this experiment's first trial (i.e., the one with the lowest ID). print("Waiting for first trial to begin...") sess = session.Session(master_url, None, None, None) while True: trials = bindings.get_GetExperimentTrials(sess, experimentId=exp_id).trials if len(trials) > 0: break else: time.sleep(0.1) first_trial_id = sorted(t_id.id for t_id in trials)[0] print("Following first trial with ID {}".format(first_trial_id)) logs.pprint_trial_logs(master_url, first_trial_id, follow=True)
def test_post_user_api(clean_auth: None) -> None: master_url = conf.make_master_url() authentication.cli_auth = authentication.Authentication( conf.make_master_url(), requested_user="******", password="", try_reauth=True) new_username = get_random_string() user = bindings.v1User(active=True, admin=False, username=new_username) body = bindings.v1PostUserRequest(password="", user=user) resp = bindings.post_PostUser(session.Session(master_url, "admin", authentication.cli_auth, None), body=body) assert resp.to_json()["user"]["username"] == new_username
def __init__( self, master: Optional[str] = None, user: Optional[str] = None, ): self._session = session.Session(master, user) userauth = api.authentication.Authentication.instance() configuration = Configuration() configuration.host = self._session._master.rstrip("/") configuration.username = userauth.token_store.get_active_user() configuration.api_key_prefix["Authorization"] = "Bearer" configuration.api_key["Authorization"] = userauth.get_session_token() self._experiments = ExperimentsApi(ApiClient(configuration)) self._internal = InternalApi(ApiClient(configuration)) self._trials = TrialsApi(ApiClient(configuration))
def test_task_logs(task_type: str, task_config: Dict[str, Any], log_regex: Any) -> None: # TODO: refactor tests to not use cli singleton auth. master_url = conf.make_master_url() certs.cli_cert = certs.default_load(conf.make_master_url()) authentication.cli_auth = authentication.Authentication( conf.make_master_url(), try_reauth=True) rps = bindings.get_GetResourcePools( session.Session(master_url, "determined", authentication.cli_auth, certs.cli_cert)) assert rps.resourcePools and len( rps.resourcePools) > 0, "missing resource pool" if (rps.resourcePools[0].type == bindings.v1ResourcePoolType.RESOURCE_POOL_TYPE_K8S and task_type == command.TaskTypeCommand): # TODO(DET-6712): Investigate intermittent slowness with K8s command logs. return body = {} if task_type == command.TaskTypeTensorBoard: exp_id = exp.run_basic_test( conf.fixtures_path("no_op/single.yaml"), conf.fixtures_path("no_op"), 1, ) body.update({"experiment_ids": [exp_id]}) resp = command.launch_command( master_url, f"api/v1/{command.RemoteTaskNewAPIs[task_type]}", task_config, "", default_body=body, ) task_id = resp[command.RemoteTaskName[task_type]]["id"] try: check_logs(master_url, task_id, log_regex, api.task_logs, api.task_log_fields) finally: command._kill(master_url, task_type, task_id)
def test_change_displayname(clean_auth: None) -> None: u_patch = create_test_user(ADMIN_CREDENTIALS, False) original_name = u_patch.username master_url = conf.make_master_url() certs.cli_cert = certs.default_load(master_url) authentication.cli_auth = authentication.Authentication( conf.make_master_url(), requested_user=original_name, password="", try_reauth=True) sess = session.Session(master_url, original_name, authentication.cli_auth, certs.cli_cert) # Get API bindings object for the created test user all_users = bindings.get_GetUsers(sess).users assert all_users is not None current_user = list( filter(lambda u: u.username == original_name, all_users))[0] assert current_user is not None and current_user.id # Rename user using display name patch_user = bindings.v1PatchUser(displayName="renamed") bindings.patch_PatchUser(sess, body=patch_user, userId=current_user.id) modded_user = bindings.get_GetUser(sess, userId=current_user.id).user assert modded_user is not None assert modded_user.displayName == "renamed" # Avoid display name of 'admin' patch_user.displayName = "Admin" with pytest.raises(errors.APIException): bindings.patch_PatchUser(sess, body=patch_user, userId=current_user.id) # Clear display name (UI will show username) patch_user.displayName = "" bindings.patch_PatchUser(sess, body=patch_user, userId=current_user.id) modded_user = bindings.get_GetUser(sess, userId=current_user.id).user assert modded_user is not None assert modded_user.displayName == ""
def create_experiment( master_url: str, config: Dict[str, Any], model_context: context.Context, template: Optional[str] = None, validate_only: bool = False, archived: bool = False, activate: bool = True, additional_body_fields: Optional[Dict[str, Any]] = None, ) -> int: body = { "activate": False, "experiment_config": yaml.safe_dump(config), "model_definition": [e.dict() for e in model_context.entries], "validate_only": validate_only, } if template: body["template"] = template if archived: body["archived"] = archived if additional_body_fields: body.update(additional_body_fields) r = req.post(master_url, "experiments", json=body) if not hasattr(r, "headers"): raise Exception(r) if validate_only: return 0 new_resource = r.headers["Location"] experiment_id = int(new_resource.split("/")[-1]) if activate: sess = session.Session(master_url, None, None, None) bindings.post_ActivateExperiment(sess, id=experiment_id) return experiment_id
def test_workspace_org() -> None: master_url = conf.make_master_url() authentication.cli_auth = authentication.Authentication(master_url, try_reauth=True) sess = session.Session(master_url, None, None, None) test_experiments: List[bindings.v1Experiment] = [] test_projects: List[bindings.v1Project] = [] test_workspaces: List[bindings.v1Workspace] = [] try: # Uncategorized workspace / project should exist already. r = bindings.get_GetWorkspaces(sess, name="Uncategorized") assert len(r.workspaces) == 1 default_workspace = r.workspaces[0] assert default_workspace.immutable r2 = bindings.get_GetWorkspaceProjects(sess, id=default_workspace.id) assert len(r2.projects) == 1 default_project = r2.projects[0] assert default_project.name == "Uncategorized" assert default_project.immutable # Add a test workspace. r3 = bindings.post_PostWorkspace( sess, body=bindings.v1PostWorkspaceRequest(name="_TestOnly")) made_workspace = r3.workspace test_workspaces.append(made_workspace) get_workspace = bindings.get_GetWorkspace( sess, id=made_workspace.id).workspace assert get_workspace.name == made_workspace.name assert not made_workspace.immutable and not get_workspace.immutable # Patch the workspace w_patch = bindings.v1PatchWorkspace.from_json(made_workspace.to_json()) w_patch.name = "_TestPatched" bindings.patch_PatchWorkspace(sess, body=w_patch, id=made_workspace.id) get_workspace = bindings.get_GetWorkspace( sess, id=made_workspace.id).workspace assert get_workspace.name == "_TestPatched" # Archive the workspace assert not made_workspace.archived bindings.post_ArchiveWorkspace(sess, id=made_workspace.id) get_workspace_2 = bindings.get_GetWorkspace( sess, id=made_workspace.id).workspace assert get_workspace_2.archived with pytest.raises(errors.APIException): # Cannot patch archived workspace bindings.patch_PatchWorkspace(sess, body=w_patch, id=made_workspace.id) with pytest.raises(errors.APIException): # Cannot create project inside archived workspace bindings.post_PostProject( sess, body=bindings.v1PostProjectRequest( name="Nope2", workspaceId=made_workspace.id), workspaceId=made_workspace.id, ) bindings.post_UnarchiveWorkspace(sess, id=made_workspace.id) get_workspace_3 = bindings.get_GetWorkspace( sess, id=made_workspace.id).workspace assert not get_workspace_3.archived # Refuse to patch, archive, unarchive, or delete the default workspace with pytest.raises(errors.APIException): bindings.patch_PatchWorkspace(sess, body=w_patch, id=default_workspace.id) with pytest.raises(errors.APIException): bindings.post_ArchiveWorkspace(sess, id=default_workspace.id) with pytest.raises(errors.APIException): bindings.post_UnarchiveWorkspace(sess, id=default_workspace.id) with pytest.raises(errors.APIException): bindings.delete_DeleteWorkspace(sess, id=default_workspace.id) # Sort test and default workspaces. workspace2 = bindings.post_PostWorkspace( sess, body=bindings.v1PostWorkspaceRequest(name="_TestWS")).workspace test_workspaces.append(workspace2) list_test_1 = bindings.get_GetWorkspaces(sess).workspaces assert ["Uncategorized", "_TestPatched", "_TestWS"] == [w.name for w in list_test_1] list_test_2 = bindings.get_GetWorkspaces( sess, orderBy=bindings.v1OrderBy.ORDER_BY_DESC).workspaces assert ["_TestWS", "_TestPatched", "Uncategorized"] == [w.name for w in list_test_2] list_test_3 = bindings.get_GetWorkspaces( sess, sortBy=bindings.v1GetWorkspacesRequestSortBy.SORT_BY_NAME ).workspaces assert ["_TestPatched", "_TestWS", "Uncategorized"] == [w.name for w in list_test_3] # Test pinned workspaces. pinned = bindings.get_GetWorkspaces( sess, pinned=True, ).workspaces assert len(pinned) == 2 bindings.post_UnpinWorkspace(sess, id=made_workspace.id) pinned = bindings.get_GetWorkspaces( sess, pinned=True, ).workspaces assert len(pinned) == 1 bindings.post_PinWorkspace(sess, id=made_workspace.id) pinned = bindings.get_GetWorkspaces( sess, pinned=True, ).workspaces assert len(pinned) == 2 # Add a test project to a workspace. r4 = bindings.post_PostProject( sess, body=bindings.v1PostProjectRequest(name="_TestOnly", workspaceId=made_workspace.id), workspaceId=made_workspace.id, ) made_project = r4.project test_projects.append(made_project) get_project = bindings.get_GetProject(sess, id=made_project.id).project assert get_project.name == made_project.name assert not made_project.immutable and not get_project.immutable # Project cannot be created in the default workspace. with pytest.raises(errors.APIException): bindings.post_PostProject( sess, body=bindings.v1PostProjectRequest( name="Nope", workspaceId=default_workspace.id), workspaceId=default_workspace.id, ) # Patch the project p_patch = bindings.v1PatchProject.from_json(made_project.to_json()) p_patch.name = "_TestPatchedProject" bindings.patch_PatchProject(sess, body=p_patch, id=made_project.id) get_project = bindings.get_GetProject(sess, id=made_project.id).project assert get_project.name == "_TestPatchedProject" # Archive the project assert not made_project.archived bindings.post_ArchiveProject(sess, id=made_project.id) get_project_2 = bindings.get_GetProject(sess, id=made_project.id).project assert get_project_2.archived # Cannot patch or move an archived project with pytest.raises(errors.APIException): bindings.patch_PatchProject(sess, body=p_patch, id=made_project.id) with pytest.raises(errors.APIException): bindings.post_MoveProject( sess, projectId=made_project.id, body=bindings.v1MoveProjectRequest( destinationWorkspaceId=workspace2.id, projectId=made_project.id, ), ) # Unarchive the project bindings.post_UnarchiveProject(sess, id=made_project.id) get_project_3 = bindings.get_GetProject(sess, id=made_project.id).project assert not get_project_3.archived # Can't archive, un-archive, or move while parent workspace is archived bindings.post_ArchiveWorkspace(sess, id=made_workspace.id) get_project_4 = bindings.get_GetProject(sess, id=made_project.id).project assert get_project_4.archived with pytest.raises(errors.APIException): bindings.post_ArchiveProject(sess, id=made_project.id) with pytest.raises(errors.APIException): bindings.post_UnarchiveProject(sess, id=made_project.id) with pytest.raises(errors.APIException): bindings.post_MoveProject( sess, projectId=made_project.id, body=bindings.v1MoveProjectRequest( destinationWorkspaceId=workspace2.id, projectId=made_project.id, ), ) bindings.post_UnarchiveWorkspace(sess, id=made_workspace.id) # Refuse to patch, archive, unarchive, or delete the default project with pytest.raises(errors.APIException): bindings.patch_PatchProject(sess, body=p_patch, id=default_project.id) with pytest.raises(errors.APIException): bindings.post_ArchiveProject(sess, id=default_project.id) with pytest.raises(errors.APIException): bindings.post_UnarchiveProject(sess, id=default_project.id) with pytest.raises(errors.APIException): bindings.delete_DeleteProject(sess, id=default_project.id) # Sort workspaces' projects. p1 = bindings.post_PostProject( sess, body=bindings.v1PostProjectRequest(name="_TestPRJ", workspaceId=made_workspace.id), workspaceId=made_workspace.id, ).project p2 = bindings.post_PostProject( sess, body=bindings.v1PostProjectRequest(name="_TestEarly", workspaceId=made_workspace.id), workspaceId=made_workspace.id, ).project test_projects += [p1, p2] list_test_4 = bindings.get_GetWorkspaceProjects( sess, id=made_workspace.id).projects assert ["_TestPatchedProject", "_TestPRJ", "_TestEarly"] == [p.name for p in list_test_4] list_test_5 = bindings.get_GetWorkspaceProjects( sess, id=made_workspace.id, orderBy=bindings.v1OrderBy.ORDER_BY_DESC).projects assert ["_TestEarly", "_TestPRJ", "_TestPatchedProject"] == [p.name for p in list_test_5] list_test_6 = bindings.get_GetWorkspaceProjects( sess, id=made_workspace.id, sortBy=bindings.v1GetWorkspaceProjectsRequestSortBy.SORT_BY_NAME, ).projects assert ["_TestEarly", "_TestPatchedProject", "_TestPRJ"] == [p.name for p in list_test_6] # Move a project to another workspace bindings.post_MoveProject( sess, projectId=made_project.id, body=bindings.v1MoveProjectRequest( destinationWorkspaceId=workspace2.id, projectId=made_project.id, ), ) get_project = bindings.get_GetProject(sess, id=made_project.id).project assert get_project.workspaceId == workspace2.id # Default project cannot be moved. with pytest.raises(errors.APIException): bindings.post_MoveProject( sess, projectId=default_project.id, body=bindings.v1MoveProjectRequest( destinationWorkspaceId=workspace2.id, projectId=default_project.id, ), ) # Project cannot be moved into the default workspace. with pytest.raises(errors.APIException): bindings.post_MoveProject( sess, projectId=made_project.id, body=bindings.v1MoveProjectRequest( destinationWorkspaceId=default_workspace.id, projectId=made_project.id, ), ) # Project cannot be moved into an archived workspace. bindings.post_ArchiveWorkspace(sess, id=made_workspace.id) with pytest.raises(errors.APIException): bindings.post_MoveProject( sess, projectId=made_project.id, body=bindings.v1MoveProjectRequest( destinationWorkspaceId=made_workspace.id, projectId=made_project.id, ), ) bindings.post_UnarchiveWorkspace(sess, id=made_workspace.id) # Add a test note to a project. note = bindings.v1Note(name="Hello", contents="Hello World") note2 = bindings.v1Note(name="Hello 2", contents="Hello World") bindings.post_AddProjectNote( sess, body=note, projectId=made_project.id, ) r5 = bindings.post_AddProjectNote( sess, body=note2, projectId=made_project.id, ) returned_notes = r5.notes assert len(returned_notes) == 2 # Put notes r6 = bindings.put_PutProjectNotes( sess, body=bindings.v1PutProjectNotesRequest(notes=[note], projectId=made_project.id), projectId=made_project.id, ) returned_notes = r6.notes assert len(returned_notes) == 1 # Create an experiment in the default project. test_exp_id = run_basic_test(conf.fixtures_path("no_op/single.yaml"), conf.fixtures_path("no_op"), 1) test_exp = bindings.get_GetExperiment( sess, experimentId=test_exp_id).experiment test_experiments.append(test_exp) wait_for_experiment_state( test_exp_id, bindings.determinedexperimentv1State.STATE_COMPLETED) assert test_exp.projectId == default_project.id # Move the test experiment into a user-made project dproj_exp = bindings.get_GetProjectExperiments( sess, id=default_project.id).experiments exp_count = len( bindings.get_GetProjectExperiments(sess, id=made_project.id).experiments) assert exp_count == 0 mbody = bindings.v1MoveExperimentRequest( destinationProjectId=made_project.id, experimentId=test_exp_id) bindings.post_MoveExperiment(sess, experimentId=test_exp_id, body=mbody) modified_exp = bindings.get_GetExperiment( sess, experimentId=test_exp_id).experiment assert modified_exp.projectId == made_project.id # Confirm the test experiment is in the new project, no longer in old project. exp_count = len( bindings.get_GetProjectExperiments(sess, id=made_project.id).experiments) assert exp_count == 1 dproj_exp2 = bindings.get_GetProjectExperiments( sess, id=default_project.id).experiments assert len(dproj_exp2) == len(dproj_exp) - 1 # Cannot move an experiment out of an archived project bindings.post_ArchiveProject(sess, id=made_project.id) mbody2 = bindings.v1MoveExperimentRequest( destinationProjectId=default_project.id, experimentId=test_exp_id) with pytest.raises(errors.APIException): bindings.post_MoveExperiment(sess, experimentId=test_exp_id, body=mbody2) bindings.post_UnarchiveProject(sess, id=made_project.id) # Moving an experiment into default project bindings.post_MoveExperiment(sess, experimentId=test_exp_id, body=mbody2) # Cannot move an experiment into an archived project bindings.post_ArchiveProject(sess, id=made_project.id) with pytest.raises(errors.APIException): bindings.post_MoveExperiment(sess, experimentId=test_exp_id, body=mbody) finally: # Clean out experiments, projects, workspaces. # In dependency order: for e in test_experiments: bindings.delete_DeleteExperiment(sess, experimentId=e.id) for p in test_projects: bindings.delete_DeleteProject(sess, id=p.id) for w in test_workspaces: bindings.delete_DeleteWorkspace(sess, id=w.id)
def follow_test_experiment_logs(master_url: str, exp_id: int) -> None: def print_progress(active_stage: int, ended: bool) -> None: # There are four sequential stages of verification. Track the # current stage with an index into this list. stages = [ "Scheduling task", "Testing training", "Testing validation", "Testing checkpointing", ] for idx, stage in enumerate(stages): if active_stage > idx: color = "green" checkbox = "✔" elif active_stage == idx: color = "red" if ended else "yellow" checkbox = "✗" if ended else " " else: color = "white" checkbox = " " print(colored(stage + (25 - len(stage)) * ".", color), end="") print(colored(" [" + checkbox + "]", color), end="") if idx == len(stages) - 1: print("\n" if ended else "\r", end="") else: print(", ", end="") sess = session.Session(master_url, None, None, None) while True: r = bindings.get_GetExperiment(sess, experimentId=exp_id).experiment trials = bindings.get_GetExperimentTrials(sess, experimentId=exp_id).trials # Wait for experiment to start and initialize a trial. if len(trials) < 1: t = {} else: trial_id = trials[0].id t = api.get(master_url, f"trials/{trial_id}").json() # Update the active_stage by examining the result from master # /api/v1/experiments/<experiment-id> endpoint. exp_state = r.state.value.replace("STATE_", "") if exp_state == constants.COMPLETED: active_stage = 4 elif t.get("runner_state") == "checkpointing": active_stage = 3 elif t.get("runner_state") == "validating": active_stage = 2 elif t.get("runner_state") in ("UNSPECIFIED", "training"): active_stage = 1 else: active_stage = 0 # If the experiment is in a terminal state, output the appropriate # message and exit. Otherwise, sleep and repeat. if exp_state == constants.COMPLETED: print_progress(active_stage, ended=True) print(colored("Model definition test succeeded! 🎉", "green")) return elif exp_state == constants.CANCELED: print_progress(active_stage, ended=True) print( colored( "Model definition test (ID: {}) canceled before " "model test could complete. Please re-run the " "command.".format(exp_id), "yellow", )) sys.exit(1) elif exp_state == constants.ERROR: print_progress(active_stage, ended=True) trial_id = trials[0].id logs.pprint_trial_logs(master_url, trial_id) sys.exit(1) else: print_progress(active_stage, ended=False) time.sleep(0.2)
def setup_session(args: Namespace) -> session.Session: master_url = args.master or util.get_default_master_address() cert = certs.default_load(master_url) return session.Session(master_url, args.user, authentication.cli_auth, cert)