def __init__(
self,
master: Optional[str] = None,
user: Optional[str] = None,
password: Optional[str] = None,
cert_path: Optional[str] = None,
cert_name: Optional[str] = None,
noverify: bool = False,
):
master = master or util.get_default_master_address()
cert = certs.default_load(
master_url=master,
explicit_path=cert_path,
explicit_cert_name=cert_name,
explicit_noverify=noverify,
)
# TODO: This should probably be try_reauth=False, but it appears that would break the case
# where the default credentials are available from the master and could be discovered by
# a REST API call against the master.
auth = authentication.Authentication(master,
user,
password,
try_reauth=True,
cert=cert)
self._session = session.Session(master, user, auth, cert)
def create_test_session() -> session.Session:
murl = conf.make_master_url()
certs.cli_cert = certs.default_load(murl)
authentication.cli_auth = authentication.Authentication(murl,
try_reauth=True)
return session.Session(murl, "determined", authentication.cli_auth,
certs.cli_cert)
def follow_experiment_logs(master_url: str, exp_id: int) -> None:
# Get the ID of this experiment's first trial (i.e., the one with the lowest ID).
print("Waiting for first trial to begin...")
sess = session.Session(master_url, None, None, None)
while True:
trials = bindings.get_GetExperimentTrials(sess,
experimentId=exp_id).trials
if len(trials) > 0:
break
else:
time.sleep(0.1)
first_trial_id = sorted(t_id.id for t_id in trials)[0]
print("Following first trial with ID {}".format(first_trial_id))
logs.pprint_trial_logs(master_url, first_trial_id, follow=True)
def test_post_user_api(clean_auth: None) -> None:
master_url = conf.make_master_url()
authentication.cli_auth = authentication.Authentication(
conf.make_master_url(),
requested_user="******",
password="",
try_reauth=True)
new_username = get_random_string()
user = bindings.v1User(active=True, admin=False, username=new_username)
body = bindings.v1PostUserRequest(password="", user=user)
resp = bindings.post_PostUser(session.Session(master_url, "admin",
authentication.cli_auth,
None),
body=body)
assert resp.to_json()["user"]["username"] == new_username
def __init__(
self,
master: Optional[str] = None,
user: Optional[str] = None,
):
self._session = session.Session(master, user)
userauth = api.authentication.Authentication.instance()
configuration = Configuration()
configuration.host = self._session._master.rstrip("/")
configuration.username = userauth.token_store.get_active_user()
configuration.api_key_prefix["Authorization"] = "Bearer"
configuration.api_key["Authorization"] = userauth.get_session_token()
self._experiments = ExperimentsApi(ApiClient(configuration))
self._internal = InternalApi(ApiClient(configuration))
self._trials = TrialsApi(ApiClient(configuration))
def test_task_logs(task_type: str, task_config: Dict[str, Any],
log_regex: Any) -> None:
# TODO: refactor tests to not use cli singleton auth.
master_url = conf.make_master_url()
certs.cli_cert = certs.default_load(conf.make_master_url())
authentication.cli_auth = authentication.Authentication(
conf.make_master_url(), try_reauth=True)
rps = bindings.get_GetResourcePools(
session.Session(master_url, "determined", authentication.cli_auth,
certs.cli_cert))
assert rps.resourcePools and len(
rps.resourcePools) > 0, "missing resource pool"
if (rps.resourcePools[0].type
== bindings.v1ResourcePoolType.RESOURCE_POOL_TYPE_K8S
and task_type == command.TaskTypeCommand):
# TODO(DET-6712): Investigate intermittent slowness with K8s command logs.
return
body = {}
if task_type == command.TaskTypeTensorBoard:
exp_id = exp.run_basic_test(
conf.fixtures_path("no_op/single.yaml"),
conf.fixtures_path("no_op"),
1,
)
body.update({"experiment_ids": [exp_id]})
resp = command.launch_command(
master_url,
f"api/v1/{command.RemoteTaskNewAPIs[task_type]}",
task_config,
"",
default_body=body,
)
task_id = resp[command.RemoteTaskName[task_type]]["id"]
try:
check_logs(master_url, task_id, log_regex, api.task_logs,
api.task_log_fields)
finally:
command._kill(master_url, task_type, task_id)
def test_change_displayname(clean_auth: None) -> None:
u_patch = create_test_user(ADMIN_CREDENTIALS, False)
original_name = u_patch.username
master_url = conf.make_master_url()
certs.cli_cert = certs.default_load(master_url)
authentication.cli_auth = authentication.Authentication(
conf.make_master_url(),
requested_user=original_name,
password="",
try_reauth=True)
sess = session.Session(master_url, original_name, authentication.cli_auth,
certs.cli_cert)
# Get API bindings object for the created test user
all_users = bindings.get_GetUsers(sess).users
assert all_users is not None
current_user = list(
filter(lambda u: u.username == original_name, all_users))[0]
assert current_user is not None and current_user.id
# Rename user using display name
patch_user = bindings.v1PatchUser(displayName="renamed")
bindings.patch_PatchUser(sess, body=patch_user, userId=current_user.id)
modded_user = bindings.get_GetUser(sess, userId=current_user.id).user
assert modded_user is not None
assert modded_user.displayName == "renamed"
# Avoid display name of 'admin'
patch_user.displayName = "Admin"
with pytest.raises(errors.APIException):
bindings.patch_PatchUser(sess, body=patch_user, userId=current_user.id)
# Clear display name (UI will show username)
patch_user.displayName = ""
bindings.patch_PatchUser(sess, body=patch_user, userId=current_user.id)
modded_user = bindings.get_GetUser(sess, userId=current_user.id).user
assert modded_user is not None
assert modded_user.displayName == ""
def create_experiment(
master_url: str,
config: Dict[str, Any],
model_context: context.Context,
template: Optional[str] = None,
validate_only: bool = False,
archived: bool = False,
activate: bool = True,
additional_body_fields: Optional[Dict[str, Any]] = None,
) -> int:
body = {
"activate": False,
"experiment_config": yaml.safe_dump(config),
"model_definition": [e.dict() for e in model_context.entries],
"validate_only": validate_only,
}
if template:
body["template"] = template
if archived:
body["archived"] = archived
if additional_body_fields:
body.update(additional_body_fields)
r = req.post(master_url, "experiments", json=body)
if not hasattr(r, "headers"):
raise Exception(r)
if validate_only:
return 0
new_resource = r.headers["Location"]
experiment_id = int(new_resource.split("/")[-1])
if activate:
sess = session.Session(master_url, None, None, None)
bindings.post_ActivateExperiment(sess, id=experiment_id)
return experiment_id
def test_workspace_org() -> None:
master_url = conf.make_master_url()
authentication.cli_auth = authentication.Authentication(master_url,
try_reauth=True)
sess = session.Session(master_url, None, None, None)
test_experiments: List[bindings.v1Experiment] = []
test_projects: List[bindings.v1Project] = []
test_workspaces: List[bindings.v1Workspace] = []
try:
# Uncategorized workspace / project should exist already.
r = bindings.get_GetWorkspaces(sess, name="Uncategorized")
assert len(r.workspaces) == 1
default_workspace = r.workspaces[0]
assert default_workspace.immutable
r2 = bindings.get_GetWorkspaceProjects(sess, id=default_workspace.id)
assert len(r2.projects) == 1
default_project = r2.projects[0]
assert default_project.name == "Uncategorized"
assert default_project.immutable
# Add a test workspace.
r3 = bindings.post_PostWorkspace(
sess, body=bindings.v1PostWorkspaceRequest(name="_TestOnly"))
made_workspace = r3.workspace
test_workspaces.append(made_workspace)
get_workspace = bindings.get_GetWorkspace(
sess, id=made_workspace.id).workspace
assert get_workspace.name == made_workspace.name
assert not made_workspace.immutable and not get_workspace.immutable
# Patch the workspace
w_patch = bindings.v1PatchWorkspace.from_json(made_workspace.to_json())
w_patch.name = "_TestPatched"
bindings.patch_PatchWorkspace(sess, body=w_patch, id=made_workspace.id)
get_workspace = bindings.get_GetWorkspace(
sess, id=made_workspace.id).workspace
assert get_workspace.name == "_TestPatched"
# Archive the workspace
assert not made_workspace.archived
bindings.post_ArchiveWorkspace(sess, id=made_workspace.id)
get_workspace_2 = bindings.get_GetWorkspace(
sess, id=made_workspace.id).workspace
assert get_workspace_2.archived
with pytest.raises(errors.APIException):
# Cannot patch archived workspace
bindings.patch_PatchWorkspace(sess,
body=w_patch,
id=made_workspace.id)
with pytest.raises(errors.APIException):
# Cannot create project inside archived workspace
bindings.post_PostProject(
sess,
body=bindings.v1PostProjectRequest(
name="Nope2", workspaceId=made_workspace.id),
workspaceId=made_workspace.id,
)
bindings.post_UnarchiveWorkspace(sess, id=made_workspace.id)
get_workspace_3 = bindings.get_GetWorkspace(
sess, id=made_workspace.id).workspace
assert not get_workspace_3.archived
# Refuse to patch, archive, unarchive, or delete the default workspace
with pytest.raises(errors.APIException):
bindings.patch_PatchWorkspace(sess,
body=w_patch,
id=default_workspace.id)
with pytest.raises(errors.APIException):
bindings.post_ArchiveWorkspace(sess, id=default_workspace.id)
with pytest.raises(errors.APIException):
bindings.post_UnarchiveWorkspace(sess, id=default_workspace.id)
with pytest.raises(errors.APIException):
bindings.delete_DeleteWorkspace(sess, id=default_workspace.id)
# Sort test and default workspaces.
workspace2 = bindings.post_PostWorkspace(
sess,
body=bindings.v1PostWorkspaceRequest(name="_TestWS")).workspace
test_workspaces.append(workspace2)
list_test_1 = bindings.get_GetWorkspaces(sess).workspaces
assert ["Uncategorized", "_TestPatched",
"_TestWS"] == [w.name for w in list_test_1]
list_test_2 = bindings.get_GetWorkspaces(
sess, orderBy=bindings.v1OrderBy.ORDER_BY_DESC).workspaces
assert ["_TestWS", "_TestPatched",
"Uncategorized"] == [w.name for w in list_test_2]
list_test_3 = bindings.get_GetWorkspaces(
sess, sortBy=bindings.v1GetWorkspacesRequestSortBy.SORT_BY_NAME
).workspaces
assert ["_TestPatched", "_TestWS",
"Uncategorized"] == [w.name for w in list_test_3]
# Test pinned workspaces.
pinned = bindings.get_GetWorkspaces(
sess,
pinned=True,
).workspaces
assert len(pinned) == 2
bindings.post_UnpinWorkspace(sess, id=made_workspace.id)
pinned = bindings.get_GetWorkspaces(
sess,
pinned=True,
).workspaces
assert len(pinned) == 1
bindings.post_PinWorkspace(sess, id=made_workspace.id)
pinned = bindings.get_GetWorkspaces(
sess,
pinned=True,
).workspaces
assert len(pinned) == 2
# Add a test project to a workspace.
r4 = bindings.post_PostProject(
sess,
body=bindings.v1PostProjectRequest(name="_TestOnly",
workspaceId=made_workspace.id),
workspaceId=made_workspace.id,
)
made_project = r4.project
test_projects.append(made_project)
get_project = bindings.get_GetProject(sess, id=made_project.id).project
assert get_project.name == made_project.name
assert not made_project.immutable and not get_project.immutable
# Project cannot be created in the default workspace.
with pytest.raises(errors.APIException):
bindings.post_PostProject(
sess,
body=bindings.v1PostProjectRequest(
name="Nope", workspaceId=default_workspace.id),
workspaceId=default_workspace.id,
)
# Patch the project
p_patch = bindings.v1PatchProject.from_json(made_project.to_json())
p_patch.name = "_TestPatchedProject"
bindings.patch_PatchProject(sess, body=p_patch, id=made_project.id)
get_project = bindings.get_GetProject(sess, id=made_project.id).project
assert get_project.name == "_TestPatchedProject"
# Archive the project
assert not made_project.archived
bindings.post_ArchiveProject(sess, id=made_project.id)
get_project_2 = bindings.get_GetProject(sess,
id=made_project.id).project
assert get_project_2.archived
# Cannot patch or move an archived project
with pytest.raises(errors.APIException):
bindings.patch_PatchProject(sess, body=p_patch, id=made_project.id)
with pytest.raises(errors.APIException):
bindings.post_MoveProject(
sess,
projectId=made_project.id,
body=bindings.v1MoveProjectRequest(
destinationWorkspaceId=workspace2.id,
projectId=made_project.id,
),
)
# Unarchive the project
bindings.post_UnarchiveProject(sess, id=made_project.id)
get_project_3 = bindings.get_GetProject(sess,
id=made_project.id).project
assert not get_project_3.archived
# Can't archive, un-archive, or move while parent workspace is archived
bindings.post_ArchiveWorkspace(sess, id=made_workspace.id)
get_project_4 = bindings.get_GetProject(sess,
id=made_project.id).project
assert get_project_4.archived
with pytest.raises(errors.APIException):
bindings.post_ArchiveProject(sess, id=made_project.id)
with pytest.raises(errors.APIException):
bindings.post_UnarchiveProject(sess, id=made_project.id)
with pytest.raises(errors.APIException):
bindings.post_MoveProject(
sess,
projectId=made_project.id,
body=bindings.v1MoveProjectRequest(
destinationWorkspaceId=workspace2.id,
projectId=made_project.id,
),
)
bindings.post_UnarchiveWorkspace(sess, id=made_workspace.id)
# Refuse to patch, archive, unarchive, or delete the default project
with pytest.raises(errors.APIException):
bindings.patch_PatchProject(sess,
body=p_patch,
id=default_project.id)
with pytest.raises(errors.APIException):
bindings.post_ArchiveProject(sess, id=default_project.id)
with pytest.raises(errors.APIException):
bindings.post_UnarchiveProject(sess, id=default_project.id)
with pytest.raises(errors.APIException):
bindings.delete_DeleteProject(sess, id=default_project.id)
# Sort workspaces' projects.
p1 = bindings.post_PostProject(
sess,
body=bindings.v1PostProjectRequest(name="_TestPRJ",
workspaceId=made_workspace.id),
workspaceId=made_workspace.id,
).project
p2 = bindings.post_PostProject(
sess,
body=bindings.v1PostProjectRequest(name="_TestEarly",
workspaceId=made_workspace.id),
workspaceId=made_workspace.id,
).project
test_projects += [p1, p2]
list_test_4 = bindings.get_GetWorkspaceProjects(
sess, id=made_workspace.id).projects
assert ["_TestPatchedProject", "_TestPRJ",
"_TestEarly"] == [p.name for p in list_test_4]
list_test_5 = bindings.get_GetWorkspaceProjects(
sess,
id=made_workspace.id,
orderBy=bindings.v1OrderBy.ORDER_BY_DESC).projects
assert ["_TestEarly", "_TestPRJ",
"_TestPatchedProject"] == [p.name for p in list_test_5]
list_test_6 = bindings.get_GetWorkspaceProjects(
sess,
id=made_workspace.id,
sortBy=bindings.v1GetWorkspaceProjectsRequestSortBy.SORT_BY_NAME,
).projects
assert ["_TestEarly", "_TestPatchedProject",
"_TestPRJ"] == [p.name for p in list_test_6]
# Move a project to another workspace
bindings.post_MoveProject(
sess,
projectId=made_project.id,
body=bindings.v1MoveProjectRequest(
destinationWorkspaceId=workspace2.id,
projectId=made_project.id,
),
)
get_project = bindings.get_GetProject(sess, id=made_project.id).project
assert get_project.workspaceId == workspace2.id
# Default project cannot be moved.
with pytest.raises(errors.APIException):
bindings.post_MoveProject(
sess,
projectId=default_project.id,
body=bindings.v1MoveProjectRequest(
destinationWorkspaceId=workspace2.id,
projectId=default_project.id,
),
)
# Project cannot be moved into the default workspace.
with pytest.raises(errors.APIException):
bindings.post_MoveProject(
sess,
projectId=made_project.id,
body=bindings.v1MoveProjectRequest(
destinationWorkspaceId=default_workspace.id,
projectId=made_project.id,
),
)
# Project cannot be moved into an archived workspace.
bindings.post_ArchiveWorkspace(sess, id=made_workspace.id)
with pytest.raises(errors.APIException):
bindings.post_MoveProject(
sess,
projectId=made_project.id,
body=bindings.v1MoveProjectRequest(
destinationWorkspaceId=made_workspace.id,
projectId=made_project.id,
),
)
bindings.post_UnarchiveWorkspace(sess, id=made_workspace.id)
# Add a test note to a project.
note = bindings.v1Note(name="Hello", contents="Hello World")
note2 = bindings.v1Note(name="Hello 2", contents="Hello World")
bindings.post_AddProjectNote(
sess,
body=note,
projectId=made_project.id,
)
r5 = bindings.post_AddProjectNote(
sess,
body=note2,
projectId=made_project.id,
)
returned_notes = r5.notes
assert len(returned_notes) == 2
# Put notes
r6 = bindings.put_PutProjectNotes(
sess,
body=bindings.v1PutProjectNotesRequest(notes=[note],
projectId=made_project.id),
projectId=made_project.id,
)
returned_notes = r6.notes
assert len(returned_notes) == 1
# Create an experiment in the default project.
test_exp_id = run_basic_test(conf.fixtures_path("no_op/single.yaml"),
conf.fixtures_path("no_op"), 1)
test_exp = bindings.get_GetExperiment(
sess, experimentId=test_exp_id).experiment
test_experiments.append(test_exp)
wait_for_experiment_state(
test_exp_id, bindings.determinedexperimentv1State.STATE_COMPLETED)
assert test_exp.projectId == default_project.id
# Move the test experiment into a user-made project
dproj_exp = bindings.get_GetProjectExperiments(
sess, id=default_project.id).experiments
exp_count = len(
bindings.get_GetProjectExperiments(sess,
id=made_project.id).experiments)
assert exp_count == 0
mbody = bindings.v1MoveExperimentRequest(
destinationProjectId=made_project.id, experimentId=test_exp_id)
bindings.post_MoveExperiment(sess,
experimentId=test_exp_id,
body=mbody)
modified_exp = bindings.get_GetExperiment(
sess, experimentId=test_exp_id).experiment
assert modified_exp.projectId == made_project.id
# Confirm the test experiment is in the new project, no longer in old project.
exp_count = len(
bindings.get_GetProjectExperiments(sess,
id=made_project.id).experiments)
assert exp_count == 1
dproj_exp2 = bindings.get_GetProjectExperiments(
sess, id=default_project.id).experiments
assert len(dproj_exp2) == len(dproj_exp) - 1
# Cannot move an experiment out of an archived project
bindings.post_ArchiveProject(sess, id=made_project.id)
mbody2 = bindings.v1MoveExperimentRequest(
destinationProjectId=default_project.id, experimentId=test_exp_id)
with pytest.raises(errors.APIException):
bindings.post_MoveExperiment(sess,
experimentId=test_exp_id,
body=mbody2)
bindings.post_UnarchiveProject(sess, id=made_project.id)
# Moving an experiment into default project
bindings.post_MoveExperiment(sess,
experimentId=test_exp_id,
body=mbody2)
# Cannot move an experiment into an archived project
bindings.post_ArchiveProject(sess, id=made_project.id)
with pytest.raises(errors.APIException):
bindings.post_MoveExperiment(sess,
experimentId=test_exp_id,
body=mbody)
finally:
# Clean out experiments, projects, workspaces.
# In dependency order:
for e in test_experiments:
bindings.delete_DeleteExperiment(sess, experimentId=e.id)
for p in test_projects:
bindings.delete_DeleteProject(sess, id=p.id)
for w in test_workspaces:
bindings.delete_DeleteWorkspace(sess, id=w.id)
def follow_test_experiment_logs(master_url: str, exp_id: int) -> None:
def print_progress(active_stage: int, ended: bool) -> None:
# There are four sequential stages of verification. Track the
# current stage with an index into this list.
stages = [
"Scheduling task",
"Testing training",
"Testing validation",
"Testing checkpointing",
]
for idx, stage in enumerate(stages):
if active_stage > idx:
color = "green"
checkbox = "✔"
elif active_stage == idx:
color = "red" if ended else "yellow"
checkbox = "✗" if ended else " "
else:
color = "white"
checkbox = " "
print(colored(stage + (25 - len(stage)) * ".", color), end="")
print(colored(" [" + checkbox + "]", color), end="")
if idx == len(stages) - 1:
print("\n" if ended else "\r", end="")
else:
print(", ", end="")
sess = session.Session(master_url, None, None, None)
while True:
r = bindings.get_GetExperiment(sess, experimentId=exp_id).experiment
trials = bindings.get_GetExperimentTrials(sess,
experimentId=exp_id).trials
# Wait for experiment to start and initialize a trial.
if len(trials) < 1:
t = {}
else:
trial_id = trials[0].id
t = api.get(master_url, f"trials/{trial_id}").json()
# Update the active_stage by examining the result from master
# /api/v1/experiments/<experiment-id> endpoint.
exp_state = r.state.value.replace("STATE_", "")
if exp_state == constants.COMPLETED:
active_stage = 4
elif t.get("runner_state") == "checkpointing":
active_stage = 3
elif t.get("runner_state") == "validating":
active_stage = 2
elif t.get("runner_state") in ("UNSPECIFIED", "training"):
active_stage = 1
else:
active_stage = 0
# If the experiment is in a terminal state, output the appropriate
# message and exit. Otherwise, sleep and repeat.
if exp_state == constants.COMPLETED:
print_progress(active_stage, ended=True)
print(colored("Model definition test succeeded! 🎉", "green"))
return
elif exp_state == constants.CANCELED:
print_progress(active_stage, ended=True)
print(
colored(
"Model definition test (ID: {}) canceled before "
"model test could complete. Please re-run the "
"command.".format(exp_id),
"yellow",
))
sys.exit(1)
elif exp_state == constants.ERROR:
print_progress(active_stage, ended=True)
trial_id = trials[0].id
logs.pprint_trial_logs(master_url, trial_id)
sys.exit(1)
else:
print_progress(active_stage, ended=False)
time.sleep(0.2)
def setup_session(args: Namespace) -> session.Session:
master_url = args.master or util.get_default_master_address()
cert = certs.default_load(master_url)
return session.Session(master_url, args.user, authentication.cli_auth,
cert)