def test_client_defaults(mocker):
get_session_mock = mocker.patch("faculty.session.get_session")
for_resource_mock = mocker.patch("faculty.clients.for_resource")
faculty.client("test-resource")
for_resource_mock.assert_called_once_with("test-resource")
get_session_mock.assert_called_once_with(
credentials_path=None,
profile_name=None,
domain=None,
protocol=None,
client_id=None,
client_secret=None,
access_token_cache=None,
)
returned_class = for_resource_mock.return_value
returned_session = get_session_mock.return_value
returned_session.service_url.assert_called_once_with(
returned_class.SERVICE_NAME)
returned_class.assert_called_once_with(
returned_session.service_url.return_value, returned_session)
def test_client(mocker):
get_session_mock = mocker.patch("faculty.session.get_session")
for_resource_mock = mocker.patch("faculty.clients.for_resource")
options = {
"credentials_path": "/path/to/credentials",
"profile_name": "my-profile",
"domain": "domain.com",
"protocol": "http",
"client_id": "client-id",
"client_secret": "client-secret",
"access_token_cache": mocker.Mock(),
}
faculty.client("test-resource", **options)
for_resource_mock.assert_called_once_with("test-resource")
get_session_mock.assert_called_once_with(**options)
returned_class = for_resource_mock.return_value
returned_session = get_session_mock.return_value
returned_session.service_url.assert_called_once_with(
returned_class.SERVICE_NAME)
returned_class.assert_called_once_with(
returned_session.service_url.return_value, returned_session)
def publish(
report_name: str,
path: Path,
show_code: bool = False,
report_id: Optional[UUID] = None,
project_id: Optional[UUID] = None,
user_id: Optional[UUID] = None,
):
"""Publish a notebook as a report.
Parameters
----------
report_name : str
The name of the report
path : Path
The path of the notebook.
report_id : Optional[UUID], optional
The report ID, if you want to publish it as a version of an existing
report (the default is None, in which case we search for an existing
report with the provided name)
project_id : Optional[UUID], optional
The project ID. Only needed if not invoking from within a project.
user_id : Optional[UUID], optional
The user ID. Only needed if not invoking from within a project.
show_code : bool, optional
Whether the code should be shown in the report or not (default False)
"""
if project_id is None:
project_id = UUID(os.getenv("SHERLOCKML_PROJECT_ID"))
if user_id is None:
user_id = faculty.client("account").authenticated_user_id()
report_client = faculty.client("report")
if report_id is None:
report_id = get_report_id_by_name(report_name, project_id)
if report_id is not None:
report_client.create_version(
report_id,
str(path.relative_to("/project/")),
user_id,
show_code=show_code,
)
click.echo("Publishing report version...")
else:
report_client.create(
project_id,
report_name,
str(path.relative_to("/project/")),
user_id,
show_code=show_code,
)
click.echo("Publishing report...")
# this is to allow farah to process the notebook before deleting it
time.sleep(5)
click.echo("Done!")
def test_client(mocker):
get_session_mock = mocker.patch("faculty.session.get_session")
for_resource_mock = mocker.patch("faculty.clients.for_resource")
options = {"foo": "bar"}
faculty.client("test-resource", **options)
get_session_mock.assert_called_once_with(**options)
for_resource_mock.assert_called_once_with("test-resource")
returned_session = get_session_mock.return_value
returned_class = for_resource_mock.return_value
returned_class.assert_called_once_with(returned_session)
def status(project, server):
"""Get the execution status for an environment."""
project_id, server_id = _resolve_server(project, server)
server_client = faculty.client("server")
server = server_client.get(project_id, server_id)
hound_url = _get_hound_url(server)
client = faculty_cli.hound.Hound(hound_url)
execution = client.latest_environment_execution()
if execution is None:
msg = "No environment has yet been applied to this server."
_print_and_exit(msg, 64)
click.echo("Latest environment execution:")
click.echo(" Status: {}".format(execution.status))
for i, environment in enumerate(execution.environments):
click.echo("")
click.echo("Environment {}".format(i))
for j, step in enumerate(environment.steps):
click.echo("")
click.echo("Step {}:".format(j))
click.echo(" Status: {}".format(step.status))
click.echo(" Command: {}".format(_format_command(step.command)))
def _server_by_name(project_id, server_name, status=None):
"""Resolve a project ID and server name to a server ID."""
client = faculty.client("server")
matching_servers = [
server
for server in client.list(project_id)
if server.name == server_name
]
if status is not None:
matching_servers = [
server
for server in matching_servers
if server.status.value == status
]
if len(matching_servers) == 1:
return matching_servers[0]
else:
if not matching_servers:
tpl = 'no {} server of name "{}" in this project'
else:
tpl = (
'more than one {} server of name "{}", please select by '
"server ID instead"
)
adjective = "available" if status is None else status
raise NoValidServer(tpl.format(adjective, server_name))
def logs(project, server, step_number):
"""Stream the logs for a server environment application."""
project_id, server_id = _resolve_server(project, server)
server_client = faculty.client("server")
server = server_client.get(project_id, server_id)
hound_url = _get_hound_url(server)
client = faculty_cli.hound.Hound(hound_url)
execution = client.latest_environment_execution()
if execution is None:
msg = "No environment has yet been applied to this server."
_print_and_exit(msg, 64)
steps = [
step for environment_execution in execution.environments
for step in environment_execution.steps
]
if step_number is not None:
try:
steps = [steps[step_number]]
except IndexError:
_print_and_exit("step {} out of range".format(step_number), 64)
for step in steps:
for line in client.stream_environment_execution_step_logs(step):
click.echo(line)
def __init__(self, project_id, job_id, clean=True, tmpdir_prefix=None):
self.project_id = project_id
self.job_id = job_id
self.job_client = client("job")
self.clean = clean
self.tmpdir_prefix = tmpdir_prefix
def instance_types(verbose):
"""List the types of servers available on dedicated infrastructure."""
client = faculty.client("cluster")
types = client.list_single_tenanted_node_types(
interactive_instances_configured=True)
types = sorted(types, key=operator.attrgetter("cost_usd_per_hour"))
if verbose:
if not types:
click.echo("No servers on dedicated infrastructure available.")
else:
headers = (
"Machine Type",
"CPUs",
"RAM",
"GPUs",
"GPU Name",
"Cost",
)
rows = [(
type_.name,
"{:.3g}".format(type_.milli_cpus / 1000),
"{:.3g} GB".format(type_.memory_mb / 1000),
type_.num_gpus or "-",
type_.gpu_name or "-",
"$ {:.3f} / hour".format(type_.cost_usd_per_hour),
) for type_ in types]
click.echo(tabulate(rows, headers, tablefmt="plain"))
else:
for type_ in types:
click.echo(type_.name)
def _list_user_servers(user_id, status=None):
"""List all servers owned by user."""
client = faculty.client("server")
servers = client.list_for_user(user_id)
if status is not None:
servers = [s for s in servers if s.status == status]
return servers
def ls(project, path):
"""List files and directories on Faculty workspace."""
if not path.startswith("/project"):
_print_and_exit("{} is outside the project workspace".format(path), 66)
project_id = _resolve_project(project)
relative_path = os.path.relpath(path, "/project")
client = faculty.client("workspace")
try:
directory_details_list = client.list(project_id=project_id,
prefix=relative_path,
depth=1)
except faculty.clients.base.NotFound:
_print_and_exit("{}: No such file or directory".format(path), 66)
try:
[directory_details] = directory_details_list
except ValueError:
_print_and_exit("Zero or more than one objects returned", 70)
for item in directory_details.content:
if hasattr(item, "content"):
click.echo("/project{}/".format(item.path))
else:
click.echo("/project{}".format(item.path))
def _get_servers(project_id, name=None, status=None):
"""List servers in the given project."""
client = faculty.client("server")
servers = client.list(project_id, name)
if status is not None:
servers = [s for s in servers if s.status == status]
return servers
def job_name_to_job_id(job_name, project_id=None):
"""
Queries faculty platform so as to convert a specified job name into its
corresponding job id.
Parameters
----------
job_name: String
Job name to query the platform for.
project_id: uuid
Unique id of the project on the platform.
Returns
-------
job_id: uuid
Unique job id corresponding to the specified job name and project.
"""
if project_id is None:
project_id = os.environ["FACULTY_PROJECT_ID"]
job_client = client("job")
for job in job_client.list(project_id):
if job.metadata.name == job_name:
return job.id
def _rsync(project, local, remote, server, rsync_opts, up):
"""Sync files from or to server."""
project_id, server_id = _resolve_server(project, server)
client = faculty.client("server")
details = client.get_ssh_details(project_id, server_id)
escaped_remote = faculty_cli.shell.quote(remote)
if up:
path_from = local
path_to = u"{}@{}:{}".format(details.username, details.hostname,
escaped_remote)
else:
path_from = u"{}@{}:{}".format(details.username, details.hostname,
escaped_remote)
path_to = local
with _save_key_to_file(details.key) as filename:
ssh_cmd = "ssh {} -p {} -i {}".format(" ".join(SSH_OPTIONS),
details.port, filename)
rsync_cmd = ["rsync", "-a", "-e", ssh_cmd, path_from, path_to]
rsync_cmd += list(rsync_opts)
_run_ssh_cmd(rsync_cmd)
def _run(self, num_subruns):
"""
Acquire the specified compute resources and execute the
cross_validation task.
Parameters
----------
num_subruns: Integer
Number of subruns within the cross-validation job. It corresponds
to the number of servers used.
"""
# Assign multiple train-test splits to each subrun.
subrun_chunks = get_chunks(self.split_ids, num_subruns)
subrun_args = []
for subrun_chunk in subrun_chunks:
in_paths = []
for split_id in subrun_chunk:
in_paths.append(
os.path.join(
self.subsub_dir.format(self.dump_time, split_id),
self.in_base))
subrun_args.append({"in_paths": ":".join(in_paths)})
# Launch parallel cross-validation.
job_client = client("job")
job_client.create_run(self.project_id, self.job_id, subrun_args)
LOGGER.info("Launching cross validation. " +
"Refer to the Jobs tab for progress information")
def list_artifacts(self, path=None):
if path is None:
path = "./"
datasets_path = self._datasets_path(path)
# Make sure path interpreted as a directory
prefix = datasets_path.rstrip("/") + "/"
# Go directly to the object store so we can get file sizes in the
# response
client = faculty.client("object")
list_response = client.list(self.project_id, prefix)
objects = list_response.objects
while list_response.next_page_token is not None:
list_response = client.list(
self.project_id, prefix, list_response.next_page_token
)
objects += list_response.objects
infos = [
faculty_object_to_mlflow_file_info(
obj, self.datasets_artifact_root
)
for obj in objects
]
# Remove root
return [i for i in infos if i.path != "/"]
def resolve_project(project):
"""Resolve a project name or ID to a project ID."""
projects_client = faculty.client("project")
try:
project_id = uuid.UUID(project)
project = projects_client.get(project_id)
except ValueError:
account_client = faculty.client("account")
user_id = account_client.authenticated_user_id()
projects = [
p for p in projects_client.list_accessible_by_user(user_id)
if p.name == project
]
if len(projects) == 1:
project = projects[0]
else:
raise ValueError("Could not resolve project.")
return project
def apply(project, server, environment):
"""Apply an environment to the server."""
project_id, server_id = _resolve_server(project, server)
environment_id = _resolve_environment(project_id, environment)
client = faculty.client("server")
client.apply_environment(server_id, environment_id)
click.echo("Applying environment {} to server {} for project {}".format(
environment, server, project))
def new_project(name):
"""Create new project."""
client = faculty.client("project")
user_id = _get_authenticated_user_id()
try:
returned_project = client.create(user_id, name)
except faculty.clients.base.BadRequest as err:
_print_and_exit(err.error, 64)
click.echo("Created project {} with ID {}".format(returned_project.name,
returned_project.id))
def job_run_logs(project, job, run):
"""Print the logs for a run."""
project_id, job_id = _resolve_job(project, job)
job_client = faculty.client("job")
run_details = job_client.get_run(project_id, job_id, run.run_number)
if run.subrun_number is not None:
subrun_number = run.subrun_number
elif len(run_details.subruns) == 1:
subrun_number = run_details.subruns[0].subrun_number
else:
_print_and_exit(
("Run {0} has {1} subruns. You must specify the subrun "
"to show logs from, e.g. '{0}.1'.").format(
run.run_number, len(run_details.subruns)),
64,
)
subrun_details = job_client.get_subrun(project_id, job_id, run.run_number,
subrun_number)
log_client = faculty.client("log")
for env_step_exec in subrun_details.environment_step_executions:
env_name = env_step_exec.environment_name
click.secho('Logs for step of environment "{}":'.format(env_name),
fg="yellow")
parts = log_client.get_subrun_environment_step_logs(
project_id,
job_id,
run_details.id,
subrun_details.id,
env_step_exec.environment_step_id,
)
click.echo("".join(part.content for part in parts), nl=False)
click.secho("Logs for job command:", fg="yellow")
parts = log_client.get_subrun_command_logs(project_id, job_id,
run_details.id,
subrun_details.id)
click.echo("".join(part.content for part in parts), nl=False)
def get_ssh_details(configuration):
client = faculty.client("server")
details = client.get_ssh_details(configuration.project.id,
configuration.server_id)
hostname = details.hostname
port = details.port
username = details.username
key = details.key
with _save_key_to_file(key) as key_file:
ssh_details = SshDetails(hostname, port, username, key_file)
yield ssh_details
def _any_server(project_id, status=None):
"""Get any running server from project."""
client = faculty.client("server")
servers_ = [server for server in client.list(project_id)]
if status is not None:
servers_ = [
server for server in servers_ if server.status.value == status
]
if not servers_:
adjective = "available" if status is None else status
message = "No {} server in project.".format(adjective)
raise NoValidServer(message)
return servers_[0].id
def _resolve_project(project):
"""Resolve a project name or ID to a project ID."""
try:
project_id = uuid.UUID(project)
except ValueError:
account_client = faculty.client("account")
user_id = account_client.authenticated_user_id()
project_client = faculty.client("project")
projects = project_client.list_accessible_by_user(user_id)
matching_projects = [p for p in projects if p.name == project]
if len(matching_projects) == 1:
project_id = matching_projects[0].id
else:
if not matching_projects:
msg = 'no project of name "{}" found'.format(project)
raise NameNotFoundError(msg)
else:
msg = ('more than one project of name "{}", please select by '
"project ID instead").format(project)
raise AmbiguousNameError(msg)
return project_id
def list_job_runs(project, job, verbose):
"""List the runs of a job."""
project_id, job_id = _resolve_job(project, job)
client = faculty.client("job")
def list_runs():
list_runs_result = client.list_runs(project_id, job_id)
for run in list_runs_result.runs:
yield run
while list_runs_result.pagination.next is not None:
list_runs_result = client.list_runs(
project_id,
job_id,
start=list_runs_result.pagination.next.start,
limit=list_runs_result.pagination.next.limit,
)
for run in list_runs_result.runs:
yield run
runs = list(list_runs())
if verbose:
if not runs:
click.echo("No runs.")
else:
rows = [(
run.run_number,
run.id,
run.state.value,
_format_datetime(run.submitted_at),
_format_datetime(run.started_at),
_format_datetime(run.ended_at),
) for run in runs]
click.echo(
tabulate(
rows,
(
"Number",
"ID",
"State",
"Submitted At",
"Started At",
"Ended At",
),
tablefmt="plain",
))
else:
for run in runs:
click.echo(run.run_number)
def _job_by_name(project_id, job_name):
"""Resolve a project ID and job name to a job ID."""
client = faculty.client("job")
jobs = client.list(project_id)
matching_jobs = [job for job in jobs if job.metadata.name == job_name]
if len(matching_jobs) == 1:
return matching_jobs[0]
else:
if not matching_jobs:
msg = 'no job of name "{}" in this project'.format(job_name)
raise NameNotFoundError(msg)
else:
msg = ('more than one job of name "{}", please select by job ID '
"instead").format(job_name)
raise AmbiguousNameError(msg)
def _get_model_version(project_id, model_id, version=None):
client = faculty.client("model")
model_versions = client.list_versions(project_id, model_id)
if version is None:
matching = model_versions[-1:]
else:
matching = [v for v in model_versions if v.version_number == version]
try:
[model_version] = matching
return model_version
except ValueError:
if len(matching) == 0:
tpl = "No version of model {} with version number {} found"
else:
tpl = "Multiple versions of model {} with version number {} found"
raise ValueError(tpl.format(model_id, version))
def _environment_by_name(project_id, environment_name):
client = faculty.client("environment")
environments = client.list(project_id)
matching_environments = [
e for e in environments if e.name == environment_name
]
if len(matching_environments) == 1:
return matching_environments[0]
else:
if not matching_environments:
msg = 'no available environment of name "{}"'.format(
environment_name)
raise NameNotFoundError(msg)
else:
msg = ('more than one environment of name "{}", please select by '
"environment ID instead").format(environment_name)
raise AmbiguousNameError(msg)
def run_job(project, job, parameter_values, num_subruns):
"""Run a job.
\b
To run a single job:
$ faculty job run PROJECT JOB
\b
To run a single job with parameters:
$ faculty job run PROJECT JOB "foo=bar,eggs=spam"
\b
To run a job multiple times with different parameters:
$ faculty job run PROJECT JOB "foo=bar,eggs=spam" "foo=bar2,eggs=spam2"
\b
To run a job multiple times with no parameters:
$ faculty job run PROJECT JOB --num-subruns 2
"""
if num_subruns is None and not parameter_values:
parameter_values = [{}]
elif num_subruns is None and parameter_values:
pass
elif num_subruns is not None and not parameter_values:
parameter_values = [{} for _ in range(num_subruns)]
else:
_print_and_exit(
"Cannot set both 'parameter_values' and 'num_subruns'.", 64)
project_id, job_id = _resolve_job(project, job)
client = faculty.client("job")
client.create_run(project_id, job_id, parameter_values)
if len(parameter_values) == 1:
run_type = "run"
suffix = ""
else:
run_type = "run array"
suffix = " with {} subruns".format(len(parameter_values))
click.echo("Submitted {} of job '{}' in project '{}'{}".format(
run_type, job, project, suffix))
def __init__(self, store_uri, **_):
parsed_uri = urllib.parse.urlparse(store_uri)
if parsed_uri.scheme != "faculty":
raise ValueError("Not a faculty URI: {}".format(store_uri))
# Test for PROJECT_ID in netloc rather than path.
elif parsed_uri.netloc != "":
raise ValueError("Invalid URI {}. Netloc is reserved. "
"Did you mean 'faculty:{}".format(
store_uri, parsed_uri.netloc))
cleaned_path = parsed_uri.path.strip("/")
try:
self._project_id = UUID(cleaned_path)
except ValueError:
raise ValueError("{} in given URI {} is not a valid UUID".format(
cleaned_path, store_uri))
self._client = faculty.client("experiment")
def list_environments(project, verbose):
"""List your environments."""
client = faculty.client("environment")
project_id = _resolve_project(project)
environments = client.list(project_id)
if verbose:
if not environments:
click.echo("No environments.")
else:
click.echo(
tabulate(
[(e.name, e.id) for e in environments],
("Environment Name", "ID"),
tablefmt="plain",
))
else:
for environment in environments:
click.echo(environment.name)
