def job_info(facade: TrainFacade, job_id, ind, sort_by, page_id):
"""
Describe a training job.
"""
try:
# ensure arguments are correctly defined
validate_inputs([job_id, ind], ['job_id', 'ind'])
# selection by index
if ind is not None:
job_id = facade.get_job_by_ind(ind)
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("info", bold=True),
click.style(job_id, bold=True, fg='green', dim=True)))
data = facade.info(job_id, sort_by, page_id)
formatted_info = render_job_info(data, sort_by)
click.echo(formatted_info)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_bundle(facade: TrainFacade, job_id, ind, out_dir, include_code, include_data, include_model, model_id):
"""
Download previously committed training bundle (code, data, models).
"""
try:
# if no inputs -> get model and code
if not (any([include_code, include_data, include_model])):
include_code = True
include_data = False
include_model = True
# ensure arguments are correctly defined
validate_inputs([job_id, ind], ['job_id', 'ind'])
# selection by index
if ind is not None:
job_id = facade.get_job_by_ind(ind)
name, content = facade.get_bundle(job_id, include_code, include_data, include_model, model_id)
extractor = Extractor(out_dir, name, job_id, label="Extracting train bundle")
click.echo("{} - Extracting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('train', bold=True, fg='blue'),
click.style(extractor.workspace_out_dir, bold=True, fg='green', dim=True)))
# build output directory (default = workspace)
extractor(content)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_logs(facade: TrainFacade, job_id, ind, out_dir):
"""
Retrieve logs from a training job.
"""
# build output directory (default = workspace)
# get logs for a specific job
try:
# ensure arguments are correctly defined
validate_inputs([job_id, ind], ['job_id', 'ind'])
# selection by index
if ind is not None:
job_id = facade.get_job_by_ind(ind)
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("logs", bold=True),
click.style(job_id, bold=True, fg='green', dim=True)))
logs = facade.get_train_logs(job_id)
click.echo_via_pager(logs)
facade.write_train_logs(job_id, logs, out_dir)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def kill_endpoint(facade: ServeFacade, endpoint, ind):
"""
Kill a running endpoint.
"""
try:
# ensure arguments are correctly defined
validate_inputs([endpoint, ind], ['endpoint', 'ind'])
# selection by index
if ind is not None:
endpoint = facade.get_endpoint_by_ind(ind)
# confirm "kill"
click.confirm('{} - Are you sure about killing endpoint {}?'.format(
click.style("Warning", bold=True, fg='yellow'),
click.style(endpoint, bold=True, fg='red')),
abort=True)
facade.delete(endpoint)
click.echo('{} - Successfully killed endpoint {}'.format(
click.style("Info", bold=True, fg='green'),
click.style(endpoint, bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_logs(facade: ServeFacade, endpoint, ind, out_dir):
"""
Retrieve logs from a running endpoint.
"""
try:
# ensure arguments are correctly defined
validate_inputs([endpoint, ind], ['endpoint', 'ind'])
# selection by index
if ind is not None:
endpoint = facade.get_endpoint_by_ind(ind)
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("logs", bold=True),
click.style(endpoint, bold=True, fg='green', dim=True)))
logs = facade.get_serve_logs(endpoint)
click.echo_via_pager(logs)
facade.write_serve_logs(endpoint, logs, out_dir)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def endpoint_provenance(facade: ServeFacade, endpoint, ind, out_dir):
"""
Retrieve provenance from a trained model.
"""
# extract provenance via DAG
try:
# ensure arguments are correctly defined
validate_inputs([endpoint, ind], ['endpoint', 'ind'])
# selection by index
if ind is not None:
endpoint = facade.get_endpoint_by_ind(ind)
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("provenance", bold=True),
click.style(endpoint, bold=True, fg='green', dim=True)))
out_fid, data = facade.provenance(out_dir, endpoint)
# render DAG (via dot)
Source(data).render(out_fid)
# remove raw DOT file
os.remove(out_fid)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_bundle(facade: ServeFacade, endpoint, ind, out_dir):
"""
Download previously committed serving bundle (code + environment).
"""
try:
# ensure arguments are correctly defined
validate_inputs([endpoint, ind], ['endpoint', 'ind'])
# selection by index
if ind is not None:
endpoint = facade.get_endpoint_by_ind(ind)
name, content = facade.get_bundle(endpoint)
extractor = Extractor(out_dir,
name,
endpoint,
label="Extracting serve bundle")
click.echo("{} - Extracting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('serve', bold=True, fg='blue'),
click.style(extractor.workspace_out_dir,
bold=True,
fg='green',
dim=True)))
# build output directory (default = workspace)
extractor(content)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def kill_notebook(facade: NotebookFacade, name, ind):
"""
Kill a running notebook.
"""
try:
# ensure arguments are correctly defined
validate_inputs([name, ind], ['name', 'ind'])
# selection by index
if ind is not None:
name = facade.get_notebook_by_ind(ind)
# confirm "kill"
click.confirm('{} - Are you sure about killing notebook {}?'.format(
click.style("Warning", bold=True, fg='yellow'),
click.style(name, bold=True, fg='red')),
abort=True)
facade.delete(name)
click.echo('{} - Successfully killed notebook {}'.format(
click.style("Info", bold=True, fg='green'),
click.style(name, bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def set_active_context(backend: BackendFacade, context: Optional[str] = None, ind: Optional[int] = None):
"""
Set current model environment workspace.
"""
try:
# ensure arguments are correctly defined
validate_inputs([context, ind], ['context', 'ind'])
# selection by index
available_contexts = backend.list()
if available_contexts:
if context:
is_context_set = backend.set_context_by_context(context)
if not is_context_set:
click.echo('Context {} invalid. It is not one of the existing deployments in {} '
.format(click.style(context, bold=True, fg='red'), click.style("kaos", bold=True)))
sys.exit(1)
click.echo("{} - Successfully set to context - {}".
format(click.style("Info", bold=True, fg='green'), click.style(context, bold=True, fg='green')))
if ind or ind == 0:
is_context_set, context = backend.set_context_by_index(ind)
if not is_context_set:
click.echo('Index {} invalid. It is not one of the existing deployments in {} '
.format(click.style(str(ind), bold=True, fg='red'), click.style("kaos", bold=True)))
sys.exit(1)
click.echo("{} - Successfully set to context - {}".
format(click.style("Info", bold=True, fg='green'), click.style(context, bold=True, fg='green')))
else:
click.echo("{} - No active builds found. Please run {} to deploy an environment".
format(click.style("Warning", bold=True, fg='yellow'),
click.style('kaos build deploy', bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def set_workspace(facade: WorkspaceFacade, name, ind):
"""
Set current model environment workspace.
"""
try:
# ensure arguments are correctly defined
validate_inputs([name, ind], ['name', 'ind'])
# selection by index
if ind is not None:
name = facade.get_workspace_by_ind(ind)
if facade.exists_by_name(name):
facade.set_by_name(name)
click.echo("{} - Successfully set {} workspace".format(
click.style("Info", bold=True, fg='green'),
click.style(name, bold=True, fg='green')))
else:
similar = facade.find_similar_workspaces(name)
if similar:
click.echo(
"{} - Workspace {} does not exist. Did you mean one of these?"
.format(click.style("Warning", bold=True, fg='yellow'),
click.style(name, bold=True, fg='green')))
click.echo('\n'.join(
map(lambda t: click.style(t, bold=True, fg='red'),
similar)))
else:
click.echo(
"{} - Workspace {} does not exist. Check the existing workspaces with `{}`"
.format(
click.style("Warning", bold=True, fg='yellow'),
click.style(name, bold=True, fg='green'),
click.style("kaos workspace list",
bold=True,
fg='white')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_build_logs(facade: ServeFacade, job_id, out_dir):
"""
Retrieve logs from a running endpoint.
"""
try:
# ensure arguments are correctly defined
validate_inputs([job_id], ['ind'])
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("build-logs", bold=True),
click.style(job_id, bold=True, fg='green', dim=True)))
logs = facade.get_build_logs(job_id)
click.echo_via_pager(logs)
facade.write_build_logs(job_id, logs, out_dir)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_build_logs(facade: NotebookFacade, job_id, out_dir):
"""
Retrieve logs from building notebook source image.
"""
# get logs for a specific job
try:
# ensure arguments are correctly defined
validate_inputs([job_id], ['job_id'])
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("build-logs", bold=True),
click.style(job_id, bold=True, fg='green', dim=True)))
logs = facade.get_build_logs(job_id)
click.echo_via_pager(logs)
facade.write_build_logs(job_id, logs, out_dir)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_template(facade: TemplateFacade, name, ind):
"""
Get template.
"""
try:
# ensure arguments are correctly defined
validate_inputs([name, ind], ['name', 'ind'])
# selection by index
if ind is not None:
name = facade.get_template_name_by_ind(ind)
name = facade.validate(name)
facade.download(name)
click.echo("{} - Successfully loaded {} template".format(
click.style("Info", bold=True, fg='green'),
click.style(name, bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def test_validate_input():
with pytest.raises(MissingArgumentError):
validate_inputs([None, None], ["a, b"])