def get_logs(facade: ServeFacade, endpoint, ind, out_dir):
"""
Retrieve logs from a running endpoint.
"""
try:
# ensure arguments are correctly defined
validate_inputs([endpoint, ind], ['endpoint', 'ind'])
# selection by index
if ind is not None:
endpoint = facade.get_endpoint_by_ind(ind)
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("logs", bold=True),
click.style(endpoint, bold=True, fg='green', dim=True)))
logs = facade.get_serve_logs(endpoint)
click.echo_via_pager(logs)
facade.write_serve_logs(endpoint, logs, out_dir)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def kill_endpoint(facade: ServeFacade, endpoint, ind):
"""
Kill a running endpoint.
"""
try:
# ensure arguments are correctly defined
validate_inputs([endpoint, ind], ['endpoint', 'ind'])
# selection by index
if ind is not None:
endpoint = facade.get_endpoint_by_ind(ind)
# confirm "kill"
click.confirm('{} - Are you sure about killing endpoint {}?'.format(
click.style("Warning", bold=True, fg='yellow'),
click.style(endpoint, bold=True, fg='red')),
abort=True)
facade.delete(endpoint)
click.echo('{} - Successfully killed endpoint {}'.format(
click.style("Info", bold=True, fg='green'),
click.style(endpoint, bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_bundle(facade: ServeFacade, endpoint, ind, out_dir):
"""
Download previously committed serving bundle (code + environment).
"""
try:
# ensure arguments are correctly defined
validate_inputs([endpoint, ind], ['endpoint', 'ind'])
# selection by index
if ind is not None:
endpoint = facade.get_endpoint_by_ind(ind)
name, content = facade.get_bundle(endpoint)
extractor = Extractor(out_dir,
name,
endpoint,
label="Extracting serve bundle")
click.echo("{} - Extracting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('serve', bold=True, fg='blue'),
click.style(extractor.workspace_out_dir,
bold=True,
fg='green',
dim=True)))
# build output directory (default = workspace)
extractor(content)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def endpoint_provenance(facade: ServeFacade, endpoint, ind, out_dir):
"""
Retrieve provenance from a trained model.
"""
# extract provenance via DAG
try:
# ensure arguments are correctly defined
validate_inputs([endpoint, ind], ['endpoint', 'ind'])
# selection by index
if ind is not None:
endpoint = facade.get_endpoint_by_ind(ind)
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("provenance", bold=True),
click.style(endpoint, bold=True, fg='green', dim=True)))
out_fid, data = facade.provenance(out_dir, endpoint)
# render DAG (via dot)
Source(data).render(out_fid)
# remove raw DOT file
os.remove(out_fid)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_build_logs(facade: ServeFacade, job_id, out_dir):
"""
Retrieve logs from a running endpoint.
"""
try:
# ensure arguments are correctly defined
validate_inputs([job_id], ['ind'])
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("build-logs", bold=True),
click.style(job_id, bold=True, fg='green', dim=True)))
logs = facade.get_build_logs(job_id)
click.echo_via_pager(logs)
facade.write_build_logs(job_id, logs, out_dir)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def list_endpoint(facade: ServeFacade):
"""
List all running endpoints.
"""
try:
data = facade.list()
building_table, n_building = render_queued_table(
data['building'],
header='BUILDING',
include_ind=False,
drop_cols={'progress'})
running_table = ""
n_running = len(data['endpoints'])
running_jobs = data['endpoints']
if n_running > 0:
running_table = \
f"\n{render_table(running_jobs, 'RUNNING', drop_cols={'code', 'image', 'model', 'progress'})}\n"
facade.cache(running_jobs)
if n_running + n_building > 30:
click.echo_via_pager(f'{building_table}{running_table}')
elif n_running + n_building > 0:
click.echo(f'{building_table}{running_table}')
else:
click.echo(
"{} - There are currently {} running endpoints - first run {}".
format(click.style("Warning", bold=True, fg='yellow'),
click.style('no', bold=True, fg='red'),
click.style("kaos serve deploy", bold=True,
fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def _create_facades(state=None, terraform=None):
template = TemplateFacade()
backend = BackendFacade(state, terraform)
workspace = WorkspaceFacade(state)
train = TrainFacade(state)
serve = ServeFacade(state)
notebook = NotebookFacade(state)
facades = {
BackendFacade: backend,
WorkspaceFacade: workspace,
TemplateFacade: template,
TrainFacade: train,
ServeFacade: serve,
NotebookFacade: notebook
}
return facades
def deploy_serve(facade: ServeFacade, source_bundle, model_id, cpu, memory,
gpu):
"""
Deploy endpoint with source (code + environment) and trained model id.
"""
try:
# inform user regarding source bundle "upload"
click.echo("{} - Submitting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('source', bold=True, fg='blue'),
click.style(source_bundle, bold=True, fg='green', dim=True)))
# process SOURCE bundle /inference/<name>/<model_id>
with Compressor(label="Compressing source bundle",
filename="source.zip",
source_path=source_bundle) as c:
data = facade.upload_source_bundle(c,
model_id,
cpu=cpu,
memory=memory,
gpu=gpu)
# inform user regarding model_id "naming"
if model_id:
click.echo(" {} Adding trained {}: {}".format(
click.style(SYM_CHECK, fg='green', bold=True),
click.style("model_id", fg='blue', bold=True),
click.style(model_id, fg='green', bold=True)))
# inform user regarding source bundle "naming"
source_glob = data['glob_name']
click.echo(" {} Setting {} bundle: {}".format(
click.style(SYM_CHECK, fg='green', bold=True),
click.style("source", fg='blue', bold=True),
click.style(f"/{source_glob}", fg='green', bold=True)))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)