def get_active_context(backend: BackendFacade):
"""
Set current model environment workspace.
"""
try:
active_context = backend.get_active_context()
if active_context:
click.echo("{} - Active context is - {}".
format(click.style("Info", bold=True, fg='green'),
click.style(active_context, bold=True, fg='green')))
else:
available_contexts = backend.list()
if available_contexts:
click.echo("{} - No {} context set. Set an active context using `{}`".
format(click.style("Warning", bold=True, fg='yellow'),
click.style("active", bold=True, fg='green'),
click.style("kaos build set", bold=True, fg='white')))
else:
click.echo("{} - No active builds found. Please run {} to deploy an environment".format(
click.style("Warning", bold=True, fg='yellow'),
click.style('kaos build deploy', bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def set_active_context(backend: BackendFacade, context: Optional[str] = None, ind: Optional[int] = None):
"""
Set current model environment workspace.
"""
try:
# ensure arguments are correctly defined
validate_inputs([context, ind], ['context', 'ind'])
# selection by index
available_contexts = backend.list()
if available_contexts:
if context:
is_context_set = backend.set_context_by_context(context)
if not is_context_set:
click.echo('Context {} invalid. It is not one of the existing deployments in {} '
.format(click.style(context, bold=True, fg='red'), click.style("kaos", bold=True)))
sys.exit(1)
click.echo("{} - Successfully set to context - {}".
format(click.style("Info", bold=True, fg='green'), click.style(context, bold=True, fg='green')))
if ind or ind == 0:
is_context_set, context = backend.set_context_by_index(ind)
if not is_context_set:
click.echo('Index {} invalid. It is not one of the existing deployments in {} '
.format(click.style(str(ind), bold=True, fg='red'), click.style("kaos", bold=True)))
sys.exit(1)
click.echo("{} - Successfully set to context - {}".
format(click.style("Info", bold=True, fg='green'), click.style(context, bold=True, fg='green')))
else:
click.echo("{} - No active builds found. Please run {} to deploy an environment".
format(click.style("Warning", bold=True, fg='yellow'),
click.style('kaos build deploy', bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def kill_endpoint(facade: ServeFacade, endpoint, ind):
"""
Kill a running endpoint.
"""
try:
# ensure arguments are correctly defined
validate_inputs([endpoint, ind], ['endpoint', 'ind'])
# selection by index
if ind is not None:
endpoint = facade.get_endpoint_by_ind(ind)
# confirm "kill"
click.confirm('{} - Are you sure about killing endpoint {}?'.format(
click.style("Warning", bold=True, fg='yellow'),
click.style(endpoint, bold=True, fg='red')),
abort=True)
facade.delete(endpoint)
click.echo('{} - Successfully killed endpoint {}'.format(
click.style("Info", bold=True, fg='green'),
click.style(endpoint, bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def job_info(facade: TrainFacade, job_id, ind, sort_by, page_id):
"""
Describe a training job.
"""
try:
# ensure arguments are correctly defined
validate_inputs([job_id, ind], ['job_id', 'ind'])
# selection by index
if ind is not None:
job_id = facade.get_job_by_ind(ind)
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("info", bold=True),
click.style(job_id, bold=True, fg='green', dim=True)))
data = facade.info(job_id, sort_by, page_id)
formatted_info = render_job_info(data, sort_by)
click.echo(formatted_info)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_bundle(facade: ServeFacade, endpoint, ind, out_dir):
"""
Download previously committed serving bundle (code + environment).
"""
try:
# ensure arguments are correctly defined
validate_inputs([endpoint, ind], ['endpoint', 'ind'])
# selection by index
if ind is not None:
endpoint = facade.get_endpoint_by_ind(ind)
name, content = facade.get_bundle(endpoint)
extractor = Extractor(out_dir,
name,
endpoint,
label="Extracting serve bundle")
click.echo("{} - Extracting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('serve', bold=True, fg='blue'),
click.style(extractor.workspace_out_dir,
bold=True,
fg='green',
dim=True)))
# build output directory (default = workspace)
extractor(content)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_logs(facade: ServeFacade, endpoint, ind, out_dir):
"""
Retrieve logs from a running endpoint.
"""
try:
# ensure arguments are correctly defined
validate_inputs([endpoint, ind], ['endpoint', 'ind'])
# selection by index
if ind is not None:
endpoint = facade.get_endpoint_by_ind(ind)
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("logs", bold=True),
click.style(endpoint, bold=True, fg='green', dim=True)))
logs = facade.get_serve_logs(endpoint)
click.echo_via_pager(logs)
facade.write_serve_logs(endpoint, logs, out_dir)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_bundle(facade: TrainFacade, job_id, ind, out_dir, include_code, include_data, include_model, model_id):
"""
Download previously committed training bundle (code, data, models).
"""
try:
# if no inputs -> get model and code
if not (any([include_code, include_data, include_model])):
include_code = True
include_data = False
include_model = True
# ensure arguments are correctly defined
validate_inputs([job_id, ind], ['job_id', 'ind'])
# selection by index
if ind is not None:
job_id = facade.get_job_by_ind(ind)
name, content = facade.get_bundle(job_id, include_code, include_data, include_model, model_id)
extractor = Extractor(out_dir, name, job_id, label="Extracting train bundle")
click.echo("{} - Extracting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('train', bold=True, fg='blue'),
click.style(extractor.workspace_out_dir, bold=True, fg='green', dim=True)))
# build output directory (default = workspace)
extractor(content)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def endpoint_provenance(facade: ServeFacade, endpoint, ind, out_dir):
"""
Retrieve provenance from a trained model.
"""
# extract provenance via DAG
try:
# ensure arguments are correctly defined
validate_inputs([endpoint, ind], ['endpoint', 'ind'])
# selection by index
if ind is not None:
endpoint = facade.get_endpoint_by_ind(ind)
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("provenance", bold=True),
click.style(endpoint, bold=True, fg='green', dim=True)))
out_fid, data = facade.provenance(out_dir, endpoint)
# render DAG (via dot)
Source(data).render(out_fid)
# remove raw DOT file
os.remove(out_fid)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def kill_notebook(facade: NotebookFacade, name, ind):
"""
Kill a running notebook.
"""
try:
# ensure arguments are correctly defined
validate_inputs([name, ind], ['name', 'ind'])
# selection by index
if ind is not None:
name = facade.get_notebook_by_ind(ind)
# confirm "kill"
click.confirm('{} - Are you sure about killing notebook {}?'.format(
click.style("Warning", bold=True, fg='yellow'),
click.style(name, bold=True, fg='red')),
abort=True)
facade.delete(name)
click.echo('{} - Successfully killed notebook {}'.format(
click.style("Info", bold=True, fg='green'),
click.style(name, bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_logs(facade: TrainFacade, job_id, ind, out_dir):
"""
Retrieve logs from a training job.
"""
# build output directory (default = workspace)
# get logs for a specific job
try:
# ensure arguments are correctly defined
validate_inputs([job_id, ind], ['job_id', 'ind'])
# selection by index
if ind is not None:
job_id = facade.get_job_by_ind(ind)
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("logs", bold=True),
click.style(job_id, bold=True, fg='green', dim=True)))
logs = facade.get_train_logs(job_id)
click.echo_via_pager(logs)
facade.write_train_logs(job_id, logs, out_dir)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def list_templates(facade: TemplateFacade):
"""
List all available templates.
"""
try:
templates = facade.list()
table = render_table(templates, include_ind=True)
click.echo(table)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def workspace_info(facade: WorkspaceFacade):
"""
Identify all running resources within a workspace.
"""
try:
info = facade.info()['response']
click.echo('\n--- Workspace {} ---\n'.format(click.style(info['name'], bold=True, fg='green')))
print_list('Pipelines', info['pipelines'])
print_list('Repos', info['repos'])
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def current_workspace(facade: WorkspaceFacade):
"""
Get current model environment workspace.
"""
try:
# return <name> as workspace
name = facade.current()
click.echo("{} - Workspace {} is currently set".format(
click.style("Info", bold=True, fg='green'),
click.style(name, bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def job_provenance(facade: TrainFacade, job_id):
"""
Kill a running training / building train job.
"""
try:
facade.kill_job(job_id)
click.echo("{} - Job {} successfully killed".format(
click.style("Info", bold=True, fg='green'),
click.style(job_id, bold=True, fg='green', dim=True)))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def create_workspace(facade: WorkspaceFacade, name):
"""
Creates a workspace.
"""
try:
facade.create(name)
click.echo("{} - Successfully set {} workspace".format(
click.style("Info", bold=True, fg='green'),
click.style(name, bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def set_workspace(facade: WorkspaceFacade, name, ind):
"""
Set current model environment workspace.
"""
try:
# ensure arguments are correctly defined
validate_inputs([name, ind], ['name', 'ind'])
# selection by index
if ind is not None:
name = facade.get_workspace_by_ind(ind)
if facade.exists_by_name(name):
facade.set_by_name(name)
click.echo("{} - Successfully set {} workspace".format(
click.style("Info", bold=True, fg='green'),
click.style(name, bold=True, fg='green')))
else:
similar = facade.find_similar_workspaces(name)
if similar:
click.echo(
"{} - Workspace {} does not exist. Did you mean one of these?"
.format(click.style("Warning", bold=True, fg='yellow'),
click.style(name, bold=True, fg='green')))
click.echo('\n'.join(
map(lambda t: click.style(t, bold=True, fg='red'),
similar)))
else:
click.echo(
"{} - Workspace {} does not exist. Check the existing workspaces with `{}`"
.format(
click.style("Warning", bold=True, fg='yellow'),
click.style(name, bold=True, fg='green'),
click.style("kaos workspace list",
bold=True,
fg='white')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def list_all(backend: BackendFacade):
"""
List all deployed kaos backend infrastructures.
"""
try:
available_contexts = backend.list()
if available_contexts:
backend.cache(available_contexts)
table = render_table(available_contexts, include_ind=False)
click.echo(table)
else:
click.echo("{} - No active builds found. Please run {} to deploy an environment".format(
click.style("Warning", bold=True, fg='yellow'),
click.style('kaos build deploy', bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def destroy(backend: BackendFacade, cloud, env, verbose, yes):
"""
Destroy kaos backend infrastructure based on selected provider.
"""
# validate ENV
env = validate_build_env(cloud, env)
if not yes:
# confirm creation of backend
if env:
click.confirm(
'{} - Are you sure about destroying {} [{}] backend in {}?'.
format(click.style("Warning", bold=True, fg='yellow'),
click.style('kaos', bold=True),
click.style(env, bold=True, fg='blue'),
click.style(cloud, bold=True, fg='red')),
abort=True)
else:
click.confirm(
'{} - Are you sure about destroying {} backend in {}?'.format(
click.style("Warning", bold=True, fg='yellow'),
click.style('kaos', bold=True),
click.style(cloud, bold=True, fg='red')),
abort=True)
try:
backend.destroy(cloud, env, verbose=verbose)
if env:
click.echo(
"{} - Successfully destroyed {} [{}] environment".format(
click.style("Info", bold=True, fg='green'),
click.style('kaos', bold=True),
click.style(env, bold=True, fg='blue')))
else:
click.echo("{} - Successfully destroyed {} environment".format(
click.style("Info", bold=True, fg='green'),
click.style('kaos', bold=True)))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_build_logs(facade: ServeFacade, job_id, out_dir):
"""
Retrieve logs from a running endpoint.
"""
try:
# ensure arguments are correctly defined
validate_inputs([job_id], ['ind'])
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("build-logs", bold=True),
click.style(job_id, bold=True, fg='green', dim=True)))
logs = facade.get_build_logs(job_id)
click.echo_via_pager(logs)
facade.write_build_logs(job_id, logs, out_dir)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def job_provenance(facade: TrainFacade, model_id, out_dir):
"""
Retrieve provenance from a trained model.
"""
# extract provenance via DAG
try:
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("provenance", bold=True),
click.style(model_id, bold=True, fg='green', dim=True)))
out, data = facade.provenance(out_dir, model_id)
# render DAG (via dot)
Source(data).render(out)
os.remove(out)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def build(self, provider, env, local_backend=False, verbose=False):
env_state = EnvironmentState.initialize(provider, env)
if not env_state.if_build_dir_exists:
build_dir(env_state.build_dir)
extra_vars = self._get_vars(provider, env_state.build_dir)
self.tf_service.cd_dir(env_state.build_dir)
self.tf_service.set_verbose(verbose)
directory = self._tf_init(provider, env, local_backend, destroying=False)
self.tf_service.plan(directory, extra_vars)
self.tf_service.apply(directory, extra_vars)
self.tf_service.execute()
# check if the deployed successfully
# Refresh environment states after terraform service operations
env_state = EnvironmentState.initialize(provider, env)
if env_state.if_tfstate_exists:
url, kubeconfig = self._parse_config(env_state.build_dir)
current_context = provider if provider in [DOCKER, MINIKUBE] else f"{provider}_{env}"
self.state_service.set(DEFAULT, user=USER)
self._set_context_list(current_context)
self._set_active_context(current_context)
self.state_service.set(current_context)
try:
self.state_service.set_section(current_context, BACKEND, url=url, token=uuid.uuid4())
self.state_service.set_section(current_context, INFRASTRUCTURE, kubeconfig=kubeconfig)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
self.state_service.write()
return True, env_state
return False, env_state
def get_build_logs(facade: NotebookFacade, job_id, out_dir):
"""
Retrieve logs from building notebook source image.
"""
# get logs for a specific job
try:
# ensure arguments are correctly defined
validate_inputs([job_id], ['job_id'])
click.echo("{} - Retrieving {} from {}".format(
click.style("Info", bold=True, fg='green'),
click.style("build-logs", bold=True),
click.style(job_id, bold=True, fg='green', dim=True)))
logs = facade.get_build_logs(job_id)
click.echo_via_pager(logs)
facade.write_build_logs(job_id, logs, out_dir)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def list_notebooks(facade: NotebookFacade):
"""
List all available running notebooks.
"""
try:
data = facade.list()
building_table, n_building = render_queued_table(
data['building'],
header='BUILDING',
include_ind=False,
drop_cols={'progress'})
running_table = ""
running_jobs = data['notebooks']
n_running = len(running_jobs)
if n_running > 0:
running_table = \
f"\n{render_table(running_jobs, 'RUNNING', drop_cols={'code', 'image', 'model', 'progress'})}\n"
facade.cache(running_jobs)
if n_running + n_building > 30:
click.echo_via_pager(f'{building_table}{running_table}')
elif n_running + n_building > 0:
click.echo(f'{building_table}{running_table}')
else:
click.echo(
"{} - There are currently {} active notebooks - first run {}".
format(
click.style("Warning", bold=True, fg='yellow'),
click.style('no', bold=True, fg='red'),
click.style("kaos notebook deploy", bold=True,
fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def list_workspaces(facade: WorkspaceFacade):
"""
List all available workspaces.
"""
try:
workspaces = facade.list()
if len(workspaces) > 0:
facade.cache(workspaces)
table = render_table(workspaces)
click.echo(table)
else:
click.echo("{} - There are currently {} active workspaces - first run {}".format(
click.style("Warning", bold=True, fg='yellow'),
click.style('no', bold=True, fg='red'),
click.style("kaos workspace create", bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def deploy_serve(facade: ServeFacade, source_bundle, model_id, cpu, memory,
gpu):
"""
Deploy endpoint with source (code + environment) and trained model id.
"""
try:
# inform user regarding source bundle "upload"
click.echo("{} - Submitting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('source', bold=True, fg='blue'),
click.style(source_bundle, bold=True, fg='green', dim=True)))
# process SOURCE bundle /inference/<name>/<model_id>
with Compressor(label="Compressing source bundle",
filename="source.zip",
source_path=source_bundle) as c:
data = facade.upload_source_bundle(c,
model_id,
cpu=cpu,
memory=memory,
gpu=gpu)
# inform user regarding model_id "naming"
if model_id:
click.echo(" {} Adding trained {}: {}".format(
click.style(SYM_CHECK, fg='green', bold=True),
click.style("model_id", fg='blue', bold=True),
click.style(model_id, fg='green', bold=True)))
# inform user regarding source bundle "naming"
source_glob = data['glob_name']
click.echo(" {} Setting {} bundle: {}".format(
click.style(SYM_CHECK, fg='green', bold=True),
click.style("source", fg='blue', bold=True),
click.style(f"/{source_glob}", fg='green', bold=True)))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def kill_workspace(facade: WorkspaceFacade):
"""
Kill all resources running in a workspace.
"""
try:
name = facade.current()
# confirm kill
click.confirm('{} - Are you sure about killing all {} resources?'.format(
click.style("Warning", bold=True, fg='yellow'),
click.style(name, bold=True, fg='red')),
abort=True)
name = facade.delete()
click.echo('{} - Successfully killed all {} resources'.format(
click.style("Info", bold=True, fg='green'),
click.style(name, bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def list_jobs(facade: TrainFacade):
"""
List all training jobs.
"""
try:
data = facade.list()['response']
building_table, n_building = render_queued_table(data['building'], header='BUILDING', include_ind=False,
drop_cols={'hyperopt', 'progress'})
ingesting_table, n_ingesting = render_queued_table(data['ingesting'], header='INGESTING', include_ind=False,
drop_cols={'hyperopt'})
training_table = ""
n_training = len(data['training'])
training_jobs = data['training']
if n_training > 0:
training_table = f"\n{render_table(training_jobs, 'TRAINING', drop_cols={'progress'})}\n"
facade.cache(training_jobs)
n_jobs = n_training + n_building + n_ingesting
if n_jobs > 30:
click.echo_via_pager(f'{ingesting_table}{building_table}{training_table}')
elif n_jobs > 0:
click.echo(f'{ingesting_table}{building_table}{training_table}')
else:
click.echo("{} - There are currently {} training jobs - first run {}".format(
click.style("Warning", bold=True, fg='yellow'),
click.style('no', bold=True, fg='red'),
click.style("kaos train deploy", bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def get_template(facade: TemplateFacade, name, ind):
"""
Get template.
"""
try:
# ensure arguments are correctly defined
validate_inputs([name, ind], ['name', 'ind'])
# selection by index
if ind is not None:
name = facade.get_template_name_by_ind(ind)
name = facade.validate(name)
facade.download(name)
click.echo("{} - Successfully loaded {} template".format(
click.style("Info", bold=True, fg='green'),
click.style(name, bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def deploy(backend: BackendFacade, cloud: str, env: str, force: bool, verbose: bool, yes: bool, local_backend: bool):
"""
Deploy kaos backend infrastructure based on selected provider.
"""
env_state = EnvironmentState.initialize(cloud, env)
if env_state.if_tfstate_exists and not force:
click.echo('{} - {} backend is already built.'.format(click.style("Aborting", bold=True, fg='red'),
click.style("kaos", bold=True)))
sys.exit(1)
elif env_state.if_tfstate_exists and force:
click.echo('{} - Performing {} build of the backend'.format(
click.style("Warning", bold=True, fg='yellow'),
click.style("force", bold=True)))
env_state.remove_terraform_files()
# set env variable appropriately
env_state.set_build_env()
cloud = env_state.cloud
env = env_state.env
if not yes:
# confirm creation of backend
if env_state.env:
click.confirm(
'{} - Are you sure about building {} [{}] backend in {}?'.format(
click.style("Warning", bold=True, fg='yellow'),
click.style('kaos', bold=True),
click.style(env_state.env, bold=True, fg='blue'),
click.style(env_state.cloud, bold=True, fg='red')),
abort=True)
else:
click.confirm(
'{} - Are you sure about building {} backend in {}?'.format(
click.style("Warning", bold=True, fg='yellow'),
click.style('kaos', bold=True),
click.style(env_state.cloud, bold=True, fg='red')),
abort=True)
if local_backend:
click.echo('{} - Building with {} terraform backend state'.format(
click.style("Info", bold=True, fg='green'),
click.style("local", bold=True)))
if local_backend and env_state.cloud in [DOCKER, MINIKUBE]:
click.echo('{} - local backend (-l/--local_backend) has no effect for {}'.format(
click.style("Info", bold=True, fg='green'),
click.style(env_state.cloud, bold=True, fg='red')))
# validate unused port for DOCKER
if env_state.cloud == DOCKER and not validate_unused_port(80):
# If the force build flag was set to True and the kaos backend
# was already built then skip the warning; otherwise, warn
# the user that the port is already taken by another service
# and issue a sys exit
if not (force and env_state.if_tfstate_exists):
click.echo(
"{} - Network port {} is used but is needed for building {} backend in {}".format(
click.style("Warning", bold=True, fg='yellow'),
click.style("80", bold=True),
click.style("kaos", bold=True),
click.style(env_state.cloud, bold=True, fg='red')))
sys.exit(1)
try:
is_built_successfully, env_state = backend.build(env_state.cloud,
env_state.env,
local_backend=local_backend,
verbose=verbose)
if is_built_successfully:
if verbose:
click.echo("\n{} - Endpoint successfully set to {}".format(
click.style("Info", bold=True, fg='green'),
click.style(backend.url, bold=True, fg='green')))
if is_cloud_provider(env_state.cloud):
kubeconfig = os.path.abspath(backend.kubeconfig)
click.echo("\n{} - To interact with the Kubernetes cluster:\n {}"
.format(click.style("Info", bold=True, fg='green'),
click.style("export KUBECONFIG=" + kubeconfig,
bold=True, fg='red')))
if env:
click.echo("{} - Successfully built {} [{}] environment".format(
click.style("Info", bold=True, fg='green'),
click.style('kaos', bold=True),
click.style(env, bold=True, fg='blue')))
else:
click.echo("{} - Successfully built {} environment".format(
click.style("Info", bold=True, fg='green'),
click.style('kaos', bold=True)))
else:
click.echo("{} - Deployment Unsuccessful while creating {} [{} {}] environment".format(
click.style("Error", bold=True, fg='red'),
click.style('kaos', bold=True),
click.style(cloud, bold=True, fg='red'),
click.style(env, bold=True, fg='red'))),
sys.exit(1)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def deploy_job(facade: TrainFacade,
source_bundle,
data_bundle,
data_manifest,
hyperparams,
parallelism,
cpu,
memory,
gpu):
"""
Deploy training job with source (code + environment) and/or data bundle (and hyperparameters).
"""
try:
# ensure either bundle exists
if not source_bundle and not data_bundle and not data_manifest and not hyperparams:
click.echo("{} - {} and/or {} and/or {} need to be defined for training"
.format(click.style('Warning', bold=True, fg='yellow'),
click.style('--source_bundle', bold=True, fg='green'),
click.style('--data_bundle or --data_manifest', bold=True, fg='green'),
click.style('--hyperparams', bold=True, fg='green')), err=True)
sys.exit(1)
# process SOURCE bundle (POST /train/<name>)
if source_bundle:
click.echo("{} - Submitting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('source', bold=True, fg='blue'),
click.style(source_bundle, bold=True, fg='green', dim=True)))
with Compressor(label="Compressing source bundle", filename="model.zip", source_path=source_bundle) as c:
data = facade.upload_source_bundle(c, cpu=cpu, memory=memory, gpu=gpu)
# inform user regarding source bundle "naming"
source_glob = data['glob_name']
click.echo(" {} Setting {} bundle: {}\n".format(
click.style(SYM_CHECK, fg='green', bold=True),
click.style("source", fg='blue', bold=True),
click.style(f"/{source_glob}", fg='green', bold=True))
)
# process DATA bundle (POST /data/<name>/features)
if data_bundle:
click.echo("{} - Submitting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('data', bold=True, fg='blue'),
click.style(data_bundle, bold=True, fg='green', dim=True)))
with Compressor(label="Compressing data bundle", filename="data.zip", source_path=data_bundle) as c:
data = facade.upload_data_bundle(c, cpu=cpu, memory=memory, gpu=gpu)
# inform user regarding data bundle "naming"
data_glob = data['glob_name']
click.echo(" {} Setting {} bundle: {}\n".format(
click.style(SYM_CHECK, fg='green', bold=True),
click.style("data", fg='blue', bold=True),
click.style(f"/{data_glob}", fg='green', bold=True))
)
# process DATA manifest (POST /data/<name>/manifest)
if data_manifest:
click.echo("{} - Submitting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('data manifest', bold=True, fg='blue'),
click.style(data_manifest, bold=True, fg='green', dim=True)))
if validate_manifest_file(data_manifest):
data = facade.upload_manifest(data_manifest, cpu=cpu, memory=memory, gpu=gpu)
else:
click.echo("The manifest file is invalid")
sys.exit(1)
# inform user regarding data manifest "naming"
data_glob = data['glob_name']
click.echo(" {} Setting {} bundle: {}\n".format(
click.style(SYM_CHECK, fg='green', bold=True),
click.style("data manifest", fg='blue', bold=True),
click.style(f"/{data_glob}", fg='green', bold=True))
)
# process HYPERPARAMS (POST /data/<name>/params)
if hyperparams:
click.echo("{} - Submitting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('hyperparams', bold=True, fg='blue'),
click.style(hyperparams, bold=True, fg='green', dim=True)))
data = facade.upload_hyperparams(hyperparams, cpu=cpu, memory=memory, gpu=gpu, parallelism=parallelism)
hyper_glob = data['glob_name']
click.echo(" {} Setting {} bundle: {}\n".format(
click.style(SYM_CHECK, fg='green', bold=True),
click.style("hyperparameters", fg='blue', bold=True),
click.style(f"/{hyper_glob}/*", fg='green', bold=True))
)
# inform user regarding actual hyperopt jobs
param_combinations = data['params']
click.echo("{} {}\n".format(
click.style("CURRENT HYPERPARAMETERS", fg='white', bold=True, underline=True),
click.style(f"({len(param_combinations)})", fg='green', bold=True)))
# set up simple table to iterate through response
table = PrettyTable(hrules=prettytable.ALL)
table.field_names = ['ind'] + list(param_combinations[0].keys())
for ind, d in enumerate(param_combinations):
table.add_row([ind] + list(d.values()))
click.echo(f"{table.get_string()}\n")
else:
facade.upload_hyperparams(cpu=cpu, memory=memory, gpu=gpu, parallelism=parallelism)
data = facade.inspect()
# update status of pipeline (i.e. its inputs)
data = [{
"Image": f"{SYM_CHECK}\n{data['image']}" if data['image'].find('null') < 0 else SYM_CROSS,
"Data": f"{SYM_CHECK}\n{data['data_glob']}" if data['data_glob'].find('null') < 0 else SYM_CROSS,
"Hyperparams": f"{SYM_CHECK}\n{data['hyper_glob']}" if data['hyper_glob'].find('null') < 0 else SYM_CROSS,
}]
# overwrite if code was "added"
if source_bundle:
data[0]["Image"] = f"{SYM_PROGRESS}\n<building>"
click.echo("{}\n".format(click.style("CURRENT TRAINING INPUTS", fg='white', bold=True, underline=True)))
click.echo(render_table(data, include_ind=False))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
