def list_templates(facade: TemplateFacade):
"""
List all available templates.
"""
try:
templates = facade.list()
table = render_table(templates, include_ind=True)
click.echo(table)
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def list_all(backend: BackendFacade):
"""
List all deployed kaos backend infrastructures.
"""
try:
available_contexts = backend.list()
if available_contexts:
backend.cache(available_contexts)
table = render_table(available_contexts, include_ind=False)
click.echo(table)
else:
click.echo("{} - No active builds found. Please run {} to deploy an environment".format(
click.style("Warning", bold=True, fg='yellow'),
click.style('kaos build deploy', bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def list_workspaces(facade: WorkspaceFacade):
"""
List all available workspaces.
"""
try:
workspaces = facade.list()
if len(workspaces) > 0:
facade.cache(workspaces)
table = render_table(workspaces)
click.echo(table)
else:
click.echo("{} - There are currently {} active workspaces - first run {}".format(
click.style("Warning", bold=True, fg='yellow'),
click.style('no', bold=True, fg='red'),
click.style("kaos workspace create", bold=True, fg='green')))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)
def deploy_job(facade: TrainFacade,
source_bundle,
data_bundle,
data_manifest,
hyperparams,
parallelism,
cpu,
memory,
gpu):
"""
Deploy training job with source (code + environment) and/or data bundle (and hyperparameters).
"""
try:
# ensure either bundle exists
if not source_bundle and not data_bundle and not data_manifest and not hyperparams:
click.echo("{} - {} and/or {} and/or {} need to be defined for training"
.format(click.style('Warning', bold=True, fg='yellow'),
click.style('--source_bundle', bold=True, fg='green'),
click.style('--data_bundle or --data_manifest', bold=True, fg='green'),
click.style('--hyperparams', bold=True, fg='green')), err=True)
sys.exit(1)
# process SOURCE bundle (POST /train/<name>)
if source_bundle:
click.echo("{} - Submitting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('source', bold=True, fg='blue'),
click.style(source_bundle, bold=True, fg='green', dim=True)))
with Compressor(label="Compressing source bundle", filename="model.zip", source_path=source_bundle) as c:
data = facade.upload_source_bundle(c, cpu=cpu, memory=memory, gpu=gpu)
# inform user regarding source bundle "naming"
source_glob = data['glob_name']
click.echo(" {} Setting {} bundle: {}\n".format(
click.style(SYM_CHECK, fg='green', bold=True),
click.style("source", fg='blue', bold=True),
click.style(f"/{source_glob}", fg='green', bold=True))
)
# process DATA bundle (POST /data/<name>/features)
if data_bundle:
click.echo("{} - Submitting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('data', bold=True, fg='blue'),
click.style(data_bundle, bold=True, fg='green', dim=True)))
with Compressor(label="Compressing data bundle", filename="data.zip", source_path=data_bundle) as c:
data = facade.upload_data_bundle(c, cpu=cpu, memory=memory, gpu=gpu)
# inform user regarding data bundle "naming"
data_glob = data['glob_name']
click.echo(" {} Setting {} bundle: {}\n".format(
click.style(SYM_CHECK, fg='green', bold=True),
click.style("data", fg='blue', bold=True),
click.style(f"/{data_glob}", fg='green', bold=True))
)
# process DATA manifest (POST /data/<name>/manifest)
if data_manifest:
click.echo("{} - Submitting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('data manifest', bold=True, fg='blue'),
click.style(data_manifest, bold=True, fg='green', dim=True)))
if validate_manifest_file(data_manifest):
data = facade.upload_manifest(data_manifest, cpu=cpu, memory=memory, gpu=gpu)
else:
click.echo("The manifest file is invalid")
sys.exit(1)
# inform user regarding data manifest "naming"
data_glob = data['glob_name']
click.echo(" {} Setting {} bundle: {}\n".format(
click.style(SYM_CHECK, fg='green', bold=True),
click.style("data manifest", fg='blue', bold=True),
click.style(f"/{data_glob}", fg='green', bold=True))
)
# process HYPERPARAMS (POST /data/<name>/params)
if hyperparams:
click.echo("{} - Submitting {} bundle: {}".format(
click.style("Info", bold=True, fg='green'),
click.style('hyperparams', bold=True, fg='blue'),
click.style(hyperparams, bold=True, fg='green', dim=True)))
data = facade.upload_hyperparams(hyperparams, cpu=cpu, memory=memory, gpu=gpu, parallelism=parallelism)
hyper_glob = data['glob_name']
click.echo(" {} Setting {} bundle: {}\n".format(
click.style(SYM_CHECK, fg='green', bold=True),
click.style("hyperparameters", fg='blue', bold=True),
click.style(f"/{hyper_glob}/*", fg='green', bold=True))
)
# inform user regarding actual hyperopt jobs
param_combinations = data['params']
click.echo("{} {}\n".format(
click.style("CURRENT HYPERPARAMETERS", fg='white', bold=True, underline=True),
click.style(f"({len(param_combinations)})", fg='green', bold=True)))
# set up simple table to iterate through response
table = PrettyTable(hrules=prettytable.ALL)
table.field_names = ['ind'] + list(param_combinations[0].keys())
for ind, d in enumerate(param_combinations):
table.add_row([ind] + list(d.values()))
click.echo(f"{table.get_string()}\n")
else:
facade.upload_hyperparams(cpu=cpu, memory=memory, gpu=gpu, parallelism=parallelism)
data = facade.inspect()
# update status of pipeline (i.e. its inputs)
data = [{
"Image": f"{SYM_CHECK}\n{data['image']}" if data['image'].find('null') < 0 else SYM_CROSS,
"Data": f"{SYM_CHECK}\n{data['data_glob']}" if data['data_glob'].find('null') < 0 else SYM_CROSS,
"Hyperparams": f"{SYM_CHECK}\n{data['hyper_glob']}" if data['hyper_glob'].find('null') < 0 else SYM_CROSS,
}]
# overwrite if code was "added"
if source_bundle:
data[0]["Image"] = f"{SYM_PROGRESS}\n<building>"
click.echo("{}\n".format(click.style("CURRENT TRAINING INPUTS", fg='white', bold=True, underline=True)))
click.echo(render_table(data, include_ind=False))
except Exception as e:
handle_specific_exception(e)
handle_exception(e)