def get_active_context(backend: BackendFacade): """ Set current model environment workspace. """ try: active_context = backend.get_active_context() if active_context: click.echo("{} - Active context is - {}". format(click.style("Info", bold=True, fg='green'), click.style(active_context, bold=True, fg='green'))) else: available_contexts = backend.list() if available_contexts: click.echo("{} - No {} context set. Set an active context using `{}`". format(click.style("Warning", bold=True, fg='yellow'), click.style("active", bold=True, fg='green'), click.style("kaos build set", bold=True, fg='white'))) else: click.echo("{} - No active builds found. Please run {} to deploy an environment".format( click.style("Warning", bold=True, fg='yellow'), click.style('kaos build deploy', bold=True, fg='green'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def set_active_context(backend: BackendFacade, context: Optional[str] = None, ind: Optional[int] = None): """ Set current model environment workspace. """ try: # ensure arguments are correctly defined validate_inputs([context, ind], ['context', 'ind']) # selection by index available_contexts = backend.list() if available_contexts: if context: is_context_set = backend.set_context_by_context(context) if not is_context_set: click.echo('Context {} invalid. It is not one of the existing deployments in {} ' .format(click.style(context, bold=True, fg='red'), click.style("kaos", bold=True))) sys.exit(1) click.echo("{} - Successfully set to context - {}". format(click.style("Info", bold=True, fg='green'), click.style(context, bold=True, fg='green'))) if ind or ind == 0: is_context_set, context = backend.set_context_by_index(ind) if not is_context_set: click.echo('Index {} invalid. It is not one of the existing deployments in {} ' .format(click.style(str(ind), bold=True, fg='red'), click.style("kaos", bold=True))) sys.exit(1) click.echo("{} - Successfully set to context - {}". format(click.style("Info", bold=True, fg='green'), click.style(context, bold=True, fg='green'))) else: click.echo("{} - No active builds found. Please run {} to deploy an environment". format(click.style("Warning", bold=True, fg='yellow'), click.style('kaos build deploy', bold=True, fg='green'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def kill_endpoint(facade: ServeFacade, endpoint, ind): """ Kill a running endpoint. """ try: # ensure arguments are correctly defined validate_inputs([endpoint, ind], ['endpoint', 'ind']) # selection by index if ind is not None: endpoint = facade.get_endpoint_by_ind(ind) # confirm "kill" click.confirm('{} - Are you sure about killing endpoint {}?'.format( click.style("Warning", bold=True, fg='yellow'), click.style(endpoint, bold=True, fg='red')), abort=True) facade.delete(endpoint) click.echo('{} - Successfully killed endpoint {}'.format( click.style("Info", bold=True, fg='green'), click.style(endpoint, bold=True, fg='green'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def job_info(facade: TrainFacade, job_id, ind, sort_by, page_id): """ Describe a training job. """ try: # ensure arguments are correctly defined validate_inputs([job_id, ind], ['job_id', 'ind']) # selection by index if ind is not None: job_id = facade.get_job_by_ind(ind) click.echo("{} - Retrieving {} from {}".format( click.style("Info", bold=True, fg='green'), click.style("info", bold=True), click.style(job_id, bold=True, fg='green', dim=True))) data = facade.info(job_id, sort_by, page_id) formatted_info = render_job_info(data, sort_by) click.echo(formatted_info) except Exception as e: handle_specific_exception(e) handle_exception(e)
def get_bundle(facade: ServeFacade, endpoint, ind, out_dir): """ Download previously committed serving bundle (code + environment). """ try: # ensure arguments are correctly defined validate_inputs([endpoint, ind], ['endpoint', 'ind']) # selection by index if ind is not None: endpoint = facade.get_endpoint_by_ind(ind) name, content = facade.get_bundle(endpoint) extractor = Extractor(out_dir, name, endpoint, label="Extracting serve bundle") click.echo("{} - Extracting {} bundle: {}".format( click.style("Info", bold=True, fg='green'), click.style('serve', bold=True, fg='blue'), click.style(extractor.workspace_out_dir, bold=True, fg='green', dim=True))) # build output directory (default = workspace) extractor(content) except Exception as e: handle_specific_exception(e) handle_exception(e)
def get_logs(facade: ServeFacade, endpoint, ind, out_dir): """ Retrieve logs from a running endpoint. """ try: # ensure arguments are correctly defined validate_inputs([endpoint, ind], ['endpoint', 'ind']) # selection by index if ind is not None: endpoint = facade.get_endpoint_by_ind(ind) click.echo("{} - Retrieving {} from {}".format( click.style("Info", bold=True, fg='green'), click.style("logs", bold=True), click.style(endpoint, bold=True, fg='green', dim=True))) logs = facade.get_serve_logs(endpoint) click.echo_via_pager(logs) facade.write_serve_logs(endpoint, logs, out_dir) except Exception as e: handle_specific_exception(e) handle_exception(e)
def get_bundle(facade: TrainFacade, job_id, ind, out_dir, include_code, include_data, include_model, model_id): """ Download previously committed training bundle (code, data, models). """ try: # if no inputs -> get model and code if not (any([include_code, include_data, include_model])): include_code = True include_data = False include_model = True # ensure arguments are correctly defined validate_inputs([job_id, ind], ['job_id', 'ind']) # selection by index if ind is not None: job_id = facade.get_job_by_ind(ind) name, content = facade.get_bundle(job_id, include_code, include_data, include_model, model_id) extractor = Extractor(out_dir, name, job_id, label="Extracting train bundle") click.echo("{} - Extracting {} bundle: {}".format( click.style("Info", bold=True, fg='green'), click.style('train', bold=True, fg='blue'), click.style(extractor.workspace_out_dir, bold=True, fg='green', dim=True))) # build output directory (default = workspace) extractor(content) except Exception as e: handle_specific_exception(e) handle_exception(e)
def endpoint_provenance(facade: ServeFacade, endpoint, ind, out_dir): """ Retrieve provenance from a trained model. """ # extract provenance via DAG try: # ensure arguments are correctly defined validate_inputs([endpoint, ind], ['endpoint', 'ind']) # selection by index if ind is not None: endpoint = facade.get_endpoint_by_ind(ind) click.echo("{} - Retrieving {} from {}".format( click.style("Info", bold=True, fg='green'), click.style("provenance", bold=True), click.style(endpoint, bold=True, fg='green', dim=True))) out_fid, data = facade.provenance(out_dir, endpoint) # render DAG (via dot) Source(data).render(out_fid) # remove raw DOT file os.remove(out_fid) except Exception as e: handle_specific_exception(e) handle_exception(e)
def kill_notebook(facade: NotebookFacade, name, ind): """ Kill a running notebook. """ try: # ensure arguments are correctly defined validate_inputs([name, ind], ['name', 'ind']) # selection by index if ind is not None: name = facade.get_notebook_by_ind(ind) # confirm "kill" click.confirm('{} - Are you sure about killing notebook {}?'.format( click.style("Warning", bold=True, fg='yellow'), click.style(name, bold=True, fg='red')), abort=True) facade.delete(name) click.echo('{} - Successfully killed notebook {}'.format( click.style("Info", bold=True, fg='green'), click.style(name, bold=True, fg='green'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def get_logs(facade: TrainFacade, job_id, ind, out_dir): """ Retrieve logs from a training job. """ # build output directory (default = workspace) # get logs for a specific job try: # ensure arguments are correctly defined validate_inputs([job_id, ind], ['job_id', 'ind']) # selection by index if ind is not None: job_id = facade.get_job_by_ind(ind) click.echo("{} - Retrieving {} from {}".format( click.style("Info", bold=True, fg='green'), click.style("logs", bold=True), click.style(job_id, bold=True, fg='green', dim=True))) logs = facade.get_train_logs(job_id) click.echo_via_pager(logs) facade.write_train_logs(job_id, logs, out_dir) except Exception as e: handle_specific_exception(e) handle_exception(e)
def list_templates(facade: TemplateFacade): """ List all available templates. """ try: templates = facade.list() table = render_table(templates, include_ind=True) click.echo(table) except Exception as e: handle_specific_exception(e) handle_exception(e)
def workspace_info(facade: WorkspaceFacade): """ Identify all running resources within a workspace. """ try: info = facade.info()['response'] click.echo('\n--- Workspace {} ---\n'.format(click.style(info['name'], bold=True, fg='green'))) print_list('Pipelines', info['pipelines']) print_list('Repos', info['repos']) except Exception as e: handle_specific_exception(e) handle_exception(e)
def current_workspace(facade: WorkspaceFacade): """ Get current model environment workspace. """ try: # return <name> as workspace name = facade.current() click.echo("{} - Workspace {} is currently set".format( click.style("Info", bold=True, fg='green'), click.style(name, bold=True, fg='green'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def job_provenance(facade: TrainFacade, job_id): """ Kill a running training / building train job. """ try: facade.kill_job(job_id) click.echo("{} - Job {} successfully killed".format( click.style("Info", bold=True, fg='green'), click.style(job_id, bold=True, fg='green', dim=True))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def create_workspace(facade: WorkspaceFacade, name): """ Creates a workspace. """ try: facade.create(name) click.echo("{} - Successfully set {} workspace".format( click.style("Info", bold=True, fg='green'), click.style(name, bold=True, fg='green'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def set_workspace(facade: WorkspaceFacade, name, ind): """ Set current model environment workspace. """ try: # ensure arguments are correctly defined validate_inputs([name, ind], ['name', 'ind']) # selection by index if ind is not None: name = facade.get_workspace_by_ind(ind) if facade.exists_by_name(name): facade.set_by_name(name) click.echo("{} - Successfully set {} workspace".format( click.style("Info", bold=True, fg='green'), click.style(name, bold=True, fg='green'))) else: similar = facade.find_similar_workspaces(name) if similar: click.echo( "{} - Workspace {} does not exist. Did you mean one of these?" .format(click.style("Warning", bold=True, fg='yellow'), click.style(name, bold=True, fg='green'))) click.echo('\n'.join( map(lambda t: click.style(t, bold=True, fg='red'), similar))) else: click.echo( "{} - Workspace {} does not exist. Check the existing workspaces with `{}`" .format( click.style("Warning", bold=True, fg='yellow'), click.style(name, bold=True, fg='green'), click.style("kaos workspace list", bold=True, fg='white'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def list_all(backend: BackendFacade): """ List all deployed kaos backend infrastructures. """ try: available_contexts = backend.list() if available_contexts: backend.cache(available_contexts) table = render_table(available_contexts, include_ind=False) click.echo(table) else: click.echo("{} - No active builds found. Please run {} to deploy an environment".format( click.style("Warning", bold=True, fg='yellow'), click.style('kaos build deploy', bold=True, fg='green'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def destroy(backend: BackendFacade, cloud, env, verbose, yes): """ Destroy kaos backend infrastructure based on selected provider. """ # validate ENV env = validate_build_env(cloud, env) if not yes: # confirm creation of backend if env: click.confirm( '{} - Are you sure about destroying {} [{}] backend in {}?'. format(click.style("Warning", bold=True, fg='yellow'), click.style('kaos', bold=True), click.style(env, bold=True, fg='blue'), click.style(cloud, bold=True, fg='red')), abort=True) else: click.confirm( '{} - Are you sure about destroying {} backend in {}?'.format( click.style("Warning", bold=True, fg='yellow'), click.style('kaos', bold=True), click.style(cloud, bold=True, fg='red')), abort=True) try: backend.destroy(cloud, env, verbose=verbose) if env: click.echo( "{} - Successfully destroyed {} [{}] environment".format( click.style("Info", bold=True, fg='green'), click.style('kaos', bold=True), click.style(env, bold=True, fg='blue'))) else: click.echo("{} - Successfully destroyed {} environment".format( click.style("Info", bold=True, fg='green'), click.style('kaos', bold=True))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def get_build_logs(facade: ServeFacade, job_id, out_dir): """ Retrieve logs from a running endpoint. """ try: # ensure arguments are correctly defined validate_inputs([job_id], ['ind']) click.echo("{} - Retrieving {} from {}".format( click.style("Info", bold=True, fg='green'), click.style("build-logs", bold=True), click.style(job_id, bold=True, fg='green', dim=True))) logs = facade.get_build_logs(job_id) click.echo_via_pager(logs) facade.write_build_logs(job_id, logs, out_dir) except Exception as e: handle_specific_exception(e) handle_exception(e)
def job_provenance(facade: TrainFacade, model_id, out_dir): """ Retrieve provenance from a trained model. """ # extract provenance via DAG try: click.echo("{} - Retrieving {} from {}".format( click.style("Info", bold=True, fg='green'), click.style("provenance", bold=True), click.style(model_id, bold=True, fg='green', dim=True))) out, data = facade.provenance(out_dir, model_id) # render DAG (via dot) Source(data).render(out) os.remove(out) except Exception as e: handle_specific_exception(e) handle_exception(e)
def build(self, provider, env, local_backend=False, verbose=False): env_state = EnvironmentState.initialize(provider, env) if not env_state.if_build_dir_exists: build_dir(env_state.build_dir) extra_vars = self._get_vars(provider, env_state.build_dir) self.tf_service.cd_dir(env_state.build_dir) self.tf_service.set_verbose(verbose) directory = self._tf_init(provider, env, local_backend, destroying=False) self.tf_service.plan(directory, extra_vars) self.tf_service.apply(directory, extra_vars) self.tf_service.execute() # check if the deployed successfully # Refresh environment states after terraform service operations env_state = EnvironmentState.initialize(provider, env) if env_state.if_tfstate_exists: url, kubeconfig = self._parse_config(env_state.build_dir) current_context = provider if provider in [DOCKER, MINIKUBE] else f"{provider}_{env}" self.state_service.set(DEFAULT, user=USER) self._set_context_list(current_context) self._set_active_context(current_context) self.state_service.set(current_context) try: self.state_service.set_section(current_context, BACKEND, url=url, token=uuid.uuid4()) self.state_service.set_section(current_context, INFRASTRUCTURE, kubeconfig=kubeconfig) except Exception as e: handle_specific_exception(e) handle_exception(e) self.state_service.write() return True, env_state return False, env_state
def get_build_logs(facade: NotebookFacade, job_id, out_dir): """ Retrieve logs from building notebook source image. """ # get logs for a specific job try: # ensure arguments are correctly defined validate_inputs([job_id], ['job_id']) click.echo("{} - Retrieving {} from {}".format( click.style("Info", bold=True, fg='green'), click.style("build-logs", bold=True), click.style(job_id, bold=True, fg='green', dim=True))) logs = facade.get_build_logs(job_id) click.echo_via_pager(logs) facade.write_build_logs(job_id, logs, out_dir) except Exception as e: handle_specific_exception(e) handle_exception(e)
def list_notebooks(facade: NotebookFacade): """ List all available running notebooks. """ try: data = facade.list() building_table, n_building = render_queued_table( data['building'], header='BUILDING', include_ind=False, drop_cols={'progress'}) running_table = "" running_jobs = data['notebooks'] n_running = len(running_jobs) if n_running > 0: running_table = \ f"\n{render_table(running_jobs, 'RUNNING', drop_cols={'code', 'image', 'model', 'progress'})}\n" facade.cache(running_jobs) if n_running + n_building > 30: click.echo_via_pager(f'{building_table}{running_table}') elif n_running + n_building > 0: click.echo(f'{building_table}{running_table}') else: click.echo( "{} - There are currently {} active notebooks - first run {}". format( click.style("Warning", bold=True, fg='yellow'), click.style('no', bold=True, fg='red'), click.style("kaos notebook deploy", bold=True, fg='green'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def list_workspaces(facade: WorkspaceFacade): """ List all available workspaces. """ try: workspaces = facade.list() if len(workspaces) > 0: facade.cache(workspaces) table = render_table(workspaces) click.echo(table) else: click.echo("{} - There are currently {} active workspaces - first run {}".format( click.style("Warning", bold=True, fg='yellow'), click.style('no', bold=True, fg='red'), click.style("kaos workspace create", bold=True, fg='green'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def deploy_serve(facade: ServeFacade, source_bundle, model_id, cpu, memory, gpu): """ Deploy endpoint with source (code + environment) and trained model id. """ try: # inform user regarding source bundle "upload" click.echo("{} - Submitting {} bundle: {}".format( click.style("Info", bold=True, fg='green'), click.style('source', bold=True, fg='blue'), click.style(source_bundle, bold=True, fg='green', dim=True))) # process SOURCE bundle /inference/<name>/<model_id> with Compressor(label="Compressing source bundle", filename="source.zip", source_path=source_bundle) as c: data = facade.upload_source_bundle(c, model_id, cpu=cpu, memory=memory, gpu=gpu) # inform user regarding model_id "naming" if model_id: click.echo(" {} Adding trained {}: {}".format( click.style(SYM_CHECK, fg='green', bold=True), click.style("model_id", fg='blue', bold=True), click.style(model_id, fg='green', bold=True))) # inform user regarding source bundle "naming" source_glob = data['glob_name'] click.echo(" {} Setting {} bundle: {}".format( click.style(SYM_CHECK, fg='green', bold=True), click.style("source", fg='blue', bold=True), click.style(f"/{source_glob}", fg='green', bold=True))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def kill_workspace(facade: WorkspaceFacade): """ Kill all resources running in a workspace. """ try: name = facade.current() # confirm kill click.confirm('{} - Are you sure about killing all {} resources?'.format( click.style("Warning", bold=True, fg='yellow'), click.style(name, bold=True, fg='red')), abort=True) name = facade.delete() click.echo('{} - Successfully killed all {} resources'.format( click.style("Info", bold=True, fg='green'), click.style(name, bold=True, fg='green'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def list_jobs(facade: TrainFacade): """ List all training jobs. """ try: data = facade.list()['response'] building_table, n_building = render_queued_table(data['building'], header='BUILDING', include_ind=False, drop_cols={'hyperopt', 'progress'}) ingesting_table, n_ingesting = render_queued_table(data['ingesting'], header='INGESTING', include_ind=False, drop_cols={'hyperopt'}) training_table = "" n_training = len(data['training']) training_jobs = data['training'] if n_training > 0: training_table = f"\n{render_table(training_jobs, 'TRAINING', drop_cols={'progress'})}\n" facade.cache(training_jobs) n_jobs = n_training + n_building + n_ingesting if n_jobs > 30: click.echo_via_pager(f'{ingesting_table}{building_table}{training_table}') elif n_jobs > 0: click.echo(f'{ingesting_table}{building_table}{training_table}') else: click.echo("{} - There are currently {} training jobs - first run {}".format( click.style("Warning", bold=True, fg='yellow'), click.style('no', bold=True, fg='red'), click.style("kaos train deploy", bold=True, fg='green'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def get_template(facade: TemplateFacade, name, ind): """ Get template. """ try: # ensure arguments are correctly defined validate_inputs([name, ind], ['name', 'ind']) # selection by index if ind is not None: name = facade.get_template_name_by_ind(ind) name = facade.validate(name) facade.download(name) click.echo("{} - Successfully loaded {} template".format( click.style("Info", bold=True, fg='green'), click.style(name, bold=True, fg='green'))) except Exception as e: handle_specific_exception(e) handle_exception(e)
def deploy(backend: BackendFacade, cloud: str, env: str, force: bool, verbose: bool, yes: bool, local_backend: bool): """ Deploy kaos backend infrastructure based on selected provider. """ env_state = EnvironmentState.initialize(cloud, env) if env_state.if_tfstate_exists and not force: click.echo('{} - {} backend is already built.'.format(click.style("Aborting", bold=True, fg='red'), click.style("kaos", bold=True))) sys.exit(1) elif env_state.if_tfstate_exists and force: click.echo('{} - Performing {} build of the backend'.format( click.style("Warning", bold=True, fg='yellow'), click.style("force", bold=True))) env_state.remove_terraform_files() # set env variable appropriately env_state.set_build_env() cloud = env_state.cloud env = env_state.env if not yes: # confirm creation of backend if env_state.env: click.confirm( '{} - Are you sure about building {} [{}] backend in {}?'.format( click.style("Warning", bold=True, fg='yellow'), click.style('kaos', bold=True), click.style(env_state.env, bold=True, fg='blue'), click.style(env_state.cloud, bold=True, fg='red')), abort=True) else: click.confirm( '{} - Are you sure about building {} backend in {}?'.format( click.style("Warning", bold=True, fg='yellow'), click.style('kaos', bold=True), click.style(env_state.cloud, bold=True, fg='red')), abort=True) if local_backend: click.echo('{} - Building with {} terraform backend state'.format( click.style("Info", bold=True, fg='green'), click.style("local", bold=True))) if local_backend and env_state.cloud in [DOCKER, MINIKUBE]: click.echo('{} - local backend (-l/--local_backend) has no effect for {}'.format( click.style("Info", bold=True, fg='green'), click.style(env_state.cloud, bold=True, fg='red'))) # validate unused port for DOCKER if env_state.cloud == DOCKER and not validate_unused_port(80): # If the force build flag was set to True and the kaos backend # was already built then skip the warning; otherwise, warn # the user that the port is already taken by another service # and issue a sys exit if not (force and env_state.if_tfstate_exists): click.echo( "{} - Network port {} is used but is needed for building {} backend in {}".format( click.style("Warning", bold=True, fg='yellow'), click.style("80", bold=True), click.style("kaos", bold=True), click.style(env_state.cloud, bold=True, fg='red'))) sys.exit(1) try: is_built_successfully, env_state = backend.build(env_state.cloud, env_state.env, local_backend=local_backend, verbose=verbose) if is_built_successfully: if verbose: click.echo("\n{} - Endpoint successfully set to {}".format( click.style("Info", bold=True, fg='green'), click.style(backend.url, bold=True, fg='green'))) if is_cloud_provider(env_state.cloud): kubeconfig = os.path.abspath(backend.kubeconfig) click.echo("\n{} - To interact with the Kubernetes cluster:\n {}" .format(click.style("Info", bold=True, fg='green'), click.style("export KUBECONFIG=" + kubeconfig, bold=True, fg='red'))) if env: click.echo("{} - Successfully built {} [{}] environment".format( click.style("Info", bold=True, fg='green'), click.style('kaos', bold=True), click.style(env, bold=True, fg='blue'))) else: click.echo("{} - Successfully built {} environment".format( click.style("Info", bold=True, fg='green'), click.style('kaos', bold=True))) else: click.echo("{} - Deployment Unsuccessful while creating {} [{} {}] environment".format( click.style("Error", bold=True, fg='red'), click.style('kaos', bold=True), click.style(cloud, bold=True, fg='red'), click.style(env, bold=True, fg='red'))), sys.exit(1) except Exception as e: handle_specific_exception(e) handle_exception(e)
def deploy_job(facade: TrainFacade, source_bundle, data_bundle, data_manifest, hyperparams, parallelism, cpu, memory, gpu): """ Deploy training job with source (code + environment) and/or data bundle (and hyperparameters). """ try: # ensure either bundle exists if not source_bundle and not data_bundle and not data_manifest and not hyperparams: click.echo("{} - {} and/or {} and/or {} need to be defined for training" .format(click.style('Warning', bold=True, fg='yellow'), click.style('--source_bundle', bold=True, fg='green'), click.style('--data_bundle or --data_manifest', bold=True, fg='green'), click.style('--hyperparams', bold=True, fg='green')), err=True) sys.exit(1) # process SOURCE bundle (POST /train/<name>) if source_bundle: click.echo("{} - Submitting {} bundle: {}".format( click.style("Info", bold=True, fg='green'), click.style('source', bold=True, fg='blue'), click.style(source_bundle, bold=True, fg='green', dim=True))) with Compressor(label="Compressing source bundle", filename="model.zip", source_path=source_bundle) as c: data = facade.upload_source_bundle(c, cpu=cpu, memory=memory, gpu=gpu) # inform user regarding source bundle "naming" source_glob = data['glob_name'] click.echo(" {} Setting {} bundle: {}\n".format( click.style(SYM_CHECK, fg='green', bold=True), click.style("source", fg='blue', bold=True), click.style(f"/{source_glob}", fg='green', bold=True)) ) # process DATA bundle (POST /data/<name>/features) if data_bundle: click.echo("{} - Submitting {} bundle: {}".format( click.style("Info", bold=True, fg='green'), click.style('data', bold=True, fg='blue'), click.style(data_bundle, bold=True, fg='green', dim=True))) with Compressor(label="Compressing data bundle", filename="data.zip", source_path=data_bundle) as c: data = facade.upload_data_bundle(c, cpu=cpu, memory=memory, gpu=gpu) # inform user regarding data bundle "naming" data_glob = data['glob_name'] click.echo(" {} Setting {} bundle: {}\n".format( click.style(SYM_CHECK, fg='green', bold=True), click.style("data", fg='blue', bold=True), click.style(f"/{data_glob}", fg='green', bold=True)) ) # process DATA manifest (POST /data/<name>/manifest) if data_manifest: click.echo("{} - Submitting {} bundle: {}".format( click.style("Info", bold=True, fg='green'), click.style('data manifest', bold=True, fg='blue'), click.style(data_manifest, bold=True, fg='green', dim=True))) if validate_manifest_file(data_manifest): data = facade.upload_manifest(data_manifest, cpu=cpu, memory=memory, gpu=gpu) else: click.echo("The manifest file is invalid") sys.exit(1) # inform user regarding data manifest "naming" data_glob = data['glob_name'] click.echo(" {} Setting {} bundle: {}\n".format( click.style(SYM_CHECK, fg='green', bold=True), click.style("data manifest", fg='blue', bold=True), click.style(f"/{data_glob}", fg='green', bold=True)) ) # process HYPERPARAMS (POST /data/<name>/params) if hyperparams: click.echo("{} - Submitting {} bundle: {}".format( click.style("Info", bold=True, fg='green'), click.style('hyperparams', bold=True, fg='blue'), click.style(hyperparams, bold=True, fg='green', dim=True))) data = facade.upload_hyperparams(hyperparams, cpu=cpu, memory=memory, gpu=gpu, parallelism=parallelism) hyper_glob = data['glob_name'] click.echo(" {} Setting {} bundle: {}\n".format( click.style(SYM_CHECK, fg='green', bold=True), click.style("hyperparameters", fg='blue', bold=True), click.style(f"/{hyper_glob}/*", fg='green', bold=True)) ) # inform user regarding actual hyperopt jobs param_combinations = data['params'] click.echo("{} {}\n".format( click.style("CURRENT HYPERPARAMETERS", fg='white', bold=True, underline=True), click.style(f"({len(param_combinations)})", fg='green', bold=True))) # set up simple table to iterate through response table = PrettyTable(hrules=prettytable.ALL) table.field_names = ['ind'] + list(param_combinations[0].keys()) for ind, d in enumerate(param_combinations): table.add_row([ind] + list(d.values())) click.echo(f"{table.get_string()}\n") else: facade.upload_hyperparams(cpu=cpu, memory=memory, gpu=gpu, parallelism=parallelism) data = facade.inspect() # update status of pipeline (i.e. its inputs) data = [{ "Image": f"{SYM_CHECK}\n{data['image']}" if data['image'].find('null') < 0 else SYM_CROSS, "Data": f"{SYM_CHECK}\n{data['data_glob']}" if data['data_glob'].find('null') < 0 else SYM_CROSS, "Hyperparams": f"{SYM_CHECK}\n{data['hyper_glob']}" if data['hyper_glob'].find('null') < 0 else SYM_CROSS, }] # overwrite if code was "added" if source_bundle: data[0]["Image"] = f"{SYM_PROGRESS}\n<building>" click.echo("{}\n".format(click.style("CURRENT TRAINING INPUTS", fg='white', bold=True, underline=True))) click.echo(render_table(data, include_ind=False)) except Exception as e: handle_specific_exception(e) handle_exception(e)