def test_submit_two_experiment_success(prepare_mocks: SubmitExperimentMocks, capsys, caplog): import logging caplog.set_level(logging.CRITICAL) prepare_mocks.mocker.patch("click.confirm", return_value=True) prepare_mocks.create_env.side_effect = [(EXPERIMENT_FOLDER), (EXPERIMENT_FOLDER)] prepare_mocks.cmd_create.side_effect = [("", 0), ("", 0)] prepare_mocks.update_conf.side_effect = [0, 0] prepare_mocks.check_run_env.side_effect = [None, None] parameters = [SCRIPT_LOCATION] parameters.extend(PR_PARAMETER) parameters.extend(PS_PARAMETER) submit_experiment(script_location=SCRIPT_LOCATION, script_folder_location=None, pack_params=[], template=None, name=None, parameter_range=PR_PARAMETER, parameter_set=PS_PARAMETER, script_parameters=[], run_kind=RunKinds.TRAINING) check_asserts(prepare_mocks, create_env_count=2, cmd_create_count=2, update_conf_count=2, submit_one_count=2, add_run_count=2) out, _ = capsys.readouterr() assert "param1=1" in out assert "param1=2" in out assert "param2=3" in out
def test_submit_depl_fail(prepare_mocks: SubmitExperimentMocks): prepare_mocks.cmd_create.side_effect = [("error message", 1)] with pytest.raises(SubmitExperimentError) as exe: submit_experiment(script_location=SCRIPT_LOCATION, script_folder_location=None, pack_params=[], template=None, name=None, parameter_range=[], parameter_set=(), script_parameters=(), run_kind=RunKinds.TRAINING) assert Texts.ENV_CREATION_ERROR_MSG in str(exe) check_asserts(prepare_mocks, update_conf_count=0, add_exp_count=0, submit_one_count=0, socat_start_count=1, del_env_count=1, add_run_count=0)
def test_submit_success(prepare_mocks: SubmitExperimentMocks): submit_experiment(script_location=SCRIPT_LOCATION, script_folder_location=None, template=None, name=None, parameter_range=[], parameter_set=[], script_parameters=[], pack_params=[], run_kind=RunKinds.TRAINING) check_asserts(prepare_mocks)
def test_submit_env_update_fail(prepare_mocks: SubmitExperimentMocks): prepare_mocks.update_conf = prepare_mocks.mocker.patch("commands.experiment.common.update_configuration", side_effect=[SubmitExperimentError]) with pytest.raises(SubmitExperimentError) as exe: submit_experiment(script_location=SCRIPT_LOCATION, script_folder_location=None, pack_params=[], template=None, name=None, parameter_range=[], parameter_set=(), script_parameters=(), run_kind=RunKinds.TRAINING) assert Texts.ENV_CREATION_ERROR_MSG in str(exe) check_asserts(prepare_mocks, add_exp_count=0, add_run_count=0, submit_one_count=0, socat_start_count=1, del_env_count=1)
def start_inference_instance(name: str, model_location: str, model_name: str, template: str = INFERENCE_TEMPLATE, local_model_location: str = None, data_location: str = None, output_location: str = None, env_variables: List[str] = None, tf_record: bool = False, pack_params: List[Tuple[str, str]] = None, requirements: str = None) -> Run: if pack_params is None: pack_params = [] else: pack_params = list(pack_params) pack_params.append(('modelName', model_name)) if model_location: pack_params.append(('modelPath', model_location)) elif local_model_location: pack_params.append(('modelPath', '/app')) if data_location: pack_params.append(('dataPath', data_location)) if output_location: pack_params.append(('outputPath', output_location)) if tf_record: pack_params.append(('inputFormat', 'tf-record')) runs, _, _ = submit_experiment(run_kind=RunKinds.INFERENCE, name=name, template=template, pack_params=pack_params, script_folder_location=local_model_location, env_variables=env_variables, requirements_file=requirements) return runs[0]
def test_submit_experiment_without_file(prepare_mocks: SubmitExperimentMocks): runs_list, _, _ = submit_experiment(script_location=None, script_folder_location=None, template='', name='', parameter_range=[], parameter_set=(), script_parameters=(), pack_params=[], run_kind=RunKinds.TRAINING) assert len(runs_list) == 1 assert runs_list[0].name == "experiment_name" check_asserts(prepare_mocks)
def test_submit_start_depl_fail(prepare_mocks: SubmitExperimentMocks): prepare_mocks.submit_one.side_effect = SubmitExperimentError() runs_list, _, _ = submit_experiment(script_location=SCRIPT_LOCATION, script_folder_location=None, pack_params=[], template=None, name=None, parameter_range=[], parameter_set=(), script_parameters=(), run_kind=RunKinds.TRAINING) assert runs_list[0].state == RunStatus.FAILED check_asserts(prepare_mocks, del_env_count=1, update_run_count=1)
def interact(ctx: click.Context, name: str, filename: str, pack_param: List[Tuple[str, str]], no_launch: bool, port_number: int, env: List[str], template: str): """ Starts an interactive session with Jupyter Notebook. """ current_namespace = get_kubectl_current_context_namespace() jupyters_number = calculate_number_of_running_jupyters(current_namespace) if jupyters_number > ACCEPTED_NUMBER_OF_NOTEBOOKS: if not click.confirm( Texts.TOO_MANY_JUPYTERS.format( jupyter_number=str(jupyters_number))): click.echo(Texts.INTERACT_ABORT_MSG) sys.exit(0) create_new_notebook = True jupyter_experiment = None if name: try: jupyter_experiment = Experiment.get(name=name, namespace=current_namespace) if jupyter_experiment and filename: handle_error(user_msg=Texts.FILENAME_BUT_SESSION_EXISTS) sys.exit(1) if jupyter_experiment: metadata = jupyter_experiment.metadata if metadata and metadata.get("labels") and metadata.get( "labels").get("script_name"): filename = metadata.get("labels").get("script_name") except Exception: handle_error(logger, Texts.EXPERIMENT_GET_ERROR_MSG, Texts.EXPERIMENT_GET_ERROR_MSG) sys.exit(1) # if experiment exists and is not based on jupyter image - we need to ask a user to choose another name if jupyter_experiment and jupyter_experiment.template_name not in JUPYTER_NOTEBOOK_TEMPLATES_NAMES: handle_error(user_msg=Texts.NAME_ALREADY_USED.format(name=name)) sys.exit(1) # if experiment exists but its state is different than RUNNING - display info about a need of purging of # this experiment if jupyter_experiment and jupyter_experiment.state not in \ [ExperimentStatus.SUBMITTED, ExperimentStatus.CREATING]: handle_error( user_msg=Texts.EXP_WITH_THE_SAME_NAME_MUST_BE_PURGED.format( name=name)) sys.exit(1) if not jupyter_experiment and ( not click.get_current_context().obj.force and not click.confirm(Texts.CONFIRM_EXPERIMENT_CREATION)): sys.exit(0) if jupyter_experiment: create_new_notebook = False else: try: check_experiment_name(value=name) except click.BadParameter as exe: handle_error(user_msg=str(exe)) sys.exit(1) number_of_retries = 0 if create_new_notebook: number_of_retries = 5 try: exp_name = name if not name and not filename: exp_name = generate_name("jup") click.echo(Texts.SUBMITTING_EXPERIMENT_USER_MSG) runs, runs_errors, filename = submit_experiment( run_kind=RunKinds.JUPYTER, script_location=filename, script_folder_location=None, template=template, name=exp_name, parameter_range=[], parameter_set=(), script_parameters=(), pack_params=pack_param, env_variables=env) click.echo( tabulate( { RUN_NAME: [run.cli_representation.name for run in runs], RUN_PARAMETERS: [run.cli_representation.parameters for run in runs], RUN_STATUS: [run.cli_representation.status for run in runs], RUN_MESSAGE: [runs_errors.get(run.name, "") for run in runs] }, headers=[ RUN_NAME, RUN_PARAMETERS, RUN_STATUS, RUN_MESSAGE ], tablefmt=TBLT_TABLE_FORMAT)) if runs: name = runs[0].name else: # run wasn't created - error raise RuntimeError("Run wasn't created") except K8sProxyCloseError as exe: handle_error(user_msg=exe.message) sys.exit(1) except SubmitExperimentError as exe: handle_error( logger, Texts.SUBMIT_ERROR_MSG.format(exception_message=exe.message), Texts.SUBMIT_ERROR_MSG.format(exception_message=exe.message)) sys.exit(1) except Exception: handle_error(logger, Texts.SUBMIT_OTHER_ERROR_MSG, Texts.SUBMIT_OTHER_ERROR_MSG) sys.exit(1) else: # if jupyter service exists - the system only connects to it click.echo(Texts.SESSION_EXISTS_MSG) url_end = "" if filename: # only Jupyter notebooks are opened directly, other files are opened in edit mode url_end = f"/notebooks/output/experiment/" if jupyter_experiment and filename.endswith(".py"): filename = filename[:filename.index(".py", -3)] + ".ipynb" if not filename.endswith(".ipynb"): url_end = "/edit/" url_end = url_end + Path(filename).name # wait until all jupyter pods are ready for i in range(JUPYTER_CHECK_POD_READY_TRIES): try: if check_pods_status(run_name=name, namespace=current_namespace, status=PodStatus.RUNNING): break except Exception: handle_error(logger, Texts.NOTEBOOK_STATE_CHECK_ERROR_MSG) sys.exit(1) time.sleep(1) else: handle_error(user_msg=Texts.NOTEBOOK_NOT_READY_ERROR_MSG) sys.exit(1) try: launch_app(k8s_app_name=NAUTAAppNames.JUPYTER, app_name=name, no_launch=no_launch, number_of_retries=number_of_retries, url_end=url_end, port=port_number) except LaunchError as exe: handle_error(logger, exe.message, exe.message) sys.exit(1) except ProxyClosingError: handle_error(user_msg=Texts.PROXY_CLOSING_ERROR_MSG) sys.exit(1) except Exception: handle_error(logger, Texts.SESSION_LAUNCH_OTHER_ERROR_MSG, Texts.SESSION_LAUNCH_OTHER_ERROR_MSG) sys.exit(1)
def submit(state: State, script_location: str, script_folder_location: str, template: str, name: str, pack_param: List[Tuple[str, str]], parameter_range: List[Tuple[str, str]], parameter_set: Tuple[str, ...], env: List[str], script_parameters: Tuple[str, ...], requirements: Optional[str]): logger.debug(Texts.SUBMIT_START_LOG_MSG) validate_script_location(script_location) validate_pack_params(pack_param) validate_pack(template) if os.path.isdir(script_location): if not requirements: requirements = get_default_requirements_location( script_directory=script_location) script_location = get_default_script_location( script_directory=script_location) if script_folder_location: validate_script_folder_location(script_folder_location) click.echo(Texts.SUBMIT_START_USER_MSG) runs_list = None # noinspection PyBroadException try: runs_list, runs_errors, _ = submit_experiment( run_kind=RunKinds.TRAINING, script_location=script_location, script_folder_location=script_folder_location, template=template, name=name, pack_params=pack_param, parameter_range=parameter_range, parameter_set=parameter_set, script_parameters=script_parameters, env_variables=env, requirements_file=requirements) except K8sProxyCloseError as exe: handle_error(user_msg=exe.message) click.echo(exe.message) if not runs_list: exit(1) except SubmitExperimentError as exe: handle_error(user_msg=Texts.SUBMIT_ERROR_MSG.format( exception_message=exe.message)) exit(1) except Exception: handle_error(user_msg=Texts.SUBMIT_OTHER_ERROR_MSG) exit(1) # display information about status of a training click.echo( tabulate( [(run.cli_representation.name, run.cli_representation.parameters, run.cli_representation.status, format_run_message(runs_errors.get(run.name, ""))) for run in runs_list], headers=[RUN_NAME, RUN_PARAMETERS, RUN_STATUS, RUN_MESSAGE], tablefmt=TBLT_TABLE_FORMAT)) # if there is at least one FAILED experiment - application has to return exit code != 0 if any(run.state == RunStatus.FAILED for run in runs_list): handle_error(logger, Texts.FAILED_RUNS_LOG_MSG) exit(1)