def test_submit_two_experiment_success(prepare_mocks: SubmitExperimentMocks,
capsys, caplog):
import logging
caplog.set_level(logging.CRITICAL)
prepare_mocks.mocker.patch("click.confirm", return_value=True)
prepare_mocks.create_env.side_effect = [(EXPERIMENT_FOLDER),
(EXPERIMENT_FOLDER)]
prepare_mocks.cmd_create.side_effect = [("", 0), ("", 0)]
prepare_mocks.update_conf.side_effect = [0, 0]
prepare_mocks.check_run_env.side_effect = [None, None]
parameters = [SCRIPT_LOCATION]
parameters.extend(PR_PARAMETER)
parameters.extend(PS_PARAMETER)
submit_experiment(script_location=SCRIPT_LOCATION,
script_folder_location=None,
pack_params=[],
template=None,
name=None,
parameter_range=PR_PARAMETER,
parameter_set=PS_PARAMETER,
script_parameters=[],
run_kind=RunKinds.TRAINING)
check_asserts(prepare_mocks,
create_env_count=2,
cmd_create_count=2,
update_conf_count=2,
submit_one_count=2,
add_run_count=2)
out, _ = capsys.readouterr()
assert "param1=1" in out
assert "param1=2" in out
assert "param2=3" in out
def test_submit_depl_fail(prepare_mocks: SubmitExperimentMocks):
prepare_mocks.cmd_create.side_effect = [("error message", 1)]
with pytest.raises(SubmitExperimentError) as exe:
submit_experiment(script_location=SCRIPT_LOCATION, script_folder_location=None, pack_params=[],
template=None, name=None, parameter_range=[], parameter_set=(), script_parameters=(),
run_kind=RunKinds.TRAINING)
assert Texts.ENV_CREATION_ERROR_MSG in str(exe)
check_asserts(prepare_mocks, update_conf_count=0, add_exp_count=0, submit_one_count=0, socat_start_count=1,
del_env_count=1, add_run_count=0)
def test_submit_success(prepare_mocks: SubmitExperimentMocks):
submit_experiment(script_location=SCRIPT_LOCATION,
script_folder_location=None,
template=None,
name=None,
parameter_range=[],
parameter_set=[],
script_parameters=[],
pack_params=[],
run_kind=RunKinds.TRAINING)
check_asserts(prepare_mocks)
def test_submit_env_update_fail(prepare_mocks: SubmitExperimentMocks):
prepare_mocks.update_conf = prepare_mocks.mocker.patch("commands.experiment.common.update_configuration",
side_effect=[SubmitExperimentError])
with pytest.raises(SubmitExperimentError) as exe:
submit_experiment(script_location=SCRIPT_LOCATION, script_folder_location=None, pack_params=[],
template=None, name=None, parameter_range=[], parameter_set=(), script_parameters=(),
run_kind=RunKinds.TRAINING)
assert Texts.ENV_CREATION_ERROR_MSG in str(exe)
check_asserts(prepare_mocks, add_exp_count=0, add_run_count=0,
submit_one_count=0, socat_start_count=1, del_env_count=1)
def start_inference_instance(name: str,
model_location: str,
model_name: str,
template: str = INFERENCE_TEMPLATE,
local_model_location: str = None,
data_location: str = None,
output_location: str = None,
env_variables: List[str] = None,
tf_record: bool = False,
pack_params: List[Tuple[str, str]] = None,
requirements: str = None) -> Run:
if pack_params is None:
pack_params = []
else:
pack_params = list(pack_params)
pack_params.append(('modelName', model_name))
if model_location:
pack_params.append(('modelPath', model_location))
elif local_model_location:
pack_params.append(('modelPath', '/app'))
if data_location:
pack_params.append(('dataPath', data_location))
if output_location:
pack_params.append(('outputPath', output_location))
if tf_record:
pack_params.append(('inputFormat', 'tf-record'))
runs, _, _ = submit_experiment(run_kind=RunKinds.INFERENCE, name=name, template=template, pack_params=pack_params,
script_folder_location=local_model_location, env_variables=env_variables,
requirements_file=requirements)
return runs[0]
def test_submit_experiment_without_file(prepare_mocks: SubmitExperimentMocks):
runs_list, _, _ = submit_experiment(script_location=None, script_folder_location=None,
template='', name='', parameter_range=[],
parameter_set=(), script_parameters=(), pack_params=[],
run_kind=RunKinds.TRAINING)
assert len(runs_list) == 1
assert runs_list[0].name == "experiment_name"
check_asserts(prepare_mocks)
def test_submit_start_depl_fail(prepare_mocks: SubmitExperimentMocks):
prepare_mocks.submit_one.side_effect = SubmitExperimentError()
runs_list, _, _ = submit_experiment(script_location=SCRIPT_LOCATION, script_folder_location=None, pack_params=[],
template=None, name=None, parameter_range=[], parameter_set=(),
script_parameters=(), run_kind=RunKinds.TRAINING)
assert runs_list[0].state == RunStatus.FAILED
check_asserts(prepare_mocks, del_env_count=1, update_run_count=1)
def interact(ctx: click.Context, name: str, filename: str,
pack_param: List[Tuple[str, str]], no_launch: bool,
port_number: int, env: List[str], template: str):
"""
Starts an interactive session with Jupyter Notebook.
"""
current_namespace = get_kubectl_current_context_namespace()
jupyters_number = calculate_number_of_running_jupyters(current_namespace)
if jupyters_number > ACCEPTED_NUMBER_OF_NOTEBOOKS:
if not click.confirm(
Texts.TOO_MANY_JUPYTERS.format(
jupyter_number=str(jupyters_number))):
click.echo(Texts.INTERACT_ABORT_MSG)
sys.exit(0)
create_new_notebook = True
jupyter_experiment = None
if name:
try:
jupyter_experiment = Experiment.get(name=name,
namespace=current_namespace)
if jupyter_experiment and filename:
handle_error(user_msg=Texts.FILENAME_BUT_SESSION_EXISTS)
sys.exit(1)
if jupyter_experiment:
metadata = jupyter_experiment.metadata
if metadata and metadata.get("labels") and metadata.get(
"labels").get("script_name"):
filename = metadata.get("labels").get("script_name")
except Exception:
handle_error(logger, Texts.EXPERIMENT_GET_ERROR_MSG,
Texts.EXPERIMENT_GET_ERROR_MSG)
sys.exit(1)
# if experiment exists and is not based on jupyter image - we need to ask a user to choose another name
if jupyter_experiment and jupyter_experiment.template_name not in JUPYTER_NOTEBOOK_TEMPLATES_NAMES:
handle_error(user_msg=Texts.NAME_ALREADY_USED.format(name=name))
sys.exit(1)
# if experiment exists but its state is different than RUNNING - display info about a need of purging of
# this experiment
if jupyter_experiment and jupyter_experiment.state not in \
[ExperimentStatus.SUBMITTED, ExperimentStatus.CREATING]:
handle_error(
user_msg=Texts.EXP_WITH_THE_SAME_NAME_MUST_BE_PURGED.format(
name=name))
sys.exit(1)
if not jupyter_experiment and (
not click.get_current_context().obj.force
and not click.confirm(Texts.CONFIRM_EXPERIMENT_CREATION)):
sys.exit(0)
if jupyter_experiment:
create_new_notebook = False
else:
try:
check_experiment_name(value=name)
except click.BadParameter as exe:
handle_error(user_msg=str(exe))
sys.exit(1)
number_of_retries = 0
if create_new_notebook:
number_of_retries = 5
try:
exp_name = name
if not name and not filename:
exp_name = generate_name("jup")
click.echo(Texts.SUBMITTING_EXPERIMENT_USER_MSG)
runs, runs_errors, filename = submit_experiment(
run_kind=RunKinds.JUPYTER,
script_location=filename,
script_folder_location=None,
template=template,
name=exp_name,
parameter_range=[],
parameter_set=(),
script_parameters=(),
pack_params=pack_param,
env_variables=env)
click.echo(
tabulate(
{
RUN_NAME:
[run.cli_representation.name for run in runs],
RUN_PARAMETERS:
[run.cli_representation.parameters for run in runs],
RUN_STATUS:
[run.cli_representation.status for run in runs],
RUN_MESSAGE:
[runs_errors.get(run.name, "") for run in runs]
},
headers=[
RUN_NAME, RUN_PARAMETERS, RUN_STATUS, RUN_MESSAGE
],
tablefmt=TBLT_TABLE_FORMAT))
if runs:
name = runs[0].name
else:
# run wasn't created - error
raise RuntimeError("Run wasn't created")
except K8sProxyCloseError as exe:
handle_error(user_msg=exe.message)
sys.exit(1)
except SubmitExperimentError as exe:
handle_error(
logger,
Texts.SUBMIT_ERROR_MSG.format(exception_message=exe.message),
Texts.SUBMIT_ERROR_MSG.format(exception_message=exe.message))
sys.exit(1)
except Exception:
handle_error(logger, Texts.SUBMIT_OTHER_ERROR_MSG,
Texts.SUBMIT_OTHER_ERROR_MSG)
sys.exit(1)
else:
# if jupyter service exists - the system only connects to it
click.echo(Texts.SESSION_EXISTS_MSG)
url_end = ""
if filename:
# only Jupyter notebooks are opened directly, other files are opened in edit mode
url_end = f"/notebooks/output/experiment/"
if jupyter_experiment and filename.endswith(".py"):
filename = filename[:filename.index(".py", -3)] + ".ipynb"
if not filename.endswith(".ipynb"):
url_end = "/edit/"
url_end = url_end + Path(filename).name
# wait until all jupyter pods are ready
for i in range(JUPYTER_CHECK_POD_READY_TRIES):
try:
if check_pods_status(run_name=name,
namespace=current_namespace,
status=PodStatus.RUNNING):
break
except Exception:
handle_error(logger, Texts.NOTEBOOK_STATE_CHECK_ERROR_MSG)
sys.exit(1)
time.sleep(1)
else:
handle_error(user_msg=Texts.NOTEBOOK_NOT_READY_ERROR_MSG)
sys.exit(1)
try:
launch_app(k8s_app_name=NAUTAAppNames.JUPYTER,
app_name=name,
no_launch=no_launch,
number_of_retries=number_of_retries,
url_end=url_end,
port=port_number)
except LaunchError as exe:
handle_error(logger, exe.message, exe.message)
sys.exit(1)
except ProxyClosingError:
handle_error(user_msg=Texts.PROXY_CLOSING_ERROR_MSG)
sys.exit(1)
except Exception:
handle_error(logger, Texts.SESSION_LAUNCH_OTHER_ERROR_MSG,
Texts.SESSION_LAUNCH_OTHER_ERROR_MSG)
sys.exit(1)
def submit(state: State, script_location: str, script_folder_location: str,
template: str, name: str, pack_param: List[Tuple[str, str]],
parameter_range: List[Tuple[str, str]],
parameter_set: Tuple[str, ...], env: List[str],
script_parameters: Tuple[str, ...], requirements: Optional[str]):
logger.debug(Texts.SUBMIT_START_LOG_MSG)
validate_script_location(script_location)
validate_pack_params(pack_param)
validate_pack(template)
if os.path.isdir(script_location):
if not requirements:
requirements = get_default_requirements_location(
script_directory=script_location)
script_location = get_default_script_location(
script_directory=script_location)
if script_folder_location:
validate_script_folder_location(script_folder_location)
click.echo(Texts.SUBMIT_START_USER_MSG)
runs_list = None
# noinspection PyBroadException
try:
runs_list, runs_errors, _ = submit_experiment(
run_kind=RunKinds.TRAINING,
script_location=script_location,
script_folder_location=script_folder_location,
template=template,
name=name,
pack_params=pack_param,
parameter_range=parameter_range,
parameter_set=parameter_set,
script_parameters=script_parameters,
env_variables=env,
requirements_file=requirements)
except K8sProxyCloseError as exe:
handle_error(user_msg=exe.message)
click.echo(exe.message)
if not runs_list:
exit(1)
except SubmitExperimentError as exe:
handle_error(user_msg=Texts.SUBMIT_ERROR_MSG.format(
exception_message=exe.message))
exit(1)
except Exception:
handle_error(user_msg=Texts.SUBMIT_OTHER_ERROR_MSG)
exit(1)
# display information about status of a training
click.echo(
tabulate(
[(run.cli_representation.name, run.cli_representation.parameters,
run.cli_representation.status,
format_run_message(runs_errors.get(run.name, "")))
for run in runs_list],
headers=[RUN_NAME, RUN_PARAMETERS, RUN_STATUS, RUN_MESSAGE],
tablefmt=TBLT_TABLE_FORMAT))
# if there is at least one FAILED experiment - application has to return exit code != 0
if any(run.state == RunStatus.FAILED for run in runs_list):
handle_error(logger, Texts.FAILED_RUNS_LOG_MSG)
exit(1)
