def test_update_configuration_failure(mocker):
modify_values_yaml_mock = mocker.patch("packs.tf_training.modify_values_yaml")
modify_dockerfile_mock = mocker.patch("packs.tf_training.modify_dockerfile")
modify_values_yaml_mock.side_effect = Exception("Test error")
with pytest.raises(RuntimeError):
tf_training.update_configuration(run_folder=EXPERIMENT_FOLDER, script_location=SCRIPT_LOCATION,
script_parameters=SCRIPT_PARAMETERS,
experiment_name='test-experiment',
cluster_registry_port= 12345, username='******',
pack_type=EXAMPLE_PACK_TYPE, pack_params=[])
assert modify_dockerfile_mock.call_count == 0, "dockerfile was modified"
assert modify_values_yaml_mock.call_count == 1, "values yaml wasn't modified"
def test_update_configuration_success(mocker):
modify_values_yaml_mock = mocker.patch("packs.tf_training.modify_values_yaml")
modify_dockerfile_mock = mocker.patch("packs.tf_training.modify_dockerfile")
output = tf_training.update_configuration(run_folder=EXPERIMENT_FOLDER, script_location=SCRIPT_LOCATION,
script_parameters=SCRIPT_PARAMETERS,
experiment_name='test-experiment', username='******',
cluster_registry_port=12345, pack_type=EXAMPLE_PACK_TYPE,
pack_params=[])
assert not output, "configuration wasn't updated"
assert modify_dockerfile_mock.call_count == 1, "dockerfile wasn't modified"
assert modify_values_yaml_mock.call_count == 1, "values yaml wasn't modified"
def prepare_experiment_environment(experiment_name: str, run_name: str, local_script_location: str,
script_parameters: Tuple[str, ...],
pack_type: str, local_registry_port: int, cluster_registry_port: int,
script_folder_location: str = None,
pack_params: List[Tuple[str, str]] = None,
env_variables: List[str] = None,
requirements_file: str = None) -> PrepareExperimentResult:
"""
Prepares draft's environment for a certain run based on provided parameters
:param experiment_name: name of an experiment
:param run_name: name of an experiment run
:param local_script_location: location of a script used for training purposes on local machine
:param script_folder_location: location of an additional folder used in training
:param script_parameters: parameters passed to a script
:param pack_type: type of a pack used to start training job
:param local_registry_port: port on which docker registry is accessible locally
:param cluster_registry_port: port on which docker registry is accessible within nauta cluster
:param pack_params: additional pack params
:param env_variables: environmental variables to be passed to training
:param requirements_file: path to a file with experiment requirements
:return: name of folder with an environment created for this run, a name of script used for training purposes
and count of Pods
In case of any problems - an exception with a description of a problem is thrown
"""
log.debug(f'Prepare run {run_name} environment - start')
run_folder = get_run_environment_path(run_name)
try:
# check environment directory
check_run_environment(run_folder)
with spinner(text=Texts.CREATING_ENVIRONMENT_MSG.format(run_name=run_name)):
# create an environment
create_environment(run_name, local_script_location, script_folder_location)
# generate draft's data
output, exit_code, log_output = cmd.create(working_directory=run_folder, pack_type=pack_type)
# copy requirements file if it was provided, create empty requirements file otherwise
dest_requirements_file = os.path.join(run_folder, 'requirements.txt')
if requirements_file:
shutil.copyfile(requirements_file, dest_requirements_file)
else:
Path(dest_requirements_file).touch()
if exit_code:
raise SubmitExperimentError(Texts.DRAFT_TEMPLATES_NOT_GENERATED_ERROR_MSG.format(reason=log_output))
# Script location on experiment container
remote_script_location = Path(local_script_location).name if local_script_location else ''
if pack_type in JUPYTER_NOTEBOOK_TEMPLATES_NAMES and remote_script_location.endswith(".py"):
# for interact (jupyter notebooks) try to convert .py file into .ipynb
py_script_location = os.path.join(run_folder, FOLDER_DIR_NAME, remote_script_location)
ipynb_file_name = convert_py_to_ipynb(py_script_location, os.path.join(run_folder, FOLDER_DIR_NAME))
local_script_location = ipynb_file_name
# reconfigure draft's templates
update_configuration(run_folder=run_folder, script_location=remote_script_location,
script_parameters=script_parameters,
experiment_name=experiment_name, run_name=run_name,
local_registry_port=local_registry_port, cluster_registry_port=cluster_registry_port,
pack_type=pack_type, pack_params=pack_params,
script_folder_location=script_folder_location,
env_variables=env_variables)
pod_count = get_pod_count(run_folder=run_folder, pack_type=pack_type)
except Exception as exe:
delete_environment(run_folder)
raise SubmitExperimentError('Problems during creation of environments.') from exe
log.debug(f'Prepare run {run_name} environment - finish')
return PrepareExperimentResult(folder_name=run_folder, script_name=local_script_location, pod_count=pod_count)
