def test_update_configuration_failure(mocker): modify_values_yaml_mock = mocker.patch("packs.tf_training.modify_values_yaml") modify_dockerfile_mock = mocker.patch("packs.tf_training.modify_dockerfile") modify_values_yaml_mock.side_effect = Exception("Test error") with pytest.raises(RuntimeError): tf_training.update_configuration(run_folder=EXPERIMENT_FOLDER, script_location=SCRIPT_LOCATION, script_parameters=SCRIPT_PARAMETERS, experiment_name='test-experiment', cluster_registry_port= 12345, username='******', pack_type=EXAMPLE_PACK_TYPE, pack_params=[]) assert modify_dockerfile_mock.call_count == 0, "dockerfile was modified" assert modify_values_yaml_mock.call_count == 1, "values yaml wasn't modified"
def test_update_configuration_success(mocker): modify_values_yaml_mock = mocker.patch("packs.tf_training.modify_values_yaml") modify_dockerfile_mock = mocker.patch("packs.tf_training.modify_dockerfile") output = tf_training.update_configuration(run_folder=EXPERIMENT_FOLDER, script_location=SCRIPT_LOCATION, script_parameters=SCRIPT_PARAMETERS, experiment_name='test-experiment', username='******', cluster_registry_port=12345, pack_type=EXAMPLE_PACK_TYPE, pack_params=[]) assert not output, "configuration wasn't updated" assert modify_dockerfile_mock.call_count == 1, "dockerfile wasn't modified" assert modify_values_yaml_mock.call_count == 1, "values yaml wasn't modified"
def prepare_experiment_environment(experiment_name: str, run_name: str, local_script_location: str, script_parameters: Tuple[str, ...], pack_type: str, local_registry_port: int, cluster_registry_port: int, script_folder_location: str = None, pack_params: List[Tuple[str, str]] = None, env_variables: List[str] = None, requirements_file: str = None) -> PrepareExperimentResult: """ Prepares draft's environment for a certain run based on provided parameters :param experiment_name: name of an experiment :param run_name: name of an experiment run :param local_script_location: location of a script used for training purposes on local machine :param script_folder_location: location of an additional folder used in training :param script_parameters: parameters passed to a script :param pack_type: type of a pack used to start training job :param local_registry_port: port on which docker registry is accessible locally :param cluster_registry_port: port on which docker registry is accessible within nauta cluster :param pack_params: additional pack params :param env_variables: environmental variables to be passed to training :param requirements_file: path to a file with experiment requirements :return: name of folder with an environment created for this run, a name of script used for training purposes and count of Pods In case of any problems - an exception with a description of a problem is thrown """ log.debug(f'Prepare run {run_name} environment - start') run_folder = get_run_environment_path(run_name) try: # check environment directory check_run_environment(run_folder) with spinner(text=Texts.CREATING_ENVIRONMENT_MSG.format(run_name=run_name)): # create an environment create_environment(run_name, local_script_location, script_folder_location) # generate draft's data output, exit_code, log_output = cmd.create(working_directory=run_folder, pack_type=pack_type) # copy requirements file if it was provided, create empty requirements file otherwise dest_requirements_file = os.path.join(run_folder, 'requirements.txt') if requirements_file: shutil.copyfile(requirements_file, dest_requirements_file) else: Path(dest_requirements_file).touch() if exit_code: raise SubmitExperimentError(Texts.DRAFT_TEMPLATES_NOT_GENERATED_ERROR_MSG.format(reason=log_output)) # Script location on experiment container remote_script_location = Path(local_script_location).name if local_script_location else '' if pack_type in JUPYTER_NOTEBOOK_TEMPLATES_NAMES and remote_script_location.endswith(".py"): # for interact (jupyter notebooks) try to convert .py file into .ipynb py_script_location = os.path.join(run_folder, FOLDER_DIR_NAME, remote_script_location) ipynb_file_name = convert_py_to_ipynb(py_script_location, os.path.join(run_folder, FOLDER_DIR_NAME)) local_script_location = ipynb_file_name # reconfigure draft's templates update_configuration(run_folder=run_folder, script_location=remote_script_location, script_parameters=script_parameters, experiment_name=experiment_name, run_name=run_name, local_registry_port=local_registry_port, cluster_registry_port=cluster_registry_port, pack_type=pack_type, pack_params=pack_params, script_folder_location=script_folder_location, env_variables=env_variables) pod_count = get_pod_count(run_folder=run_folder, pack_type=pack_type) except Exception as exe: delete_environment(run_folder) raise SubmitExperimentError('Problems during creation of environments.') from exe log.debug(f'Prepare run {run_name} environment - finish') return PrepareExperimentResult(folder_name=run_folder, script_name=local_script_location, pod_count=pod_count)