def test_launch_webui_with_kube_config_loading_success(mocked_browser_check,
mocker):
spf_mock = mocker.patch("util.launcher.K8sProxy")
kube_config_mock = mocker.patch(
'util.k8s.k8s_info.config.load_kube_config')
kube_client_mock = mocker.patch(
'kubernetes.client.configuration.Configuration')
wfc_mock = mocker.patch("util.launcher.wait_for_connection")
browser_mock = mocker.patch("util.launcher.webbrowser.open_new")
input_mock = mocker.patch("util.launcher.wait_for_ctrl_c")
if get_current_os() in (OS.WINDOWS, OS.MACOS):
socat_mock = mocker.patch("util.launcher.socat")
runner = CliRunner()
runner.invoke(launch.launch, [APP_NAME])
assert spf_mock.call_count == 1, "port wasn't forwarded"
assert kube_config_mock.call_count == 1, "kube config wasn't loaded"
assert kube_client_mock.call_count == 1, "kubernetes api key wasn't read"
assert wfc_mock.call_count == 1, "connection wasn't checked"
assert browser_mock.call_count == 1, "browser wasn't started"
assert input_mock.call_count == 1, "enter wasn't prompted"
if get_current_os() in (OS.WINDOWS, OS.MACOS):
# noinspection PyUnboundLocalVariable
assert socat_mock.start.call_count == 1, "socat wasn't started"
def test_launch_webui_unsupported_browser(mocked_k8s_config,
mocked_browser_check, mocker):
mocked_browser_check.return_value = False
spf_mock = mocker.patch("util.launcher.K8sProxy")
spf_mock.return_value = 0
wfc_mock = mocker.patch("util.launcher.wait_for_connection")
browser_mock = mocker.patch("util.launcher.webbrowser.open_new")
input_mock = mocker.patch("util.launcher.wait_for_ctrl_c")
if get_current_os() in (OS.WINDOWS, OS.MACOS):
socat_mock = mocker.patch("util.launcher.socat")
runner = CliRunner()
result = runner.invoke(launch.launch, [APP_NAME])
assert spf_mock.call_count == 1, "port wasn't forwarded"
assert wfc_mock.call_count == 0, "connection was checked"
assert browser_mock.call_count == 0, "browser was not started"
assert input_mock.call_count == 0, "enter was prompted"
if get_current_os() in (OS.WINDOWS, OS.MACOS):
# noinspection PyUnboundLocalVariable
assert socat_mock.start.call_count == 0, "socat was started"
assert result.exit_code == 1
def validate_config_path(path: str) -> bool:
if os.path.isdir(path):
directory_content = os.listdir(path)
expected_content = ['draft.exe', 'helm.exe'] if system.get_current_os() == system.OS.WINDOWS \
else ['draft', 'helm']
return set(expected_content).issubset(directory_content)
return False
def validate_config_path(path: str) -> bool:
if os.path.isdir(path):
directory_content = os.listdir(path)
expected_content = {'helm.exe'} if system.get_current_os() == system.OS.WINDOWS \
else {'helm'}
return expected_content.issubset(directory_content)
return False
def set_frames_string():
fallback_frames = "\\|/-"
if get_current_os() == OS.WINDOWS:
return fallback_frames
try:
utf_frames = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
utf_frames.encode(sys.stdout.encoding)
return utf_frames
except UnicodeEncodeError:
return fallback_frames
def test_extract_pack_name_from_path_lack_of_file():
pack_name = "test_pack_name"
if get_current_os() == OS.WINDOWS:
path_to_check = f"C:\\dist\\config\\packs\\{pack_name}\\charts"
else:
path_to_check = f"dist/config/packs/{pack_name}/charts"
ret_pack_name = template.extract_pack_name_from_path(path_to_check)
assert not ret_pack_name
def test_extract_pack_name_from_path_folder_too_short():
pack_name = "test_pack_name"
if get_current_os() == OS.WINDOWS:
path_to_check = f"C:\\{pack_name}\\charts"
else:
path_to_check = f"/{pack_name}/charts"
ret_pack_name = template.extract_pack_name_from_path(path_to_check)
assert not ret_pack_name
def test_validate_config_path_for_existing_config_dir_with_valid_data(mocker):
os_path_mock = mocker.patch('os.path.isdir')
os_path_mock.return_value = True
os_listdir_mock = mocker.patch('os.listdir')
os_listdir_mock.return_value = ['draft.exe', 'helm.exe'] if system.get_current_os() == system.OS.WINDOWS \
else ['draft', 'helm']
result = Config.validate_config_path(FAKE_PATH)
assert result
def test_extract_pack_name_from_path_success():
pack_name = "test_pack_name"
if get_current_os() == OS.WINDOWS:
path_to_check = f"C:\\dist\\config\\packs\\{pack_name}\\charts\\values.yaml"
else:
path_to_check = f"dist/config/packs/{pack_name}/charts/values.yaml"
ret_pack_name = template.extract_pack_name_from_path(path_to_check)
assert ret_pack_name == pack_name
def test_launch_webui_with_browser_success(mocked_k8s_config,
mocked_browser_check, mocker):
spf_mock = mocker.patch("util.launcher.K8sProxy")
wfc_mock = mocker.patch("util.launcher.wait_for_connection")
browser_mock = mocker.patch("util.launcher.webbrowser.open_new")
input_mock = mocker.patch("util.launcher.wait_for_ctrl_c")
if get_current_os() in (OS.WINDOWS, OS.MACOS):
socat_mock = mocker.patch("util.launcher.socat")
runner = CliRunner()
runner.invoke(launch.launch, [APP_NAME])
assert spf_mock.call_count == 1, "port wasn't forwarded"
assert wfc_mock.call_count == 1, "connection wasn't checked"
assert browser_mock.call_count == 1, "browser wasn't started"
assert input_mock.call_count == 1, "enter wasn't prompted"
if get_current_os() in (OS.WINDOWS, OS.MACOS):
# noinspection PyUnboundLocalVariable
assert socat_mock.start.call_count == 1, "socat wasn't started"
def create_environment(experiment_name: str, file_location: str, folder_location: str) -> str:
"""
Creates a complete environment for executing a training using draft.
:param experiment_name: name of an experiment used to create a folder
with content of an experiment
:param file_location: location of a training script
:param folder_location: location of a folder with additional data
:return: (experiment_folder)
experiment_folder - folder with experiment's artifacts
In case of any problems during creation of an enviornment it throws an
exception with a description of a problem
"""
log.debug("Create environment - start")
message_prefix = Texts.CREATE_ENV_MSG_PREFIX
# create a folder for experiment's purposes
run_environment_path = get_run_environment_path(experiment_name)
folder_path = os.path.join(run_environment_path, FOLDER_DIR_NAME)
try:
if not os.path.exists(folder_path):
os.makedirs(folder_path)
except Exception:
log.exception("Create environment - creating experiment folder error.")
raise SubmitExperimentError(message_prefix.format(reason=Texts.EXP_DIR_CANT_BE_CREATED))
# create a semaphore saying that experiment is under submission
Path(os.path.join(run_environment_path, EXP_SUB_SEMAPHORE_FILENAME)).touch()
# copy training script - it overwrites the file taken from a folder_location
if file_location:
try:
shutil.copy2(file_location, folder_path)
if get_current_os() == OS.WINDOWS:
os.chmod(os.path.join(folder_path, os.path.basename(file_location)), 0o666)
except Exception:
log.exception("Create environment - copying training script error.")
raise SubmitExperimentError(message_prefix.format(reason=Texts.TRAINING_SCRIPT_CANT_BE_CREATED))
# copy folder content
if folder_location:
try:
copy_tree(folder_location, folder_path)
except Exception:
log.exception("Create environment - copying training folder error.")
raise SubmitExperimentError(message_prefix.format(reason=Texts.DIR_CANT_BE_COPIED_ERROR_TEXT))
log.debug("Create environment - end")
return run_environment_path
def check_asserts(prepare_mocks: SubmitExperimentMocks, get_namespace_count=1, get_exp_name_count=1, create_env_count=1,
cmd_create_count=1, update_conf_count=1, k8s_proxy_count=1, add_exp_count=1, add_run_count=1,
update_run_count=0, submit_one_count=1, del_env_count=0, socat_start_count=1,
delete_k8s_object_count=0):
assert prepare_mocks.get_namespace.call_count == get_namespace_count, "current user namespace was not fetched"
assert prepare_mocks.gen_exp_name.call_count == get_exp_name_count, "experiment name wasn't created"
assert prepare_mocks.create_env.call_count == create_env_count, "environment wasn't created"
assert prepare_mocks.cmd_create.call_count == cmd_create_count, "deployment wasn't created"
assert prepare_mocks.update_conf.call_count == update_conf_count, "configuration wasn't updated"
assert prepare_mocks.k8s_proxy.call_count == k8s_proxy_count, "port wasn't forwarded"
assert prepare_mocks.add_exp.call_count == add_exp_count, "experiment model was not created"
assert prepare_mocks.add_run.call_count == add_run_count, "run model was not created"
assert prepare_mocks.update_run.call_count == update_run_count, "run model was not updated"
assert prepare_mocks.submit_one.call_count == submit_one_count, "training wasn't deployed"
assert prepare_mocks.del_env.call_count == del_env_count, "environment folder was deleted"
assert prepare_mocks.delete_k8s_object_mock.call_count == delete_k8s_object_count, "experiment was not deleted"
if get_current_os() in (OS.WINDOWS, OS.MACOS):
assert prepare_mocks.socat.start.call_count == socat_start_count, "socat wasn't started"
if socat_start_count > 0:
prepare_mocks.socat.start.assert_called_with(FAKE_NODE_PORT)
def __init__(self, mocker) -> None:
self.mocker = mocker
self.get_namespace = mocker.patch("commands.experiment.common.get_kubectl_current_context_namespace",
side_effect=[EXPERIMENT_NAMESPACE])
self.gen_exp_name = mocker.patch("commands.experiment.common.generate_exp_name_and_labels",
side_effect=[(EXPERIMENT_NAME, {})])
self.add_exp = mocker.patch("platform_resources.experiment.Experiment.create")
self.update_experiment = mocker.patch("platform_resources.experiment.Experiment.update")
self.add_run = mocker.patch("platform_resources.experiment.Run.create")
self.update_run = mocker.patch("platform_resources.experiment.Run.update")
self.cmd_create = mocker.patch("draft.cmd.create", side_effect=[("", 0, "")])
self.submit_one = mocker.patch("commands.experiment.common.submit_draft_pack")
self.update_conf = mocker.patch("commands.experiment.common.update_configuration", side_effect=[0])
self.create_env = mocker.patch("commands.experiment.common.create_environment",
side_effect=[(EXPERIMENT_FOLDER, "")])
self.check_run_env = mocker.patch("commands.experiment.common.check_run_environment",
side_effect=[(EXPERIMENT_FOLDER, "")])
self.del_env = mocker.patch("commands.experiment.common.delete_environment")
self.k8s_proxy = mocker.patch("commands.experiment.common.K8sProxy")
self.k8s_proxy.return_value.__enter__.return_value.tunnel_port = FAKE_NODE_PORT
self.k8s_get_node_port = mocker.patch("commands.experiment.common.get_app_service_node_port")
self.k8s_get_node_port.return_value = FAKE_NODE_PORT
self.socat = mocker.patch("commands.experiment.common.socat") \
if get_current_os() in (OS.WINDOWS, OS.MACOS) else None
self.isdir = mocker.patch("os.path.isdir", return_value=True)
self.isfile = mocker.patch("os.path.isfile", return_value=True)
self.touch = mocker.patch("commands.experiment.common.Path.touch")
self.config_mock = mocker.patch('commands.experiment.common.Config')
self.config_mock.return_value.config_path = FAKE_CLI_CONFIG_DIR_PATH
self.delete_k8s_object_mock = mocker.patch('commands.experiment.common.delete_k8s_object')
self.get_pod_count_mock = mocker.patch('commands.experiment.common.get_pod_count', return_value=1)
self.remove_files = mocker.patch('os.remove')
def submit_experiment(template: str, name: str = None, run_kind: RunKinds = RunKinds.TRAINING,
script_location: str = None, script_parameters: Tuple[str, ...] = None,
pack_params: List[Tuple[str, str]] = None, parameter_range: List[Tuple[str, str]] = None,
parameter_set: Tuple[str, ...] = None,
script_folder_location: str = None,
env_variables: List[str] = None,
requirements_file: str = None) -> (List[Run], Dict[str, str], str):
script_parameters = script_parameters if script_parameters else ()
parameter_set = parameter_set if parameter_set else ()
parameter_range = parameter_range if parameter_range else []
log.debug("Submit experiment - start")
try:
namespace = get_kubectl_current_context_namespace()
global submitted_namespace
submitted_namespace = namespace
except Exception:
message = Texts.GET_NAMESPACE_ERROR_MSG
log.exception(message)
raise SubmitExperimentError(message)
try:
with spinner(text=Texts.PREPARING_RESOURCE_DEFINITIONS_MSG):
experiment_name, labels = generate_exp_name_and_labels(script_name=script_location,
namespace=namespace, name=name,
run_kind=run_kind)
runs_list = prepare_list_of_runs(experiment_name=experiment_name, parameter_range=parameter_range,
parameter_set=parameter_set, template_name=template)
except SubmitExperimentError as exe:
log.exception(str(exe))
raise exe
except Exception:
message = Texts.SUBMIT_PREPARATION_ERROR_MSG
log.exception(message)
raise SubmitExperimentError(message)
global submitted_experiment
submitted_experiment = experiment_name
# Ctrl-C handling
signal.signal(signal.SIGINT, ctrl_c_handler_for_submit)
signal.signal(signal.SIGTERM, ctrl_c_handler_for_submit)
try:
config = Config()
# start port forwarding
# noinspection PyBroadException
with K8sProxy(NAUTAAppNames.DOCKER_REGISTRY, port=config.local_registry_port) as proxy:
# Save port that was actually used in configuration
if proxy.tunnel_port != config.local_registry_port:
config.local_registry_port = proxy.tunnel_port
experiment_run_folders = [] # List of local directories used by experiment's runs
try:
# run socat if on Windows or Mac OS
if get_current_os() in (OS.WINDOWS, OS.MACOS):
# noinspection PyBroadException
try:
with spinner(text=Texts.CLUSTER_CONNECTION_MSG):
socat.start(proxy.tunnel_port)
except Exception:
error_msg = Texts.LOCAL_DOCKER_TUNNEL_ERROR_MSG
log.exception(error_msg)
raise SubmitExperimentError(error_msg)
cluster_registry_port = get_app_service_node_port(nauta_app_name=NAUTAAppNames.DOCKER_REGISTRY)
# prepare environments for all experiment's runs
for experiment_run in runs_list:
if script_parameters and experiment_run.parameters:
current_script_parameters = script_parameters + experiment_run.parameters
elif script_parameters:
current_script_parameters = script_parameters
elif experiment_run.parameters:
current_script_parameters = experiment_run.parameters
else:
current_script_parameters = ""
run_folder, script_location, pod_count = \
prepare_experiment_environment(experiment_name=experiment_name,
run_name=experiment_run.name,
local_script_location=script_location,
script_folder_location=script_folder_location, # noqa: E501
script_parameters=current_script_parameters,
pack_type=template, pack_params=pack_params,
local_registry_port=proxy.tunnel_port,
cluster_registry_port=cluster_registry_port,
env_variables=env_variables,
requirements_file=requirements_file)
# Set correct pod count
if not pod_count or pod_count < 1:
raise SubmitExperimentError('Unable to determine pod count: make sure that values.yaml '
'file in your pack has podCount field with positive integer value.')
experiment_run.pod_count = pod_count
experiment_run_folders.append(run_folder)
script_name = None
if script_location is not None:
script_name = os.path.basename(script_location)
# Prepend script_name parameter to run description only for display purposes.
experiment_run.parameters = script_parameters if not experiment_run.parameters \
else experiment_run.parameters + script_parameters
if experiment_run.parameters and script_name:
experiment_run.parameters = (script_name, ) + experiment_run.parameters
elif script_name:
experiment_run.parameters = (script_name, )
except SubmitExperimentError as e:
log.exception(Texts.ENV_CREATION_ERROR_MSG)
e.message += f' {Texts.ENV_CREATION_ERROR_MSG}'
raise
except Exception:
# any error in this step breaks execution of this command
message = Texts.ENV_CREATION_ERROR_MSG
log.exception(message)
# just in case - remove folders that were created with a success
for experiment_run_folder in experiment_run_folders:
delete_environment(experiment_run_folder)
# if ps or pr option is used - first ask whether experiment(s) should be submitted
if parameter_range or parameter_set:
click.echo(Texts.CONFIRM_SUBMIT_MSG)
click.echo(tabulate({RUN_NAME: [run.name for run in runs_list],
RUN_PARAMETERS: ["\n".join(run.parameters) if run.parameters
else "" for run in runs_list]},
headers=[RUN_NAME, RUN_PARAMETERS], tablefmt="orgtbl"))
if not click.confirm(Texts.CONFIRM_SUBMIT_QUESTION_MSG, default=True):
for experiment_run_folder in experiment_run_folders:
delete_environment(experiment_run_folder)
exit()
# create Experiment model
# TODO template_name & template_namespace should be filled after Template implementation
parameter_range_spec = [f'-pr {param_name} {param_value}' for param_name, param_value in parameter_range]
parameter_set_spec = [f'-ps {ps_spec}' for ps_spec in parameter_set]
experiment_parameters_spec = list(script_parameters) + parameter_range_spec + parameter_set_spec
experiment = experiments_model.Experiment(name=experiment_name, template_name=template,
parameters_spec=experiment_parameters_spec,
template_namespace="template-namespace")
experiment.create(namespace=namespace, labels=labels)
# submit runs
run_errors = {}
for run, run_folder in zip(runs_list, experiment_run_folders):
try:
run.state = RunStatus.QUEUED
with spinner(text=Texts.CREATING_RESOURCES_MSG.format(run_name=run.name)):
# Add Run object with runKind label and pack params as annotations
run.create(namespace=namespace, labels={'runKind': run_kind.value},
annotations={pack_param_name: pack_param_value
for pack_param_name, pack_param_value in pack_params})
submitted_runs.append(run)
submit_draft_pack(run_folder, namespace=namespace)
except Exception as exe:
delete_environment(run_folder)
try:
run.state = RunStatus.FAILED
run_errors[run.name] = str(exe)
run.update()
except Exception as rexe:
# update of non-existing run may fail
log.debug(Texts.ERROR_DURING_PATCHING_RUN.format(str(rexe)))
# Delete experiment if no Runs were submitted
if not submitted_runs:
click.echo(Texts.SUBMISSION_FAIL_ERROR_MSG)
delete_k8s_object("experiment", experiment_name)
# Change experiment status to submitted
experiment.state = experiments_model.ExperimentStatus.SUBMITTED
experiment.update()
except LocalPortOccupiedError as exe:
click.echo(exe.message)
raise SubmitExperimentError(exe.message)
except K8sProxyCloseError:
log.exception('Error during closing of a proxy for a {}'.format(NAUTAAppNames.DOCKER_REGISTRY))
raise K8sProxyCloseError(Texts.PROXY_CLOSE_ERROR_MSG)
except K8sProxyOpenError:
error_msg = Texts.PROXY_OPEN_ERROR_MSG
log.exception(error_msg)
raise SubmitExperimentError(error_msg)
except SubmitExperimentError:
raise
except Exception as exe:
error_msg = Texts.SUBMIT_OTHER_ERROR_MSG
log.exception(error_msg)
raise SubmitExperimentError(error_msg) from exe
finally:
with spinner(text=Texts.CLUSTER_CONNECTION_CLOSING_MSG):
# noinspection PyBroadException
try:
socat.stop()
except Exception:
log.exception("Error during closing of a proxy for a local docker-host tunnel")
raise K8sProxyCloseError(Texts.DOCKER_TUNNEL_CLOSE_ERROR_MSG)
# remove semaphores from all exp folders
remove_sempahore(experiment_name)
log.debug("Submit - finish")
return runs_list, run_errors, script_location
def create_environment(experiment_name: str,
file_location: str = None,
folder_location: str = None,
show_folder_size_warning=True,
max_folder_size_in_bytes=1024 * 1024,
spinner_to_hide=None) -> str:
"""
Creates a complete environment for executing a training using draft.
:param experiment_name: name of an experiment used to create a folder
with content of an experiment
:param file_location: location of a training script
:param folder_location: location of a folder with additional data
:param show_folder_size_warning: if True, a warning will be shown if script folder size exceeds
value in max_folder_size_in_bytes param
:param max_folder_size_in_bytes: maximum script folder size,
:param spinner_to_hide: provide spinner, if it should be hidden before folder size warning
:return: (experiment_folder)
experiment_folder - folder with experiment's artifacts
In case of any problems during creation of an enviornment it throws an
exception with a description of a problem
"""
log.debug("Create environment - start")
message_prefix = Texts.CREATE_ENV_MSG_PREFIX
# create a folder for experiment's purposes
run_environment_path = get_run_environment_path(experiment_name)
folder_path = os.path.join(run_environment_path, FOLDER_DIR_NAME)
try:
if not os.path.exists(folder_path):
os.makedirs(folder_path)
except Exception:
log.exception("Create environment - creating experiment folder error.")
raise SubmitExperimentError(
message_prefix.format(reason=Texts.EXP_DIR_CANT_BE_CREATED))
# create a semaphore saying that experiment is under submission
Path(os.path.join(run_environment_path,
EXP_SUB_SEMAPHORE_FILENAME)).touch()
# copy training script - it overwrites the file taken from a folder_location
if file_location:
try:
shutil.copy2(file_location, folder_path)
if get_current_os() == OS.WINDOWS:
os.chmod(
os.path.join(folder_path, os.path.basename(file_location)),
0o666) # nosec
except Exception:
log.exception(
"Create environment - copying training script error.")
raise SubmitExperimentError(
message_prefix.format(
reason=Texts.TRAINING_SCRIPT_CANT_BE_CREATED))
# copy folder content
if folder_location:
folder_size = get_total_directory_size_in_bytes(folder_location)
if show_folder_size_warning and folder_size >= max_folder_size_in_bytes:
if spinner_to_hide:
spinner_to_hide.hide()
if not click.confirm(
f'Experiment\'s script folder location size ({folder_size/1024/1024:.2f} MB) '
f'exceeds {max_folder_size_in_bytes/1024/1024:.2f} MB. '
f'It is highly recommended to use input/output shares for large amounts of data '
f'instead of submitting them along with experiment. Do you want to continue?'
):
exit(2)
if spinner_to_hide:
spinner_to_hide.show()
try:
copy_tree(folder_location, folder_path)
except Exception:
log.exception(
"Create environment - copying training folder error.")
raise SubmitExperimentError(
message_prefix.format(
reason=Texts.DIR_CANT_BE_COPIED_ERROR_TEXT))
log.debug("Create environment - end")
return run_environment_path
def launch_app(k8s_app_name: NAUTAAppNames = None,
no_launch: bool = False,
port: int = None,
app_name: str = None,
number_of_retries: int = 0,
url_end: str = "",
namespace: str = None):
try:
with spinner(text=Texts.LAUNCHING_APP_MSG) as proxy_spinner, \
K8sProxy(nauta_app_name=k8s_app_name, port=port, app_name=app_name,
number_of_retries=number_of_retries, namespace=namespace) as proxy:
url = FORWARDED_URL.format(proxy.tunnel_port, url_end)
# run socat if on Windows or Mac OS
if get_current_os() in (OS.WINDOWS, OS.MACOS):
# noinspection PyBroadException
try:
socat.start(proxy.container_port)
except Exception:
err_message = Texts.LOCAL_DOCKER_TUNNEL_ERROR_MSG
logger.exception(err_message)
raise LaunchError(err_message)
if k8s_app_name == NAUTAAppNames.INGRESS:
config.load_kube_config()
user_token = configuration.Configuration().api_key.get(
'authorization')
prepared_user_token = user_token.replace('Bearer ', '')
url = f'{url}?token={prepared_user_token}'
if not no_launch:
if is_gui_browser_available():
wait_for_connection(url)
webbrowser.open_new(url)
proxy_spinner.hide()
else:
click.echo(Texts.NO_WEB_BROWSER_ERROR_MSG)
if port and port != proxy.tunnel_port:
click.echo(
Texts.CANNOT_USE_PORT.format(
required_port=port, random_port=proxy.tunnel_port))
proxy_spinner.hide()
click.echo(Texts.GO_TO_MSG.format(url=url))
click.echo(Texts.PROXY_CREATED_MSG)
wait_for_ctrl_c()
except K8sProxyCloseError:
err_message = Texts.PROXY_CLOSE_ERROR_MSG.format(app_name=k8s_app_name)
raise ProxyClosingError(err_message)
except LocalPortOccupiedError as exe:
err_message = Texts.PROXY_CREATED_EXTENDED_ERROR_MSG.format(
app_name=k8s_app_name, reason=exe.message)
raise LaunchError(err_message)
except K8sProxyOpenError:
error_msg = Texts.PROXY_CREATED_ERROR_MSG.format(app_name=k8s_app_name)
logger.exception(error_msg)
raise LaunchError(error_msg)
except LaunchError as e:
raise e
except Exception:
err_message = Texts.WEB_APP_LAUCH_FAIL_MSG
logger.exception(err_message)
raise LaunchError(err_message)
finally:
# noinspection PyBroadException
# terminate socat if on Windows or Mac OS
if get_current_os() in (OS.WINDOWS, OS.MACOS):
# noinspection PyBroadException
try:
with spinner(text=Texts.WEB_APP_CLOSING_MSG):
socat.stop()
except Exception:
err_message = Texts.PROXY_CLOSE_ERROR_MSG.format(k8s_app_name)
raise ProxyClosingError(err_message)
def start_port_forwarding(
k8s_app_name: NAUTAAppNames,
port: int = None,
app_name: str = None,
number_of_retries: int = 0,
namespace: str = None) -> (subprocess.Popen, Optional[int], int):
"""
Creates a proxy responsible for forwarding requests to and from a
kubernetes' local docker proxy. In case of any errors during creating the
process - throws a RuntimeError exception with a short description of
a cause of a problem.
When proxy created by this function is no longer needed - it should
be closed by calling kill() function on a process returned by this
function.
:param k8s_app_name: name of kubernetes application for tunnel creation
value taken from NAUTAAppNames enum
:param port: if given - the system will try to use it as a local port. Random port will be used
if that port is not available
:return:
instance of a process with proxy, tunneled port and container port
"""
logger.debug("Start port forwarding")
try:
service_node_port = None
service_container_port = None
app_services = get_app_services(nauta_app_name=k8s_app_name,
namespace=namespace,
app_name=app_name)
if app_services:
service_node_port = app_services[0].spec.ports[0].node_port
if service_node_port:
logger.debug('Service node port pod has been found: {}'.format(
service_node_port))
service_container_port = app_services[0].spec.ports[0].port
if service_container_port:
logger.debug(
'Service container port has been found: {}'.format(
service_container_port))
service_name = app_services[0].metadata.name
namespace = app_services[0].metadata.namespace
if not service_node_port and not service_container_port:
logger.error(f'Cannot find open ports for {k8s_app_name} app')
raise KubernetesError(Texts.PROXY_CREATION_MISSING_PORT_ERROR_MSG)
if port:
if check_port_availability(port):
tunnel_port = port
else:
tunnel_port = find_random_available_port()
else:
tunnel_port = find_random_available_port()
if system.get_current_os() == system.OS.WINDOWS:
port_forward_command = [
'FOR', '/L', '%N', 'IN', '()', 'DO', 'kubectl', 'port-forward',
f'--namespace={namespace}', f'service/{service_name}',
f'{tunnel_port}:{service_container_port}'
]
else:
port_forward_command = [
'while', 'true;', 'do', 'kubectl', 'port-forward',
f'--namespace={namespace} ', f'service/{service_name}',
f'{tunnel_port}:{service_container_port};', 'done'
]
logger.debug(port_forward_command)
process = None
if number_of_retries:
for i in range(number_of_retries - 1):
try:
process = system.execute_subprocess_command(
port_forward_command, shell=True, join=True)
except Exception:
logger.exception(
"Error during setting up proxy - retrying.")
else:
break
time.sleep(5)
if not process:
process = system.execute_subprocess_command(port_forward_command,
shell=True,
join=True)
except KubernetesError as exe:
raise RuntimeError(exe)
except LocalPortOccupiedError as exe:
raise exe
except Exception:
raise RuntimeError(Texts.PROXY_CREATION_OTHER_ERROR_MSG)
logger.info("Port forwarding - proxy set up")
return process, tunnel_port, service_container_port