def _execute_in_conda_env(conda_env_path, command, install_mlflow, command_env=None):
if command_env is None:
command_env = os.environ
env_id = os.environ.get("MLFLOW_HOME", VERSION) if install_mlflow else None
conda_env_name = get_or_create_conda_env(conda_env_path, env_id=env_id)
activate_conda_env = get_conda_command(conda_env_name)
if install_mlflow:
if "MLFLOW_HOME" in os.environ: # dev version
install_mlflow = "pip install -e {} 1>&2".format(os.environ["MLFLOW_HOME"])
else:
install_mlflow = "pip install mlflow=={} 1>&2".format(VERSION)
activate_conda_env += [install_mlflow]
if os.name != "nt":
separator = " && "
else:
separator = " & "
command = separator.join(activate_conda_env + [command])
_logger.info("=== Running command '%s'", command)
if os.name != "nt":
child = subprocess.Popen(["bash", "-c", command], close_fds=True, env=command_env)
else:
child = subprocess.Popen(["cmd", "/c", command], close_fds=True, env=command_env)
rc = child.wait()
if rc != 0:
raise Exception(
"Command '{0}' returned non zero return code. Return code = {1}".format(command, rc)
)
def run(self, project_uri, entry_point, params, version, backend_config,
tracking_uri, experiment_id):
work_dir = fetch_and_validate_project(project_uri, version,
entry_point, params)
project = load_project(work_dir)
if MLFLOW_LOCAL_BACKEND_RUN_ID_CONFIG in backend_config:
run_id = backend_config[MLFLOW_LOCAL_BACKEND_RUN_ID_CONFIG]
else:
run_id = None
active_run = get_or_create_run(run_id, project_uri, experiment_id,
work_dir, version, entry_point, params)
command_args = []
command_separator = " "
use_conda = backend_config[PROJECT_USE_CONDA]
synchronous = backend_config[PROJECT_SYNCHRONOUS]
docker_args = backend_config[PROJECT_DOCKER_ARGS]
storage_dir = backend_config[PROJECT_STORAGE_DIR]
# If a docker_env attribute is defined in MLproject then it takes precedence over conda yaml
# environments, so the project will be executed inside a docker container.
if project.docker_env:
tracking.MlflowClient().set_tag(active_run.info.run_id,
MLFLOW_PROJECT_ENV, "docker")
validate_docker_env(project)
validate_docker_installation()
image = build_docker_image(
work_dir=work_dir,
repository_uri=project.name,
base_image=project.docker_env.get('image'),
run_id=active_run.info.run_id)
command_args += _get_docker_command(
image=image,
active_run=active_run,
docker_args=docker_args,
volumes=project.docker_env.get("volumes"),
user_env_vars=project.docker_env.get("environment"))
# Synchronously create a conda environment (even though this may take some time)
# to avoid failures due to multiple concurrent attempts to create the same conda env.
elif use_conda:
tracking.MlflowClient().set_tag(active_run.info.run_id,
MLFLOW_PROJECT_ENV, "conda")
command_separator = " && "
conda_env_name = get_or_create_conda_env(project.conda_env_path)
command_args += get_conda_command(conda_env_name)
# In synchronous mode, run the entry point command in a blocking fashion, sending status
# updates to the tracking server when finished. Note that the run state may not be
# persisted to the tracking server if interrupted
if synchronous:
command_args += get_entry_point_command(project, entry_point,
params, storage_dir)
command_str = command_separator.join(command_args)
return _run_entry_point(command_str,
work_dir,
experiment_id,
run_id=active_run.info.run_id)
# Otherwise, invoke `mlflow run` in a subprocess
return _invoke_mlflow_run_subprocess(work_dir=work_dir,
entry_point=entry_point,
parameters=params,
experiment_id=experiment_id,
use_conda=use_conda,
storage_dir=storage_dir,
run_id=active_run.info.run_id)
def _execute_in_conda_env(
conda_env_name,
command,
install_mlflow,
command_env=None,
synchronous=True,
preexec_fn=None,
stdout=None,
stderr=None,
env_root_dir=None,
):
"""
:param conda_env_path conda: conda environment file path
:param command: command to run on the restored conda environment.
:param install_mlflow: whether to install mlflow
:param command_env: environment for child process.
:param synchronous: If True, wait until server process exit and return 0, if process exit
with non-zero return code, raise exception.
If False, return the server process `Popen` instance immediately.
:param stdout: Redirect server stdout
:param stderr: Redirect server stderr
:param env_root_dir: See doc of PyFuncBackend constructor argument `env_root_dir`.
"""
if command_env is None:
command_env = os.environ.copy()
if env_root_dir is not None:
command_env = {
**command_env,
**_get_conda_extra_env_vars(env_root_dir)
}
activate_conda_env = get_conda_command(conda_env_name)
if install_mlflow:
pip_install_mlflow = _get_pip_install_mlflow()
activate_conda_env += [pip_install_mlflow]
if _IS_UNIX:
separator = " && "
else:
separator = " & "
command = separator.join(activate_conda_env + [command])
_logger.info("=== Running command '%s'", command)
if _IS_UNIX:
child = subprocess.Popen(
["bash", "-c", command],
close_fds=True,
env=command_env,
preexec_fn=preexec_fn,
stdout=stdout,
stderr=stderr,
)
else:
child = subprocess.Popen(
["cmd", "/c", command],
close_fds=True,
env=command_env,
preexec_fn=preexec_fn,
stdout=stdout,
stderr=stderr,
)
if synchronous:
rc = child.wait()
if rc != 0:
raise Exception(
"Command '{0}' returned non zero return code. Return code = {1}"
.format(command, rc))
return 0
else:
return child
def _execute_in_conda_env(
conda_env_path,
command,
install_mlflow,
command_env=None,
synchronous=True,
preexec_fn=None,
stdout=None,
stderr=None,
):
"""
:param conda_env_path conda: conda environment file path
:param command: command to run on the restored conda environment.
:install_mlflow: whether to install mlflow
:command_env: environment for child process.
:param synchronous: If True, wait until server process exit and return 0, if process exit
with non-zero return code, raise exception.
If False, return the server process `Popen` instance immediately.
:param stdout: Redirect server stdout
:param stderr: Redirect server stderr
"""
if command_env is None:
command_env = os.environ
# PIP_NO_INPUT=1 make pip run in non-interactive mode,
# otherwise pip might prompt "yes or no" and ask stdin input
command_env["PIP_NO_INPUT"] = "1"
env_id = os.environ.get("MLFLOW_HOME", VERSION) if install_mlflow else None
conda_env_name = get_or_create_conda_env(conda_env_path, env_id=env_id)
activate_conda_env = get_conda_command(conda_env_name)
if install_mlflow:
if "MLFLOW_HOME" in os.environ: # dev version
install_mlflow = "pip install -e {} 1>&2".format(os.environ["MLFLOW_HOME"])
else:
install_mlflow = "pip install mlflow=={} 1>&2".format(VERSION)
activate_conda_env += [install_mlflow]
if os.name != "nt":
separator = " && "
# Add "exec" before the starting scoring server command, so that the scoring server
# process replaces the bash process, otherwise the scoring server process is created
# as a child process of the bash process.
# Note we in `mlflow.pyfunc.spark_udf`, use prctl PR_SET_PDEATHSIG to ensure scoring
# server process being killed when UDF process exit. The PR_SET_PDEATHSIG can only
# send signal to the bash process, if the scoring server process is created as a
# child process of the bash process, then it cannot receive the signal sent by prctl.
# TODO: For Windows, there's no equivalent things of Unix shell's exec. Windows also
# does not support prctl. We need to find an approach to address it.
command = "exec " + command
else:
separator = " & "
command = separator.join(activate_conda_env + [command])
_logger.info("=== Running command '%s'", command)
if os.name != "nt":
child = subprocess.Popen(
["bash", "-c", command],
close_fds=True,
env=command_env,
preexec_fn=preexec_fn,
stdout=stdout,
stderr=stderr,
)
else:
child = subprocess.Popen(
["cmd", "/c", command],
close_fds=True,
env=command_env,
preexec_fn=preexec_fn,
stdout=stdout,
stderr=stderr,
)
if synchronous:
rc = child.wait()
if rc != 0:
raise Exception(
"Command '{0}' returned non zero return code. Return code = {1}".format(command, rc)
)
return 0
else:
return child
def run(
self, project_uri, entry_point, params, version, backend_config, tracking_uri, experiment_id
):
work_dir = fetch_and_validate_project(project_uri, version, entry_point, params)
project = load_project(work_dir)
if MLFLOW_LOCAL_BACKEND_RUN_ID_CONFIG in backend_config:
run_id = backend_config[MLFLOW_LOCAL_BACKEND_RUN_ID_CONFIG]
else:
run_id = None
active_run = get_or_create_run(
run_id, project_uri, experiment_id, work_dir, version, entry_point, params
)
command_args = []
command_separator = " "
env_manager = backend_config[PROJECT_ENV_MANAGER]
synchronous = backend_config[PROJECT_SYNCHRONOUS]
docker_args = backend_config[PROJECT_DOCKER_ARGS]
storage_dir = backend_config[PROJECT_STORAGE_DIR]
# Select an appropriate env manager for the project env type
if env_manager is None:
env_manager = _env_type_to_env_manager(project.env_type)
else:
if project.env_type == env_type.PYTHON and env_manager == _EnvManager.CONDA:
raise MlflowException.invalid_parameter_value(
"python_env project cannot be executed using conda. Set `--env-manager` to "
"'virtualenv' or 'local' to execute this project."
)
# If a docker_env attribute is defined in MLproject then it takes precedence over conda yaml
# environments, so the project will be executed inside a docker container.
if project.docker_env:
from mlflow.projects.docker import (
validate_docker_env,
validate_docker_installation,
build_docker_image,
)
tracking.MlflowClient().set_tag(active_run.info.run_id, MLFLOW_PROJECT_ENV, "docker")
validate_docker_env(project)
validate_docker_installation()
image = build_docker_image(
work_dir=work_dir,
repository_uri=project.name,
base_image=project.docker_env.get("image"),
run_id=active_run.info.run_id,
)
command_args += _get_docker_command(
image=image,
active_run=active_run,
docker_args=docker_args,
volumes=project.docker_env.get("volumes"),
user_env_vars=project.docker_env.get("environment"),
)
# Synchronously create a conda environment (even though this may take some time)
# to avoid failures due to multiple concurrent attempts to create the same conda env.
elif env_manager == _EnvManager.VIRTUALENV:
tracking.MlflowClient().set_tag(
active_run.info.run_id, MLFLOW_PROJECT_ENV, "virtualenv"
)
command_separator = " && "
if project.env_type == env_type.CONDA:
python_env = _PythonEnv.from_conda_yaml(project.env_config_path)
else:
python_env = _PythonEnv.from_yaml(project.env_config_path)
python_bin_path = _install_python(python_env.python)
env_root = _get_mlflow_virtualenv_root()
work_dir_path = Path(work_dir)
env_name = _get_virtualenv_name(python_env, work_dir_path)
env_dir = Path(env_root).joinpath(env_name)
activate_cmd = _create_virtualenv(work_dir_path, python_bin_path, env_dir, python_env)
command_args += [activate_cmd]
elif env_manager == _EnvManager.CONDA:
tracking.MlflowClient().set_tag(active_run.info.run_id, MLFLOW_PROJECT_ENV, "conda")
command_separator = " && "
conda_env_name = get_or_create_conda_env(project.env_config_path)
command_args += get_conda_command(conda_env_name)
# In synchronous mode, run the entry point command in a blocking fashion, sending status
# updates to the tracking server when finished. Note that the run state may not be
# persisted to the tracking server if interrupted
if synchronous:
command_args += get_entry_point_command(project, entry_point, params, storage_dir)
command_str = command_separator.join(command_args)
return _run_entry_point(
command_str, work_dir, experiment_id, run_id=active_run.info.run_id
)
# Otherwise, invoke `mlflow run` in a subprocess
return _invoke_mlflow_run_subprocess(
work_dir=work_dir,
entry_point=entry_point,
parameters=params,
experiment_id=experiment_id,
env_manager=env_manager,
docker_args=docker_args,
storage_dir=storage_dir,
run_id=active_run.info.run_id,
)
