def validate_docker_installation():
"""
Verify if Docker is installed on host machine.
"""
try:
docker_path = "docker"
process._exec_cmd([docker_path, "--help"], throw_on_error=False)
except EnvironmentError:
raise ExecutionException(
"Could not find Docker executable. "
"Ensure Docker is installed as per the instructions "
"at https://docs.docker.com/install/overview/.")
def validate_docker_installation():
"""
Verify if Docker is installed and running on host machine.
"""
if shutil.which("docker") is None:
raise ExecutionException(
"Could not find Docker executable. "
"Ensure Docker is installed as per the instructions "
"at https://docs.docker.com/install/overview/."
)
cmd = ["docker", "info"]
prc = process._exec_cmd(
cmd,
throw_on_error=False,
capture_output=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
if prc.returncode != 0:
joined_cmd = " ".join(cmd)
raise ExecutionException(
f"Ran `{joined_cmd}` to ensure docker daemon is running but it failed "
f"with the following output:\n{prc.stdout}"
)
def _run_server(
file_store_path,
default_artifact_root,
serve_artifacts,
artifacts_only,
artifacts_destination,
host,
port,
static_prefix=None,
workers=None,
gunicorn_opts=None,
waitress_opts=None,
expose_prometheus=None,
):
"""
Run the MLflow server, wrapping it in gunicorn or waitress on windows
:param static_prefix: If set, the index.html asset will be served from the path static_prefix.
If left None, the index.html asset will be served from the root path.
:return: None
"""
env_map = {}
if file_store_path:
env_map[BACKEND_STORE_URI_ENV_VAR] = file_store_path
if default_artifact_root:
env_map[ARTIFACT_ROOT_ENV_VAR] = default_artifact_root
if serve_artifacts:
env_map[SERVE_ARTIFACTS_ENV_VAR] = "true"
if artifacts_only:
env_map[ARTIFACTS_ONLY_ENV_VAR] = "true"
if artifacts_destination:
env_map[ARTIFACTS_DESTINATION_ENV_VAR] = artifacts_destination
if static_prefix:
env_map[STATIC_PREFIX_ENV_VAR] = static_prefix
if expose_prometheus:
env_map[PROMETHEUS_EXPORTER_ENV_VAR] = expose_prometheus
# TODO: eventually may want waitress on non-win32
if sys.platform == "win32":
full_command = _build_waitress_command(waitress_opts, host, port)
else:
full_command = _build_gunicorn_command(gunicorn_opts, host, port,
workers or 4)
_exec_cmd(full_command, extra_env=env_map, capture_output=False)
def _list_conda_environments(extra_env=None):
"""
Return a list of names of conda environments.
:param extra_env: extra environment variables for running "conda env list" command.
"""
prc = process._exec_cmd(
[get_conda_bin_executable("conda"), "env", "list", "--json"],
extra_env=extra_env)
return list(map(os.path.basename, json.loads(prc.stdout).get("envs", [])))
def test_run_local_conda_env():
with open(os.path.join(TEST_PROJECT_DIR, "conda.yaml"), "r") as handle:
conda_env_contents = handle.read()
expected_env_name = "mlflow-%s" % hashlib.sha1(
conda_env_contents.encode("utf-8")).hexdigest()
try:
process._exec_cmd(
cmd=["conda", "env", "remove", "--name", expected_env_name])
except process.ShellCommandException:
_logger.error(
"Unable to remove conda environment %s. The environment may not have been present, "
"continuing with running the test.",
expected_env_name,
)
invoke_cli_runner(
cli.run,
[
TEST_PROJECT_DIR, "-e", "check_conda_env", "-P",
"conda_env_name=%s" % expected_env_name
],
)
def _install_python(version, pyenv_root=None, capture_output=False):
"""
Installs a specified version of python with pyenv and returns a path to the installed python
binary.
:param version: Python version to install.
:param pyenv_root: The value of the "PYENV_ROOT" environment variable used when running
`pyenv install` which installs python in `{PYENV_ROOT}/versions/{version}`.
:param capture_output: Set the `capture_output` argument when calling `_exec_cmd`
:return: Path to the installed python binary.
"""
version = (version if _SEMANTIC_VERSION_REGEX.match(version) else
_find_latest_installable_python_version(version))
_logger.info("Installing python %s if it does not exist", version)
# pyenv-win doesn't support `--skip-existing` but its behavior is enabled by default
# https://github.com/pyenv-win/pyenv-win/pull/314
pyenv_install_options = ("--skip-existing", ) if _IS_UNIX else ()
extra_env = {"PYENV_ROOT": pyenv_root} if pyenv_root else None
pyenv_bin_path = _get_pyenv_bin_path()
_exec_cmd(
[pyenv_bin_path, "install", *pyenv_install_options, version],
capture_output=capture_output,
# Windows fails to find pyenv and throws `FileNotFoundError` without `shell=True`
shell=not _IS_UNIX,
extra_env=extra_env,
)
if _IS_UNIX:
if pyenv_root is None:
pyenv_root = _exec_cmd([pyenv_bin_path, "root"],
capture_output=True).stdout.strip()
path_to_bin = ("bin", "python")
else:
# pyenv-win doesn't provide the `pyenv root` command
pyenv_root = pyenv_root or os.getenv("PYENV_ROOT")
if pyenv_root is None:
raise MlflowException(
"Environment variable 'PYENV_ROOT' must be set")
path_to_bin = ("python.exe", )
return Path(pyenv_root).joinpath("versions", version, *path_to_bin)
def _create_virtualenv(local_model_path,
python_bin_path,
env_dir,
python_env,
extra_env=None,
capture_output=False):
# Created a command to activate the environment
paths = ("bin", "activate") if _IS_UNIX else ("Scripts", "activate.bat")
activate_cmd = env_dir.joinpath(*paths)
activate_cmd = f"source {activate_cmd}" if _IS_UNIX else activate_cmd
if env_dir.exists():
_logger.info("Environment %s already exists", env_dir)
return activate_cmd
_logger.info("Creating a new environment %s", env_dir)
_exec_cmd(["virtualenv", "--python", python_bin_path, env_dir],
capture_output=capture_output)
_logger.info("Installing dependencies")
for deps in filter(
None, [python_env.build_dependencies, python_env.dependencies]):
# Create a temporary requirements file in the model directory to resolve the references
# in it correctly.
tmp_req_file = f"requirements.{uuid.uuid4().hex}.txt"
local_model_path.joinpath(tmp_req_file).write_text("\n".join(deps))
try:
cmd = _join_commands(activate_cmd,
f"python -m pip install -r {tmp_req_file}")
_exec_cmd(cmd,
capture_output=capture_output,
cwd=local_model_path,
extra_env=extra_env)
finally:
local_model_path.joinpath(tmp_req_file).unlink()
return activate_cmd
def _find_latest_installable_python_version(version_prefix):
"""
Find the latest installable python version that matches the given version prefix
from the output of `pyenv install --list`. For example, `version_prefix("3.8")` returns '3.8.x'
where 'x' represents the latest micro version in 3.8.
"""
lines = _exec_cmd([_get_pyenv_bin_path(), "install", "--list"],
capture_output=True).stdout.splitlines()
semantic_versions = filter(_SEMANTIC_VERSION_REGEX.match,
map(str.strip, lines))
matched = [v for v in semantic_versions if v.startswith(version_prefix)]
if not matched:
raise MlflowException(
(f"Could not find python version that matches {version_prefix}"))
return sorted(matched, key=Version)[-1]
def _execute_in_virtualenv(
activate_cmd,
command,
install_mlflow,
command_env=None,
synchronous=True,
capture_output=False,
env_root_dir=None,
**kwargs,
):
"""
Runs a command in a specified virtualenv environment.
:param activate_cmd: Command to activate the virtualenv environment.
:param command: Command to run in the virtualenv environment.
:param install_mlflow: Flag to determine whether to install mlflow in the virtualenv
environment.
:param command_env: Environment variables passed to a process running the command.
:param synchronous: Set the `synchronous` argument when calling `_exec_cmd`.
:param capture_output: Set the `capture_output` argument when calling `_exec_cmd`.
:param env_root_dir: See doc of PyFuncBackend constructor argument `env_root_dir`.
:param kwargs: Set the `kwargs` argument when calling `_exec_cmd`
"""
if command_env is None:
command_env = os.environ.copy()
if env_root_dir is not None:
command_env = {
**command_env,
**_get_virtualenv_extra_env_vars(env_root_dir)
}
pre_command = [activate_cmd]
if install_mlflow:
pre_command.append(_get_pip_install_mlflow())
cmd = _join_commands(*pre_command, command)
_logger.info("Running command: %s", " ".join(cmd))
return _exec_cmd(cmd,
capture_output=capture_output,
env=command_env,
synchronous=synchronous,
**kwargs)
def clean_envs_and_cache():
yield
if get_free_disk_space() < 7.0: # unit: GiB
process._exec_cmd(["./dev/remove-conda-envs.sh"])
def remove_conda_env(env_name):
process._exec_cmd(
["conda", "remove", "--name", env_name, "--yes", "--all"])
def get_conda_envs():
stdout = process._exec_cmd(["conda", "env", "list", "--json"]).stdout
return [os.path.basename(env) for env in json.loads(stdout)["envs"]]
def test_command_example(directory, command):
cwd_dir = os.path.join(EXAMPLES_DIR, directory)
process._exec_cmd(command, cwd=cwd_dir)
def _fetch_dbfs(uri, local_path):
_logger.info(
"=== Downloading DBFS file %s to local path %s ===", uri, os.path.abspath(local_path)
)
process._exec_cmd(cmd=["databricks", "fs", "cp", "-r", uri, local_path])
def get_or_create_conda_env(conda_env_path,
env_id=None,
capture_output=False,
env_root_dir=None):
"""
Given a `Project`, creates a conda environment containing the project's dependencies if such a
conda environment doesn't already exist. Returns the name of the conda environment.
:param conda_env_path: Path to a conda yaml file.
:param env_id: Optional string that is added to the contents of the yaml file before
calculating the hash. It can be used to distinguish environments that have the
same conda dependencies but are supposed to be different based on the context.
For example, when serving the model we may install additional dependencies to the
environment after the environment has been activated.
:param capture_output: Specify the capture_output argument while executing the
"conda env create" command.
:param env_root_dir: See doc of PyFuncBackend constructor argument `env_root_dir`.
"""
conda_path = get_conda_bin_executable("conda")
conda_env_create_path = _get_conda_executable_for_create_env()
try:
process._exec_cmd([conda_path, "--help"], throw_on_error=False)
except EnvironmentError:
raise ExecutionException(
"Could not find Conda executable at {0}. "
"Ensure Conda is installed as per the instructions at "
"https://conda.io/projects/conda/en/latest/"
"user-guide/install/index.html. "
"You can also configure MLflow to look for a specific "
"Conda executable by setting the {1} environment variable "
"to the path of the Conda executable".format(
conda_path, MLFLOW_CONDA_HOME))
try:
process._exec_cmd([conda_env_create_path, "--help"],
throw_on_error=False)
except EnvironmentError:
raise ExecutionException(
"You have set the env variable {0}, but {1} does not exist or "
"it is not working properly. Note that {1} and the conda executable need to be "
"in the same conda environment. You can change the search path by"
"modifying the env variable {2}".format(
MLFLOW_CONDA_CREATE_ENV_CMD,
conda_env_create_path,
MLFLOW_CONDA_HOME,
))
conda_extra_env_vars = _get_conda_extra_env_vars(env_root_dir)
# Include the env_root_dir hash in the project_env_name,
# this is for avoid conda env name conflicts between different CONDA_ENVS_PATH.
project_env_name = _get_conda_env_name(conda_env_path,
env_id=env_id,
env_root_dir=env_root_dir)
if env_root_dir is not None:
project_env_path = os.path.join(env_root_dir, _CONDA_ENVS_DIR,
project_env_name)
else:
project_env_path = project_env_name
if project_env_name in _list_conda_environments(conda_extra_env_vars):
_logger.info("Conda environment %s already exists.", project_env_path)
return project_env_name
_logger.info("=== Creating conda environment %s ===", project_env_path)
try:
if conda_env_path:
process._exec_cmd(
[
conda_env_create_path,
"env",
"create",
"-n",
project_env_name,
"--file",
conda_env_path,
],
extra_env=conda_extra_env_vars,
capture_output=capture_output,
)
else:
process._exec_cmd(
[
conda_env_create_path,
"create",
"--channel",
"conda-forge",
"--yes",
"--override-channels",
"-n",
project_env_name,
"python",
],
extra_env=conda_extra_env_vars,
capture_output=capture_output,
)
return project_env_name
except Exception:
try:
if project_env_name in _list_conda_environments(
conda_extra_env_vars):
_logger.warning(
"Encountered unexpected error while creating conda environment. "
"Removing %s.",
project_env_path,
)
process._exec_cmd(
[
conda_path,
"remove",
"--yes",
"--name",
project_env_name,
"--all",
],
extra_env=conda_extra_env_vars,
capture_output=False,
)
except Exception as e:
_logger.warning(
"Removing conda environment %s failed (error: %s)",
project_env_path,
repr(e),
)
raise
def _list_conda_environments():
prc = process._exec_cmd([get_conda_bin_executable("conda"), "env", "list", "--json"])
return list(map(os.path.basename, json.loads(prc.stdout).get("envs", [])))
