def push_image_to_registry(image_tag):
client = docker.from_env(timeout=_DOCKER_API_TIMEOUT)
_logger.info("=== Pushing docker image %s ===", image_tag)
for line in client.images.push(repository=image_tag,
stream=True,
decode=True):
if "error" in line and line["error"]:
raise ExecutionException("Error while pushing to docker registry: "
"{error}".format(error=line["error"]))
return client.images.get_registry_data(image_tag).id
def _validate_parameters(self, user_parameters):
missing_params = []
for name in self.parameters:
if name not in user_parameters and self.parameters[
name].default is None:
missing_params.append(name)
if missing_params:
raise ExecutionException(
"No value given for missing parameters: %s" %
", ".join(["'%s'" % name for name in missing_params]))
def run(self, project_uri, entry_point, params, version, backend_config,
tracking_uri, experiment_id):
# doing this handling bc positional argument fix not in mlflow <= 1.10.0 release
# https://github.com/mlflow/mlflow/issues/3138
if _AZUREML_URI not in tracking_uri and _AZUREML_URI in experiment_id:
tracking_uri, experiment_id = experiment_id, tracking_uri
# use_conda value from mlflow.project.run call propagated to backend_config
# release after 1.10.0, so if use_conda key not in backend config, assume to be True
# if the user hasn't passed a backend_config and has set backend="azureml", assume
# that it's a local run using local conda environment (i.e. use_conda = False)
if backend_config is None:
backend_config = {PROJECT_USE_CONDA: False}
elif PROJECT_USE_CONDA not in backend_config:
backend_config[PROJECT_USE_CONDA] = True
use_conda = backend_config[PROJECT_USE_CONDA]
stream_output = backend_config[
STREAM_OUTPUT] if STREAM_OUTPUT in backend_config else True
compute = backend_config[COMPUTE] if COMPUTE in backend_config else None
try:
work_dir = fetch_and_validate_project(project_uri, version,
entry_point, params)
mlproject = load_project(work_dir)
except ExecutionException as e:
raise ExecutionException(e)
# process mlflow parameters into a format usable for AzureML ScriptRunConfig
command_args = []
command_args += get_entry_point_command(mlproject, entry_point, params,
None)
# components for launching an AzureML ScriptRun
workspace = load_azure_workspace()
experiment = _load_azure_experiment(workspace, experiment_id)
# TODO: mlflow system tag mlflow.source.name is null after the switch from script, args to command
src = ScriptRunConfig(source_directory=work_dir, command=command_args)
# in case customer sets target to local
if compute and compute != _LOCAL and compute != _LOCAL.upper():
remote_environment = _load_remote_environment(mlproject)
registered_env = remote_environment.register(workspace=workspace)
cpu_cluster = _load_compute_target(workspace, backend_config)
src.run_config.target = cpu_cluster.name
src.run_config.environment = registered_env
else:
local_environment = _load_local_environment(mlproject, use_conda)
src.run_config.environment = local_environment
submitted_run = experiment.submit(config=src)
_logger.info(
_CONSOLE_MSG.format(
"AzureML-Mlflow {} Experiment submitted".format(
experiment.name)))
return AzureMLSubmittedRun(submitted_run, stream_output)
def _compute_path_value(self, user_param_value, storage_dir):
if not data.is_uri(user_param_value):
if not os.path.exists(user_param_value):
raise ExecutionException("Got value %s for parameter %s, but no such file or "
"directory was found." % (user_param_value, self.name))
return os.path.abspath(user_param_value)
basename = os.path.basename(user_param_value)
dest_path = os.path.join(storage_dir, basename)
if dest_path != user_param_value:
data.download_uri(uri=user_param_value, output_path=dest_path)
return os.path.abspath(dest_path)
def _parse_subdirectory(uri):
# Parses a uri and returns the uri and subdirectory as separate values.
# Uses '#' as a delimiter.
subdirectory = ''
parsed_uri = uri
if '#' in uri:
subdirectory = uri[uri.find('#') + 1:]
parsed_uri = uri[:uri.find('#')]
if subdirectory and '.' in subdirectory:
raise ExecutionException("'.' is not allowed in project subdirectory paths.")
return parsed_uri, subdirectory
def _validate_docker_installation():
"""
Verify if Docker is installed on host machine.
"""
try:
docker_path = "docker"
process.exec_cmd([docker_path, "--help"], throw_on_error=False)
except EnvironmentError:
raise ExecutionException("Could not find Docker executable. "
"Ensure Docker is installed as per the instructions "
"at https://docs.docker.com/install/overview/.")
def before_run_validations(tracking_uri, backend_config):
"""Validations to perform before running a project on Databricks."""
if backend_config is None:
raise ExecutionException(
"Backend spec must be provided when launching MLflow project "
"runs on Databricks.")
elif "existing_cluster_id" in backend_config:
raise MlflowException(message=(
"MLflow Project runs on Databricks must provide a *new cluster* specification."
" Project execution against existing clusters is not currently supported. For more"
" information, see https://mlflow.org/docs/latest/projects.html"
"#run-an-mlflow-project-on-databricks"),
error_code=INVALID_PARAMETER_VALUE)
if not is_databricks_uri(tracking_uri) and \
not is_http_uri(tracking_uri):
raise ExecutionException(
"When running on Databricks, the MLflow tracking URI must be of the form "
"'databricks' or 'databricks://profile', or a remote HTTP URI accessible to both the "
"current client and code running on Databricks. Got local tracking URI %s. "
"Please specify a valid tracking URI via mlflow.set_tracking_uri or by setting the "
"MLFLOW_TRACKING_URI environment variable." % tracking_uri)
def _compute_path_value(self, user_param_value, storage_dir, key_position):
local_path = get_local_path_or_none(user_param_value)
if local_path:
if not os.path.exists(local_path):
raise ExecutionException(
"Got value %s for parameter %s, but no such file or "
"directory was found." % (user_param_value, self.name))
return os.path.abspath(local_path)
target_sub_dir = "param_{}".format(key_position)
download_dir = os.path.join(storage_dir, target_sub_dir)
os.mkdir(download_dir)
return artifact_utils._download_artifact_from_uri(
artifact_uri=user_param_value, output_path=download_dir)
def _fetch_zip_repo(uri):
import requests
from io import BytesIO
# TODO (dbczumar): Replace HTTP resolution via ``requests.get`` with an invocation of
# ```mlflow.data.download_uri()`` when the API supports the same set of available stores as
# the artifact repository (Azure, FTP, etc). See the following issue:
# https://github.com/mlflow/mlflow/issues/763.
response = requests.get(uri)
try:
response.raise_for_status()
except requests.HTTPError as error:
raise ExecutionException("Unable to retrieve ZIP file. Reason: %s" % str(error))
return BytesIO(response.content)
def _fetch_project(uri, force_tempdir, version=None):
"""
Fetch a project into a local directory, returning the path to the local project directory.
:param force_tempdir: If True, will fetch the project into a temporary directory. Otherwise,
will fetch ZIP or Git projects into a temporary directory but simply
return the path of local projects (i.e. perform a no-op for local
projects).
"""
parsed_uri, subdirectory = _parse_subdirectory(uri)
use_temp_dst_dir = force_tempdir or _is_zip_uri(
parsed_uri) or not _is_local_uri(parsed_uri)
dst_dir = tempfile.mkdtemp() if use_temp_dst_dir else parsed_uri
if use_temp_dst_dir:
_logger.info("=== Fetching project from %s into %s ===", uri, dst_dir)
if _is_zip_uri(parsed_uri):
if _is_file_uri(parsed_uri):
from six.moves import urllib
parsed_file_uri = urllib.parse.urlparse(
urllib.parse.unquote(parsed_uri))
parsed_uri = os.path.join(parsed_file_uri.netloc,
parsed_file_uri.path)
_unzip_repo(zip_file=(parsed_uri if _is_local_uri(parsed_uri) else
_fetch_zip_repo(parsed_uri)),
dst_dir=dst_dir)
elif _is_local_uri(uri):
if version is not None:
raise ExecutionException(
"Setting a version is only supported for Git project URIs")
if use_temp_dst_dir:
dir_util.copy_tree(src=parsed_uri, dst=dst_dir)
else:
assert _GIT_URI_REGEX.match(
parsed_uri), "Non-local URI %s should be a Git URI" % parsed_uri
_fetch_git_repo(parsed_uri, version, dst_dir)
res = os.path.abspath(os.path.join(dst_dir, subdirectory))
if not os.path.exists(res):
raise ExecutionException("Could not find subdirectory %s of %s" %
(subdirectory, dst_dir))
return res
def _parse_subdirectory(uri):
# Parses a uri and returns the uri and subdirectory as separate values.
# Uses '#' as a delimiter.
unquoted_uri = _strip_quotes(uri)
subdirectory = ""
parsed_uri = unquoted_uri
if "#" in unquoted_uri:
subdirectory = unquoted_uri[unquoted_uri.find("#") + 1:]
parsed_uri = unquoted_uri[:unquoted_uri.find("#")]
if subdirectory and "." in subdirectory:
raise ExecutionException(
"'.' is not allowed in project subdirectory paths.")
return parsed_uri, subdirectory
def _fetch_git_repo(uri, version, dst_dir, git_username, git_password):
"""
Clone the git repo at ``uri`` into ``dst_dir``, checking out commit ``version`` (or defaulting
to the head commit of the repository's master branch if version is unspecified).
If ``git_username`` and ``git_password`` are specified, uses them to authenticate while fetching
the repo. Otherwise, assumes authentication parameters are specified by the environment,
e.g. by a Git credential helper.
"""
# We defer importing git until the last moment, because the import requires that the git
# executable is availble on the PATH, so we only want to fail if we actually need it.
import git
repo = git.Repo.init(dst_dir)
origin = repo.create_remote("origin", uri)
git_args = [git_username, git_password]
if not (all(arg is not None
for arg in git_args) or all(arg is None for arg in git_args)):
raise ExecutionException(
"Either both or neither of git_username and git_password must be "
"specified.")
if git_username:
git_credentials = "url=%s\nusername=%s\npassword=%s" % (
uri, git_username, git_password)
repo.git.config("--local", "credential.helper", "cache")
process.exec_cmd(cmd=["git", "credential-cache", "store"],
cwd=dst_dir,
cmd_stdin=git_credentials)
origin.fetch()
if version is not None:
try:
repo.git.checkout(version)
except git.exc.GitCommandError as e:
raise ExecutionException(
"Unable to checkout version '%s' of git repo %s"
"- please ensure that the version exists in the repo. "
"Error: %s" % (version, uri, e))
else:
repo.create_head("master", origin.refs.master)
repo.heads.master.checkout()
def get_entry_point(self, entry_point):
if entry_point in self._entry_points:
return self._entry_points[entry_point]
_, file_extension = os.path.splitext(entry_point)
ext_to_cmd = {".py": "python", ".sh": os.environ.get("SHELL", "bash")}
if file_extension in ext_to_cmd:
command = "%s %s" % (ext_to_cmd[file_extension], shlex_quote(entry_point))
if type(command) not in six.string_types:
command = command.encode("utf-8")
return EntryPoint(name=entry_point, parameters={}, command=command)
raise ExecutionException("Could not find {0} among entry points {1} or interpret {0} as a "
"runnable script. Supported script file extensions: "
"{2}".format(entry_point, list(self._entry_points.keys()),
list(ext_to_cmd.keys())))
def _fetch_git_repo(uri, version, dst_dir):
"""
Clone the git repo at ``uri`` into ``dst_dir``, checking out commit ``version`` (or defaulting
to the head commit of the repository's master branch if version is unspecified).
Assumes authentication parameters are specified by the environment, e.g. by a Git credential
helper.
"""
# We defer importing git until the last moment, because the import requires that the git
# executable is available on the PATH, so we only want to fail if we actually need it.
import git
repo = git.Repo.init(dst_dir)
origin = repo.create_remote("origin", uri)
if version is not None:
try:
origin.fetch(refspec=version, depth=GIT_FETCH_DEPTH)
repo.git.checkout(version)
except git.exc.GitCommandError as e:
raise ExecutionException(
"Unable to checkout version '%s' of git repo %s"
"- please ensure that the version exists in the repo. "
"Error: %s" % (version, uri, e))
else:
g = git.cmd.Git(dst_dir)
cmd = ["git", "remote", "show", "origin"]
output = g.execute(cmd)
head_branch = _get_head_branch(output)
if head_branch is None:
raise ExecutionException(
"Failed to find HEAD branch. Output of `{cmd}`:\n{output}".
format(cmd=" ".join(cmd), output=output))
origin.fetch(head_branch, depth=GIT_FETCH_DEPTH)
ref = origin.refs[0]
_logger.info("Fetched '%s' branch", head_branch)
repo.create_head(head_branch, ref)
repo.heads[head_branch].checkout()
repo.submodule_update(init=True, recursive=True)
def _fetch_project(uri, force_tempdir, version=None, git_username=None, git_password=None):
"""
Fetch a project into a local directory, returning the path to the local project directory.
:param force_tempdir: If True, will fetch the project into a temporary directory. Otherwise,
will fetch Git projects into a temporary directory but simply return the
path of local projects (i.e. perform a no-op for local projects).
"""
parsed_uri, subdirectory = _parse_subdirectory(uri)
use_temp_dst_dir = force_tempdir or not _is_local_uri(parsed_uri)
dst_dir = tempfile.mkdtemp() if use_temp_dst_dir else parsed_uri
if use_temp_dst_dir:
eprint("=== Fetching project from %s into %s ===" % (uri, dst_dir))
if _is_local_uri(uri):
if version is not None:
raise ExecutionException("Setting a version is only supported for Git project URIs")
if use_temp_dst_dir:
dir_util.copy_tree(src=parsed_uri, dst=dst_dir)
else:
assert _GIT_URI_REGEX.match(parsed_uri), "Non-local URI %s should be a Git URI" % parsed_uri
_fetch_git_repo(parsed_uri, version, dst_dir, git_username, git_password)
res = os.path.abspath(os.path.join(dst_dir, subdirectory))
if not os.path.exists(res):
raise ExecutionException("Could not find subdirectory %s of %s" % (subdirectory, dst_dir))
return res
def load_project(directory):
mlproject_path = os.path.join(directory, MLPROJECT_FILE_NAME)
# TODO: Validate structure of YAML loaded from the file
if os.path.exists(mlproject_path):
with open(mlproject_path) as mlproject_file:
yaml_obj = yaml.safe_load(mlproject_file.read())
else:
yaml_obj = {}
project_name = yaml_obj.get("name")
if not project_name:
project_name = None
conda_path = yaml_obj.get("conda_env")
docker_env = yaml_obj.get("docker_env")
if docker_env and not docker_env.get("image"):
raise ExecutionException("Docker environment specified but no image "
"attribute found.")
if conda_path and docker_env:
raise ExecutionException("Project cannot contain both a docker and conda environment.")
entry_points = {}
for name, entry_point_yaml in yaml_obj.get("entry_points", {}).items():
parameters = entry_point_yaml.get("parameters", {})
command = entry_point_yaml.get("command")
entry_points[name] = EntryPoint(name, parameters, command)
if conda_path:
conda_env_path = os.path.join(directory, conda_path)
if not os.path.exists(conda_env_path):
raise ExecutionException("Project specified conda environment file %s, but no such "
"file was found." % conda_env_path)
return Project(conda_env_path=conda_env_path, entry_points=entry_points,
docker_env=docker_env, name=project_name,)
default_conda_path = os.path.join(directory, DEFAULT_CONDA_FILE_NAME)
if os.path.exists(default_conda_path):
return Project(conda_env_path=default_conda_path, entry_points=entry_points,
docker_env=docker_env, name=project_name)
return Project(conda_env_path=None, entry_points=entry_points,
docker_env=docker_env, name=project_name)
def _parse_kubernetes_config(backend_config):
"""
Creates build context tarfile containing Dockerfile and project code, returning path to tarfile
"""
if not backend_config:
raise ExecutionException("Backend_config file not found.")
kube_config = backend_config.copy()
if 'kube-job-template-path' not in backend_config.keys():
raise ExecutionException("'kube-job-template-path' attribute must be specified in "
"backend_config.")
kube_job_template = backend_config['kube-job-template-path']
if os.path.exists(kube_job_template):
with open(kube_job_template, 'r') as job_template:
yaml_obj = yaml.safe_load(job_template.read())
kube_job_template = yaml_obj
kube_config['kube-job-template'] = kube_job_template
else:
raise ExecutionException("Could not find 'kube-job-template-path': {}".format(
kube_job_template))
if 'kube-context' not in backend_config.keys():
raise ExecutionException("Could not find kube-context in backend_config.")
if 'repository-uri' not in backend_config.keys():
raise ExecutionException("Could not find 'repository-uri' in backend_config.")
return kube_config
def _translate_to_runstate(self, app_state: str) -> RunStatus:
if app_state == skein.model.ApplicationState.FINISHED:
return RunStatus.FINISHED
elif app_state == skein.model.ApplicationState.KILLED:
return RunStatus.KILLED
elif app_state == skein.model.ApplicationState.FAILED:
return RunStatus.FAILED
elif (app_state == skein.model.ApplicationState.NEW_SAVING
or app_state == skein.model.ApplicationState.ACCEPTED
or app_state == skein.model.ApplicationState.SUBMITTED):
return RunStatus.SCHEDULED
elif app_state == skein.model.ApplicationState.RUNNING:
return RunStatus.RUNNING
raise ExecutionException(f"YARN Application {self._skein_app_id}"
f" has invalid status: {app_state}")
def _dbfs_path_exists(self, dbfs_uri):
"""
Returns True if the passed-in path exists in DBFS for the workspace corresponding to the
default Databricks CLI profile.
"""
dbfs_path = _parse_dbfs_uri_path(dbfs_uri)
json_response_obj = self.databricks_api_request(
endpoint="/api/2.0/dbfs/get-status", method="GET", json={"path": dbfs_path})
# If request fails with a RESOURCE_DOES_NOT_EXIST error, the file does not exist on DBFS
error_code_field = "error_code"
if error_code_field in json_response_obj:
if json_response_obj[error_code_field] == "RESOURCE_DOES_NOT_EXIST":
return False
raise ExecutionException("Got unexpected error response when checking whether file %s "
"exists in DBFS: %s" % json_response_obj)
return True
def _load_azure_experiment(workspace, experiment_id):
''' Experiment name is registered by MLflow from the script and is
referenced by its ID, which we can use to retrieve the experiment name.
:param workspace: AzureML Workspace object
:param experiment_id: MLflow experiment ID
:return experiment: AzureML Experiment object
'''
experiment_name = MlflowClient().get_experiment(experiment_id).name
try:
experiment = Experiment(workspace=workspace, name=experiment_name)
_logger.info(
_CONSOLE_MSG.format(
"Experiment {} found.".format(experiment_name)))
except ExperimentExecutionException as e:
raise ExecutionException(
"Mlflow Experiment name not found. /n{}".format(e))
return experiment
def _dbfs_path_exists(self, dbfs_path):
"""
Return True if the passed-in path exists in DBFS for the workspace corresponding to the
default Databricks CLI profile. The path is expected to be a relative path to the DBFS root
directory, e.g. 'path/to/file'.
"""
response = self._databricks_api_request(
endpoint="/api/2.0/dbfs/get-status", method="GET", json={"path": "/%s" % dbfs_path})
json_response_obj = json.loads(response.text)
# If request fails with a RESOURCE_DOES_NOT_EXIST error, the file does not exist on DBFS
error_code_field = "error_code"
if error_code_field in json_response_obj:
if json_response_obj[error_code_field] == "RESOURCE_DOES_NOT_EXIST":
return False
raise ExecutionException("Got unexpected error response when checking whether file %s "
"exists in DBFS: %s" % (dbfs_path, json_response_obj))
return True
def _wait_for(submitted_run_obj):
"""Wait on the passed-in submitted run, reporting its status to the tracking server."""
run_id = submitted_run_obj.run_id
active_run = None
# Note: there's a small chance we fail to report the run's status to the tracking server if
# we're interrupted before we reach the try block below
try:
active_run = tracking.MlflowClient().get_run(run_id) if run_id is not None else None
if submitted_run_obj.wait():
_logger.info("=== Run (ID '%s') succeeded ===", run_id)
_maybe_set_run_terminated(active_run, "FINISHED")
else:
_maybe_set_run_terminated(active_run, "FAILED")
raise ExecutionException("Run (ID '%s') failed" % run_id)
except KeyboardInterrupt:
_logger.error("=== Run (ID '%s') interrupted, cancelling run ===", run_id)
submitted_run_obj.cancel()
_maybe_set_run_terminated(active_run, "FAILED")
raise
def _run(uri, entry_point="main", version=None, parameters=None, experiment_id=None,
mode=None, cluster_spec=None, git_username=None, git_password=None, use_conda=True,
storage_dir=None, block=True, run_id=None):
"""
Helper that delegates to the project-running method corresponding to the passed-in mode.
Returns a ``SubmittedRun`` corresponding to the project run.
"""
exp_id = experiment_id or _get_experiment_id()
parameters = parameters or {}
work_dir = _fetch_project(uri=uri, force_tempdir=False, version=version,
git_username=git_username, git_password=git_password)
project = _project_spec.load_project(work_dir)
project.get_entry_point(entry_point)._validate_parameters(parameters)
if run_id:
active_run = tracking.get_service().get_run(run_id)
else:
active_run = _create_run(uri, exp_id, work_dir, entry_point, parameters)
if mode == "databricks":
from mlflow.projects.databricks import run_databricks
return run_databricks(
remote_run=active_run,
uri=uri, entry_point=entry_point, work_dir=work_dir, parameters=parameters,
experiment_id=exp_id, cluster_spec=cluster_spec)
elif mode == "local" or mode is None:
# Synchronously create a conda environment (even though this may take some time) to avoid
# failures due to multiple concurrent attempts to create the same conda env.
conda_env_name = _get_or_create_conda_env(project.conda_env_path) if use_conda else None
# In blocking mode, run the entry point command in blocking fashion, sending status updates
# to the tracking server when finished. Note that the run state may not be persisted to the
# tracking server if interrupted
if block:
command = _get_entry_point_command(
project, entry_point, parameters, conda_env_name, storage_dir)
return _run_entry_point(command, work_dir, exp_id, run_id=active_run.info.run_uuid)
# Otherwise, invoke `mlflow run` in a subprocess
return _invoke_mlflow_run_subprocess(
work_dir=work_dir, entry_point=entry_point, parameters=parameters, experiment_id=exp_id,
use_conda=use_conda, storage_dir=storage_dir, run_id=active_run.info.run_uuid)
supported_modes = ["local", "databricks"]
raise ExecutionException("Got unsupported execution mode %s. Supported "
"values: %s" % (mode, supported_modes))
def _build_docker_image(work_dir, project, active_run):
"""
Build a docker image containing the project in `work_dir`, using the base image and tagging the
built image with the project name specified by `project`.
"""
if not project.name:
raise ExecutionException(
"Project name in MLProject must be specified when using docker "
"for image tagging.")
tag_name = "mlflow-{name}-{version}".format(
name=(project.name if project.name else "docker-project"),
version=_get_git_commit(work_dir)[:7],
)
dockerfile = ("FROM {imagename}\n"
"LABEL Name={tag_name}\n"
"COPY {build_context_path}/* /mlflow/projects/code/\n"
"WORKDIR /mlflow/projects/code/\n").format(
imagename=project.docker_env.get('image'),
tag_name=tag_name,
build_context_path=_PROJECT_TAR_ARCHIVE_NAME)
build_ctx_path = _create_docker_build_ctx(work_dir, dockerfile)
with open(build_ctx_path, 'rb') as docker_build_ctx:
_logger.info("=== Building docker image %s ===", tag_name)
client = docker.from_env()
image = client.images.build(tag=tag_name,
forcerm=True,
dockerfile=posixpath.join(
_PROJECT_TAR_ARCHIVE_NAME,
_GENERATED_DOCKERFILE_NAME),
fileobj=docker_build_ctx,
custom_context=True,
encoding="gzip")
try:
os.remove(build_ctx_path)
except Exception: # pylint: disable=broad-except
_logger.info("Temporary docker context file %s was not deleted.",
build_ctx_path)
tracking.MlflowClient().set_tag(active_run.info.run_uuid,
MLFLOW_DOCKER_IMAGE_NAME, tag_name)
tracking.MlflowClient().set_tag(active_run.info.run_uuid,
MLFLOW_DOCKER_IMAGE_ID, image[0].id)
return tag_name
def _get_or_create_conda_env(conda_env_path, env_id=None):
"""
Given a `Project`, creates a conda environment containing the project's dependencies if such a
conda environment doesn't already exist. Returns the name of the conda environment.
:param conda_env_path: Path to a conda yaml file.
:param env_id: Optional string that is added to the contents of the yaml file before
calculating the hash. It can be used to distinguish environments that have the
same conda dependencies but are supposed to be different based on the context.
For example, when serving the model we may install additional dependencies to the
environment after the environment has been activated.
"""
conda_path = _get_conda_bin_executable("conda")
try:
process.exec_cmd([conda_path, "--help"], throw_on_error=False)
except EnvironmentError:
raise ExecutionException(
"Could not find Conda executable at {0}. "
"Ensure Conda is installed as per the instructions "
"at https://conda.io/docs/user-guide/install/index.html. You can "
"also configure MLflow to look for a specific Conda executable "
"by setting the {1} environment variable to the path of the Conda "
"executable".format(conda_path, MLFLOW_CONDA_HOME))
# The approach here is to directly run the user's conda executable (e.g. on Databricks or other
# environments where MLFLOW_CONDA_HOME is set), and set up the shell to detect the conda
# bash function otherwise
(_, stdout, _) = process.exec_cmd([conda_path, "env", "list", "--json"])
env_names = [os.path.basename(env) for env in json.loads(stdout)['envs']]
project_env_name = _get_conda_env_name(conda_env_path, env_id)
if project_env_name not in env_names:
_logger.info('=== Creating conda environment %s ===', project_env_name)
if conda_env_path:
process.exec_cmd([
conda_path, "env", "create", "-n", project_env_name, "--file",
conda_env_path
],
stream_output=True)
else:
process.exec_cmd([
conda_path, "env", "create", "-n", project_env_name, "python"
],
stream_output=True)
return project_env_name
def _get_conda_command(conda_env_name, direct_output_to_err=False):
conda_path = _get_conda_bin_executable("conda")
activate_path = _get_conda_bin_executable("activate")
try:
process.exec_cmd([conda_path, "--help"], throw_on_error=False)
except EnvironmentError:
raise ExecutionException(
"Could not find Conda executable at {0}. "
"Ensure Conda is installed as per the instructions "
"at https://conda.io/docs/user-guide/install/index.html. You can "
"also configure MLflow to look for a specific Conda executable "
"by setting the {1} environment variable to the path of the Conda "
"executable".format(conda_path, MLFLOW_CONDA_HOME))
(_, stdout, _) = process.exec_cmd([conda_path, "info", "--json"])
conda_env_version = json.loads(stdout)['conda_env_version']
conda_env_version_major = int(conda_env_version.split(".")[0])
conda_env_version_minor = int(conda_env_version.split(".")[1])
output_direct = ""
if direct_output_to_err:
output_direct = " 1>&2"
# in case os name is not 'nt', we are not running on windows. It introduces
# bash command otherwise.
if os.name != "nt" and (conda_env_version_major == 4
and conda_env_version_minor < 6):
return [
"source %s %s%s" % (activate_path, conda_env_name, output_direct)
]
else:
# TODO Need to fix, getting conda.sh is not simple
# As per https://github.com/conda/conda/issues/7126
# Notes:
# 1. $(dirname $CONDA_EXE)/../etc/profile.d/conda.sh will break in cases where conda and conda.sh is in expected directories, ie. /usr/bin/conda, /etc/profile.d/conda.sh
# 2. $(dirname $CONDA_EXE)/activate <env> will not work if activate and deactivate does not stick around.
return [
"source /etc/profile.d/conda.sh",
"%s activate %s%s" % (conda_path, conda_env_name, output_direct)
]
def load_azure_workspace():
"""
Load existing Azure Workspace from Tracking Store
:rtype: AzureML Workspace object
"""
from .store import AzureMLRestStore
from mlflow.exceptions import ExecutionException
from mlflow.tracking.client import MlflowClient
try:
def_store = MlflowClient()._tracking_client.store
except ExecutionException:
logger.warning(
VERSION_WARNING.format("MlflowClient()._tracking_client.store"))
def_store = MlflowClient().store
if isinstance(def_store, AzureMLRestStore):
workspace = Workspace._from_service_context(def_store.service_context,
_location=None)
return workspace
else:
raise ExecutionException(
"Workspace not found, please set the tracking URI in your script to AzureML."
)
def _fetch_git_repo(uri, version, dst_dir):
"""
Clone the git repo at ``uri`` into ``dst_dir``, checking out commit ``version`` (or defaulting
to the head commit of the repository's master branch if version is unspecified).
Assumes authentication parameters are specified by the environment, e.g. by a Git credential
helper.
"""
# We defer importing git until the last moment, because the import requires that the git
# executable is availble on the PATH, so we only want to fail if we actually need it.
import git
repo = git.Repo.init(dst_dir)
origin = repo.create_remote("origin", uri)
origin.fetch()
if version is not None:
try:
repo.git.checkout(version)
except git.exc.GitCommandError as e:
raise ExecutionException("Unable to checkout version '%s' of git repo %s"
"- please ensure that the version exists in the repo. "
"Error: %s" % (version, uri, e))
else:
repo.create_head("master", origin.refs.master)
repo.heads.master.checkout()
def load_project(directory):
mlproject_path = os.path.join(directory, MLPROJECT_FILE_NAME)
# TODO: Validate structure of YAML loaded from the file
if os.path.exists(mlproject_path):
with open(mlproject_path) as mlproject_file:
yaml_obj = yaml.safe_load(mlproject_file.read())
else:
yaml_obj = {}
entry_points = {}
for name, entry_point_yaml in yaml_obj.get("entry_points", {}).items():
parameters = entry_point_yaml.get("parameters", {})
command = entry_point_yaml.get("command")
entry_points[name] = EntryPoint(name, parameters, command)
conda_path = yaml_obj.get("conda_env")
if conda_path:
conda_env_path = os.path.join(directory, conda_path)
if not os.path.exists(conda_env_path):
raise ExecutionException("Project specified conda environment file %s, but no such "
"file was found." % conda_env_path)
return Project(conda_env_path=conda_env_path, entry_points=entry_points)
default_conda_path = os.path.join(directory, DEFAULT_CONDA_FILE_NAME)
if os.path.exists(default_conda_path):
return Project(conda_env_path=default_conda_path, entry_points=entry_points)
return Project(conda_env_path=None, entry_points=entry_points)
def _dbfs_path_exists(self, dbfs_path):
"""
Return True if the passed-in path exists in DBFS for the workspace corresponding to the
default Databricks CLI profile. The path is expected to be a relative path to the DBFS root
directory, e.g. 'path/to/file'.
"""
host_creds = databricks_utils.get_databricks_host_creds(self.databricks_profile)
response = rest_utils.http_request(
host_creds=host_creds, endpoint="/api/2.0/dbfs/get-status", method="GET",
json={"path": "/%s" % dbfs_path})
try:
json_response_obj = json.loads(response.text)
except ValueError:
raise MlflowException(
"API request to check existence of file at DBFS path %s failed with status code "
"%s. Response body: %s" % (dbfs_path, response.status_code, response.text))
# If request fails with a RESOURCE_DOES_NOT_EXIST error, the file does not exist on DBFS
error_code_field = "error_code"
if error_code_field in json_response_obj:
if json_response_obj[error_code_field] == "RESOURCE_DOES_NOT_EXIST":
return False
raise ExecutionException("Got unexpected error response when checking whether file %s "
"exists in DBFS: %s" % (dbfs_path, json_response_obj))
return True
