def get_underlying_uri(uri):
# Note: to support a registry URI that is different from the tracking URI here,
# we'll need to add setting of registry URIs via environment variables.
from mlflow.tracking import MlflowClient
databricks_profile_uri = (
get_databricks_profile_uri_from_artifact_uri(uri) or mlflow.get_registry_uri()
)
client = MlflowClient(registry_uri=databricks_profile_uri)
(name, version) = get_model_name_and_version(client, uri)
download_uri = client.get_model_version_download_uri(name, version)
return add_databricks_profile_info_to_artifact_uri(download_uri, databricks_profile_uri)
def test_register_model_raises_exception_with_unsupported_registry_store():
"""
This test case ensures that the `register_model` operation fails with an informative error
message when the registry store URI refers to a store that does not support Model Registry
features (e.g., FileStore).
"""
with TempDir() as tmp:
old_registry_uri = get_registry_uri()
try:
set_registry_uri(tmp.path())
with pytest.raises(MlflowException) as exc:
register_model(model_uri="runs:/1234/some_model", name="testmodel")
assert exc.value.error_code == ErrorCode.Name(FEATURE_DISABLED)
finally:
set_registry_uri(old_registry_uri)
def get_underlying_uri(uri):
# Note: to support a registry URI that is different from the tracking URI here,
# we'll need to add setting of registry URIs via environment variables.
from mlflow.tracking import MlflowClient
databricks_profile_uri = (
get_databricks_profile_uri_from_artifact_uri(uri)
or mlflow.get_registry_uri())
client = MlflowClient(registry_uri=databricks_profile_uri)
(name, version, stage) = ModelsArtifactRepository._parse_uri(uri)
if stage is not None:
latest = client.get_latest_versions(name, [stage])
if len(latest) == 0:
raise MlflowException(
"No versions of model with name '{name}' and "
"stage '{stage}' found".format(name=name, stage=stage))
version = latest[0].version
download_uri = client.get_model_version_download_uri(name, version)
return add_databricks_profile_info_to_artifact_uri(
download_uri, databricks_profile_uri)
def handle_model_uri(model_uri, service_name):
"""
Handle the various types of model uris we could receive.
:param model_uri:
:type model_uri: str
:param service_name:
:type service_name: str
:return:
:rtype:
"""
client = MlflowClient()
if model_uri.startswith("models:/"):
model_name = model_uri.split("/")[-2]
model_stage_or_version = model_uri.split("/")[-1]
if model_stage_or_version in client.get_model_version_stages(None, None):
# TODO: Add exception handling for no models found with specified stage
model_version = client.get_latest_versions(model_name, [model_stage_or_version])[0].version
else:
model_version = model_stage_or_version
elif (model_uri.startswith("runs:/") or model_uri.startswith("file://")) \
and get_tracking_uri().startswith("azureml") and get_registry_uri().startswith("azureml"):
# We will register the model for the user
model_name = service_name + "-model"
mlflow_model = mlflow_register_model(model_uri, model_name)
model_version = mlflow_model.version
_logger.info(
"Registered an Azure Model with name: `%s` and version: `%s`",
mlflow_model.name,
mlflow_model.version,
)
else:
raise MlflowException("Unsupported model uri provided, or tracking or registry uris are not set to "
"an AzureML uri.")
return model_name, model_version
#
# Code snippet for https://mlflow.org/docs/latest/python_api/mlflow.html#get_registry_uri
#
import warnings
import mlflow
if __name__ == "__main__":
warnings.filterwarnings("ignore")
print(mlflow.__version__)
# Get the current model registry uri
mr_uri = mlflow.get_registry_uri()
print("Current model registry uri: {}".format(mr_uri))
# Get the current tracking uri
tracking_uri = mlflow.get_tracking_uri()
print("Current tracking uri: {}".format(tracking_uri))
# They should be the same
assert mr_uri == tracking_uri
def deploy(
model_uri,
workspace,
deployment_config=None,
service_name=None,
model_name=None,
tags=None,
mlflow_home=None,
synchronous=True,
):
"""
Register an MLflow model with Azure ML and deploy a websevice to Azure Container Instances (ACI)
or Azure Kubernetes Service (AKS).
The deployed service will contain a webserver that processes model queries.
For information about the input data formats accepted by this webserver, see the
:ref:`MLflow deployment tools documentation <azureml_deployment>`.
:param model_uri: The location, in URI format, of the MLflow model used to build the Azure
ML deployment image. For example:
- ``/Users/me/path/to/local/model``
- ``relative/path/to/local/model``
- ``s3://my_bucket/path/to/model``
- ``runs:/<mlflow_run_id>/run-relative/path/to/model``
- ``models:/<model_name>/<model_version>``
- ``models:/<model_name>/<stage>``
For more information about supported URI schemes, see
`Referencing Artifacts <https://www.mlflow.org/docs/latest/concepts.html#
artifact-locations>`_.
:param workspace: The AzureML workspace in which to deploy the service. This is a
`azureml.core.Workspace` object.
:param deployment_config: The configuration for the Azure web service. This configuration
allows you to specify the resources the webservice will use and
the compute cluster it will be deployed in. If unspecified, the web
service will be deployed into a Azure Container Instance. This is a
`azureml.core.DeploymentConfig` object. For more information, see
`<https://docs.microsoft.com/python/api/azureml-core/
azureml.core.webservice.aks.aksservicedeploymentconfiguration>`_ and
`<https://docs.microsoft.com/en-us/python/api/azureml-core/azureml
.core.webservice.aci.aciservicedeploymentconfiguration>`_
:param service_name: The name to assign the Azure Machine learning webservice that will be
created. If unspecified, a unique name will be generated.
:param model_name: The name to assign the Azure Model will be created. If unspecified,
a unique model name will be generated. Only used if the model is not
already registered with Azure.
:param tags: A collection of tags, represented as a dictionary of string key-value pairs, to
associate with the Azure Model and Deployment that will be created.
These tags are added to a set of default tags that include the model uri,
and more. For more information, see
`<https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.model(class)?view=azure-ml-py>`_.
:param mlflow_home: Path to a local copy of the MLflow GitHub repository. If specified, the
image will install MLflow from this directory. Otherwise, it will install
MLflow from pip.
:param synchronous: If ``True``, this method blocks until the image creation procedure
terminates before returning. If ``False``, the method returns immediately,
but the returned image will not be available until the asynchronous
creation process completes. Use the
``azureml.core.Webservice.wait_for_deployment()`` function to wait
for the deployment process to complete.
:return: A tuple containing the following elements in order:
- An ``azureml.core.webservice.Webservice`` object containing metadata for the
new service.
- An ``azureml.core.model.Model`` object containing metadata for the new model.
.. code-block:: python
:caption: Example
import mlflow.azureml
from azureml.core import Workspace
from azureml.core.webservice import AciWebservice, Webservice
# Load or create an Azure ML Workspace
workspace_name = "<Name of your Azure ML workspace>"
subscription_id = "<Your Azure subscription ID>"
resource_group = "<Name of the Azure resource group in which to create Azure ML resources>"
location = "<Name of the Azure location (region) in which to create Azure ML resources>"
azure_workspace = Workspace.create(name=workspace_name,
subscription_id=subscription_id,
resource_group=resource_group,
location=location,
create_resource_group=True,
exist_ok=True)
# Create an Azure Container Instance webservice for an MLflow model
azure_service, azure_model = mlflow.azureml.deploy(model_uri="<model_uri>",
service_name="<deployment-name>",
workspace=azure_workspace,
synchronous=True)
"""
# The Azure ML SDK is only compatible with Python 3. However, the `mlflow.azureml` module should
# still be accessible for import from Python 2. Therefore, we will only import from the SDK
# upon method invocation.
# pylint: disable=import-error
from azureml.core.model import Model as AzureModel, InferenceConfig
from azureml.core import Environment as AzureEnvironment
from azureml.core import VERSION as AZUREML_VERSION
from azureml.core.webservice import AciWebservice
absolute_model_path = _download_artifact_from_uri(model_uri)
model_pyfunc_conf, model = _load_pyfunc_conf_with_model(
model_path=absolute_model_path)
model_python_version = model_pyfunc_conf.get(pyfunc.PY_VERSION, None)
run_id = None
run_id_tag = None
try:
run_id = model.run_id
run_id_tag = run_id
except AttributeError:
run_id = str(uuid.uuid4())
if model_python_version is not None and StrictVersion(
model_python_version) < StrictVersion("3.0.0"):
raise MlflowException(
message=
("Azure ML can only deploy models trained in Python 3 and above. See"
" the following MLflow GitHub issue for a thorough explanation of this"
" limitation and a workaround to enable support for deploying models"
" trained in Python 2: https://github.com/mlflow/mlflow/issues/668"
),
error_code=INVALID_PARAMETER_VALUE,
)
tags = _build_tags(
model_uri=model_uri,
model_python_version=model_python_version,
user_tags=tags,
run_id=run_id_tag,
)
if service_name is None:
service_name = _get_mlflow_azure_name(run_id)
if model_name is None:
model_name = _get_mlflow_azure_name(run_id)
with TempDir(chdr=True) as tmp:
model_directory_path = tmp.path("model")
tmp_model_path = os.path.join(
model_directory_path,
_copy_file_or_tree(src=absolute_model_path,
dst=model_directory_path),
)
registered_model = None
azure_model_id = None
# If we are passed a 'models' uri, we will attempt to extract a name and version which
# can be used to retreive an AzureML Model. This will ignore stage based model uris,
# which is alright until we have full deployment plugin support.
#
# If instead we are passed a 'runs' uri while the user is using the AzureML tracking
# and registry stores, we will be able to register the model on their behalf using
# the AzureML plugin, which will maintain lineage between the model and the run that
# produced it. This returns an MLFlow Model object however, so we'll still need the
# name and ID in order to retrieve the AzureML Model object which is currently
# needed to deploy.
if model_uri.startswith("models:/"):
m_name = model_uri.split("/")[-2]
m_version = int(model_uri.split("/")[-1])
azure_model_id = "{}:{}".format(m_name, m_version)
elif (model_uri.startswith("runs:/")
and get_tracking_uri().startswith("azureml")
and get_registry_uri().startswith("azureml")):
mlflow_model = mlflow_register_model(model_uri, model_name)
azure_model_id = "{}:{}".format(mlflow_model.name,
mlflow_model.version)
_logger.info(
"Registered an Azure Model with name: `%s` and version: `%s`",
mlflow_model.name,
azure_model_id,
)
# Attempt to retrieve an AzureML Model object which we intend to deploy
if azure_model_id:
try:
registered_model = AzureModel(workspace, id=azure_model_id)
_logger.info("Found registered model in AzureML with ID '%s'",
azure_model_id)
except Exception as e: # pylint: disable=broad-except
_logger.info(
"Unable to find model in AzureML with ID '%s', will register the model.\n"
"Exception was: %s",
azure_model_id,
e,
)
# If we have not found a registered model by this point, we will register it on the users'
# behalf. It is required for a Model to be registered in some way with Azure in order to
# deploy to Azure, so this is expected for Azure users.
if not registered_model:
registered_model = AzureModel.register(workspace=workspace,
model_path=tmp_model_path,
model_name=model_name,
tags=tags)
_logger.info(
"Registered an Azure Model with name: `%s` and version: `%s`",
registered_model.name,
registered_model.version,
)
# Create an execution script (entry point) for the image's model server. Azure ML requires
# the container's execution script to be located in the current working directory during
# image creation, so we create the execution script as a temporary file in the current
# working directory.
execution_script_path = tmp.path("execution_script.py")
_create_execution_script(output_path=execution_script_path,
azure_model=registered_model)
environment = None
if pyfunc.ENV in model_pyfunc_conf:
environment = AzureEnvironment.from_conda_specification(
_get_mlflow_azure_name(run_id),
os.path.join(tmp_model_path, model_pyfunc_conf[pyfunc.ENV]),
)
else:
environment = AzureEnvironment(_get_mlflow_azure_name(run_id))
if mlflow_home is not None:
path = tmp.path("dist")
_logger.info("Bulding temporary MLFlow wheel in %s", path)
wheel = _create_mlflow_wheel(mlflow_home, path)
whl_url = AzureEnvironment.add_private_pip_wheel(
workspace=workspace, file_path=wheel, exist_ok=True)
environment.python.conda_dependencies.add_pip_package(whl_url)
else:
environment.python.conda_dependencies.add_pip_package(
"mlflow=={}".format(mlflow_version))
# AzureML requires azureml-defaults to be installed to include
# flask for the inference server.
environment.python.conda_dependencies.add_pip_package(
"azureml-defaults=={}".format(AZUREML_VERSION))
inference_config = InferenceConfig(entry_script=execution_script_path,
environment=environment)
if deployment_config is not None:
if deployment_config.tags is not None:
# We want more narrowly-scoped tags to win on merge
tags.update(deployment_config.tags)
deployment_config.tags = tags
else:
deployment_config = AciWebservice.deploy_configuration(tags=tags)
# Finally, deploy the AzureML Model object to a webservice, and return back
webservice = AzureModel.deploy(
workspace=workspace,
name=service_name,
models=[registered_model],
inference_config=inference_config,
deployment_config=deployment_config,
)
_logger.info("Deploying an Azure Webservice with name: `%s`",
webservice.name)
if synchronous:
webservice.wait_for_deployment(show_output=True)
return webservice, registered_model
model_name = "demo-cmr"
input_data_path = "dbfs:/databricks-datasets/wine-quality/winequality-red.csv"
now = datetime.now()
parent_run_name = now.strftime("%Y%m%d-%H%M")
# use central model registry
scope = "demo-cmr"
key = "cmr"
registry_uri = "databricks://" + scope + ":" + key
mlflow.set_registry_uri(registry_uri)
# COMMAND ----------
import mlflow
mlflow.get_registry_uri()
# COMMAND ----------
def evaluate_hyperparams_wrapper(X_train, X_test, y_train, y_test):
def evaluate_hyperparams(params):
min_samples_leaf = int(params['min_samples_leaf'])
max_depth = params['max_depth']
n_estimators = int(params['n_estimators'])
rf = RandomForestRegressor(
max_depth=max_depth,
min_samples_leaf=min_samples_leaf,
n_estimators=n_estimators,
)
