def to_literal( self, ctx: FlyteContext, python_val: PipelineModel, python_type: Type[PipelineModel], expected: LiteralType, ) -> Literal: local_path = ctx.file_access.get_random_local_path() pathlib.Path(local_path).parent.mkdir(parents=True, exist_ok=True) python_val.save(local_path) remote_dir = ctx.file_access.get_random_remote_directory() ctx.file_access.upload_directory(local_path, remote_dir) return Literal(scalar=Scalar(blob=Blob( uri=remote_dir, metadata=BlobMetadata(type=self._TYPE_INFO))))
def log_model( spark_model, artifact_path, conda_env=None, dfs_tmpdir=None, sample_input=None, registered_model_name=None, signature: ModelSignature = None, input_example: ModelInputExample = None, await_registration_for=DEFAULT_AWAIT_MAX_SLEEP_SECONDS, pip_requirements=None, extra_pip_requirements=None, ): """ Log a Spark MLlib model as an MLflow artifact for the current run. This uses the MLlib persistence format and produces an MLflow Model with the Spark flavor. Note: If no run is active, it will instantiate a run to obtain a run_id. :param spark_model: Spark model to be saved - MLflow can only save descendants of pyspark.ml.Model which implement MLReadable and MLWritable. :param artifact_path: Run relative artifact path. :param conda_env: Either a dictionary representation of a Conda environment or the path to a Conda environment yaml file. If provided, this decsribes the environment this model should be run in. At minimum, it should specify the dependencies contained in :func:`get_default_conda_env()`. If `None`, the default :func:`get_default_conda_env()` environment is added to the model. The following is an *example* dictionary representation of a Conda environment:: { 'name': 'mlflow-env', 'channels': ['defaults'], 'dependencies': [ 'python=3.7.0', 'pyspark=2.3.0' ] } :param dfs_tmpdir: Temporary directory path on Distributed (Hadoop) File System (DFS) or local filesystem if running in local mode. The model is written in this destination and then copied into the model's artifact directory. This is necessary as Spark ML models read from and write to DFS if running on a cluster. If this operation completes successfully, all temporary files created on the DFS are removed. Defaults to ``/tmp/mlflow``. :param sample_input: A sample input used to add the MLeap flavor to the model. This must be a PySpark DataFrame that the model can evaluate. If ``sample_input`` is ``None``, the MLeap flavor is not added. :param registered_model_name: If given, create a model version under ``registered_model_name``, also creating a registered model if one with the given name does not exist. :param signature: :py:class:`ModelSignature <mlflow.models.ModelSignature>` describes model input and output :py:class:`Schema <mlflow.types.Schema>`. The model signature can be :py:func:`inferred <mlflow.models.infer_signature>` from datasets with valid model input (e.g. the training dataset with target column omitted) and valid model output (e.g. model predictions generated on the training dataset), for example: .. code-block:: python from mlflow.models.signature import infer_signature train = df.drop_column("target_label") predictions = ... # compute model predictions signature = infer_signature(train, predictions) :param input_example: Input example provides one or several instances of valid model input. The example can be used as a hint of what data to feed the model. The given example will be converted to a Pandas DataFrame and then serialized to json using the Pandas split-oriented format. Bytes are base64-encoded. :param await_registration_for: Number of seconds to wait for the model version to finish being created and is in ``READY`` status. By default, the function waits for five minutes. Specify 0 or None to skip waiting. :param pip_requirements: {{ pip_requirements }} :param extra_pip_requirements: {{ extra_pip_requirements }} :return: A :py:class:`ModelInfo <mlflow.models.model.ModelInfo>` instance that contains the metadata of the logged model. .. code-block:: python :caption: Example from pyspark.ml import Pipeline from pyspark.ml.classification import LogisticRegression from pyspark.ml.feature import HashingTF, Tokenizer training = spark.createDataFrame([ (0, "a b c d e spark", 1.0), (1, "b d", 0.0), (2, "spark f g h", 1.0), (3, "hadoop mapreduce", 0.0) ], ["id", "text", "label"]) tokenizer = Tokenizer(inputCol="text", outputCol="words") hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features") lr = LogisticRegression(maxIter=10, regParam=0.001) pipeline = Pipeline(stages=[tokenizer, hashingTF, lr]) model = pipeline.fit(training) mlflow.spark.log_model(model, "spark-model") """ from py4j.protocol import Py4JError _validate_model(spark_model) from pyspark.ml import PipelineModel if not isinstance(spark_model, PipelineModel): spark_model = PipelineModel([spark_model]) run_id = mlflow.tracking.fluent._get_or_start_run().info.run_id run_root_artifact_uri = mlflow.get_artifact_uri() # If the artifact URI is a local filesystem path, defer to Model.log() to persist the model, # since Spark may not be able to write directly to the driver's filesystem. For example, # writing to `file:/uri` will write to the local filesystem from each executor, which will # be incorrect on multi-node clusters - to avoid such issues we just use the Model.log() path # here. if is_local_uri(run_root_artifact_uri): return Model.log( artifact_path=artifact_path, flavor=mlflow.spark, spark_model=spark_model, conda_env=conda_env, dfs_tmpdir=dfs_tmpdir, sample_input=sample_input, registered_model_name=registered_model_name, signature=signature, input_example=input_example, await_registration_for=await_registration_for, pip_requirements=pip_requirements, extra_pip_requirements=extra_pip_requirements, ) model_dir = os.path.join(run_root_artifact_uri, artifact_path) # Try to write directly to the artifact repo via Spark. If this fails, defer to Model.log() # to persist the model try: spark_model.save(posixpath.join(model_dir, _SPARK_MODEL_PATH_SUB)) except Py4JError: return Model.log( artifact_path=artifact_path, flavor=mlflow.spark, spark_model=spark_model, conda_env=conda_env, dfs_tmpdir=dfs_tmpdir, sample_input=sample_input, registered_model_name=registered_model_name, signature=signature, input_example=input_example, await_registration_for=await_registration_for, pip_requirements=pip_requirements, extra_pip_requirements=extra_pip_requirements, ) # Otherwise, override the default model log behavior and save model directly to artifact repo mlflow_model = Model(artifact_path=artifact_path, run_id=run_id) with TempDir() as tmp: tmp_model_metadata_dir = tmp.path() _save_model_metadata( tmp_model_metadata_dir, spark_model, mlflow_model, sample_input, conda_env, signature=signature, input_example=input_example, ) mlflow.tracking.fluent.log_artifacts(tmp_model_metadata_dir, artifact_path) if registered_model_name is not None: mlflow.register_model( "runs:/%s/%s" % (run_id, artifact_path), registered_model_name, await_registration_for, ) return mlflow_model.get_model_info()
def save_model( spark_model, path, mlflow_model=None, conda_env=None, dfs_tmpdir=None, sample_input=None, signature: ModelSignature = None, input_example: ModelInputExample = None, pip_requirements=None, extra_pip_requirements=None, ): """ Save a Spark MLlib Model to a local path. By default, this function saves models using the Spark MLlib persistence mechanism. Additionally, if a sample input is specified using the ``sample_input`` parameter, the model is also serialized in MLeap format and the MLeap flavor is added. :param spark_model: Spark model to be saved - MLflow can only save descendants of pyspark.ml.Model which implement MLReadable and MLWritable. :param path: Local path where the model is to be saved. :param mlflow_model: MLflow model config this flavor is being added to. :param conda_env: Either a dictionary representation of a Conda environment or the path to a Conda environment yaml file. If provided, this decsribes the environment this model should be run in. At minimum, it should specify the dependencies contained in :func:`get_default_conda_env()`. If `None`, the default :func:`get_default_conda_env()` environment is added to the model. The following is an *example* dictionary representation of a Conda environment:: { 'name': 'mlflow-env', 'channels': ['defaults'], 'dependencies': [ 'python=3.7.0', 'pyspark=2.3.0' ] } :param dfs_tmpdir: Temporary directory path on Distributed (Hadoop) File System (DFS) or local filesystem if running in local mode. The model is be written in this destination and then copied to the requested local path. This is necessary as Spark ML models read from and write to DFS if running on a cluster. All temporary files created on the DFS are removed if this operation completes successfully. Defaults to ``/tmp/mlflow``. :param sample_input: A sample input that is used to add the MLeap flavor to the model. This must be a PySpark DataFrame that the model can evaluate. If ``sample_input`` is ``None``, the MLeap flavor is not added. :param signature: :py:class:`ModelSignature <mlflow.models.ModelSignature>` describes model input and output :py:class:`Schema <mlflow.types.Schema>`. The model signature can be :py:func:`inferred <mlflow.models.infer_signature>` from datasets with valid model input (e.g. the training dataset with target column omitted) and valid model output (e.g. model predictions generated on the training dataset), for example: .. code-block:: python from mlflow.models.signature import infer_signature train = df.drop_column("target_label") predictions = ... # compute model predictions signature = infer_signature(train, predictions) :param input_example: Input example provides one or several instances of valid model input. The example can be used as a hint of what data to feed the model. The given example will be converted to a Pandas DataFrame and then serialized to json using the Pandas split-oriented format. Bytes are base64-encoded. :param pip_requirements: {{ pip_requirements }} :param extra_pip_requirements: {{ extra_pip_requirements }} .. code-block:: python :caption: Example from mlflow import spark from pyspark.ml.pipeline.PipelineModel # your pyspark.ml.pipeline.PipelineModel type model = ... mlflow.spark.save_model(model, "spark-model") """ _validate_model(spark_model) _validate_env_arguments(conda_env, pip_requirements, extra_pip_requirements) from pyspark.ml import PipelineModel if not isinstance(spark_model, PipelineModel): spark_model = PipelineModel([spark_model]) if mlflow_model is None: mlflow_model = Model() # Spark ML stores the model on DFS if running on a cluster # Save it to a DFS temp dir first and copy it to local path if dfs_tmpdir is None: dfs_tmpdir = DFS_TMP tmp_path = _tmp_path(dfs_tmpdir) spark_model.save(tmp_path) sparkml_data_path = os.path.abspath( os.path.join(path, _SPARK_MODEL_PATH_SUB)) # We're copying the Spark model from DBFS to the local filesystem if (a) the temporary DFS URI # we saved the Spark model to is a DBFS URI ("dbfs:/my-directory"), or (b) if we're running # on a Databricks cluster and the URI is schemeless (e.g. looks like a filesystem absolute path # like "/my-directory") copying_from_dbfs = is_valid_dbfs_uri(tmp_path) or ( databricks_utils.is_in_cluster() and posixpath.abspath(tmp_path) == tmp_path) if copying_from_dbfs and databricks_utils.is_dbfs_fuse_available(): tmp_path_fuse = dbfs_hdfs_uri_to_fuse_path(tmp_path) shutil.move(src=tmp_path_fuse, dst=sparkml_data_path) else: _HadoopFileSystem.copy_to_local_file(tmp_path, sparkml_data_path, remove_src=True) _save_model_metadata( dst_dir=path, spark_model=spark_model, mlflow_model=mlflow_model, sample_input=sample_input, conda_env=conda_env, signature=signature, input_example=input_example, pip_requirements=pip_requirements, extra_pip_requirements=extra_pip_requirements, )
def save_model(spark_model, path, mlflow_model=Model(), conda_env=None, dfs_tmpdir=None, sample_input=None): """ Save a Spark MLlib Model to a local path. By default, this function saves models using the Spark MLlib persistence mechanism. Additionally, if a sample input is specified using the ``sample_input`` parameter, the model is also serialized in MLeap format and the MLeap flavor is added. :param spark_model: Spark model to be saved - MLflow can only save descendants of pyspark.ml.Model which implement MLReadable and MLWritable. :param path: Local path where the model is to be saved. :param mlflow_model: MLflow model config this flavor is being added to. :param conda_env: Either a dictionary representation of a Conda environment or the path to a Conda environment yaml file. If provided, this decsribes the environment this model should be run in. At minimum, it should specify the dependencies contained in :func:`get_default_conda_env()`. If `None`, the default :func:`get_default_conda_env()` environment is added to the model. The following is an *example* dictionary representation of a Conda environment:: { 'name': 'mlflow-env', 'channels': ['defaults'], 'dependencies': [ 'python=3.7.0', 'pyspark=2.3.0' ] } :param dfs_tmpdir: Temporary directory path on Distributed (Hadoop) File System (DFS) or local filesystem if running in local mode. The model is be written in this destination and then copied to the requested local path. This is necessary as Spark ML models read from and write to DFS if running on a cluster. All temporary files created on the DFS are removed if this operation completes successfully. Defaults to ``/tmp/mlflow``. :param sample_input: A sample input that is used to add the MLeap flavor to the model. This must be a PySpark DataFrame that the model can evaluate. If ``sample_input`` is ``None``, the MLeap flavor is not added. .. code-block:: python :caption: Example from mlflow import spark from pyspark.ml.pipeline.PipelineModel # your pyspark.ml.pipeline.PipelineModel type model = ... mlflow.spark.save_model(model, "spark-model") """ _validate_model(spark_model) from pyspark.ml import PipelineModel if not isinstance(spark_model, PipelineModel): spark_model = PipelineModel([spark_model]) # Spark ML stores the model on DFS if running on a cluster # Save it to a DFS temp dir first and copy it to local path if dfs_tmpdir is None: dfs_tmpdir = DFS_TMP tmp_path = _tmp_path(dfs_tmpdir) spark_model.save(tmp_path) sparkml_data_path = os.path.abspath( os.path.join(path, _SPARK_MODEL_PATH_SUB)) _HadoopFileSystem.copy_to_local_file(tmp_path, sparkml_data_path, remove_src=True) _save_model_metadata(dst_dir=path, spark_model=spark_model, mlflow_model=mlflow_model, sample_input=sample_input, conda_env=conda_env)
def log_model(spark_model, artifact_path, conda_env=None, dfs_tmpdir=None, sample_input=None, registered_model_name=None): """ Log a Spark MLlib model as an MLflow artifact for the current run. This uses the MLlib persistence format and produces an MLflow Model with the Spark flavor. :param spark_model: Spark model to be saved - MLFlow can only save descendants of pyspark.ml.Model which implement MLReadable and MLWritable. :param artifact_path: Run relative artifact path. :param conda_env: Either a dictionary representation of a Conda environment or the path to a Conda environment yaml file. If provided, this decsribes the environment this model should be run in. At minimum, it should specify the dependencies contained in :func:`get_default_conda_env()`. If `None`, the default :func:`get_default_conda_env()` environment is added to the model. The following is an *example* dictionary representation of a Conda environment:: { 'name': 'mlflow-env', 'channels': ['defaults'], 'dependencies': [ 'python=3.7.0', 'pyspark=2.3.0' ] } :param dfs_tmpdir: Temporary directory path on Distributed (Hadoop) File System (DFS) or local filesystem if running in local mode. The model is written in this destination and then copied into the model's artifact directory. This is necessary as Spark ML models read from and write to DFS if running on a cluster. If this operation completes successfully, all temporary files created on the DFS are removed. Defaults to ``/tmp/mlflow``. :param sample_input: A sample input used to add the MLeap flavor to the model. This must be a PySpark DataFrame that the model can evaluate. If ``sample_input`` is ``None``, the MLeap flavor is not added. :param registered_model_name: Note:: Experimental: This argument may change or be removed in a future release without warning. If given, create a model version under ``registered_model_name``, also creating a registered model if one with the given name does not exist. >>> from pyspark.ml import Pipeline >>> from pyspark.ml.classification import LogisticRegression >>> from pyspark.ml.feature import HashingTF, Tokenizer >>> training = spark.createDataFrame([ ... (0, "a b c d e spark", 1.0), ... (1, "b d", 0.0), ... (2, "spark f g h", 1.0), ... (3, "hadoop mapreduce", 0.0) ], ["id", "text", "label"]) >>> tokenizer = Tokenizer(inputCol="text", outputCol="words") >>> hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features") >>> lr = LogisticRegression(maxIter=10, regParam=0.001) >>> pipeline = Pipeline(stages=[tokenizer, hashingTF, lr]) >>> model = pipeline.fit(training) >>> mlflow.spark.log_model(model, "spark-model") """ from py4j.protocol import Py4JJavaError _validate_model(spark_model) from pyspark.ml import PipelineModel if not isinstance(spark_model, PipelineModel): spark_model = PipelineModel([spark_model]) run_id = mlflow.tracking.fluent._get_or_start_run().info.run_id run_root_artifact_uri = mlflow.get_artifact_uri() # If the artifact URI is a local filesystem path, defer to Model.log() to persist the model, # since Spark may not be able to write directly to the driver's filesystem. For example, # writing to `file:/uri` will write to the local filesystem from each executor, which will # be incorrect on multi-node clusters - to avoid such issues we just use the Model.log() path # here. if is_local_uri(run_root_artifact_uri): return Model.log(artifact_path=artifact_path, flavor=mlflow.spark, spark_model=spark_model, conda_env=conda_env, dfs_tmpdir=dfs_tmpdir, sample_input=sample_input, registered_model_name=registered_model_name) # If Spark cannot write directly to the artifact repo, defer to Model.log() to persist the # model model_dir = os.path.join(run_root_artifact_uri, artifact_path) try: spark_model.save(os.path.join(model_dir, _SPARK_MODEL_PATH_SUB)) except Py4JJavaError: return Model.log(artifact_path=artifact_path, flavor=mlflow.spark, spark_model=spark_model, conda_env=conda_env, dfs_tmpdir=dfs_tmpdir, sample_input=sample_input, registered_model_name=registered_model_name) # Otherwise, override the default model log behavior and save model directly to artifact repo mlflow_model = Model(artifact_path=artifact_path, run_id=run_id) with TempDir() as tmp: tmp_model_metadata_dir = tmp.path() _save_model_metadata(tmp_model_metadata_dir, spark_model, mlflow_model, sample_input, conda_env) mlflow.tracking.fluent.log_artifacts(tmp_model_metadata_dir, artifact_path) if registered_model_name is not None: mlflow.register_model("runs:/%s/%s" % (run_id, artifact_path), registered_model_name)
def save_pipeline_model(pipeline_model: PipelineModel, foldername: str): pipeline_model.save(foldername)