def test_log_model_persists_specified_conda_env_in_mlflow_model_directory( gluon_model, gluon_custom_env ): artifact_path = "model" with mlflow.start_run(): mlflow.gluon.log_model( gluon_model=gluon_model, artifact_path=artifact_path, conda_env=gluon_custom_env ) model_path = _download_artifact_from_uri( "runs:/{run_id}/{artifact_path}".format( run_id=mlflow.active_run().info.run_id, artifact_path=artifact_path ) ) pyfunc_conf = _get_flavor_configuration(model_path=model_path, flavor_name=pyfunc.FLAVOR_NAME) saved_conda_env_path = os.path.join(model_path, pyfunc_conf[pyfunc.ENV]) assert os.path.exists(saved_conda_env_path) assert saved_conda_env_path != gluon_custom_env with open(gluon_custom_env, "r") as f: gluon_custom_env_parsed = yaml.safe_load(f) with open(saved_conda_env_path, "r") as f: saved_conda_env_parsed = yaml.safe_load(f) assert saved_conda_env_parsed == gluon_custom_env_parsed
def test_node_hook_logging_below_limit_all_strategy(tmp_path, config_dir, dummy_run_params, dummy_node, param_length, strategy): # mocker.patch("kedro_mlflow.utils._is_kedro_project", return_value=True) _write_yaml( tmp_path / "conf" / "base" / "mlflow.yml", dict(hooks=dict(node=dict(long_parameters_strategy=strategy)), ), ) mlflow_tracking_uri = (tmp_path / "mlruns").as_uri() mlflow.set_tracking_uri(mlflow_tracking_uri) mlflow_node_hook = MlflowNodeHook() param_value = param_length * "a" node_inputs = {"params:my_param": param_value} with mlflow.start_run(): mlflow_node_hook.before_pipeline_run(run_params=dummy_run_params, pipeline=Pipeline([]), catalog=DataCatalog()) mlflow_node_hook.before_node_run( node=node(func=lambda x: x, inputs=dict(x="a"), outputs=None), catalog=DataCatalog(), # can be empty inputs=node_inputs, is_async=False, run_id="132", ) run_id = mlflow.active_run().info.run_id mlflow_client = MlflowClient(mlflow_tracking_uri) current_run = mlflow_client.get_run(run_id) assert current_run.data.params == {"my_param": param_value}
def _original_fn(*_og_args, **_og_kwargs): if is_testing(): _validate_args( autologging_integration, function_name, args, kwargs, og_args, og_kwargs, ) # By the time `original` is called by the patch implementation, we # assume that either: 1. the patch implementation has already # created an MLflow run or 2. the patch code will not create an # MLflow run during the current execution. Here, we capture a # reference to the active run, which we will use later on to # determine whether or not the patch implementation created # a run and perform validation if necessary nonlocal patch_function_run_for_testing patch_function_run_for_testing = mlflow.active_run( ) nonlocal original_has_been_called original_has_been_called = True nonlocal original_result # Show all non-MLflow warnings as normal (i.e. not as event logs) # during original function execution, even if silent mode is enabled # (`silent=True`), since these warnings originate from the ML framework # or one of its dependencies and are likely relevant to the caller with set_non_mlflow_warnings_behavior_for_current_thread( disable_warnings=False, reroute_warnings=False, ): original_result = original( *_og_args, **_og_kwargs) return original_result
def test_log_model_persists_specified_conda_env_in_mlflow_model_directory( saved_tf_iris_model, tf_custom_env): artifact_path = "model" with mlflow.start_run(): mlflow.tensorflow.log_model(tf_saved_model_dir=saved_tf_iris_model.path, tf_meta_graph_tags=saved_tf_iris_model.meta_graph_tags, tf_signature_def_key=saved_tf_iris_model.signature_def_key, artifact_path=artifact_path, conda_env=tf_custom_env) model_uri = "runs:/{run_id}/{artifact_path}".format( run_id=mlflow.active_run().info.run_id, artifact_path=artifact_path) model_path = _download_artifact_from_uri(artifact_uri=model_uri) pyfunc_conf = _get_flavor_configuration(model_path=model_path, flavor_name=pyfunc.FLAVOR_NAME) saved_conda_env_path = os.path.join(model_path, pyfunc_conf[pyfunc.ENV]) assert os.path.exists(saved_conda_env_path) assert saved_conda_env_path != tf_custom_env with open(tf_custom_env, "r") as f: tf_custom_env_text = f.read() with open(saved_conda_env_path, "r") as f: saved_conda_env_text = f.read() assert saved_conda_env_text == tf_custom_env_text
def test_spark_sklearn_autologging_context_provider(spark_session, data_format, file_path): mlflow.spark.autolog() mlflow.sklearn.autolog() df = (spark_session.read.format(data_format).option( "header", "true").option("inferSchema", "true").load(file_path).select("number1", "number2")) pandas_df = df.toPandas() # DF info should be logged to the first run (it should be added to our context provider after # the toPandas() call above & then logged here) with mlflow.start_run(): run = _fit_sklearn_model(pandas_df) _assert_spark_data_logged(run, file_path, data_format) with mlflow.start_run(): pandas_df2 = df.filter("number1 > 0").toPandas() run2 = _fit_sklearn_model(pandas_df2) assert run2.info.run_id != run.info.run_id _assert_spark_data_logged(run2, file_path, data_format) time.sleep(1) assert mlflow.active_run() is None
def _log_posttraining_metadata(estimator, *args, **kwargs): """ Records metadata for a scikit-learn estimator after training has completed. This is intended to be invoked within a patched scikit-learn training routine (e.g., `fit()`, `fit_transform()`, ...) and assumes the existence of an active MLflow run that can be referenced via the fluent Tracking API. :param estimator: The scikit-learn estimator for which to log metadata. :param args: The arguments passed to the scikit-learn training routine (e.g., `fit()`, `fit_transform()`, ...). :param kwargs: The keyword arguments passed to the scikit-learn training routine. """ if hasattr(estimator, "score"): try: score_args = _get_args_for_score(estimator.score, estimator.fit, args, kwargs) training_score = estimator.score(*score_args) except Exception as e: msg = ( estimator.score.__qualname__ + " failed. The 'training_score' metric will not be recorded. Scoring error: " + str(e)) _logger.warning(msg) else: try_mlflow_log(mlflow.log_metric, "training_score", training_score) # log common metrics and artifacts for estimators (classifier, regressor) _log_specialized_estimator_content(estimator, mlflow.active_run().info.run_id, args, kwargs) def get_input_example(): # Fetch an input example using the first several rows of the array-like # training data supplied to the training routine (e.g., `fit()`) fit_arg_names = _get_arg_names(estimator.fit) X_var_name, y_var_name = fit_arg_names[:2] input_example = _get_Xy(args, kwargs, X_var_name, y_var_name)[0][:INPUT_EXAMPLE_SAMPLE_ROWS] return input_example def infer_model_signature(input_example): if not hasattr(estimator, "predict"): raise Exception( "the trained model does not specify a `predict` function, " + "which is required in order to infer the signature") return infer_signature(input_example, estimator.predict(input_example)) if log_models: # Will only resolve `input_example` and `signature` if `log_models` is `True`. input_example, signature = resolve_input_example_and_signature( get_input_example, infer_model_signature, log_input_examples, log_model_signatures, _logger, ) try_mlflow_log( log_model, estimator, artifact_path="model", signature=signature, input_example=input_example, ) if _is_parameter_search_estimator(estimator): if hasattr(estimator, "best_estimator_") and log_models: try_mlflow_log( log_model, estimator.best_estimator_, artifact_path="best_estimator", signature=signature, input_example=input_example, ) if hasattr(estimator, "best_score_"): try_mlflow_log(mlflow.log_metric, "best_cv_score", estimator.best_score_) if hasattr(estimator, "best_params_"): best_params = { "best_{param_name}".format(param_name=param_name): param_value for param_name, param_value in estimator.best_params_.items() } try_mlflow_log(mlflow.log_params, best_params) if hasattr(estimator, "cv_results_"): try: # Fetch environment-specific tags (e.g., user and source) to ensure that lineage # information is consistent with the parent run child_tags = context_registry.resolve_tags() child_tags.update({MLFLOW_AUTOLOGGING: FLAVOR_NAME}) _create_child_runs_for_parameter_search( cv_estimator=estimator, parent_run=mlflow.active_run(), child_tags=child_tags, ) except Exception as e: msg = ( "Encountered exception during creation of child runs for parameter search." " Child runs may be missing. Exception: {}".format( str(e))) _logger.warning(msg) try: cv_results_df = pd.DataFrame.from_dict( estimator.cv_results_) _log_parameter_search_results_as_artifact( cv_results_df, mlflow.active_run().info.run_id) except Exception as e: msg = ( "Failed to log parameter search results as an artifact." " Exception: {}".format(str(e))) _logger.warning(msg)
def test_log_fn_args_as_params_ignores_unwanted_parameters(start_run): # pylint: disable=W0613 args, kwargs, unlogged = ("arg1", {"arg2": "value"}, ["arg1", "arg2", "arg3"]) log_fn_args_as_params(dummy_fn, args, kwargs, unlogged) client = mlflow.tracking.MlflowClient() params = client.get_run(mlflow.active_run().info.run_id).data.params assert len(params.keys()) == 0
def test_node_hook_logging( tmp_path, mocker, monkeypatch, dummy_run_params, dummy_catalog, dummy_pipeline, dummy_node, config_dir, flatten_dict_params, expected, ): mocker.patch("logging.config.dictConfig") mocker.patch("kedro_mlflow.utils._is_kedro_project", return_value=True) monkeypatch.chdir(tmp_path) # config = KedroMlflowConfig( # project_path=tmp_path, # node_hook_opts={"flatten_dict_params": flatten_dict_params, "sep": "-"}, # ) # # the function is imported inside the other file antd this is the file to patch # # see https://stackoverflow.com/questions/30987973/python-mock-patch-doesnt-work-as-expected-for-public-method # mocker.patch( # "kedro_mlflow.framework.hooks.node_hook.get_mlflow_config", return_value=config # ) _write_yaml( tmp_path / "conf" / "base" / "mlflow.yml", dict( hooks=dict( node=dict( flatten_dict_params=flatten_dict_params, recursive=False, sep="-" ) ), ), ), mlflow_node_hook = MlflowNodeHook() node_inputs = { v: dummy_catalog._data_sets.get(v) for k, v in dummy_node._inputs.items() } mlflow_tracking_uri = (tmp_path / "mlruns").as_uri() mlflow.set_tracking_uri(mlflow_tracking_uri) with mlflow.start_run(): mlflow_node_hook.before_pipeline_run( run_params=dummy_run_params, pipeline=dummy_pipeline, catalog=dummy_catalog ) mlflow_node_hook.before_node_run( node=dummy_node, catalog=dummy_catalog, inputs=node_inputs, is_async=False, run_id="132", ) run_id = mlflow.active_run().info.run_id mlflow_client = MlflowClient(mlflow_tracking_uri) current_run = mlflow_client.get_run(run_id) assert current_run.data.params == expected
def test_tf_estimator_autolog_ends_auto_created_run(tmpdir, export): directory = tmpdir.mkdir("test") mlflow.tensorflow.autolog() create_tf_estimator_model(str(directory), export) assert mlflow.active_run() is None
def main(): """ Trains a model on the MNIST dataset. """ print('{} Starting mlflow_demo.py.'.format(get_time())) # Load configuration file and default column ordering for df_models. config_root = os.path.join(os.path.dirname(os.getcwd()), 'config') config_file = 'config.yml' run_name = get_time() with mlflow.start_run(run_name=run_name): with open(os.path.join(config_root, config_file)) as fin: cfg = yaml.load(fin, Loader=yaml.FullLoader) print('{} Configuration parameters for run_name = {} run_id = {}.'. format(get_time(), run_name, mlflow.active_run().info.run_id)) pprint(cfg) mlflow.log_params(cfg) x_train, y_train, x_test, y_test = get_dataset(cfg['train_size'], cfg['test_size']) model = create_model(cfg) model.fit(x=x_train, y=y_train, validation_split=cfg['validation_split'], callbacks=[get_tensorboard_callbacks(run_name)], epochs=cfg['epochs']) # Export SavedModel model_local_path = get_model_path_local(run_name) if not os.path.exists(model_local_path): os.makedirs(model_local_path) model.save(os.path.join(model_local_path, 'final_model')) mlflow.tensorflow.log_model(tf_saved_model_dir=os.path.join( model_local_path, 'final_model'), tf_meta_graph_tags=[tag_constants.SERVING], tf_signature_def_key='serving_default', artifact_path='model') # Log a pickle file. pickle.dump(x_train, open(os.path.join(model_local_path, 'x_train.p'), 'wb')) mlflow.log_artifact(os.path.join(model_local_path, 'x_train.p')) # Log an image. fig, ax = plt.subplots() ax.scatter(np.random.normal(0.0, 1.0, 1000), np.random.normal(0.0, 1.0, 1000), alpha=0.3) ax.set_title('Random plot') fig.savefig(os.path.join(model_local_path, 'random_figure.png'), dpi=300, bbox_inches='tight') mlflow.log_artifact(os.path.join(model_local_path, 'random_figure.png'))
def access_run(self): return mlflow.active_run()
def safe_patch_function(*args, **kwargs): """ A safe wrapper around the specified `patch_function` implementation designed to handle exceptions thrown during the execution of `patch_function`. This wrapper distinguishes exceptions thrown from the underlying / original function (`<destination>.<function_name>`) from exceptions thrown from other parts of `patch_function`. This distinction is made by passing an augmented version of the underlying / original function to `patch_function` that uses nonlocal state to track whether or not it has been executed and whether or not it threw an exception. Exceptions thrown from the underlying / original function are propagated to the caller, while exceptions thrown from other parts of `patch_function` are caught and logged as warnings. """ # Reroute warnings encountered during the patch function implementation to an MLflow event # logger, and enforce silent mode if applicable (i.e. if the corresponding autologging # integration was called with `silent=True`), hiding MLflow event logging statements and # hiding all warnings in the autologging preamble and postamble (i.e. the code surrounding # the user's original / underlying ML function). Non-MLflow warnings are enabled during the # execution of the original / underlying ML function # # Note that we've opted *not* to apply this context manager as a decorator on # `safe_patch_function` because the context-manager-as-decorator pattern uses # `contextlib.ContextDecorator`, which creates generator expressions that cannot be pickled # during model serialization by ML frameworks such as scikit-learn is_silent_mode = get_autologging_config(autologging_integration, "silent", False) with set_mlflow_events_and_warnings_behavior_globally( # MLflow warnings emitted during autologging training sessions are likely not # actionable and result from the autologging implementation invoking another MLflow # API. Accordingly, we reroute these warnings to the MLflow event logger with level # WARNING For reference, see recommended warning and event logging behaviors from # https://docs.python.org/3/howto/logging.html#when-to-use-logging reroute_warnings=True, disable_event_logs=is_silent_mode, disable_warnings=is_silent_mode, ), set_non_mlflow_warnings_behavior_for_current_thread( # non-MLflow Warnings emitted during the autologging preamble (before the original / # underlying ML function is called) and postamble (after the original / underlying ML # function is called) are likely not actionable and result from the autologging # implementation invoking an API from a dependent library. Accordingly, we reroute # these warnings to the MLflow event logger with level WARNING. For reference, see # recommended warning and event logging behaviors from # https://docs.python.org/3/howto/logging.html#when-to-use-logging reroute_warnings=True, disable_warnings=is_silent_mode, ): if is_testing(): preexisting_run_for_testing = mlflow.active_run() # Whether or not to exclude autologged content from user-created fluent runs # (i.e. runs created manually via `mlflow.start_run()`) exclusive = get_autologging_config(autologging_integration, "exclusive", False) user_created_fluent_run_is_active = ( mlflow.active_run() and not _AutologgingSessionManager.active_session() ) active_session_failed = ( _AutologgingSessionManager.active_session() is not None and _AutologgingSessionManager.active_session().state == "failed" ) if ( active_session_failed or autologging_is_disabled(autologging_integration) or (user_created_fluent_run_is_active and exclusive) or mlflow.utils.autologging_utils._AUTOLOGGING_GLOBALLY_DISABLED ): # If the autologging integration associated with this patch is disabled, # or if the current autologging integration is in exclusive mode and a user-created # fluent run is active, call the original function and return. Restore the original # warning behavior during original function execution, since autologging is being # skipped with set_non_mlflow_warnings_behavior_for_current_thread( disable_warnings=False, reroute_warnings=False, ): return original(*args, **kwargs) # Whether or not the original / underlying function has been called during the # execution of patched code original_has_been_called = False # The value returned by the call to the original / underlying function during # the execution of patched code original_result = None # Whether or not an exception was raised from within the original / underlying function # during the execution of patched code failed_during_original = False # The active MLflow run (if any) associated with patch code execution patch_function_run_for_testing = None def try_log_autologging_event(log_fn, *args): try: log_fn(*args) except Exception as e: _logger.debug( "Failed to log autologging event via '%s'. Exception: %s", log_fn, e, ) with _AutologgingSessionManager.start_session(autologging_integration) as session: try: def call_original(*og_args, **og_kwargs): try: try_log_autologging_event( AutologgingEventLogger.get_logger().log_original_function_start, session, destination, function_name, og_args, og_kwargs, ) if is_testing(): _validate_args(args, kwargs, og_args, og_kwargs) # By the time `original` is called by the patch implementation, we # assume that either: 1. the patch implementation has already # created an MLflow run or 2. the patch code will not create an # MLflow run during the current execution. Here, we capture a # reference to the active run, which we will use later on to # determine whether or not the patch implementation created # a run and perform validation if necessary nonlocal patch_function_run_for_testing patch_function_run_for_testing = mlflow.active_run() nonlocal original_has_been_called original_has_been_called = True nonlocal original_result # Show all non-MLflow warnings as normal (i.e. not as event logs) # during original function execution, even if silent mode is enabled # (`silent=True`), since these warnings originate from the ML framework # or one of its dependencies and are likely relevant to the caller with set_non_mlflow_warnings_behavior_for_current_thread( disable_warnings=False, reroute_warnings=False, ): original_result = original(*og_args, **og_kwargs) try_log_autologging_event( AutologgingEventLogger.get_logger().log_original_function_success, session, destination, function_name, og_args, og_kwargs, ) return original_result except Exception as e: try_log_autologging_event( AutologgingEventLogger.get_logger().log_original_function_error, session, destination, function_name, og_args, og_kwargs, e, ) nonlocal failed_during_original failed_during_original = True raise # Apply the name, docstring, and signature of `original` to `call_original`. # This is important because several autologging patch implementations inspect # the signature of the `original` argument during execution call_original = update_wrapper_extended(call_original, original) try_log_autologging_event( AutologgingEventLogger.get_logger().log_patch_function_start, session, destination, function_name, args, kwargs, ) if patch_is_class: patch_function.call(call_original, *args, **kwargs) else: patch_function(call_original, *args, **kwargs) session.state = "succeeded" try_log_autologging_event( AutologgingEventLogger.get_logger().log_patch_function_success, session, destination, function_name, args, kwargs, ) except Exception as e: session.state = "failed" # Exceptions thrown during execution of the original function should be # propagated to the caller. Additionally, exceptions encountered during test # mode should be reraised to detect bugs in autologging implementations if failed_during_original or is_testing(): raise try_log_autologging_event( AutologgingEventLogger.get_logger().log_patch_function_error, session, destination, function_name, args, kwargs, e, ) _logger.warning( "Encountered unexpected error during %s autologging: %s", autologging_integration, e, ) if is_testing() and not preexisting_run_for_testing: # If an MLflow run was created during the execution of patch code, verify that # it is no longer active and that it contains expected autologging tags assert not mlflow.active_run(), ( "Autologging integration %s leaked an active run" % autologging_integration ) if patch_function_run_for_testing: _validate_autologging_run( autologging_integration, patch_function_run_for_testing.info.run_id ) if original_has_been_called: return original_result else: return original(*args, **kwargs)
def run_train_cv(self) -> None: """クロスバリデーションでの学習・評価を行う 学習・評価とともに、各foldのモデルの保存、スコアのログ出力についても行う """ # mlflow mlflow.set_experiment(self.exp_name) mlflow.start_run(run_name=self.run_name) logger.info(f"{self.run_name} - start training cv") scores = [] va_idxes = [] preds = [] # Adversarial validation if self.advanced and "adversarial_validation" in self.advanced: X_train = self.X_train X_test = self.X_test X_train["target"] = 0 X_test["target"] = 1 X_train = pd.concat([X_train, X_test], sort=False).reset_index(drop=True) y_train = X_train["target"] X_train.drop("target", axis=1, inplace=True) X_test.drop("target", axis=1, inplace=True) self.X_train = X_train self.y_train = y_train # 各foldで学習を行う for i_fold in range(self.cv.n_splits): # 学習を行う logger.info(f"{self.run_name} fold {i_fold} - start training") model, va_idx, va_pred, score = self.train_fold(i_fold) logger.info( f"{self.run_name} fold {i_fold} - end training - score {score}" ) # モデルを保存する model.save_model() # 結果を保持する va_idxes.append(va_idx) scores.append(score) preds.append(va_pred) # 各foldの結果をまとめる va_idxes = np.concatenate(va_idxes) order = np.argsort(va_idxes) preds = np.concatenate(preds, axis=0) preds = preds[order] if self.evaluation_metric == "log_loss": cv_score = log_loss(self.y_train, preds, eps=1e-15, normalize=True) elif self.evaluation_metric == "mean_absolute_error": cv_score = mean_absolute_error(self.y_train, preds) elif self.evaluation_metric == "rmse": cv_score = np.sqrt(mean_squared_error(self.y_train, preds)) elif self.evaluation_metric == "auc": cv_score = roc_auc_score(self.y_train, preds) elif self.evaluation_metric == "prauc": cv_score = average_precision_score(self.y_train, preds) logger.info(f"{self.run_name} - end training cv - score {cv_score}") # 予測結果の保存 Data.dump(preds, f"../output/pred/{self.run_name}-train.pkl") # mlflow self.run_id = mlflow.active_run().info.run_id log_param("model_name", self.model_cls.__class__.__name__) log_param("fe_name", self.fe_name) log_param("train_params", self.params) log_param("cv_strategy", str(self.cv)) log_param("evaluation_metric", self.evaluation_metric) log_metric("cv_score", cv_score) log_param( "fold_scores", dict( zip([f"fold_{i}" for i in range(len(scores))], [round(s, 4) for s in scores])), ) log_param("cols_definition", self.cols_definition) log_param("description", self.description) mlflow.end_run()
import random import tempfile import mlflow from mlflow import log_metric, log_param, log_artifacts, get_artifact_uri, active_run,\ get_tracking_uri, log_artifact if __name__ == "__main__": print("Running {} with tracking URI {}".format(sys.argv[0], get_tracking_uri())) log_param("param1", 5) log_metric("foo", 5) log_metric("foo", 6) log_metric("foo", 7) log_metric("random_int", random.randint(0, 100)) run_id = active_run().info.run_id # Get run metadata & data from the tracking server service = mlflow.tracking.MlflowClient() run = service.get_run(run_id) print("Metadata & data for run with UUID %s: %s" % (run_id, run)) local_dir = tempfile.mkdtemp() message = "test artifact written during run %s within artifact URI %s\n" \ % (active_run().info.run_id, get_artifact_uri()) try: file_path = os.path.join(local_dir, "some_output_file.txt") with open(file_path, "w") as handle: handle.write(message) log_artifacts(local_dir, "some_subdir") log_artifact(file_path, "another_dir") finally: shutil.rmtree(local_dir)
def train_models(self, args, base_line=True): """ Train the model and log all the MLflow Metrics :param args: command line arguments. If no arguments then use default :param base_line: Default flag. Create Baseline model """ # Create TensorFlow Session sess = tf.InteractiveSession() # Configure output_dir output_dir = tempfile.mkdtemp() # # initialize some classes # kdata_cls = KIMDB_Data_Utils() ktrain_cls = KTrain() kplot_cls = KPlot() # # get IMDB Data # (train_data, train_labels), (test_data, test_labels) = kdata_cls.fetch_imdb_data() # # prepare and vectorize data # x_train = kdata_cls.prepare_vectorized_sequences(train_data) x_test = kdata_cls.prepare_vectorized_sequences(test_data) y_train = kdata_cls.prepare_vectorized_labels(train_labels) y_test = kdata_cls.prepare_vectorized_labels(test_labels) image_dir = ktrain_cls.get_directory_path("images") model_dir = ktrain_cls.get_directory_path("models") graph_label_loss = 'Baseline Model: Training and Validation Loss' graph_label_acc = 'Baseline Model: Training and Validation Accuracy' graph_image_loss_png = os.path.join(image_dir, 'baseline_loss.png') graph_image_acc_png = os.path.join(image_dir, 'baseline_accuracy.png') if not base_line: graph_label_loss = 'Experimental: Training and Validation Loss' graph_label_acc = 'Experimental Model: Training and Validation Accuracy' graph_image_loss_png = os.path.join(image_dir, 'experimental_loss.png') graph_image_acc_png = os.path.join(image_dir, 'experimental_accuracy.png') kmodel = KModel() if base_line: print("Baseline Model:") model = kmodel.build_basic_model() else: print("Experiment Model:") model = kmodel.build_experimental_model(args.hidden_layers, args.output) history = ktrain_cls.compile_and_fit_model(model, x_train, y_train, epochs=args.epochs, loss=args.loss, output_dir=output_dir) model.summary() ktrain_cls.print_metrics(history) figure_loss = kplot_cls.plot_loss_graph(history, graph_label_loss) figure_loss.savefig(graph_image_loss_png) figure_acc = kplot_cls.plot_accuracy_graph(history, graph_label_acc) figure_acc.savefig(graph_image_acc_png) results = ktrain_cls.evaluate_model(model, x_test, y_test) print("Average Probability Results:") print(results) print() print("Predictions Results:") predictions = model.predict(x_test) print(predictions) mlflow_server = args.tracking_server # # We don't want to force people to have tracking server # running on localhost as it tracks in mlruns directory if mlflow_server: # Tracking URI if not mlflow_server.startswith("http"): mlflow_tracking_uri = 'http://' + mlflow_server + ':5000' else: mlflow_tracking_uri = mlflow_server # Set the Tracking URI mlflow.set_tracking_uri(mlflow_tracking_uri) print("MLflow Tracking URI: %s" % mlflow_tracking_uri) else: print("MLflow Tracking URI: %s" % "local directory 'mlruns'") with mlflow.start_run(): # print out current run_uuid run_uuid = mlflow.active_run().info.run_uuid print("MLflow Run ID: %s" % run_uuid) # log parameters mlflow.log_param("hidden_layers", args.hidden_layers) mlflow.log_param("output", args.output) mlflow.log_param("epochs", args.epochs) mlflow.log_param("loss_function", args.loss) # calculate metrics binary_loss = ktrain_cls.get_binary_loss(history) binary_acc = ktrain_cls.get_binary_acc(history) validation_loss = ktrain_cls.get_validation_loss(history) validation_acc = ktrain_cls.get_validation_acc(history) average_loss = results[0] average_acc = results[1] # log metrics mlflow.log_metric("binary_loss", binary_loss) mlflow.log_metric("binary_acc", binary_acc) mlflow.log_metric("validation_loss", validation_loss) mlflow.log_metric("validation_acc", validation_acc) mlflow.log_metric("average_loss", average_loss) mlflow.log_metric("average_acc", average_acc) # log artifacts mlflow.log_artifacts(image_dir, "images") # log model mlflow.keras.log_model(model, "models") # save model locally pathdir = "keras_models/" + run_uuid model_dir = self.get_directory_path(pathdir, False) ktrain_cls.keras_save_model(model, model_dir) # Write out TensorFlow events as a run artifact print("Uploading TensorFlow events as a run artifact.") mlflow.log_artifacts(output_dir, artifact_path="events") print("loss function use", args.loss)
def test_statsmodels_autolog_ends_auto_created_run(): mlflow.statsmodels.autolog() arma_model() assert mlflow.active_run() is None
def test_statsmodels_autolog_persists_manually_created_run(): mlflow.statsmodels.autolog() with mlflow.start_run() as run: ols_model() assert mlflow.active_run() assert mlflow.active_run().info.run_id == run.info.run_id
# Log the model within the MLflow run # https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html # https://www.mlflow.org/docs/latest/python_api/mlflow.pyspark.ml.html mlflow.mleap.log_model(spark_model=tunedModel.bestModel, sample_input=trainingData.limit(1), artifact_path="spark-model-mleap") mlflow.pyfunc.log_model(tunedModel.bestModel, "spark-model-pyfunc") # We log other artifacts model_path = "/dbfs/mnt/nycitibike/spark-model" mlflow.spark.save_model(tunedModel.bestModel, model_path) run = mlflow.active_run() print("Active run_id: {}".format(run.info.run_id)) mlflow.end_run() # COMMAND ---------- model = tunedModel.bestModel # COMMAND ---------- # MAGIC %fs ls /mnt/nycitibike/spark-model # COMMAND ---------- # MAGIC %md https://docs.microsoft.com/fr-fr/azure/databricks/_static/notebooks/mleap-model-export-demo-python.html
file.write('GENE_' + str(indexes[idx]) + '_WEIGHTS:\n\t') file.write('{0: ' + str(w[0]) + '\n\t' + ' 1: ' + str(w[1]) + '}\n') weights = {0: w[0], 1: w[1]} model = RandomForestClassifier(n_estimators=150, class_weight=weights) model.fit(trn_x, trn_y) mlflow.sklearn.log_model( model, 'model_fold_' + str(i + 1) + '_' + str(indexes[idx])) print('Model saved in run %s' % mlflow.active_run().info.run_uuid) pred = model.predict_proba(val_x)[:, 1] fold_scores.append(roc_auc_score(val_y, pred)) scores.append(fold_scores) scores = np.asanyarray(scores) mean_auc = np.mean(scores, axis=0) mlflow.log_metric('Gene_' + str(indexes[idx]) + '_AUC', mean_auc[0]) for i in range(5): ma.log_file(file_name='model_fold_' + str(i + 1) + '.txt', artifact_path=mlflow.get_artifact_uri(), delete_local=False) os.remove('model_fold_' + str(i + 1) + '.txt')
def test_lgb_autolog_ends_auto_created_run(bst_params, train_set): mlflow.lightgbm.autolog() lgb.train(bst_params, train_set, num_boost_round=1) assert mlflow.active_run() is None
#!/usr/bin/env python3 # -*- coding:utf-8 -*- # datetime:2019/11/11 11:56 import numpy as np from sklearn.linear_model import LogisticRegression import mlflow from mlflow import sklearn if __name__ == "__main__": mlflow.set_tracking_uri("http://localhost:5001") # mlflow.create_experiment("sklearn logistic regression") mlflow.set_experiment("sklearn logistic regression") with mlflow.start_run() as active_run: print(mlflow.active_run().info) X = np.array([-2, -1, 0, 1, 2, 1]).reshape(-1, 1) y = np.array([0, 0, 1, 1, 1, 0]) lr = LogisticRegression() lr.fit(X, y) score = lr.score(X, y) print("Score: %s" % score) mlflow.log_metric("score", score) # sklearn.log_model(lr, "model") mlflow.sklearn.log_model(lr, "model2") # print("Model saved in run %s" % mlflow.active_run().info.run_uuid)
def test_lgb_autolog_persists_manually_created_run(bst_params, train_set): mlflow.lightgbm.autolog() with mlflow.start_run() as run: lgb.train(bst_params, train_set, num_boost_round=1) assert mlflow.active_run() assert mlflow.active_run().info.run_id == run.info.run_id
def test_tf_estimator_autolog_persists_manually_created_run(tmpdir, export): directory = tmpdir.mkdir("test") with mlflow.start_run() as run: create_tf_estimator_model(str(directory), export) assert mlflow.active_run() assert mlflow.active_run().info.run_id == run.info.run_id
def test_pytorch_autolog_ends_auto_created_run(pytorch_model): assert mlflow.active_run() is None
def train(original, *args, **kwargs): def record_eval_results(eval_results, metrics_logger): """ Create a callback function that records evaluation results. """ @exception_safe_function def callback(env): res = {} for data_name, eval_name, value, _ in env.evaluation_result_list: key = data_name + "-" + eval_name res[key] = value metrics_logger.record_metrics(res, env.iteration) eval_results.append(res) return callback def log_feature_importance_plot(features, importance, importance_type): """ Log feature importance plot. """ import matplotlib.pyplot as plt indices = np.argsort(importance) features = np.array(features)[indices] importance = importance[indices] num_features = len(features) # If num_features > 10, increase the figure height to prevent the plot # from being too dense. w, h = [6.4, 4.8] # matplotlib's default figure size h = h + 0.1 * num_features if num_features > 10 else h fig, ax = plt.subplots(figsize=(w, h)) yloc = np.arange(num_features) ax.barh(yloc, importance, align="center", height=0.5) ax.set_yticks(yloc) ax.set_yticklabels(features) ax.set_xlabel("Importance") ax.set_title("Feature Importance ({})".format(importance_type)) fig.tight_layout() tmpdir = tempfile.mkdtemp() try: # pylint: disable=undefined-loop-variable filepath = os.path.join(tmpdir, "feature_importance_{}.png".format(imp_type)) fig.savefig(filepath) mlflow.log_artifact(filepath) finally: plt.close(fig) shutil.rmtree(tmpdir) autologging_client = MlflowAutologgingQueueingClient() # logging booster params separately via mlflow.log_params to extract key/value pairs # and make it easier to compare them across runs. booster_params = args[0] if len(args) > 0 else kwargs["params"] autologging_client.log_params(run_id=mlflow.active_run().info.run_id, params=booster_params) unlogged_params = [ "params", "train_set", "valid_sets", "valid_names", "fobj", "feval", "init_model", "evals_result", "learning_rates", "callbacks", ] params_to_log_for_fn = get_mlflow_run_params_for_fn_args( original, args, kwargs, unlogged_params ) autologging_client.log_params( run_id=mlflow.active_run().info.run_id, params=params_to_log_for_fn ) param_logging_operations = autologging_client.flush(synchronous=False) all_arg_names = _get_arg_names(original) num_pos_args = len(args) # adding a callback that records evaluation results. eval_results = [] callbacks_index = all_arg_names.index("callbacks") run_id = mlflow.active_run().info.run_id with batch_metrics_logger(run_id) as metrics_logger: callback = record_eval_results(eval_results, metrics_logger) if num_pos_args >= callbacks_index + 1: tmp_list = list(args) tmp_list[callbacks_index] += [callback] args = tuple(tmp_list) elif "callbacks" in kwargs and kwargs["callbacks"] is not None: kwargs["callbacks"] += [callback] else: kwargs["callbacks"] = [callback] # training model model = original(*args, **kwargs) # If early_stopping_rounds is present, logging metrics at the best iteration # as extra metrics with the max step + 1. early_stopping_index = all_arg_names.index("early_stopping_rounds") early_stopping = ( num_pos_args >= early_stopping_index + 1 or "early_stopping_rounds" in kwargs ) if early_stopping: extra_step = len(eval_results) autologging_client.log_metrics( run_id=mlflow.active_run().info.run_id, metrics={ "stopped_iteration": extra_step, # best_iteration is set even if training does not stop early. "best_iteration": model.best_iteration, }, ) # iteration starts from 1 in LightGBM. last_iter_results = eval_results[model.best_iteration - 1] autologging_client.log_metrics( run_id=mlflow.active_run().info.run_id, metrics=last_iter_results, step=extra_step, ) early_stopping_logging_operations = autologging_client.flush(synchronous=False) # logging feature importance as artifacts. for imp_type in ["split", "gain"]: features = model.feature_name() importance = model.feature_importance(importance_type=imp_type) try: log_feature_importance_plot(features, importance, imp_type) except Exception: _logger.exception( "Failed to log feature importance plot. LightGBM autologging " "will ignore the failure and continue. Exception: " ) imp = {ft: imp for ft, imp in zip(features, importance.tolist())} tmpdir = tempfile.mkdtemp() try: filepath = os.path.join(tmpdir, "feature_importance_{}.json".format(imp_type)) with open(filepath, "w") as f: json.dump(imp, f, indent=2) mlflow.log_artifact(filepath) finally: shutil.rmtree(tmpdir) # train_set must exist as the original train function already ran successfully train_set = args[1] if len(args) > 1 else kwargs.get("train_set") # it is possible that the dataset was constructed before the patched # constructor was applied, so we cannot assume the input_example_info exists input_example_info = getattr(train_set, "input_example_info", None) def get_input_example(): if input_example_info is None: raise Exception(ENSURE_AUTOLOGGING_ENABLED_TEXT) if input_example_info.error_msg is not None: raise Exception(input_example_info.error_msg) return input_example_info.input_example def infer_model_signature(input_example): model_output = model.predict(input_example) model_signature = infer_signature(input_example, model_output) return model_signature # Whether to automatically log the trained model based on boolean flag. if log_models: # Will only resolve `input_example` and `signature` if `log_models` is `True`. input_example, signature = resolve_input_example_and_signature( get_input_example, infer_model_signature, log_input_examples, log_model_signatures, _logger, ) log_model( model, artifact_path="model", signature=signature, input_example=input_example, ) param_logging_operations.await_completion() if early_stopping: early_stopping_logging_operations.await_completion() return model
def _log_posttraining_metadata(estimator, *args, **kwargs): """ Records metadata for a scikit-learn estimator after training has completed. This is intended to be invoked within a patched scikit-learn training routine (e.g., `fit()`, `fit_transform()`, ...) and assumes the existence of an active MLflow run that can be referenced via the fluent Tracking API. :param estimator: The scikit-learn estimator for which to log metadata. :param args: The arguments passed to the scikit-learn training routine (e.g., `fit()`, `fit_transform()`, ...). :param kwargs: The keyword arguments passed to the scikit-learn training routine. """ if hasattr(estimator, "score"): try: score_args = _get_args_for_score(estimator.score, estimator.fit, args, kwargs) training_score = estimator.score(*score_args) except Exception as e: # pylint: disable=broad-except msg = ( estimator.score.__qualname__ + " failed. The 'training_score' metric will not be recorded. Scoring error: " + str(e)) _logger.warning(msg) else: try_mlflow_log(mlflow.log_metric, "training_score", training_score) # log common metrics and artifacts for estimators (classifier, regressor) _log_specialized_estimator_content(estimator, mlflow.active_run().info.run_id, args, kwargs) input_example = None signature = None if hasattr(estimator, "predict"): try: # Fetch an input example using the first several rows of the array-like # training data supplied to the training routine (e.g., `fit()`) SAMPLE_ROWS = 5 fit_arg_names = _get_arg_names(estimator.fit) X_var_name, y_var_name = fit_arg_names[:2] input_example = _get_Xy(args, kwargs, X_var_name, y_var_name)[0][:SAMPLE_ROWS] model_output = estimator.predict(input_example) signature = infer_signature(input_example, model_output) except Exception as e: # pylint: disable=broad-except input_example = None msg = "Failed to infer an input example and model signature: " + str( e) _logger.warning(msg) try_mlflow_log( log_model, estimator, artifact_path="model", signature=signature, input_example=input_example, ) if _is_parameter_search_estimator(estimator): if hasattr(estimator, "best_estimator_"): try_mlflow_log( log_model, estimator.best_estimator_, artifact_path="best_estimator", signature=signature, input_example=input_example, ) if hasattr(estimator, "best_params_"): best_params = { "best_{param_name}".format(param_name=param_name): param_value for param_name, param_value in estimator.best_params_.items() } try_mlflow_log(mlflow.log_params, best_params) if hasattr(estimator, "cv_results_"): try: # Fetch environment-specific tags (e.g., user and source) to ensure that lineage # information is consistent with the parent run environment_tags = context_registry.resolve_tags() _create_child_runs_for_parameter_search( cv_estimator=estimator, parent_run=mlflow.active_run(), child_tags=environment_tags, ) except Exception as e: # pylint: disable=broad-except msg = ( "Encountered exception during creation of child runs for parameter search." " Child runs may be missing. Exception: {}".format( str(e))) _logger.warning(msg) try: cv_results_df = pd.DataFrame.from_dict( estimator.cv_results_) _log_parameter_search_results_as_artifact( cv_results_df, mlflow.active_run().info.run_id) except Exception as e: # pylint: disable=broad-except msg = ( "Failed to log parameter search results as an artifact." " Exception: {}".format(str(e))) _logger.warning(msg)
def train(*args, **kwargs): def record_eval_results(eval_results): """ Create a callback function that records evaluation results. """ def callback(env): res = {} for data_name, eval_name, value, _ in env.evaluation_result_list: key = data_name + '-' + eval_name res[key] = value eval_results.append(res) return callback def log_feature_importance_plot(features, importance, importance_type): """ Log feature importance plot. """ import matplotlib.pyplot as plt indices = np.argsort(importance) features = np.array(features)[indices] importance = importance[indices] num_features = len(features) # If num_features > 10, increase the figure height to prevent the plot # from being too dense. w, h = [6.4, 4.8] # matplotlib's default figure size h = h + 0.1 * num_features if num_features > 10 else h fig, ax = plt.subplots(figsize=(w, h)) yloc = np.arange(num_features) ax.barh(yloc, importance, align='center', height=0.5) ax.set_yticks(yloc) ax.set_yticklabels(features) ax.set_xlabel('Importance') ax.set_title('Feature Importance ({})'.format(importance_type)) fig.tight_layout() tmpdir = tempfile.mkdtemp() try: # pylint: disable=undefined-loop-variable filepath = os.path.join(tmpdir, 'feature_importance_{}.png'.format(imp_type)) fig.savefig(filepath) try_mlflow_log(mlflow.log_artifact, filepath) finally: plt.close(fig) shutil.rmtree(tmpdir) if not mlflow.active_run(): try_mlflow_log(mlflow.start_run) auto_end_run = True else: auto_end_run = False original = gorilla.get_original_attribute(lightgbm, 'train') # logging booster params separately via mlflow.log_params to extract key/value pairs # and make it easier to compare them across runs. params = args[0] if len(args) > 0 else kwargs['params'] try_mlflow_log(mlflow.log_params, params) unlogged_params = ['params', 'train_set', 'valid_sets', 'valid_names', 'fobj', 'feval', 'init_model', 'evals_result', 'learning_rates', 'callbacks'] log_fn_args_as_params(original, args, kwargs, unlogged_params) all_arg_names = inspect.getargspec(original)[0] # pylint: disable=W1505 num_pos_args = len(args) # adding a callback that records evaluation results. eval_results = [] callbacks_index = all_arg_names.index('callbacks') callback = record_eval_results(eval_results) if num_pos_args >= callbacks_index + 1: tmp_list = list(args) tmp_list[callbacks_index] += [callback] args = tuple(tmp_list) elif 'callbacks' in kwargs and kwargs['callbacks'] is not None: kwargs['callbacks'] += [callback] else: kwargs['callbacks'] = [callback] # training model model = original(*args, **kwargs) # logging metrics on each iteration. for idx, metrics in enumerate(eval_results): try_mlflow_log(mlflow.log_metrics, metrics, step=idx) # If early_stopping_rounds is present, logging metrics at the best iteration # as extra metrics with the max step + 1. early_stopping_index = all_arg_names.index('early_stopping_rounds') early_stopping = (num_pos_args >= early_stopping_index + 1 or 'early_stopping_rounds' in kwargs) if early_stopping: extra_step = len(eval_results) try_mlflow_log(mlflow.log_metric, 'stopped_iteration', len(eval_results)) # best_iteration is set even if training does not stop early. try_mlflow_log(mlflow.log_metric, 'best_iteration', model.best_iteration) # iteration starts from 1 in LightGBM. try_mlflow_log(mlflow.log_metrics, eval_results[model.best_iteration - 1], step=extra_step) # logging feature importance as artifacts. for imp_type in ['split', 'gain']: features = model.feature_name() importance = model.feature_importance(importance_type=imp_type) try: log_feature_importance_plot(features, importance, imp_type) except Exception: # pylint: disable=broad-except _logger.exception('Failed to log feature importance plot. LightGBM autologging ' 'will ignore the failure and continue. Exception: ') imp = {ft: imp for ft, imp in zip(features, importance.tolist())} tmpdir = tempfile.mkdtemp() try: filepath = os.path.join(tmpdir, 'feature_importance_{}.json'.format(imp_type)) with open(filepath, 'w') as f: json.dump(imp, f, indent=2) try_mlflow_log(mlflow.log_artifact, filepath) finally: shutil.rmtree(tmpdir) try_mlflow_log(log_model, model, artifact_path='model') if auto_end_run: try_mlflow_log(mlflow.end_run) return model
def _fit_keras_model_with_active_run(pandas_df, epochs): run_id = mlflow.active_run().info.run_id _fit_keras(pandas_df, epochs) run_id = run_id return mlflow.get_run(run_id)
from __future__ import print_function import sys import numpy as np from sklearn.linear_model import LogisticRegression import mlflow import mlflow.sklearn if __name__ == "__main__": penalty = sys.argv[1] # either 'l1', 'l2', 'elasticnet', or 'none' C = float(sys.argv[2]) if len(sys.argv) > 2 else 1.0 tol = float(sys.argv[3]) if len(sys.argv) > 3 else 1e-4 X = np.array([-2, -1, 0, 1, 2, 1]).reshape(-1, 1) y = np.array([0, 0, 1, 1, 1, 0]) lr = LogisticRegression(penalty=penalty, C=C, tol=tol) lr.fit(X, y) score = lr.score(X, y) mlflow.log_metric("score", score) mlflow.sklearn.log_model(lr, "model") print("Score: %s" % score) print("Model saved in run %s" % mlflow.active_run().info.run_uuid)
def _fit_keras_model(pandas_df, epochs): active_run = mlflow.active_run() if active_run: return _fit_keras_model_with_active_run(pandas_df, epochs) else: return _fit_keras_model_no_active_run(pandas_df, epochs)