def test_log_model_persists_specified_conda_env_in_mlflow_model_directory(
gluon_model, gluon_custom_env
):
artifact_path = "model"
with mlflow.start_run():
mlflow.gluon.log_model(
gluon_model=gluon_model, artifact_path=artifact_path, conda_env=gluon_custom_env
)
model_path = _download_artifact_from_uri(
"runs:/{run_id}/{artifact_path}".format(
run_id=mlflow.active_run().info.run_id, artifact_path=artifact_path
)
)
pyfunc_conf = _get_flavor_configuration(model_path=model_path, flavor_name=pyfunc.FLAVOR_NAME)
saved_conda_env_path = os.path.join(model_path, pyfunc_conf[pyfunc.ENV])
assert os.path.exists(saved_conda_env_path)
assert saved_conda_env_path != gluon_custom_env
with open(gluon_custom_env, "r") as f:
gluon_custom_env_parsed = yaml.safe_load(f)
with open(saved_conda_env_path, "r") as f:
saved_conda_env_parsed = yaml.safe_load(f)
assert saved_conda_env_parsed == gluon_custom_env_parsed
def test_node_hook_logging_below_limit_all_strategy(tmp_path, config_dir,
dummy_run_params,
dummy_node, param_length,
strategy):
# mocker.patch("kedro_mlflow.utils._is_kedro_project", return_value=True)
_write_yaml(
tmp_path / "conf" / "base" / "mlflow.yml",
dict(hooks=dict(node=dict(long_parameters_strategy=strategy)), ),
)
mlflow_tracking_uri = (tmp_path / "mlruns").as_uri()
mlflow.set_tracking_uri(mlflow_tracking_uri)
mlflow_node_hook = MlflowNodeHook()
param_value = param_length * "a"
node_inputs = {"params:my_param": param_value}
with mlflow.start_run():
mlflow_node_hook.before_pipeline_run(run_params=dummy_run_params,
pipeline=Pipeline([]),
catalog=DataCatalog())
mlflow_node_hook.before_node_run(
node=node(func=lambda x: x, inputs=dict(x="a"), outputs=None),
catalog=DataCatalog(), # can be empty
inputs=node_inputs,
is_async=False,
run_id="132",
)
run_id = mlflow.active_run().info.run_id
mlflow_client = MlflowClient(mlflow_tracking_uri)
current_run = mlflow_client.get_run(run_id)
assert current_run.data.params == {"my_param": param_value}
def _original_fn(*_og_args, **_og_kwargs):
if is_testing():
_validate_args(
autologging_integration,
function_name,
args,
kwargs,
og_args,
og_kwargs,
)
# By the time `original` is called by the patch implementation, we
# assume that either: 1. the patch implementation has already
# created an MLflow run or 2. the patch code will not create an
# MLflow run during the current execution. Here, we capture a
# reference to the active run, which we will use later on to
# determine whether or not the patch implementation created
# a run and perform validation if necessary
nonlocal patch_function_run_for_testing
patch_function_run_for_testing = mlflow.active_run(
)
nonlocal original_has_been_called
original_has_been_called = True
nonlocal original_result
# Show all non-MLflow warnings as normal (i.e. not as event logs)
# during original function execution, even if silent mode is enabled
# (`silent=True`), since these warnings originate from the ML framework
# or one of its dependencies and are likely relevant to the caller
with set_non_mlflow_warnings_behavior_for_current_thread(
disable_warnings=False,
reroute_warnings=False,
):
original_result = original(
*_og_args, **_og_kwargs)
return original_result
def test_log_model_persists_specified_conda_env_in_mlflow_model_directory(
saved_tf_iris_model, tf_custom_env):
artifact_path = "model"
with mlflow.start_run():
mlflow.tensorflow.log_model(tf_saved_model_dir=saved_tf_iris_model.path,
tf_meta_graph_tags=saved_tf_iris_model.meta_graph_tags,
tf_signature_def_key=saved_tf_iris_model.signature_def_key,
artifact_path=artifact_path,
conda_env=tf_custom_env)
model_uri = "runs:/{run_id}/{artifact_path}".format(
run_id=mlflow.active_run().info.run_id,
artifact_path=artifact_path)
model_path = _download_artifact_from_uri(artifact_uri=model_uri)
pyfunc_conf = _get_flavor_configuration(model_path=model_path, flavor_name=pyfunc.FLAVOR_NAME)
saved_conda_env_path = os.path.join(model_path, pyfunc_conf[pyfunc.ENV])
assert os.path.exists(saved_conda_env_path)
assert saved_conda_env_path != tf_custom_env
with open(tf_custom_env, "r") as f:
tf_custom_env_text = f.read()
with open(saved_conda_env_path, "r") as f:
saved_conda_env_text = f.read()
assert saved_conda_env_text == tf_custom_env_text
def test_spark_sklearn_autologging_context_provider(spark_session, data_format,
file_path):
mlflow.spark.autolog()
mlflow.sklearn.autolog()
df = (spark_session.read.format(data_format).option(
"header",
"true").option("inferSchema",
"true").load(file_path).select("number1", "number2"))
pandas_df = df.toPandas()
# DF info should be logged to the first run (it should be added to our context provider after
# the toPandas() call above & then logged here)
with mlflow.start_run():
run = _fit_sklearn_model(pandas_df)
_assert_spark_data_logged(run, file_path, data_format)
with mlflow.start_run():
pandas_df2 = df.filter("number1 > 0").toPandas()
run2 = _fit_sklearn_model(pandas_df2)
assert run2.info.run_id != run.info.run_id
_assert_spark_data_logged(run2, file_path, data_format)
time.sleep(1)
assert mlflow.active_run() is None
def _log_posttraining_metadata(estimator, *args, **kwargs):
"""
Records metadata for a scikit-learn estimator after training has completed.
This is intended to be invoked within a patched scikit-learn training routine
(e.g., `fit()`, `fit_transform()`, ...) and assumes the existence of an active
MLflow run that can be referenced via the fluent Tracking API.
:param estimator: The scikit-learn estimator for which to log metadata.
:param args: The arguments passed to the scikit-learn training routine (e.g.,
`fit()`, `fit_transform()`, ...).
:param kwargs: The keyword arguments passed to the scikit-learn training routine.
"""
if hasattr(estimator, "score"):
try:
score_args = _get_args_for_score(estimator.score,
estimator.fit, args, kwargs)
training_score = estimator.score(*score_args)
except Exception as e:
msg = (
estimator.score.__qualname__ +
" failed. The 'training_score' metric will not be recorded. Scoring error: "
+ str(e))
_logger.warning(msg)
else:
try_mlflow_log(mlflow.log_metric, "training_score",
training_score)
# log common metrics and artifacts for estimators (classifier, regressor)
_log_specialized_estimator_content(estimator,
mlflow.active_run().info.run_id,
args, kwargs)
def get_input_example():
# Fetch an input example using the first several rows of the array-like
# training data supplied to the training routine (e.g., `fit()`)
fit_arg_names = _get_arg_names(estimator.fit)
X_var_name, y_var_name = fit_arg_names[:2]
input_example = _get_Xy(args, kwargs, X_var_name,
y_var_name)[0][:INPUT_EXAMPLE_SAMPLE_ROWS]
return input_example
def infer_model_signature(input_example):
if not hasattr(estimator, "predict"):
raise Exception(
"the trained model does not specify a `predict` function, "
+ "which is required in order to infer the signature")
return infer_signature(input_example,
estimator.predict(input_example))
if log_models:
# Will only resolve `input_example` and `signature` if `log_models` is `True`.
input_example, signature = resolve_input_example_and_signature(
get_input_example,
infer_model_signature,
log_input_examples,
log_model_signatures,
_logger,
)
try_mlflow_log(
log_model,
estimator,
artifact_path="model",
signature=signature,
input_example=input_example,
)
if _is_parameter_search_estimator(estimator):
if hasattr(estimator, "best_estimator_") and log_models:
try_mlflow_log(
log_model,
estimator.best_estimator_,
artifact_path="best_estimator",
signature=signature,
input_example=input_example,
)
if hasattr(estimator, "best_score_"):
try_mlflow_log(mlflow.log_metric, "best_cv_score",
estimator.best_score_)
if hasattr(estimator, "best_params_"):
best_params = {
"best_{param_name}".format(param_name=param_name):
param_value
for param_name, param_value in
estimator.best_params_.items()
}
try_mlflow_log(mlflow.log_params, best_params)
if hasattr(estimator, "cv_results_"):
try:
# Fetch environment-specific tags (e.g., user and source) to ensure that lineage
# information is consistent with the parent run
child_tags = context_registry.resolve_tags()
child_tags.update({MLFLOW_AUTOLOGGING: FLAVOR_NAME})
_create_child_runs_for_parameter_search(
cv_estimator=estimator,
parent_run=mlflow.active_run(),
child_tags=child_tags,
)
except Exception as e:
msg = (
"Encountered exception during creation of child runs for parameter search."
" Child runs may be missing. Exception: {}".format(
str(e)))
_logger.warning(msg)
try:
cv_results_df = pd.DataFrame.from_dict(
estimator.cv_results_)
_log_parameter_search_results_as_artifact(
cv_results_df,
mlflow.active_run().info.run_id)
except Exception as e:
msg = (
"Failed to log parameter search results as an artifact."
" Exception: {}".format(str(e)))
_logger.warning(msg)
def test_log_fn_args_as_params_ignores_unwanted_parameters(start_run): # pylint: disable=W0613
args, kwargs, unlogged = ("arg1", {"arg2": "value"}, ["arg1", "arg2", "arg3"])
log_fn_args_as_params(dummy_fn, args, kwargs, unlogged)
client = mlflow.tracking.MlflowClient()
params = client.get_run(mlflow.active_run().info.run_id).data.params
assert len(params.keys()) == 0
def test_node_hook_logging(
tmp_path,
mocker,
monkeypatch,
dummy_run_params,
dummy_catalog,
dummy_pipeline,
dummy_node,
config_dir,
flatten_dict_params,
expected,
):
mocker.patch("logging.config.dictConfig")
mocker.patch("kedro_mlflow.utils._is_kedro_project", return_value=True)
monkeypatch.chdir(tmp_path)
# config = KedroMlflowConfig(
# project_path=tmp_path,
# node_hook_opts={"flatten_dict_params": flatten_dict_params, "sep": "-"},
# )
# # the function is imported inside the other file antd this is the file to patch
# # see https://stackoverflow.com/questions/30987973/python-mock-patch-doesnt-work-as-expected-for-public-method
# mocker.patch(
# "kedro_mlflow.framework.hooks.node_hook.get_mlflow_config", return_value=config
# )
_write_yaml(
tmp_path / "conf" / "base" / "mlflow.yml",
dict(
hooks=dict(
node=dict(
flatten_dict_params=flatten_dict_params, recursive=False, sep="-"
)
),
),
),
mlflow_node_hook = MlflowNodeHook()
node_inputs = {
v: dummy_catalog._data_sets.get(v) for k, v in dummy_node._inputs.items()
}
mlflow_tracking_uri = (tmp_path / "mlruns").as_uri()
mlflow.set_tracking_uri(mlflow_tracking_uri)
with mlflow.start_run():
mlflow_node_hook.before_pipeline_run(
run_params=dummy_run_params, pipeline=dummy_pipeline, catalog=dummy_catalog
)
mlflow_node_hook.before_node_run(
node=dummy_node,
catalog=dummy_catalog,
inputs=node_inputs,
is_async=False,
run_id="132",
)
run_id = mlflow.active_run().info.run_id
mlflow_client = MlflowClient(mlflow_tracking_uri)
current_run = mlflow_client.get_run(run_id)
assert current_run.data.params == expected
def test_tf_estimator_autolog_ends_auto_created_run(tmpdir, export):
directory = tmpdir.mkdir("test")
mlflow.tensorflow.autolog()
create_tf_estimator_model(str(directory), export)
assert mlflow.active_run() is None
def main():
"""
Trains a model on the MNIST dataset.
"""
print('{} Starting mlflow_demo.py.'.format(get_time()))
# Load configuration file and default column ordering for df_models.
config_root = os.path.join(os.path.dirname(os.getcwd()), 'config')
config_file = 'config.yml'
run_name = get_time()
with mlflow.start_run(run_name=run_name):
with open(os.path.join(config_root, config_file)) as fin:
cfg = yaml.load(fin, Loader=yaml.FullLoader)
print('{} Configuration parameters for run_name = {} run_id = {}.'.
format(get_time(), run_name,
mlflow.active_run().info.run_id))
pprint(cfg)
mlflow.log_params(cfg)
x_train, y_train, x_test, y_test = get_dataset(cfg['train_size'],
cfg['test_size'])
model = create_model(cfg)
model.fit(x=x_train,
y=y_train,
validation_split=cfg['validation_split'],
callbacks=[get_tensorboard_callbacks(run_name)],
epochs=cfg['epochs'])
# Export SavedModel
model_local_path = get_model_path_local(run_name)
if not os.path.exists(model_local_path):
os.makedirs(model_local_path)
model.save(os.path.join(model_local_path, 'final_model'))
mlflow.tensorflow.log_model(tf_saved_model_dir=os.path.join(
model_local_path, 'final_model'),
tf_meta_graph_tags=[tag_constants.SERVING],
tf_signature_def_key='serving_default',
artifact_path='model')
# Log a pickle file.
pickle.dump(x_train,
open(os.path.join(model_local_path, 'x_train.p'), 'wb'))
mlflow.log_artifact(os.path.join(model_local_path, 'x_train.p'))
# Log an image.
fig, ax = plt.subplots()
ax.scatter(np.random.normal(0.0, 1.0, 1000),
np.random.normal(0.0, 1.0, 1000),
alpha=0.3)
ax.set_title('Random plot')
fig.savefig(os.path.join(model_local_path, 'random_figure.png'),
dpi=300,
bbox_inches='tight')
mlflow.log_artifact(os.path.join(model_local_path,
'random_figure.png'))
def access_run(self):
return mlflow.active_run()
def safe_patch_function(*args, **kwargs):
"""
A safe wrapper around the specified `patch_function` implementation designed to
handle exceptions thrown during the execution of `patch_function`. This wrapper
distinguishes exceptions thrown from the underlying / original function
(`<destination>.<function_name>`) from exceptions thrown from other parts of
`patch_function`. This distinction is made by passing an augmented version of the
underlying / original function to `patch_function` that uses nonlocal state to track
whether or not it has been executed and whether or not it threw an exception.
Exceptions thrown from the underlying / original function are propagated to the caller,
while exceptions thrown from other parts of `patch_function` are caught and logged as
warnings.
"""
# Reroute warnings encountered during the patch function implementation to an MLflow event
# logger, and enforce silent mode if applicable (i.e. if the corresponding autologging
# integration was called with `silent=True`), hiding MLflow event logging statements and
# hiding all warnings in the autologging preamble and postamble (i.e. the code surrounding
# the user's original / underlying ML function). Non-MLflow warnings are enabled during the
# execution of the original / underlying ML function
#
# Note that we've opted *not* to apply this context manager as a decorator on
# `safe_patch_function` because the context-manager-as-decorator pattern uses
# `contextlib.ContextDecorator`, which creates generator expressions that cannot be pickled
# during model serialization by ML frameworks such as scikit-learn
is_silent_mode = get_autologging_config(autologging_integration, "silent", False)
with set_mlflow_events_and_warnings_behavior_globally(
# MLflow warnings emitted during autologging training sessions are likely not
# actionable and result from the autologging implementation invoking another MLflow
# API. Accordingly, we reroute these warnings to the MLflow event logger with level
# WARNING For reference, see recommended warning and event logging behaviors from
# https://docs.python.org/3/howto/logging.html#when-to-use-logging
reroute_warnings=True,
disable_event_logs=is_silent_mode,
disable_warnings=is_silent_mode,
), set_non_mlflow_warnings_behavior_for_current_thread(
# non-MLflow Warnings emitted during the autologging preamble (before the original /
# underlying ML function is called) and postamble (after the original / underlying ML
# function is called) are likely not actionable and result from the autologging
# implementation invoking an API from a dependent library. Accordingly, we reroute
# these warnings to the MLflow event logger with level WARNING. For reference, see
# recommended warning and event logging behaviors from
# https://docs.python.org/3/howto/logging.html#when-to-use-logging
reroute_warnings=True,
disable_warnings=is_silent_mode,
):
if is_testing():
preexisting_run_for_testing = mlflow.active_run()
# Whether or not to exclude autologged content from user-created fluent runs
# (i.e. runs created manually via `mlflow.start_run()`)
exclusive = get_autologging_config(autologging_integration, "exclusive", False)
user_created_fluent_run_is_active = (
mlflow.active_run() and not _AutologgingSessionManager.active_session()
)
active_session_failed = (
_AutologgingSessionManager.active_session() is not None
and _AutologgingSessionManager.active_session().state == "failed"
)
if (
active_session_failed
or autologging_is_disabled(autologging_integration)
or (user_created_fluent_run_is_active and exclusive)
or mlflow.utils.autologging_utils._AUTOLOGGING_GLOBALLY_DISABLED
):
# If the autologging integration associated with this patch is disabled,
# or if the current autologging integration is in exclusive mode and a user-created
# fluent run is active, call the original function and return. Restore the original
# warning behavior during original function execution, since autologging is being
# skipped
with set_non_mlflow_warnings_behavior_for_current_thread(
disable_warnings=False, reroute_warnings=False,
):
return original(*args, **kwargs)
# Whether or not the original / underlying function has been called during the
# execution of patched code
original_has_been_called = False
# The value returned by the call to the original / underlying function during
# the execution of patched code
original_result = None
# Whether or not an exception was raised from within the original / underlying function
# during the execution of patched code
failed_during_original = False
# The active MLflow run (if any) associated with patch code execution
patch_function_run_for_testing = None
def try_log_autologging_event(log_fn, *args):
try:
log_fn(*args)
except Exception as e:
_logger.debug(
"Failed to log autologging event via '%s'. Exception: %s", log_fn, e,
)
with _AutologgingSessionManager.start_session(autologging_integration) as session:
try:
def call_original(*og_args, **og_kwargs):
try:
try_log_autologging_event(
AutologgingEventLogger.get_logger().log_original_function_start,
session,
destination,
function_name,
og_args,
og_kwargs,
)
if is_testing():
_validate_args(args, kwargs, og_args, og_kwargs)
# By the time `original` is called by the patch implementation, we
# assume that either: 1. the patch implementation has already
# created an MLflow run or 2. the patch code will not create an
# MLflow run during the current execution. Here, we capture a
# reference to the active run, which we will use later on to
# determine whether or not the patch implementation created
# a run and perform validation if necessary
nonlocal patch_function_run_for_testing
patch_function_run_for_testing = mlflow.active_run()
nonlocal original_has_been_called
original_has_been_called = True
nonlocal original_result
# Show all non-MLflow warnings as normal (i.e. not as event logs)
# during original function execution, even if silent mode is enabled
# (`silent=True`), since these warnings originate from the ML framework
# or one of its dependencies and are likely relevant to the caller
with set_non_mlflow_warnings_behavior_for_current_thread(
disable_warnings=False, reroute_warnings=False,
):
original_result = original(*og_args, **og_kwargs)
try_log_autologging_event(
AutologgingEventLogger.get_logger().log_original_function_success,
session,
destination,
function_name,
og_args,
og_kwargs,
)
return original_result
except Exception as e:
try_log_autologging_event(
AutologgingEventLogger.get_logger().log_original_function_error,
session,
destination,
function_name,
og_args,
og_kwargs,
e,
)
nonlocal failed_during_original
failed_during_original = True
raise
# Apply the name, docstring, and signature of `original` to `call_original`.
# This is important because several autologging patch implementations inspect
# the signature of the `original` argument during execution
call_original = update_wrapper_extended(call_original, original)
try_log_autologging_event(
AutologgingEventLogger.get_logger().log_patch_function_start,
session,
destination,
function_name,
args,
kwargs,
)
if patch_is_class:
patch_function.call(call_original, *args, **kwargs)
else:
patch_function(call_original, *args, **kwargs)
session.state = "succeeded"
try_log_autologging_event(
AutologgingEventLogger.get_logger().log_patch_function_success,
session,
destination,
function_name,
args,
kwargs,
)
except Exception as e:
session.state = "failed"
# Exceptions thrown during execution of the original function should be
# propagated to the caller. Additionally, exceptions encountered during test
# mode should be reraised to detect bugs in autologging implementations
if failed_during_original or is_testing():
raise
try_log_autologging_event(
AutologgingEventLogger.get_logger().log_patch_function_error,
session,
destination,
function_name,
args,
kwargs,
e,
)
_logger.warning(
"Encountered unexpected error during %s autologging: %s",
autologging_integration,
e,
)
if is_testing() and not preexisting_run_for_testing:
# If an MLflow run was created during the execution of patch code, verify that
# it is no longer active and that it contains expected autologging tags
assert not mlflow.active_run(), (
"Autologging integration %s leaked an active run" % autologging_integration
)
if patch_function_run_for_testing:
_validate_autologging_run(
autologging_integration, patch_function_run_for_testing.info.run_id
)
if original_has_been_called:
return original_result
else:
return original(*args, **kwargs)
def run_train_cv(self) -> None:
"""クロスバリデーションでの学習・評価を行う
学習・評価とともに、各foldのモデルの保存、スコアのログ出力についても行う
"""
# mlflow
mlflow.set_experiment(self.exp_name)
mlflow.start_run(run_name=self.run_name)
logger.info(f"{self.run_name} - start training cv")
scores = []
va_idxes = []
preds = []
# Adversarial validation
if self.advanced and "adversarial_validation" in self.advanced:
X_train = self.X_train
X_test = self.X_test
X_train["target"] = 0
X_test["target"] = 1
X_train = pd.concat([X_train, X_test],
sort=False).reset_index(drop=True)
y_train = X_train["target"]
X_train.drop("target", axis=1, inplace=True)
X_test.drop("target", axis=1, inplace=True)
self.X_train = X_train
self.y_train = y_train
# 各foldで学習を行う
for i_fold in range(self.cv.n_splits):
# 学習を行う
logger.info(f"{self.run_name} fold {i_fold} - start training")
model, va_idx, va_pred, score = self.train_fold(i_fold)
logger.info(
f"{self.run_name} fold {i_fold} - end training - score {score}"
)
# モデルを保存する
model.save_model()
# 結果を保持する
va_idxes.append(va_idx)
scores.append(score)
preds.append(va_pred)
# 各foldの結果をまとめる
va_idxes = np.concatenate(va_idxes)
order = np.argsort(va_idxes)
preds = np.concatenate(preds, axis=0)
preds = preds[order]
if self.evaluation_metric == "log_loss":
cv_score = log_loss(self.y_train, preds, eps=1e-15, normalize=True)
elif self.evaluation_metric == "mean_absolute_error":
cv_score = mean_absolute_error(self.y_train, preds)
elif self.evaluation_metric == "rmse":
cv_score = np.sqrt(mean_squared_error(self.y_train, preds))
elif self.evaluation_metric == "auc":
cv_score = roc_auc_score(self.y_train, preds)
elif self.evaluation_metric == "prauc":
cv_score = average_precision_score(self.y_train, preds)
logger.info(f"{self.run_name} - end training cv - score {cv_score}")
# 予測結果の保存
Data.dump(preds, f"../output/pred/{self.run_name}-train.pkl")
# mlflow
self.run_id = mlflow.active_run().info.run_id
log_param("model_name", self.model_cls.__class__.__name__)
log_param("fe_name", self.fe_name)
log_param("train_params", self.params)
log_param("cv_strategy", str(self.cv))
log_param("evaluation_metric", self.evaluation_metric)
log_metric("cv_score", cv_score)
log_param(
"fold_scores",
dict(
zip([f"fold_{i}" for i in range(len(scores))],
[round(s, 4) for s in scores])),
)
log_param("cols_definition", self.cols_definition)
log_param("description", self.description)
mlflow.end_run()
import random
import tempfile
import mlflow
from mlflow import log_metric, log_param, log_artifacts, get_artifact_uri, active_run,\
get_tracking_uri, log_artifact
if __name__ == "__main__":
print("Running {} with tracking URI {}".format(sys.argv[0],
get_tracking_uri()))
log_param("param1", 5)
log_metric("foo", 5)
log_metric("foo", 6)
log_metric("foo", 7)
log_metric("random_int", random.randint(0, 100))
run_id = active_run().info.run_id
# Get run metadata & data from the tracking server
service = mlflow.tracking.MlflowClient()
run = service.get_run(run_id)
print("Metadata & data for run with UUID %s: %s" % (run_id, run))
local_dir = tempfile.mkdtemp()
message = "test artifact written during run %s within artifact URI %s\n" \
% (active_run().info.run_id, get_artifact_uri())
try:
file_path = os.path.join(local_dir, "some_output_file.txt")
with open(file_path, "w") as handle:
handle.write(message)
log_artifacts(local_dir, "some_subdir")
log_artifact(file_path, "another_dir")
finally:
shutil.rmtree(local_dir)
def train_models(self, args, base_line=True):
"""
Train the model and log all the MLflow Metrics
:param args: command line arguments. If no arguments then use default
:param base_line: Default flag. Create Baseline model
"""
# Create TensorFlow Session
sess = tf.InteractiveSession()
# Configure output_dir
output_dir = tempfile.mkdtemp()
#
# initialize some classes
#
kdata_cls = KIMDB_Data_Utils()
ktrain_cls = KTrain()
kplot_cls = KPlot()
#
# get IMDB Data
#
(train_data,
train_labels), (test_data, test_labels) = kdata_cls.fetch_imdb_data()
#
# prepare and vectorize data
#
x_train = kdata_cls.prepare_vectorized_sequences(train_data)
x_test = kdata_cls.prepare_vectorized_sequences(test_data)
y_train = kdata_cls.prepare_vectorized_labels(train_labels)
y_test = kdata_cls.prepare_vectorized_labels(test_labels)
image_dir = ktrain_cls.get_directory_path("images")
model_dir = ktrain_cls.get_directory_path("models")
graph_label_loss = 'Baseline Model: Training and Validation Loss'
graph_label_acc = 'Baseline Model: Training and Validation Accuracy'
graph_image_loss_png = os.path.join(image_dir, 'baseline_loss.png')
graph_image_acc_png = os.path.join(image_dir, 'baseline_accuracy.png')
if not base_line:
graph_label_loss = 'Experimental: Training and Validation Loss'
graph_label_acc = 'Experimental Model: Training and Validation Accuracy'
graph_image_loss_png = os.path.join(image_dir,
'experimental_loss.png')
graph_image_acc_png = os.path.join(image_dir,
'experimental_accuracy.png')
kmodel = KModel()
if base_line:
print("Baseline Model:")
model = kmodel.build_basic_model()
else:
print("Experiment Model:")
model = kmodel.build_experimental_model(args.hidden_layers,
args.output)
history = ktrain_cls.compile_and_fit_model(model,
x_train,
y_train,
epochs=args.epochs,
loss=args.loss,
output_dir=output_dir)
model.summary()
ktrain_cls.print_metrics(history)
figure_loss = kplot_cls.plot_loss_graph(history, graph_label_loss)
figure_loss.savefig(graph_image_loss_png)
figure_acc = kplot_cls.plot_accuracy_graph(history, graph_label_acc)
figure_acc.savefig(graph_image_acc_png)
results = ktrain_cls.evaluate_model(model, x_test, y_test)
print("Average Probability Results:")
print(results)
print()
print("Predictions Results:")
predictions = model.predict(x_test)
print(predictions)
mlflow_server = args.tracking_server
#
# We don't want to force people to have tracking server
# running on localhost as it tracks in mlruns directory
if mlflow_server:
# Tracking URI
if not mlflow_server.startswith("http"):
mlflow_tracking_uri = 'http://' + mlflow_server + ':5000'
else:
mlflow_tracking_uri = mlflow_server
# Set the Tracking URI
mlflow.set_tracking_uri(mlflow_tracking_uri)
print("MLflow Tracking URI: %s" % mlflow_tracking_uri)
else:
print("MLflow Tracking URI: %s" % "local directory 'mlruns'")
with mlflow.start_run():
# print out current run_uuid
run_uuid = mlflow.active_run().info.run_uuid
print("MLflow Run ID: %s" % run_uuid)
# log parameters
mlflow.log_param("hidden_layers", args.hidden_layers)
mlflow.log_param("output", args.output)
mlflow.log_param("epochs", args.epochs)
mlflow.log_param("loss_function", args.loss)
# calculate metrics
binary_loss = ktrain_cls.get_binary_loss(history)
binary_acc = ktrain_cls.get_binary_acc(history)
validation_loss = ktrain_cls.get_validation_loss(history)
validation_acc = ktrain_cls.get_validation_acc(history)
average_loss = results[0]
average_acc = results[1]
# log metrics
mlflow.log_metric("binary_loss", binary_loss)
mlflow.log_metric("binary_acc", binary_acc)
mlflow.log_metric("validation_loss", validation_loss)
mlflow.log_metric("validation_acc", validation_acc)
mlflow.log_metric("average_loss", average_loss)
mlflow.log_metric("average_acc", average_acc)
# log artifacts
mlflow.log_artifacts(image_dir, "images")
# log model
mlflow.keras.log_model(model, "models")
# save model locally
pathdir = "keras_models/" + run_uuid
model_dir = self.get_directory_path(pathdir, False)
ktrain_cls.keras_save_model(model, model_dir)
# Write out TensorFlow events as a run artifact
print("Uploading TensorFlow events as a run artifact.")
mlflow.log_artifacts(output_dir, artifact_path="events")
print("loss function use", args.loss)
def test_statsmodels_autolog_ends_auto_created_run():
mlflow.statsmodels.autolog()
arma_model()
assert mlflow.active_run() is None
def test_statsmodels_autolog_persists_manually_created_run():
mlflow.statsmodels.autolog()
with mlflow.start_run() as run:
ols_model()
assert mlflow.active_run()
assert mlflow.active_run().info.run_id == run.info.run_id
# Log the model within the MLflow run
# https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html
# https://www.mlflow.org/docs/latest/python_api/mlflow.pyspark.ml.html
mlflow.mleap.log_model(spark_model=tunedModel.bestModel,
sample_input=trainingData.limit(1),
artifact_path="spark-model-mleap")
mlflow.pyfunc.log_model(tunedModel.bestModel, "spark-model-pyfunc")
# We log other artifacts
model_path = "/dbfs/mnt/nycitibike/spark-model"
mlflow.spark.save_model(tunedModel.bestModel, model_path)
run = mlflow.active_run()
print("Active run_id: {}".format(run.info.run_id))
mlflow.end_run()
# COMMAND ----------
model = tunedModel.bestModel
# COMMAND ----------
# MAGIC %fs ls /mnt/nycitibike/spark-model
# COMMAND ----------
# MAGIC %md https://docs.microsoft.com/fr-fr/azure/databricks/_static/notebooks/mleap-model-export-demo-python.html
file.write('GENE_' + str(indexes[idx]) +
'_WEIGHTS:\n\t')
file.write('{0: ' + str(w[0]) + '\n\t' + ' 1: ' +
str(w[1]) + '}\n')
weights = {0: w[0], 1: w[1]}
model = RandomForestClassifier(n_estimators=150,
class_weight=weights)
model.fit(trn_x, trn_y)
mlflow.sklearn.log_model(
model,
'model_fold_' + str(i + 1) + '_' + str(indexes[idx]))
print('Model saved in run %s' %
mlflow.active_run().info.run_uuid)
pred = model.predict_proba(val_x)[:, 1]
fold_scores.append(roc_auc_score(val_y, pred))
scores.append(fold_scores)
scores = np.asanyarray(scores)
mean_auc = np.mean(scores, axis=0)
mlflow.log_metric('Gene_' + str(indexes[idx]) + '_AUC',
mean_auc[0])
for i in range(5):
ma.log_file(file_name='model_fold_' + str(i + 1) + '.txt',
artifact_path=mlflow.get_artifact_uri(),
delete_local=False)
os.remove('model_fold_' + str(i + 1) + '.txt')
def test_lgb_autolog_ends_auto_created_run(bst_params, train_set):
mlflow.lightgbm.autolog()
lgb.train(bst_params, train_set, num_boost_round=1)
assert mlflow.active_run() is None
#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# datetime:2019/11/11 11:56
import numpy as np
from sklearn.linear_model import LogisticRegression
import mlflow
from mlflow import sklearn
if __name__ == "__main__":
mlflow.set_tracking_uri("http://localhost:5001")
# mlflow.create_experiment("sklearn logistic regression")
mlflow.set_experiment("sklearn logistic regression")
with mlflow.start_run() as active_run:
print(mlflow.active_run().info)
X = np.array([-2, -1, 0, 1, 2, 1]).reshape(-1, 1)
y = np.array([0, 0, 1, 1, 1, 0])
lr = LogisticRegression()
lr.fit(X, y)
score = lr.score(X, y)
print("Score: %s" % score)
mlflow.log_metric("score", score)
# sklearn.log_model(lr, "model")
mlflow.sklearn.log_model(lr, "model2")
# print("Model saved in run %s" % mlflow.active_run().info.run_uuid)
def test_lgb_autolog_persists_manually_created_run(bst_params, train_set):
mlflow.lightgbm.autolog()
with mlflow.start_run() as run:
lgb.train(bst_params, train_set, num_boost_round=1)
assert mlflow.active_run()
assert mlflow.active_run().info.run_id == run.info.run_id
def test_tf_estimator_autolog_persists_manually_created_run(tmpdir, export):
directory = tmpdir.mkdir("test")
with mlflow.start_run() as run:
create_tf_estimator_model(str(directory), export)
assert mlflow.active_run()
assert mlflow.active_run().info.run_id == run.info.run_id
def test_pytorch_autolog_ends_auto_created_run(pytorch_model):
assert mlflow.active_run() is None
def train(original, *args, **kwargs):
def record_eval_results(eval_results, metrics_logger):
"""
Create a callback function that records evaluation results.
"""
@exception_safe_function
def callback(env):
res = {}
for data_name, eval_name, value, _ in env.evaluation_result_list:
key = data_name + "-" + eval_name
res[key] = value
metrics_logger.record_metrics(res, env.iteration)
eval_results.append(res)
return callback
def log_feature_importance_plot(features, importance, importance_type):
"""
Log feature importance plot.
"""
import matplotlib.pyplot as plt
indices = np.argsort(importance)
features = np.array(features)[indices]
importance = importance[indices]
num_features = len(features)
# If num_features > 10, increase the figure height to prevent the plot
# from being too dense.
w, h = [6.4, 4.8] # matplotlib's default figure size
h = h + 0.1 * num_features if num_features > 10 else h
fig, ax = plt.subplots(figsize=(w, h))
yloc = np.arange(num_features)
ax.barh(yloc, importance, align="center", height=0.5)
ax.set_yticks(yloc)
ax.set_yticklabels(features)
ax.set_xlabel("Importance")
ax.set_title("Feature Importance ({})".format(importance_type))
fig.tight_layout()
tmpdir = tempfile.mkdtemp()
try:
# pylint: disable=undefined-loop-variable
filepath = os.path.join(tmpdir, "feature_importance_{}.png".format(imp_type))
fig.savefig(filepath)
mlflow.log_artifact(filepath)
finally:
plt.close(fig)
shutil.rmtree(tmpdir)
autologging_client = MlflowAutologgingQueueingClient()
# logging booster params separately via mlflow.log_params to extract key/value pairs
# and make it easier to compare them across runs.
booster_params = args[0] if len(args) > 0 else kwargs["params"]
autologging_client.log_params(run_id=mlflow.active_run().info.run_id, params=booster_params)
unlogged_params = [
"params",
"train_set",
"valid_sets",
"valid_names",
"fobj",
"feval",
"init_model",
"evals_result",
"learning_rates",
"callbacks",
]
params_to_log_for_fn = get_mlflow_run_params_for_fn_args(
original, args, kwargs, unlogged_params
)
autologging_client.log_params(
run_id=mlflow.active_run().info.run_id, params=params_to_log_for_fn
)
param_logging_operations = autologging_client.flush(synchronous=False)
all_arg_names = _get_arg_names(original)
num_pos_args = len(args)
# adding a callback that records evaluation results.
eval_results = []
callbacks_index = all_arg_names.index("callbacks")
run_id = mlflow.active_run().info.run_id
with batch_metrics_logger(run_id) as metrics_logger:
callback = record_eval_results(eval_results, metrics_logger)
if num_pos_args >= callbacks_index + 1:
tmp_list = list(args)
tmp_list[callbacks_index] += [callback]
args = tuple(tmp_list)
elif "callbacks" in kwargs and kwargs["callbacks"] is not None:
kwargs["callbacks"] += [callback]
else:
kwargs["callbacks"] = [callback]
# training model
model = original(*args, **kwargs)
# If early_stopping_rounds is present, logging metrics at the best iteration
# as extra metrics with the max step + 1.
early_stopping_index = all_arg_names.index("early_stopping_rounds")
early_stopping = (
num_pos_args >= early_stopping_index + 1 or "early_stopping_rounds" in kwargs
)
if early_stopping:
extra_step = len(eval_results)
autologging_client.log_metrics(
run_id=mlflow.active_run().info.run_id,
metrics={
"stopped_iteration": extra_step,
# best_iteration is set even if training does not stop early.
"best_iteration": model.best_iteration,
},
)
# iteration starts from 1 in LightGBM.
last_iter_results = eval_results[model.best_iteration - 1]
autologging_client.log_metrics(
run_id=mlflow.active_run().info.run_id,
metrics=last_iter_results,
step=extra_step,
)
early_stopping_logging_operations = autologging_client.flush(synchronous=False)
# logging feature importance as artifacts.
for imp_type in ["split", "gain"]:
features = model.feature_name()
importance = model.feature_importance(importance_type=imp_type)
try:
log_feature_importance_plot(features, importance, imp_type)
except Exception:
_logger.exception(
"Failed to log feature importance plot. LightGBM autologging "
"will ignore the failure and continue. Exception: "
)
imp = {ft: imp for ft, imp in zip(features, importance.tolist())}
tmpdir = tempfile.mkdtemp()
try:
filepath = os.path.join(tmpdir, "feature_importance_{}.json".format(imp_type))
with open(filepath, "w") as f:
json.dump(imp, f, indent=2)
mlflow.log_artifact(filepath)
finally:
shutil.rmtree(tmpdir)
# train_set must exist as the original train function already ran successfully
train_set = args[1] if len(args) > 1 else kwargs.get("train_set")
# it is possible that the dataset was constructed before the patched
# constructor was applied, so we cannot assume the input_example_info exists
input_example_info = getattr(train_set, "input_example_info", None)
def get_input_example():
if input_example_info is None:
raise Exception(ENSURE_AUTOLOGGING_ENABLED_TEXT)
if input_example_info.error_msg is not None:
raise Exception(input_example_info.error_msg)
return input_example_info.input_example
def infer_model_signature(input_example):
model_output = model.predict(input_example)
model_signature = infer_signature(input_example, model_output)
return model_signature
# Whether to automatically log the trained model based on boolean flag.
if log_models:
# Will only resolve `input_example` and `signature` if `log_models` is `True`.
input_example, signature = resolve_input_example_and_signature(
get_input_example,
infer_model_signature,
log_input_examples,
log_model_signatures,
_logger,
)
log_model(
model, artifact_path="model", signature=signature, input_example=input_example,
)
param_logging_operations.await_completion()
if early_stopping:
early_stopping_logging_operations.await_completion()
return model
def _log_posttraining_metadata(estimator, *args, **kwargs):
"""
Records metadata for a scikit-learn estimator after training has completed.
This is intended to be invoked within a patched scikit-learn training routine
(e.g., `fit()`, `fit_transform()`, ...) and assumes the existence of an active
MLflow run that can be referenced via the fluent Tracking API.
:param estimator: The scikit-learn estimator for which to log metadata.
:param args: The arguments passed to the scikit-learn training routine (e.g.,
`fit()`, `fit_transform()`, ...).
:param kwargs: The keyword arguments passed to the scikit-learn training routine.
"""
if hasattr(estimator, "score"):
try:
score_args = _get_args_for_score(estimator.score,
estimator.fit, args, kwargs)
training_score = estimator.score(*score_args)
except Exception as e: # pylint: disable=broad-except
msg = (
estimator.score.__qualname__ +
" failed. The 'training_score' metric will not be recorded. Scoring error: "
+ str(e))
_logger.warning(msg)
else:
try_mlflow_log(mlflow.log_metric, "training_score",
training_score)
# log common metrics and artifacts for estimators (classifier, regressor)
_log_specialized_estimator_content(estimator,
mlflow.active_run().info.run_id,
args, kwargs)
input_example = None
signature = None
if hasattr(estimator, "predict"):
try:
# Fetch an input example using the first several rows of the array-like
# training data supplied to the training routine (e.g., `fit()`)
SAMPLE_ROWS = 5
fit_arg_names = _get_arg_names(estimator.fit)
X_var_name, y_var_name = fit_arg_names[:2]
input_example = _get_Xy(args, kwargs, X_var_name,
y_var_name)[0][:SAMPLE_ROWS]
model_output = estimator.predict(input_example)
signature = infer_signature(input_example, model_output)
except Exception as e: # pylint: disable=broad-except
input_example = None
msg = "Failed to infer an input example and model signature: " + str(
e)
_logger.warning(msg)
try_mlflow_log(
log_model,
estimator,
artifact_path="model",
signature=signature,
input_example=input_example,
)
if _is_parameter_search_estimator(estimator):
if hasattr(estimator, "best_estimator_"):
try_mlflow_log(
log_model,
estimator.best_estimator_,
artifact_path="best_estimator",
signature=signature,
input_example=input_example,
)
if hasattr(estimator, "best_params_"):
best_params = {
"best_{param_name}".format(param_name=param_name):
param_value
for param_name, param_value in
estimator.best_params_.items()
}
try_mlflow_log(mlflow.log_params, best_params)
if hasattr(estimator, "cv_results_"):
try:
# Fetch environment-specific tags (e.g., user and source) to ensure that lineage
# information is consistent with the parent run
environment_tags = context_registry.resolve_tags()
_create_child_runs_for_parameter_search(
cv_estimator=estimator,
parent_run=mlflow.active_run(),
child_tags=environment_tags,
)
except Exception as e: # pylint: disable=broad-except
msg = (
"Encountered exception during creation of child runs for parameter search."
" Child runs may be missing. Exception: {}".format(
str(e)))
_logger.warning(msg)
try:
cv_results_df = pd.DataFrame.from_dict(
estimator.cv_results_)
_log_parameter_search_results_as_artifact(
cv_results_df,
mlflow.active_run().info.run_id)
except Exception as e: # pylint: disable=broad-except
msg = (
"Failed to log parameter search results as an artifact."
" Exception: {}".format(str(e)))
_logger.warning(msg)
def train(*args, **kwargs):
def record_eval_results(eval_results):
"""
Create a callback function that records evaluation results.
"""
def callback(env):
res = {}
for data_name, eval_name, value, _ in env.evaluation_result_list:
key = data_name + '-' + eval_name
res[key] = value
eval_results.append(res)
return callback
def log_feature_importance_plot(features, importance, importance_type):
"""
Log feature importance plot.
"""
import matplotlib.pyplot as plt
indices = np.argsort(importance)
features = np.array(features)[indices]
importance = importance[indices]
num_features = len(features)
# If num_features > 10, increase the figure height to prevent the plot
# from being too dense.
w, h = [6.4, 4.8] # matplotlib's default figure size
h = h + 0.1 * num_features if num_features > 10 else h
fig, ax = plt.subplots(figsize=(w, h))
yloc = np.arange(num_features)
ax.barh(yloc, importance, align='center', height=0.5)
ax.set_yticks(yloc)
ax.set_yticklabels(features)
ax.set_xlabel('Importance')
ax.set_title('Feature Importance ({})'.format(importance_type))
fig.tight_layout()
tmpdir = tempfile.mkdtemp()
try:
# pylint: disable=undefined-loop-variable
filepath = os.path.join(tmpdir, 'feature_importance_{}.png'.format(imp_type))
fig.savefig(filepath)
try_mlflow_log(mlflow.log_artifact, filepath)
finally:
plt.close(fig)
shutil.rmtree(tmpdir)
if not mlflow.active_run():
try_mlflow_log(mlflow.start_run)
auto_end_run = True
else:
auto_end_run = False
original = gorilla.get_original_attribute(lightgbm, 'train')
# logging booster params separately via mlflow.log_params to extract key/value pairs
# and make it easier to compare them across runs.
params = args[0] if len(args) > 0 else kwargs['params']
try_mlflow_log(mlflow.log_params, params)
unlogged_params = ['params', 'train_set', 'valid_sets', 'valid_names', 'fobj', 'feval',
'init_model', 'evals_result', 'learning_rates', 'callbacks']
log_fn_args_as_params(original, args, kwargs, unlogged_params)
all_arg_names = inspect.getargspec(original)[0] # pylint: disable=W1505
num_pos_args = len(args)
# adding a callback that records evaluation results.
eval_results = []
callbacks_index = all_arg_names.index('callbacks')
callback = record_eval_results(eval_results)
if num_pos_args >= callbacks_index + 1:
tmp_list = list(args)
tmp_list[callbacks_index] += [callback]
args = tuple(tmp_list)
elif 'callbacks' in kwargs and kwargs['callbacks'] is not None:
kwargs['callbacks'] += [callback]
else:
kwargs['callbacks'] = [callback]
# training model
model = original(*args, **kwargs)
# logging metrics on each iteration.
for idx, metrics in enumerate(eval_results):
try_mlflow_log(mlflow.log_metrics, metrics, step=idx)
# If early_stopping_rounds is present, logging metrics at the best iteration
# as extra metrics with the max step + 1.
early_stopping_index = all_arg_names.index('early_stopping_rounds')
early_stopping = (num_pos_args >= early_stopping_index + 1 or
'early_stopping_rounds' in kwargs)
if early_stopping:
extra_step = len(eval_results)
try_mlflow_log(mlflow.log_metric, 'stopped_iteration', len(eval_results))
# best_iteration is set even if training does not stop early.
try_mlflow_log(mlflow.log_metric, 'best_iteration', model.best_iteration)
# iteration starts from 1 in LightGBM.
try_mlflow_log(mlflow.log_metrics, eval_results[model.best_iteration - 1],
step=extra_step)
# logging feature importance as artifacts.
for imp_type in ['split', 'gain']:
features = model.feature_name()
importance = model.feature_importance(importance_type=imp_type)
try:
log_feature_importance_plot(features, importance, imp_type)
except Exception: # pylint: disable=broad-except
_logger.exception('Failed to log feature importance plot. LightGBM autologging '
'will ignore the failure and continue. Exception: ')
imp = {ft: imp for ft, imp in zip(features, importance.tolist())}
tmpdir = tempfile.mkdtemp()
try:
filepath = os.path.join(tmpdir, 'feature_importance_{}.json'.format(imp_type))
with open(filepath, 'w') as f:
json.dump(imp, f, indent=2)
try_mlflow_log(mlflow.log_artifact, filepath)
finally:
shutil.rmtree(tmpdir)
try_mlflow_log(log_model, model, artifact_path='model')
if auto_end_run:
try_mlflow_log(mlflow.end_run)
return model
def _fit_keras_model_with_active_run(pandas_df, epochs):
run_id = mlflow.active_run().info.run_id
_fit_keras(pandas_df, epochs)
run_id = run_id
return mlflow.get_run(run_id)
from __future__ import print_function
import sys
import numpy as np
from sklearn.linear_model import LogisticRegression
import mlflow
import mlflow.sklearn
if __name__ == "__main__":
penalty = sys.argv[1] # either 'l1', 'l2', 'elasticnet', or 'none'
C = float(sys.argv[2]) if len(sys.argv) > 2 else 1.0
tol = float(sys.argv[3]) if len(sys.argv) > 3 else 1e-4
X = np.array([-2, -1, 0, 1, 2, 1]).reshape(-1, 1)
y = np.array([0, 0, 1, 1, 1, 0])
lr = LogisticRegression(penalty=penalty, C=C, tol=tol)
lr.fit(X, y)
score = lr.score(X, y)
mlflow.log_metric("score", score)
mlflow.sklearn.log_model(lr, "model")
print("Score: %s" % score)
print("Model saved in run %s" % mlflow.active_run().info.run_uuid)
def _fit_keras_model(pandas_df, epochs):
active_run = mlflow.active_run()
if active_run:
return _fit_keras_model_with_active_run(pandas_df, epochs)
else:
return _fit_keras_model_no_active_run(pandas_df, epochs)
