def mlflow_run(self, df, r_name="Lab-1:RF Petrol Regression Experiment"):
"""
This method trains, computes metrics, and logs all metrics, parameters,
and artifacts for the current run
:param df: pandas dataFrame
:param r_name: Name of the experiment as logged by MLflow
:return: MLflow Tuple (ExperimentID, runID)
"""
with mlflow.start_run(run_name=r_name) as run:
# get all rows and columns but the last column
X = dataset.iloc[:, 0:4].values
# get all the last columns, which is what we want to predict
y = dataset.iloc[:, 4].values
# create train and test data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
# Feature Scaling
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
# train and predict
self.rf.fit(X_train, y_train)
y_pred = self.rf.predict(X_test)
# Log model and params using the MLflow sklearn APIs
mlflow.sklearn.log_model(self.rf, "random-forest-reg-model")
mlflow.log_params(self.params)
# compute metrics
mae = metrics.mean_absolute_error(y_test, y_pred)
mse = metrics.mean_squared_error(y_test, y_pred)
rsme = np.sqrt(mse)
r2 = metrics.r2_score(y_test, y_pred)
# Log metrics
mlflow.log_metric("mae", mae)
mlflow.log_metric("mse", mse)
mlflow.log_metric("rsme", rsme)
mlflow.log_metric("r2", r2)
# update global class instance variable with values
self.rsme.append(rsme)
self.r2.append(r2)
self.estimators.append(params["n_estimators"])
# plot RSME graph and save as artifacts
(fig, ax) = plot_graphs(rfr.estimators, rfr.rsme, "Random Forest Estimators", "Root Mean Square", "Root Mean Square vs Estimators")
# get current run and experiment id
runID = run.info.run_uuid
experimentID = run.info.experiment_id
# create temporary artifact file name and log artifact
temp_file_name = get_temporary_directory_path("rsme_estimators-", ".png")
temp_name = temp_file_name.name
try:
fig.savefig(temp_name)
mlflow.log_artifact(temp_name, "rsme_estimators_plots")
finally:
temp_file_name.close() # Delete the temp file
# plot R2 graph and save as artifacts
(fig_2, ax) = plot_graphs(rfr.estimators, rfr.r2, "Random Forest Estimators", "R2", "R2 vs Estimators")
# create temporary artifact file name and log artifact
temp_file_name = get_temporary_directory_path("r2-estimators-", ".png")
temp_name = temp_file_name.name
try:
fig_2.savefig(temp_name)
mlflow.log_artifact(temp_name, "r2_estimators_plots")
finally:
temp_file_name.close() # Delete the temp file
# print some data
print("-" * 100)
print("Inside MLflow Run with run_id {} and experiment_id {}".format(runID, experimentID))
print("Estimator trees :", self.params["n_estimators"])
print('Mean Absolute Error :', mae)
print('Mean Squared Error :', mse)
print('Root Mean Squared Error:', rsme)
print('R2 :', r2)
return (experimentID, runID)
def mlflow_run(self,
df,
r_name="Lab-2:RF Bank Note Classification Experiment"):
"""
This method trains, computes metrics, and logs all metrics, parameters,
and artifacts for the current run
:param df: pandas dataFrame
:param r_name: Name of the experiment as logged by MLflow
:return: MLflow Tuple (ExperimentID, runID)
"""
with mlflow.start_run(run_name=r_name) as run:
# get all rows and columns but the last column, which is our class
X = df.iloc[:, 0:4].values
# get all observed values in the last columns, which is what we want to predict
y = df.iloc[:, 4].values
# create train and test data
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.2,
random_state=0)
# Feature Scaling
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
# train and predict
self.rf.fit(X_train, y_train)
y_pred = self.rf.predict(X_test)
# Log model and params using the MLflow sklearn APIs
mlflow.sklearn.log_model(self.rf, "random-forest-class-model")
mlflow.log_params(self.params)
# compute evaluation metrics
acc = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred)
conf_matrix = confusion_matrix(y_test, y_pred)
# get confusion matrix values
true_positive = conf_matrix[0][0]
true_negative = conf_matrix[1][1]
false_positive = conf_matrix[0][1]
false_negative = conf_matrix[1][0]
# get confusion matrix as a dictionary
class_report = classification_report(y_test,
y_pred,
output_dict=True)
recall_0 = class_report['0']['recall']
f1_score_0 = class_report['0']['f1-score']
recall_1 = class_report['1']['recall']
f1_score_1 = class_report['1']['f1-score']
# log metrics
mlflow.log_metric("accuracy_score", acc)
mlflow.log_metric("precision", precision)
mlflow.log_metric("true_positive", true_positive)
mlflow.log_metric("true_negative", true_negative)
mlflow.log_metric("false_positive", false_positive)
mlflow.log_metric("false_negative", false_negative)
mlflow.log_metric("recall_0", recall_0)
mlflow.log_metric("f1_score_0", f1_score_0)
mlflow.log_metric("recall_1", recall_1)
mlflow.log_metric("f1_score_1", f1_score_1)
# get current run and experiment id
runID = run.info.run_uuid
experimentID = run.info.experiment_id
# create confusion matrix images
(plt, fig, ax) = plot_confusion_matrix(
y_test,
y_pred,
y,
title="Bank Note Classification Confusion Matrix")
# create temporary artifact file name and log artifact
temp_file_name = get_temporary_directory_path(
"confusion_matrix-", ".png")
temp_name = temp_file_name.name
try:
fig.savefig(temp_name)
mlflow.log_artifact(temp_name, "confusion_matrix_plots")
finally:
temp_file_name.close() # Delete the temp file
# print some data
print("-" * 100)
print(
"Inside MLflow Run with run_id {} and experiment_id {}".format(
runID, experimentID))
print("Estimators trees:", self.params["n_estimators"])
print(conf_matrix)
print(classification_report(y_test, y_pred))
print("Accuracy Score:", acc)
print("Precision :", precision)
return (experimentID, runID)
def mlflow_run(self, df, r_name="Lab-4:RF Experiment Model"):
"""
Override the base class mlflow_run for this epxerimental runs
This method trains the model, evaluates, computes the metrics, logs
all the relevant metrics, artifacts, and models.
:param df: pandas dataFrame
:param r_name: name of the experiment run
:return: MLflow Tuple (ExperimentID, runID)
"""
with mlflow.start_run(run_name=r_name) as run:
X_train, X_test, y_train, y_test = train_test_split(
df.drop(["price"], axis=1),
df[["price"]].values.ravel(),
random_state=42)
self.rf.fit(X_train, y_train)
predictions = self.rf.predict(X_test)
# Log model and parameters
mlflow.sklearn.log_model(self.rf, "random-forest-model")
# Note we are logging as a dictionary of all params instead of logging each parameter
mlflow.log_params(self.params)
# Log params
#[mlflow.log_param(param, value) for param, value in self.params.items()]
# Create metrics
mse = metrics.mean_squared_error(y_test, predictions)
rmse = np.sqrt(mse)
mae = metrics.mean_absolute_error(y_test, predictions)
r2 = metrics.r2_score(y_test, predictions)
# Log metrics
mlflow.log_metric("mse", mse)
mlflow.log_metric("mae", mae)
mlflow.log_metric("rsme", rmse)
mlflow.log_metric("r2", r2)
# get experimentalID and runID
runID = run.info.run_uuid
experimentID = run.info.experiment_id
# Create feature importance and save them as artifact
# This allows us to remove least important features from the dataset
# with each iteration if they don't have any effect on the predictive power of
# the prediction.
importance = pd.DataFrame(
list(zip(df.columns, self.rf.feature_importances_)),
columns=["Feature", "Importance"]).sort_values("Importance",
ascending=False)
# Log importance file as feature artifact
temp_file_name = get_temporary_directory_path(
"feature-importance-", ".csv")
temp_name = temp_file_name.name
try:
importance.to_csv(temp_name, index=False)
mlflow.log_artifact(temp_name, "feature-importance-files")
finally:
temp_file_name.close() # Delete the temp file
# Create residual plots and image directory
# Residuals R = observed value - predicted value
(plt, fig,
ax) = plot_residual_graphs(predictions, y_test,
"Predicted values for Price ($)",
"Residual", "Residual Plot")
# Log residuals images
temp_file_name = get_temporary_directory_path("residuals-", ".png")
temp_name = temp_file_name.name
try:
fig.savefig(temp_name)
mlflow.log_artifact(temp_name, "residuals-plots")
finally:
temp_file_name.close() # Delete the temp file
print("-" * 100)
print("Inside MLflow {} Run with run_id {} and experiment_id {}".
format(r_name, runID, experimentID))
print(" mse: {}".format(mse))
print(" rmse: {}".format(rmse))
print(" mae: {}".format(mae))
print(" R2 : {}".format(r2))
return (experimentID, runID)
