def __evaluate(confusion_matrix: tf.Tensor, auc: float, precision: np.ndarray, recall: np.ndarray) -> None: # Confusion Matrix fig, ax = plt.subplots(figsize=(16, 8)) sns.heatmap(confusion_matrix, annot=True, fmt='d', cmap=sns.color_palette("Blues"), ax=ax) ax.set_xlabel('Predicted') ax.set_ylabel('Ground Truth') mlflow.log_figure(fig, f'confusion_matrix.png') plt.close(fig) # AUC mlflow.log_metric('val_auc', auc) # Precision Recall fig, ax = plt.subplots(figsize=(16, 8)) sns.lineplot(x=recall, y=precision, ax=ax) ax.set_xlabel('Recall') ax.set_xlim(0., 1.) ax.set_ylabel('Precision') ax.set_ylim(0., 1.) mlflow.log_figure(fig, f'precision_recall.png') plt.close(fig)
def log_plots(exp_id, run_id, density, boxplot): # If model reside in MLflow runs if (exp_id and run_id) is not None: with mlflow.start_run(experiment_id=exp_id, run_id=run_id): print(f"\nCalculate and log model's residuals...") mlflow.log_figure(density, "./plots/eval_distplot.png") mlflow.log_figure(boxplot, "./plots/eval_boxplots.png")
def feature_conductance(test_input_tensor): ig = IntegratedGradients(net) test_input_tensor.requires_grad_() attr, _ = ig.attribute(test_input_tensor, target=1, return_convergence_delta=True) attr = attr.detach().numpy() # To understand these attributions, we can first average them across all the inputs # and print and visualize the average attribution for each feature. feature_imp, feature_imp_dict = visualize_importances(feature_names, np.mean(attr, axis=0)) mlflow.log_metrics(feature_imp_dict) mlflow.log_text(str(feature_imp), "feature_imp_summary.txt") fig, (ax1, ax2) = plt.subplots(2, 1) fig.tight_layout(pad=3) ax1.hist(attr[:, 1], 100) ax1.set(title="Distribution of Sibsp Attribution Values") # we can bucket the examples by the value of the sibsp feature and # plot the average attribution for the feature. # In the plot below, the size of the dot is proportional to # the number of examples with that value. bin_means, bin_edges, _ = stats.binned_statistic( test_features[:, 1], attr[:, 1], statistic="mean", bins=6 ) bin_count, _, _ = stats.binned_statistic( test_features[:, 1], attr[:, 1], statistic="count", bins=6 ) bin_width = bin_edges[1] - bin_edges[0] bin_centers = bin_edges[1:] - bin_width / 2 ax2.scatter(bin_centers, bin_means, s=bin_count) ax2.set(xlabel="Average Sibsp Feature Value", ylabel="Average Attribution") mlflow.log_figure(fig, "Average_Sibsp_Feature_Value.png")
def layer_conductance(net, test_input_tensor): """ To use Layer Conductance, we create a LayerConductance object passing in the model as well as the module (layer) whose output we would like to understand. In this case, we choose net.sigmoid1, the output of the first hidden layer. Now obtain the conductance values for all the test examples by calling attribute on the LayerConductance object. LayerConductance also requires a target index for networks with mutliple outputs, defining the index of the output for which gradients are computed. Similar to feature attributions, we provide target = 1, corresponding to survival. LayerConductance also utilizes a baseline, but we simply use the default zero baseline as in integrated gradients. """ cond = LayerConductance(net, net.sigmoid1) cond_vals = cond.attribute(test_input_tensor, target=1) cond_vals = cond_vals.detach().numpy() # We can begin by visualizing the average conductance for each neuron. neuron_names = ["neuron " + str(x) for x in range(12)] avg_neuron_imp, neuron_imp_dict = visualize_importances( neuron_names, np.mean(cond_vals, axis=0), title="Average Neuron Importances", axis_title="Neurons", ) mlflow.log_metrics(neuron_imp_dict) mlflow.log_text(str(avg_neuron_imp), "neuron_imp_summary.txt") # We can also look at the distribution of each neuron's attributions. Below we look at the distributions for neurons 7 and 9, # and we can confirm that their attribution distributions are very close to 0, suggesting they are not learning substantial features. fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(9, 6)) fig.tight_layout(pad=3) ax1.hist(cond_vals[:, 9], 100) ax1.set(title="Neuron 9 Distribution") ax2.hist(cond_vals[:, 7], 100) ax2.set(title="Neuron 7 Distribution") mlflow.log_figure(fig, "Neurons_Distribution.png")
def plotSamples(train): msg = """ ==================================================== Saving a sample in MLFlow to later visaully validate results... ==================================================== """ messages(msg) fig = plt.figure(figsize=(20, 12)) for i in range(0, 10): plt.subplot(2, 5, i + 1) for X_batch, Y_batch in train: image = X_batch[0] dic = {0: 'NORMAL', 1: 'PNEUMONIA'} plt.title(dic.get(Y_batch[0])) plt.axis('off') plt.imshow(np.squeeze(image), cmap='gray', interpolation='nearest') break plt.tight_layout() # plt.savefig("data/batch_images.pdf") # mlflow.log_artifact("data/batch_images.pdf","data") mlflow.log_figure(fig, "samples/sample.png") msg = """ ==================================================== done...move to next section ==================================================== """ messages(msg)
def log_model_residuals(model, X_train, y_train, X_val, y_val): tic = time.time() print(f"\nCalculate model's residuals and log them...") fig = plot_residuals_errors(model, X_train, y_train, X_val, y_val) mlflow.log_figure(fig, "./plots/residuals_errors.png") min, sec = divmod(time.time() - tic, 60) print(f"Calculating residuals took: {int(min)}min {int(sec)}sec")
def main(cfg: DictConfig) -> None: # read WP thresholds from mlflow artifacts path_to_mlflow = to_absolute_path(cfg['create_df']['path_to_mlflow']) mlflow.set_tracking_uri(f"file://{path_to_mlflow}") experiment_id = cfg['create_df']['experiment_id'] run_id = cfg['create_df']['run_id'] path_to_wp_definitions = f"{path_to_mlflow}/{experiment_id}/{run_id}/artifacts/working_points.json" with open(path_to_wp_definitions, 'r') as f: wp_definitions = json.load(f) # instantiate partial create_df object from hydra cfg OmegaConf.register_new_resolver("get_method", hydra.utils.get_method) partial_create_df = instantiate(cfg["create_df"]) partial_plot_efficiency = instantiate(cfg["var_cfg"]) # make dataframes with specified input features and predictions for each tau_type vs_type = cfg['vs_type'] df_sel = {} if not cfg['from_skims']: # create dataframes and log to mlflow with mlflow.start_run(experiment_id=experiment_id, run_id=run_id) as active_run: for tau_type in ['tau', vs_type]: df = partial_create_df( tau_type_to_select=tau_type, pred_samples=cfg['pred_samples'][tau_type]) df.to_csv(f'{tau_type}.csv') mlflow.log_artifact(f'{tau_type}.csv', cfg['output_skim_folder']) df_sel[tau_type] = df else: # read already existing skimmed dataframes for tau_type in ['tau', vs_type]: df = pd.read_csv( f"{path_to_mlflow}/{experiment_id}/{run_id}/artifacts/{cfg['output_skim_folder']}/{tau_type}.csv" ) df_sel[tau_type] = df # compute and plot efficiency curves wp_thrs, wp_names = list(wp_definitions[vs_type].values()), list( wp_definitions[vs_type].keys()) WPs_to_require = OmegaConf.to_object(cfg['WPs_to_require']) del WPs_to_require[vs_type] # remove vs_type key eff, eff_up, eff_down = differential_efficiency( df_sel['tau'], df_sel[vs_type], cfg['var_cfg']['var_name'], cfg['var_cfg']['var_bins'], vs_type, 'score_vs_', wp_thrs, cfg['require_WPs_in_numerator'], cfg['require_WPs_in_denominator'], WPs_to_require, wp_definitions) fig = partial_plot_efficiency(eff=eff, eff_up=eff_up, eff_down=eff_down, labels=wp_names) # log to mlflow with mlflow.start_run(experiment_id=experiment_id, run_id=run_id) as active_run: mlflow.log_figure(fig, cfg['output_filename']) print( f'\n[INFO] logged plots to artifacts for experiment ({experiment_id}), run ({run_id})\n' )
def _log_figures_with_mlflow(self, report) -> None: """Log figures with MLflow in the artifact folder.""" if report.feature_importance_figure is not None: mlflow.log_figure( report.feature_importance_figure, "figures/weight_plot.html" ) for key, figure in report.data_series_figures.items(): mlflow.log_figure(figure, f"figures/{key}.html") self.logger.info("Logged figures to MLflow.")
def __evaluate( weights: tff.learning.ModelWeights, client_states: Dict[int, ClientState], dataset: FederatedDataset, evaluation_fn: Callable[ [tff.learning.ModelWeights, List[tf.data.Dataset], List[ClientState]], Tuple[tf.Tensor, Dict[Text, tf.Tensor], Dict[Text, tf.Tensor]]] ) -> None: confusion_matrix, aggregated_metrics, client_metrics = evaluation_fn( weights, [dataset.data[client] for client in dataset.clients], [client_states[client] for client in dataset.clients]) # Confusion Matrix fig, ax = plt.subplots(figsize=(16, 8)) sns.heatmap(confusion_matrix, annot=True, fmt='d', cmap=sns.color_palette("Blues"), ax=ax) ax.set_xlabel('Predicted') ax.set_ylabel('Ground Truth') mlflow.log_figure(fig, f'confusion_matrix.png') plt.close(fig) # Precision Recall fig, ax = plt.subplots(figsize=(16, 8)) sns.lineplot(x=aggregated_metrics['recall'], y=aggregated_metrics['precision'], ax=ax) ax.set_xlabel('Recall') ax.set_xlim(0., 1.) ax.set_ylabel('Precision') ax.set_ylim(0., 1.) mlflow.log_figure(fig, f'precision_recall.png') plt.close(fig) # Client Metrics auc = metrics.SigmoidDecorator(tf.keras.metrics.AUC(curve='PR'), name='auc') accuracy = metrics.SigmoidDecorator(tf.keras.metrics.BinaryAccuracy(), name='accuracy') for client, metric in zip(client_states.keys(), iter(client_metrics)): tf.nest.map_structure(lambda v, t: v.assign(t), auc.variables, list(metric['auc'])) tf.nest.map_structure(lambda v, t: v.assign(t), accuracy.variables, list(metric['accuracy'])) mlflow.log_metric(f'client_{client}_val_auc', auc.result().numpy()) mlflow.log_metric(f'client_{client}_val_acc', accuracy.result().numpy())
def test_log_figure_plotly(subdir): from plotly import graph_objects as go filename = "figure.html" artifact_file = filename if subdir is None else posixpath.join(subdir, filename) fig = go.Figure(go.Scatter(x=[0, 1], y=[2, 3])) with mlflow.start_run(): mlflow.log_figure(fig, artifact_file) artifact_path = None if subdir is None else posixpath.normpath(subdir) artifact_uri = mlflow.get_artifact_uri(artifact_path) run_artifact_dir = local_file_uri_to_path(artifact_uri) assert os.listdir(run_artifact_dir) == [filename]
def test_log_figure_matplotlib(subdir): import matplotlib.pyplot as plt filename = "figure.png" artifact_file = filename if subdir is None else posixpath.join(subdir, filename) fig, ax = plt.subplots() ax.plot([0, 1], [2, 3]) with mlflow.start_run(): mlflow.log_figure(fig, artifact_file) plt.close(fig) artifact_path = None if subdir is None else posixpath.normpath(subdir) artifact_uri = mlflow.get_artifact_uri(artifact_path) run_artifact_dir = local_file_uri_to_path(artifact_uri) assert os.listdir(run_artifact_dir) == [filename]
def log_evaluated_results( eval_results, mlflow_tracking, fraction, n_splits, n_repeats, ): """Logs the genereted plot Args: plots (list of pandas.plot): genereted plots via Pandas df_names (list of strings): short names of dataset used model_name (string): short name of chosen model """ if mlflow_tracking: print(f"Log artifacts for model evaluated...") model_name = eval_results[0] plots = eval_results[1] scores = eval_results[2] df_names = eval_results[3] time_spent = eval_results[4] exp_id = mlflow_set_exp_id("Model:Choose") run_name = f"{model_name} : Best" with mlflow.start_run(experiment_id=exp_id, run_name=run_name): for i in range(2): fig = plots[i].get_figure() path = f"./plots/{model_name}_on_{df_names[i]}.png" mlflow.log_figure(fig, path) mlflow.log_params( { "time_spent": time_spent, "fraction": fraction, "cv_n_splits": n_splits, "cv_n_repeats": n_repeats, "random_state": rnd_state, } ) mlflow.log_metrics( { "score_on_train": scores[0], "score_on_val": scores[1], } )
def visualize_importances( feature_names, importances, title="Average Feature Importances", plot=True, axis_title="Features", ): feature_imp = PrettyTable(["feature_name", "importances"]) feature_imp_dict = {} for i in range(len(feature_names)): print(feature_names[i], ": ", "%.3f" % (importances[i])) feature_imp.add_row([feature_names[i], importances[i]]) feature_imp_dict[str(feature_names[i])] = importances[i] x_pos = np.arange(len(feature_names)) if plot: fig, ax = plt.subplots(figsize=(12, 6)) ax.bar(x_pos, importances, align="center") ax.set(title=title, xlabel=axis_title) ax.set_xticks(x_pos) ax.set_xticklabels(feature_names, rotation="vertical") mlflow.log_figure(fig, title + ".png") return feature_imp, feature_imp_dict
def main(): with open("data/featurized.pkl", "rb") as f: dataset = pickle.load(f) with open("params.yaml", "r") as f: cfg = yaml.safe_load(f) mlflow.set_tracking_uri("http://localhost:5000") with mlflow.start_run(): mlflow.sklearn.autolog() clf = LogisticRegression(**cfg["evaluate_model"]) clf.fit(dataset["train"]["X"], dataset["train"]["y"]) y_pred = clf.predict(dataset["test"]["X"]) report = classification_report( y_pred=y_pred, y_true=dataset["test"]["y"], output_dict=True, ) metrics = { "accuracy": report["accuracy"], "f1-score": report["macro avg"]["f1-score"], } fig = plot_metrics_per_class(report, labels=dataset["labels"]) mlflow.log_params(cfg["download_data"]) mlflow.log_metrics(metrics=metrics) mlflow.sklearn.eval_and_log_metrics( clf, X=dataset["test"]["X"], y_true=dataset["test"]["y"], prefix="test_" ) mlflow.log_figure(fig, artifact_file="metrics.png") with open("data/results.json", "w") as f: json.dump(obj=metrics, fp=f)
def test_log_figure_raises_error_for_unsupported_figure_object_type(): with mlflow.start_run(), pytest.raises(TypeError, match="Unsupported figure object type"): mlflow.log_figure("not_figure", "figure.png")
# setup figure ax = fig.add_subplot(5, 5, idx) xs,ys = score_df[param], score_df[metric] # scatter ax.scatter(xs, ys) # fit line z = np.polyfit(xs, ys, 1) p = np.poly1d(z) sample_xs = np.linspace(xs.min(), xs.max(), 100) ax.plot(sample_xs, p(sample_xs), linestyle='--', color='k') # label axes ax.set(xlabel=param.replace('_',' ').title(), ylabel=ylabel_map[metric]) # prepare for next round idx += 1 fig.tight_layout() mlflow.log_figure(fig, 'param_vs_metric.png') # visualization via pca from sklearn.decomposition import PCA pca = PCA(n_components=2, svd_solver='full') pca.fit(score_df.iloc[:,:5].T) # we're only using the parameters # - per param fig = plt.figure(figsize=[20,4]) idx = 1 for param in ['n_estimators','max_depth','n_bins','max_samples','max_features']: ax = fig.add_subplot(1, 5, idx) ax.scatter(pca.components_[0], pca.components_[1], c=score_df[param]) ax.set(xlabel='PC1', ylabel='PC2', title=param) idx += 1 fig.tight_layout() mlflow.log_figure(fig, 'pca_params.png')
# Define mlflow artifacts to log with the experiment run mlflow.log_metric("precision", metrics.precision(1.0)) mlflow.log_metric("recall", metrics.recall(1.0)) mlflow.log_metric("f1", metrics.fMeasure(1.0)) mlflow.spark.log_model(pipelineTrained, "turbine_anomalies") mlflow.set_tag("model", "gbt") # Add confusion matrix to the model labels = pipelineTrained.stages[2].labels fig = plt.figure() sn.heatmap(pd.DataFrame(metrics.confusionMatrix().toArray()), annot=True, fmt='g', xticklabels=labels, yticklabels=labels) plt.suptitle("Turbine Damage Prediction. F1={:.2f}".format(metrics.fMeasure(1.0)), fontsize = 18) plt.xlabel("Predicted Labels") plt.ylabel("True Labels") mlflow.log_figure(fig, "confusion_matrix.png") # needs mlflow version >=1.13.1 # COMMAND ---------- # MAGIC %md ## Saving our model to MLFLow registry # COMMAND ---------- # DBTITLE 1,Save our new model to the registry as a new version # Get the best model having the best metrics.AUROC from the registry that fits our search criteria best_models = mlflow.search_runs(filter_string='tags.model="gbt" and attributes.status = "FINISHED" and metrics.f1 > 0', order_by=['metrics.f1 DESC'], max_results=1) model_uri = best_models.iloc[0].artifact_uri print(f"Model is stored at '{model_uri}'.") # Register the model
import mlflow import matplotlib.pyplot as plt if __name__ == '__main__': fig, ax = plt.subplots() ax.plot([0, 1], [2, 3]) with mlflow.start_run(): mlflow.log_figure(fig, "figure.png")
def run(experiment_name: str, run_name: str, config: Config) -> None: mlflow.set_experiment(experiment_name) with mlflow.start_run(run_name=run_name): mlflow.log_params(config._asdict()) def reduce_fn(state, data): model = __model_fn(config.window_size, config.hidden_size, config.dropout) loss_fn = losses.WeightedBinaryCrossEntropy(config.pos_weight) optimizer = __optimizer_fn(config.learning_rate) input_spec = (tf.TensorSpec( (None, config.window_size, len(SENSORS)), dtype=tf.float32), tf.TensorSpec((None, 1), dtype=tf.float32)) train_step = tf.function(partial(__train_step, model=model, optimizer=optimizer, loss_fn=loss_fn), input_signature=input_spec) val_state = tf.zeros((2, 2), dtype=tf.int32) val_metrics = __metrics_fn() val_step = partial(__validation_step, model=model, metrics=val_metrics) for _ in range(1, config.epochs + 1): for X, y in data[0]: train_step(X, y) confusion_matrix = data[1].reduce(val_state, val_step) auc = val_metrics[0].result().numpy() precision = val_metrics[1].result().numpy() recall = val_metrics[2].result().numpy() return (state[0] + confusion_matrix, state[1] + [auc], state[2] + [precision], state[3] + [recall]) confusion_matrix, auc, precision, recall = reduce( reduce_fn, __load_data(config.path, config.window_size, config.batch_size), (tf.zeros((2, 2), dtype=tf.int32), [], [], [])) # Confusion matrix fig, ax = plt.subplots(figsize=(16, 8)) sns.heatmap(confusion_matrix, annot=True, fmt='d', cmap=sns.color_palette("Blues"), ax=ax) ax.set_xlabel('Predicted') ax.set_ylabel('Ground Truth') mlflow.log_figure(fig, 'confusion_matrix.png') plt.close(fig) # AUC mlflow.log_metric('val_auc', np.mean(auc)) # Precision Recall fig, ax = plt.subplots(figsize=(16, 8)) sns.lineplot(x=np.mean(recall, axis=0), y=np.mean(precision, axis=0), ax=ax) ax.set_xlabel('Recall') ax.set_xlim(0., 1.) ax.set_ylabel('Precision') ax.set_ylim(0., 1.) mlflow.log_figure(fig, 'precision_recall.png') plt.close(fig)
hparams = json.load(hfile) mlflow.set_tracking_uri(hparams['meta']["mlflow_uri"]) mlflow.set_experiment(hparams['meta']["name"]) with mlflow.start_run(run_name=hparams['meta']["method"] + "-" + hparams["meta"]["optimizer"]) as run: mlflow.log_dict(hparams, artifact_file="hparams/hparams.json") mlflow.log_text("", artifact_file="output/_touch.txt") artifact_uri = mlflow.get_artifact_uri("output/") hparams["meta"]["output_dir"] = artifact_uri print(f"URI: {artifact_uri}") start_time = datetime.now() if hparams["n_perturbs"] > 1: for perturb in range(hparams["n_perturbs"]): print(f"Running perturb {perturb}") continuation = ContinuationCreator( problem=problem, hparams=hparams, key=perturb).get_continuation_method() continuation.run() else: continuation = ContinuationCreator( problem=problem, hparams=hparams).get_continuation_method() continuation.run() end_time = datetime.now() print(f"Duration: {end_time-start_time}") figure = bif_plot(hparams["meta"]["output_dir"], pick_array) mlflow.log_figure(figure, artifact_file="plots/fig.png")
def plot_predictions( y, ypred, metrics, use_mlflow=False, pdf_path="prediction_plots.pdf", persistence=False, lead=None, unit="minutes", ): if use_mlflow: import mlflow elif pdf_path is not None: # Save plots into pdf instead from matplotlib.backends.backend_pdf import PdfPages pdf = PdfPages(pdf_path) # Use last metric if there are more than one if isinstance(metrics, (list, tuple)): if len(metrics) > 1: metric = metrics[-1] else: metric = metrics[0] else: metric = metrics y = y.squeeze() ypred = ypred.squeeze() if has_storm_index(y): fig = [] ax = [] # Plot predictions for each storm individually for storm in y.storms.level: storm_fig, storm_ax = plot_prediction( y, ypred, metric, storm=storm, persistence=persistence, lead=lead, unit=unit, ) fig.append(storm_fig) ax.append(storm_ax) if use_mlflow: mlflow.log_figure(fig, f"prediction_plots/storm_{storm}.png") elif pdf_path is not None: pdf.savefig(fig) else: fig, ax = plot_prediction(y, ypred, metric, persistence=persistence, lead=lead, unit=unit) if use_mlflow: mlflow.log_figure(fig, "prediction_plot.png") elif pdf_path is not None: pdf.savefig(fig) if not use_mlflow: pdf.close() return fig, ax
f'f1': model_scores['f1'], f'f1_micro': model_scores['f1_micro'], f'f1_macro': model_scores['f1_macro'], f'precision': model_scores['precision'], f'recall': model_scores['recall'], f'roc_auc': model_scores['roc_auc'] }) f1_timestep = calc_score_and_std_per_timestep( X_test_df, y_test, y_pred) fig, ax = plt.subplots() ax.plot(range(LAST_TRAIN_TIMESTEP + 1, LAST_TIMESTEP + 1), f1_timestep) ax.set_xlabel('timestep') ax.set_ylabel('f1') mlflow.log_figure(fig, f'f1_timestep_{m}.png') plot_confusion_matrix( y_test, y_pred, path= f'{PLOTS_ROOT}/{m}_{cv}_{c_weight}_confusion_matrix.png' ) plot_precision_recall_roc( y_test, y_prob, path=f'{PLOTS_ROOT}/{m}_{cv}_{c_weight}') mlflow.log_artifact( f'{PLOTS_ROOT}/{m}_{cv}_{c_weight}_confusion_matrix.png' ) mlflow.log_artifact(
for fold_n, (train_idx, test_idx) in enumerate(folds.split(X_train_all)): print(f'FOLD: {fold_n}') print(f'TRAIN: {train_idx}, TEST: {test_idx}') X_train, y_train = X_train_all[train_idx], y_train_all[train_idx] X_valid, y_valid = X_train_all[test_idx], y_train_all[test_idx] print(f'Train shape: {X_train.shape}') print(f'Val shape: {X_valid.shape}') assert X_train.shape[0] == y_train.shape[0] assert X_valid.shape[0] == y_valid.shape[0] fig, ax = plt.subplots() sns.countplot(y=y_train, label='train', color='slateblue') sns.countplot(y=y_valid, label='val', color='turquoise') ax.set_title('Train/validation labels count') plt.legend() mlflow.log_figure(fig, f'labels_count_fold_{fold_n}.png') mlflow.lightgbm.autolog() run_name = f'{EXPERIMENT_NAME}_fold_{fold_n}' with mlflow.start_run(run_name=run_name, nested=True): dtrain = lgb.Dataset( X_train, label=y_train ) # weight=get_unbalanced_weights(y_train, 0.3, 0.7)) dvalid = lgb.Dataset( X_valid, label=y_valid ) # weight=get_unbalanced_weights(y_valid, 0.3, 0.7)) res = {} clf = lgb.train(params, dtrain, valid_sets=[dtrain, dvalid], evals_result=res, early_stopping_rounds=50,
def train_model( model, X_dev, y_dev, X_train, y_train, X_val, y_val, mlflow_tracking=True, log_residuals=True, save_mlmodel_separatly=True, ): print(f"\nTrain final model on Development set...") tic = time.time() model_name = type(model).__name__ if mlflow_tracking: # Setup MLflow tracking server exp_id = mlflow_set_exp_id("Model:Train") run_name = f"{model_name}" ## Enable autologging mlflow.sklearn.autolog() ##* Fit model with MLflow logging with mlflow.start_run(experiment_id=exp_id, run_name=run_name) as run: run_id = run.info.run_id print(f"Active run_id: {run_id} ...\n") model = model.fit(X_dev, y_dev) toc = time.time() ## Disable autologging mlflow.sklearn.autolog(disable=True) ##* Log custom metrics mare_on_dev = mare(y_dev, model.predict(X_dev)) mare_on_train = mare(y_train, model.predict(X_train)) mare_on_val = mare(y_val, model.predict(X_val)) print(f"\nMARE on DEV: {mare_on_dev}") print(f"MARE on TRAIN: {mare_on_train}") print(f"MARE on VAL: {mare_on_val}") mlflow.log_metrics({ "mare_on_dev": mare_on_dev, "mare_on_train": mare_on_train, "mare_on_val": mare_on_val, }) ##* Log custom plots if log_residuals: print(f"\nCalculate and log model's residuals...") fig = plot_residuals_errors(model, X_train, y_train, X_val, y_val) mlflow.log_figure(fig, "./plots/residuals_errors.png") else: ##* Fit trivial model = model.fit(X_dev, y_dev) toc = time.time() exp_id, run_id = None, None ## Evaluate time spent min, sec = divmod(toc - tic, 60) print(f"Model training took: {int(min)}min {int(sec)}sec\n") ## Save trained pipeline if save_mlmodel_separatly: folder = save_mlmodel_aside(model, run_id) else: print(f"No one model was NOT saved separatly...") folder = None print(f"\nExperiment ID: {exp_id}") print(f"Run ID: {run_id}") print(f"Folder: {folder}") return exp_id, run_id, folder
def run(experiment_name: str, run_name: str, config: Config) -> None: mlflow.set_experiment(experiment_name) train, val, test = __load_data(config.path, config.window_size, config.batch_size) model = __model_fn(config.window_size, config.hidden_size, config.dropout) loss_fn = losses.WeightedBinaryCrossEntropy(config.pos_weight) optimizer = __optimizer_fn(config.learning_rate) input_spec = (tf.TensorSpec((None, config.window_size, len(SENSORS)), dtype=tf.float32), tf.TensorSpec((None, 1), dtype=tf.float32)) ckpt = tf.train.Checkpoint(model=model, optimizer=optimizer) ckpt_manager = tf.train.CheckpointManager(ckpt, config.output, max_to_keep=5) train_loss = tf.keras.metrics.Mean(name='loss') train_metrics = __training_metrics_fn() train_step = tf.function(partial(__train_step, model=model, optimizer=optimizer, loss_fn=loss_fn, loss=train_loss, metrics=train_metrics), input_signature=input_spec) val_loss = tf.keras.metrics.Mean(name='loss') val_metrics = __training_metrics_fn() val_step = tf.function(partial(__validation_step, model=model, loss_fn=loss_fn, loss=val_loss, metrics=val_metrics), input_signature=input_spec) eval_state = tf.zeros((2, 2), dtype=tf.int32) eval_metrics = __evaluation_metrics_fn() eval_step = partial(__evaluation_step, model=model, metrics=eval_metrics) with mlflow.start_run(run_name=run_name): mlflow.log_params(config._asdict()) # Fitting for epoch in range(1, config.epochs + 1): train_loss.reset_states() for metrics in train_metrics: metrics.reset_states() val_loss.reset_states() for metric in val_metrics: metric.reset_states() # Training for X, y in train: train_step(X, y) mlflow.log_metric(train_loss.name, train_loss.result().numpy(), step=epoch) mlflow.log_metrics( { metric.name: metric.result().numpy() for metric in train_metrics }, step=epoch) # Validation for X, y in val: val_step(X, y) mlflow.log_metric(f'val_{val_loss.name}', val_loss.result().numpy(), step=epoch) mlflow.log_metrics( { f'val_{metric.name}': metric.result().numpy() for metric in val_metrics }, step=epoch) # Checkpoint if epoch % config.checkpoint_rate == 0: ckpt_manager.save() # Evaluation def evaluate(confusion_matrix, client): # Reset PR-AUC and Accuracy metrics eval_metrics[0].reset_states() eval_metrics[3].reset_states() results = test[client].reduce(eval_state, eval_step) mlflow.log_metrics(f'client_{client}_val_auc', eval_metrics[0].result().numpy()) mlflow.log_metrics(f'client_{client}_val_acc', eval_metrics[3].result().numpy()) return confusion_matrix + results confusion_matrix = reduce(evaluate, test.clients, tf.zeros((2, 2), dtype=tf.int32)) # Confusion matrix fig, ax = plt.subplots(figsize=(16, 8)) sns.heatmap(confusion_matrix, annot=True, fmt='d', cmap=sns.color_palette("Blues"), ax=ax) ax.set_xlabel('Predicted') ax.set_ylabel('Ground Truth') mlflow.log_figure(fig, 'confusion_matrix.png') plt.close(fig) # Precision Recall fig, ax = plt.subplots(figsize=(16, 8)) sns.lineplot(x=eval_metrics[2].results().numpy(), y=eval_metrics[1].results().numpy(), ax=ax) ax.set_xlabel('Recall') ax.set_xlim(0., 1.) ax.set_ylabel('Precision') ax.set_ylim(0., 1.) mlflow.log_figure(fig, 'precision_recall.png') plt.close(fig)
f'f1': model_scores['f1'], f'f1_micro': model_scores['f1_micro'], f'f1_macro': model_scores['f1_macro'], f'precision': model_scores['precision'], f'recall': model_scores['recall'], f'roc_auc': model_scores['roc_auc'] }) f1_timestep = calc_score_and_std_per_timestep( X_test_df, y_test, y_pred) fig, ax = plt.subplots() ax.plot(range(LAST_TRAIN_TIMESTEP + 1, LAST_TIMESTEP + 1), f1_timestep) ax.set_xlabel('timestep') ax.set_ylabel('f1') mlflow.log_figure(fig, f'f1_timestep_{m}_automl.png') plot_confusion_matrix( y_test, y_pred, path=f'{PLOTS_ROOT}/automl_{m}_confusion_matrix.png') plot_precision_recall_roc(y_test, y_prob, path=f'{PLOTS_ROOT}/automl_{m}') mlflow.log_artifact( f'{PLOTS_ROOT}/automl_{m}_confusion_matrix.png') mlflow.log_artifact( f'{PLOTS_ROOT}/automl_{m}_precision_recall.png') mlflow.log_artifact(f'{PLOTS_ROOT}/automl_{m}_roc.png') plt.close('all')
def mlflow_log_figure(config: DictConfig, fig, target=None): figure_config = config["training"]["figure"] figure_name = (target + "_" + figure_config["name"] if target is not None else figure_config["name"]) mlflow.log_figure(fig, "fig/" + figure_name)