def training(context: MLClientCtx, p1: int = 1, p2: int = 2) -> None: """Train a model. :param context: The runtime context object. :param p1: A model parameter. :param p2: Another model parameter. """ # access input metadata, values, and inputs print(f"Run: {context.name} (uid={context.uid})") print(f"Params: p1={p1}, p2={p2}") context.logger.info("started training") # <insert training code here> # log the run results (scalar values) context.log_result("accuracy", p1 * 2) context.log_result("loss", p1 * 3) # add a lable/tag to this run context.set_label("category", "tests") # log a simple artifact + label the artifact # If you want to upload a local file to the artifact repo add src_path=<local-path> context.log_artifact("somefile", body=b"abc is 123", local_path="myfile.txt") # create a dataframe artifact df = pd.DataFrame([{ "A": 10, "B": 100 }, { "A": 11, "B": 110 }, { "A": 12, "B": 120 }]) context.log_dataset("mydf", df=df) # Log an ML Model artifact, add metrics, params, and labels to it # and place it in a subdir ('models') under artifacts path context.log_model( "mymodel", body=b"abc is 123", model_file="model.txt", metrics={"accuracy": 0.85}, parameters={"xx": "abc"}, labels={"framework": "xgboost"}, artifact_path=context.artifact_subpath("models"), )
def fit(context: MLClientCtx, dataset: DataItem, num_boost_round: int = 10, evals: List[Tuple[DMatrix, str]] = [], obj: Union[Callable, str] = "", feval: Union[Callable, str] = None, maximize: bool = False, early_stopping_rounds: int = None, evals_result: dict = {}, verbose_eval: bool = True, xgb_model: DataItem = None, callbacks: List[Callable] = [], label_column: str = "labels", encode_cols: dict = {}, sample: int = -1, test_size: float = 0.25, valid_size: float = 0.75, random_state: int = 1994, models_dest: str = "models", plots_dest: str = "plots", file_ext: str = "csv", test_set_key: str = "test-set", gpus: bool = False) -> None: """low level xgboost train api for the xgboost `train` params see: https://xgboost.readthedocs.io/en/latest/python/python_api.html#xgboost.train Note: the first parameter of xgboost's `train` method is a dict of parameters supplied to the booster (engine). To modify one of those simply add a task parameter (when running you supply an mlrun NewTask) with the prefix "XGB_". So for example, to set the 'tree_method' parameter to 'approx', add {"XGB_tree_method":"approx"} to the task params key. :param context: the function context :param dataset: the full data set, train, valid and test will be extracted and each converted to a DMatrix for input to xgboost's `train` :param label_column: ground-truth (y) labels :param encode_cols: dictionary of names and prefixes for columns that are to hot be encoded. :param sample: Selects the first n rows, or select a sample starting from the first. If negative <-1, select a random sample :param test_size: (0.05) test set size :param valid_size: (0.75) Once the test set has been removed the training set gets this proportion. :param random_state: (1) sklearn rng seed :param models_dest: destination subfolder for model artifacts :param plots_dest: destination subfolder for plot artifacts :param file_ext: format for test_set_key hold out data :param test_set_key: (test-set), key of held out data in artifact store :param gpus: (False), run on gpus """ raw, labels, header = get_sample(dataset, sample, label_column) # hot-encode if encode_cols: raw = pd.get_dummies(raw, columns=list(encode_cols.keys()), prefix=list(encode_cols.values()), drop_first=True) # split the sample into train validate, test and calibration sets: (xtrain, ytrain), (xvalid, yvalid), (xtest, ytest) = \ get_splits(raw, labels, 3, test_size, valid_size, random_state) # save test data as regular dataframe as it may be used by other process context.log_dataset(test_set_key, df=pd.concat([xtest, ytest], axis=1), format=file_ext, index=False) # convert to xgboost DMatrix (todo - dask, gpu) dtrain = DMatrix(xtrain, label=ytrain) dvalid = DMatrix(xvalid, label=yvalid) boost_params = { "tree_method": "gpu_hist" if gpus else "hist", "seed": random_state, "disable_default_eval_metric": 1, "objective": "reg:squaredlogerror", "eval_metric": "rmsle" } # enable user to customize `booster param` parameters for k, v in context.parameters.items(): if k.startswith('XGB_'): boost_params[k[4:]] = v # collect learning curves / training history results = dict() booster = train( boost_params, dtrain=dtrain, num_boost_round=num_boost_round, evals=[(dtrain, "train"), (dvalid, "valid")], evals_result=results, obj=squared_log, feval=rmsle, maximize=maximize, early_stopping_rounds=early_stopping_rounds, verbose_eval=verbose_eval, # xgb_model=xgb_model, # callbacks: List[Callable] = [] ) context.log_model("model", body=dumps(booster), model_file="model.pkl", artifact_path='/User/artifacts/tttt') learning_curves(context, results)
def train_model( context: MLClientCtx, model_pkg_class: str, dataset: DataItem, label_column: str = "labels", encode_cols: List[str] = [], sample: int = -1, test_size: float = 0.30, train_val_split: float = 0.75, test_set_key: str = "test_set", model_evaluator=None, models_dest: str = "", plots_dest: str = "plots", file_ext: str = "parquet", model_pkg_file: str = "", random_state: int = 1, ) -> None: """train a classifier An optional cutom model evaluator can be supplied that should have the signature: `my_custom_evaluator(context, xvalid, yvalid, model)` and return a dictionary of scalar "results", a "plots" keys with a list of PlotArtifacts, and and "tables" key containing a returned list of TableArtifacts. :param context: the function context :param model_pkg_class: the model to train, e.g, "sklearn.neural_networks.MLPClassifier", or json model config :param dataset: ("data") name of raw data file :param label_column: ground-truth (y) labels :param encode_cols: dictionary of names and prefixes for columns that are to hot be encoded. :param sample: Selects the first n rows, or select a sample starting from the first. If negative <-1, select a random sample :param test_size: (0.05) test set size :param train_val_split: (0.75) Once the test set has been removed the training set gets this proportion. :param test_set_key: key of held out data in artifact store :param model_evaluator: (None) a custom model evaluator can be specified :param models_dest: ("") models subfolder on artifact path :param plots_dest: plot subfolder on artifact path :param file_ext: ("parquet") format for test_set_key hold out data :param random_state: (1) sklearn rng seed """ models_dest = models_dest or "model" raw, labels, header = get_sample(dataset, sample, label_column) if encode_cols: raw = pd.get_dummies(raw, columns=list(encode_cols.keys()), prefix=list(encode_cols.values()), drop_first=True) (xtrain, ytrain), (xvalid, yvalid), (xtest, ytest) = get_splits( raw, labels, 3, test_size, 1 - train_val_split, random_state) context.log_dataset(test_set_key, df=pd.concat([xtest, ytest.to_frame()], axis=1), format=file_ext, index=False, labels={"data-type": "held-out"}, artifact_path=context.artifact_subpath('data')) model_config = gen_sklearn_model(model_pkg_class, context.parameters.items()) model_config["FIT"].update({"X": xtrain, "y": ytrain.values}) ClassifierClass = create_class(model_config["META"]["class"]) model = ClassifierClass(**model_config["CLASS"]) model.fit(**model_config["FIT"]) artifact_path = context.artifact_subpath(models_dest) plots_path = context.artifact_subpath(models_dest, plots_dest) if model_evaluator: eval_metrics = model_evaluator(context, xvalid, yvalid, model, plots_artifact_path=plots_path) else: eval_metrics = eval_model_v2(context, xvalid, yvalid, model, plots_artifact_path=plots_path) context.set_label('class', model_pkg_class) context.log_model("model", body=dumps(model), artifact_path=artifact_path, extra_data=eval_metrics, model_file="model.pkl", metrics=context.results, labels={"class": model_pkg_class})
def train_model(context: MLClientCtx, dataset: DataItem, model_pkg_class: str, label_column: str = "label", train_validation_size: float = 0.75, sample: float = 1.0, models_dest: str = "models", test_set_key: str = "test_set", plots_dest: str = "plots", dask_key: str = "dask_key", dask_persist: bool = False, scheduler_key: str = '', file_ext: str = "parquet", random_state: int = 42) -> None: """ Train a sklearn classifier with Dask :param context: Function context. :param dataset: Raw data file. :param model_pkg_class: Model to train, e.g, "sklearn.ensemble.RandomForestClassifier", or json model config. :param label_column: (label) Ground-truth y labels. :param train_validation_size: (0.75) Train validation set proportion out of the full dataset. :param sample: (1.0) Select sample from dataset (n-rows/% of total), randomzie rows as default. :param models_dest: (models) Models subfolder on artifact path. :param test_set_key: (test_set) Mlrun db key of held out data in artifact store. :param plots_dest: (plots) Plot subfolder on artifact path. :param dask_key: (dask key) Key of dataframe in dask client "datasets" attribute. :param dask_persist: (False) Should the data be persisted (through the `client.persist`) :param scheduler_key: (scheduler) Dask scheduler configuration, json also logged as an artifact. :param file_ext: (parquet) format for test_set_key hold out data :param random_state: (42) sklearn seed """ if scheduler_key: client = Client(scheduler_key) else: client = Client() context.logger.info("Read Data") df = dataset.as_df(df_module=dd) context.logger.info("Prep Data") numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64'] df = df.select_dtypes(include=numerics) if df.isna().any().any().compute() == True: raise Exception('NAs valus found') df_header = df.columns df = df.sample(frac=sample).reset_index(drop=True) encoder = LabelEncoder() encoder = encoder.fit(df[label_column]) X = df.drop(label_column, axis=1).to_dask_array(lengths=True) y = encoder.transform(df[label_column]) classes = df[label_column].drop_duplicates() # no unique values in dask classes = [str(i) for i in classes] context.logger.info("Split and Train") X_train, X_test, y_train, y_test = model_selection.train_test_split( X, y, train_size=train_validation_size, random_state=random_state) scaler = StandardScaler() scaler = scaler.fit(X_train) X_train_transformed = scaler.transform(X_train) X_test_transformed = scaler.transform(X_test) model_config = gen_sklearn_model(model_pkg_class, context.parameters.items()) model_config["FIT"].update({"X": X_train_transformed, "y": y_train}) ClassifierClass = create_class(model_config["META"]["class"]) model = ClassifierClass(**model_config["CLASS"]) with joblib.parallel_backend("dask"): model = model.fit(**model_config["FIT"]) artifact_path = context.artifact_subpath(models_dest) plots_path = context.artifact_subpath(models_dest, plots_dest) context.logger.info("Evaluate") extra_data_dict = {} for report in (ROCAUC, ClassificationReport, ConfusionMatrix): report_name = str(report.__name__) plt.cla() plt.clf() plt.close() viz = report(model, classes=classes, per_class=True, is_fitted=True) viz.fit(X_train_transformed, y_train) # Fit the training data to the visualizer viz.score(X_test_transformed, y_test.compute()) # Evaluate the model on the test data plot = context.log_artifact(PlotArtifact(report_name, body=viz.fig, title=report_name), db_key=False) extra_data_dict[str(report)] = plot if report_name == 'ROCAUC': context.log_results({ "micro": viz.roc_auc.get("micro"), "macro": viz.roc_auc.get("macro") }) elif report_name == 'ClassificationReport': for score_name in viz.scores_: for score_class in viz.scores_[score_name]: context.log_results({ score_name + "-" + score_class: viz.scores_[score_name].get(score_class) }) viz = FeatureImportances(model, classes=classes, per_class=True, is_fitted=True, labels=df_header.delete( df_header.get_loc(label_column))) viz.fit(X_train_transformed, y_train) viz.score(X_test_transformed, y_test) plot = context.log_artifact(PlotArtifact("FeatureImportances", body=viz.fig, title="FeatureImportances"), db_key=False) extra_data_dict[str("FeatureImportances")] = plot plt.cla() plt.clf() plt.close() context.logger.info("Log artifacts") artifact_path = context.artifact_subpath(models_dest) plots_path = context.artifact_subpath(models_dest, plots_dest) context.set_label('class', model_pkg_class) context.log_model("model", body=dumps(model), artifact_path=artifact_path, model_file="model.pkl", extra_data=extra_data_dict, metrics=context.results, labels={"class": model_pkg_class}) context.log_artifact("standard_scaler", body=dumps(scaler), artifact_path=artifact_path, model_file="scaler.gz", label="standard_scaler") context.log_artifact("label_encoder", body=dumps(encoder), artifact_path=artifact_path, model_file="encoder.gz", label="label_encoder") df_to_save = delayed(np.column_stack)((X_test, y_test)).compute() context.log_dataset( test_set_key, df=pd.DataFrame(df_to_save, columns=df_header), # improve log dataset ability format=file_ext, index=False, labels={"data-type": "held-out"}, artifact_path=context.artifact_subpath('data')) context.logger.info("Done!")
def data_clean(context: MLClientCtx, src: DataItem, file_ext: str = "csv", models_dest: str = "models/encoders", cleaned_key: str = "cleaned-data", encoded_key: str = "encoded-data"): df = src.as_df() # drop columns drop_cols_list = ["customerID", "TotalCharges"] df.drop(drop_cols_list, axis=1, inplace=True) # header transformations old_cols = df.columns rename_cols_map = { "SeniorCitizen": "senior", "Partner": "partner", "Dependents": "deps", "Churn": "labels" } df.rename(rename_cols_map, axis=1, inplace=True) # add drop column to logs: for col in drop_cols_list: rename_cols_map.update({col: "_DROPPED_"}) # log the op tp = os.path.join(models_dest, "preproc-column_map.json") context.log_artifact("preproc-column_map.json", body=json.dumps(rename_cols_map), local_path=tp) df = df.applymap(lambda x: "No" if str(x).startswith("No ") else x) # encode numerical type as category bins (ordinal) bins = [0, 12, 24, 36, 48, 60, np.inf] labels = [0, 1, 2, 3, 4, 5] tenure = df.tenure.copy(deep=True) df["tenure_map"] = pd.cut(df.tenure, bins, labels=False) tenure_map = dict(zip(bins, labels)) # save this transformation tp = os.path.join(models_dest, "preproc-numcat_map.json") context.log_artifact("preproc-numcat_map.json", body=bytes(json.dumps(tenure_map).encode("utf-8")), local_path=tp) context.log_dataset(cleaned_key, df=df, format=file_ext, index=False) fix_cols = [ "gender", "partner", "deps", "OnlineSecurity", "OnlineBackup", "DeviceProtection", "TechSupport", "StreamingTV", "StreamingMovies", "PhoneService", "MultipleLines", "PaperlessBilling", "InternetService", "Contract", "PaymentMethod", "labels" ] d = defaultdict(LabelEncoder) df[fix_cols] = df[fix_cols].apply( lambda x: d[x.name].fit_transform(x.astype(str))) context.log_dataset(encoded_key, df=df, format=file_ext, index=False) model_bin = dumps(d) context.log_model("model", body=model_bin, artifact_path=os.path.join(context.artifact_path, models_dest), model_file="model.pkl")
def data_clean( context: MLClientCtx, src: DataItem, file_ext: str = "csv", models_dest: str = "models/encoders", cleaned_key: str = "cleaned-data", encoded_key: str = "encoded-data", ): """process a raw churn data file Data has 3 states here: `raw`, `cleaned` and `encoded` * `raw` kept by default, the pipeline begins with a raw data artifact * `cleaned` kept for charts, presentations * `encoded` is input for a cross validation and training function steps (not necessarily in correct order, some parallel) * column name maps * deal with nans and other types of missings/junk * label encode binary and ordinal category columns * create category ranges from numerical columns And finally, * test Why we don't one-hot-encode here? One hot encoding isn't a necessary step for all algorithms. It can also generate a very large feature matrix that doesn't need to be serialized (even if sparse). So we leave one-hot-encoding for the training step. What about scaling numerical columns? Same as why we don't one hot encode here. Do we scale before train-test split? IMHO, no. Scaling before splitting introduces a type of data leakage. In addition, many estimators are completely immune to the monotonic transformations implied by scaling, so why waste the cycles? TODO: * parallelize where possible * more abstraction (more parameters, chain sklearn transformers) * convert to marketplace function :param context: the function execution context :param src: an artifact or file path :param file_ext: file type for artifacts :param models_dest: label encoders and other preprocessing steps should be saved together with other pipeline models :param cleaned_key: key of cleaned data table in artifact store :param encoded_key: key of encoded data table in artifact store """ df = src.as_df() # drop columns drop_cols_list = ["customerID", "TotalCharges"] df.drop(drop_cols_list, axis=1, inplace=True) # header transformations rename_cols_map = { "SeniorCitizen": "senior", "Partner": "partner", "Dependents": "deps", "Churn": "labels", } df.rename(rename_cols_map, axis=1, inplace=True) # add drop column to logs: for col in drop_cols_list: rename_cols_map.update({col: "_DROPPED_"}) # log the op tp = os.path.join(models_dest, "preproc-column_map.json") context.log_artifact("preproc-column_map.json", body=json.dumps(rename_cols_map), local_path=tp) # VALUE transformations # clean # truncate reply to "No" df = df.applymap(lambda x: "No" if str(x).startswith("No ") else x) # encode numerical type as category bins (ordinal) bins = [0, 12, 24, 36, 48, 60, np.inf] labels = [0, 1, 2, 3, 4, 5] df["tenure_map"] = pd.cut(df.tenure, bins, labels=False) tenure_map = dict(zip(bins, labels)) # save this transformation tp = os.path.join(models_dest, "preproc-numcat_map.json") context.log_artifact( "preproc-numcat_map.json", body=bytes(json.dumps(tenure_map).encode("utf-8")), local_path=tp, ) context.log_dataset(cleaned_key, df=df, format=file_ext, index=False) # label encoding - generate model for each column saved in dict # some of these columns may be hot encoded in the training step fix_cols = [ "gender", "partner", "deps", "OnlineSecurity", "OnlineBackup", "DeviceProtection", "TechSupport", "StreamingTV", "StreamingMovies", "PhoneService", "MultipleLines", "PaperlessBilling", "InternetService", "Contract", "PaymentMethod", "labels", ] d = defaultdict(LabelEncoder) df[fix_cols] = df[fix_cols].apply( lambda x: d[x.name].fit_transform(x.astype(str))) context.log_dataset(encoded_key, df=df, format=file_ext, index=False) model_bin = dumps(d) context.log_model( "model", body=model_bin, artifact_path=os.path.join(context.artifact_path, models_dest), model_file="model.pkl", )
def train_model( context: MLClientCtx, model_type: str, dataset: Union[DataItem, pd.core.frame.DataFrame], label_column: str = "labels", encode_cols: dict = {}, sample: int = -1, imbal_vec=[], test_size: float = 0.25, valid_size: float = 0.75, random_state: int = 1, models_dest: str = "models", plots_dest: str = "plots", eval_metrics: list = ["error", "auc"], file_ext: str = "parquet", test_set: str = "test_set", ) -> None: """train an xgboost model. Note on imabalanced data: the `imbal_vec` parameter represents the measured class representations in the sample and can be used as a first step in tuning an XGBoost model. This isn't a hyperparamter, merely an estimate that should be set as 'constant' throughout tuning process. :param context: the function context :param model_type: the model type to train, "classifier", "regressor"... :param dataset: ("data") name of raw data file :param label_column: ground-truth (y) labels :param encode_cols: dictionary of names and prefixes for columns that are to hot be encoded. :param sample: Selects the first n rows, or select a sample starting from the first. If negative <-1, select a random sample :param imbal_vec: ([]) vector of class weights seen in sample :param test_size: (0.05) test set size :param valid_size: (0.75) Once the test set has been removed the training set gets this proportion. :param random_state: (1) sklearn rng seed :param models_dest: destination subfolder for model artifacts :param plots_dest: destination subfolder for plot artifacts :param eval_metrics: (["error", "auc"]) learning curve metrics :param file_ext: format for test_set_key hold out data :param test-set: (test_set) key of held out data in artifact store """ models_dest = models_dest or "models" plots_dest = plots_dest or f"plots/{context.name}" raw, labels, header = get_sample(dataset, sample, label_column) if encode_cols: raw = pd.get_dummies( raw, columns=list(encode_cols.keys()), prefix=list(encode_cols.values()), drop_first=True, ) (xtrain, ytrain), (xvalid, yvalid), (xtest, ytest) = get_splits( raw, labels, 3, test_size, valid_size, random_state) context.log_dataset(test_set, df=pd.concat([xtest, ytest], axis=1), format=file_ext, index=False) model_config = _gen_xgb_model(model_type, context.parameters.items()) XGBBoostClass = create_class(model_config["META"]["class"]) model = XGBBoostClass(**model_config["CLASS"]) model_config["FIT"].update({ "X": xtrain, "y": ytrain.values, "eval_set": [(xtrain, ytrain), (xvalid, yvalid)], "eval_metric": eval_metrics, }) model.fit(**model_config["FIT"]) eval_metrics = eval_model_v2(context, xvalid, yvalid, model) model_bin = dumps(model) context.log_model( "model", body=model_bin, artifact_path=os.path.join(context.artifact_path, models_dest), model_file="model.pkl", )