def test_enough_time_to_train(self): model_type = "Xgboost" model_type_2 = "LightGBM" model_type = "Xgboost" automl = AutoML( results_path=self.automl_dir, total_time_limit=10, model_time_limit=10, algorithms=[model_type, model_type_2], ) automl._time_ctrl = TimeController( time.time(), 10, 10, ["simple_algorithms", "not_so_random"], [model_type, model_type_2], ) for i in range(5): automl._time_ctrl.log_time(f"Xgboost_{i}", model_type, "not_so_random", 1) # should be always true self.assertTrue( automl._time_ctrl.enough_time(model_type, "not_so_random")) for i in range(5): automl._time_ctrl.log_time(f"LightGBM_{i}", model_type_2, "not_so_random", 1) # should be always true self.assertTrue( automl._time_ctrl.enough_time(model_type_2, "not_so_random"))
def load_progress(self): state = {} fname = os.path.join(self._results_path, "progress.json") if not os.path.exists(fname): return state = json.load(open(fname, "r")) self._fit_level = state.get("fit_level", self._fit_level) self._all_params = state.get("all_params", self._all_params) self._time_ctrl = TimeController.from_json(state.get("time_controller"))
def test_set_model_time_limit(self): model_type = "Xgboost" automl = AutoML(results_path=self.automl_dir, model_time_limit=10, algorithms=[model_type]) automl._time_ctrl = TimeController( time.time(), None, 10, ["simple_algorithms", "not_so_random"], "Xgboost") for i in range(12): automl._time_ctrl.log_time(f"Xgboost_{i}", model_type, "not_so_random", 10) # should be always true self.assertTrue( automl._time_ctrl.enough_time(model_type, "not_so_random"))
def test_set_total_time_limit(self): model_type = "Xgboost" automl = AutoML(results_path=self.automl_dir, total_time_limit=100, algorithms=[model_type]) automl._time_ctrl = TimeController( time.time(), 100, None, ["simple_algorithms", "not_so_random"], "Xgboost") time_spend = 0 for i in range(12): automl._start_time -= 10 automl._time_ctrl.log_time(f"Xgboost_{i}", model_type, "not_so_random", 10) if automl._time_ctrl.enough_time(model_type, "not_so_random"): time_spend += 10 self.assertTrue(time_spend < 100)
def test_to_and_from_json(self): tc = TimeController( start_time=time.time(), total_time_limit=10, model_time_limit=None, steps=["simple_algorithms"], algorithms=["Baseline"], ) tc.log_time("1_Baseline", "Baseline", "simple_algorithms", 123.1) tc2 = TimeController.from_json(tc.to_json()) assert_almost_equal(tc2.step_spend("simple_algorithms"), 123.1) assert_almost_equal(tc2.model_spend("Baseline"), 123.1)
def _fit(self, X, y): """Fits the AutoML model with data""" if self._fit_level == "finished": print( "This model has already been fitted. You can use predict methods or select a new 'results_path' for a new a 'fit()'." ) return # Validate input and build dataframes X, y = self._build_dataframe(X, y) self.n_features_in_ = X.shape[1] self.n_classes = len(np.unique(y[~pd.isnull(y)])) # Get attributes (__init__ params) self._mode = self._get_mode() self._ml_task = self._get_ml_task() self._results_path = self._get_results_path() self._total_time_limit = self._get_total_time_limit() self._model_time_limit = self._get_model_time_limit() self._algorithms = self._get_algorithms() self._train_ensemble = self._get_train_ensemble() self._stack_models = self._get_stack_models() self._eval_metric = self._get_eval_metric() self._validation_strategy = self._get_validation_strategy() self._verbose = self._get_verbose() self._explain_level = self._get_explain_level() self._golden_features = self._get_golden_features() self._feature_selection = self._get_feature_selection() self._start_random_models = self._get_start_random_models() self._hill_climbing_steps = self._get_hill_climbing_steps() self._top_models_to_improve = self._get_top_models_to_improve() self._random_state = self._get_random_state() try: self.load_progress() if self._fit_level == "finished": print( "This model has already been fitted. You can use predict methods or select a new 'results_path' for a new 'fit()'." ) return self._check_can_load() self.verbose_print(f"AutoML directory: {self._results_path}") self.verbose_print( f"The task is {self._ml_task} with evaluation metric {self._eval_metric}" ) self.verbose_print( f"AutoML will use algorithms: {self._algorithms}") if self._stack_models: self.verbose_print("AutoML will stack models") if self._train_ensemble: self.verbose_print("AutoML will ensemble availabe models") self._start_time = time.time() if self._time_ctrl is not None: self._start_time -= self._time_ctrl.already_spend() # Automatic Exloratory Data Analysis if self._explain_level == 2: EDA.compute(X, y, os.path.join(self._results_path, "EDA")) # Save data self._save_data(X.copy(deep=False), y) tuner = MljarTuner( self._get_tuner_params( self._start_random_models, self._hill_climbing_steps, self._top_models_to_improve, ), self._algorithms, self._ml_task, self._validation_strategy, self._explain_level, self._data_info, self._golden_features, self._feature_selection, self._train_ensemble, self._stack_models, self._random_state, ) self.tuner = tuner steps = tuner.steps() self.verbose_print(f"AutoML steps: {steps}") if self._time_ctrl is None: self._time_ctrl = TimeController( self._start_time, self._total_time_limit, self._model_time_limit, steps, self._algorithms, ) self._time_ctrl.log_time( "prepare_data", "prepare_data", "prepare_data", time.time() - self._start_time, ) for step in steps: self._fit_level = step start = time.time() # self._time_start[step] = start if step == "stack": self.prepare_for_stacking() generated_params = [] if step in self._all_params: generated_params = self._all_params[step] else: generated_params = tuner.generate_params( step, self._models, self._results_path, self._stacked_models) if generated_params is None or not generated_params: self.verbose_print( f"Skip {step} because no parameters were generated.") continue if generated_params: if "learner" in generated_params[ 0] and not self._time_ctrl.enough_time( generated_params[0]["learner"]["model_type"], self._fit_level): self.verbose_print( f"Skip {step} because of the time limit.") else: model_str = "models" if len( generated_params) > 1 else "model" self.verbose_print( f"* Step {step} will try to check up to {len(generated_params)} {model_str}" ) for params in generated_params: if params.get("status", "") in ["trained", "skipped", "error"]: self.verbose_print( f"{params['name']}: {params['status']}.") continue try: trained = False if "ensemble" in step: trained = self.ensemble_step( is_stacked=params["is_stacked"]) else: trained = self.train_model(params) params["status"] = "trained" if trained else "skipped" params["final_loss"] = self._models[-1].get_final_loss( ) params["train_time"] = self._models[-1].get_train_time( ) except Exception as e: self._update_errors_report(params.get("name"), str(e)) params["status"] = "error" self.save_progress(step, generated_params) self._fit_level = "finished" self.save_progress() self.verbose_print( f"AutoML fit time: {np.round(time.time() - self._start_time,2)} seconds" ) except Exception as e: raise e finally: if self._X_path is not None: self._load_data_variables(X) return self
class BaseAutoML(BaseEstimator, ABC): """ Automated Machine Learning for supervised tasks (binary classification, multiclass classification, regression). Warning: This class should not be used directly. Use derived classes instead. """ def __init__(self): logger.debug("BaseAutoML.__init__") self._results_path = None self._models = [ ] # instances of iterative learner framework or ensemble self._best_model = None self._verbose = True self._threshold = None # used only in classification self._metrics_details = None self._max_metrics = None self._confusion_matrix = None self._X_path, self._y_path = None, None self._data_info = None self._model_paths = [] self._stacked_models = None self._fit_level = None self._start_time = time.time() self._time_ctrl = None self._all_params = {} # https://scikit-learn.org/stable/developers/develop.html#universal-attributes self.n_features_in_ = None # for scikit-learn api def _get_tuner_params(self, start_random_models, hill_climbing_steps, top_models_to_improve): return { "start_random_models": start_random_models, "hill_climbing_steps": hill_climbing_steps, "top_models_to_improve": top_models_to_improve, } def _check_can_load(self): """ Checks if AutoML can be loaded from a folder""" if self.results_path is not None: # Dir exists and can be loaded if os.path.exists(self.results_path) and os.path.exists( os.path.join(self.results_path, "params.json")): self.load(self.results_path) self._results_path = self.results_path def load(self, path): logger.info("Loading AutoML models ...") try: params = json.load(open(os.path.join(path, "params.json"))) self._model_paths = params["saved"] self._ml_task = params["ml_task"] self._eval_metric = params["eval_metric"] stacked_models = params.get("stacked") models_map = {} for model_path in self._model_paths: if model_path.endswith("Ensemble") or model_path.endswith( "Ensemble_Stacked"): ens = Ensemble.load(model_path, models_map) self._models += [ens] models_map[ens.get_name()] = ens else: m = ModelFramework.load(model_path) self._models += [m] models_map[m.get_name()] = m if stacked_models is not None: self._stacked_models = [] for stacked_model_name in stacked_models: self._stacked_models += [models_map[stacked_model_name]] best_model_name = None with open(os.path.join(path, "best_model.txt"), "r") as fin: best_model_name = fin.read() self._best_model = models_map[best_model_name] data_info_path = os.path.join(path, "data_info.json") self._data_info = json.load(open(data_info_path)) self.n_features_in_ = self._data_info["n_features"] if "n_classes" in self._data_info: self.n_classes = self._data_info["n_classes"] self._fit_level = "finished" except Exception as e: raise AutoMLException(f"Cannot load AutoML directory. {str(e)}") def get_leaderboard(self): ldb = { "name": [], "model_type": [], "metric_type": [], "metric_value": [], "train_time": [], } for m in self._models: ldb["name"] += [m.get_name()] ldb["model_type"] += [m.get_type()] ldb["metric_type"] += [self._eval_metric] ldb["metric_value"] += [m.get_final_loss()] ldb["train_time"] += [np.round(m.get_train_time(), 2)] return pd.DataFrame(ldb) def keep_model(self, model, model_path): if model is None: return self._models += [model] self._model_paths += [model_path] self.select_and_save_best() self.verbose_print("{} {} {} trained in {} seconds".format( model.get_name(), self._eval_metric, np.round(model.get_final_loss(), 6), np.round(model.get_train_time(), 2), )) self._time_ctrl.log_time(model.get_name(), model.get_type(), self._fit_level, model.get_train_time()) def create_dir(self, model_path): if not os.path.exists(model_path): try: os.mkdir(model_path) except Exception as e: raise AutoMLException( f"Cannot create directory {model_path}. {str(e)}") def train_model(self, params): # do we have enough time to train? # if not, skip if not self._time_ctrl.enough_time(params["learner"]["model_type"], self._fit_level): logger.info( f"Cannot train {params['name']} because of the time constraint" ) return False # let's create directory to log all training artifacts model_path = os.path.join(self._results_path, params["name"]) self.create_dir(model_path) # prepare callbacks early_stop = EarlyStopping({ "metric": { "name": self._eval_metric }, "log_to_dir": model_path }) learner_time_constraint = LearnerTimeConstraint({ "learner_time_limit": self._time_ctrl.learner_time_limit( params["learner"]["model_type"], self._fit_level, self._validation_strategy.get("k_folds", 1.0), ), "min_steps": params["additional"].get("min_steps"), }) total_time_constraint = TotalTimeConstraint({ "total_time_limit": self._total_time_limit if self._model_time_limit is None else None, "total_time_start": self._start_time, }) # create model framework mf = ModelFramework( params, callbacks=[ early_stop, learner_time_constraint, total_time_constraint ], ) # start training logger.info( f"Train model #{len(self._models)+1} / Model name: {params['name']}" ) mf.train(model_path) # save the model mf.save(model_path) # and keep info about the model self.keep_model(mf, model_path) return True def verbose_print(self, msg): if self._verbose > 0: # self._progress_bar.write(msg) print(msg) def ensemble_step(self, is_stacked=False): if self._train_ensemble and len(self._models) > 1: ensemble_path = os.path.join( self._results_path, "Ensemble_Stacked" if is_stacked else "Ensemble") self.create_dir(ensemble_path) self.ensemble = Ensemble(self._eval_metric, self._ml_task, is_stacked=is_stacked) oofs, target = self.ensemble.get_oof_matrix(self._models) self.ensemble.fit(oofs, target) self.ensemble.save(ensemble_path) self.keep_model(self.ensemble, ensemble_path) return True return False def can_we_stack_them(self, y): # if multiclass and too many classes then No return True def get_stacked_data(self, X, mode="training"): # mode can be `training` or `predict` if self._stacked_models is None: return X all_oofs = [] for m in self._stacked_models: oof = None if mode == "training": oof = m.get_out_of_folds() else: oof = m.predict(X) if self._ml_task == BINARY_CLASSIFICATION: cols = [f for f in oof.columns if "prediction" in f] if len(cols) == 2: oof = pd.DataFrame({"prediction": oof[cols[1]]}) cols = [f for f in oof.columns if "prediction" in f] oof = oof[cols] oof.columns = [f"{m.get_name()}_{c}" for c in cols] all_oofs += [oof] org_index = X.index.copy() X.reset_index(drop=True, inplace=True) X_stacked = pd.concat(all_oofs + [X], axis=1) X_stacked.index = org_index.copy() X.index = org_index.copy() return X_stacked def _perform_model_stacking(self): if self._stacked_models is not None: return ldb = self.get_leaderboard() ldb = ldb.sort_values(by="metric_value", ascending=True) models_map = { m.get_name(): m for m in self._models if not m._is_stacked } self._stacked_models = [] models_limit = 10 for model_type in np.unique(ldb.model_type): if model_type in ["Baseline"]: continue ds = ldb[ldb.model_type == model_type].copy() ds.sort_values(by="metric_value", inplace=True) for n in list(ds.name.iloc[:models_limit].values): self._stacked_models += [models_map[n]] scores = [m.get_final_loss() for m in self._stacked_models] self._stacked_models = [ self._stacked_models[i] for i in np.argsort(scores).tolist() ] def prepare_for_stacking(self): # print("Stacked models ....") # do we have enough models? if len(self._models) < 5: return # do we have time? if self._total_time_limit is not None: time_left = self._total_time_limit - (time.time() - self._start_time) # we need at least 60 seconds to do anything if time_left < 60: return self._perform_model_stacking() X_stacked_path = os.path.join(self._results_path, "X_stacked.parquet") if os.path.exists(X_stacked_path): return X = pd.read_parquet(self._X_path) org_columns = X.columns.tolist() X_stacked = self.get_stacked_data(X) new_columns = X_stacked.columns.tolist() added_columns = [c for c in new_columns if c not in org_columns] # save stacked train data X_stacked.to_parquet(X_stacked_path, index=False) """ # resue old params for m in self._stacked_models: # print(m.get_type()) # use only Xgboost, LightGBM and CatBoost as stacked models if m.get_type() not in ["Xgboost", "LightGBM", "CatBoost"]: continue params = copy.deepcopy(m.params) params["validation"]["X_train_path"] = X_train_stacked_path params["name"] = params["name"] + "_Stacked" params["is_stacked"] = True # print(params) if "model_architecture_json" in params["learner"]: # the new model will be created with wider input size del params["learner"]["model_architecture_json"] if self._ml_task == REGRESSION: # scale added predictions in regression if the target was scaled (in the case of NN) target_preprocessing = params["preprocessing"]["target_preprocessing"] scale = None if "scale_log_and_normal" in target_preprocessing: scale = "scale_log_and_normal" elif "scale_normal" in target_preprocessing: scale = "scale_normal" if scale is not None: for col in added_columns: params["preprocessing"]["columns_preprocessing"][col] = [ scale] self.train_model(params) """ def _save_data(self, X, y): self._X_path = os.path.join(self._results_path, "X.parquet") self._y_path = os.path.join(self._results_path, "y.parquet") X.to_parquet(self._X_path, index=False) # let's check before any conversions target_is_numeric = pd.api.types.is_numeric_dtype(y) if self._ml_task == MULTICLASS_CLASSIFICATION: y = y.astype(str) pd.DataFrame({"target": y}).to_parquet(self._y_path, index=False) self._validation_strategy["X_path"] = self._X_path self._validation_strategy["y_path"] = self._y_path self._validation_strategy["results_path"] = self._results_path columns_and_target_info = DataInfo.compute(X, y, self._ml_task) self.n_features_in_ = X.shape[1] self.n_classes = len(np.unique(y[~pd.isnull(y)])) self._data_info = { "columns": X.columns.tolist(), "rows": y.shape[0], "cols": X.shape[1], "target_is_numeric": target_is_numeric, "columns_info": columns_and_target_info["columns_info"], "target_info": columns_and_target_info["target_info"], "n_features": self.n_features_in_, } # Add n_classes if not regression if self._ml_task != REGRESSION: self._data_info["n_classes"] = self.n_classes if columns_and_target_info.get("num_class") is not None: self._data_info["num_class"] = columns_and_target_info["num_class"] data_info_path = os.path.join(self._results_path, "data_info.json") with open(data_info_path, "w") as fout: fout.write(json.dumps(self._data_info, indent=4)) self._drop_data_variables(X) def _drop_data_variables(self, X): X.drop(X.columns, axis=1, inplace=True) def _load_data_variables(self, X_train): if X_train.shape[1] == 0: X = pd.read_parquet(self._X_path) for c in X.columns: X_train.insert(loc=X_train.shape[1], column=c, value=X[c]) os.remove(self._X_path) os.remove(self._y_path) def save_progress(self, step=None, generated_params=None): if step is not None and generated_params is not None: self._all_params[step] = generated_params state = {} state["fit_level"] = self._fit_level state["time_controller"] = self._time_ctrl.to_json() state["all_params"] = self._all_params fname = os.path.join(self._results_path, "progress.json") with open(fname, "w") as fout: fout.write(json.dumps(state, indent=4)) def load_progress(self): state = {} fname = os.path.join(self._results_path, "progress.json") if not os.path.exists(fname): return state = json.load(open(fname, "r")) self._fit_level = state.get("fit_level", self._fit_level) self._all_params = state.get("all_params", self._all_params) self._time_ctrl = TimeController.from_json( state.get("time_controller")) def _validate_X_predict(self, X): """Validate X whenever one tries to predict, apply, predict_proba""" # X = check_array(X, ensure_2d=False) X = np.atleast_2d(X) n_features = X.shape[1] if self.n_features_in_ != n_features: raise ValueError( f"Number of features of the model must match the input. Model n_features_in_ is {self.n_features_in_} and input n_features is {n_features}. Reshape your data." ) # This method builds pandas.Dataframe from input. The input can be numpy.ndarray, matrix, or pandas.Dataframe # This method is used to build dataframes in `fit()` and in `predict`. That's the reason y can be None (`predict()` method) def _build_dataframe(self, X, y=None): # If Inputs are not pandas dataframes use scikit-learn validation for X array if not isinstance(X, pd.DataFrame): # Validate X as array X = check_array(X, ensure_2d=False) # Force X to be 2D X = np.atleast_2d(X) # Create Pandas dataframe from np.arrays, columns get names with the schema: feature_{index} X = pd.DataFrame( X, columns=["feature_" + str(i) for i in range(1, len(X[0]) + 1)]) # Enforce column names # Enforce X_train columns to be string X.columns = X.columns.astype(str) X.reset_index(drop=True, inplace=True) if y is None: return X # Check if y is np.ndarray, transform to pd.Series if isinstance(y, np.ndarray): y = check_array(y, ensure_2d=False) y = pd.Series(np.array(y), name="target") # if pd.DataFrame, slice first column elif isinstance(y, pd.DataFrame): y = np.array(y.iloc[:, 0]) y = check_array(y, ensure_2d=False) y = pd.Series(np.array(y), name="target") X, y = ExcludeRowsMissingTarget.transform(X, y, warn=True) X.reset_index(drop=True, inplace=True) y.reset_index(drop=True, inplace=True) return X, y def _fit(self, X, y): """Fits the AutoML model with data""" if self._fit_level == "finished": print( "This model has already been fitted. You can use predict methods or select a new 'results_path' for a new a 'fit()'." ) return # Validate input and build dataframes X, y = self._build_dataframe(X, y) self.n_features_in_ = X.shape[1] self.n_classes = len(np.unique(y[~pd.isnull(y)])) # Get attributes (__init__ params) self._mode = self._get_mode() self._ml_task = self._get_ml_task() self._results_path = self._get_results_path() self._total_time_limit = self._get_total_time_limit() self._model_time_limit = self._get_model_time_limit() self._algorithms = self._get_algorithms() self._train_ensemble = self._get_train_ensemble() self._stack_models = self._get_stack_models() self._eval_metric = self._get_eval_metric() self._validation_strategy = self._get_validation_strategy() self._verbose = self._get_verbose() self._explain_level = self._get_explain_level() self._golden_features = self._get_golden_features() self._feature_selection = self._get_feature_selection() self._start_random_models = self._get_start_random_models() self._hill_climbing_steps = self._get_hill_climbing_steps() self._top_models_to_improve = self._get_top_models_to_improve() self._random_state = self._get_random_state() try: self.load_progress() if self._fit_level == "finished": print( "This model has already been fitted. You can use predict methods or select a new 'results_path' for a new 'fit()'." ) return self._check_can_load() self.verbose_print(f"AutoML directory: {self._results_path}") self.verbose_print( f"The task is {self._ml_task} with evaluation metric {self._eval_metric}" ) self.verbose_print( f"AutoML will use algorithms: {self._algorithms}") if self._stack_models: self.verbose_print("AutoML will stack models") if self._train_ensemble: self.verbose_print("AutoML will ensemble availabe models") self._start_time = time.time() if self._time_ctrl is not None: self._start_time -= self._time_ctrl.already_spend() # Automatic Exloratory Data Analysis if self._explain_level == 2: EDA.compute(X, y, os.path.join(self._results_path, "EDA")) # Save data self._save_data(X.copy(deep=False), y) tuner = MljarTuner( self._get_tuner_params( self._start_random_models, self._hill_climbing_steps, self._top_models_to_improve, ), self._algorithms, self._ml_task, self._validation_strategy, self._explain_level, self._data_info, self._golden_features, self._feature_selection, self._train_ensemble, self._stack_models, self._random_state, ) self.tuner = tuner steps = tuner.steps() self.verbose_print(f"AutoML steps: {steps}") if self._time_ctrl is None: self._time_ctrl = TimeController( self._start_time, self._total_time_limit, self._model_time_limit, steps, self._algorithms, ) self._time_ctrl.log_time( "prepare_data", "prepare_data", "prepare_data", time.time() - self._start_time, ) for step in steps: self._fit_level = step start = time.time() # self._time_start[step] = start if step == "stack": self.prepare_for_stacking() generated_params = [] if step in self._all_params: generated_params = self._all_params[step] else: generated_params = tuner.generate_params( step, self._models, self._results_path, self._stacked_models) if generated_params is None or not generated_params: self.verbose_print( f"Skip {step} because no parameters were generated.") continue if generated_params: if "learner" in generated_params[ 0] and not self._time_ctrl.enough_time( generated_params[0]["learner"]["model_type"], self._fit_level): self.verbose_print( f"Skip {step} because of the time limit.") else: model_str = "models" if len( generated_params) > 1 else "model" self.verbose_print( f"* Step {step} will try to check up to {len(generated_params)} {model_str}" ) for params in generated_params: if params.get("status", "") in ["trained", "skipped", "error"]: self.verbose_print( f"{params['name']}: {params['status']}.") continue try: trained = False if "ensemble" in step: trained = self.ensemble_step( is_stacked=params["is_stacked"]) else: trained = self.train_model(params) params["status"] = "trained" if trained else "skipped" params["final_loss"] = self._models[-1].get_final_loss( ) params["train_time"] = self._models[-1].get_train_time( ) except Exception as e: self._update_errors_report(params.get("name"), str(e)) params["status"] = "error" self.save_progress(step, generated_params) self._fit_level = "finished" self.save_progress() self.verbose_print( f"AutoML fit time: {np.round(time.time() - self._start_time,2)} seconds" ) except Exception as e: raise e finally: if self._X_path is not None: self._load_data_variables(X) return self def _update_errors_report(self, model_name, error_msg): """Append error message to errors.md file. """ errors_filename = os.path.join(self._get_results_path(), "errors.md") with open(errors_filename, "a") as fout: self.verbose_print( f"There was an error during {model_name} training.") self.verbose_print(f"Please check {errors_filename} for details.") fout.write(f"## Error for {model_name}\n\n") fout.write(error_msg) link = "https://github.com/mljar/mljar-supervised/issues/new" fout.write( f"\n\nPlease set a GitHub issue with above error message at: {link}" ) fout.write("\n\n") def select_and_save_best(self): # Select best model (lowest loss) self._best_model = min(self._models, key=lambda x: x.get_final_loss()) with open(os.path.join(self._results_path, "best_model.txt"), "w") as fout: fout.write(f"{self._best_model.get_name()}") with open(os.path.join(self._results_path, "params.json"), "w") as fout: params = { "ml_task": self._ml_task, "eval_metric": self._eval_metric, "saved": self._model_paths, } if self._stacked_models is not None: params["stacked"] = [ m.get_name() for m in self._stacked_models ] fout.write(json.dumps(params, indent=4)) ldb = self.get_leaderboard() ldb.to_csv(os.path.join(self._results_path, "leaderboard.csv"), index=False) # save report ldb["Link"] = [ f"[Results link]({m}/README.md)" for m in ldb["name"].values ] ldb.insert(loc=0, column="Best model", value="") ldb.loc[ldb.name == self._best_model.get_name(), "Best model"] = "**the best**" with open(os.path.join(self._results_path, "README.md"), "w") as fout: fout.write(f"# AutoML Leaderboard\n\n") fout.write(tabulate(ldb.values, ldb.columns, tablefmt="pipe")) LeaderboardPlots.compute(ldb, self._results_path, fout) def _check_is_fitted(self): # First check if model can be loaded self._check_can_load() # Check if fitted if self._fit_level != "finished": raise AutoMLException( "This model has not been fitted yet. Please call `fit()` first." ) def _base_predict(self, X): self._check_is_fitted() X = self._build_dataframe(X) if not isinstance(X.columns[0], str): X.columns = [str(c) for c in X.columns] input_columns = X.columns.tolist() for column in self._data_info["columns"]: if column not in input_columns: raise AutoMLException( f"Missing column: {column} in input data. Cannot predict") X = X[self._data_info["columns"]] self._validate_X_predict(X) # is stacked model if self._best_model._is_stacked: self._perform_model_stacking() X_stacked = self.get_stacked_data(X, mode="predict") if self._best_model.get_type() == "Ensemble": # Ensemble is using both original and stacked data predictions = self._best_model.predict(X, X_stacked) else: predictions = self._best_model.predict(X_stacked) else: predictions = self._best_model.predict(X) if self._ml_task == BINARY_CLASSIFICATION: # need to predict the label based on predictions and threshold neg_label, pos_label = ( predictions.columns[0][11:], predictions.columns[1][11:], ) if neg_label == "0" and pos_label == "1": neg_label, pos_label = 0, 1 target_is_numeric = self._data_info.get("target_is_numeric", False) if target_is_numeric: neg_label = int() pos_label = int(pos_label) # assume that it is binary classification predictions[ "label"] = predictions.iloc[:, 1] > self._best_model._threshold predictions["label"] = predictions["label"].map({ True: pos_label, False: neg_label }) return predictions elif self._ml_task == MULTICLASS_CLASSIFICATION: target_is_numeric = self._data_info.get("target_is_numeric", False) if target_is_numeric: predictions["label"] = predictions["label"].astype(np.int32) return predictions # Regression else: return predictions def _predict(self, X): predictions = self._base_predict(X) # Return predictions # If classification task the result is in column 'label' # If regression task the result is in column 'prediction' return (predictions["label"].to_numpy() if self._ml_task != REGRESSION else predictions["prediction"].to_numpy()) def _predict_proba(self, X): # Check is task type is correct if self._ml_task == REGRESSION: raise AutoMLException( f"Method `predict_proba()` can only be used when in classification tasks. Current task: '{self._ml_task}'." ) # Make and return predictions # If classification task the result is in column 'label' # Need to drop `label` column. return self._base_predict(X).drop(["label"], axis=1).to_numpy() def _predict_all(self, X): # Check is task type is correct if self._ml_task == REGRESSION: raise AutoMLException( f"Method `predict_all()` can only be used when in classification tasks. Current task: '{self._ml_task}'." ) # Make and return predictions return self._base_predict(X) def _score(self, X, y=None): # y default must be None for scikit-learn compatibility # Check if y is None if y is None: raise AutoMLException("y must be specified.") predictions = self._predict(X) return (r2_score(y, predictions) if self._ml_task == REGRESSION else accuracy_score(y, predictions)) def _get_mode(self): """ Gets the current mode""" self._validate_mode() return deepcopy(self.mode) def _get_ml_task(self): """ Gets the current ml_task. If "auto" it is determined""" self._validate_ml_task() if self.ml_task == "auto": classes_number = self.n_classes if classes_number == 2: self._estimator_type = "classifier" # for sk-learn api return BINARY_CLASSIFICATION elif classes_number <= 20: self._estimator_type = "classifier" # for sk-learn api return MULTICLASS_CLASSIFICATION else: self._estimator_type = "regressor" # for sk-learn api return REGRESSION else: return deepcopy(self.ml_task) def _get_results_path(self): """ Gets the current results_path""" # if we already have the results path set, please return it if self._results_path is not None: return self._results_path self._validate_results_path() path = self.results_path if path is None: for i in range(1, 10001): name = f"AutoML_{i}" if not os.path.exists(name): self.create_dir(name) self._results_path = name return name # If it got here, could not create, raise expection raise AutoMLException("Cannot create directory for AutoML results") elif os.path.exists(self.results_path) and os.path.exists( os.path.join( self.results_path, "params.json")): # AutoML already loaded, return path self._results_path = path return path # Dir does not exist, create it elif not os.path.exists(path): self.create_dir(path) self._results_path = path return path # Dir exists and is empty, use it elif os.path.exists(path) and not len(os.listdir(path)): self._results_path = path return path elif os.path.exists(path) and len(os.listdir(path)): raise AutoMLException( f"Cannot set directory for AutoML. Directory '{path}' is not empty." ) raise AutoMLException("Cannot set directory for AutoML results") def _get_total_time_limit(self): """ Gets the current total_time_limit""" self._validate_total_time_limit() return deepcopy(self.total_time_limit) def _get_model_time_limit(self): """ Gets the current model_time_limit""" self._validate_model_time_limit() return deepcopy(self.model_time_limit) def _get_algorithms(self): """ Gets the current algorithms. If "auto" it is determined""" self._validate_algorithms() if self.algorithms == "auto": if self._get_mode() == "Explain": return [ "Baseline", "Linear", "Decision Tree", "Random Forest", "Xgboost", "Neural Network", ] if self._get_mode() == "Perform": return [ "Linear", "Random Forest", "LightGBM", "Xgboost", "CatBoost", "Neural Network", ] if self._get_mode() == "Compete": return [ "Linear", "Decision Tree", "Random Forest", "Extra Trees", "LightGBM", "Xgboost", "CatBoost", "Neural Network", "Nearest Neighbors", ] else: return deepcopy(self.algorithms) def _get_train_ensemble(self): """ Gets the current train_ensemble""" self._validate_train_ensemble() return deepcopy(self.train_ensemble) def _get_stack_models(self): """ Gets the current stack_models""" self._validate_stack_models() if self.stack_models == "auto": return True if self.mode == "Compete" else False else: return deepcopy(self.stack_models) def _get_eval_metric(self): """ Gets the current eval_metric""" self._validate_eval_metric() if self.eval_metric == "auto": if self._get_ml_task() == BINARY_CLASSIFICATION: return "logloss" elif self._get_ml_task() == MULTICLASS_CLASSIFICATION: return "logloss" elif self._get_ml_task() == REGRESSION: return "rmse" else: return deepcopy(self.eval_metric) def _get_validation_strategy(self): """ Gets the current validation_strategy""" strat = {} self._validate_validation_strategy() if self.validation_strategy == "auto": if self._get_mode() == "Explain": strat = { "validation_type": "split", "train_ratio": 0.75, "shuffle": True, "stratify": True, } elif self._get_mode() == "Perform": strat = { "validation_type": "kfold", "k_folds": 5, "shuffle": True, "stratify": True, } elif self._get_mode() == "Compete": strat = { "validation_type": "kfold", "k_folds": 10, "shuffle": True, "stratify": True, } if self._get_ml_task() == REGRESSION: if "stratify" in strat: # it's better to always check # before delete (trust me) del strat["stratify"] return strat else: strat = deepcopy(self.validation_strategy) if "stratify" in strat: del strat["stratify"] return strat def _get_verbose(self): """Gets the current verbose""" self._validate_verbose() return deepcopy(self.verbose) def _get_explain_level(self): """ Gets the current explain_level""" self._validate_explain_level() if self.explain_level == "auto": if self._get_mode() == "Explain": return 2 if self._get_mode() == "Perform": return 1 if self._get_mode() == "Compete": return 0 else: return deepcopy(self.explain_level) def _get_golden_features(self): self._validate_golden_features() if self.golden_features == "auto": if self._get_mode() == "Explain": return False if self._get_mode() == "Perform": return True if self._get_mode() == "Compete": return True else: return deepcopy(self.golden_features) def _get_feature_selection(self): """ Gets the current feature_selection""" self._validate_feature_selection() if self.feature_selection == "auto": if self._get_mode() == "Explain": return False if self._get_mode() == "Perform": return True if self._get_mode() == "Compete": return True else: return deepcopy(self.feature_selection) def _get_start_random_models(self): """ Gets the current start_random_models""" self._validate_start_random_models() if self.start_random_models == "auto": if self._get_mode() == "Explain": return 1 if self._get_mode() == "Perform": return 5 if self._get_mode() == "Compete": return 10 else: return deepcopy(self.start_random_models) def _get_hill_climbing_steps(self): """ Gets the current hill_climbing_steps""" self._validate_hill_climbing_steps() if self.hill_climbing_steps == "auto": if self._get_mode() == "Explain": return 0 if self._get_mode() == "Perform": return 2 if self._get_mode() == "Compete": return 2 else: return deepcopy(self.hill_climbing_steps) def _get_top_models_to_improve(self): """ Gets the current top_models_to_improve""" self._validate_top_models_to_improve() if self.top_models_to_improve == "auto": if self._get_mode() == "Explain": return 0 if self._get_mode() == "Perform": return 2 if self._get_mode() == "Compete": return 3 else: return deepcopy(self.top_models_to_improve) def _get_random_state(self): """ Gets the current random_state""" self._validate_random_state() return deepcopy(self.random_state) def _validate_mode(self): """ Validates mode parameter""" valid_modes = ["Explain", "Perform", "Compete"] if self.mode not in valid_modes: raise ValueError( f"Expected 'mode' to be {' or '.join(valid_modes)}, got '{self.mode}'" ) def _validate_ml_task(self): """ Validates ml_task parameter""" if isinstance(self.ml_task, str) and self.ml_task == "auto": return if self.ml_task not in AlgorithmsRegistry.get_supported_ml_tasks(): raise ValueError( f"Expected 'ml_task' to be {' or '.join(AlgorithmsRegistry.get_supported_ml_tasks())}, got '{self.ml_task}''" ) def _validate_results_path(self): """ Validates path parameter""" if self.results_path is None or isinstance(self.results_path, str): return raise ValueError( f"Expected 'results_path' to be of type string, got '{type(self.results_path)}''" ) def _validate_total_time_limit(self): """ Validates total_time_limit parameter""" check_greater_than_zero_integer(self.total_time_limit, "total_time_limit") def _validate_model_time_limit(self): """ Validates model_time_limit parameter""" if self.model_time_limit is not None: check_greater_than_zero_integer(self.model_time_limit, "model_time_limit") def _validate_algorithms(self): """ Validates algorithms parameter""" if isinstance(self.algorithms, str) and self.algorithms == "auto": return for algo in self.algorithms: if algo not in list( AlgorithmsRegistry.registry[self._ml_task].keys()): raise ValueError( f"The algorithm {algo} is not allowed to use for ML task: {self._ml_task}. Allowed algorithms: {list(AlgorithmsRegistry.registry[self._ml_task].keys())}" ) def _validate_train_ensemble(self): """ Validates train_ensemble parameter""" # `train_ensemble` defaults to True, no further checking required check_bool(self.train_ensemble, "train_ensemble") def _validate_stack_models(self): """ Validates stack_models parameter""" # `stack_models` defaults to "auto". If "auto" return, else check if is valid bool if isinstance(self.stack_models, str) and self.stack_models == "auto": return check_bool(self.stack_models, "stack_models") def _validate_eval_metric(self): """ Validates eval_metric parameter""" # `stack_models` defaults to "auto". If not "auto", check if is valid bool if isinstance(self.eval_metric, str) and self.eval_metric == "auto": return if (self._get_ml_task() == BINARY_CLASSIFICATION or self._get_ml_task() == MULTICLASS_CLASSIFICATION ) and self.eval_metric != "logloss": raise ValueError( f"Metric {self.eval_metric} is not allowed in ML task: {self._get_ml_task()}. \ Use 'log_loss'") elif self._get_ml_task() == REGRESSION and self.eval_metric != "rmse": raise ValueError( f"Metric {self.eval_metric} is not allowed in ML task: {self._get_ml_task()}. \ Use 'rmse'") def _validate_validation_strategy(self): """ Validates validation parameter""" if (isinstance(self.validation_strategy, str) and self.validation_strategy == "auto"): return # only validation_type is mandatory # other parameters of validations # have defaults set in their constructors required_keys = ["validation_type"] if type(self.validation_strategy) is not dict: raise ValueError( f"Expected 'validation_strategy' to be a dict, got '{type(self.validation_strategy)}'" ) if not all(key in self.validation_strategy for key in required_keys): raise ValueError( f"Expected dict with keys: {' , '.join(required_keys)}") def _validate_verbose(self): """ Validates verbose parameter""" check_positive_integer(self.verbose, "verbose") def _validate_explain_level(self): """ Validates explain_level parameter""" if isinstance(self.explain_level, str) and self.explain_level == "auto": return valid_explain_levels = [0, 1, 2] # Check if explain level is 0 or greater integer if not (isinstance(self.explain_level, int) and self.explain_level in valid_explain_levels): raise ValueError( f"Expected 'explain_level' to be {' or '.join([str(x) for x in valid_explain_levels])}, got '{self.explain_level}'" ) def _validate_golden_features(self): """ Validates golden_features parameter""" if isinstance(self.golden_features, str) and self.golden_features == "auto": return check_bool(self.golden_features, "golden_features") def _validate_feature_selection(self): """ Validates feature_selection parameter""" if isinstance(self.feature_selection, str) and self.feature_selection == "auto": return check_bool(self.feature_selection, "feature_selection") def _validate_start_random_models(self): """ Validates start_random_models parameter""" if (isinstance(self.start_random_models, str) and self.start_random_models == "auto"): return check_greater_than_zero_integer(self.start_random_models, "start_random_models") def _validate_hill_climbing_steps(self): """ Validates hill_climbing_steps parameter""" if (isinstance(self.hill_climbing_steps, str) and self.hill_climbing_steps == "auto"): return check_positive_integer(self.hill_climbing_steps, "hill_climbing_steps") def _validate_top_models_to_improve(self): """ Validates top_models_to_improve parameter""" if (isinstance(self.top_models_to_improve, str) and self.top_models_to_improve == "auto"): return check_positive_integer(self.top_models_to_improve, "top_models_to_improve") def _validate_random_state(self): """ Validates random_state parameter""" check_positive_integer(self.random_state, "random_state") def to_json(self): if self._best_model is None: return None return { "best_model": self._best_model.to_json(), "threshold": self._threshold, "ml_task": self._ml_task, } def from_json(self, json_data): if json_data["best_model"]["algorithm_short_name"] == "Ensemble": self._best_model = Ensemble() self._best_model.from_json(json_data["best_model"]) else: self._best_model = ModelFramework( json_data["best_model"].get("params")) self._best_model.from_json(json_data["best_model"]) self._threshold = json_data.get("threshold") self._ml_task = json_data.get("ml_task")
def fit(self, X_train, y_train, X_validation=None, y_validation=None): """ Fit AutoML :param X_train: Pandas DataFrame with training data. :param y_train: Numpy Array with target training data. :param X_validation: Pandas DataFrame with validation data. (Not implemented yet) :param y_validation: Numpy Array with target of validation data. (Not implemented yet) """ try: self.load_progress() if self._fit_level == "finished": print("AutoML is trained. Skipping fit step ...") return # if self._best_model is not None: # print("Best model is already set, no need to run fit. Skipping ...") # return self._start_time = time.time() if self._time_ctrl is not None: self._start_time -= self._time_ctrl.already_spend() if not isinstance(X_train, pd.DataFrame): raise AutoMLException( "AutoML needs X_train matrix to be a Pandas DataFrame" ) # Automatic Exloratory Data Analysis if self._explain_level == 2: EDA.compute(X_train, y_train, os.path.join(self._results_path, "EDA")) self._set_ml_task(y_train) if X_train is not None: X_train = X_train.copy(deep=False) X_train, y_train, X_validation, y_validation = self._initial_prep( X_train, y_train, X_validation, y_validation ) self._save_data(X_train, y_train, X_validation, y_validation) self._set_algorithms() self._set_metric() if self._ml_task in [BINARY_CLASSIFICATION, MULTICLASS_CLASSIFICATION]: self._check_imbalanced(y_train) tuner = MljarTuner( self._tuner_params, self._algorithms, self._ml_task, self._validation, self._explain_level, self._data_info, self._golden_features, self._feature_selection, self._train_ensemble, self._stack_models, self._seed, ) self.tuner = tuner steps = tuner.steps() if self._time_ctrl is None: self._time_ctrl = TimeController( self._start_time, self._total_time_limit, self._model_time_limit, steps, self._algorithms, ) self._time_ctrl.log_time( "prepare_data", "prepare_data", "prepare_data", time.time() - self._start_time ) for step in steps: self._fit_level = step start = time.time() # self._time_start[step] = start if step == "stack": self.prepare_for_stacking() generated_params = [] if step in self._all_params: generated_params = self._all_params[step] else: generated_params = tuner.generate_params( step, self._models, self._results_path, self._stacked_models ) if generated_params is None: continue if generated_params: print("-" * 72) print(f"{step} with {len(generated_params)} models to train ...") for params in generated_params: if params.get("status", "") == "trained": print(f"Skipping {params['name']}, already trained.") continue if params.get("status", "") == "skipped": print(f"Skipped {params['name']}.") continue trained = False if "ensemble" in step: trained = self.ensemble_step(is_stacked=params["is_stacked"]) else: trained = self.train_model(params) params["status"] = "trained" if trained else "skipped" params["final_loss"] = self._models[-1].get_final_loss() params["train_time"] = self._models[-1].get_train_time() self.save_progress(step, generated_params) self._fit_level = "finished" self.save_progress() print(f"AutoML fit time: {time.time() - self._start_time}") except Exception as e: raise e finally: if self._X_train_path is not None: self._load_data_variables(X_train)
class AutoML: """ Automated Machine Learning for supervised tasks (binary classification, multiclass classification, regression). """ def __init__( self, results_path=None, total_time_limit=30 * 60, mode="Explain", **kwargs ): """ Initialize the AutoML object. :param results_path: The path with results. If left `None` then the name of directory will be generated, with template: AutoML_{number}, where the number can be from 1 to 1,000 - depends which direcory name will be available. If the `results_path` will point to directory with AutoML results (`params.json` must be present), then all models will be loaded. :param total_time_limit: The time limit in seconds for AutoML training. It is not used when `model_time_limit` is not `None`. :params mode ## Additional (optional parameters) :param model_time_limit: The time limit for training a single model, in seconds. If `model_time_limit` is set, the `total_time_limit` is not respected. The single model can contain several learners. The time limit for single learner is computed based on `model_time_limit`. For example, in the case of 10-fold cross-validation, one model will have 10 learners. The `model_time_limit` is time for all 10 learners. :param algorithms: The list of algorithms that will be used in the training. The algorithms can be: [ "Baseline", "Linear", "Decision Tree", "Random Forest", "Extra Trees", "LightGBM", "Xgboost", "CatBoost", "Neural Network", "Nearest Neighbors", ] :param tuning_mode: The mode for tuning. It can be: `Normal`, `Sport`, `Insane`, `Perfect`. The names are kept the same as in https://mljar.com application. Each mode describe how many models will be checked: - `Normal` - about 5-10 models of each algorithm will be trained, - `Sport` - about 10-15 models of each algorithm will be trained, - `Insane` - about 15-20 models of each algorithm will be trained, - `Perfect` - about 25-35 models of each algorithm will be trained. You can also set how many models will be trained with `set_advanced` method. :param train_ensemble: If true then at the end of models training the ensemble will be created. (Default is `True`) :param stack_models: If true then stacked models will be created. Stack level is 1. (Default is `True`) :param optimize_metric: The metric to be optimized. (not implemented yet, please left `None`) :param validation: The JSON with validation type. Right now only Cross-Validation is supported. The example JSON parameters for validation: ``` {"validation_type": "kfold", "k_folds": 5, "shuffle": True, "stratify": True, "random_seed": 123} ``` :param verbose: Not implemented yet. :param ml_task: The machine learning task that will be solved. Can be: `"binary_classification", "multiclass_classification", "regression"`. If left `None` AutoML will try to guess the task based on target values. If there will be only 2 values in the target, then task will be set to `"binary_classification"`. If number of values in the target will be between 2 and 20 (included), then task will be set to `"multiclass_classification"`. In all other casses, the task is set to `"regression"`. :param explain_level: The level of explanations included to each model. `explain_level = 0` means no explanations `explain_level = 1` means produce importance plot (with permutation method), for decision trees produce tree plots, for linear models save coefficients `explain_level = 2` the same as for `1` plus SHAP explanations :param seed: The seed for random generator. """ logger.debug("AutoML.__init__") self._results_path = results_path """ total_time_limit is the time for computing for all models model_time_limit is the time for computing a single model if model_time_limit is None then its value is computed from total_time_limit if total_time_limit is set and model_time_limit is set, then total_time_limit constraint will be omitted """ self._total_time_limit = total_time_limit if mode not in ["Explain", "Perform", "Compete"]: print("mode should be: Explain, Perform or Compete") print("Setting mode=Explain") mode = "Explain" self._mode = mode if self._mode == "Explain": self._train_ensemble = True self._stack_models = False self._validation = { "validation_type": "split", "train_ratio": 0.75, "shuffle": True, "stratify": True, } self._algorithms = [ "Baseline", "Linear", "Decision Tree", "Random Forest", "Xgboost", "Neural Network", ] self._explain_level = 2 self._start_random_models = 1 self._hill_climbing_steps = 0 self._top_models_to_improve = 0 self._golden_features = False self._feature_selection = False elif self._mode == "Perform": self._train_ensemble = True self._stack_models = False self._validation = { "validation_type": "kfold", "k_folds": 5, "shuffle": True, "stratify": True, } self._algorithms = [ "Linear", "Random Forest", "LightGBM", "Xgboost", "CatBoost", "Neural Network", ] self._explain_level = 1 self._start_random_models = 5 self._hill_climbing_steps = 2 self._top_models_to_improve = 2 self._golden_features = True self._feature_selection = True elif self._mode == "Compete": self._train_ensemble = True self._stack_models = True self._validation = { "validation_type": "kfold", "k_folds": 10, "shuffle": True, "stratify": True, } self._algorithms = [ "Linear", "Decision Tree", "Random Forest", "Extra Trees", "LightGBM", "Xgboost", "CatBoost", "Neural Network", "Nearest Neighbors", ] self._explain_level = 0 self._start_random_models = 10 self._hill_climbing_steps = 2 self._top_models_to_improve = 3 self._golden_features = True self._feature_selection = True self._model_time_limit = None if "model_time_limit" in kwargs: self._model_time_limit = kwargs["model_time_limit"] # algorithms are checked during the fit method call, # we need to know the data to tell if allgorithm is proper if "algorithms" in kwargs: self._algorithms = kwargs["algorithms"] if "validation" in kwargs: self._validation = kwargs["validation"] if "train_ensemble" in kwargs: self._train_ensemble = kwargs["train_ensemble"] if "stack_models" in kwargs: self._stack_models = kwargs["stack_models"] if "explain_level" in kwargs: self._explain_level = kwargs["explain_level"] self._ml_task = None if "ml_task" in kwargs: self._ml_task = kwargs["ml_task"] self._user_set_optimize_metric = None if "optimize_metric" in kwargs: self._user_set_optimize_metric = kwargs["optimize_metric"] if "tuning_mode" in kwargs: self.set_tuning_mode(kwargs["tuning_mode"]) if "golden_features" in kwargs: self._golden_features = kwargs["golden_features"] if "feature_selection" in kwargs: self._feature_selection = kwargs["feature_selection"] self._verbose = True if "verbose" is kwargs: self._verbose = kwargs["verbose"] self._seed = 1234 if "seed" in kwargs: self._seed = kwargs["seed"] self._tuner_params = { "start_random_models": self._start_random_models, "hill_climbing_steps": self._hill_climbing_steps, "top_models_to_improve": self._top_models_to_improve, } self._models = [] # instances of iterative learner framework or ensemble # it is instance of model framework or ensemble self._best_model = None self._verbose = True self._threshold = None self._metrics_details = None self._max_metrics = None self._confusion_matrix = None self._X_train_path, self._y_train_path = None, None self._X_validation_path, self._y_validation_path = None, None self._data_info = None self._model_paths = [] self._stacked_models = None self._fit_level = None self._start_time = time.time() # it will be updated in `fit` method self._time_ctrl = None if self._validation["validation_type"] != "kfold" and self._stack_models: print( "Models cannot be stacked. Please set validation to k-fold to stack models." ) # stacking only available of k-fold validation self._stack_models = False self._all_params = {} # this should be last in the constrcutor # in case there is a dir, it might load models self._set_results_dir() def set_tuning_mode(self, mode="Normal"): if mode == "Sport": self._start_random_models = 10 self._hill_climbing_steps = 2 self._top_models_to_improve = 3 elif mode == "Insane": self._start_random_models = 15 self._hill_climbing_steps = 3 self._top_models_to_improve = 4 elif mode == "Perfect": self._start_random_models = 25 self._hill_climbing_steps = 5 self._top_models_to_improve = 5 else: # Normal self._start_random_models = 5 self._hill_climbing_steps = 1 self._top_models_to_improve = 2 self._tuner_params = { "start_random_models": self._start_random_models, "hill_climbing_steps": self._hill_climbing_steps, "top_models_to_improve": self._top_models_to_improve, } def set_advanced( self, start_random_models=1, hill_climbing_steps=0, top_models_to_improve=0 ): """ Advanced set of tuning parameters. :param start_random_models: Number of not-so-random models to check for each algorithm. :param hill_climbing_steps: Number of hill climbing steps during tuning. :param top_models_to_improve: Number of top models (of each algorithm) which will be considered for improving in hill climbing steps. """ self._start_random_models = start_random_models self._hill_climbing_steps = hill_climbing_steps self._top_models_to_improve = top_models_to_improve self._tuner_params = { "start_random_models": self._start_random_models, "hill_climbing_steps": self._hill_climbing_steps, "top_models_to_improve": self._top_models_to_improve, } def _set_results_dir(self): if self._results_path is None: found = False for i in range(1, 10001): self._results_path = f"AutoML_{i}" if not os.path.exists(self._results_path): found = True break if not found: raise AutoMLException("Cannot create directory for AutoML results") if os.path.exists(self._results_path) and os.path.exists( os.path.join(self._results_path, "params.json") ): print(f"Directory {self._results_path} already exists") self.load() elif self._results_path is not None: if not os.path.exists(self._results_path): print(f"Create directory {self._results_path}") try: os.mkdir(self._results_path) except Exception as e: raise AutoMLException( f"Cannot create directory {self._results_path}" ) elif os.path.exists(self._results_path) and len( os.listdir(self._results_path) ): raise AutoMLException( f"Cannot set directory for AutoML. Directory {self._results_path} is not empty." ) else: raise AutoMLException("Cannot set directory for AutoML results") def load(self): logger.info("Loading AutoML models ...") try: params = json.load(open(os.path.join(self._results_path, "params.json"))) self._model_paths = params["saved"] self._ml_task = params["ml_task"] self._optimize_metric = params["optimize_metric"] stacked_models = params.get("stacked") models_map = {} for model_path in self._model_paths: if model_path.endswith("Ensemble") or model_path.endswith( "Ensemble_Stacked" ): ens = Ensemble.load(model_path, models_map) self._models += [ens] models_map[ens.get_name()] = ens else: m = ModelFramework.load(model_path) self._models += [m] models_map[m.get_name()] = m if stacked_models is not None: self._stacked_models = [] for stacked_model_name in stacked_models: self._stacked_models += [models_map[stacked_model_name]] best_model_name = None with open(os.path.join(self._results_path, "best_model.txt"), "r") as fin: best_model_name = fin.read() self._best_model = models_map[best_model_name] data_info_path = os.path.join(self._results_path, "data_info.json") self._data_info = json.load(open(data_info_path)) except Exception as e: raise AutoMLException(f"Cannot load AutoML directory. {str(e)}") def get_leaderboard(self): ldb = { "name": [], "model_type": [], "metric_type": [], "metric_value": [], "train_time": [], } for m in self._models: ldb["name"] += [m.get_name()] ldb["model_type"] += [m.get_type()] ldb["metric_type"] += [self._optimize_metric] ldb["metric_value"] += [m.get_final_loss()] ldb["train_time"] += [np.round(m.get_train_time(), 2)] return pd.DataFrame(ldb) def keep_model(self, model, model_path): if model is None: return self._models += [model] self._model_paths += [model_path] self.select_and_save_best() self.verbose_print( "{} {} {} trained in {} seconds".format( model.get_name(), self._optimize_metric, np.round(model.get_final_loss(), 6), np.round(model.get_train_time(), 2), ) ) self._time_ctrl.log_time( model.get_name(), model.get_type(), self._fit_level, model.get_train_time() ) def create_dir(self, model_path): if not os.path.exists(model_path): try: os.mkdir(model_path) except Exception as e: raise AutoMLException(f"Cannot create directory {model_path}. {str(e)}") def train_model(self, params): # do we have enough time to train? # if not, skip if not self._time_ctrl.enough_time( params["learner"]["model_type"], self._fit_level ): logger.info(f"Cannot train {params['name']} because of the time constraint") return False # let's create directory to log all training artifacts model_path = os.path.join(self._results_path, params["name"]) self.create_dir(model_path) # prepare callbacks early_stop = EarlyStopping( {"metric": {"name": self._optimize_metric}, "log_to_dir": model_path} ) learner_time_constraint = LearnerTimeConstraint( { "learner_time_limit": self._time_ctrl.learner_time_limit( params["learner"]["model_type"], self._fit_level, self._validation.get("k_folds", 1.0), ), "min_steps": params["additional"].get("min_steps"), } ) total_time_constraint = TotalTimeConstraint( { "total_time_limit": self._total_time_limit if self._model_time_limit is None else None, "total_time_start": self._start_time, } ) # create model framework mf = ModelFramework( params, callbacks=[early_stop, learner_time_constraint, total_time_constraint], ) # start training logger.info( f"Train model #{len(self._models)+1} / Model name: {params['name']}" ) mf.train(model_path) # save the model mf.save(model_path) # and keep info about the model self.keep_model(mf, model_path) return True def verbose_print(self, msg): if self._verbose: # self._progress_bar.write(msg) print(msg) def ensemble_step(self, is_stacked=False): if self._train_ensemble and len(self._models) > 1: ensemble_path = os.path.join( self._results_path, "Ensemble_Stacked" if is_stacked else "Ensemble" ) self.create_dir(ensemble_path) self.ensemble = Ensemble( self._optimize_metric, self._ml_task, is_stacked=is_stacked ) oofs, target = self.ensemble.get_oof_matrix(self._models) self.ensemble.fit(oofs, target) self.ensemble.save(ensemble_path) self.keep_model(self.ensemble, ensemble_path) return True return False def can_we_stack_them(self, y): # if multiclass and too many classes then No return True def get_stacked_data(self, X, mode="training"): # mode can be `training` or `predict` if self._stacked_models is None: return X all_oofs = [] for m in self._stacked_models: oof = None if mode == "training": oof = m.get_out_of_folds() else: oof = m.predict(X) if self._ml_task == BINARY_CLASSIFICATION: cols = [f for f in oof.columns if "prediction" in f] if len(cols) == 2: oof = pd.DataFrame({"prediction": oof[cols[1]]}) cols = [f for f in oof.columns if "prediction" in f] oof = oof[cols] oof.columns = [f"{m.get_name()}_{c}" for c in cols] all_oofs += [oof] org_index = X.index.copy() X.reset_index(drop=True, inplace=True) X_stacked = pd.concat(all_oofs + [X], axis=1) X_stacked.index = org_index.copy() X.index = org_index.copy() return X_stacked def stack_models(self): if self._stacked_models is not None: return ldb = self.get_leaderboard() ldb = ldb.sort_values(by="metric_value", ascending=True) models_map = {m.get_name(): m for m in self._models if not m._is_stacked} self._stacked_models = [] models_limit = 10 for model_type in np.unique(ldb.model_type): if model_type in ["Baseline"]: continue ds = ldb[ldb.model_type == model_type].copy() ds.sort_values(by="metric_value", inplace=True) for n in list(ds.name.iloc[:models_limit].values): self._stacked_models += [models_map[n]] scores = [m.get_final_loss() for m in self._stacked_models] self._stacked_models = [ self._stacked_models[i] for i in np.argsort(scores).tolist() ] def prepare_for_stacking(self): # print("Stacked models ....") # do we have enough models? if len(self._models) < 5: return # do we have time? if self._total_time_limit is not None: time_left = self._total_time_limit - (time.time() - self._start_time) # we need at least 60 seconds to do anything if time_left < 60: return self.stack_models() X_train_stacked_path = os.path.join( self._results_path, "X_train_stacked.parquet" ) if os.path.exists(X_train_stacked_path): return X = pd.read_parquet(self._X_train_path) org_columns = X.columns.tolist() X_stacked = self.get_stacked_data(X) new_columns = X_stacked.columns.tolist() added_columns = [c for c in new_columns if c not in org_columns] # save stacked train data X_stacked.to_parquet(X_train_stacked_path, index=False) """ # resue old params for m in self._stacked_models: # print(m.get_type()) # use only Xgboost, LightGBM and CatBoost as stacked models if m.get_type() not in ["Xgboost", "LightGBM", "CatBoost"]: continue params = copy.deepcopy(m.params) params["validation"]["X_train_path"] = X_train_stacked_path params["name"] = params["name"] + "_Stacked" params["is_stacked"] = True # print(params) if "model_architecture_json" in params["learner"]: # the new model will be created with wider input size del params["learner"]["model_architecture_json"] if self._ml_task == REGRESSION: # scale added predictions in regression if the target was scaled (in the case of NN) target_preprocessing = params["preprocessing"]["target_preprocessing"] scale = None if "scale_log_and_normal" in target_preprocessing: scale = "scale_log_and_normal" elif "scale_normal" in target_preprocessing: scale = "scale_normal" if scale is not None: for col in added_columns: params["preprocessing"]["columns_preprocessing"][col] = [scale] self.train_model(params) """ def _set_ml_task(self, y): """ Set and validate the ML task. If ML task is not set, it trys to guess ML task based on count of unique values in the target. Then it performs validation. """ # if not set, guess if self._ml_task is None: target_unique_cnt = len(np.unique(y[~pd.isnull(y)])) if target_unique_cnt == 2: self._ml_task = BINARY_CLASSIFICATION elif target_unique_cnt <= 20: self._ml_task = MULTICLASS_CLASSIFICATION else: self._ml_task = REGRESSION # validation if self._ml_task not in AlgorithmsRegistry.get_supported_ml_tasks(): raise Exception( "Unknow Machine Learning task {}." " Supported tasks are: {}".format( self._ml_task, AlgorithmsRegistry.get_supported_ml_tasks() ) ) if self._ml_task == REGRESSION: if "stratify" in self._validation: del self._validation["stratify"] logger.info("AutoML task to be solved: {}".format(self._ml_task)) print(f"AutoML task to be solved: { self._ml_task}") def _set_algorithms(self): """ Set and validate available algorithms. If algorithms are not set, all algorithms from registry are used. Then perform vadlidation of algorithms. """ if len(self._algorithms) == 0: self._algorithms = list(AlgorithmsRegistry.registry[self._ml_task].keys()) for a in self._algorithms: if a not in list(AlgorithmsRegistry.registry[self._ml_task].keys()): raise AutoMLException( "The algorithm {} is not allowed to use for ML task: {}. Allowed algorithms: {}".format( a, self._ml_task, list(AlgorithmsRegistry.registry[self._ml_task].keys()), ) ) logger.info("AutoML will use algorithms: {}".format(self._algorithms)) print(f"AutoML will use algorithms: {self._algorithms}") def _set_metric(self): """ Set and validate the metric to be optimized. """ if self._ml_task == BINARY_CLASSIFICATION: if self._user_set_optimize_metric is None: self._optimize_metric = "logloss" elif self._user_set_optimize_metric not in ["logloss", "auc"]: raise AutoMLException( "Metric {} is not allowed in ML task: {}".format( self._user_set_optimize_metric, self._ml_task ) ) else: self._optimize_metric = self._user_set_optimize_metric elif self._ml_task == MULTICLASS_CLASSIFICATION: if self._user_set_optimize_metric is None: self._optimize_metric = "logloss" elif self._user_set_optimize_metric not in ["logloss"]: raise AutoMLException( "Metric {} is not allowed in ML task: {}".format( self._user_set_optimize_metric, self._ml_task ) ) else: self._optimize_metric = self._user_set_optimize_metric elif self._ml_task == REGRESSION: if self._user_set_optimize_metric is None: self._optimize_metric = "rmse" elif self._user_set_optimize_metric not in ["rmse"]: raise AutoMLException( "Metric {} is not allowed in ML task: {}".format( self._user_set_optimize_metric, self._ml_task ) ) else: self._optimize_metric = self._user_set_optimize_metric logger.info( "AutoML will optimize for metric: {0}".format(self._optimize_metric) ) print(f"AutoML will optimize for metric: {self._optimize_metric}") def _check_imbalanced(self, y): v = y.value_counts() # at least 10 samples of each class ii = v < 10 if np.sum(ii): raise AutoMLException( f"There need to be at least 10 samples of each class, for class {list(v[ii].index)} there is {v[ii].values} samples" ) # at least 1% of all samples for each class v = y.value_counts(normalize=True) * 100.0 ii = v < 1.0 if np.sum(ii): raise AutoMLException( f"There need to be at least 1% of samples of each class, for class {list(v[ii].index)} there is {v[ii].values} % of samples" ) def _initial_prep(self, X_train, y_train, X_validation=None, y_validation=None): if not isinstance(X_train, pd.DataFrame): X_train = pd.DataFrame(X_train) if not isinstance(X_train.columns[0], str): X_train.columns = [str(c) for c in X_train.columns] X_train.reset_index(drop=True, inplace=True) if isinstance(y_train, pd.DataFrame): if "target" not in y_train.columns: raise AutoMLException( "y_train should be Numpy array, Pandas Series or DataFrame with column 'target' " ) else: y_train = y_train["target"] y_train = pd.Series(np.array(y_train), name="target") X_train, y_train = ExcludeRowsMissingTarget.transform( X_train, y_train, warn=True ) return X_train, y_train, X_validation, y_validation def _save_data(self, X_train, y_train, X_validation=None, y_validation=None): self._X_train_path = os.path.join(self._results_path, "X_train.parquet") self._y_train_path = os.path.join(self._results_path, "y_train.parquet") X_train.to_parquet(self._X_train_path, index=False) if self._ml_task == MULTICLASS_CLASSIFICATION: y_train = y_train.astype(str) pd.DataFrame({"target": y_train}).to_parquet(self._y_train_path, index=False) self._validation["X_train_path"] = self._X_train_path self._validation["y_train_path"] = self._y_train_path self._validation["results_path"] = self._results_path columns_and_target_info = DataInfo.compute(X_train, y_train, self._ml_task) self._data_info = { "columns": X_train.columns.tolist(), "rows": X_train.shape[0], "cols": X_train.shape[1], "target_is_numeric": pd.api.types.is_numeric_dtype(y_train), "columns_info": columns_and_target_info["columns_info"], "target_info": columns_and_target_info["target_info"], } if columns_and_target_info.get("num_class") is not None: self._data_info["num_class"] = columns_and_target_info["num_class"] data_info_path = os.path.join(self._results_path, "data_info.json") with open(data_info_path, "w") as fout: fout.write(json.dumps(self._data_info, indent=4)) self._drop_data_variables(X_train) def _drop_data_variables(self, X_train): X_train.drop(X_train.columns, axis=1, inplace=True) def _load_data_variables(self, X_train): if X_train.shape[1] == 0: X = pd.read_parquet(self._X_train_path) for c in X.columns: X_train.insert(loc=X_train.shape[1], column=c, value=X[c]) os.remove(self._X_train_path) os.remove(self._y_train_path) def save_progress(self, step=None, generated_params=None): if step is not None and generated_params is not None: self._all_params[step] = generated_params state = {} state["fit_level"] = self._fit_level state["time_controller"] = self._time_ctrl.to_json() state["all_params"] = self._all_params fname = os.path.join(self._results_path, "progress.json") with open(fname, "w") as fout: fout.write(json.dumps(state, indent=4)) def load_progress(self): state = {} fname = os.path.join(self._results_path, "progress.json") if not os.path.exists(fname): return state = json.load(open(fname, "r")) self._fit_level = state.get("fit_level", self._fit_level) self._all_params = state.get("all_params", self._all_params) self._time_ctrl = TimeController.from_json(state.get("time_controller")) def fit(self, X_train, y_train, X_validation=None, y_validation=None): """ Fit AutoML :param X_train: Pandas DataFrame with training data. :param y_train: Numpy Array with target training data. :param X_validation: Pandas DataFrame with validation data. (Not implemented yet) :param y_validation: Numpy Array with target of validation data. (Not implemented yet) """ try: self.load_progress() if self._fit_level == "finished": print("AutoML is trained. Skipping fit step ...") return # if self._best_model is not None: # print("Best model is already set, no need to run fit. Skipping ...") # return self._start_time = time.time() if self._time_ctrl is not None: self._start_time -= self._time_ctrl.already_spend() if not isinstance(X_train, pd.DataFrame): raise AutoMLException( "AutoML needs X_train matrix to be a Pandas DataFrame" ) # Automatic Exloratory Data Analysis if self._explain_level == 2: EDA.compute(X_train, y_train, os.path.join(self._results_path, "EDA")) self._set_ml_task(y_train) if X_train is not None: X_train = X_train.copy(deep=False) X_train, y_train, X_validation, y_validation = self._initial_prep( X_train, y_train, X_validation, y_validation ) self._save_data(X_train, y_train, X_validation, y_validation) self._set_algorithms() self._set_metric() if self._ml_task in [BINARY_CLASSIFICATION, MULTICLASS_CLASSIFICATION]: self._check_imbalanced(y_train) tuner = MljarTuner( self._tuner_params, self._algorithms, self._ml_task, self._validation, self._explain_level, self._data_info, self._golden_features, self._feature_selection, self._train_ensemble, self._stack_models, self._seed, ) self.tuner = tuner steps = tuner.steps() if self._time_ctrl is None: self._time_ctrl = TimeController( self._start_time, self._total_time_limit, self._model_time_limit, steps, self._algorithms, ) self._time_ctrl.log_time( "prepare_data", "prepare_data", "prepare_data", time.time() - self._start_time ) for step in steps: self._fit_level = step start = time.time() # self._time_start[step] = start if step == "stack": self.prepare_for_stacking() generated_params = [] if step in self._all_params: generated_params = self._all_params[step] else: generated_params = tuner.generate_params( step, self._models, self._results_path, self._stacked_models ) if generated_params is None: continue if generated_params: print("-" * 72) print(f"{step} with {len(generated_params)} models to train ...") for params in generated_params: if params.get("status", "") == "trained": print(f"Skipping {params['name']}, already trained.") continue if params.get("status", "") == "skipped": print(f"Skipped {params['name']}.") continue trained = False if "ensemble" in step: trained = self.ensemble_step(is_stacked=params["is_stacked"]) else: trained = self.train_model(params) params["status"] = "trained" if trained else "skipped" params["final_loss"] = self._models[-1].get_final_loss() params["train_time"] = self._models[-1].get_train_time() self.save_progress(step, generated_params) self._fit_level = "finished" self.save_progress() print(f"AutoML fit time: {time.time() - self._start_time}") except Exception as e: raise e finally: if self._X_train_path is not None: self._load_data_variables(X_train) def select_and_save_best(self): max_loss = 10e14 for i, m in enumerate(self._models): if m.get_final_loss() < max_loss: self._best_model = m max_loss = m.get_final_loss() with open(os.path.join(self._results_path, "best_model.txt"), "w") as fout: fout.write(f"{self._best_model.get_name()}") with open(os.path.join(self._results_path, "params.json"), "w") as fout: params = { "ml_task": self._ml_task, "optimize_metric": self._optimize_metric, "saved": self._model_paths, } if self._stacked_models is not None: params["stacked"] = [m.get_name() for m in self._stacked_models] fout.write(json.dumps(params, indent=4)) ldb = self.get_leaderboard() ldb.to_csv(os.path.join(self._results_path, "leaderboard.csv"), index=False) # save report ldb["Link"] = [f"[Results link]({m}/README.md)" for m in ldb["name"].values] ldb.insert(loc=0, column="Best model", value="") ldb.loc[ldb.name == self._best_model.get_name(), "Best model"] = "**the best**" with open(os.path.join(self._results_path, "README.md"), "w") as fout: fout.write(f"# AutoML Leaderboard\n\n") fout.write(tabulate(ldb.values, ldb.columns, tablefmt="pipe")) LeaderboardPlots.compute(ldb, self._results_path, fout) def predict(self, X): """ Computes predictions from AutoML best model. :param X: The Pandas DataFrame with input data. The input data should have the same columns as data used for training, otherwise the `AutoMLException` will be raised. """ if self._best_model is None: return None if not isinstance(X.columns[0], str): X.columns = [str(c) for c in X.columns] input_columns = X.columns.tolist() for column in self._data_info["columns"]: if column not in input_columns: raise AutoMLException( f"Missing column: {column} in input data. Cannot predict" ) X = X[self._data_info["columns"]] # is stacked model if self._best_model._is_stacked: self.stack_models() X_stacked = self.get_stacked_data(X, mode="predict") if self._best_model.get_type() == "Ensemble": # Ensemble is using both original and stacked data predictions = self._best_model.predict(X, X_stacked) else: predictions = self._best_model.predict(X_stacked) else: predictions = self._best_model.predict(X) if self._ml_task == BINARY_CLASSIFICATION: # need to predict the label based on predictions and threshold neg_label, pos_label = ( predictions.columns[0][11:], predictions.columns[1][11:], ) if neg_label == "0" and pos_label == "1": neg_label, pos_label = 0, 1 target_is_numeric = self._data_info.get("target_is_numeric", False) if target_is_numeric: neg_label = int(neg_label) pos_label = int(pos_label) # assume that it is binary classification predictions["label"] = predictions.iloc[:, 1] > self._best_model._threshold predictions["label"] = predictions["label"].map( {True: pos_label, False: neg_label} ) return predictions elif self._ml_task == MULTICLASS_CLASSIFICATION: target_is_numeric = self._data_info.get("target_is_numeric", False) if target_is_numeric: predictions["label"] = predictions["label"].astype(int) return predictions else: return predictions def to_json(self): if self._best_model is None: return None return { "best_model": self._best_model.to_json(), "threshold": self._threshold, "ml_task": self._ml_task, } def from_json(self, json_data): if json_data["best_model"]["algorithm_short_name"] == "Ensemble": self._best_model = Ensemble() self._best_model.from_json(json_data["best_model"]) else: self._best_model = ModelFramework(json_data["best_model"].get("params")) self._best_model.from_json(json_data["best_model"]) self._threshold = json_data.get("threshold") self._ml_task = json_data.get("ml_task")
def test_enough_time_for_stacking(self): for t in [5, 10, 20]: tc = TimeController( start_time=time.time(), total_time_limit=100, model_time_limit=None, steps=[ "default_algorithms", "not_so_random", "golden_features", "insert_random_feature", "features_selection", "hill_climbing_1", "hill_climbing_3", "hill_climbing_5", "ensemble", "stack", "ensemble_stacked", ], algorithms=["Xgboost"], ) tc.log_time("1_Xgboost", "Xgboost", "default_algorithms", t) tc.log_time("2_Xgboost", "Xgboost", "not_so_random", t) tc.log_time("3_Xgboost", "Xgboost", "insert_random_feature", t) tc.log_time("4_Xgboost", "Xgboost", "features_selection", t) tc.log_time("5_Xgboost", "Xgboost", "hill_climbing_1", t) tc.log_time("6_Xgboost", "Xgboost", "hill_climbing_2", t) tc.log_time("7_Xgboost", "Xgboost", "hill_climbing_3", t) assert_almost_equal(tc.already_spend(), 7 * t) if t < 20: self.assertTrue(tc.enough_time("Xgboost", "stack")) else: self.assertFalse(tc.enough_time("Xgboost", "stack")) self.assertTrue( tc.enough_time("Ensemble_Stacked", "ensemble_stacked"))
def _fit(self, X, y): """Fits the AutoML model with data""" if self._fit_level == "finished": return print( "This model has already been fitted. You can use predict methods or select a new 'results_path' for a new a 'fit()'." ) # Validate input and build dataframes X, y = self._build_dataframe(X, y) self.n_features = X.shape[1] self.n_classes = len(np.unique(y[~pd.isnull(y)])) # Get attributes (__init__ params) self._mode = self._get_mode() self._ml_task = self._get_ml_task() self._tuning_mode = self._get_tuning_mode() self._results_path = self._get_results_path() self._total_time_limit = self._get_total_time_limit() self._model_time_limit = self._get_model_time_limit() self._algorithms = self._get_algorithms() self._train_ensemble = self._get_train_ensemble() self._stack_models = self._get_stack_models() self._eval_metric = self._get_eval_metric() self._validation_strategy = self._get_validation_strategy() self._verbose = self._get_verbose() self._explain_level = self._get_explain_level() self._golden_features = self._get_golden_features() self._feature_selection = self._get_feature_selection() self._start_random_models = self._get_start_random_models() self._hill_climbing_steps = self._get_hill_climbing_steps() self._top_models_to_improve = self._get_top_models_to_improve() self._random_state = self._get_random_state() try: self.load_progress() if self._fit_level == "finished": print( "This model has already been fitted. You can use predict methods or select a new 'results_path' for a new 'fit()'." ) return # Validate input and build dataframes X, y = self._build_dataframe(X, y) self.n_features = X.shape[1] self.n_classes = len(np.unique(y[~pd.isnull(y)])) self.verbose_print( f"AutoML current directory: {self._results_path}") self.verbose_print(f"AutoML current task: {self._ml_task}") self.verbose_print( f"AutoML will use algorithms : {self._algorithms}") self.verbose_print( f"AutoML will optimize for metric : {self._eval_metric}") self._start_time = time.time() if self._time_ctrl is not None: self._start_time -= self._time_ctrl.already_spend() # Automatic Exloratory Data Analysis if self._explain_level == 2: EDA.compute(X, y, os.path.join(self._results_path, "EDA")) # Save data self._save_data(X, y) tuner = MljarTuner( self._get_tuner_params( self._start_random_models, self._hill_climbing_steps, self._top_models_to_improve, ), self._algorithms, self._ml_task, self._validation_strategy, self._explain_level, self._data_info, self._golden_features, self._feature_selection, self._train_ensemble, self._stack_models, self._random_state, ) self.tuner = tuner steps = tuner.steps() self.verbose_print(f"AutoML steps: {steps}") if self._time_ctrl is None: self._time_ctrl = TimeController( self._start_time, self._total_time_limit, self._model_time_limit, steps, self._algorithms, ) self._time_ctrl.log_time( "prepare_data", "prepare_data", "prepare_data", time.time() - self._start_time, ) for step in steps: self._fit_level = step start = time.time() # self._time_start[step] = start if step == "stack": self.prepare_for_stacking() generated_params = [] if step in self._all_params: generated_params = self._all_params[step] else: generated_params = tuner.generate_params( step, self._models, self._results_path, self._stacked_models) if generated_params is None: continue if generated_params: self.verbose_print("-" * 72) self.verbose_print( f"{step} with {len(generated_params)} models to train ..." ) for params in generated_params: if params.get("status", "") == "trained": self.verbose_print( f"Skipping {params['name']}, already trained.") continue if params.get("status", "") == "skipped": self.verbose_print(f"Skipped {params['name']}.") continue trained = False if "ensemble" in step: trained = self.ensemble_step( is_stacked=params["is_stacked"]) else: trained = self.train_model(params) params["status"] = "trained" if trained else "skipped" params["final_loss"] = self._models[-1].get_final_loss() params["train_time"] = self._models[-1].get_train_time() self.save_progress(step, generated_params) self._fit_level = "finished" self.save_progress() self.verbose_print( f"AutoML fit time: {time.time() - self._start_time}") except Exception as e: raise e finally: if self._X_path is not None: self._load_data_variables(X) return self