def run(self): # update status mlmodel = MLModel.objects.get(pk=self.job_params.get("db_id")) mlmodel.status = "started" mlmodel.save() mlexperiment = MLExperiment.objects.get( pk=mlmodel.parent_experiment_id) print("mlexperiment", mlexperiment.id) print(mlexperiment.parent_columns_usage) # prepare data columns_usage = mlexperiment.parent_columns_usage.columns_usage print("cols", columns_usage) training_dataframe = mlexperiment.parent_training_dataframe print("training", training_dataframe.absolute_path) metric_params = mlexperiment.params.get("metric") validation_params = mlexperiment.params.get("validation") preprocessing_params = mlexperiment.params.get("preprocessing") df_train = DataServe.get(training_dataframe.absolute_path) training_data = { "train": { "X": df_train[columns_usage.get("input")], "y": df_train[columns_usage.get("target")], } } # prepare model hyper parameters learner_params = { "learner_type": mlmodel.model_type, "max_iters": 3, "max_depth": 1, } for k, v in mlmodel.params.items(): learner_params[k] = v train_params = { "preprocessing": preprocessing_params, "validation": validation_params, "learner": learner_params, } print(train_params) # prepare needed callbacks early_stop = EarlyStopping({"metric": {"name": "logloss"}}) metric_logger = MetricLogger({"metric_names": ["logloss", "auc"]}) # run the training il = IterativeLearner(train_params, callbacks=[early_stop, metric_logger]) il.train(training_data) # save the model save_details = il.save() logger.info(save_details) # store model details in platform database mlmodel.status = "done" mlmodel.save_details = save_details mlmodel.all_params = ( train_params) # all parameters will be needed for models loading mlmodel.save()
def from_json(self, json_data): # pretty sure that this can be easily refactored if json_data["algorithm_short_name"] == "Ensemble": self._best_model = Ensemble() self._best_model.from_json(json_data) else: self._best_model = IterativeLearner(json_data.get("params")) self._best_model.from_json(json_data)
def test_fit_and_predict(self): il = IterativeLearner(self.train_params, callbacks=[]) il.train(self.data) y_predicted = il.predict(self.X) metric = Metric({"name": "logloss"}) loss = metric(self.y, y_predicted) self.assertTrue(loss < 0.4)
def test_fit_and_predict(self): early_stop = EarlyStopping({"metric": {"name": "logloss"}}) metric_logger = MetricLogger({"metric_names": ["logloss", "auc"]}) il = IterativeLearner(self.train_params, callbacks=[early_stop, metric_logger]) il.train(self.data) y_predicted = il.predict(self.X) metric = Metric({"name": "logloss"}) loss = metric(self.y, y_predicted) self.assertTrue(loss < 0.4)
def test_fit_and_predict_split(self): self.assertTrue("Private" in list(self.data["train"]["X"]["workclass"])) early_stop = EarlyStopping({"metric": {"name": "logloss"}}) metric_logger = MetricLogger({"metric_names": ["logloss", "auc"]}) il = IterativeLearner(self.train_params, callbacks=[early_stop, metric_logger]) il.train(self.data) self.assertTrue("Private" in list(self.data["train"]["X"]["workclass"])) y_predicted = il.predict(self.data["train"]["X"]) self.assertTrue("Private" in list(self.data["train"]["X"]["workclass"])) metric = Metric({"name": "logloss"}) loss = metric(self.data["train"]["y"], y_predicted) self.assertTrue(loss < 0.6)
def test_fit_and_predict(self): MAX_STEPS = 100 additional["max_steps"] = MAX_STEPS iters_cnt = 5 max_iters = MaxItersConstraint({"max_iters": iters_cnt}) metric_logger = MetricLogger({"metric_names": ["logloss"]}) il = IterativeLearner(self.train_params, callbacks=[max_iters, metric_logger]) il.train(self.data) metric_logs = il.get_metric_logs() for k in range(self.kfolds): self.assertEqual( len(metric_logs[il.learners[k].uid]["train"]["logloss"]), iters_cnt ) self.assertNotEqual( len(metric_logs[il.learners[k].uid]["train"]["logloss"]), MAX_STEPS )
def run(self): # read data logger.info("ComputeBatchPrediction::run") batch = MLBatchPrediction.objects.get(pk=self.job_params.get("db_id")) logger.info("batch", batch) # {'parent_mlmodel': 9, 'parent_dataframe': 1, 'db_id': 1, 'created_by_id': 1, 'parent_organization_id': 1, 'parent_project_id': 1} mlmodel = MLModel.objects.get(pk=self.job_params.get("parent_mlmodel")) logger.info(mlmodel.save_details) logger.info(mlmodel.all_params) il = IterativeLearner(mlmodel.all_params) il.load(mlmodel.save_details) logger.info(batch.parent_dataframe.absolute_path) input_df = DataServe.get(batch.parent_dataframe.absolute_path) predictions = il.predict(input_df) logger.info(predictions) filename = "predictions-{0}.csv".format(str(uuid.uuid4())[:8]) organization_slug = batch.parent_organization.slug project_id = batch.parent_project.id relative_dir = "org_{0}_proj_{1}".format(organization_slug, project_id) relative_path = os.path.join(relative_dir, filename) result_absolute_path = Storage().get_path(relative_dir, filename) logger.info(result_absolute_path) df = pd.DataFrame({"prediction": predictions}) df.to_csv(result_absolute_path, index=False) # create mljar data frame result_df = DataFrame( source_id=self.job_params.get("parent_dataframe"), # fix this absolute_path=result_absolute_path, file_size=1, # TODO fix the file size columns_details="", # we can describe any data frame (always :-)) preview_absolute_path="", created_by_id=self.job_params["created_by_id"], parent_organization_id=self.job_params["parent_organization_id"], parent_project_id=self.job_params["parent_project_id"], ) result_df.save() batch.result_dataframe = result_df batch.status = "done" batch.save()
def test_fit_and_predict(self): MAX_STEPS = 10 additional["max_steps"] = MAX_STEPS metric_logger = MetricLogger({"metric_names": ["logloss", "auc"]}) il = IterativeLearner(self.train_params, callbacks=[metric_logger]) il.train(self.data) metric_logs = il.get_metric_logs() self.assertEqual( len(metric_logs[il.learners[0].uid]["train"]["logloss"]), len(metric_logs[il.learners[0].uid]["train"]["auc"]), ) self.assertEqual( len(metric_logs[il.learners[0].uid]["train"]["logloss"]), len(metric_logs[il.learners[0].uid]["iters"]), ) self.assertEqual( len(metric_logs[il.learners[0].uid]["train"]["logloss"]), MAX_STEPS)
def test_fit_and_predict_kfold(self): self.assertTrue( "Private" in list(self.data["train"]["X"]["workclass"])) early_stop = EarlyStopping({"metric": {"name": "logloss"}}) metric_logger = MetricLogger({"metric_names": ["logloss", "auc"]}) params = copy.deepcopy(self.train_params) params["validation"] = { "validation_type": "kfold", "k_folds": 5, "shuffle": True, } il = IterativeLearner(params, callbacks=[early_stop, metric_logger]) il.train(self.data) oof = il.get_out_of_folds() self.assertEqual(len(np.unique(oof.index)), oof.shape[0]) self.assertTrue( np.array_equal(oof.index, self.data["train"]["X"].index)) self.assertTrue(oof.shape[0], self.data["train"]["X"].shape[0]) self.assertTrue( "Private" in list(self.data["train"]["X"]["workclass"])) y_predicted = il.predict(self.data["train"]["X"]) self.assertTrue( "Private" in list(self.data["train"]["X"]["workclass"])) metric = Metric({"name": "logloss"}) loss = metric(self.data["train"]["y"], y_predicted) self.assertTrue(loss < 0.6)
def from_json(self, json_desc): self.library_version = json_desc.get("library_version", self.library_version) self.algorithm_name = json_desc.get("algorithm_name", self.algorithm_name) self.algorithm_short_name = json_desc.get("algorithm_short_name", self.algorithm_short_name) self.uid = json_desc.get("uid", self.uid) self.selected_models = [] models_json = json_desc.get("models") for selected in models_json: model = selected["model"] repeat = selected["repeat"] il = IterativeLearner(model.get("params")) il.from_json(model) self.selected_models += [ # {"model": LearnerFactory.load(model), "repeat": repeat} { "model": il, "repeat": repeat } ]
def train_model(self, params, X, y): early_stop = EarlyStopping({"metric": {"name": "logloss"}}) time_constraint = TimeConstraint({"train_seconds_time_limit": self._time_limit}) il = IterativeLearner(params, callbacks=[early_stop, time_constraint]) il_key = il.get_params_key() if il_key in self._models_params_keys: return None self._models_params_keys += [il_key] if self.should_train_next(il.get_name()): il.train({"train": {"X": X, "y": y}}) return il return None
def train_model(self, params, X, y): metric_logger = MetricLogger({"metric_names": ["logloss", "auc"]}) early_stop = EarlyStopping({"metric": {"name": self._optimize_metric}}) time_constraint = TimeConstraint({"train_seconds_time_limit": self._time_limit}) il = IterativeLearner( params, callbacks=[early_stop, time_constraint, metric_logger] ) il_key = il.get_params_key() if il_key in self._models_params_keys: self._progress_bar.update(1) return None self._models_params_keys += [il_key] if self.should_train_next(il.get_name()): il.train({"train": {"X": X, "y": y}}) self._progress_bar.update(1) return il self._progress_bar.update(1) return None
def test_save_and_load(self): il = IterativeLearner(self.train_params, callbacks=[]) il.train(self.data) metric = Metric({"name": "logloss"}) loss = metric(self.y, il.predict(self.X)) json_desc = il.to_json() il2 = IterativeLearner(json_desc.get("params"), callbacks=[]) self.assertTrue(il.uid != il2.uid) il2.from_json(json_desc) self.assertTrue(il.uid == il2.uid) loss2 = metric(self.y, il2.predict(self.X)) assert_almost_equal(loss, loss2) uids = [i.uid for i in il.learners] uids2 = [i.uid for i in il2.learners] for u in uids: self.assertTrue(u in uids2)
def test_save_and_load(self): self.assertTrue( "Private" in list(self.data["train"]["X"]["workclass"])) early_stop = EarlyStopping({"metric": {"name": "logloss"}}) metric_logger = MetricLogger({"metric_names": ["logloss", "auc"]}) il = IterativeLearner(self.train_params, callbacks=[early_stop, metric_logger]) il.train(self.data) y_predicted = il.predict(self.data["train"]["X"]) metric = Metric({"name": "logloss"}) loss_1 = metric(self.data["train"]["y"], y_predicted) json_desc = il.to_json() il2 = IterativeLearner(self.train_params, callbacks=[]) self.assertTrue(il.uid != il2.uid) il2.from_json(json_desc) self.assertTrue(il.uid == il2.uid) y_predicted_2 = il2.predict(self.data["train"]["X"]) loss_2 = metric(self.data["train"]["y"], y_predicted_2) assert_almost_equal(loss_1, loss_2) uids = [i.uid for i in il.learners] uids2 = [i.uid for i in il2.learners] for u in uids: self.assertTrue(u in uids2)
class AutoML: def __init__( self, total_time_limit=60 * 60, learner_time_limit=120, algorithms=["CatBoost", "Xgboost", "RF", "LightGBM", "NN"], start_random_models=10, hill_climbing_steps=3, top_models_to_improve=5, train_ensemble=True, verbose=True, ): self._total_time_limit = total_time_limit self._time_limit = ( learner_time_limit ) # time limit in seconds for single learner self._train_ensemble = train_ensemble self._models = [] # instances of iterative learner framework or ensemble self._models_params_keys = [] self._best_model = ( None ) # it is instance of iterative learner framework or ensemble self._validation = {"validation_type": "kfold", "k_folds": 5, "shuffle": True} self._start_random_models = start_random_models self._hill_climbing_steps = hill_climbing_steps self._top_models_to_improve = top_models_to_improve self._algorithms = algorithms self._verbose = verbose if self._total_time_limit is not None: estimated_models_to_check = ( len(self._algorithms) * ( self._start_random_models + self._top_models_to_improve * self._hill_climbing_steps * 2 ) * 5 ) # set time limit for single model training # the 0.85 is safe scale factor, to not exceed time limit self._time_limit = self._total_time_limit * 0.85 / estimated_models_to_check if len(self._algorithms) == 0: self._algorithms = list( ModelsRegistry.registry[BINARY_CLASSIFICATION].keys() ) self._fit_time = None self._models_train_time = {} self._threshold, self._metrics_details, self._max_metrics, self._confusion_matrix = ( None, None, None, None, ) def get_additional_metrics(self): # 'target' - the target after processing used for model training # 'prediction' - out of folds predictions of model oof_predictions = self._best_model.get_out_of_folds() self._metrics_details, self._max_metrics, self._confusion_matrix = ComputeAdditionalMetrics.compute( oof_predictions["target"], oof_predictions["prediction"], BINARY_CLASSIFICATION ) self._threshold = self._max_metrics["f1"]["threshold"] # print(self._metrics_details, self._max_metrics, self._confusion_matrix) def _get_model_params(self, model_type, X, y): model_info = ModelsRegistry.registry[BINARY_CLASSIFICATION][model_type] model_params = RandomParameters.get(model_info["params"]) required_preprocessing = model_info["required_preprocessing"] model_additional = model_info["additional"] preprocessing_params = PreprocessingTuner.get( required_preprocessing, {"train": {"X": X, "y": y}}, BINARY_CLASSIFICATION ) return { "additional": model_additional, "preprocessing": preprocessing_params, "validation": self._validation, "learner": { "model_type": model_info["class"].algorithm_short_name, **model_params, }, } def train_model(self, params, X, y): early_stop = EarlyStopping({"metric": {"name": "logloss"}}) time_constraint = TimeConstraint({"train_seconds_time_limit": self._time_limit}) il = IterativeLearner(params, callbacks=[early_stop, time_constraint]) il_key = il.get_params_key() if il_key in self._models_params_keys: return None self._models_params_keys += [il_key] if self.should_train_next(il.get_name()): il.train({"train": {"X": X, "y": y}}) return il return None def verbose_print(self, msg): if self._verbose: print(msg) def log_train_time(self, model_type, train_time): if model_type in self._models_train_time: self._models_train_time[model_type] += [train_time] else: self._models_train_time[model_type] = [train_time] def should_train_next(self, model_type): # no time limit, just train, dont ask if self._total_time_limit is None: return True total_time_already_spend = ( 0 if model_type not in self._models_train_time else np.sum(self._models_train_time[model_type]) ) mean_time_already_spend = ( 0 if model_type not in self._models_train_time else np.mean(self._models_train_time[model_type]) ) if ( total_time_already_spend + mean_time_already_spend < 0.85 * self._total_time_limit / float(len(self._algorithms)) ): return True return False def not_so_random_step(self, X, y): for model_type in self._algorithms: for i in range(self._start_random_models): params = self._get_model_params(model_type, X, y) m = self.train_model(params, X, y) if m is not None: self._models += [m] self.verbose_print( "Learner {} final loss {} time {}".format( m.get_name(), m.get_final_loss(), m.get_train_time() ) ) self.log_train_time(m.get_name(), m.get_train_time()) def hill_climbing_step(self, X, y): for hill_climbing in range(self._hill_climbing_steps): # get models orderer by loss models = [] for m in self._models: models += [(m.callbacks.callbacks[0].final_loss, m)] models = sorted(models, key=lambda x: x[0]) for i in range(min(self._top_models_to_improve, len(models))): m = models[i][1] for p in HillClimbing.get(m.params.get("learner")): if p is not None: all_params = copy.deepcopy(m.params) all_params["learner"] = p new_model = self.train_model(all_params, X, y) if new_model is not None: self._models += [new_model] self.verbose_print( "Learner {} final loss {} time {}".format( new_model.get_name(), new_model.get_final_loss(), new_model.get_train_time(), ) ) self.log_train_time( new_model.get_name(), new_model.get_train_time() ) def ensemble_step(self, y): if self._train_ensemble: self.ensemble = Ensemble() X_oof = self.ensemble.get_oof_matrix(self._models) self.ensemble.fit(X_oof, y) self._models += [self.ensemble] self.verbose_print( "Learner {} final loss {} time {}".format( self.ensemble.get_name(), self.ensemble.get_final_loss(), self.ensemble.get_train_time(), ) ) self.log_train_time( self.ensemble.get_name(), self.ensemble.get_train_time() ) def fit(self, X, y): start_time = time.time() X.reset_index(drop=True, inplace=True) y = np.array(y) if not isinstance(y, pd.DataFrame): y = pd.DataFrame({"target": y}) y.reset_index(drop=True, inplace=True) y = y["target"] # drops rows with missing target X, y = PreprocessingExcludeMissingValues.transform(X, y) # start with not-so-random models self.not_so_random_step(X, y) # perform hill climbing steps on best models self.hill_climbing_step(X, y) # train ensemble self.ensemble_step(y) max_loss = 10e12 for i, m in enumerate(self._models): if m.get_final_loss() < max_loss: self._best_model = m max_loss = m.get_final_loss() self.get_additional_metrics() self._fit_time = time.time() - start_time def predict(self, X): if self._best_model is not None: predictions = self._best_model.predict(X) neg_label, pos_label = predictions.columns[0][2:], predictions.columns[1][2:] if neg_label == '0' and pos_label == '1': neg_label, pos_label = 0, 1 # assume that it is binary classification predictions['label'] = predictions.iloc[:, 1] > self._threshold predictions['label'] = predictions['label'].map({True: pos_label, False: neg_label}) return predictions #return pd.DataFrame( # { # "prediction": self._best_model.predict(X), # "label": self._best_model.predict(X) > self._threshold, # } #) return None def to_json(self): if self._best_model is None: return None return {"best_model": self._best_model.to_json(), "threshold": self._threshold} def from_json(self, json_data): # pretty sure that this can be easily refactored if json_data["best_model"]["algorithm_short_name"] == "Ensemble": self._best_model = Ensemble() self._best_model.from_json(json_data["best_model"]) else: self._best_model = IterativeLearner(json_data["best_model"].get("params")) self._best_model.from_json(json_data["best_model"]) self._threshold = json_data.get("threshold")