def test_individual_op_freeze_trainable(self): from lale.lib.sklearn import LogisticRegression liquid = LogisticRegression(C=0.1, solver='liblinear') self.assertIn('penalty', liquid.free_hyperparams()) self.assertFalse(liquid.is_frozen_trainable()) liquid_grid = get_grid_search_parameter_grids(liquid) self.assertTrue(len(liquid_grid) > 1, f'grid size {len(liquid_grid)}') frozen = liquid.freeze_trainable() self.assertEqual(len(frozen.free_hyperparams()), 0) self.assertTrue(frozen.is_frozen_trainable()) frozen_grid = get_grid_search_parameter_grids(frozen) self.assertEqual(len(frozen_grid), 1)
class HyperoptImpl: def __init__(self, estimator=None, max_evals=50, frac_evals_with_defaults = 0, algo='tpe', cv=5, handle_cv_failure=False, scoring='accuracy', best_score=0.0, max_opt_time=None, max_eval_time=None, pgo:Optional[PGO]=None, show_progressbar=True, args_to_scorer=None, verbose=False): self.max_evals = max_evals if estimator is None: self.estimator = LogisticRegression() else: self.estimator = estimator if frac_evals_with_defaults > 0: self.evals_with_defaults = int(frac_evals_with_defaults*max_evals) else: self.evals_with_defaults = 0 self.algo = algo self.scoring = scoring self.best_score = best_score self.handle_cv_failure = handle_cv_failure self.cv = cv self._trials = hyperopt.Trials() self._default_trials = hyperopt.Trials() self.max_opt_time = max_opt_time self.max_eval_time = max_eval_time self.pgo = pgo self.show_progressbar = show_progressbar if args_to_scorer is not None: self.args_to_scorer = args_to_scorer else: self.args_to_scorer = {} self.verbose = verbose def fit(self, X_train, y_train): opt_start_time = time.time() is_clf = self.estimator.is_classifier() self.cv = check_cv(self.cv, y = y_train, classifier=is_clf) data_schema = lale.helpers.fold_schema( X_train, y_train, self.cv, is_clf) self.search_space = hyperopt.hp.choice( 'meta_model', [hyperopt_search_space(self.estimator, pgo=self.pgo, data_schema=data_schema)]) #Create a search space with default hyperparameters for all trainable parts of the pipeline. #This search space is used for `frac_evals_with_defaults` fraction of the total trials. try: self.search_space_with_defaults = hyperopt.hp.choice('meta_model', [hyperopt_search_space(self.estimator.freeze_trainable(), pgo=self.pgo, data_schema=data_schema)]) except: logger.warning(f"Exception caught during generation of default search space, setting frac_evals_with_defaults to zero.") self.evals_with_defaults = 0 def hyperopt_train_test(params, X_train, y_train): warnings.filterwarnings("ignore") trainable = create_instance_from_hyperopt_search_space(self.estimator, params) try: cv_score, logloss, execution_time = cross_val_score_track_trials(trainable, X_train, y_train, cv=self.cv, scoring=self.scoring, args_to_scorer=self.args_to_scorer) logger.debug("Successful trial of hyperopt with hyperparameters:{}".format(params)) except BaseException as e: #If there is any error in cross validation, use the score based on a random train-test split as the evaluation criterion if self.handle_cv_failure: X_train_part, X_validation, y_train_part, y_validation = train_test_split(X_train, y_train, test_size=0.20) start = time.time() trained = trainable.fit(X_train_part, y_train_part) scorer = check_scoring(trainable, scoring=self.scoring) cv_score = scorer(trained, X_validation, y_validation, **self.args_to_scorer) execution_time = time.time() - start y_pred_proba = trained.predict_proba(X_validation) try: logloss = log_loss(y_true=y_validation, y_pred=y_pred_proba) except BaseException: logloss = 0 logger.debug("Warning, log loss cannot be computed") else: logger.debug(e) logger.debug("Error {} with pipeline:{}".format(e, trainable.to_json())) raise e return cv_score, logloss, execution_time def merge_trials(trials1, trials2): max_tid = max([trial['tid'] for trial in trials1.trials]) for trial in trials2: tid = trial['tid'] + max_tid + 1 hyperopt_trial = hyperopt.Trials().new_trial_docs( tids=[None], specs=[None], results=[None], miscs=[None]) hyperopt_trial[0] = trial hyperopt_trial[0]['tid'] = tid hyperopt_trial[0]['misc']['tid'] = tid for key in hyperopt_trial[0]['misc']['idxs'].keys(): hyperopt_trial[0]['misc']['idxs'][key] = [tid] trials1.insert_trial_docs(hyperopt_trial) trials1.refresh() return trials1 def proc_train_test(params, X_train, y_train, return_dict): return_dict['params'] = copy.deepcopy(params) try: score, logloss, execution_time = hyperopt_train_test(params, X_train=X_train, y_train=y_train) return_dict['loss'] = self.best_score - score return_dict['time'] = execution_time return_dict['log_loss'] = logloss return_dict['status'] = hyperopt.STATUS_OK except BaseException as e: logger.warning(f"Exception caught in Hyperopt:{type(e)}, {traceback.format_exc()} with hyperparams: {params}, setting status to FAIL") return_dict['status'] = hyperopt.STATUS_FAIL return_dict['error_msg'] = f"Exception caught in Hyperopt:{type(e)}, {traceback.format_exc()} with hyperparams: {params}" if self.verbose: print(return_dict['error_msg']) def get_final_trained_estimator(params, X_train, y_train): warnings.filterwarnings("ignore") trainable = create_instance_from_hyperopt_search_space(self.estimator, params) trained = trainable.fit(X_train, y_train) return trained def f(params): current_time = time.time() if (self.max_opt_time is not None) and ((current_time - opt_start_time) > self.max_opt_time) : # if max optimization time set, and we have crossed it, exit optimization completely sys.exit(0) if self.max_eval_time: # Run hyperopt in a subprocess that can be interupted manager = multiprocessing.Manager() proc_dict = manager.dict() p = multiprocessing.Process( target=proc_train_test, args=(params, X_train, y_train, proc_dict)) p.start() p.join(self.max_eval_time) if p.is_alive(): p.terminate() p.join() logger.warning(f"Maximum alloted evaluation time exceeded. with hyperparams: {params}, setting status to FAIL") proc_dict['status'] = hyperopt.STATUS_FAIL if 'status' not in proc_dict: logger.warning(f"Corrupted results, setting status to FAIL") proc_dict['status'] = hyperopt.STATUS_FAIL else: proc_dict = {} proc_train_test(params, X_train, y_train, proc_dict) return proc_dict algo = getattr(hyperopt, self.algo) #Search in the search space with defaults if self.evals_with_defaults > 0: try: hyperopt.fmin(f, self.search_space_with_defaults, algo=algo.suggest, max_evals=self.evals_with_defaults, trials=self._default_trials, rstate=np.random.RandomState(SEED), show_progressbar=self.show_progressbar) except SystemExit : logger.warning('Maximum alloted optimization time exceeded. Optimization exited prematurely') except AllTrialsFailed: self._best_estimator = None if hyperopt.STATUS_OK not in self._trials.statuses(): raise ValueError('Error from hyperopt, none of the trials succeeded.') try : hyperopt.fmin(f, self.search_space, algo=algo.suggest, max_evals=self.max_evals-self.evals_with_defaults, trials=self._trials, rstate=np.random.RandomState(SEED), show_progressbar=self.show_progressbar) except SystemExit : logger.warning('Maximum alloted optimization time exceeded. Optimization exited prematurely') except AllTrialsFailed: self._best_estimator = None if hyperopt.STATUS_OK not in self._trials.statuses(): raise ValueError('Error from hyperopt, none of the trials succeeded.') self._trials = merge_trials(self._trials, self._default_trials) try : best_trial = self._trials.best_trial val_loss = self._trials.best_trial['result']['loss'] if len(self._default_trials) > 0: default_val_loss = self._default_trials.best_trial['result']['loss'] if default_val_loss < val_loss: best_trial = self._default_trials.best_trial best_params = best_trial['result']['params'] logger.info( 'best score: {:.1%}\nbest hyperparams found using {} hyperopt trials: {}'.format( self.best_score - self._trials.average_best_error(), self.max_evals, best_params ) ) trained = get_final_trained_estimator(best_params, X_train, y_train) self._best_estimator = trained except BaseException as e : logger.warning('Unable to extract the best parameters from optimization, the error: {}'.format(e)) self._best_estimator = None return self def predict(self, X_eval): import warnings warnings.filterwarnings("ignore") if self._best_estimator is None: raise ValueError("Can not predict as the best estimator is None. Either an attempt to call `predict` " "before calling `fit` or all the trials during `fit` failed.") trained = self._best_estimator try: predictions = trained.predict(X_eval) except ValueError as e: logger.warning("ValueError in predicting using Hyperopt:{}, the error is:{}".format(trained, e)) predictions = None return predictions def summary(self): """Table summarizing the trial results (ID, loss, time, log_loss, status). Returns ------- result : DataFrame""" def make_record(trial_dict): try: loss = trial_dict['result']['loss'] except BaseException: loss = np.nan try: time = trial_dict['result']['time'] except BaseException: time = '-' try: log_loss = trial_dict['result']['log_loss'] except BaseException: log_loss = np.nan return { 'name': f'p{trial_dict["tid"]}', 'tid': trial_dict['tid'], 'loss': trial_dict['result'].get('loss', float('nan')), 'time': trial_dict['result'].get('time', float('nan')), 'log_loss': trial_dict['result'].get('log_loss', float('nan')), 'status': trial_dict['result']['status']} records = [make_record(td) for td in self._trials.trials] result = pd.DataFrame.from_records(records, index='name') return result def get_pipeline(self, pipeline_name=None, astype='lale'): """Retrieve one of the trials. Parameters ---------- pipeline_name : union type, default None - string Key for table returned by summary(), return a trainable pipeline. - None When not specified, return the best trained pipeline found. astype : 'lale' or 'sklearn', default 'lale' Type of resulting pipeline. Returns ------- result : Trained operator if best, trainable operator otherwise. """ if pipeline_name is None: result = getattr(self, '_best_estimator', None) else: tid = int(pipeline_name[1:]) params = self._trials.trials[tid]['result']['params'] result = create_instance_from_hyperopt_search_space( self.estimator, params) if result is None or astype == 'lale': return result assert astype == 'sklearn', astype return result.export_to_sklearn_pipeline()
class _HyperoptImpl: def __init__( self, *, estimator=None, scoring=None, best_score=0.0, args_to_scorer=None, cv=5, handle_cv_failure=False, verbose=False, show_progressbar=True, algo="tpe", max_evals=50, frac_evals_with_defaults=0, max_opt_time=None, max_eval_time=None, pgo: Optional[PGO] = None, ): self.max_evals = max_evals if estimator is None: self.estimator = LogisticRegression() else: self.estimator = estimator if frac_evals_with_defaults > 0: self.evals_with_defaults = int(frac_evals_with_defaults * max_evals) else: self.evals_with_defaults = 0 self.algo = algo self.scoring = scoring if self.scoring is None: is_clf = self.estimator.is_classifier() if is_clf: self.scoring = "accuracy" else: self.scoring = "r2" self.best_score = best_score self.handle_cv_failure = handle_cv_failure self.cv = cv self._trials = hyperopt.Trials() self._default_trials = hyperopt.Trials() self.max_opt_time = max_opt_time self.max_eval_time = max_eval_time self.pgo = pgo self.show_progressbar = show_progressbar if args_to_scorer is not None: self.args_to_scorer = args_to_scorer else: self.args_to_scorer = {} self.verbose = verbose def _summarize_statuses(self): status_list = self._trials.statuses() status_hist = {} for status in status_list: status_hist[status] = 1 + status_hist.get(status, 0) if hyperopt.STATUS_FAIL in status_hist: print( f"{status_hist[hyperopt.STATUS_FAIL]} out of {len(status_list)} trials failed, call summary() for details." ) if not self.verbose: print("Run with verbose=True to see per-trial exceptions.") def fit(self, X_train, y_train, **fit_params): opt_start_time = time.time() is_clf = self.estimator.is_classifier() self.cv = check_cv(self.cv, y=y_train, classifier=is_clf) data_schema = lale.helpers.fold_schema(X_train, y_train, self.cv, is_clf) self.search_space = hyperopt.hp.choice( "meta_model", [ hyperopt_search_space( self.estimator, pgo=self.pgo, data_schema=data_schema ) ], ) # Create a search space with default hyperparameters for all trainable parts of the pipeline. # This search space is used for `frac_evals_with_defaults` fraction of the total trials. try: self.search_space_with_defaults = hyperopt.hp.choice( "meta_model", [ hyperopt_search_space( self.estimator.freeze_trainable(), pgo=self.pgo, data_schema=data_schema, ) ], ) except Exception: logger.warning( "Exception caught during generation of default search space, setting frac_evals_with_defaults to zero." ) self.evals_with_defaults = 0 def hyperopt_train_test(params, X_train, y_train): warnings.filterwarnings("ignore") trainable = create_instance_from_hyperopt_search_space( self.estimator, params ) try: cv_score, logloss, execution_time = cross_val_score_track_trials( trainable, X_train, y_train, cv=self.cv, scoring=self.scoring, args_to_scorer=self.args_to_scorer, ) logger.debug( "Successful trial of hyperopt with hyperparameters:{}".format( params ) ) except BaseException as e: # If there is any error in cross validation, use the score based on a random train-test split as the evaluation criterion if self.handle_cv_failure and trainable is not None: ( X_train_part, X_validation, y_train_part, y_validation, ) = train_test_split(X_train, y_train, test_size=0.20) start = time.time() trained = trainable.fit(X_train_part, y_train_part, **fit_params) scorer = check_scoring(trainable, scoring=self.scoring) cv_score = scorer( trained, X_validation, y_validation, **self.args_to_scorer ) execution_time = time.time() - start y_pred_proba = trained.predict_proba(X_validation) try: logloss = log_loss(y_true=y_validation, y_pred=y_pred_proba) except BaseException: logloss = 0 logger.debug("Warning, log loss cannot be computed") else: logger.debug(e) if trainable is None: logger.debug( "Error {} with uncreatable pipeline with parameters:{}".format( e, lale.pretty_print.hyperparams_to_string(params) ) ) else: logger.debug( "Error {} with pipeline:{}".format(e, trainable.to_json()) ) raise e return cv_score, logloss, execution_time def merge_trials(trials1, trials2): max_tid = max([trial["tid"] for trial in trials1.trials]) for trial in trials2: tid = trial["tid"] + max_tid + 1 hyperopt_trial = hyperopt.Trials().new_trial_docs( tids=[None], specs=[None], results=[None], miscs=[None] ) hyperopt_trial[0] = trial hyperopt_trial[0]["tid"] = tid hyperopt_trial[0]["misc"]["tid"] = tid for key in hyperopt_trial[0]["misc"]["idxs"].keys(): hyperopt_trial[0]["misc"]["idxs"][key] = [tid] trials1.insert_trial_docs(hyperopt_trial) trials1.refresh() return trials1 def proc_train_test(params, X_train, y_train, return_dict): return_dict["params"] = copy.deepcopy(params) try: score, logloss, execution_time = hyperopt_train_test( params, X_train=X_train, y_train=y_train ) return_dict["loss"] = self.best_score - score return_dict["time"] = execution_time return_dict["log_loss"] = logloss return_dict["status"] = hyperopt.STATUS_OK except BaseException as e: exception_type = f"{type(e).__module__}.{type(e).__name__}" try: trainable = create_instance_from_hyperopt_search_space( self.estimator, params ) if trainable is None: trial_info = f"hyperparams: {params}" else: trial_info = f'pipeline: """{trainable.pretty_print(show_imports=False)}"""' except BaseException: trial_info = f"hyperparams: {params}" error_msg = f"Exception caught in Hyperopt: {exception_type}, {traceback.format_exc()}with {trial_info}" logger.warning(error_msg + ", setting status to FAIL") return_dict["status"] = hyperopt.STATUS_FAIL return_dict["error_msg"] = error_msg if self.verbose: print(return_dict["error_msg"]) def get_final_trained_estimator(params, X_train, y_train): warnings.filterwarnings("ignore") trainable = create_instance_from_hyperopt_search_space( self.estimator, params ) if trainable is None: return None else: trained = trainable.fit(X_train, y_train, **fit_params) return trained def f(params): current_time = time.time() if (self.max_opt_time is not None) and ( (current_time - opt_start_time) > self.max_opt_time ): # if max optimization time set, and we have crossed it, exit optimization completely sys.exit(0) if self.max_eval_time: # Run hyperopt in a subprocess that can be interupted manager = multiprocessing.Manager() proc_dict: Dict[str, Any] = manager.dict() p = multiprocessing.Process( target=proc_train_test, args=(params, X_train, y_train, proc_dict) ) p.start() p.join(self.max_eval_time) if p.is_alive(): p.terminate() p.join() logger.warning( f"Maximum alloted evaluation time exceeded. with hyperparams: {params}, setting status to FAIL" ) proc_dict["status"] = hyperopt.STATUS_FAIL if "status" not in proc_dict: logger.warning("Corrupted results, setting status to FAIL") proc_dict["status"] = hyperopt.STATUS_FAIL else: proc_dict = {} proc_train_test(params, X_train, y_train, proc_dict) return proc_dict algo = getattr(hyperopt, self.algo) # Search in the search space with defaults if self.evals_with_defaults > 0: try: hyperopt.fmin( f, self.search_space_with_defaults, algo=algo.suggest, max_evals=self.evals_with_defaults, trials=self._default_trials, rstate=np.random.RandomState(SEED), show_progressbar=self.show_progressbar, ) except SystemExit: logger.warning( "Maximum alloted optimization time exceeded. Optimization exited prematurely" ) except AllTrialsFailed: self._best_estimator = None if hyperopt.STATUS_OK not in self._trials.statuses(): raise ValueError( "Error from hyperopt, none of the trials succeeded." ) try: hyperopt.fmin( f, self.search_space, algo=algo.suggest, max_evals=self.max_evals - self.evals_with_defaults, trials=self._trials, rstate=np.random.RandomState(SEED), show_progressbar=self.show_progressbar, ) except SystemExit: logger.warning( "Maximum alloted optimization time exceeded. Optimization exited prematurely" ) except AllTrialsFailed: self._best_estimator = None if hyperopt.STATUS_OK not in self._trials.statuses(): self._summarize_statuses() raise ValueError("Error from hyperopt, none of the trials succeeded.") self._trials = merge_trials(self._trials, self._default_trials) if self.show_progressbar: self._summarize_statuses() try: best_trial = self._trials.best_trial val_loss = self._trials.best_trial["result"]["loss"] if len(self._default_trials) > 0: default_val_loss = self._default_trials.best_trial["result"]["loss"] if default_val_loss < val_loss: best_trial = self._default_trials.best_trial best_params = best_trial["result"]["params"] logger.info( "best score: {:.1%}\nbest hyperparams found using {} hyperopt trials: {}".format( self.best_score - self._trials.average_best_error(), self.max_evals, best_params, ) ) trained = get_final_trained_estimator(best_params, X_train, y_train) self._best_estimator = trained except BaseException as e: logger.warning( "Unable to extract the best parameters from optimization, the error: {}".format( e ) ) self._best_estimator = None return self def predict(self, X_eval, **predict_params): import warnings warnings.filterwarnings("ignore") if self._best_estimator is None: raise ValueError( "Can not predict as the best estimator is None. Either an attempt to call `predict` " "before calling `fit` or all the trials during `fit` failed." ) trained = self._best_estimator try: predictions = trained.predict(X_eval, **predict_params) except ValueError as e: logger.warning( "ValueError in predicting using Hyperopt:{}, the error is:{}".format( trained, e ) ) predictions = None return predictions def summary(self): """Table summarizing the trial results (ID, loss, time, log_loss, status). Returns ------- result : DataFrame""" def make_record(trial_dict): return { "name": f'p{trial_dict["tid"]}', "tid": trial_dict["tid"], "loss": trial_dict["result"].get("loss", float("nan")), "time": trial_dict["result"].get("time", float("nan")), "log_loss": trial_dict["result"].get("log_loss", float("nan")), "status": trial_dict["result"]["status"], } records = [make_record(td) for td in self._trials.trials] result = pd.DataFrame.from_records(records, index="name") return result def get_pipeline(self, pipeline_name=None, astype="lale"): """Retrieve one of the trials. Parameters ---------- pipeline_name : union type, default None - string Key for table returned by summary(), return a trainable pipeline. - None When not specified, return the best trained pipeline found. astype : 'lale' or 'sklearn', default 'lale' Type of resulting pipeline. Returns ------- result : Trained operator if best, trainable operator otherwise.""" best_name = None if self._best_estimator is not None: best_name = f'p{self._trials.best_trial["tid"]}' if pipeline_name is None: pipeline_name = best_name if pipeline_name == best_name: result = getattr(self, "_best_estimator", None) else: assert pipeline_name is not None tid = int(pipeline_name[1:]) params = self._trials.trials[tid]["result"]["params"] result = create_instance_from_hyperopt_search_space(self.estimator, params) if result is None or astype == "lale": return result assert astype == "sklearn", astype return result.export_to_sklearn_pipeline()