def _predict(self, test_data: DataNode): if self.ensemble_method is not None: if self.es is None and self.eval_type == 'cv': raise AttributeError( "Please call refit() for cross-validation!") elif self.es is None: raise AttributeError("AutoML is not fitted!") return self.es.predict(test_data) else: try: best_op_list, estimator = load_combined_transformer_estimator( self.output_dir, self.incumbent, self.timestamp) except Exception as e: if self.eval_type == 'cv': raise AttributeError( "Please call refit() for cross-validation!") else: raise e test_data_node = test_data.copy_() test_data_node = construct_node(test_data_node, best_op_list) if self.task_type in CLS_TASKS: return estimator.predict_proba(test_data_node.data[0]) else: return estimator.predict(test_data_node.data[0])
def train_valid_split(node: DataNode): X, y = node.copy_().data sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=1) for train_index, test_index in sss.split(X, y): X_train, X_val = X[train_index], X[test_index] y_train, y_val = y[train_index], y[test_index] train_data = DataNode(data=[X_train, y_train], feature_type=node.feature_types.copy()) valid_data = DataNode(data=[X_val, y_val], feature_type=node.feature_types.copy()) return train_data, valid_data
def apply(self, data_node: DataNode, ref_node: DataNode, phase='test'): input_node = data_node.copy_() fe_config = ref_node.config if ref_node is None: return data_node elif fe_config in self.node_dict: fe_trans_list = self.node_dict[fe_config][1] for i, tran in enumerate(fe_trans_list): if phase == 'test' and i == 2: # Disable balancer continue if tran is not None: input_node = tran.operate(input_node) else: self.logger.info("Ref node config:") self.logger.info(str(fe_config)) self.logger.info("Node history in optimizer:") self.logger.info(str(self.node_dict)) raise ValueError("Ref node not in history!") return input_node
def _predict(self, test_data: DataNode): if self.ensemble_method is not None: if self.es is None: raise AttributeError("AutoML is not fitted!") return self.es.predict(test_data) else: if self.inner_opt_algorithm == 'combined': best_op_list, estimator = load_combined_transformer_estimator( self.output_dir, self.best_config, self.timestamp) else: best_op_list, estimator = load_transformer_estimator( self.output_dir, self.optimal_algo_id, self.best_hpo_config, self.best_fe_config, self.timestamp) test_data_node = test_data.copy_() test_data_node = construct_node(test_data_node, best_op_list) if self.task_type in CLS_TASKS: return estimator.predict_proba(test_data_node.data[0]) else: return estimator.predict(test_data_node.data[0])
def parse_config(data_node: DataNode, config: dict, record=False, skip_balance=False, if_imbal=False): """ Transform the data node based on the pipeline specified by configuration. :param data_node: :param config: :param record: :return: the resulting data node. """ _preprocessor_candidates = get_combined_fe_candidtates(_preprocessor, _gen_addons) _preprocessor_candidates = get_combined_fe_candidtates(_preprocessor_candidates, _sel_addons) _rescaler_candidates = get_combined_fe_candidtates(_rescaler, _res_addons) if not if_imbal: _balancer_candidates = get_combined_fe_candidtates(_bal_balancer, _bal_addons) else: _balancer_candidates = get_combined_fe_candidtates(_imb_balancer, _bal_addons) # Remove the indicator in config_dict. config_dict = config.copy() image_pre_id = config_dict.get('image_preprocessor', None) if image_pre_id: config_dict.pop('image_preprocessor') text_pre_id = config_dict.get('text_preprocessor', None) if text_pre_id: config_dict.pop('text_preprocessor') def tran_operate(id, tran_set, config, node): _config = {} for key in config: if id in key: config_name = key.split(':')[1] _config[config_name] = config[key] tran = tran_set[id](**_config) output_node = tran.operate(node) return output_node, tran _node = data_node.copy_() tran_dict = dict() # Image preprocessor if image_pre_id: _node, image_tran = tran_operate(image_pre_id, _image_preprocessor, config_dict, _node) tran_dict['image_preprocessor'] = image_tran # Text preprocessor if text_pre_id: _node, text_tran = tran_operate(text_pre_id, _text_preprocessor, config_dict, _node) tran_dict['text_preprocessor'] = text_tran for stage in stage_list: if stage == 'balancer': if skip_balance: op_id = 'empty' else: if stage in config_dict: op_id = config_dict[stage] config_dict.pop(stage) else: op_id = 'empty' else: op_id = config_dict[stage] config_dict.pop(stage) if stage == 'preprocessor': _node, tran = tran_operate(op_id, _preprocessor_candidates, config_dict, _node) elif stage == 'rescaler': _node, tran = tran_operate(op_id, _rescaler_candidates, config_dict, _node) elif stage == 'balancer': _node, tran = tran_operate(op_id, _balancer_candidates, config_dict, _node) else: # Third party stage _node, tran = tran_operate(op_id, thirdparty_candidates_dict[stage], config_dict, _node) tran_dict[stage] = tran _node.config = config if record: return _node, tran_dict return _node
def __init__(self, task_type, trial_num, classifier_ids: List[str], data: DataNode, metric='acc', ensemble_method='ensemble_selection', ensemble_size=10, per_run_time_limit=300, output_dir=None, dataset_name='default_dataset', eval_type='holdout', share_feature=False, inner_opt_algorithm='rb', fe_algo='bo', time_limit=None, n_jobs=1, seed=1): """ :param classifier_ids: subset of {'adaboost','bernoulli_nb','decision_tree','extra_trees','gaussian_nb','gradient_boosting', 'gradient_boosting','k_nearest_neighbors','lda','liblinear_svc','libsvm_svc','multinomial_nb','passive_aggressive','qda', 'random_forest','sgd'} """ self.timestamp = time.time() self.task_type = task_type self.metric = get_metric(metric) self.original_data = data.copy_() self.ensemble_method = ensemble_method self.ensemble_size = ensemble_size self.trial_num = trial_num self.n_jobs = n_jobs self.alpha = 4 self.B = 0.01 self.seed = seed self.shared_mode = share_feature self.output_dir = output_dir np.random.seed(self.seed) # Best configuration. self.optimal_algo_id = None self.nbest_algo_ids = None self.best_lower_bounds = None self.es = None # Set up backend. self.dataset_name = dataset_name self.time_limit = time_limit self.start_time = time.time() self.logger = get_logger('Soln-ml: %s' % dataset_name) # Bandit settings. self.incumbent_perf = -1. self.arms = classifier_ids self.include_algorithms = classifier_ids self.rewards = dict() self.sub_bandits = dict() self.evaluation_cost = dict() self.fe_datanodes = dict() self.eval_type = eval_type self.fe_algo = fe_algo self.inner_opt_algorithm = inner_opt_algorithm for arm in self.arms: self.rewards[arm] = list() self.evaluation_cost[arm] = list() self.fe_datanodes[arm] = list() self.sub_bandits[arm] = SecondLayerBandit( self.task_type, arm, self.original_data, metric=self.metric, output_dir=output_dir, per_run_time_limit=per_run_time_limit, share_fe=self.shared_mode, seed=self.seed, eval_type=eval_type, dataset_id=dataset_name, n_jobs=self.n_jobs, fe_algo=fe_algo, mth=inner_opt_algorithm, ) self.action_sequence = list() self.final_rewards = list() self.start_time = time.time() self.time_records = list()
def __init__(self, task_type, trial_num, classifier_ids: List[str], data: DataNode, include_preprocessors=None, time_limit=None, metric='acc', ensemble_method='ensemble_selection', ensemble_size=50, per_run_time_limit=300, output_dir="logs", dataset_name='default_dataset', eval_type='holdout', inner_opt_algorithm='fixed', enable_fe=True, fe_algo='bo', n_jobs=1, seed=1): """ :param classifier_ids: subset of {'adaboost','bernoulli_nb','decision_tree','extra_trees','gaussian_nb','gradient_boosting', 'gradient_boosting','k_nearest_neighbors','lda','liblinear_svc','libsvm_svc','multinomial_nb','passive_aggressive','qda', 'random_forest','sgd'} """ self.timestamp = time.time() self.task_type = task_type self.include_preprocessors = include_preprocessors self.metric = get_metric(metric) self.original_data = data.copy_() self.ensemble_method = ensemble_method self.ensemble_size = ensemble_size self.trial_num = trial_num self.n_jobs = n_jobs self.alpha = 4 self.seed = seed self.output_dir = output_dir self.early_stop_flag = False # np.random.seed(self.seed) # Best configuration. self.optimal_algo_id = None self.nbest_algo_ids = None self.best_lower_bounds = None self.es = None # Set up backend. self.dataset_name = dataset_name self.time_limit = time_limit self.start_time = time.time() self.logger = get_logger('Soln-ml: %s' % dataset_name) # Bandit settings. self.incumbent_perf = -float("INF") self.arms = classifier_ids self.include_algorithms = classifier_ids self.rewards = dict() self.sub_bandits = dict() self.evaluation_cost = dict() self.eval_type = eval_type self.enable_fe = enable_fe self.fe_algo = fe_algo self.inner_opt_algorithm = inner_opt_algorithm # Record the execution cost for each arm. if not (self.time_limit is None) ^ (self.trial_num is None): raise ValueError('Please set one of time_limit or trial_num.') self.arm_cost_stats = dict() for _arm in self.arms: self.arm_cost_stats[_arm] = list() for arm in self.arms: self.rewards[arm] = list() self.evaluation_cost[arm] = list() self.sub_bandits[arm] = SecondLayerBandit( self.task_type, arm, self.original_data, include_preprocessors=self.include_preprocessors, metric=self.metric, output_dir=output_dir, per_run_time_limit=per_run_time_limit, seed=self.seed, eval_type=eval_type, dataset_id=dataset_name, n_jobs=self.n_jobs, fe_algo=fe_algo, mth=self.inner_opt_algorithm, timestamp=self.timestamp) self.action_sequence = list() self.final_rewards = list() self.start_time = time.time() self.time_records = list()
def __init__(self, node_list, node_index, task_type, timestamp, fe_config_space: ConfigurationSpace, cash_config_space: ConfigurationSpace, data: DataNode, fixed_config=None, trial_num=0, time_limit=None, metric='acc', ensemble_method='ensemble_selection', ensemble_size=50, per_run_time_limit=300, output_dir="logs", dataset_name='default_dataset', eval_type='holdout', resampling_params=None, n_jobs=1, seed=1): # Tree setting self.node_list = node_list self.node_index = node_index # Set up backend. self.dataset_name = dataset_name self.trial_num = trial_num self.time_limit = time_limit self.per_run_time_limit = per_run_time_limit self.start_time = time.time() self.logger = get_logger('Soln-ml: %s' % dataset_name) # Basic settings. self.eval_type = eval_type self.resampling_params = resampling_params self.task_type = task_type self.timestamp = timestamp self.fe_config_space = fe_config_space self.cash_config_space = cash_config_space self.fixed_config = fixed_config self.original_data = data.copy_() self.metric = get_metric(metric) self.ensemble_method = ensemble_method self.ensemble_size = ensemble_size self.n_jobs = n_jobs self.seed = seed self.output_dir = output_dir self.early_stop_flag = False self.timeout_flag = False self.incumbent_perf = -float("INF") self.incumbent = None self.eval_dict = dict() if self.task_type in CLS_TASKS: self.if_imbal = is_imbalanced_dataset(self.original_data) else: self.if_imbal = False self.es = None
def _parse(self, data_node: DataNode, config, record=False, skip_balance=False): """ Transform the data node based on the pipeline specified by configuration. :param data_node: :param config: :param record: :return: the resulting data node. """ # Remove the indicator in config_dict. config_dict = config.get_dictionary().copy() pre1_id = config_dict['preprocessor1'] config_dict.pop('preprocessor1') pre2_id = config_dict['preprocessor2'] config_dict.pop('preprocessor2') if skip_balance: bal_id = 'empty' else: if 'balancer' in config_dict: bal_id = config_dict['balancer'] config_dict.pop('balancer') else: bal_id = 'empty' gen_id = config_dict['generator'] config_dict.pop('generator') res_id = config_dict['rescaler'] config_dict.pop('rescaler') sel_id = config_dict['selector'] config_dict.pop('selector') def tran_operate(id, tran_set, config, node): if id != "empty": _config = {} for key in config: if id in key: config_name = key.split(':')[1] _config[config_name] = config[key] tran = tran_set[id](**_config) output_node = tran.operate(node) return output_node, tran return node, None _node = data_node.copy_() # Preprocessor1 _node, pre1_tran = tran_operate(pre1_id, _preprocessor1, config_dict, _node) # Preprocessor2 _node, pre2_tran = tran_operate(pre2_id, _preprocessor2, config_dict, _node) # Balancer _balancer = _bal_balancer if self.if_bal else _imb_balancer _node, bal_tran = tran_operate(bal_id, _balancer, config_dict, _node) # Rescaler _node, res_tran = tran_operate(res_id, _rescaler, config_dict, _node) # Generator _node, gen_tran = tran_operate(gen_id, _generator, config_dict, _node) # Selector _node, sel_tran = tran_operate(sel_id, _selector, config_dict, _node) _node.config = config if record: return _node, [pre1_tran, pre2_tran, bal_tran, res_tran, gen_tran, sel_tran] return _node
def __init__(self, task_type, estimator_id: str, data: DataNode, metric, include_preprocessors=None, share_fe=False, output_dir='logs', per_run_time_limit=120, per_run_mem_limit=5120, dataset_id='default', eval_type='holdout', mth='rb', sw_size=3, n_jobs=1, seed=1, enable_fe=True, fe_algo='bo', number_of_unit_resource=2, total_resource=30, timestamp=None): self.task_type = task_type self.metric = metric self.number_of_unit_resource = number_of_unit_resource # One unit of resource, that's, the number of trials per iteration. self.one_unit_of_resource = 5 self.total_resource = total_resource self.per_run_time_limit = per_run_time_limit self.per_run_mem_limit = per_run_mem_limit self.estimator_id = estimator_id self.include_preprocessors = include_preprocessors self.evaluation_type = eval_type self.original_data = data.copy_() self.share_fe = share_fe self.output_dir = output_dir self.n_jobs = n_jobs self.mth = mth self.seed = seed self.sliding_window_size = sw_size task_id = '%s-%d-%s' % (dataset_id, seed, estimator_id) self.logger = get_logger(self.__class__.__name__ + '-' + task_id) # Bandit settings. # self.arms = ['fe', 'hpo'] self.arms = ['hpo', 'fe'] self.rewards = dict() self.optimizer = dict() self.evaluation_cost = dict() self.update_flag = dict() # Global incumbent. self.inc = dict() self.local_inc = dict() self.local_hist = {'fe': [], 'hpo': []} self.inc_record = {'fe': list(), 'hpo': list()} self.exp_output = dict() self.eval_dict = {'fe': dict(), 'hpo': dict()} for arm in self.arms: self.rewards[arm] = list() self.update_flag[arm] = False self.evaluation_cost[arm] = list() self.exp_output[arm] = dict() self.pull_cnt = 0 self.action_sequence = list() self.final_rewards = list() self.incumbent_config = None self.incumbent_perf = float("-INF") self.early_stopped_flag = False self.first_start = True self.include_text = True if TEXT in self.original_data.feature_types else False self.include_image = True if IMAGE in self.original_data.feature_types else False # Fetch hyperparameter space. if self.task_type in CLS_TASKS: from solnml.components.models.classification import _classifiers, _addons _candidates = get_combined_candidtates(_classifiers, _addons) if estimator_id in _candidates: clf_class = _candidates[estimator_id] else: raise ValueError("Algorithm %s not supported!" % estimator_id) cs = clf_class.get_hyperparameter_search_space() elif self.task_type in RGS_TASKS: from solnml.components.models.regression import _regressors, _addons _candidates = get_combined_candidtates(_regressors, _addons) if estimator_id in _candidates: reg_class = _candidates[estimator_id] else: raise ValueError("Algorithm %s not supported!" % estimator_id) cs = reg_class.get_hyperparameter_search_space() else: raise ValueError("Unknown task type %s!" % self.task_type) self.config_space = cs self.default_config = cs.get_default_configuration() self.config_space.seed(self.seed) self.if_imbal = is_imbalanced_dataset(self.original_data) self.fe_config_space = get_task_hyperparameter_space( self.task_type, self.estimator_id, include_preprocessors=self.include_preprocessors, include_text=self.include_text, include_image=self.include_image, if_imbal=self.if_imbal) self.fe_default_config = self.fe_config_space.get_default_configuration( ) self.timestamp = timestamp # Build the Feature Engineering component. if self.task_type in CLS_TASKS: fe_evaluator = ClassificationEvaluator( self.default_config, self.fe_default_config, estimator_id, scorer=self.metric, data_node=self.original_data, name='fe', resampling_strategy=self.evaluation_type, if_imbal=self.if_imbal, seed=self.seed, output_dir=self.output_dir, timestamp=self.timestamp) hpo_evaluator = ClassificationEvaluator( self.default_config, self.fe_default_config, estimator_id, scorer=self.metric, data_node=self.original_data, name='hpo', resampling_strategy=self.evaluation_type, if_imbal=self.if_imbal, seed=self.seed, output_dir=self.output_dir, timestamp=self.timestamp) elif self.task_type in RGS_TASKS: fe_evaluator = RegressionEvaluator( self.default_config, self.fe_default_config, estimator_id, scorer=self.metric, data_node=self.original_data, name='fe', resampling_strategy=self.evaluation_type, seed=self.seed, output_dir=self.output_dir, timestamp=self.timestamp) hpo_evaluator = RegressionEvaluator( self.default_config, self.fe_default_config, estimator_id, scorer=self.metric, data_node=self.original_data, name='hpo', resampling_strategy=self.evaluation_type, seed=self.seed, output_dir=self.output_dir, timestamp=self.timestamp) else: raise ValueError('Invalid task type!') if self.mth != 'combined': self.enable_fe = enable_fe trials_per_iter = self.one_unit_of_resource * self.number_of_unit_resource self.optimizer['fe'] = build_fe_optimizer( self.evaluation_type, fe_evaluator, self.fe_config_space, per_run_time_limit=per_run_time_limit, per_run_mem_limit=per_run_mem_limit, inner_iter_num_per_iter=trials_per_iter, output_dir=output_dir, seed=self.seed, n_jobs=n_jobs) self.inc['fe'], self.local_inc[ 'fe'] = self.fe_default_config, self.fe_default_config # Build the HPO component. # trials_per_iter = max(len(self.optimizer['fe'].trans_types), 20) trials_per_iter = self.one_unit_of_resource * self.number_of_unit_resource self.optimizer['hpo'] = build_hpo_optimizer( self.evaluation_type, hpo_evaluator, cs, output_dir=output_dir, per_run_time_limit=per_run_time_limit, inner_iter_num_per_iter=trials_per_iter, seed=self.seed, n_jobs=n_jobs) self.inc['hpo'], self.local_inc[ 'hpo'] = self.default_config, self.default_config self.init_config = cs.get_default_configuration() self.local_hist['fe'].append(self.fe_default_config) self.local_hist['hpo'].append(self.default_config) else: self.rewards = list() self.evaluation_cost = list() self.eval_dict = {} trials_per_iter = self.one_unit_of_resource * self.number_of_unit_resource if self.task_type in CLS_TASKS: from solnml.utils.combined_cls_evaluator import get_combined_cs from solnml.utils.combined_cls_evaluator import CombinedClassificationEvaluator as CombinedEvaluator else: from solnml.utils.combined_rgs_evaluator import get_combined_cs from solnml.utils.combined_rgs_evaluator import CombinedRegressionEvaluator as CombinedEvaluator self.evaluator = CombinedEvaluator( estimator_id, scorer=self.metric, data_node=self.original_data, if_imbal=self.if_imbal, timestamp=self.timestamp, output_dir=self.output_dir, resampling_strategy=self.evaluation_type) cs = get_combined_cs( self.estimator_id, self.task_type, include_image=self.include_image, include_text=self.include_text, include_preprocessors=self.include_preprocessors, if_imbal=self.if_imbal) self.optimizer = build_hpo_optimizer( self.evaluation_type, self.evaluator, cs, output_dir=self.output_dir, per_run_time_limit=self.per_run_time_limit, inner_iter_num_per_iter=trials_per_iter, seed=self.seed, n_jobs=self.n_jobs)
def __init__(self, node_list, node_index, task_type, timestamp, fe_config_space: ConfigurationSpace, cash_config_space: ConfigurationSpace, data: DataNode, fixed_config=None, time_limit=None, trial_num=0, metric='acc', ensemble_method='ensemble_selection', ensemble_size=50, per_run_time_limit=300, output_dir="logs", dataset_name='default_dataset', eval_type='holdout', resampling_params=None, n_jobs=1, seed=1): """ :param classifier_ids: subset of {'adaboost','bernoulli_nb','decision_tree','extra_trees','gaussian_nb','gradient_boosting', 'gradient_boosting','k_nearest_neighbors','lda','liblinear_svc','libsvm_svc','multinomial_nb','passive_aggressive','qda', 'random_forest','sgd'} """ super(ConditioningBlock, self).__init__(node_list, node_index, task_type, timestamp, fe_config_space, cash_config_space, data, fixed_config=fixed_config, time_limit=time_limit, trial_num=trial_num, metric=metric, ensemble_method=ensemble_method, ensemble_size=ensemble_size, per_run_time_limit=per_run_time_limit, output_dir=output_dir, dataset_name=dataset_name, eval_type=eval_type, resampling_params=resampling_params, n_jobs=n_jobs, seed=seed) # Best configuration. self.optimal_arm = None self.best_lower_bounds = None # Bandit settings. self.alpha = 4 self.arms = list( cash_config_space.get_hyperparameter('algorithm').choices) self.rewards = dict() self.sub_bandits = dict() self.evaluation_cost = dict() self.arm_cost_stats = dict() for _arm in self.arms: self.arm_cost_stats[_arm] = list() for arm in self.arms: self.rewards[arm] = list() self.evaluation_cost[arm] = list() hps = cash_config_space.get_hyperparameters() cs = ConfigurationSpace() cs.add_hyperparameter(Constant('algorithm', arm)) for hp in hps: if hp.name.split(':')[0] == arm: cs.add_hyperparameter(hp) # Add active conditions conds = cash_config_space.get_conditions() for cond in conds: try: cs.add_condition(cond) except: pass # Add active forbidden clauses forbids = cash_config_space.get_forbiddens() for forbid in forbids: try: cs.add_forbidden_clause(forbid) except: pass from solnml.blocks.block_utils import get_node_type child_type = get_node_type(node_list, node_index + 1) self.sub_bandits[arm] = child_type( node_list, node_index + 1, task_type, timestamp, deepcopy(fe_config_space), deepcopy(cs), data.copy_(), fixed_config=fixed_config, time_limit=time_limit, metric=metric, ensemble_method=ensemble_method, ensemble_size=ensemble_size, per_run_time_limit=per_run_time_limit, output_dir=output_dir, dataset_name=dataset_name, eval_type=eval_type, resampling_params=resampling_params, n_jobs=n_jobs, seed=seed) self.action_sequence = list() self.final_rewards = list() self.start_time = time.time() self.time_records = list() # Initialize the parameters. self.pull_cnt = 0 self.pick_id = 0 self.update_cnt = 0 arm_num = len(self.arms) self.optimal_algo_id = None self.arm_candidate = self.arms.copy() self.best_lower_bounds = np.zeros(arm_num) _iter_id = 0 if self.time_limit is None: if arm_num * self.alpha > self.trial_num: raise ValueError('Trial number should be larger than %d.' % (arm_num * self.alpha)) else: self.trial_num = MAX_INT
def __init__(self, task_type, estimator_id: str, data: DataNode, metric, share_fe=False, output_dir='logs', per_run_time_limit=120, per_run_mem_limit=5120, dataset_id='default', eval_type='holdout', mth='rb', sw_size=3, n_jobs=1, seed=1, fe_algo='tree_based', enable_intersection=True, number_of_unit_resource=2, total_resource=30): self.task_type = task_type self.metric = metric self.number_of_unit_resource = number_of_unit_resource # One unit of resource, that's, the number of trials per iteration. self.one_unit_of_resource = 5 self.total_resource = total_resource self.per_run_time_limit = per_run_time_limit self.per_run_mem_limit = per_run_mem_limit self.estimator_id = estimator_id self.evaluation_type = eval_type self.original_data = data.copy_() self.share_fe = share_fe self.output_dir = output_dir self.n_jobs = n_jobs self.mth = mth self.seed = seed self.sliding_window_size = sw_size task_id = '%s-%d-%s' % (dataset_id, seed, estimator_id) self.logger = get_logger(self.__class__.__name__ + '-' + task_id) np.random.seed(self.seed) # Bandit settings. # self.arms = ['fe', 'hpo'] self.arms = ['hpo', 'fe'] self.rewards = dict() self.optimizer = dict() self.evaluation_cost = dict() self.update_flag = dict() # Global incumbent. self.inc = dict() self.local_inc = dict() self.local_hist = {'fe': [], 'hpo': []} for arm in self.arms: self.rewards[arm] = list() self.update_flag[arm] = False self.evaluation_cost[arm] = list() self.pull_cnt = 0 self.action_sequence = list() self.final_rewards = list() self.incumbent_perf = float("-INF") self.early_stopped_flag = False self.enable_intersection = enable_intersection # Fetch hyperparameter space. if self.task_type in CLS_TASKS: from solnml.components.models.classification import _classifiers, _addons if estimator_id in _classifiers: clf_class = _classifiers[estimator_id] elif estimator_id in _addons.components: clf_class = _addons.components[estimator_id] else: raise ValueError("Algorithm %s not supported!" % estimator_id) cs = clf_class.get_hyperparameter_search_space() model = UnParametrizedHyperparameter("estimator", estimator_id) cs.add_hyperparameter(model) elif self.task_type in REG_TASKS: from solnml.components.models.regression import _regressors, _addons if estimator_id in _regressors: reg_class = _regressors[estimator_id] elif estimator_id in _addons.components: reg_class = _addons.components[estimator_id] else: raise ValueError("Algorithm %s not supported!" % estimator_id) cs = reg_class.get_hyperparameter_search_space() model = UnParametrizedHyperparameter("estimator", estimator_id) cs.add_hyperparameter(model) else: raise ValueError("Unknown task type %s!" % self.task_type) self.config_space = cs self.default_config = cs.get_default_configuration() self.config_space.seed(self.seed) # Build the Feature Engineering component. if self.task_type in CLS_TASKS: fe_evaluator = ClassificationEvaluator( self.default_config, scorer=self.metric, name='fe', resampling_strategy=self.evaluation_type, seed=self.seed) hpo_evaluator = ClassificationEvaluator( self.default_config, scorer=self.metric, data_node=self.original_data, name='hpo', resampling_strategy=self.evaluation_type, seed=self.seed) elif self.task_type in REG_TASKS: fe_evaluator = RegressionEvaluator( self.default_config, scorer=self.metric, name='fe', resampling_strategy=self.evaluation_type, seed=self.seed) hpo_evaluator = RegressionEvaluator( self.default_config, scorer=self.metric, data_node=self.original_data, name='hpo', resampling_strategy=self.evaluation_type, seed=self.seed) else: raise ValueError('Invalid task type!') self.fe_algo = fe_algo self.optimizer['fe'] = build_fe_optimizer(self.fe_algo, self.evaluation_type, self.task_type, self.original_data, fe_evaluator, estimator_id, per_run_time_limit, per_run_mem_limit, self.seed, shared_mode=self.share_fe, n_jobs=n_jobs) self.inc['fe'], self.local_inc[ 'fe'] = self.original_data, self.original_data # Build the HPO component. # trials_per_iter = max(len(self.optimizer['fe'].trans_types), 20) trials_per_iter = self.one_unit_of_resource * self.number_of_unit_resource self.optimizer['hpo'] = build_hpo_optimizer( self.evaluation_type, hpo_evaluator, cs, output_dir=output_dir, per_run_time_limit=per_run_time_limit, trials_per_iter=trials_per_iter, seed=self.seed, n_jobs=n_jobs) self.inc['hpo'], self.local_inc[ 'hpo'] = self.default_config, self.default_config self.init_config = cs.get_default_configuration() self.local_hist['fe'].append(self.original_data) self.local_hist['hpo'].append(self.default_config)
def __init__(self, node_list, node_index, task_type, timestamp, fe_config_space: ConfigurationSpace, cash_config_space: ConfigurationSpace, data: DataNode, fixed_config=None, time_limit=None, trial_num=0, metric='acc', ensemble_method='ensemble_selection', ensemble_size=50, per_run_time_limit=300, output_dir="logs", dataset_name='default_dataset', eval_type='holdout', resampling_params=None, n_jobs=1, seed=1): super(AlternatingBlock, self).__init__(node_list, node_index, task_type, timestamp, fe_config_space, cash_config_space, data, fixed_config=fixed_config, time_limit=time_limit, trial_num=trial_num, metric=metric, ensemble_method=ensemble_method, ensemble_size=ensemble_size, per_run_time_limit=per_run_time_limit, output_dir=output_dir, dataset_name=dataset_name, eval_type=eval_type, resampling_params=resampling_params, n_jobs=n_jobs, seed=seed) self.arms = ['hpo', 'fe'] self.optimal_algo_id = None self.first_start = True self.sub_bandits = dict() self.rewards = dict() self.evaluation_cost = dict() self.update_flag = dict() # Global incumbent. self.init_config = { 'fe': fe_config_space.get_default_configuration().get_dictionary().copy( ), 'hpo': cash_config_space.get_default_configuration().get_dictionary(). copy() } self.inc = { 'fe': fe_config_space.get_default_configuration().get_dictionary().copy( ), 'hpo': cash_config_space.get_default_configuration().get_dictionary(). copy() } self.local_inc = { 'fe': fe_config_space.get_default_configuration().get_dictionary().copy( ), 'hpo': cash_config_space.get_default_configuration().get_dictionary(). copy() } self.local_hist = {'fe': [], 'hpo': []} self.inc_record = {'fe': list(), 'hpo': list()} self.exp_output = dict() self.eval_dict = dict() self.arm_eval_dict = {'fe': dict(), 'hpo': dict()} for arm in self.arms: self.rewards[arm] = list() self.update_flag[arm] = False self.evaluation_cost[arm] = list() self.exp_output[arm] = dict() self.pull_cnt = 0 self.action_sequence = list() self.final_rewards = list() for arm in self.arms: if arm == 'fe': from solnml.blocks.block_utils import get_node_type child_type = get_node_type(node_list, node_index + 1) self.sub_bandits[arm] = child_type( node_list, node_index + 1, task_type, timestamp, fe_config_space, None, data.copy_(), fixed_config=self.init_config['hpo'], time_limit=time_limit, metric=metric, ensemble_method=ensemble_method, ensemble_size=ensemble_size, per_run_time_limit=per_run_time_limit, output_dir=output_dir, dataset_name=dataset_name, eval_type=eval_type, resampling_params=resampling_params, n_jobs=n_jobs, seed=seed) else: from solnml.blocks.block_utils import get_node_type child_type = get_node_type(node_list, node_index + 2) self.sub_bandits[arm] = child_type( node_list, node_index + 2, task_type, timestamp, None, cash_config_space, data.copy_(), fixed_config=self.init_config['fe'], time_limit=time_limit, metric=metric, ensemble_method=ensemble_method, ensemble_size=ensemble_size, per_run_time_limit=per_run_time_limit, output_dir=output_dir, dataset_name=dataset_name, eval_type=eval_type, resampling_params=resampling_params, n_jobs=n_jobs, seed=seed) self.topk_saver = CombinedTopKModelSaver(k=50, model_dir=self.output_dir, identifier=self.timestamp)