def _predict(self, test_data: DataNode):
if self.ensemble_method is not None:
if self.es is None and self.eval_type == 'cv':
raise AttributeError(
"Please call refit() for cross-validation!")
elif self.es is None:
raise AttributeError("AutoML is not fitted!")
return self.es.predict(test_data)
else:
try:
best_op_list, estimator = load_combined_transformer_estimator(
self.output_dir, self.incumbent, self.timestamp)
except Exception as e:
if self.eval_type == 'cv':
raise AttributeError(
"Please call refit() for cross-validation!")
else:
raise e
test_data_node = test_data.copy_()
test_data_node = construct_node(test_data_node, best_op_list)
if self.task_type in CLS_TASKS:
return estimator.predict_proba(test_data_node.data[0])
else:
return estimator.predict(test_data_node.data[0])
def train_valid_split(node: DataNode):
X, y = node.copy_().data
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=1)
for train_index, test_index in sss.split(X, y):
X_train, X_val = X[train_index], X[test_index]
y_train, y_val = y[train_index], y[test_index]
train_data = DataNode(data=[X_train, y_train],
feature_type=node.feature_types.copy())
valid_data = DataNode(data=[X_val, y_val],
feature_type=node.feature_types.copy())
return train_data, valid_data
def apply(self, data_node: DataNode, ref_node: DataNode, phase='test'):
input_node = data_node.copy_()
fe_config = ref_node.config
if ref_node is None:
return data_node
elif fe_config in self.node_dict:
fe_trans_list = self.node_dict[fe_config][1]
for i, tran in enumerate(fe_trans_list):
if phase == 'test' and i == 2: # Disable balancer
continue
if tran is not None:
input_node = tran.operate(input_node)
else:
self.logger.info("Ref node config:")
self.logger.info(str(fe_config))
self.logger.info("Node history in optimizer:")
self.logger.info(str(self.node_dict))
raise ValueError("Ref node not in history!")
return input_node
def _predict(self, test_data: DataNode):
if self.ensemble_method is not None:
if self.es is None:
raise AttributeError("AutoML is not fitted!")
return self.es.predict(test_data)
else:
if self.inner_opt_algorithm == 'combined':
best_op_list, estimator = load_combined_transformer_estimator(
self.output_dir, self.best_config, self.timestamp)
else:
best_op_list, estimator = load_transformer_estimator(
self.output_dir, self.optimal_algo_id,
self.best_hpo_config, self.best_fe_config, self.timestamp)
test_data_node = test_data.copy_()
test_data_node = construct_node(test_data_node, best_op_list)
if self.task_type in CLS_TASKS:
return estimator.predict_proba(test_data_node.data[0])
else:
return estimator.predict(test_data_node.data[0])
def parse_config(data_node: DataNode, config: dict, record=False, skip_balance=False, if_imbal=False):
"""
Transform the data node based on the pipeline specified by configuration.
:param data_node:
:param config:
:param record:
:return: the resulting data node.
"""
_preprocessor_candidates = get_combined_fe_candidtates(_preprocessor, _gen_addons)
_preprocessor_candidates = get_combined_fe_candidtates(_preprocessor_candidates, _sel_addons)
_rescaler_candidates = get_combined_fe_candidtates(_rescaler, _res_addons)
if not if_imbal:
_balancer_candidates = get_combined_fe_candidtates(_bal_balancer, _bal_addons)
else:
_balancer_candidates = get_combined_fe_candidtates(_imb_balancer, _bal_addons)
# Remove the indicator in config_dict.
config_dict = config.copy()
image_pre_id = config_dict.get('image_preprocessor', None)
if image_pre_id:
config_dict.pop('image_preprocessor')
text_pre_id = config_dict.get('text_preprocessor', None)
if text_pre_id:
config_dict.pop('text_preprocessor')
def tran_operate(id, tran_set, config, node):
_config = {}
for key in config:
if id in key:
config_name = key.split(':')[1]
_config[config_name] = config[key]
tran = tran_set[id](**_config)
output_node = tran.operate(node)
return output_node, tran
_node = data_node.copy_()
tran_dict = dict()
# Image preprocessor
if image_pre_id:
_node, image_tran = tran_operate(image_pre_id, _image_preprocessor, config_dict, _node)
tran_dict['image_preprocessor'] = image_tran
# Text preprocessor
if text_pre_id:
_node, text_tran = tran_operate(text_pre_id, _text_preprocessor, config_dict, _node)
tran_dict['text_preprocessor'] = text_tran
for stage in stage_list:
if stage == 'balancer':
if skip_balance:
op_id = 'empty'
else:
if stage in config_dict:
op_id = config_dict[stage]
config_dict.pop(stage)
else:
op_id = 'empty'
else:
op_id = config_dict[stage]
config_dict.pop(stage)
if stage == 'preprocessor':
_node, tran = tran_operate(op_id, _preprocessor_candidates, config_dict, _node)
elif stage == 'rescaler':
_node, tran = tran_operate(op_id, _rescaler_candidates, config_dict, _node)
elif stage == 'balancer':
_node, tran = tran_operate(op_id, _balancer_candidates, config_dict, _node)
else:
# Third party stage
_node, tran = tran_operate(op_id, thirdparty_candidates_dict[stage], config_dict, _node)
tran_dict[stage] = tran
_node.config = config
if record:
return _node, tran_dict
return _node
def __init__(self,
task_type,
trial_num,
classifier_ids: List[str],
data: DataNode,
metric='acc',
ensemble_method='ensemble_selection',
ensemble_size=10,
per_run_time_limit=300,
output_dir=None,
dataset_name='default_dataset',
eval_type='holdout',
share_feature=False,
inner_opt_algorithm='rb',
fe_algo='bo',
time_limit=None,
n_jobs=1,
seed=1):
"""
:param classifier_ids: subset of {'adaboost','bernoulli_nb','decision_tree','extra_trees','gaussian_nb','gradient_boosting',
'gradient_boosting','k_nearest_neighbors','lda','liblinear_svc','libsvm_svc','multinomial_nb','passive_aggressive','qda',
'random_forest','sgd'}
"""
self.timestamp = time.time()
self.task_type = task_type
self.metric = get_metric(metric)
self.original_data = data.copy_()
self.ensemble_method = ensemble_method
self.ensemble_size = ensemble_size
self.trial_num = trial_num
self.n_jobs = n_jobs
self.alpha = 4
self.B = 0.01
self.seed = seed
self.shared_mode = share_feature
self.output_dir = output_dir
np.random.seed(self.seed)
# Best configuration.
self.optimal_algo_id = None
self.nbest_algo_ids = None
self.best_lower_bounds = None
self.es = None
# Set up backend.
self.dataset_name = dataset_name
self.time_limit = time_limit
self.start_time = time.time()
self.logger = get_logger('Soln-ml: %s' % dataset_name)
# Bandit settings.
self.incumbent_perf = -1.
self.arms = classifier_ids
self.include_algorithms = classifier_ids
self.rewards = dict()
self.sub_bandits = dict()
self.evaluation_cost = dict()
self.fe_datanodes = dict()
self.eval_type = eval_type
self.fe_algo = fe_algo
self.inner_opt_algorithm = inner_opt_algorithm
for arm in self.arms:
self.rewards[arm] = list()
self.evaluation_cost[arm] = list()
self.fe_datanodes[arm] = list()
self.sub_bandits[arm] = SecondLayerBandit(
self.task_type,
arm,
self.original_data,
metric=self.metric,
output_dir=output_dir,
per_run_time_limit=per_run_time_limit,
share_fe=self.shared_mode,
seed=self.seed,
eval_type=eval_type,
dataset_id=dataset_name,
n_jobs=self.n_jobs,
fe_algo=fe_algo,
mth=inner_opt_algorithm,
)
self.action_sequence = list()
self.final_rewards = list()
self.start_time = time.time()
self.time_records = list()
def __init__(self,
task_type,
trial_num,
classifier_ids: List[str],
data: DataNode,
include_preprocessors=None,
time_limit=None,
metric='acc',
ensemble_method='ensemble_selection',
ensemble_size=50,
per_run_time_limit=300,
output_dir="logs",
dataset_name='default_dataset',
eval_type='holdout',
inner_opt_algorithm='fixed',
enable_fe=True,
fe_algo='bo',
n_jobs=1,
seed=1):
"""
:param classifier_ids: subset of {'adaboost','bernoulli_nb','decision_tree','extra_trees','gaussian_nb','gradient_boosting',
'gradient_boosting','k_nearest_neighbors','lda','liblinear_svc','libsvm_svc','multinomial_nb','passive_aggressive','qda',
'random_forest','sgd'}
"""
self.timestamp = time.time()
self.task_type = task_type
self.include_preprocessors = include_preprocessors
self.metric = get_metric(metric)
self.original_data = data.copy_()
self.ensemble_method = ensemble_method
self.ensemble_size = ensemble_size
self.trial_num = trial_num
self.n_jobs = n_jobs
self.alpha = 4
self.seed = seed
self.output_dir = output_dir
self.early_stop_flag = False
# np.random.seed(self.seed)
# Best configuration.
self.optimal_algo_id = None
self.nbest_algo_ids = None
self.best_lower_bounds = None
self.es = None
# Set up backend.
self.dataset_name = dataset_name
self.time_limit = time_limit
self.start_time = time.time()
self.logger = get_logger('Soln-ml: %s' % dataset_name)
# Bandit settings.
self.incumbent_perf = -float("INF")
self.arms = classifier_ids
self.include_algorithms = classifier_ids
self.rewards = dict()
self.sub_bandits = dict()
self.evaluation_cost = dict()
self.eval_type = eval_type
self.enable_fe = enable_fe
self.fe_algo = fe_algo
self.inner_opt_algorithm = inner_opt_algorithm
# Record the execution cost for each arm.
if not (self.time_limit is None) ^ (self.trial_num is None):
raise ValueError('Please set one of time_limit or trial_num.')
self.arm_cost_stats = dict()
for _arm in self.arms:
self.arm_cost_stats[_arm] = list()
for arm in self.arms:
self.rewards[arm] = list()
self.evaluation_cost[arm] = list()
self.sub_bandits[arm] = SecondLayerBandit(
self.task_type,
arm,
self.original_data,
include_preprocessors=self.include_preprocessors,
metric=self.metric,
output_dir=output_dir,
per_run_time_limit=per_run_time_limit,
seed=self.seed,
eval_type=eval_type,
dataset_id=dataset_name,
n_jobs=self.n_jobs,
fe_algo=fe_algo,
mth=self.inner_opt_algorithm,
timestamp=self.timestamp)
self.action_sequence = list()
self.final_rewards = list()
self.start_time = time.time()
self.time_records = list()
def __init__(self,
node_list,
node_index,
task_type,
timestamp,
fe_config_space: ConfigurationSpace,
cash_config_space: ConfigurationSpace,
data: DataNode,
fixed_config=None,
trial_num=0,
time_limit=None,
metric='acc',
ensemble_method='ensemble_selection',
ensemble_size=50,
per_run_time_limit=300,
output_dir="logs",
dataset_name='default_dataset',
eval_type='holdout',
resampling_params=None,
n_jobs=1,
seed=1):
# Tree setting
self.node_list = node_list
self.node_index = node_index
# Set up backend.
self.dataset_name = dataset_name
self.trial_num = trial_num
self.time_limit = time_limit
self.per_run_time_limit = per_run_time_limit
self.start_time = time.time()
self.logger = get_logger('Soln-ml: %s' % dataset_name)
# Basic settings.
self.eval_type = eval_type
self.resampling_params = resampling_params
self.task_type = task_type
self.timestamp = timestamp
self.fe_config_space = fe_config_space
self.cash_config_space = cash_config_space
self.fixed_config = fixed_config
self.original_data = data.copy_()
self.metric = get_metric(metric)
self.ensemble_method = ensemble_method
self.ensemble_size = ensemble_size
self.n_jobs = n_jobs
self.seed = seed
self.output_dir = output_dir
self.early_stop_flag = False
self.timeout_flag = False
self.incumbent_perf = -float("INF")
self.incumbent = None
self.eval_dict = dict()
if self.task_type in CLS_TASKS:
self.if_imbal = is_imbalanced_dataset(self.original_data)
else:
self.if_imbal = False
self.es = None
def _parse(self, data_node: DataNode, config, record=False, skip_balance=False):
"""
Transform the data node based on the pipeline specified by configuration.
:param data_node:
:param config:
:param record:
:return: the resulting data node.
"""
# Remove the indicator in config_dict.
config_dict = config.get_dictionary().copy()
pre1_id = config_dict['preprocessor1']
config_dict.pop('preprocessor1')
pre2_id = config_dict['preprocessor2']
config_dict.pop('preprocessor2')
if skip_balance:
bal_id = 'empty'
else:
if 'balancer' in config_dict:
bal_id = config_dict['balancer']
config_dict.pop('balancer')
else:
bal_id = 'empty'
gen_id = config_dict['generator']
config_dict.pop('generator')
res_id = config_dict['rescaler']
config_dict.pop('rescaler')
sel_id = config_dict['selector']
config_dict.pop('selector')
def tran_operate(id, tran_set, config, node):
if id != "empty":
_config = {}
for key in config:
if id in key:
config_name = key.split(':')[1]
_config[config_name] = config[key]
tran = tran_set[id](**_config)
output_node = tran.operate(node)
return output_node, tran
return node, None
_node = data_node.copy_()
# Preprocessor1
_node, pre1_tran = tran_operate(pre1_id, _preprocessor1, config_dict, _node)
# Preprocessor2
_node, pre2_tran = tran_operate(pre2_id, _preprocessor2, config_dict, _node)
# Balancer
_balancer = _bal_balancer if self.if_bal else _imb_balancer
_node, bal_tran = tran_operate(bal_id, _balancer, config_dict, _node)
# Rescaler
_node, res_tran = tran_operate(res_id, _rescaler, config_dict, _node)
# Generator
_node, gen_tran = tran_operate(gen_id, _generator, config_dict, _node)
# Selector
_node, sel_tran = tran_operate(sel_id, _selector, config_dict, _node)
_node.config = config
if record:
return _node, [pre1_tran, pre2_tran, bal_tran, res_tran, gen_tran, sel_tran]
return _node
def __init__(self,
task_type,
estimator_id: str,
data: DataNode,
metric,
include_preprocessors=None,
share_fe=False,
output_dir='logs',
per_run_time_limit=120,
per_run_mem_limit=5120,
dataset_id='default',
eval_type='holdout',
mth='rb',
sw_size=3,
n_jobs=1,
seed=1,
enable_fe=True,
fe_algo='bo',
number_of_unit_resource=2,
total_resource=30,
timestamp=None):
self.task_type = task_type
self.metric = metric
self.number_of_unit_resource = number_of_unit_resource
# One unit of resource, that's, the number of trials per iteration.
self.one_unit_of_resource = 5
self.total_resource = total_resource
self.per_run_time_limit = per_run_time_limit
self.per_run_mem_limit = per_run_mem_limit
self.estimator_id = estimator_id
self.include_preprocessors = include_preprocessors
self.evaluation_type = eval_type
self.original_data = data.copy_()
self.share_fe = share_fe
self.output_dir = output_dir
self.n_jobs = n_jobs
self.mth = mth
self.seed = seed
self.sliding_window_size = sw_size
task_id = '%s-%d-%s' % (dataset_id, seed, estimator_id)
self.logger = get_logger(self.__class__.__name__ + '-' + task_id)
# Bandit settings.
# self.arms = ['fe', 'hpo']
self.arms = ['hpo', 'fe']
self.rewards = dict()
self.optimizer = dict()
self.evaluation_cost = dict()
self.update_flag = dict()
# Global incumbent.
self.inc = dict()
self.local_inc = dict()
self.local_hist = {'fe': [], 'hpo': []}
self.inc_record = {'fe': list(), 'hpo': list()}
self.exp_output = dict()
self.eval_dict = {'fe': dict(), 'hpo': dict()}
for arm in self.arms:
self.rewards[arm] = list()
self.update_flag[arm] = False
self.evaluation_cost[arm] = list()
self.exp_output[arm] = dict()
self.pull_cnt = 0
self.action_sequence = list()
self.final_rewards = list()
self.incumbent_config = None
self.incumbent_perf = float("-INF")
self.early_stopped_flag = False
self.first_start = True
self.include_text = True if TEXT in self.original_data.feature_types else False
self.include_image = True if IMAGE in self.original_data.feature_types else False
# Fetch hyperparameter space.
if self.task_type in CLS_TASKS:
from solnml.components.models.classification import _classifiers, _addons
_candidates = get_combined_candidtates(_classifiers, _addons)
if estimator_id in _candidates:
clf_class = _candidates[estimator_id]
else:
raise ValueError("Algorithm %s not supported!" % estimator_id)
cs = clf_class.get_hyperparameter_search_space()
elif self.task_type in RGS_TASKS:
from solnml.components.models.regression import _regressors, _addons
_candidates = get_combined_candidtates(_regressors, _addons)
if estimator_id in _candidates:
reg_class = _candidates[estimator_id]
else:
raise ValueError("Algorithm %s not supported!" % estimator_id)
cs = reg_class.get_hyperparameter_search_space()
else:
raise ValueError("Unknown task type %s!" % self.task_type)
self.config_space = cs
self.default_config = cs.get_default_configuration()
self.config_space.seed(self.seed)
self.if_imbal = is_imbalanced_dataset(self.original_data)
self.fe_config_space = get_task_hyperparameter_space(
self.task_type,
self.estimator_id,
include_preprocessors=self.include_preprocessors,
include_text=self.include_text,
include_image=self.include_image,
if_imbal=self.if_imbal)
self.fe_default_config = self.fe_config_space.get_default_configuration(
)
self.timestamp = timestamp
# Build the Feature Engineering component.
if self.task_type in CLS_TASKS:
fe_evaluator = ClassificationEvaluator(
self.default_config,
self.fe_default_config,
estimator_id,
scorer=self.metric,
data_node=self.original_data,
name='fe',
resampling_strategy=self.evaluation_type,
if_imbal=self.if_imbal,
seed=self.seed,
output_dir=self.output_dir,
timestamp=self.timestamp)
hpo_evaluator = ClassificationEvaluator(
self.default_config,
self.fe_default_config,
estimator_id,
scorer=self.metric,
data_node=self.original_data,
name='hpo',
resampling_strategy=self.evaluation_type,
if_imbal=self.if_imbal,
seed=self.seed,
output_dir=self.output_dir,
timestamp=self.timestamp)
elif self.task_type in RGS_TASKS:
fe_evaluator = RegressionEvaluator(
self.default_config,
self.fe_default_config,
estimator_id,
scorer=self.metric,
data_node=self.original_data,
name='fe',
resampling_strategy=self.evaluation_type,
seed=self.seed,
output_dir=self.output_dir,
timestamp=self.timestamp)
hpo_evaluator = RegressionEvaluator(
self.default_config,
self.fe_default_config,
estimator_id,
scorer=self.metric,
data_node=self.original_data,
name='hpo',
resampling_strategy=self.evaluation_type,
seed=self.seed,
output_dir=self.output_dir,
timestamp=self.timestamp)
else:
raise ValueError('Invalid task type!')
if self.mth != 'combined':
self.enable_fe = enable_fe
trials_per_iter = self.one_unit_of_resource * self.number_of_unit_resource
self.optimizer['fe'] = build_fe_optimizer(
self.evaluation_type,
fe_evaluator,
self.fe_config_space,
per_run_time_limit=per_run_time_limit,
per_run_mem_limit=per_run_mem_limit,
inner_iter_num_per_iter=trials_per_iter,
output_dir=output_dir,
seed=self.seed,
n_jobs=n_jobs)
self.inc['fe'], self.local_inc[
'fe'] = self.fe_default_config, self.fe_default_config
# Build the HPO component.
# trials_per_iter = max(len(self.optimizer['fe'].trans_types), 20)
trials_per_iter = self.one_unit_of_resource * self.number_of_unit_resource
self.optimizer['hpo'] = build_hpo_optimizer(
self.evaluation_type,
hpo_evaluator,
cs,
output_dir=output_dir,
per_run_time_limit=per_run_time_limit,
inner_iter_num_per_iter=trials_per_iter,
seed=self.seed,
n_jobs=n_jobs)
self.inc['hpo'], self.local_inc[
'hpo'] = self.default_config, self.default_config
self.init_config = cs.get_default_configuration()
self.local_hist['fe'].append(self.fe_default_config)
self.local_hist['hpo'].append(self.default_config)
else:
self.rewards = list()
self.evaluation_cost = list()
self.eval_dict = {}
trials_per_iter = self.one_unit_of_resource * self.number_of_unit_resource
if self.task_type in CLS_TASKS:
from solnml.utils.combined_cls_evaluator import get_combined_cs
from solnml.utils.combined_cls_evaluator import CombinedClassificationEvaluator as CombinedEvaluator
else:
from solnml.utils.combined_rgs_evaluator import get_combined_cs
from solnml.utils.combined_rgs_evaluator import CombinedRegressionEvaluator as CombinedEvaluator
self.evaluator = CombinedEvaluator(
estimator_id,
scorer=self.metric,
data_node=self.original_data,
if_imbal=self.if_imbal,
timestamp=self.timestamp,
output_dir=self.output_dir,
resampling_strategy=self.evaluation_type)
cs = get_combined_cs(
self.estimator_id,
self.task_type,
include_image=self.include_image,
include_text=self.include_text,
include_preprocessors=self.include_preprocessors,
if_imbal=self.if_imbal)
self.optimizer = build_hpo_optimizer(
self.evaluation_type,
self.evaluator,
cs,
output_dir=self.output_dir,
per_run_time_limit=self.per_run_time_limit,
inner_iter_num_per_iter=trials_per_iter,
seed=self.seed,
n_jobs=self.n_jobs)
def __init__(self,
node_list,
node_index,
task_type,
timestamp,
fe_config_space: ConfigurationSpace,
cash_config_space: ConfigurationSpace,
data: DataNode,
fixed_config=None,
time_limit=None,
trial_num=0,
metric='acc',
ensemble_method='ensemble_selection',
ensemble_size=50,
per_run_time_limit=300,
output_dir="logs",
dataset_name='default_dataset',
eval_type='holdout',
resampling_params=None,
n_jobs=1,
seed=1):
"""
:param classifier_ids: subset of {'adaboost','bernoulli_nb','decision_tree','extra_trees','gaussian_nb','gradient_boosting',
'gradient_boosting','k_nearest_neighbors','lda','liblinear_svc','libsvm_svc','multinomial_nb','passive_aggressive','qda',
'random_forest','sgd'}
"""
super(ConditioningBlock,
self).__init__(node_list,
node_index,
task_type,
timestamp,
fe_config_space,
cash_config_space,
data,
fixed_config=fixed_config,
time_limit=time_limit,
trial_num=trial_num,
metric=metric,
ensemble_method=ensemble_method,
ensemble_size=ensemble_size,
per_run_time_limit=per_run_time_limit,
output_dir=output_dir,
dataset_name=dataset_name,
eval_type=eval_type,
resampling_params=resampling_params,
n_jobs=n_jobs,
seed=seed)
# Best configuration.
self.optimal_arm = None
self.best_lower_bounds = None
# Bandit settings.
self.alpha = 4
self.arms = list(
cash_config_space.get_hyperparameter('algorithm').choices)
self.rewards = dict()
self.sub_bandits = dict()
self.evaluation_cost = dict()
self.arm_cost_stats = dict()
for _arm in self.arms:
self.arm_cost_stats[_arm] = list()
for arm in self.arms:
self.rewards[arm] = list()
self.evaluation_cost[arm] = list()
hps = cash_config_space.get_hyperparameters()
cs = ConfigurationSpace()
cs.add_hyperparameter(Constant('algorithm', arm))
for hp in hps:
if hp.name.split(':')[0] == arm:
cs.add_hyperparameter(hp)
# Add active conditions
conds = cash_config_space.get_conditions()
for cond in conds:
try:
cs.add_condition(cond)
except:
pass
# Add active forbidden clauses
forbids = cash_config_space.get_forbiddens()
for forbid in forbids:
try:
cs.add_forbidden_clause(forbid)
except:
pass
from solnml.blocks.block_utils import get_node_type
child_type = get_node_type(node_list, node_index + 1)
self.sub_bandits[arm] = child_type(
node_list,
node_index + 1,
task_type,
timestamp,
deepcopy(fe_config_space),
deepcopy(cs),
data.copy_(),
fixed_config=fixed_config,
time_limit=time_limit,
metric=metric,
ensemble_method=ensemble_method,
ensemble_size=ensemble_size,
per_run_time_limit=per_run_time_limit,
output_dir=output_dir,
dataset_name=dataset_name,
eval_type=eval_type,
resampling_params=resampling_params,
n_jobs=n_jobs,
seed=seed)
self.action_sequence = list()
self.final_rewards = list()
self.start_time = time.time()
self.time_records = list()
# Initialize the parameters.
self.pull_cnt = 0
self.pick_id = 0
self.update_cnt = 0
arm_num = len(self.arms)
self.optimal_algo_id = None
self.arm_candidate = self.arms.copy()
self.best_lower_bounds = np.zeros(arm_num)
_iter_id = 0
if self.time_limit is None:
if arm_num * self.alpha > self.trial_num:
raise ValueError('Trial number should be larger than %d.' %
(arm_num * self.alpha))
else:
self.trial_num = MAX_INT
def __init__(self,
task_type,
estimator_id: str,
data: DataNode,
metric,
share_fe=False,
output_dir='logs',
per_run_time_limit=120,
per_run_mem_limit=5120,
dataset_id='default',
eval_type='holdout',
mth='rb',
sw_size=3,
n_jobs=1,
seed=1,
fe_algo='tree_based',
enable_intersection=True,
number_of_unit_resource=2,
total_resource=30):
self.task_type = task_type
self.metric = metric
self.number_of_unit_resource = number_of_unit_resource
# One unit of resource, that's, the number of trials per iteration.
self.one_unit_of_resource = 5
self.total_resource = total_resource
self.per_run_time_limit = per_run_time_limit
self.per_run_mem_limit = per_run_mem_limit
self.estimator_id = estimator_id
self.evaluation_type = eval_type
self.original_data = data.copy_()
self.share_fe = share_fe
self.output_dir = output_dir
self.n_jobs = n_jobs
self.mth = mth
self.seed = seed
self.sliding_window_size = sw_size
task_id = '%s-%d-%s' % (dataset_id, seed, estimator_id)
self.logger = get_logger(self.__class__.__name__ + '-' + task_id)
np.random.seed(self.seed)
# Bandit settings.
# self.arms = ['fe', 'hpo']
self.arms = ['hpo', 'fe']
self.rewards = dict()
self.optimizer = dict()
self.evaluation_cost = dict()
self.update_flag = dict()
# Global incumbent.
self.inc = dict()
self.local_inc = dict()
self.local_hist = {'fe': [], 'hpo': []}
for arm in self.arms:
self.rewards[arm] = list()
self.update_flag[arm] = False
self.evaluation_cost[arm] = list()
self.pull_cnt = 0
self.action_sequence = list()
self.final_rewards = list()
self.incumbent_perf = float("-INF")
self.early_stopped_flag = False
self.enable_intersection = enable_intersection
# Fetch hyperparameter space.
if self.task_type in CLS_TASKS:
from solnml.components.models.classification import _classifiers, _addons
if estimator_id in _classifiers:
clf_class = _classifiers[estimator_id]
elif estimator_id in _addons.components:
clf_class = _addons.components[estimator_id]
else:
raise ValueError("Algorithm %s not supported!" % estimator_id)
cs = clf_class.get_hyperparameter_search_space()
model = UnParametrizedHyperparameter("estimator", estimator_id)
cs.add_hyperparameter(model)
elif self.task_type in REG_TASKS:
from solnml.components.models.regression import _regressors, _addons
if estimator_id in _regressors:
reg_class = _regressors[estimator_id]
elif estimator_id in _addons.components:
reg_class = _addons.components[estimator_id]
else:
raise ValueError("Algorithm %s not supported!" % estimator_id)
cs = reg_class.get_hyperparameter_search_space()
model = UnParametrizedHyperparameter("estimator", estimator_id)
cs.add_hyperparameter(model)
else:
raise ValueError("Unknown task type %s!" % self.task_type)
self.config_space = cs
self.default_config = cs.get_default_configuration()
self.config_space.seed(self.seed)
# Build the Feature Engineering component.
if self.task_type in CLS_TASKS:
fe_evaluator = ClassificationEvaluator(
self.default_config,
scorer=self.metric,
name='fe',
resampling_strategy=self.evaluation_type,
seed=self.seed)
hpo_evaluator = ClassificationEvaluator(
self.default_config,
scorer=self.metric,
data_node=self.original_data,
name='hpo',
resampling_strategy=self.evaluation_type,
seed=self.seed)
elif self.task_type in REG_TASKS:
fe_evaluator = RegressionEvaluator(
self.default_config,
scorer=self.metric,
name='fe',
resampling_strategy=self.evaluation_type,
seed=self.seed)
hpo_evaluator = RegressionEvaluator(
self.default_config,
scorer=self.metric,
data_node=self.original_data,
name='hpo',
resampling_strategy=self.evaluation_type,
seed=self.seed)
else:
raise ValueError('Invalid task type!')
self.fe_algo = fe_algo
self.optimizer['fe'] = build_fe_optimizer(self.fe_algo,
self.evaluation_type,
self.task_type,
self.original_data,
fe_evaluator,
estimator_id,
per_run_time_limit,
per_run_mem_limit,
self.seed,
shared_mode=self.share_fe,
n_jobs=n_jobs)
self.inc['fe'], self.local_inc[
'fe'] = self.original_data, self.original_data
# Build the HPO component.
# trials_per_iter = max(len(self.optimizer['fe'].trans_types), 20)
trials_per_iter = self.one_unit_of_resource * self.number_of_unit_resource
self.optimizer['hpo'] = build_hpo_optimizer(
self.evaluation_type,
hpo_evaluator,
cs,
output_dir=output_dir,
per_run_time_limit=per_run_time_limit,
trials_per_iter=trials_per_iter,
seed=self.seed,
n_jobs=n_jobs)
self.inc['hpo'], self.local_inc[
'hpo'] = self.default_config, self.default_config
self.init_config = cs.get_default_configuration()
self.local_hist['fe'].append(self.original_data)
self.local_hist['hpo'].append(self.default_config)
def __init__(self,
node_list,
node_index,
task_type,
timestamp,
fe_config_space: ConfigurationSpace,
cash_config_space: ConfigurationSpace,
data: DataNode,
fixed_config=None,
time_limit=None,
trial_num=0,
metric='acc',
ensemble_method='ensemble_selection',
ensemble_size=50,
per_run_time_limit=300,
output_dir="logs",
dataset_name='default_dataset',
eval_type='holdout',
resampling_params=None,
n_jobs=1,
seed=1):
super(AlternatingBlock,
self).__init__(node_list,
node_index,
task_type,
timestamp,
fe_config_space,
cash_config_space,
data,
fixed_config=fixed_config,
time_limit=time_limit,
trial_num=trial_num,
metric=metric,
ensemble_method=ensemble_method,
ensemble_size=ensemble_size,
per_run_time_limit=per_run_time_limit,
output_dir=output_dir,
dataset_name=dataset_name,
eval_type=eval_type,
resampling_params=resampling_params,
n_jobs=n_jobs,
seed=seed)
self.arms = ['hpo', 'fe']
self.optimal_algo_id = None
self.first_start = True
self.sub_bandits = dict()
self.rewards = dict()
self.evaluation_cost = dict()
self.update_flag = dict()
# Global incumbent.
self.init_config = {
'fe':
fe_config_space.get_default_configuration().get_dictionary().copy(
),
'hpo':
cash_config_space.get_default_configuration().get_dictionary().
copy()
}
self.inc = {
'fe':
fe_config_space.get_default_configuration().get_dictionary().copy(
),
'hpo':
cash_config_space.get_default_configuration().get_dictionary().
copy()
}
self.local_inc = {
'fe':
fe_config_space.get_default_configuration().get_dictionary().copy(
),
'hpo':
cash_config_space.get_default_configuration().get_dictionary().
copy()
}
self.local_hist = {'fe': [], 'hpo': []}
self.inc_record = {'fe': list(), 'hpo': list()}
self.exp_output = dict()
self.eval_dict = dict()
self.arm_eval_dict = {'fe': dict(), 'hpo': dict()}
for arm in self.arms:
self.rewards[arm] = list()
self.update_flag[arm] = False
self.evaluation_cost[arm] = list()
self.exp_output[arm] = dict()
self.pull_cnt = 0
self.action_sequence = list()
self.final_rewards = list()
for arm in self.arms:
if arm == 'fe':
from solnml.blocks.block_utils import get_node_type
child_type = get_node_type(node_list, node_index + 1)
self.sub_bandits[arm] = child_type(
node_list,
node_index + 1,
task_type,
timestamp,
fe_config_space,
None,
data.copy_(),
fixed_config=self.init_config['hpo'],
time_limit=time_limit,
metric=metric,
ensemble_method=ensemble_method,
ensemble_size=ensemble_size,
per_run_time_limit=per_run_time_limit,
output_dir=output_dir,
dataset_name=dataset_name,
eval_type=eval_type,
resampling_params=resampling_params,
n_jobs=n_jobs,
seed=seed)
else:
from solnml.blocks.block_utils import get_node_type
child_type = get_node_type(node_list, node_index + 2)
self.sub_bandits[arm] = child_type(
node_list,
node_index + 2,
task_type,
timestamp,
None,
cash_config_space,
data.copy_(),
fixed_config=self.init_config['fe'],
time_limit=time_limit,
metric=metric,
ensemble_method=ensemble_method,
ensemble_size=ensemble_size,
per_run_time_limit=per_run_time_limit,
output_dir=output_dir,
dataset_name=dataset_name,
eval_type=eval_type,
resampling_params=resampling_params,
n_jobs=n_jobs,
seed=seed)
self.topk_saver = CombinedTopKModelSaver(k=50,
model_dir=self.output_dir,
identifier=self.timestamp)