def test_choose_next(self):
configspace = ConfigurationSpace()
configspace.add_hyperparameter(UniformFloatHyperparameter('a', 0, 1))
configspace.add_hyperparameter(UniformFloatHyperparameter('b', 0, 1))
dataset_name = 'foo'
func_eval_time_limit = 15
total_walltime_limit = 15
memory_limit = 3000
auto = AutoMLSMBO(None, dataset_name, None, func_eval_time_limit,
total_walltime_limit, memory_limit, None)
auto.config_space = configspace
scenario = Scenario({'cs': configspace,
'cutoff-time': func_eval_time_limit,
'wallclock-limit': total_walltime_limit,
'memory-limit': memory_limit,
'run-obj': 'quality'})
smac = SMAC(scenario)
self.assertRaisesRegex(ValueError, 'Cannot use SMBO algorithm on '
'empty runhistory',
auto.choose_next, smac)
runhistory = smac.solver.runhistory
runhistory.add(config=Configuration(configspace,
values={'a': 0.1, 'b': 0.2}),
cost=0.5, time=0.5, status=StatusType.SUCCESS)
auto.choose_next(smac)
def test_smbo_metalearning_configurations(backend, context, dask_client):
# Get the inputs to the optimizer
X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
config_space = AutoML(backend=backend,
metric=autosklearn.metrics.accuracy,
time_left_for_this_task=20,
per_run_time_limit=5).fit(
X_train,
Y_train,
task=BINARY_CLASSIFICATION,
only_return_configuration_space=True)
watcher = StopWatch()
# Create an optimizer
smbo = AutoMLSMBO(
config_space=config_space,
dataset_name='iris',
backend=backend,
total_walltime_limit=10,
func_eval_time_limit=5,
memory_limit=4096,
metric=autosklearn.metrics.accuracy,
watcher=watcher,
n_jobs=1,
dask_client=dask_client,
port=logging.handlers.DEFAULT_TCP_LOGGING_PORT,
start_num_run=1,
data_memory_limit=None,
num_metalearning_cfgs=25,
pynisher_context=context,
)
assert smbo.pynisher_context == context
# Create the inputs to metalearning
datamanager = XYDataManager(
X_train,
Y_train,
X_test,
Y_test,
task=BINARY_CLASSIFICATION,
dataset_name='iris',
feat_type={i: 'numerical'
for i in range(X_train.shape[1])},
)
backend.save_datamanager(datamanager)
smbo.task = BINARY_CLASSIFICATION
smbo.reset_data_manager()
metalearning_configurations = smbo.get_metalearning_suggestions()
# We should have 25 metalearning configurations
assert len(metalearning_configurations) == 25
assert [
isinstance(config, Configuration)
for config in metalearning_configurations
]
def get_meta_learning_configs(X,
y,
task_type,
dataset_name='default',
metric='accuracy',
num_cfgs=5):
if X is None or y is None:
X, y, _ = load_data(dataset_name)
backend = create(temporary_directory=None,
output_directory=None,
delete_tmp_folder_after_terminate=False,
delete_output_folder_after_terminate=False,
shared_mode=True)
dm = XYDataManager(X, y, None, None, task_type, None, dataset_name)
configuration_space = pipeline.get_configuration_space(
dm.info,
include_estimators=None,
exclude_estimators=None,
include_preprocessors=None,
exclude_preprocessors=None)
watcher = StopWatch()
name = os.path.basename(dm.name)
watcher.start_task(name)
def reset_data_manager(max_mem=None):
pass
automlsmbo = AutoMLSMBO(
config_space=configuration_space,
dataset_name=dataset_name,
backend=backend,
total_walltime_limit=1e5,
func_eval_time_limit=1e5,
memory_limit=1e5,
metric=metric,
watcher=watcher,
metadata_directory='components/meta_learning/meta_resource',
num_metalearning_cfgs=num_cfgs)
automlsmbo.reset_data_manager = reset_data_manager
automlsmbo.task = task_type
automlsmbo.datamanager = dm
configs = automlsmbo.get_metalearning_suggestions()
return configs
def test_choose_next(self):
configspace = ConfigurationSpace()
configspace.add_hyperparameter(UniformFloatHyperparameter('a', 0, 1))
configspace.add_hyperparameter(UniformFloatHyperparameter('b', 0, 1))
dataset_name = 'foo'
func_eval_time_limit = 15
total_walltime_limit = 15
memory_limit = 3072
auto = AutoMLSMBO(config_space=None,
dataset_name=dataset_name,
backend=None,
func_eval_time_limit=func_eval_time_limit,
total_walltime_limit=total_walltime_limit,
memory_limit=memory_limit,
watcher=None,
metric=accuracy)
auto.config_space = configspace
scenario = Scenario({
'cs': configspace,
'cutoff_time': func_eval_time_limit,
'wallclock_limit': total_walltime_limit,
'memory_limit': memory_limit,
'run_obj': 'quality',
})
smac = SMAC(scenario)
self.assertRaisesRegex(
ValueError, 'Cannot use SMBO algorithm on empty runhistory',
auto.choose_next, smac)
config = Configuration(configspace, values={'a': 0.1, 'b': 0.2})
# TODO make sure the incumbent is always set?
smac.solver.incumbent = config
runhistory = smac.solver.runhistory
runhistory.add(config=config,
cost=0.5,
time=0.5,
status=StatusType.SUCCESS)
auto.choose_next(smac)
def _fit(self, datamanager, metric):
# Reset learnt stuff
self.models_ = None
self.ensemble_ = None
# Check arguments prior to doing anything!
if not isinstance(self._disable_evaluator_output, (bool, list)):
raise ValueError('disable_evaluator_output must be of type bool '
'or list.')
if isinstance(self._disable_evaluator_output, list):
allowed_elements = ['model', 'y_optimization']
for element in self._disable_evaluator_output:
if element not in allowed_elements:
raise ValueError("List member '%s' for argument "
"'disable_evaluator_output' must be one "
"of " + str(allowed_elements))
if self._resampling_strategy not in [
'holdout', 'holdout-iterative-fit', 'cv', 'partial-cv',
'partial-cv-iterative-fit'
]:
raise ValueError('Illegal resampling strategy: %s' %
self._resampling_strategy)
if self._resampling_strategy in ['partial-cv', 'partial-cv-iterative-fit'] \
and self._ensemble_size != 0:
raise ValueError("Resampling strategy %s cannot be used "
"together with ensembles." %
self._resampling_strategy)
if self._resampling_strategy in ['partial-cv', 'cv',
'partial-cv-iterative-fit'] and \
not 'folds' in self._resampling_strategy_arguments:
self._resampling_strategy_arguments['folds'] = 5
self._backend._make_internals_directory()
if self._keep_models:
try:
os.makedirs(self._backend.get_model_dir())
except (OSError, FileExistsError) as e:
if not self._shared_mode:
raise
self._metric = metric
self._task = datamanager.info['task']
self._label_num = datamanager.info['label_num']
# == Pickle the data manager to speed up loading
data_manager_path = self._backend.save_datamanager(datamanager)
time_for_load_data = self._stopwatch.wall_elapsed(self._dataset_name)
if self._debug_mode:
self._print_load_time(self._dataset_name, self._time_for_task,
time_for_load_data, self._logger)
# == Perform dummy predictions
num_run = 1
#if self._resampling_strategy in ['holdout', 'holdout-iterative-fit']:
num_run = self._do_dummy_prediction(datamanager, num_run)
# = Create a searchspace
# Do this before One Hot Encoding to make sure that it creates a
# search space for a dense classifier even if one hot encoding would
# make it sparse (tradeoff; if one hot encoding would make it sparse,
# densifier and truncatedSVD would probably lead to a MemoryError,
# like this we can't use some of the preprocessing methods in case
# the data became sparse)
self.configuration_space, configspace_path = self._create_search_space(
self._backend.temporary_directory,
self._backend,
datamanager,
include_estimators=self._include_estimators,
exclude_estimators=self._exclude_estimators,
include_preprocessors=self._include_preprocessors,
exclude_preprocessors=self._exclude_preprocessors)
# == RUN ensemble builder
# Do this before calculating the meta-features to make sure that the
# dummy predictions are actually included in the ensemble even if
# calculating the meta-features takes very long
ensemble_task_name = 'runEnsemble'
self._stopwatch.start_task(ensemble_task_name)
time_left_for_ensembles = max(0,self._time_for_task \
- self._stopwatch.wall_elapsed(self._dataset_name))
if self._logger:
self._logger.info('Start Ensemble with %5.2fsec time left' %
time_left_for_ensembles)
if time_left_for_ensembles <= 0:
self._logger.warning("Not starting ensemble builder because there "
"is no time left!")
self._proc_ensemble = None
else:
self._proc_ensemble = self._get_ensemble_process(
time_left_for_ensembles)
if self._ensemble_size > 0:
self._proc_ensemble.start()
else:
self._logger.info('Not starting ensemble builder because '
'ensemble size is <= 0.')
self._stopwatch.stop_task(ensemble_task_name)
# kill the datamanager as it will be re-loaded anyways from sub processes
try:
del self._datamanager
except Exception:
pass
# => RUN SMAC
smac_task_name = 'runSMAC'
self._stopwatch.start_task(smac_task_name)
time_left_for_smac = max(
0, self._time_for_task -
self._stopwatch.wall_elapsed(self._dataset_name))
if self._logger:
self._logger.info('Start SMAC with %5.2fsec time left' %
time_left_for_smac)
if time_left_for_smac <= 0:
self._logger.warning("Not starting SMAC because there is no time "
"left.")
_proc_smac = None
else:
if self._per_run_time_limit is None or \
self._per_run_time_limit > time_left_for_smac:
print('Time limit for a single run is higher than total time '
'limit. Capping the limit for a single run to the total '
'time given to SMAC (%f)' % time_left_for_smac)
per_run_time_limit = time_left_for_smac
else:
per_run_time_limit = self._per_run_time_limit
_proc_smac = AutoMLSMBO(
config_space=self.configuration_space,
dataset_name=self._dataset_name,
backend=self._backend,
total_walltime_limit=time_left_for_smac,
func_eval_time_limit=per_run_time_limit,
memory_limit=self._ml_memory_limit,
data_memory_limit=self._data_memory_limit,
watcher=self._stopwatch,
start_num_run=num_run,
num_metalearning_cfgs=self.
_initial_configurations_via_metalearning,
config_file=configspace_path,
seed=self._seed,
metadata_directory=self._metadata_directory,
metric=self._metric,
resampling_strategy=self._resampling_strategy,
resampling_strategy_args=self._resampling_strategy_arguments,
shared_mode=self._shared_mode,
include_estimators=self._include_estimators,
exclude_estimators=self._exclude_estimators,
include_preprocessors=self._include_preprocessors,
exclude_preprocessors=self._exclude_preprocessors,
disable_file_output=self._disable_evaluator_output,
get_smac_object_callback=self._get_smac_object_callback,
smac_scenario_args=self._smac_scenario_args,
)
self.runhistory_, self.trajectory_ = \
_proc_smac.run_smbo()
trajectory_filename = os.path.join(
self._backend.get_smac_output_directory_for_run(self._seed),
'trajectory.json')
saveable_trajectory = \
[list(entry[:2]) + [entry[2].get_dictionary()] + list(entry[3:])
for entry in self.trajectory_]
with open(trajectory_filename, 'w') as fh:
json.dump(saveable_trajectory, fh)
# Wait until the ensemble process is finished to avoid shutting down
# while the ensemble builder tries to access the data
if self._proc_ensemble is not None and self._ensemble_size > 0:
self._proc_ensemble.join()
self._proc_ensemble = None
self._load_models()
return self
def _fit(self, datamanager):
# Reset learnt stuff
self.models_ = None
self.ensemble_ = None
# Check arguments prior to doing anything!
if self._resampling_strategy not in [
'holdout', 'holdout-iterative-fit', 'cv', 'nested-cv',
'partial-cv'
]:
raise ValueError('Illegal resampling strategy: %s' %
self._resampling_strategy)
if self._resampling_strategy == 'partial-cv' and \
self._ensemble_size != 0:
raise ValueError("Resampling strategy partial-cv cannot be used "
"together with ensembles.")
acquisition_functions = ['EI', 'EIPS']
if self.acquisition_function not in acquisition_functions:
raise ValueError(
'Illegal acquisition %s: Must be one of %s.' %
(self.acquisition_function, acquisition_functions))
self._backend._make_internals_directory()
if self._keep_models:
try:
os.mkdir(self._backend.get_model_dir())
except OSError:
self._logger.warning("model directory already exists")
if not self._shared_mode:
raise
self._metric = datamanager.info['metric']
self._task = datamanager.info['task']
self._label_num = datamanager.info['label_num']
# == Pickle the data manager to speed up loading
data_manager_path = self._backend.save_datamanager(datamanager)
self._save_ensemble_data(datamanager.data['X_train'],
datamanager.data['Y_train'])
time_for_load_data = self._stopwatch.wall_elapsed(self._dataset_name)
if self._debug_mode:
self._print_load_time(self._dataset_name, self._time_for_task,
time_for_load_data, self._logger)
# == Perform dummy predictions
num_run = 1
#if self._resampling_strategy in ['holdout', 'holdout-iterative-fit']:
num_run = self._do_dummy_prediction(datamanager, num_run)
# = Create a searchspace
# Do this before One Hot Encoding to make sure that it creates a
# search space for a dense classifier even if one hot encoding would
# make it sparse (tradeoff; if one hot encoding would make it sparse,
# densifier and truncatedSVD would probably lead to a MemoryError,
# like this we can't use some of the preprocessing methods in case
# the data became sparse)
self.configuration_space, configspace_path = self._create_search_space(
self._tmp_dir, self._backend, datamanager,
self._include_estimators, self._include_preprocessors)
# == RUN ensemble builder
# Do this before calculating the meta-features to make sure that the
# dummy predictions are actually included in the ensemble even if
# calculating the meta-features takes very long
ensemble_task_name = 'runEnsemble'
self._stopwatch.start_task(ensemble_task_name)
time_left_for_ensembles = max(0,self._time_for_task \
- self._stopwatch.wall_elapsed(self._dataset_name))
if self._logger:
self._logger.info('Start Ensemble with %5.2fsec time left' %
time_left_for_ensembles)
if time_left_for_ensembles <= 0:
self._logger.warning("Not starting ensemble builder because there "
"is no time left!")
self._proc_ensemble = None
else:
self._proc_ensemble = self._get_ensemble_process(
time_left_for_ensembles)
self._proc_ensemble.start()
self._stopwatch.stop_task(ensemble_task_name)
# == RUN SMBO
# kill the datamanager as it will be re-loaded anyways from sub processes
try:
del self._datamanager
except Exception:
pass
# => RUN SMAC
smac_task_name = 'runSMAC'
self._stopwatch.start_task(smac_task_name)
time_left_for_smac = max(
0, self._time_for_task -
self._stopwatch.wall_elapsed(self._dataset_name))
if self._logger:
self._logger.info('Start SMAC with %5.2fsec time left' %
time_left_for_smac)
if time_left_for_smac <= 0:
self._logger.warning("Not starting SMAC because there is no time "
"left.")
self._procsmac = None
else:
self._proc_smac = AutoMLSMBO(
config_space=self.configuration_space,
dataset_name=self._dataset_name,
tmp_dir=self._tmp_dir,
output_dir=self._output_dir,
total_walltime_limit=time_left_for_smac,
func_eval_time_limit=self._per_run_time_limit,
memory_limit=self._ml_memory_limit,
data_memory_limit=self._data_memory_limit,
watcher=self._stopwatch,
start_num_run=num_run,
num_metalearning_cfgs=self.
_initial_configurations_via_metalearning,
config_file=configspace_path,
smac_iters=self._max_iter_smac,
seed=self._seed,
metadata_directory=self._metadata_directory,
resampling_strategy=self._resampling_strategy,
resampling_strategy_args=self._resampling_strategy_arguments,
acquisition_function=self.acquisition_function,
shared_mode=self._shared_mode)
self._proc_smac.start()
psutil_procs = []
procs = []
if self._proc_smac is not None:
proc_smac = psutil.Process(self._proc_smac.pid)
psutil_procs.append(proc_smac)
procs.append(self._proc_smac)
if self._proc_ensemble is not None:
proc_ensemble = psutil.Process(self._proc_ensemble.pid)
psutil_procs.append(proc_ensemble)
procs.append(self._proc_ensemble)
if self._queue is not None:
self._queue.put(
[time_for_load_data, data_manager_path, psutil_procs])
else:
for proc in procs:
proc.join()
if self._queue is None:
self._load_models()
return self
def fit(
self,
X: np.ndarray,
y: np.ndarray,
task: int,
X_test: Optional[np.ndarray] = None,
y_test: Optional[np.ndarray] = None,
feat_type: Optional[List[str]] = None,
dataset_name: Optional[str] = None,
only_return_configuration_space: Optional[bool] = False,
load_models: bool = True,
):
# Reset learnt stuff
self.models_ = None
self.cv_models_ = None
self.ensemble_ = None
# The metric must exist as of this point
# It can be provided in the constructor, or automatically
# defined in the estimator fit call
if self._metric is None:
raise ValueError('No metric given.')
if not isinstance(self._metric, Scorer):
raise ValueError('Metric must be instance of '
'autosklearn.metrics.Scorer.')
if self._shared_mode:
# If this fails, it's likely that this is the first call to get
# the data manager
try:
D = self._backend.load_datamanager()
dataset_name = D.name
except IOError:
pass
if dataset_name is None:
dataset_name = hash_array_or_matrix(X)
self._backend.save_start_time(self._seed)
self._stopwatch = StopWatch()
self._dataset_name = dataset_name
self._stopwatch.start_task(self._dataset_name)
self._logger = self._get_logger(dataset_name)
if feat_type is not None and len(feat_type) != X.shape[1]:
raise ValueError('Array feat_type does not have same number of '
'variables as X has features. %d vs %d.' %
(len(feat_type), X.shape[1]))
if feat_type is not None and not all(
[isinstance(f, str) for f in feat_type]):
raise ValueError('Array feat_type must only contain strings.')
if feat_type is not None:
for ft in feat_type:
if ft.lower() not in ['categorical', 'numerical']:
raise ValueError('Only `Categorical` and `Numerical` are '
'valid feature types, you passed `%s`' %
ft)
datamanager = XYDataManager(
X,
y,
X_test=X_test,
y_test=y_test,
task=task,
feat_type=feat_type,
dataset_name=dataset_name,
)
self._backend._make_internals_directory()
try:
os.makedirs(self._backend.get_model_dir())
except (OSError, FileExistsError):
if not self._shared_mode:
raise
try:
os.makedirs(self._backend.get_cv_model_dir())
except (OSError, FileExistsError):
if not self._shared_mode:
raise
self._task = datamanager.info['task']
self._label_num = datamanager.info['label_num']
# == Pickle the data manager to speed up loading
self._backend.save_datamanager(datamanager)
time_for_load_data = self._stopwatch.wall_elapsed(self._dataset_name)
if self._debug_mode:
self._print_load_time(self._dataset_name, self._time_for_task,
time_for_load_data, self._logger)
# == Perform dummy predictions
num_run = 1
# if self._resampling_strategy in ['holdout', 'holdout-iterative-fit']:
num_run = self._do_dummy_prediction(datamanager, num_run)
# = Create a searchspace
# Do this before One Hot Encoding to make sure that it creates a
# search space for a dense classifier even if one hot encoding would
# make it sparse (tradeoff; if one hot encoding would make it sparse,
# densifier and truncatedSVD would probably lead to a MemoryError,
# like this we can't use some of the preprocessing methods in case
# the data became sparse)
self.configuration_space, configspace_path = self._create_search_space(
self._backend.temporary_directory,
self._backend,
datamanager,
include_estimators=self._include_estimators,
exclude_estimators=self._exclude_estimators,
include_preprocessors=self._include_preprocessors,
exclude_preprocessors=self._exclude_preprocessors)
if only_return_configuration_space:
return self.configuration_space
# == RUN ensemble builder
# Do this before calculating the meta-features to make sure that the
# dummy predictions are actually included in the ensemble even if
# calculating the meta-features takes very long
ensemble_task_name = 'runEnsemble'
self._stopwatch.start_task(ensemble_task_name)
elapsed_time = self._stopwatch.wall_elapsed(self._dataset_name)
time_left_for_ensembles = max(0, self._time_for_task - elapsed_time)
if time_left_for_ensembles <= 0:
self._proc_ensemble = None
# Fit only raises error when ensemble_size is not zero but
# time_left_for_ensembles is zero.
if self._ensemble_size > 0:
raise ValueError("Not starting ensemble builder because there "
"is no time left. Try increasing the value "
"of time_left_for_this_task.")
elif self._ensemble_size <= 0:
self._proc_ensemble = None
self._logger.info('Not starting ensemble builder because '
'ensemble size is <= 0.')
else:
self._logger.info('Start Ensemble with %5.2fsec time left' %
time_left_for_ensembles)
self._proc_ensemble = self._get_ensemble_process(
time_left_for_ensembles)
self._proc_ensemble.start()
self._stopwatch.stop_task(ensemble_task_name)
# kill the datamanager as it will be re-loaded anyways from sub processes
try:
del self._datamanager
except Exception:
pass
# => RUN SMAC
smac_task_name = 'runSMAC'
self._stopwatch.start_task(smac_task_name)
elapsed_time = self._stopwatch.wall_elapsed(self._dataset_name)
time_left_for_smac = max(0, self._time_for_task - elapsed_time)
if self._logger:
self._logger.info('Start SMAC with %5.2fsec time left' %
time_left_for_smac)
if time_left_for_smac <= 0:
self._logger.warning("Not starting SMAC because there is no time "
"left.")
_proc_smac = None
self._budget_type = None
else:
if self._per_run_time_limit is None or \
self._per_run_time_limit > time_left_for_smac:
self._logger.warning(
'Time limit for a single run is higher than total time '
'limit. Capping the limit for a single run to the total '
'time given to SMAC (%f)' % time_left_for_smac)
per_run_time_limit = time_left_for_smac
else:
per_run_time_limit = self._per_run_time_limit
# Make sure that at least 2 models are created for the ensemble process
num_models = time_left_for_smac // per_run_time_limit
if num_models < 2:
per_run_time_limit = time_left_for_smac // 2
self._logger.warning(
"Capping the per_run_time_limit to {} to have "
"time for a least 2 models in each process.".format(
per_run_time_limit))
_proc_smac = AutoMLSMBO(
config_space=self.configuration_space,
dataset_name=self._dataset_name,
backend=self._backend,
total_walltime_limit=time_left_for_smac,
func_eval_time_limit=per_run_time_limit,
memory_limit=self._ml_memory_limit,
data_memory_limit=self._data_memory_limit,
watcher=self._stopwatch,
start_num_run=num_run,
num_metalearning_cfgs=self.
_initial_configurations_via_metalearning,
config_file=configspace_path,
seed=self._seed,
metadata_directory=self._metadata_directory,
metric=self._metric,
resampling_strategy=self._resampling_strategy,
resampling_strategy_args=self._resampling_strategy_arguments,
shared_mode=self._shared_mode,
include_estimators=self._include_estimators,
exclude_estimators=self._exclude_estimators,
include_preprocessors=self._include_preprocessors,
exclude_preprocessors=self._exclude_preprocessors,
disable_file_output=self._disable_evaluator_output,
get_smac_object_callback=self._get_smac_object_callback,
smac_scenario_args=self._smac_scenario_args,
)
try:
self.runhistory_, self.trajectory_, self._budget_type = \
_proc_smac.run_smbo()
trajectory_filename = os.path.join(
self._backend.get_smac_output_directory_for_run(
self._seed), 'trajectory.json')
saveable_trajectory = \
[list(entry[:2]) + [entry[2].get_dictionary()] + list(entry[3:])
for entry in self.trajectory_]
with open(trajectory_filename, 'w') as fh:
json.dump(saveable_trajectory, fh)
except Exception as e:
self._logger.exception(e)
raise
# Wait until the ensemble process is finished to avoid shutting down
# while the ensemble builder tries to access the data
if self._proc_ensemble is not None and self._ensemble_size > 0:
self._proc_ensemble.join()
self._proc_ensemble = None
if load_models:
self._load_models()
return self
