def test_inject_dependencies(self):
# initialize objects with missing dependencies
ta = ExecuteTAFuncDict(lambda x: x**2)
rh = RunHistory(aggregate_func=None)
acqu_func = EI(model=None)
intensifier = Intensifier(tae_runner=None,
stats=None,
traj_logger=None,
rng=np.random.RandomState(),
instances=None)
init_design = DefaultConfiguration(tae_runner=None,
scenario=None,
stats=None,
traj_logger=None,
rng=np.random.RandomState())
rh2epm = RunHistory2EPM4Cost(scenario=self.scenario, num_params=0)
rh2epm.scenario = None
# assert missing dependencies
self.assertIsNone(rh.aggregate_func)
self.assertIsNone(acqu_func.model)
self.assertIsNone(intensifier.tae_runner)
self.assertIsNone(intensifier.stats)
self.assertIsNone(intensifier.traj_logger)
self.assertIsNone(init_design.tae_runner)
self.assertIsNone(init_design.scenario)
self.assertIsNone(init_design.stats)
self.assertIsNone(init_design.traj_logger)
self.assertIsNone(rh2epm.scenario)
# initialize smac-object
SMAC(scenario=self.scenario,
tae_runner=ta,
runhistory=rh,
intensifier=intensifier,
acquisition_function=acqu_func,
runhistory2epm=rh2epm,
initial_design=init_design)
# assert that missing dependencies are injected
self.assertIsNotNone(rh.aggregate_func, AbstractAcquisitionFunction)
self.assertIsInstance(acqu_func.model, AbstractEPM)
self.assertIsInstance(intensifier.tae_runner, ExecuteTARun)
self.assertIsInstance(intensifier.stats, Stats)
self.assertIsInstance(intensifier.traj_logger, TrajLogger)
self.assertIsInstance(init_design.tae_runner, ExecuteTARun)
self.assertIsInstance(init_design.scenario, Scenario)
self.assertIsInstance(init_design.stats, Stats)
self.assertIsInstance(init_design.traj_logger, TrajLogger)
self.assertIsInstance(rh2epm.scenario, Scenario)
def get_smac_object(
scenario_dict,
seed,
ta,
backend,
metalearning_configurations,
runhistory,
):
scenario_dict['input_psmac_dirs'] = backend.get_smac_output_glob(
smac_run_id=seed if not scenario_dict['shared-model'] else '*', )
scenario = Scenario(scenario_dict)
if len(metalearning_configurations) > 0:
default_config = scenario.cs.get_default_configuration()
#TODO for Milad
#write function here which load the configuration from openml and put to metalearningconfiguration
initial_configurations = [default_config] + metalearning_configurations
else:
initial_configurations = None
rh2EPM = RunHistory2EPM4Cost(
num_params=len(scenario.cs.get_hyperparameters()),
scenario=scenario,
success_states=[
StatusType.SUCCESS,
StatusType.MEMOUT,
StatusType.TIMEOUT,
# As long as we don't have a model for crashes yet!
StatusType.CRASHED,
],
impute_censored_data=False,
impute_state=None,
)
return SMAC(
scenario=scenario,
rng=seed,
runhistory2epm=rh2EPM,
tae_runner=ta,
initial_configurations=initial_configurations,
runhistory=runhistory,
run_id=seed,
)
def convert_data_for_epm(scenario: Scenario,
runhistory: RunHistory,
impute_inactive_parameters=False,
rng=None,
logger=None):
"""
converts data from runhistory into EPM format
Parameters
----------
scenario: Scenario
smac.scenario.scenario.Scenario Object
runhistory: RunHistory
smac.runhistory.runhistory.RunHistory Object with all necessary data
impute_inactive_parameters: bool
whether to impute all inactive parameters in all configurations - this is needed for random forests, as they do not accept nan-values
Returns
-------
X: np.array
X matrix with configuartion x features for all observed samples
y: np.array
y matrix with all observations
types: np.array
types of X cols -- necessary to train our RF implementation
"""
if rng is None:
rng = np.random.RandomState(42)
if impute_inactive_parameters:
runhistory = force_finite_runhistory(runhistory)
types, bounds = get_types(scenario.cs, scenario.feature_array)
if logger is not None:
logger.debug("Types: " + str(types) + ", Bounds: " + str(bounds))
model = RandomForestWithInstances(scenario.cs, types, bounds,
rng.randint(MAXINT))
params = scenario.cs.get_hyperparameters()
num_params = len(params)
run_obj = scenario.run_obj
if run_obj == "runtime":
# if we log the performance data,
# the RFRImputator will already get
# log transform data from the runhistory
cutoff = np.log10(scenario.cutoff)
threshold = np.log10(scenario.cutoff * scenario.par_factor)
imputor = RFRImputator(rng=rng,
cutoff=cutoff,
threshold=threshold,
model=model,
change_threshold=0.01,
max_iter=10)
# TODO: Adapt runhistory2EPM object based on scenario
rh2EPM = RunHistory2EPM4LogCost(scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=True,
impute_state=[
StatusType.TIMEOUT,
],
imputor=imputor)
X, Y = rh2EPM.transform(runhistory)
else:
rh2EPM = RunHistory2EPM4Cost(scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=False,
impute_state=None)
X, Y = rh2EPM.transform(runhistory)
return X, Y, types
def create_server(self) -> SMAC:
"""Function to create the server.
Returns
-------
smac : SMAC
Return the fully configured SMAC object.
"""
# 首先创建输出文件夹
output_dir = self.temp_folder + "server-output_%s" % (
datetime.datetime.fromtimestamp(time.time()).strftime(
'%Y-%m-%d_%H:%M:%S_%f'))
# 创建scenario
scenario_dict = {
"cs": self.cs,
"run_obj": "quality",
"cutoff_time": self.cutoff,
"initial_incumbent": "RANDOM",
"output_dir": output_dir
}
scenario = Scenario(scenario_dict)
# Runhistory实例
runhistory = RunHistory(aggregate_func=average_cost)
runhistory2epm = RunHistory2EPM4Cost(scenario=scenario,
num_params=1,
success_states=[
StatusType.SUCCESS,
StatusType.CRASHED],
impute_censored_data=False,
impute_state=None)
# 创建server对象
server = Server(self.server_args, self.cs)
# 创建ta函数，不存放数据，因为server不真正运行ta
ta = CustomizedTA(np.array(0), np.array(0), np.array(0), np.array(0))
# 创建smac
if self.our_work is not None:
# our_work原则上是打开gradient文件路径的str
hoag = self._cal_hoag()
# 创建epm
gpr = GaussianGradientEPM
smac = SMAC(
scenario=scenario,
tae_runner=ta,
model=gpr(),
hoag=hoag,
runhistory2epm=runhistory2epm,
runhistory=runhistory,
server=server
)
else:
smac = SMAC(scenario=scenario, tae_runner=ta,
runhistory2epm=runhistory2epm,
runhistory=runhistory,
server=server)
# 返回smac对象
return smac
def convert_data(scenario: Scenario, runhistory: RunHistory):
'''
converts data from runhistory into EPM format
Parameters
----------
scenario: Scenario
smac.scenario.scenario.Scenario Object
runhistory: RunHistory
smac.runhistory.runhistory.RunHistory Object with all necessary data
Returns
-------
np.array
X matrix with configuartion x features for all observed samples
np.array
y matrix with all observations
np.array
types of X cols -- necessary to train our RF implementation
'''
types = np.zeros(len(scenario.cs.get_hyperparameters()), dtype=np.uint)
for i, param in enumerate(scenario.cs.get_hyperparameters()):
if isinstance(param, (CategoricalHyperparameter)):
n_cats = len(param.choices)
types[i] = n_cats
if scenario.feature_array is not None:
types = np.hstack((types, np.zeros((scenario.feature_array.shape[1]))))
types = np.array(types, dtype=np.uint)
model = RandomForestWithInstances(types, scenario.feature_array)
params = scenario.cs.get_hyperparameters()
num_params = len(params)
if scenario.run_obj == "runtime":
if scenario.run_obj == "runtime":
# if we log the performance data,
# the RFRImputator will already get
# log transform data from the runhistory
cutoff = np.log10(scenario.cutoff)
threshold = np.log10(scenario.cutoff * scenario.par_factor)
else:
cutoff = scenario.cutoff
threshold = scenario.cutoff * scenario.par_factor
imputor = RFRImputator(cs=scenario.cs,
rs=np.random.RandomState(42),
cutoff=cutoff,
threshold=threshold,
model=model,
change_threshold=0.01,
max_iter=10)
# TODO: Adapt runhistory2EPM object based on scenario
rh2EPM = RunHistory2EPM4LogCost(scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=False,
impute_state=[
StatusType.TIMEOUT,
],
imputor=imputor)
else:
rh2EPM = RunHistory2EPM4Cost(scenario=scenario,
num_params=num_params,
success_states=None,
impute_censored_data=False,
impute_state=None)
X, Y = rh2EPM.transform(runhistory)
return X, Y, types
def __init__(
self,
scenario: Scenario,
tae_runner: typing.Union[ExecuteTARun, typing.Callable] = None,
runhistory: RunHistory = None,
intensifier: Intensifier = None,
acquisition_function: AbstractAcquisitionFunction = None,
acquisition_function_optimizer: AcquisitionFunctionMaximizer = None,
model: AbstractEPM = None,
runhistory2epm: AbstractRunHistory2EPM = None,
initial_design: InitialDesign = None,
initial_configurations: typing.List[Configuration] = None,
stats: Stats = None,
restore_incumbent: Configuration = None,
rng: typing.Union[np.random.RandomState, int] = None,
smbo_class: SMBO = None,
run_id: int = 1):
"""Constructor
Parameters
----------
scenario : ~smac.scenario.scenario.Scenario
Scenario object
tae_runner : ~smac.tae.execute_ta_run.ExecuteTARun or callable
Callable or implementation of
:class:`~smac.tae.execute_ta_run.ExecuteTARun`. In case a
callable is passed it will be wrapped by
:class:`~smac.tae.execute_func.ExecuteTAFuncDict`.
If not set, it will be initialized with the
:class:`~smac.tae.execute_ta_run_old.ExecuteTARunOld`.
runhistory : RunHistory
runhistory to store all algorithm runs
intensifier : Intensifier
intensification object to issue a racing to decide the current
incumbent
acquisition_function : ~smac.optimizer.acquisition.AbstractAcquisitionFunction
Object that implements the :class:`~smac.optimizer.acquisition.AbstractAcquisitionFunction`.
Will use :class:`~smac.optimizer.acquisition.EI` if not set.
acquisition_function_optimizer : ~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer
Object that implements the :class:`~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer`.
Will use :class:`smac.optimizer.ei_optimization.InterleavedLocalAndRandomSearch` if not set.
model : AbstractEPM
Model that implements train() and predict(). Will use a
:class:`~smac.epm.rf_with_instances.RandomForestWithInstances` if not set.
runhistory2epm : ~smac.runhistory.runhistory2epm.RunHistory2EMP
Object that implements the AbstractRunHistory2EPM. If None,
will use :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4Cost`
if objective is cost or
:class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4LogCost`
if objective is runtime.
initial_design : InitialDesign
initial sampling design
initial_configurations : typing.List[Configuration]
list of initial configurations for initial design --
cannot be used together with initial_design
stats : Stats
optional stats object
rng : np.random.RandomState
Random number generator
restore_incumbent : Configuration
incumbent used if restoring to previous state
smbo_class : ~smac.optimizer.smbo.SMBO
Class implementing the SMBO interface which will be used to
instantiate the optimizer class.
run_id: int, (default: 1)
Run ID will be used as subfolder for output_dir.
"""
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
aggregate_func = average_cost
self.output_dir = create_output_directory(scenario, run_id)
scenario.write()
# initialize stats object
if stats:
self.stats = stats
else:
self.stats = Stats(scenario)
# initialize empty runhistory
if runhistory is None:
runhistory = RunHistory(aggregate_func=aggregate_func)
# inject aggr_func if necessary
if runhistory.aggregate_func is None:
runhistory.aggregate_func = aggregate_func
# initial random number generator
num_run, rng = self._get_rng(rng=rng)
# reset random number generator in config space to draw different
# random configurations with each seed given to SMAC
scenario.cs.seed(rng.randint(MAXINT))
# initial Trajectory Logger
traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats)
# initial EPM
types, bounds = get_types(scenario.cs, scenario.feature_array)
if model is None:
model = RandomForestWithInstances(
types=types,
bounds=bounds,
instance_features=scenario.feature_array,
seed=rng.randint(MAXINT),
pca_components=scenario.PCA_DIM)
# initial acquisition function
if acquisition_function is None:
if scenario.run_obj == "runtime":
acquisition_function = LogEI(model=model)
else:
acquisition_function = EI(model=model)
# inject model if necessary
if acquisition_function.model is None:
acquisition_function.model = model
# initialize optimizer on acquisition function
if acquisition_function_optimizer is None:
acquisition_function_optimizer = InterleavedLocalAndRandomSearch(
acquisition_function, scenario.cs,
np.random.RandomState(seed=rng.randint(MAXINT)))
elif not isinstance(
acquisition_function_optimizer,
AcquisitionFunctionMaximizer,
):
raise ValueError(
"Argument 'acquisition_function_optimizer' must be of type"
"'AcquisitionFunctionMaximizer', but is '%s'" %
type(acquisition_function_optimizer))
# initialize tae_runner
# First case, if tae_runner is None, the target algorithm is a call
# string in the scenario file
if tae_runner is None:
tae_runner = ExecuteTARunOld(
ta=scenario.ta,
stats=self.stats,
run_obj=scenario.run_obj,
runhistory=runhistory,
par_factor=scenario.par_factor,
cost_for_crash=scenario.cost_for_crash)
# Second case, the tae_runner is a function to be optimized
elif callable(tae_runner):
tae_runner = ExecuteTAFuncDict(
ta=tae_runner,
stats=self.stats,
run_obj=scenario.run_obj,
memory_limit=scenario.memory_limit,
runhistory=runhistory,
par_factor=scenario.par_factor,
cost_for_crash=scenario.cost_for_crash)
# Third case, if it is an ExecuteTaRun we can simply use the
# instance. Otherwise, the next check raises an exception
elif not isinstance(tae_runner, ExecuteTARun):
raise TypeError("Argument 'tae_runner' is %s, but must be "
"either a callable or an instance of "
"ExecuteTaRun. Passing 'None' will result in the "
"creation of target algorithm runner based on the "
"call string in the scenario file." %
type(tae_runner))
# Check that overall objective and tae objective are the same
if tae_runner.run_obj != scenario.run_obj:
raise ValueError("Objective for the target algorithm runner and "
"the scenario must be the same, but are '%s' and "
"'%s'" % (tae_runner.run_obj, scenario.run_obj))
# inject stats if necessary
if tae_runner.stats is None:
tae_runner.stats = self.stats
# inject runhistory if necessary
if tae_runner.runhistory is None:
tae_runner.runhistory = runhistory
# inject cost_for_crash
if tae_runner.crash_cost != scenario.cost_for_crash:
tae_runner.crash_cost = scenario.cost_for_crash
# initialize intensification
if intensifier is None:
intensifier = Intensifier(tae_runner=tae_runner,
stats=self.stats,
traj_logger=traj_logger,
rng=rng,
instances=scenario.train_insts,
cutoff=scenario.cutoff,
deterministic=scenario.deterministic,
run_obj_time=scenario.run_obj == "runtime",
always_race_against=scenario.cs.get_default_configuration() \
if scenario.always_race_default else None,
instance_specifics=scenario.instance_specific,
minR=scenario.minR,
maxR=scenario.maxR)
# inject deps if necessary
if intensifier.tae_runner is None:
intensifier.tae_runner = tae_runner
if intensifier.stats is None:
intensifier.stats = self.stats
if intensifier.traj_logger is None:
intensifier.traj_logger = traj_logger
# initial design
if initial_design is not None and initial_configurations is not None:
raise ValueError(
"Either use initial_design or initial_configurations; but not both"
)
if initial_configurations is not None:
initial_design = MultiConfigInitialDesign(
tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
runhistory=runhistory,
rng=rng,
configs=initial_configurations,
intensifier=intensifier,
aggregate_func=aggregate_func)
elif initial_design is None:
if scenario.initial_incumbent == "DEFAULT":
initial_design = DefaultConfiguration(tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
elif scenario.initial_incumbent == "RANDOM":
initial_design = RandomConfiguration(tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
else:
raise ValueError("Don't know what kind of initial_incumbent "
"'%s' is" % scenario.initial_incumbent)
# inject deps if necessary
if initial_design.tae_runner is None:
initial_design.tae_runner = tae_runner
if initial_design.scenario is None:
initial_design.scenario = scenario
if initial_design.stats is None:
initial_design.stats = self.stats
if initial_design.traj_logger is None:
initial_design.traj_logger = traj_logger
# initial conversion of runhistory into EPM data
if runhistory2epm is None:
num_params = len(scenario.cs.get_hyperparameters())
if scenario.run_obj == "runtime":
# if we log the performance data,
# the RFRImputator will already get
# log transform data from the runhistory
cutoff = np.log10(scenario.cutoff)
threshold = np.log10(scenario.cutoff * scenario.par_factor)
imputor = RFRImputator(rng=rng,
cutoff=cutoff,
threshold=threshold,
model=model,
change_threshold=0.01,
max_iter=2)
runhistory2epm = RunHistory2EPM4LogCost(
scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=True,
impute_state=[
StatusType.CAPPED,
],
imputor=imputor)
elif scenario.run_obj == 'quality':
runhistory2epm = RunHistory2EPM4Cost(
scenario=scenario,
num_params=num_params,
success_states=[StatusType.SUCCESS, StatusType.CRASHED],
impute_censored_data=False,
impute_state=None)
else:
raise ValueError('Unknown run objective: %s. Should be either '
'quality or runtime.' % self.scenario.run_obj)
# inject scenario if necessary:
if runhistory2epm.scenario is None:
runhistory2epm.scenario = scenario
smbo_args = {
'scenario': scenario,
'stats': self.stats,
'initial_design': initial_design,
'runhistory': runhistory,
'runhistory2epm': runhistory2epm,
'intensifier': intensifier,
'aggregate_func': aggregate_func,
'num_run': num_run,
'model': model,
'acq_optimizer': acquisition_function_optimizer,
'acquisition_func': acquisition_function,
'rng': rng,
'restore_incumbent': restore_incumbent
}
if smbo_class is None:
self.solver = SMBO(**smbo_args)
else:
self.solver = smbo_class(**smbo_args)
def __init__(self,
scenario: Scenario,
tae_runner: Optional[Union[Type[ExecuteTARun], Callable]] = None,
tae_runner_kwargs: Optional[dict] = None,
runhistory: Optional[Union[Type[RunHistory], RunHistory]] = None,
runhistory_kwargs: Optional[dict] = None,
intensifier: Optional[Type[Intensifier]] = None,
intensifier_kwargs: Optional[dict] = None,
acquisition_function: Optional[Type[AbstractAcquisitionFunction]] = None,
acquisition_function_kwargs: Optional[dict] = None,
integrate_acquisition_function: bool = False,
acquisition_function_optimizer: Optional[Type[AcquisitionFunctionMaximizer]] = None,
acquisition_function_optimizer_kwargs: Optional[dict] = None,
model: Optional[Type[AbstractEPM]] = None,
model_kwargs: Optional[dict] = None,
runhistory2epm: Optional[Type[AbstractRunHistory2EPM]] = None,
runhistory2epm_kwargs: Optional[dict] = None,
initial_design: Optional[Type[InitialDesign]] = None,
initial_design_kwargs: Optional[dict] = None,
initial_configurations: Optional[List[Configuration]] = None,
stats: Optional[Stats] = None,
restore_incumbent: Optional[Configuration] = None,
rng: Optional[Union[np.random.RandomState, int]] = None,
smbo_class: Optional[SMBO] = None,
run_id: Optional[int] = None,
random_configuration_chooser: Optional[Type[RandomConfigurationChooser]] = None,
random_configuration_chooser_kwargs: Optional[dict] = None
):
"""
Constructor
Parameters
----------
scenario : ~smac.scenario.scenario.Scenario
Scenario object
tae_runner : ~smac.tae.execute_ta_run.ExecuteTARun or callable
Callable or implementation of
:class:`~smac.tae.execute_ta_run.ExecuteTARun`. In case a
callable is passed it will be wrapped by
:class:`~smac.tae.execute_func.ExecuteTAFuncDict`.
If not set, it will be initialized with the
:class:`~smac.tae.execute_ta_run_old.ExecuteTARunOld`.
tae_runner_kwargs: Optional[dict]
arguments passed to constructor of '~tae_runner'
runhistory : RunHistory
runhistory to store all algorithm runs
runhistory_kwargs : Optional[dict]
arguments passed to constructor of runhistory.
We strongly advise against changing the aggregation function,
since it will break some code assumptions
intensifier : Intensifier
intensification object to issue a racing to decide the current
incumbent
intensifier_kwargs: Optional[dict]
arguments passed to the constructor of '~intensifier'
acquisition_function : ~smac.optimizer.acquisition.AbstractAcquisitionFunction
Class or object that implements the :class:`~smac.optimizer.acquisition.AbstractAcquisitionFunction`.
Will use :class:`~smac.optimizer.acquisition.EI` or :class:`~smac.optimizer.acquisition.LogEI` if not set.
`~acquisition_function_kwargs` is passed to the class constructor.
acquisition_function_kwargs : Optional[dict]
dictionary to pass specific arguments to ~acquisition_function
integrate_acquisition_function : bool, default=False
Whether to integrate the acquisition function. Works only with models which can sample their
hyperparameters (i.e. GaussianProcessMCMC).
acquisition_function_optimizer : ~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer
Object that implements the :class:`~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer`.
Will use :class:`smac.optimizer.ei_optimization.InterleavedLocalAndRandomSearch` if not set.
acquisition_function_optimizer_kwargs: Optional[dict]
Arguments passed to constructor of '~acquisition_function_optimizer'
model : AbstractEPM
Model that implements train() and predict(). Will use a
:class:`~smac.epm.rf_with_instances.RandomForestWithInstances` if not set.
model_kwargs : Optional[dict]
Arguments passed to constructor of '~model'
runhistory2epm : ~smac.runhistory.runhistory2epm.RunHistory2EMP
Object that implements the AbstractRunHistory2EPM. If None,
will use :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4Cost`
if objective is cost or
:class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4LogCost`
if objective is runtime.
runhistory2epm_kwargs: Optional[dict]
Arguments passed to the constructor of '~runhistory2epm'
initial_design : InitialDesign
initial sampling design
initial_design_kwargs: Optional[dict]
arguments passed to constructor of `~initial_design'
initial_configurations : List[Configuration]
list of initial configurations for initial design --
cannot be used together with initial_design
stats : Stats
optional stats object
rng : np.random.RandomState
Random number generator
restore_incumbent : Configuration
incumbent used if restoring to previous state
smbo_class : ~smac.optimizer.smbo.SMBO
Class implementing the SMBO interface which will be used to
instantiate the optimizer class.
run_id : int (optional)
Run ID will be used as subfolder for output_dir. If no ``run_id`` is given, a random ``run_id`` will be
chosen.
random_configuration_chooser : ~smac.optimizer.random_configuration_chooser.RandomConfigurationChooser
How often to choose a random configuration during the intensification procedure.
random_configuration_chooser_kwargs : Optional[dict]
arguments of constructor for '~random_configuration_chooser'
"""
self.logger = logging.getLogger(
self.__module__ + "." + self.__class__.__name__)
aggregate_func = average_cost
self.scenario = scenario
self.output_dir = ""
if not restore_incumbent:
# restore_incumbent is used by the CLI interface which provides a method for restoring a SMAC run given an
# output directory. This is the default path.
# initial random number generator
# run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger)
# run_id=datetime.now().strftime("%Y%m%d%H%M%S%f")
run_id=uuid1()
self.output_dir = create_output_directory(scenario, run_id) # fixme run_id
elif scenario.output_dir is not None:
run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger)
# output-directory is created in CLI when restoring from a
# folder. calling the function again in the facade results in two
# folders being created: run_X and run_X.OLD. if we are
# restoring, the output-folder exists already and we omit creating it,
# but set the self-output_dir to the dir.
# necessary because we want to write traj to new output-dir in CLI.
self.output_dir = scenario.output_dir_for_this_run
if (
scenario.deterministic is True
and getattr(scenario, 'tuner_timeout', None) is None
and scenario.run_obj == 'quality'
):
self.logger.info('Optimizing a deterministic scenario for quality without a tuner timeout - will make '
'SMAC deterministic and only evaluate one configuration per iteration!')
scenario.intensification_percentage = 1e-10
scenario.min_chall = 1
scenario.write()
# initialize stats object
if stats:
self.stats = stats
else:
self.stats = Stats(scenario,file_system=scenario.file_system)
if self.scenario.run_obj == "runtime" and not self.scenario.transform_y == "LOG":
self.logger.warning("Runtime as objective automatically activates log(y) transformation")
self.scenario.transform_y = "LOG"
# initialize empty runhistory
runhistory_def_kwargs = {'aggregate_func': aggregate_func}
if runhistory_kwargs is not None:
runhistory_def_kwargs.update(runhistory_kwargs)
if runhistory is None:
runhistory = RunHistory(**runhistory_def_kwargs,file_system=scenario.file_system)
elif inspect.isclass(runhistory):
runhistory = runhistory(**runhistory_def_kwargs)
else:
if runhistory.aggregate_func is None:
runhistory.aggregate_func = aggregate_func
rand_conf_chooser_kwargs = {
'rng': rng
}
if random_configuration_chooser_kwargs is not None:
rand_conf_chooser_kwargs.update(random_configuration_chooser_kwargs)
if random_configuration_chooser is None:
if 'prob' not in rand_conf_chooser_kwargs:
rand_conf_chooser_kwargs['prob'] = scenario.rand_prob
random_configuration_chooser = ChooserProb(**rand_conf_chooser_kwargs)
elif inspect.isclass(random_configuration_chooser):
random_configuration_chooser = random_configuration_chooser(**rand_conf_chooser_kwargs)
elif not isinstance(random_configuration_chooser, RandomConfigurationChooser):
raise ValueError("random_configuration_chooser has to be"
" a class or object of RandomConfigurationChooser")
# reset random number generator in config space to draw different
# random configurations with each seed given to SMAC
scenario.cs.seed(rng.randint(MAXINT))
# initial Trajectory Logger
traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats,file_system=scenario.file_system)
# initial EPM
types, bounds = get_types(scenario.cs, scenario.feature_array)
model_def_kwargs = {
'types': types,
'bounds': bounds,
'instance_features': scenario.feature_array,
'seed': rng.randint(MAXINT),
'pca_components': scenario.PCA_DIM,
}
if model_kwargs is not None:
model_def_kwargs.update(model_kwargs)
if model is None:
for key, value in {
'log_y': scenario.transform_y in ["LOG", "LOGS"],
'num_trees': scenario.rf_num_trees,
'do_bootstrapping': scenario.rf_do_bootstrapping,
'ratio_features': scenario.rf_ratio_features,
'min_samples_split': scenario.rf_min_samples_split,
'min_samples_leaf': scenario.rf_min_samples_leaf,
'max_depth': scenario.rf_max_depth,
}.items():
if key not in model_def_kwargs:
model_def_kwargs[key] = value
model_def_kwargs['configspace'] = self.scenario.cs
model = RandomForestWithInstances(**model_def_kwargs)
elif inspect.isclass(model):
model_def_kwargs['configspace'] = self.scenario.cs
model = model(**model_def_kwargs)
else:
raise TypeError(
"Model not recognized: %s" %(type(model)))
# initial acquisition function
acq_def_kwargs = {'model': model}
if acquisition_function_kwargs is not None:
acq_def_kwargs.update(acquisition_function_kwargs)
if acquisition_function is None:
if scenario.transform_y in ["LOG", "LOGS"]:
acquisition_function = LogEI(**acq_def_kwargs)
else:
acquisition_function = EI(**acq_def_kwargs)
elif inspect.isclass(acquisition_function):
acquisition_function = acquisition_function(**acq_def_kwargs)
else:
raise TypeError(
"Argument acquisition_function must be None or an object implementing the "
"AbstractAcquisitionFunction, not %s."
% type(acquisition_function)
)
if integrate_acquisition_function:
acquisition_function = IntegratedAcquisitionFunction(
acquisition_function=acquisition_function,
**acq_def_kwargs
)
# initialize optimizer on acquisition function
acq_func_opt_kwargs = {
'acquisition_function': acquisition_function,
'config_space': scenario.cs,
'rng': rng,
}
if acquisition_function_optimizer_kwargs is not None:
acq_func_opt_kwargs.update(acquisition_function_optimizer_kwargs)
if acquisition_function_optimizer is None:
for key, value in {
'max_steps': scenario.sls_max_steps,
'n_steps_plateau_walk': scenario.sls_n_steps_plateau_walk,
}.items():
if key not in acq_func_opt_kwargs:
acq_func_opt_kwargs[key] = value
acquisition_function_optimizer = InterleavedLocalAndRandomSearch(**acq_func_opt_kwargs)
elif inspect.isclass(acquisition_function_optimizer):
acquisition_function_optimizer = acquisition_function_optimizer(**acq_func_opt_kwargs)
else:
raise TypeError(
"Argument acquisition_function_optimizer must be None or an object implementing the "
"AcquisitionFunctionMaximizer, but is '%s'" %
type(acquisition_function_optimizer)
)
# initialize tae_runner
# First case, if tae_runner is None, the target algorithm is a call
# string in the scenario file
tae_def_kwargs = {
'stats': self.stats,
'run_obj': scenario.run_obj,
'runhistory': runhistory,
'par_factor': scenario.par_factor,
'cost_for_crash': scenario.cost_for_crash,
'abort_on_first_run_crash': scenario.abort_on_first_run_crash
}
if tae_runner_kwargs is not None:
tae_def_kwargs.update(tae_runner_kwargs)
if 'ta' not in tae_def_kwargs:
tae_def_kwargs['ta'] = scenario.ta
if tae_runner is None:
tae_def_kwargs['ta'] = scenario.ta
tae_runner = ExecuteTARunOld(**tae_def_kwargs)
elif inspect.isclass(tae_runner):
tae_runner = tae_runner(**tae_def_kwargs)
elif callable(tae_runner):
tae_def_kwargs['ta'] = tae_runner
tae_runner = ExecuteTAFuncDict(**tae_def_kwargs)
else:
raise TypeError("Argument 'tae_runner' is %s, but must be "
"either None, a callable or an object implementing "
"ExecuteTaRun. Passing 'None' will result in the "
"creation of target algorithm runner based on the "
"call string in the scenario file."
% type(tae_runner))
# Check that overall objective and tae objective are the same
if tae_runner.run_obj != scenario.run_obj:
raise ValueError("Objective for the target algorithm runner and "
"the scenario must be the same, but are '%s' and "
"'%s'" % (tae_runner.run_obj, scenario.run_obj))
# initialize intensification
intensifier_def_kwargs = {
'tae_runner': tae_runner,
'stats': self.stats,
'traj_logger': traj_logger,
'rng': rng,
'instances': scenario.train_insts,
'cutoff': scenario.cutoff,
'deterministic': scenario.deterministic,
'run_obj_time': scenario.run_obj == "runtime",
'always_race_against': scenario.cs.get_default_configuration()
if scenario.always_race_default else None,
'use_ta_time_bound': scenario.use_ta_time,
'instance_specifics': scenario.instance_specific,
'minR': scenario.minR,
'maxR': scenario.maxR,
'adaptive_capping_slackfactor': scenario.intens_adaptive_capping_slackfactor,
'min_chall': scenario.intens_min_chall
}
if intensifier_kwargs is not None:
intensifier_def_kwargs.update(intensifier_kwargs)
if intensifier is None:
intensifier = Intensifier(**intensifier_def_kwargs)
elif inspect.isclass(intensifier):
intensifier = intensifier(**intensifier_def_kwargs)
else:
raise TypeError(
"Argument intensifier must be None or an object implementing the Intensifier, but is '%s'" %
type(intensifier)
)
# initial design
if initial_design is not None and initial_configurations is not None:
raise ValueError(
"Either use initial_design or initial_configurations; but not both")
init_design_def_kwargs = {
'tae_runner': tae_runner,
'scenario': scenario,
'stats': self.stats,
'traj_logger': traj_logger,
'runhistory': runhistory,
'rng': rng,
'configs': initial_configurations,
'intensifier': intensifier,
'aggregate_func': aggregate_func,
'n_configs_x_params': 0,
'max_config_fracs': 0.0
}
if initial_design_kwargs is not None:
init_design_def_kwargs.update(initial_design_kwargs)
if initial_configurations is not None:
initial_design = InitialDesign(**init_design_def_kwargs)
elif initial_design is None:
if scenario.initial_incumbent == "DEFAULT":
init_design_def_kwargs['max_config_fracs'] = 0.0
initial_design = DefaultConfiguration(**init_design_def_kwargs)
elif scenario.initial_incumbent == "RANDOM":
init_design_def_kwargs['max_config_fracs'] = 0.0
initial_design = RandomConfigurations(**init_design_def_kwargs)
elif scenario.initial_incumbent == "LHD":
initial_design = LHDesign(**init_design_def_kwargs)
elif scenario.initial_incumbent == "FACTORIAL":
initial_design = FactorialInitialDesign(**init_design_def_kwargs)
elif scenario.initial_incumbent == "SOBOL":
initial_design = SobolDesign(**init_design_def_kwargs)
else:
raise ValueError("Don't know what kind of initial_incumbent "
"'%s' is" % scenario.initial_incumbent)
elif inspect.isclass(initial_design):
initial_design = initial_design(**init_design_def_kwargs)
else:
raise TypeError(
"Argument initial_design must be None or an object implementing the InitialDesign, but is '%s'" %
type(initial_design)
)
# if we log the performance data,
# the RFRImputator will already get
# log transform data from the runhistory
if scenario.transform_y in ["LOG", "LOGS"]:
cutoff = np.log(np.nanmin([np.inf, np.float_(scenario.cutoff)]))
threshold = cutoff + np.log(scenario.par_factor)
else:
cutoff = np.nanmin([np.inf, np.float_(scenario.cutoff)])
threshold = cutoff * scenario.par_factor
num_params = len(scenario.cs.get_hyperparameters())
imputor = RFRImputator(rng=rng,
cutoff=cutoff,
threshold=threshold,
model=model,
change_threshold=0.01,
max_iter=2)
r2e_def_kwargs = {
'scenario': scenario,
'num_params': num_params,
'success_states': [StatusType.SUCCESS, ],
'impute_censored_data': True,
'impute_state': [StatusType.CAPPED, ],
'imputor': imputor,
'scale_perc': 5
}
if scenario.run_obj == 'quality':
r2e_def_kwargs.update({
'success_states': [StatusType.SUCCESS, StatusType.CRASHED],
'impute_censored_data': False,
'impute_state': None,
})
if runhistory2epm_kwargs is not None:
r2e_def_kwargs.update(runhistory2epm_kwargs)
if runhistory2epm is None:
if scenario.run_obj == 'runtime':
runhistory2epm = RunHistory2EPM4LogCost(**r2e_def_kwargs)
elif scenario.run_obj == 'quality':
if scenario.transform_y == "NONE":
runhistory2epm = RunHistory2EPM4Cost(**r2e_def_kwargs)
elif scenario.transform_y == "LOG":
runhistory2epm = RunHistory2EPM4LogCost(**r2e_def_kwargs)
elif scenario.transform_y == "LOGS":
runhistory2epm = RunHistory2EPM4LogScaledCost(**r2e_def_kwargs)
elif scenario.transform_y == "INVS":
runhistory2epm = RunHistory2EPM4InvScaledCost(**r2e_def_kwargs)
else:
raise ValueError('Unknown run objective: %s. Should be either '
'quality or runtime.' % self.scenario.run_obj)
elif inspect.isclass(runhistory2epm):
runhistory2epm = runhistory2epm(**r2e_def_kwargs)
else:
raise TypeError(
"Argument runhistory2epm must be None or an object implementing the RunHistory2EPM, but is '%s'" %
type(runhistory2epm)
)
smbo_args = {
'scenario': scenario,
'stats': self.stats,
'initial_design': initial_design,
'runhistory': runhistory,
'runhistory2epm': runhistory2epm,
'intensifier': intensifier,
'aggregate_func': aggregate_func,
'num_run': run_id,
'model': model,
'acq_optimizer': acquisition_function_optimizer,
'acquisition_func': acquisition_function,
'rng': rng,
'restore_incumbent': restore_incumbent,
'random_configuration_chooser': random_configuration_chooser
}
if smbo_class is None:
self.solver = SMBO(**smbo_args)
else:
self.solver = smbo_class(**smbo_args)
def __init__(
self,
X_train: np.ndarray,
y_train: np.ndarray,
X_valid: np.ndarray,
y_valid: np.ndarray,
dirs: typing.List[str],
smbo_id: int,
#server: Server = None,
cs: ConfigurationSpace = None,
our_work: bool = False):
if smbo_id >= len(dirs):
raise ValueError("SMBO ID exceeds size of the pool.")
super().__init__()
# 创建一个ta函数
ta = CustomizedTA(X_train, y_train, X_valid, y_valid)
# 如果未指定configspace，则赋值默认
if cs is None:
cs = ConfigurationSpace()
# 超参搜索空间，使用[1e-6, 1]
alpha = UniformFloatHyperparameter(name="alpha",
lower=1e-3,
upper=1,
default_value=1,
log=False)
cs.add_hyperparameters([alpha])
# 创建scenario
scenario_dict = {
"cs": cs,
"run_obj": "quality",
"cutoff_time": 100,
"shared_model": True if len(dirs) > 1 else False,
#"initial_incumbent": 'RANDOM"
"input_psmac_dirs": dirs,
"output_dir": dirs[smbo_id]
}
scenario = Scenario(scenario_dict)
runhistory = RunHistory(aggregate_func=average_cost)
runhistory2epm = RunHistory2EPM4Cost(
scenario=scenario,
num_params=1,
success_states=[StatusType.SUCCESS, StatusType.CRASHED],
impute_censored_data=False,
impute_state=None)
# 创建smac
if our_work is not None:
# 读入梯度信息
with open(our_work, "r") as fp:
lines = fp.readlines()
# 计算loss的值
loss = [float(line.strip().split()[0]) for line in lines]
gradient = []
for i in range(1, len(loss)):
gradient.append((loss[i] - loss[i - 1]) * len(loss))
hoag = DummyHOAG(0.00095, 1, np.array(gradient))
# 创建epm
gpr = GaussianGradientEPM
self.smac = SMAC(
scenario=scenario,
tae_runner=ta,
model=gpr(),
hoag=hoag,
runhistory2epm=runhistory2epm,
runhistory=runhistory,
#server=server
)
else:
self.smac = SMAC(scenario=scenario,
tae_runner=ta,
runhistory2epm=runhistory2epm,
runhistory=runhistory)
def run_experiment():
pipeline_space = PipelineSpace()
o_s = OneHotEncodingStep()
i_s = ImputationStep()
r_s = RescalingStep()
b_s = BalancingStep()
p_s = PreprocessingStep()
c_s = ClassificationStep()
pipeline_space.add_pipeline_steps([o_s, i_s, r_s, b_s, p_s, c_s])
runhistory = PCRunHistory(average_cost)
cs_builder = ConfigSpaceBuilder(pipeline_space)
config_space = cs_builder.build_config_space()
args = {
'cs': config_space,
'run_obj': "quality",
'runcount_limit': 100,
'wallclock_limit': 100,
'memory_limit': 100,
'cutoff_time': 100,
'deterministic': "true"
}
scenario = Scenario(args)
# Build stats
stats = Stats(scenario, output_dir=None, stamp="")
types, bounds = get_types(scenario.cs, scenario.feature_array)
model = RandomForestWithInstances(types=types, bounds=bounds)
constant_pipeline_steps = [
"one_hot_encoder", "imputation", "rescaling", "balancing",
"feature_preprocessor"
]
variable_pipeline_steps = ["classifier"]
rng = np.random.RandomState()
num_params = len(scenario.cs.get_hyperparameters())
acquisition_func = EI(model)
acq_func_wrapper = PCAquisitionFunctionWrapper(
acquisition_func=acquisition_func,
config_space=scenario.cs,
runhistory=runhistory,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps)
runhistory2epm = RunHistory2EPM4Cost(scenario,
num_params,
success_states=[StatusType.SUCCESS])
local_search = LocalSearch(acquisition_function=acq_func_wrapper,
config_space=scenario.cs)
select_configuration = SelectConfigurationsWithMarginalization(
scenario=scenario,
stats=stats,
runhistory=runhistory,
model=model,
acq_optimizer=local_search,
acquisition_func=acq_func_wrapper,
rng=rng,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps,
num_marginalized_configurations_by_random_search=40,
num_configs_for_marginalization=200)
# sample configurations to fill runhistory
sample_configs = config_space.sample_configuration(size=10)
for config in sample_configs:
runhistory.add(config, 1, 1, StatusType.SUCCESS)
# test select_configurations procedure
X, Y = runhistory2epm.transform(runhistory)
challengers = select_configuration.run(
X,
Y,
sample_configs[0],
num_configurations_by_random_search_sorted=100,
num_configurations_by_local_search=10,
random_leaf_size=1)
print(challengers[0])
def build_pc_smbo(self,
tae_runner,
stats,
scenario,
runhistory,
aggregate_func,
acq_func_name,
model_target_names,
logging_directory,
double_intensification=False,
constant_pipeline_steps=None,
variable_pipeline_steps=None,
cached_pipeline_steps=None,
seed=None,
intensification_instances=None,
num_marginalized_configurations_by_random_search=20,
num_configs_for_marginalization=40,
random_splitting_number=5,
random_splitting_enabled=False):
# Build intensifier
rng = np.random.RandomState(seed)
traj_logger = TrajLogger(logging_directory, stats)
intensifier = Intensifier(tae_runner=tae_runner,
stats=stats,
traj_logger=traj_logger,
rng=rng,
cutoff=scenario.cutoff,
deterministic=scenario.deterministic,
run_obj_time=scenario.run_obj == "runtime",
run_limit=scenario.ta_run_limit,
instances=intensification_instances,
maxR=len(intensification_instances))
# Build model
types, bounds = get_types(scenario.cs, scenario.feature_array)
#types = get_types(scenario.cs)
if len(model_target_names) > 1:
# model_target_names = ['cost','time']
model = UncorrelatedMultiObjectiveRandomForestWithInstances(
target_names=model_target_names, bounds=bounds, types=types)
# UncorrelatedMultiObjectiveRandomForestWithInstances(target_names=model_target_names,
# types=types)
elif len(model_target_names) == 1:
model = RandomForestWithInstances(types=types, bounds=bounds)
else:
model = RandomEPM(rng=rng)
# model = RandomForestWithInstances(types=types)
# Build acquisition function, runhistory2epm and local search
num_params = len(scenario.cs.get_hyperparameters())
if acq_func_name in ["ei", "pc-ei"]:
acquisition_func = EI(model)
acq_func_wrapper = PCAquisitionFunctionWrapper(
acquisition_func=acquisition_func,
config_space=scenario.cs,
runhistory=runhistory,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps)
runhistory2epm = RunHistory2EPM4Cost(
scenario, num_params, success_states=[StatusType.SUCCESS])
local_search = LocalSearch(acquisition_function=acq_func_wrapper,
config_space=scenario.cs)
select_configuration = SelectConfigurations(
scenario=scenario,
stats=stats,
runhistory=runhistory,
model=model,
acq_optimizer=local_search,
acquisition_func=acq_func_wrapper,
rng=rng,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps)
elif acq_func_name in ["m-ei", "pc-m-ei"]:
#acquisition_func = MEI(model)
acquisition_func = EI(model)
acq_func_wrapper = PCAquisitionFunctionWrapper(
acquisition_func=acquisition_func,
config_space=scenario.cs,
runhistory=runhistory,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps)
runhistory2epm = RunHistory2EPM4Cost(
scenario, num_params, success_states=[StatusType.SUCCESS])
local_search = LocalSearch(acquisition_function=acq_func_wrapper,
config_space=scenario.cs)
# TODO: num_configs_for_marginalization
select_configuration = SelectConfigurationsWithMarginalization(
scenario=scenario,
stats=stats,
runhistory=runhistory,
model=model,
acq_optimizer=local_search,
acquisition_func=acq_func_wrapper,
rng=rng,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps,
num_marginalized_configurations_by_random_search=
num_marginalized_configurations_by_random_search,
num_configs_for_marginalization=num_configs_for_marginalization
)
elif acq_func_name in ['eips', 'pc-eips']:
acquisition_func = EIPS(model)
acq_func_wrapper = PCAquisitionFunctionWrapper(
acquisition_func=acquisition_func,
config_space=scenario.cs,
runhistory=runhistory,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps)
runhistory2epm = RunHistory2EPM4EIPS(
scenario, num_params, success_states=[StatusType.SUCCESS])
local_search = LocalSearch(acquisition_function=acq_func_wrapper,
config_space=scenario.cs)
select_configuration = SelectConfigurations(
scenario=scenario,
stats=stats,
runhistory=runhistory,
model=model,
acq_optimizer=local_search,
acquisition_func=acq_func_wrapper,
rng=rng,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps)
elif acq_func_name in ["m-eips", "pc-m-eips"]:
acquisition_func = EIPS(model)
acq_func_wrapper = PCAquisitionFunctionWrapper(
acquisition_func=acquisition_func,
config_space=scenario.cs,
runhistory=runhistory,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps)
runhistory2epm = RunHistory2EPM4EIPS(
scenario, num_params, success_states=[StatusType.SUCCESS])
local_search = LocalSearch(acquisition_function=acq_func_wrapper,
config_space=scenario.cs)
# TODO: num_configs_for_marginalization
select_configuration = SelectConfigurationsWithMarginalization(
scenario=scenario,
stats=stats,
runhistory=runhistory,
model=model,
acq_optimizer=local_search,
acquisition_func=acq_func_wrapper,
rng=rng,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps,
num_marginalized_configurations_by_random_search=
num_marginalized_configurations_by_random_search,
num_configs_for_marginalization=num_configs_for_marginalization
)
elif acq_func_name == 'pceips':
acquisition_func = PCEIPS(model)
acq_func_wrapper = PCAquisitionFunctionWrapperWithCachingReduction(
acquisition_func=acquisition_func,
config_space=scenario.cs,
runhistory=runhistory,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps,
cached_pipeline_steps=cached_pipeline_steps)
runhistory2epm = RunHistory2EPM4EIPS(
scenario, num_params, success_states=[StatusType.SUCCESS])
local_search = LocalSearch(acquisition_function=acq_func_wrapper,
config_space=scenario.cs)
if constant_pipeline_steps == None or variable_pipeline_steps == None or cached_pipeline_steps == None:
raise ValueError(
"Constant_pipeline_steps and variable pipeline steps should not be none\
when using PCEIPS")
select_configuration = SelectConfigurations(
scenario=scenario,
stats=stats,
runhistory=runhistory,
model=model,
acq_optimizer=local_search,
acquisition_func=acq_func_wrapper,
rng=rng,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps)
elif acq_func_name == 'pc-m-pceips':
acquisition_func = PCEIPS(model)
acq_func_wrapper = PCAquisitionFunctionWrapperWithCachingReduction(
acquisition_func=acquisition_func,
config_space=scenario.cs,
runhistory=runhistory,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps,
cached_pipeline_steps=cached_pipeline_steps)
runhistory2epm = RunHistory2EPM4EIPS(
scenario, num_params, success_states=[StatusType.SUCCESS])
local_search = LocalSearch(acquisition_function=acq_func_wrapper,
config_space=scenario.cs)
if constant_pipeline_steps == None or variable_pipeline_steps == None or cached_pipeline_steps == None:
raise ValueError(
"Constant_pipeline_steps and variable pipeline steps should not be none\
when using PCEIPS")
select_configuration = SelectConfigurationsWithMarginalization(
scenario=scenario,
stats=stats,
runhistory=runhistory,
model=model,
acq_optimizer=local_search,
acquisition_func=acq_func_wrapper,
rng=rng,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps,
num_marginalized_configurations_by_random_search=
num_marginalized_configurations_by_random_search,
num_configs_for_marginalization=num_configs_for_marginalization
)
elif acq_func_name == "roar":
runhistory2epm = RunHistory2EPM4Cost(
scenario, num_params, success_states=[StatusType.SUCCESS])
select_configuration = SelectConfigurationsRandom(
scenario=scenario)
elif acq_func_name == "pc-roar-mrs":
runhistory2epm = RunHistory2EPM4Cost(
scenario, num_params, success_states=[StatusType.SUCCESS])
select_configuration = SelectConfigurationsMRS(
scenario=scenario,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps,
splitting_number=random_splitting_number,
random_splitting_enabled=random_splitting_enabled)
elif acq_func_name == "pc-roar-sigmoid-rs":
runhistory2epm = RunHistory2EPM4Cost(
scenario, num_params, success_states=[StatusType.SUCCESS])
select_configuration = SelectConfigurationsSigmoidRS(
scenario=scenario,
constant_pipeline_steps=constant_pipeline_steps,
variable_pipeline_steps=variable_pipeline_steps,
fraction=random_splitting_number)
else:
# Not a valid acquisition function
raise ValueError("The provided acquisition function is not valid")
# Build initial design
# initial_design = RandomConfiguration(tae_runner=tae_runner,
# scenario=scenario,
# stats=stats,
# traj_logger=traj_logger,
# rng=rng)
initial_configs = scenario.cs.sample_configuration(size=2)
for config in initial_configs:
config._populate_values()
initial_design = MultiConfigInitialDesign(
tae_runner=tae_runner,
scenario=scenario,
stats=stats,
traj_logger=traj_logger,
runhistory=runhistory,
rng=rng,
configs=initial_configs,
intensifier=intensifier,
aggregate_func=aggregate_func)
# run id
num_run = rng.randint(1234567980)
# Build pc_smbo
if acq_func_name not in ['pc-roar-sigmoid-rs']:
smbo = PCSMBO(scenario=scenario,
stats=stats,
initial_design=initial_design,
runhistory=runhistory,
runhistory2epm=runhistory2epm,
intensifier=intensifier,
aggregate_func=aggregate_func,
num_run=num_run,
model=model,
rng=rng,
select_configuration=select_configuration,
double_intensification=double_intensification)
else:
smbo = PCSMBOSigmoidRandomSearch(
scenario=scenario,
stats=stats,
initial_design=initial_design,
runhistory=runhistory,
runhistory2epm=runhistory2epm,
intensifier=intensifier,
aggregate_func=aggregate_func,
num_run=num_run,
model=model,
rng=rng,
select_configuration=select_configuration)
return smbo
def __init__(
self,
scenario: Scenario,
tae_runner: Optional[Union[Type[BaseRunner], Callable]] = None,
tae_runner_kwargs: Optional[Dict] = None,
runhistory: Optional[Union[Type[RunHistory], RunHistory]] = None,
runhistory_kwargs: Optional[Dict] = None,
intensifier: Optional[Type[AbstractRacer]] = None,
intensifier_kwargs: Optional[Dict] = None,
acquisition_function: Optional[
Type[AbstractAcquisitionFunction]] = None,
acquisition_function_kwargs: Optional[Dict] = None,
integrate_acquisition_function: bool = False,
acquisition_function_optimizer: Optional[
Type[AcquisitionFunctionMaximizer]] = None,
acquisition_function_optimizer_kwargs: Optional[Dict] = None,
model: Optional[Type[AbstractEPM]] = None,
model_kwargs: Optional[Dict] = None,
runhistory2epm: Optional[Type[AbstractRunHistory2EPM]] = None,
runhistory2epm_kwargs: Optional[Dict] = None,
initial_design: Optional[Type[InitialDesign]] = None,
initial_design_kwargs: Optional[Dict] = None,
initial_configurations: Optional[List[Configuration]] = None,
stats: Optional[Stats] = None,
restore_incumbent: Optional[Configuration] = None,
rng: Optional[Union[np.random.RandomState, int]] = None,
smbo_class: Optional[Type[SMBO]] = None,
run_id: Optional[int] = None,
random_configuration_chooser: Optional[
Type[RandomConfigurationChooser]] = None,
random_configuration_chooser_kwargs: Optional[Dict] = None,
dask_client: Optional[dask.distributed.Client] = None,
n_jobs: Optional[int] = 1,
):
"""
Constructor
Parameters
----------
scenario : ~smac.scenario.scenario.Scenario
Scenario object
tae_runner : ~smac.tae.base.BaseRunner or callable
Callable or implementation of
:class:`~smac.tae.base.BaseRunner`. In case a
callable is passed it will be wrapped by
:class:`~smac.tae.execute_func.ExecuteTAFuncDict`.
If not set, it will be initialized with the
:class:`~smac.tae.execute_ta_run_old.ExecuteTARunOld`.
tae_runner_kwargs: Optional[Dict]
arguments passed to constructor of '~tae_runner'
runhistory : RunHistory
runhistory to store all algorithm runs
runhistory_kwargs : Optional[Dict]
arguments passed to constructor of runhistory.
We strongly advise against changing the aggregation function,
since it will break some code assumptions
intensifier : Intensifier
intensification object to issue a racing to decide the current
incumbent
intensifier_kwargs: Optional[Dict]
arguments passed to the constructor of '~intensifier'
acquisition_function : ~smac.optimizer.acquisition.AbstractAcquisitionFunction
Class or object that implements the :class:`~smac.optimizer.acquisition.AbstractAcquisitionFunction`.
Will use :class:`~smac.optimizer.acquisition.EI` or :class:`~smac.optimizer.acquisition.LogEI` if not set.
`~acquisition_function_kwargs` is passed to the class constructor.
acquisition_function_kwargs : Optional[Dict]
dictionary to pass specific arguments to ~acquisition_function
integrate_acquisition_function : bool, default=False
Whether to integrate the acquisition function. Works only with models which can sample their
hyperparameters (i.e. GaussianProcessMCMC).
acquisition_function_optimizer : ~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer
Object that implements the :class:`~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer`.
Will use :class:`smac.optimizer.ei_optimization.LocalAndSortedRandomSearch` if not set.
acquisition_function_optimizer_kwargs: Optional[Dict]
Arguments passed to constructor of '~acquisition_function_optimizer'
model : AbstractEPM
Model that implements train() and predict(). Will use a
:class:`~smac.epm.rf_with_instances.RandomForestWithInstances` if not set.
model_kwargs : Optional[Dict]
Arguments passed to constructor of '~model'
runhistory2epm : ~smac.runhistory.runhistory2epm.RunHistory2EMP
Object that implements the AbstractRunHistory2EPM. If None,
will use :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4Cost`
if objective is cost or
:class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4LogCost`
if objective is runtime.
runhistory2epm_kwargs: Optional[Dict]
Arguments passed to the constructor of '~runhistory2epm'
initial_design : InitialDesign
initial sampling design
initial_design_kwargs: Optional[Dict]
arguments passed to constructor of `~initial_design'
initial_configurations : List[Configuration]
list of initial configurations for initial design --
cannot be used together with initial_design
stats : Stats
optional stats object
rng : np.random.RandomState
Random number generator
restore_incumbent : Configuration
incumbent used if restoring to previous state
smbo_class : ~smac.optimizer.smbo.SMBO
Class implementing the SMBO interface which will be used to
instantiate the optimizer class.
run_id : int (optional)
Run ID will be used as subfolder for output_dir. If no ``run_id`` is given, a random ``run_id`` will be
chosen.
random_configuration_chooser : ~smac.optimizer.random_configuration_chooser.RandomConfigurationChooser
How often to choose a random configuration during the intensification procedure.
random_configuration_chooser_kwargs : Optional[Dict]
arguments of constructor for '~random_configuration_chooser'
dask_client : dask.distributed.Client
User-created dask client, can be used to start a dask cluster and then attach SMAC to it.
n_jobs : int, optional
Number of jobs. If > 1 or -1, this creates a dask client if ``dask_client`` is ``None``. Will
be ignored if ``dask_client`` is not ``None``.
If ``None``, this value will be set to 1, if ``-1``, this will be set to the number of cpu cores.
"""
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
self.scenario = scenario
self.output_dir = ""
if not restore_incumbent:
# restore_incumbent is used by the CLI interface which provides a method for restoring a SMAC run given an
# output directory. This is the default path.
# initial random number generator
run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger)
self.output_dir = create_output_directory(scenario, run_id)
elif scenario.output_dir is not None: # type: ignore[attr-defined] # noqa F821
run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger)
# output-directory is created in CLI when restoring from a
# folder. calling the function again in the facade results in two
# folders being created: run_X and run_X.OLD. if we are
# restoring, the output-folder exists already and we omit creating it,
# but set the self-output_dir to the dir.
# necessary because we want to write traj to new output-dir in CLI.
self.output_dir = cast(str, scenario.output_dir_for_this_run
) # type: ignore[attr-defined] # noqa F821
rng = cast(np.random.RandomState, rng)
if (scenario.deterministic is
True # type: ignore[attr-defined] # noqa F821
and getattr(scenario, 'tuner_timeout', None) is None
and scenario.run_obj ==
'quality' # type: ignore[attr-defined] # noqa F821
):
self.logger.info(
'Optimizing a deterministic scenario for quality without a tuner timeout - will make '
'SMAC deterministic and only evaluate one configuration per iteration!'
)
scenario.intensification_percentage = 1e-10 # type: ignore[attr-defined] # noqa F821
scenario.min_chall = 1 # type: ignore[attr-defined] # noqa F821
scenario.write()
# initialize stats object
if stats:
self.stats = stats
else:
self.stats = Stats(scenario)
if self.scenario.run_obj == "runtime" and not self.scenario.transform_y == "LOG": # type: ignore[attr-defined] # noqa F821
self.logger.warning(
"Runtime as objective automatically activates log(y) transformation"
)
self.scenario.transform_y = "LOG" # type: ignore[attr-defined] # noqa F821
# initialize empty runhistory
runhistory_def_kwargs = {}
if runhistory_kwargs is not None:
runhistory_def_kwargs.update(runhistory_kwargs)
if runhistory is None:
runhistory = RunHistory(**runhistory_def_kwargs)
elif inspect.isclass(runhistory):
runhistory = runhistory(
**runhistory_def_kwargs) # type: ignore[operator] # noqa F821
elif isinstance(runhistory, RunHistory):
pass
else:
raise ValueError(
'runhistory has to be a class or an object of RunHistory')
rand_conf_chooser_kwargs = {'rng': rng}
if random_configuration_chooser_kwargs is not None:
rand_conf_chooser_kwargs.update(
random_configuration_chooser_kwargs)
if random_configuration_chooser is None:
if 'prob' not in rand_conf_chooser_kwargs:
rand_conf_chooser_kwargs[
'prob'] = scenario.rand_prob # type: ignore[attr-defined] # noqa F821
random_configuration_chooser_instance = (
ChooserProb(**rand_conf_chooser_kwargs
) # type: ignore[arg-type] # noqa F821
) # type: RandomConfigurationChooser
elif inspect.isclass(random_configuration_chooser):
random_configuration_chooser_instance = random_configuration_chooser(
**
rand_conf_chooser_kwargs) # type: ignore[arg-type] # noqa F821
elif not isinstance(random_configuration_chooser,
RandomConfigurationChooser):
raise ValueError(
"random_configuration_chooser has to be"
" a class or object of RandomConfigurationChooser")
# reset random number generator in config space to draw different
# random configurations with each seed given to SMAC
scenario.cs.seed(
rng.randint(MAXINT)) # type: ignore[attr-defined] # noqa F821
# initial Trajectory Logger
traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats)
# initial EPM
types, bounds = get_types(
scenario.cs,
scenario.feature_array) # type: ignore[attr-defined] # noqa F821
model_def_kwargs = {
'types': types,
'bounds': bounds,
'instance_features': scenario.feature_array,
'seed': rng.randint(MAXINT),
'pca_components': scenario.PCA_DIM,
}
if model_kwargs is not None:
model_def_kwargs.update(model_kwargs)
if model is None:
for key, value in {
'log_y': scenario.transform_y
in ["LOG",
"LOGS"], # type: ignore[attr-defined] # noqa F821
'num_trees': scenario.
rf_num_trees, # type: ignore[attr-defined] # noqa F821
'do_bootstrapping': scenario.
rf_do_bootstrapping, # type: ignore[attr-defined] # noqa F821
'ratio_features': scenario.
rf_ratio_features, # type: ignore[attr-defined] # noqa F821
'min_samples_split': scenario.
rf_min_samples_split, # type: ignore[attr-defined] # noqa F821
'min_samples_leaf': scenario.
rf_min_samples_leaf, # type: ignore[attr-defined] # noqa F821
'max_depth': scenario.
rf_max_depth, # type: ignore[attr-defined] # noqa F821
}.items():
if key not in model_def_kwargs:
model_def_kwargs[key] = value
model_def_kwargs[
'configspace'] = self.scenario.cs # type: ignore[attr-defined] # noqa F821
model_instance = (
RandomForestWithInstances(
**model_def_kwargs) # type: ignore[arg-type] # noqa F821
) # type: AbstractEPM
elif inspect.isclass(model):
model_def_kwargs[
'configspace'] = self.scenario.cs # type: ignore[attr-defined] # noqa F821
model_instance = model(
**model_def_kwargs) # type: ignore[arg-type] # noqa F821
else:
raise TypeError("Model not recognized: %s" % (type(model)))
# initial acquisition function
acq_def_kwargs = {'model': model_instance}
if acquisition_function_kwargs is not None:
acq_def_kwargs.update(acquisition_function_kwargs)
if acquisition_function is None:
if scenario.transform_y in [
"LOG", "LOGS"
]: # type: ignore[attr-defined] # noqa F821
acquisition_function_instance = (
LogEI(**
acq_def_kwargs) # type: ignore[arg-type] # noqa F821
) # type: AbstractAcquisitionFunction
else:
acquisition_function_instance = EI(
**acq_def_kwargs) # type: ignore[arg-type] # noqa F821
elif inspect.isclass(acquisition_function):
acquisition_function_instance = acquisition_function(
**acq_def_kwargs)
else:
raise TypeError(
"Argument acquisition_function must be None or an object implementing the "
"AbstractAcquisitionFunction, not %s." %
type(acquisition_function))
if integrate_acquisition_function:
acquisition_function_instance = IntegratedAcquisitionFunction(
acquisition_function=acquisition_function_instance,
**acq_def_kwargs)
# initialize optimizer on acquisition function
acq_func_opt_kwargs = {
'acquisition_function': acquisition_function_instance,
'config_space':
scenario.cs, # type: ignore[attr-defined] # noqa F821
'rng': rng,
}
if acquisition_function_optimizer_kwargs is not None:
acq_func_opt_kwargs.update(acquisition_function_optimizer_kwargs)
if acquisition_function_optimizer is None:
for key, value in {
'max_steps': scenario.
sls_max_steps, # type: ignore[attr-defined] # noqa F821
'n_steps_plateau_walk': scenario.
sls_n_steps_plateau_walk, # type: ignore[attr-defined] # noqa F821
}.items():
if key not in acq_func_opt_kwargs:
acq_func_opt_kwargs[key] = value
acquisition_function_optimizer_instance = (
LocalAndSortedRandomSearch(
**
acq_func_opt_kwargs) # type: ignore[arg-type] # noqa F821
) # type: AcquisitionFunctionMaximizer
elif inspect.isclass(acquisition_function_optimizer):
acquisition_function_optimizer_instance = acquisition_function_optimizer(
**acq_func_opt_kwargs) # type: ignore[arg-type] # noqa F821
else:
raise TypeError(
"Argument acquisition_function_optimizer must be None or an object implementing the "
"AcquisitionFunctionMaximizer, but is '%s'" %
type(acquisition_function_optimizer))
# initialize tae_runner
# First case, if tae_runner is None, the target algorithm is a call
# string in the scenario file
tae_def_kwargs = {
'stats': self.stats,
'run_obj': scenario.run_obj,
'par_factor':
scenario.par_factor, # type: ignore[attr-defined] # noqa F821
'cost_for_crash':
scenario.cost_for_crash, # type: ignore[attr-defined] # noqa F821
'abort_on_first_run_crash': scenario.
abort_on_first_run_crash # type: ignore[attr-defined] # noqa F821
}
if tae_runner_kwargs is not None:
tae_def_kwargs.update(tae_runner_kwargs)
if 'ta' not in tae_def_kwargs:
tae_def_kwargs[
'ta'] = scenario.ta # type: ignore[attr-defined] # noqa F821
if tae_runner is None:
tae_def_kwargs[
'ta'] = scenario.ta # type: ignore[attr-defined] # noqa F821
tae_runner_instance = (
ExecuteTARunOld(
**tae_def_kwargs) # type: ignore[arg-type] # noqa F821
) # type: BaseRunner
elif inspect.isclass(tae_runner):
tae_runner_instance = cast(BaseRunner, tae_runner(
**tae_def_kwargs)) # type: ignore[arg-type] # noqa F821
elif callable(tae_runner):
tae_def_kwargs['ta'] = tae_runner
tae_def_kwargs[
'use_pynisher'] = scenario.limit_resources # type: ignore[attr-defined] # noqa F821
tae_runner_instance = ExecuteTAFuncDict(
**tae_def_kwargs) # type: ignore[arg-type] # noqa F821
else:
raise TypeError(
"Argument 'tae_runner' is %s, but must be "
"either None, a callable or an object implementing "
"BaseRunner. Passing 'None' will result in the "
"creation of target algorithm runner based on the "
"call string in the scenario file." % type(tae_runner))
# In case of a parallel run, wrap the single worker in a parallel
# runner
if n_jobs is None or n_jobs == 1:
_n_jobs = 1
elif n_jobs == -1:
_n_jobs = joblib.cpu_count()
elif n_jobs > 0:
_n_jobs = n_jobs
else:
raise ValueError(
'Number of tasks must be positive, None or -1, but is %s' %
str(n_jobs))
if _n_jobs > 1 or dask_client is not None:
tae_runner_instance = DaskParallelRunner(
tae_runner_instance,
n_workers=_n_jobs,
output_directory=self.output_dir,
dask_client=dask_client,
)
# Check that overall objective and tae objective are the same
# TODO: remove these two ignores once the scenario object knows all its attributes!
if tae_runner_instance.run_obj != scenario.run_obj: # type: ignore[union-attr] # noqa F821
raise ValueError(
"Objective for the target algorithm runner and "
"the scenario must be the same, but are '%s' and "
"'%s'" %
(tae_runner_instance.run_obj,
scenario.run_obj)) # type: ignore[union-attr] # noqa F821
# initialize intensification
intensifier_def_kwargs = {
'stats': self.stats,
'traj_logger': traj_logger,
'rng': rng,
'instances':
scenario.train_insts, # type: ignore[attr-defined] # noqa F821
'cutoff':
scenario.cutoff, # type: ignore[attr-defined] # noqa F821
'deterministic':
scenario.deterministic, # type: ignore[attr-defined] # noqa F821
'run_obj_time': scenario.run_obj ==
"runtime", # type: ignore[attr-defined] # noqa F821
'instance_specifics': scenario.
instance_specific, # type: ignore[attr-defined] # noqa F821
'adaptive_capping_slackfactor': scenario.
intens_adaptive_capping_slackfactor, # type: ignore[attr-defined] # noqa F821
'min_chall':
scenario.intens_min_chall # type: ignore[attr-defined] # noqa F821
}
if isinstance(intensifier, Intensifier) \
or (intensifier is not None and inspect.isclass(intensifier) and issubclass(intensifier, Intensifier)):
intensifier_def_kwargs[
'always_race_against'] = scenario.cs.get_default_configuration(
) # type: ignore[attr-defined] # noqa F821
intensifier_def_kwargs[
'use_ta_time_bound'] = scenario.use_ta_time # type: ignore[attr-defined] # noqa F821
intensifier_def_kwargs[
'minR'] = scenario.minR # type: ignore[attr-defined] # noqa F821
intensifier_def_kwargs[
'maxR'] = scenario.maxR # type: ignore[attr-defined] # noqa F821
if intensifier_kwargs is not None:
intensifier_def_kwargs.update(intensifier_kwargs)
if intensifier is None:
intensifier_instance = (
Intensifier(**intensifier_def_kwargs
) # type: ignore[arg-type] # noqa F821
) # type: AbstractRacer
elif inspect.isclass(intensifier):
intensifier_instance = intensifier(
**intensifier_def_kwargs) # type: ignore[arg-type] # noqa F821
else:
raise TypeError(
"Argument intensifier must be None or an object implementing the AbstractRacer, but is '%s'"
% type(intensifier))
# initial design
if initial_design is not None and initial_configurations is not None:
raise ValueError(
"Either use initial_design or initial_configurations; but not both"
)
init_design_def_kwargs = {
'cs': scenario.cs, # type: ignore[attr-defined] # noqa F821
'traj_logger': traj_logger,
'rng': rng,
'ta_run_limit':
scenario.ta_run_limit, # type: ignore[attr-defined] # noqa F821
'configs': initial_configurations,
'n_configs_x_params': 0,
'max_config_fracs': 0.0
}
if initial_design_kwargs is not None:
init_design_def_kwargs.update(initial_design_kwargs)
if initial_configurations is not None:
initial_design_instance = InitialDesign(**init_design_def_kwargs)
elif initial_design is None:
if scenario.initial_incumbent == "DEFAULT": # type: ignore[attr-defined] # noqa F821
init_design_def_kwargs['max_config_fracs'] = 0.0
initial_design_instance = DefaultConfiguration(
**init_design_def_kwargs)
elif scenario.initial_incumbent == "RANDOM": # type: ignore[attr-defined] # noqa F821
init_design_def_kwargs['max_config_fracs'] = 0.0
initial_design_instance = RandomConfigurations(
**init_design_def_kwargs)
elif scenario.initial_incumbent == "LHD": # type: ignore[attr-defined] # noqa F821
initial_design_instance = LHDesign(**init_design_def_kwargs)
elif scenario.initial_incumbent == "FACTORIAL": # type: ignore[attr-defined] # noqa F821
initial_design_instance = FactorialInitialDesign(
**init_design_def_kwargs)
elif scenario.initial_incumbent == "SOBOL": # type: ignore[attr-defined] # noqa F821
initial_design_instance = SobolDesign(**init_design_def_kwargs)
else:
raise ValueError("Don't know what kind of initial_incumbent "
"'%s' is" % scenario.initial_incumbent
) # type: ignore[attr-defined] # noqa F821
elif inspect.isclass(initial_design):
initial_design_instance = initial_design(**init_design_def_kwargs)
else:
raise TypeError(
"Argument initial_design must be None or an object implementing the InitialDesign, but is '%s'"
% type(initial_design))
# if we log the performance data,
# the RFRImputator will already get
# log transform data from the runhistory
if scenario.transform_y in [
"LOG", "LOGS"
]: # type: ignore[attr-defined] # noqa F821
cutoff = np.log(np.nanmin([
np.inf, np.float_(scenario.cutoff)
])) # type: ignore[attr-defined] # noqa F821
threshold = cutoff + np.log(
scenario.par_factor) # type: ignore[attr-defined] # noqa F821
else:
cutoff = np.nanmin([np.inf, np.float_(scenario.cutoff)
]) # type: ignore[attr-defined] # noqa F821
threshold = cutoff * scenario.par_factor # type: ignore[attr-defined] # noqa F821
num_params = len(scenario.cs.get_hyperparameters()
) # type: ignore[attr-defined] # noqa F821
imputor = RFRImputator(rng=rng,
cutoff=cutoff,
threshold=threshold,
model=model_instance,
change_threshold=0.01,
max_iter=2)
r2e_def_kwargs = {
'scenario': scenario,
'num_params': num_params,
'success_states': [
StatusType.SUCCESS,
],
'impute_censored_data': True,
'impute_state': [
StatusType.CAPPED,
],
'imputor': imputor,
'scale_perc': 5
}
if scenario.run_obj == 'quality':
r2e_def_kwargs.update({
'success_states':
[StatusType.SUCCESS, StatusType.CRASHED, StatusType.MEMOUT],
'impute_censored_data':
False,
'impute_state':
None,
})
if isinstance(
intensifier_instance,
(SuccessiveHalving, Hyperband)) and scenario.run_obj == "quality":
r2e_def_kwargs.update({
'success_states': [
StatusType.SUCCESS,
StatusType.CRASHED,
StatusType.MEMOUT,
StatusType.DONOTADVANCE,
],
'consider_for_higher_budgets_state': [
StatusType.DONOTADVANCE,
StatusType.TIMEOUT,
StatusType.CRASHED,
StatusType.MEMOUT,
],
})
if runhistory2epm_kwargs is not None:
r2e_def_kwargs.update(runhistory2epm_kwargs)
if runhistory2epm is None:
if scenario.run_obj == 'runtime':
rh2epm = (
RunHistory2EPM4LogCost(
**r2e_def_kwargs) # type: ignore[arg-type] # noqa F821
) # type: AbstractRunHistory2EPM
elif scenario.run_obj == 'quality':
if scenario.transform_y == "NONE": # type: ignore[attr-defined] # noqa F821
rh2epm = RunHistory2EPM4Cost(
**r2e_def_kwargs) # type: ignore[arg-type] # noqa F821
elif scenario.transform_y == "LOG": # type: ignore[attr-defined] # noqa F821
rh2epm = RunHistory2EPM4LogCost(
**r2e_def_kwargs) # type: ignore[arg-type] # noqa F821
elif scenario.transform_y == "LOGS": # type: ignore[attr-defined] # noqa F821
rh2epm = RunHistory2EPM4LogScaledCost(
**r2e_def_kwargs) # type: ignore[arg-type] # noqa F821
elif scenario.transform_y == "INVS": # type: ignore[attr-defined] # noqa F821
rh2epm = RunHistory2EPM4InvScaledCost(
**r2e_def_kwargs) # type: ignore[arg-type] # noqa F821
else:
raise ValueError('Unknown run objective: %s. Should be either '
'quality or runtime.' % self.scenario.run_obj)
elif inspect.isclass(runhistory2epm):
rh2epm = runhistory2epm(
**r2e_def_kwargs) # type: ignore[arg-type] # noqa F821
else:
raise TypeError(
"Argument runhistory2epm must be None or an object implementing the RunHistory2EPM, but is '%s'"
% type(runhistory2epm))
smbo_args = {
'scenario': scenario,
'stats': self.stats,
'initial_design': initial_design_instance,
'runhistory': runhistory,
'runhistory2epm': rh2epm,
'intensifier': intensifier_instance,
'num_run': run_id,
'model': model_instance,
'acq_optimizer': acquisition_function_optimizer_instance,
'acquisition_func': acquisition_function_instance,
'rng': rng,
'restore_incumbent': restore_incumbent,
'random_configuration_chooser':
random_configuration_chooser_instance,
'tae_runner': tae_runner_instance,
} # type: Dict[str, Any]
if smbo_class is None:
self.solver = SMBO(**
smbo_args) # type: ignore[arg-type] # noqa F821
else:
self.solver = smbo_class(
**smbo_args) # type: ignore[arg-type] # noqa F821
def __init__(self,
scenario: Scenario,
tae_runner: ExecuteTARun = None,
runhistory: RunHistory = None,
intensifier: Intensifier = None,
initial_design: InitialDesign = None,
initial_configurations: typing.List[Configuration] = None,
stats: Stats = None,
rng: np.random.RandomState = None,
run_id: int = 1):
"""
Constructor
Parameters
----------
scenario: smac.scenario.scenario.Scenario
Scenario object
tae_runner: smac.tae.execute_ta_run.ExecuteTARun or callable
Callable or implementation of
:class:`~smac.tae.execute_ta_run.ExecuteTARun`. In case a
callable is passed it will be wrapped by
:class:`~smac.tae.execute_func.ExecuteTAFuncDict`.
If not set, it will be initialized with the
:class:`~smac.tae.execute_ta_run_old.ExecuteTARunOld`.
runhistory: RunHistory
Runhistory to store all algorithm runs
intensifier: Intensifier
intensification object to issue a racing to decide the current incumbent
initial_design: InitialDesign
initial sampling design
initial_configurations: typing.List[Configuration]
list of initial configurations for initial design --
cannot be used together with initial_design
stats: Stats
optional stats object
rng: np.random.RandomState
Random number generator
run_id: int, (default: 1)
Run ID will be used as subfolder for output_dir.
"""
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
# initial random number generator
_, rng = get_rng(rng=rng, logger=self.logger)
# initial conversion of runhistory into EPM data
# since ROAR does not really use it the converted data
# we simply use a cheap RunHistory2EPM here
num_params = len(scenario.cs.get_hyperparameters())
runhistory2epm = RunHistory2EPM4Cost(scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=False,
impute_state=None)
aggregate_func = average_cost
# initialize empty runhistory
if runhistory is None:
runhistory = RunHistory(aggregate_func=aggregate_func)
# inject aggr_func if necessary
if runhistory.aggregate_func is None:
runhistory.aggregate_func = aggregate_func
self.stats = Stats(scenario)
rs = RandomSearch(
acquisition_function=None,
config_space=scenario.cs,
)
# use SMAC facade
super().__init__(
scenario=scenario,
tae_runner=tae_runner,
runhistory=runhistory,
intensifier=intensifier,
runhistory2epm=runhistory2epm,
initial_design=initial_design,
initial_configurations=initial_configurations,
stats=stats,
rng=rng,
run_id=run_id,
acquisition_function_optimizer=rs,
)
def run(self):
"""
Implementation of the forward selection loop.
Uses SMACs EPM (RF) wrt the feature space to minimize the OOB error.
Returns
-------
feature_importance: OrderedDict
dict_keys (first key -> most important) -> OOB error
"""
parameters = [p.name for p in self.scenario.cs.get_hyperparameters()]
self.logger.debug("Parameters: %s", parameters)
rh2epm = RunHistory2EPM4Cost(scenario=self.scenario,
num_params=len(parameters),
success_states=[
StatusType.SUCCESS, StatusType.CAPPED,
StatusType.CRASHED
],
impute_censored_data=False,
impute_state=None)
X, y = rh2epm.transform(self.rh)
# reduce sample size to speedup computation
if X.shape[0] > self.MAX_SAMPLES:
idx = np.random.choice(X.shape[0],
size=self.MAX_SAMPLES,
replace=False)
X = X[idx, :]
y = y[idx]
self.logger.debug(
"Shape of X: %s, of y: %s, #parameters: %s, #feats: %s", X.shape,
y.shape, len(parameters), len(self.scenario.feature_names))
names = copy.deepcopy(self.scenario.feature_names)
self.logger.debug("Features: %s", names)
used = list(range(0, len(parameters)))
feat_ids = {f: i for i, f in enumerate(names, len(used))}
ids_feat = {i: f for f, i in feat_ids.items()}
self.logger.debug("Used: %s", used)
evaluated_feature_importance = OrderedDict()
types, bounds = get_types(self.scenario.cs,
self.scenario.feature_array)
last_error = np.inf
for _round in range(self.to_evaluate): # Main Loop
errors = []
for f in names:
i = feat_ids[f]
self.logger.debug('Evaluating %s', f)
used.append(i)
self.logger.debug(
'Used features: %s',
str([ids_feat[j] for j in used[len(parameters):]]))
start = time.time()
self._refit_model(types[sorted(used)], bounds, X[:,
sorted(used)],
y) # refit the model every round
errors.append(self.model.rf.out_of_bag_error())
used.pop()
self.logger.debug('Refitted RF (sec %.2f; error: %.4f)' %
(time.time() - start, errors[-1]))
else:
self.logger.debug('Evaluating None')
start = time.time()
self._refit_model(types[sorted(used)], bounds, X[:,
sorted(used)],
y) # refit the model every round
errors.append(self.model.rf.out_of_bag_error())
self.logger.debug('Refitted RF (sec %.2f; error: %.4f)' %
(time.time() - start, errors[-1]))
if _round == 0:
evaluated_feature_importance['None'] = errors[-1]
best_idx = np.argmin(errors)
lowest_error = errors[best_idx]
if best_idx == len(errors) - 1:
self.logger.info('Best thing to do is add nothing')
best_feature = 'None'
# evaluated_feature_importance[best_feature] = lowest_error
break
elif lowest_error >= last_error:
break
else:
last_error = lowest_error
best_feature = names.pop(best_idx)
used.append(feat_ids[best_feature])
self.logger.debug('%s: %.4f' % (best_feature, lowest_error))
evaluated_feature_importance[best_feature] = lowest_error
self.logger.debug(evaluated_feature_importance)
self.evaluated_feature_importance = evaluated_feature_importance
return evaluated_feature_importance
def plot_cost_over_time(self,
rh,
traj,
output="performance_over_time.png",
validator=None):
""" Plot performance over time, using all trajectory entries
with max_time = wallclock_limit or (if inf) the highest
recorded time
Parameters
----------
rh: RunHistory
runhistory to use
traj: List
trajectory to take times/incumbents from
output: str
path to output-png
epm: RandomForestWithInstances
emperical performance model (expecting trained on all runs)
"""
self.logger.debug("Estimating costs over time for best run.")
validator.traj = traj # set trajectory
time, configs = [], []
for entry in traj:
time.append(entry["wallclock_time"])
configs.append(entry["incumbent"])
self.logger.debug("Using %d samples (%d distinct) from trajectory.",
len(time), len(set(configs)))
if validator.epm: # not log as validator epm is trained on cost, not log cost
epm = validator.epm
else:
self.logger.debug(
"No EPM passed! Training new one from runhistory.")
# Train random forest and transform training data (from given rh)
# Not using validator because we want to plot uncertainties
rh2epm = RunHistory2EPM4Cost(num_params=len(
self.scenario.cs.get_hyperparameters()),
scenario=self.scenario)
X, y = rh2epm.transform(rh)
self.logger.debug("Training model with data of shape X: %s, y:%s",
str(X.shape), str(y.shape))
types, bounds = get_types(self.scenario.cs,
self.scenario.feature_array)
epm = RandomForestWithInstances(
types=types,
bounds=bounds,
instance_features=self.scenario.feature_array,
#seed=self.rng.randint(MAXINT),
ratio_features=1.0)
epm.train(X, y)
## not necessary right now since the EPM only knows the features
## of the training instances
# use only training instances
#=======================================================================
# if self.scenario.feature_dict:
# feat_array = []
# for inst in self.scenario.train_insts:
# feat_array.append(self.scenario.feature_dict[inst])
# backup_features_epm = epm.instance_features
# epm.instance_features = np.array(feat_array)
#=======================================================================
# predict performance for all configurations in trajectory
config_array = convert_configurations_to_array(configs)
mean, var = epm.predict_marginalized_over_instances(config_array)
#=======================================================================
# # restore feature array in epm
# if self.scenario.feature_dict:
# epm.instance_features = backup_features_epm
#=======================================================================
mean = mean[:, 0]
var = var[:, 0]
uncertainty_upper = mean + np.sqrt(var)
uncertainty_lower = mean - np.sqrt(var)
if self.scenario.run_obj == 'runtime': # We have to clip at 0 as we want to put y on the logscale
uncertainty_lower[uncertainty_lower < 0] = 0
uncertainty_upper[uncertainty_upper < 0] = 0
# plot
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_ylabel('performance')
ax.set_xlabel('time [sec]')
ax.plot(time, mean, 'r-', label="estimated performance")
ax.fill_between(time,
uncertainty_upper,
uncertainty_lower,
alpha=0.8,
label="standard deviation")
ax.set_xscale("log", nonposx='clip')
if self.scenario.run_obj == 'runtime':
ax.set_yscale('log')
# ax.set_ylim(min(mean)*0.8, max(mean)*1.2)
# start after 1% of the configuration budget
ax.set_xlim(min(time) + (max(time) - min(time)) * 0.01, max(time))
ax.legend()
plt.tight_layout()
fig.savefig(output)
plt.close(fig)
def __init__(self, api_config, config_space, parallel_setting="LS"):
super(SMAC4EPMOpimizer, self).__init__(api_config)
self.cs = config_space
self.num_hps = len(self.cs.get_hyperparameters())
if parallel_setting not in ["CL_min", "CL_max", "CL_mean", "KB", "LS"]:
raise ValueError(
"parallel_setting can only be one of the following: "
"CL_min, CL_max, CL_mean, KB, LS")
self.parallel_setting = parallel_setting
rng = np.random.RandomState(seed=0)
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alt. to runtime)
"runcount-limit": 128,
"cs": self.cs, # configuration space
"deterministic": True,
"limit_resources": False,
})
self.stats = Stats(scenario)
# traj = TrajLogger(output_dir=None, stats=self.stats)
self.runhistory = RunHistory()
r2e_def_kwargs = {
"scenario": scenario,
"num_params": self.num_hps,
"success_states": [
StatusType.SUCCESS,
],
"impute_censored_data": False,
"scale_perc": 5,
}
self.random_chooser = ChooserProb(rng=rng, prob=0.0)
types, bounds = get_types(self.cs, instance_features=None)
model_kwargs = {
"configspace": self.cs,
"types": types,
"bounds": bounds,
"seed": rng.randint(MAXINT),
}
models = []
cov_amp = ConstantKernel(
2.0,
constant_value_bounds=(np.exp(-10), np.exp(2)),
prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng),
)
cont_dims = np.array(np.where(np.array(types) == 0)[0], dtype=np.int)
cat_dims = np.where(np.array(types) != 0)[0]
if len(cont_dims) > 0:
exp_kernel = Matern(
np.ones([len(cont_dims)]),
[(np.exp(-6.754111155189306), np.exp(0.0858637988771976))
for _ in range(len(cont_dims))],
nu=2.5,
operate_on=cont_dims,
)
if len(cat_dims) > 0:
ham_kernel = HammingKernel(
np.ones([len(cat_dims)]),
[(np.exp(-6.754111155189306), np.exp(0.0858637988771976))
for _ in range(len(cat_dims))],
operate_on=cat_dims,
)
assert len(cont_dims) + len(cat_dims) == len(
scenario.cs.get_hyperparameters())
noise_kernel = WhiteKernel(
noise_level=1e-8,
noise_level_bounds=(np.exp(-25), np.exp(2)),
prior=HorseshoePrior(scale=0.1, rng=rng),
)
if len(cont_dims) > 0 and len(cat_dims) > 0:
# both
kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel
elif len(cont_dims) > 0 and len(cat_dims) == 0:
# only cont
kernel = cov_amp * exp_kernel + noise_kernel
elif len(cont_dims) == 0 and len(cat_dims) > 0:
# only cont
kernel = cov_amp * ham_kernel + noise_kernel
else:
raise ValueError()
gp_kwargs = {"kernel": kernel}
rf_kwargs = {}
rf_kwargs["num_trees"] = model_kwargs.get("num_trees", 10)
rf_kwargs["do_bootstrapping"] = model_kwargs.get(
"do_bootstrapping", True)
rf_kwargs["ratio_features"] = model_kwargs.get("ratio_features", 1.0)
rf_kwargs["min_samples_split"] = model_kwargs.get(
"min_samples_split", 2)
rf_kwargs["min_samples_leaf"] = model_kwargs.get("min_samples_leaf", 1)
rf_kwargs["log_y"] = model_kwargs.get("log_y", True)
rf_log = RandomForestWithInstances(**model_kwargs, **rf_kwargs)
rf_kwargs = copy.deepcopy(rf_kwargs)
rf_kwargs["log_y"] = False
rf_no_log = RandomForestWithInstances(**model_kwargs, **rf_kwargs)
rh2epm_cost = RunHistory2EPM4Cost(**r2e_def_kwargs)
rh2epm_log_cost = RunHistory2EPM4LogScaledCost(**r2e_def_kwargs)
rh2epm_copula = RunHistory2EPM4GaussianCopulaCorrect(**r2e_def_kwargs)
self.combinations = []
# 2 models * 4 acquisition functions
acq_funcs = [EI, PI, LogEI, LCB]
acq_func_instances = []
# acq_func_maximizer_instances = []
n_sls_iterations = {
1: 10,
2: 10,
3: 10,
4: 10,
5: 10,
6: 10,
7: 8,
8: 6,
}.get(len(self.cs.get_hyperparameters()), 5)
acq_func_maximizer_kwargs = {
"config_space": self.cs,
"rng": rng,
"max_steps": 5,
"n_steps_plateau_walk": 5,
"n_sls_iterations": n_sls_iterations,
}
self.idx_ei = 0
self.num_models = len(models)
self.num_acq_funcs = len(acq_funcs)
no_transform_gp = GaussianProcess(**copy.deepcopy(model_kwargs),
**copy.deepcopy(gp_kwargs))
ei = EI(model=no_transform_gp)
acq_func_maximizer_kwargs["acquisition_function"] = ei
ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((no_transform_gp, ei, ei_opt, rh2epm_cost))
pi = PI(model=no_transform_gp)
acq_func_maximizer_kwargs["acquisition_function"] = pi
pi_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((no_transform_gp, pi, pi_opt, rh2epm_cost))
lcb = LCB(model=no_transform_gp)
acq_func_maximizer_kwargs["acquisition_function"] = lcb
lcb_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((no_transform_gp, lcb, lcb_opt, rh2epm_cost))
gp = GaussianProcess(**copy.deepcopy(model_kwargs),
**copy.deepcopy(gp_kwargs))
ei = EI(model=gp)
acq_func_maximizer_kwargs["acquisition_function"] = ei
ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((gp, ei, ei_opt, rh2epm_copula))
gp = GaussianProcess(**copy.deepcopy(model_kwargs),
**copy.deepcopy(gp_kwargs))
ei = LogEI(model=gp)
acq_func_maximizer_kwargs["acquisition_function"] = ei
ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((gp, ei, ei_opt, rh2epm_log_cost))
ei = EI(model=rf_no_log)
acq_func_maximizer_kwargs["acquisition_function"] = ei
ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((rf_no_log, ei, ei_opt, rh2epm_cost))
ei = LogEI(model=rf_log)
acq_func_maximizer_kwargs["acquisition_function"] = ei
ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((rf_log, ei, ei_opt, rh2epm_log_cost))
ei = EI(model=rf_no_log)
acq_func_maximizer_kwargs["acquisition_function"] = ei
ei_opt = LocalAndSortedRandomSearch(**acq_func_maximizer_kwargs)
self.combinations.append((rf_no_log, ei, ei_opt, rh2epm_copula))
self.num_acq_instances = len(acq_func_instances)
self.best_observation = np.inf
self.next_evaluations = []
def __init__(
self,
scenario: Scenario,
tae_runner: typing.Optional[typing.Union[ExecuteTARun,
typing.Callable]] = None,
runhistory: typing.Optional[RunHistory] = None,
intensifier: typing.Optional[Intensifier] = None,
acquisition_function: typing.
Optional[AbstractAcquisitionFunction] = None,
acquisition_function_optimizer: typing.
Optional[AcquisitionFunctionMaximizer] = None,
model: typing.Optional[AbstractEPM] = None,
runhistory2epm: typing.Optional[AbstractRunHistory2EPM] = None,
initial_design: typing.Optional[InitialDesign] = None,
initial_configurations: typing.Optional[
typing.List[Configuration]] = None,
stats: typing.Optional[Stats] = None,
restore_incumbent: typing.Optional[Configuration] = None,
rng: typing.Optional[typing.Union[np.random.RandomState, int]] = None,
smbo_class: typing.Optional[SMBO] = None,
run_id: typing.Optional[int] = None,
random_configuration_chooser: typing.
Optional[RandomConfigurationChooser] = None):
"""
Constructor
Parameters
----------
scenario : ~smac.scenario.scenario.Scenario
Scenario object
tae_runner : ~smac.tae.execute_ta_run.ExecuteTARun or callable
Callable or implementation of
:class:`~smac.tae.execute_ta_run.ExecuteTARun`. In case a
callable is passed it will be wrapped by
:class:`~smac.tae.execute_func.ExecuteTAFuncDict`.
If not set, it will be initialized with the
:class:`~smac.tae.execute_ta_run_old.ExecuteTARunOld`.
runhistory : RunHistory
runhistory to store all algorithm runs
intensifier : Intensifier
intensification object to issue a racing to decide the current
incumbent
acquisition_function : ~smac.optimizer.acquisition.AbstractAcquisitionFunction
Object that implements the :class:`~smac.optimizer.acquisition.AbstractAcquisitionFunction`.
Will use :class:`~smac.optimizer.acquisition.EI` if not set.
acquisition_function_optimizer : ~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer
Object that implements the :class:`~smac.optimizer.ei_optimization.AcquisitionFunctionMaximizer`.
Will use :class:`smac.optimizer.ei_optimization.InterleavedLocalAndRandomSearch` if not set.
model : AbstractEPM
Model that implements train() and predict(). Will use a
:class:`~smac.epm.rf_with_instances.RandomForestWithInstances` if not set.
runhistory2epm : ~smac.runhistory.runhistory2epm.RunHistory2EMP
Object that implements the AbstractRunHistory2EPM. If None,
will use :class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4Cost`
if objective is cost or
:class:`~smac.runhistory.runhistory2epm.RunHistory2EPM4LogCost`
if objective is runtime.
initial_design : InitialDesign
initial sampling design
initial_configurations : typing.List[Configuration]
list of initial configurations for initial design --
cannot be used together with initial_design
stats : Stats
optional stats object
rng : np.random.RandomState
Random number generator
restore_incumbent : Configuration
incumbent used if restoring to previous state
smbo_class : ~smac.optimizer.smbo.SMBO
Class implementing the SMBO interface which will be used to
instantiate the optimizer class.
run_id : int (optional)
Run ID will be used as subfolder for output_dir. If no ``run_id`` is given, a random ``run_id`` will be
chosen.
random_configuration_chooser : ~smac.optimizer.random_configuration_chooser.RandomConfigurationChooser
How often to choose a random configuration during the intensification procedure.
"""
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
aggregate_func = average_cost
self.scenario = scenario
self.output_dir = ""
if not restore_incumbent:
# restore_incumbent is used by the CLI interface which provides a method for restoring a SMAC run given an
# output directory. This is the default path.
# initial random number generator
run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger)
self.output_dir = create_output_directory(scenario, run_id)
elif scenario.output_dir is not None:
run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger)
# output-directory is created in CLI when restoring from a
# folder. calling the function again in the facade results in two
# folders being created: run_X and run_X.OLD. if we are
# restoring, the output-folder exists already and we omit creating it,
# but set the self-output_dir to the dir.
# necessary because we want to write traj to new output-dir in CLI.
self.output_dir = scenario.output_dir_for_this_run
if (scenario.deterministic is True
and getattr(scenario, 'tuner_timeout', None) is None
and scenario.run_obj == 'quality'):
self.logger.info('Optimizing a deterministic scenario for '
'quality without a tuner timeout - will make '
'SMAC deterministic!')
scenario.intensification_percentage = 1e-10
scenario.write()
# initialize stats object
if stats:
self.stats = stats
else:
self.stats = Stats(scenario)
if self.scenario.run_obj == "runtime" and not self.scenario.transform_y == "LOG":
self.logger.warn(
"Runtime as objective automatically activates log(y) transformation"
)
self.scenario.transform_y = "LOG"
# initialize empty runhistory
if runhistory is None:
runhistory = RunHistory(aggregate_func=aggregate_func)
# inject aggr_func if necessary
if runhistory.aggregate_func is None:
runhistory.aggregate_func = aggregate_func
if not random_configuration_chooser:
random_configuration_chooser = ChooserProb(prob=scenario.rand_prob,
rng=rng)
# reset random number generator in config space to draw different
# random configurations with each seed given to SMAC
scenario.cs.seed(rng.randint(MAXINT))
# initial Trajectory Logger
traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats)
# initial EPM
types, bounds = get_types(scenario.cs, scenario.feature_array)
if model is None:
model = RandomForestWithInstances(
types=types,
bounds=bounds,
instance_features=scenario.feature_array,
seed=rng.randint(MAXINT),
pca_components=scenario.PCA_DIM,
log_y=scenario.transform_y in ["LOG", "LOGS"],
num_trees=scenario.rf_num_trees,
do_bootstrapping=scenario.rf_do_bootstrapping,
ratio_features=scenario.rf_ratio_features,
min_samples_split=scenario.rf_min_samples_split,
min_samples_leaf=scenario.rf_min_samples_leaf,
max_depth=scenario.rf_max_depth)
# initial acquisition function
if acquisition_function is None:
if scenario.transform_y in ["LOG", "LOGS"]:
acquisition_function = LogEI(model=model)
else:
acquisition_function = EI(model=model)
# inject model if necessary
if acquisition_function.model is None:
acquisition_function.model = model
# initialize optimizer on acquisition function
if acquisition_function_optimizer is None:
acquisition_function_optimizer = InterleavedLocalAndRandomSearch(
acquisition_function=acquisition_function,
config_space=scenario.cs,
rng=np.random.RandomState(seed=rng.randint(MAXINT)),
max_steps=scenario.sls_max_steps,
n_steps_plateau_walk=scenario.sls_n_steps_plateau_walk)
elif not isinstance(
acquisition_function_optimizer,
AcquisitionFunctionMaximizer,
):
raise ValueError(
"Argument 'acquisition_function_optimizer' must be of type"
"'AcquisitionFunctionMaximizer', but is '%s'" %
type(acquisition_function_optimizer))
# initialize tae_runner
# First case, if tae_runner is None, the target algorithm is a call
# string in the scenario file
if tae_runner is None:
tae_runner = ExecuteTARunOld(
ta=scenario.ta,
stats=self.stats,
run_obj=scenario.run_obj,
runhistory=runhistory,
par_factor=scenario.par_factor,
cost_for_crash=scenario.cost_for_crash,
abort_on_first_run_crash=scenario.abort_on_first_run_crash)
# Second case, the tae_runner is a function to be optimized
elif callable(tae_runner):
tae_runner = ExecuteTAFuncDict(
ta=tae_runner,
stats=self.stats,
run_obj=scenario.run_obj,
memory_limit=scenario.memory_limit,
runhistory=runhistory,
par_factor=scenario.par_factor,
cost_for_crash=scenario.cost_for_crash,
abort_on_first_run_crash=scenario.abort_on_first_run_crash)
# Third case, if it is an ExecuteTaRun we can simply use the
# instance. Otherwise, the next check raises an exception
elif not isinstance(tae_runner, ExecuteTARun):
raise TypeError("Argument 'tae_runner' is %s, but must be "
"either a callable or an instance of "
"ExecuteTaRun. Passing 'None' will result in the "
"creation of target algorithm runner based on the "
"call string in the scenario file." %
type(tae_runner))
# Check that overall objective and tae objective are the same
if tae_runner.run_obj != scenario.run_obj:
raise ValueError("Objective for the target algorithm runner and "
"the scenario must be the same, but are '%s' and "
"'%s'" % (tae_runner.run_obj, scenario.run_obj))
# inject stats if necessary
if tae_runner.stats is None:
tae_runner.stats = self.stats
# inject runhistory if necessary
if tae_runner.runhistory is None:
tae_runner.runhistory = runhistory
# inject cost_for_crash
if tae_runner.crash_cost != scenario.cost_for_crash:
tae_runner.crash_cost = scenario.cost_for_crash
# initialize intensification
if intensifier is None:
intensifier = Intensifier(
tae_runner=tae_runner,
stats=self.stats,
traj_logger=traj_logger,
rng=rng,
instances=scenario.train_insts,
cutoff=scenario.cutoff,
deterministic=scenario.deterministic,
run_obj_time=scenario.run_obj == "runtime",
always_race_against=scenario.cs.get_default_configuration()
if scenario.always_race_default else None,
use_ta_time_bound=scenario.use_ta_time,
instance_specifics=scenario.instance_specific,
minR=scenario.minR,
maxR=scenario.maxR,
adaptive_capping_slackfactor=scenario.
intens_adaptive_capping_slackfactor,
min_chall=scenario.intens_min_chall)
# inject deps if necessary
if intensifier.tae_runner is None:
intensifier.tae_runner = tae_runner
if intensifier.stats is None:
intensifier.stats = self.stats
if intensifier.traj_logger is None:
intensifier.traj_logger = traj_logger
# initial design
if initial_design is not None and initial_configurations is not None:
raise ValueError(
"Either use initial_design or initial_configurations; but not both"
)
if initial_configurations is not None:
initial_design = MultiConfigInitialDesign(
tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
runhistory=runhistory,
rng=rng,
configs=initial_configurations,
intensifier=intensifier,
aggregate_func=aggregate_func)
elif initial_design is None:
if scenario.initial_incumbent == "DEFAULT":
initial_design = DefaultConfiguration(tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
elif scenario.initial_incumbent == "RANDOM":
initial_design = RandomConfiguration(tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
elif scenario.initial_incumbent == "LHD":
initial_design = LHDesign(runhistory=runhistory,
intensifier=intensifier,
aggregate_func=aggregate_func,
tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
elif scenario.initial_incumbent == "FACTORIAL":
initial_design = FactorialInitialDesign(
runhistory=runhistory,
intensifier=intensifier,
aggregate_func=aggregate_func,
tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
elif scenario.initial_incumbent == "SOBOL":
initial_design = SobolDesign(runhistory=runhistory,
intensifier=intensifier,
aggregate_func=aggregate_func,
tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
else:
raise ValueError("Don't know what kind of initial_incumbent "
"'%s' is" % scenario.initial_incumbent)
# inject deps if necessary
if initial_design.tae_runner is None:
initial_design.tae_runner = tae_runner
if initial_design.scenario is None:
initial_design.scenario = scenario
if initial_design.stats is None:
initial_design.stats = self.stats
if initial_design.traj_logger is None:
initial_design.traj_logger = traj_logger
# initial conversion of runhistory into EPM data
if runhistory2epm is None:
num_params = len(scenario.cs.get_hyperparameters())
if scenario.run_obj == 'runtime':
# if we log the performance data,
# the RFRImputator will already get
# log transform data from the runhistory
cutoff = np.log(scenario.cutoff)
threshold = np.log(scenario.cutoff * scenario.par_factor)
imputor = RFRImputator(rng=rng,
cutoff=cutoff,
threshold=threshold,
model=model,
change_threshold=0.01,
max_iter=2)
runhistory2epm = RunHistory2EPM4LogCost(
scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=True,
impute_state=[
StatusType.CAPPED,
],
imputor=imputor)
elif scenario.run_obj == 'quality':
if scenario.transform_y == "NONE":
runhistory2epm = RunHistory2EPM4Cost(
scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS, StatusType.CRASHED
],
impute_censored_data=False,
impute_state=None)
elif scenario.transform_y == "LOG":
runhistory2epm = RunHistory2EPM4LogCost(
scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS, StatusType.CRASHED
],
impute_censored_data=False,
impute_state=None)
elif scenario.transform_y == "LOGS":
runhistory2epm = RunHistory2EPM4LogScaledCost(
scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS, StatusType.CRASHED
],
impute_censored_data=False,
impute_state=None)
elif scenario.transform_y == "INVS":
runhistory2epm = RunHistory2EPM4InvScaledCost(
scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS, StatusType.CRASHED
],
impute_censored_data=False,
impute_state=None)
else:
raise ValueError('Unknown run objective: %s. Should be either '
'quality or runtime.' % self.scenario.run_obj)
# inject scenario if necessary:
if runhistory2epm.scenario is None:
runhistory2epm.scenario = scenario
smbo_args = {
'scenario': scenario,
'stats': self.stats,
'initial_design': initial_design,
'runhistory': runhistory,
'runhistory2epm': runhistory2epm,
'intensifier': intensifier,
'aggregate_func': aggregate_func,
'num_run': run_id,
'model': model,
'acq_optimizer': acquisition_function_optimizer,
'acquisition_func': acquisition_function,
'rng': rng,
'restore_incumbent': restore_incumbent,
'random_configuration_chooser': random_configuration_chooser
}
if smbo_class is None:
self.solver = SMBO(**smbo_args)
else:
self.solver = smbo_class(**smbo_args)
def __init__(
self,
scenario: Scenario,
# TODO: once we drop python3.4 add type hint
# typing.Union[ExecuteTARun, callable]
tae_runner=None,
runhistory: RunHistory = None,
intensifier: Intensifier = None,
acquisition_function: AbstractAcquisitionFunction = None,
model: AbstractEPM = None,
runhistory2epm: AbstractRunHistory2EPM = None,
initial_design: InitialDesign = None,
initial_configurations: typing.List[Configuration] = None,
stats: Stats = None,
rng: np.random.RandomState = None,
run_id: int = 1):
"""Constructor"""
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
aggregate_func = average_cost
self.runhistory = None
self.trajectory = None
# initialize stats object
if stats:
self.stats = stats
else:
self.stats = Stats(scenario)
self.output_dir = create_output_directory(scenario, run_id)
scenario.write()
# initialize empty runhistory
if runhistory is None:
runhistory = RunHistory(aggregate_func=aggregate_func)
# inject aggr_func if necessary
if runhistory.aggregate_func is None:
runhistory.aggregate_func = aggregate_func
# initial random number generator
num_run, rng = self._get_rng(rng=rng)
# reset random number generator in config space to draw different
# random configurations with each seed given to SMAC
scenario.cs.seed(rng.randint(MAXINT))
# initial Trajectory Logger
traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats)
# initial EPM
types, bounds = get_types(scenario.cs, scenario.feature_array)
if model is None:
model = RandomForestWithInstances(
types=types,
bounds=bounds,
instance_features=scenario.feature_array,
seed=rng.randint(MAXINT),
pca_components=scenario.PCA_DIM,
num_trees=scenario.rf_num_trees,
do_bootstrapping=scenario.rf_do_bootstrapping,
ratio_features=scenario.rf_ratio_features,
min_samples_split=scenario.rf_min_samples_split,
min_samples_leaf=scenario.rf_min_samples_leaf,
max_depth=scenario.rf_max_depth)
# initial acquisition function
if acquisition_function is None:
if scenario.run_obj == "runtime":
acquisition_function = LogEI(model=model)
else:
acquisition_function = EI(model=model)
# inject model if necessary
if acquisition_function.model is None:
acquisition_function.model = model
# initialize optimizer on acquisition function
local_search = LocalSearch(
acquisition_function,
scenario.cs,
max_steps=scenario.sls_max_steps,
n_steps_plateau_walk=scenario.sls_n_steps_plateau_walk)
# initialize tae_runner
# First case, if tae_runner is None, the target algorithm is a call
# string in the scenario file
if tae_runner is None:
tae_runner = ExecuteTARunOld(
ta=scenario.ta,
stats=self.stats,
run_obj=scenario.run_obj,
runhistory=runhistory,
par_factor=scenario.par_factor,
cost_for_crash=scenario.cost_for_crash)
# Second case, the tae_runner is a function to be optimized
elif callable(tae_runner):
tae_runner = ExecuteTAFuncDict(
ta=tae_runner,
stats=self.stats,
run_obj=scenario.run_obj,
memory_limit=scenario.memory_limit,
runhistory=runhistory,
par_factor=scenario.par_factor,
cost_for_crash=scenario.cost_for_crash)
# Third case, if it is an ExecuteTaRun we can simply use the
# instance. Otherwise, the next check raises an exception
elif not isinstance(tae_runner, ExecuteTARun):
raise TypeError("Argument 'tae_runner' is %s, but must be "
"either a callable or an instance of "
"ExecuteTaRun. Passing 'None' will result in the "
"creation of target algorithm runner based on the "
"call string in the scenario file." %
type(tae_runner))
# Check that overall objective and tae objective are the same
if tae_runner.run_obj != scenario.run_obj:
raise ValueError("Objective for the target algorithm runner and "
"the scenario must be the same, but are '%s' and "
"'%s'" % (tae_runner.run_obj, scenario.run_obj))
# inject stats if necessary
if tae_runner.stats is None:
tae_runner.stats = self.stats
# inject runhistory if necessary
if tae_runner.runhistory is None:
tae_runner.runhistory = runhistory
# inject cost_for_crash
if tae_runner.crash_cost != scenario.cost_for_crash:
tae_runner.crash_cost = scenario.cost_for_crash
# initialize intensification
if intensifier is None:
intensifier = Intensifier(
tae_runner=tae_runner,
stats=self.stats,
traj_logger=traj_logger,
rng=rng,
instances=scenario.train_insts,
cutoff=scenario.cutoff,
deterministic=scenario.deterministic,
run_obj_time=scenario.run_obj == "runtime",
always_race_against=scenario.cs.get_default_configuration()
if scenario.always_race_default else None,
instance_specifics=scenario.instance_specific,
minR=scenario.minR,
maxR=scenario.maxR,
adaptive_capping_slackfactor=scenario.
intens_adaptive_capping_slackfactor,
min_chall=scenario.intens_min_chall)
# inject deps if necessary
if intensifier.tae_runner is None:
intensifier.tae_runner = tae_runner
if intensifier.stats is None:
intensifier.stats = self.stats
if intensifier.traj_logger is None:
intensifier.traj_logger = traj_logger
# initial design
if initial_design is not None and initial_configurations is not None:
raise ValueError(
"Either use initial_design or initial_configurations; but not both"
)
if initial_configurations is not None:
initial_design = MultiConfigInitialDesign(
tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
runhistory=runhistory,
rng=rng,
configs=initial_configurations,
intensifier=intensifier,
aggregate_func=aggregate_func)
elif initial_design is None:
if scenario.initial_incumbent == "DEFAULT":
initial_design = DefaultConfiguration(tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
elif scenario.initial_incumbent == "RANDOM":
initial_design = RandomConfiguration(tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
else:
raise ValueError("Don't know what kind of initial_incumbent "
"'%s' is" % scenario.initial_incumbent)
# inject deps if necessary
if initial_design.tae_runner is None:
initial_design.tae_runner = tae_runner
if initial_design.scenario is None:
initial_design.scenario = scenario
if initial_design.stats is None:
initial_design.stats = self.stats
if initial_design.traj_logger is None:
initial_design.traj_logger = traj_logger
# initial conversion of runhistory into EPM data
if runhistory2epm is None:
num_params = len(scenario.cs.get_hyperparameters())
if scenario.run_obj == "runtime":
# if we log the performance data,
# the RFRImputator will already get
# log transform data from the runhistory
cutoff = np.log(scenario.cutoff)
threshold = np.log(scenario.cutoff * scenario.par_factor)
imputor = RFRImputator(rng=rng,
cutoff=cutoff,
threshold=threshold,
model=model,
change_threshold=0.01,
max_iter=2)
runhistory2epm = RunHistory2EPM4LogCost(
scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=True,
impute_state=[
StatusType.CAPPED,
],
imputor=imputor)
elif scenario.run_obj == 'quality':
runhistory2epm = RunHistory2EPM4Cost(
scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=False,
impute_state=None)
else:
raise ValueError('Unknown run objective: %s. Should be either '
'quality or runtime.' % self.scenario.run_obj)
# inject scenario if necessary:
if runhistory2epm.scenario is None:
runhistory2epm.scenario = scenario
self.solver = EPILS_Solver(scenario=scenario,
stats=self.stats,
initial_design=initial_design,
runhistory=runhistory,
runhistory2epm=runhistory2epm,
intensifier=intensifier,
aggregate_func=aggregate_func,
num_run=num_run,
model=model,
acq_optimizer=local_search,
acquisition_func=acquisition_function,
rng=rng)
def validate_epm(
self,
config_mode: Union[str, typing.List[Configuration]] = 'def',
instance_mode: Union[str, typing.List[str]] = 'test',
repetitions: int = 1,
runhistory: typing.Optional[RunHistory] = None,
output_fn: typing.Optional[str] = None,
reuse_epm: bool = True,
) -> RunHistory:
"""
Use EPM to predict costs/runtimes for unknown config/inst-pairs.
side effect: if output is specified, saves runhistory to specified
output directory.
Parameters
----------
output_fn: str
path to runhistory to be saved. if the suffix is not '.json', will
be interpreted as directory and filename will be
'validated_runhistory_EPM.json'
config_mode: str or list<Configuration>
string or directly a list of Configuration, string from [def, inc, def+inc, wallclock_time, cpu_time, all].
time evaluates at cpu- or wallclock-timesteps of:
[max_time/2^0, max_time/2^1, max_time/2^3, ..., default] with max_time being the highest recorded time
instance_mode: str or list<str>
what instances to use for validation, either from
[train, test, train+test] or directly a list of instances
repetitions: int
number of repetitions in nondeterministic algorithms
runhistory: RunHistory
optional, RunHistory-object to reuse runs
reuse_epm: bool
if true (and if `self.epm`), reuse epm to validate runs
Returns
-------
runhistory: RunHistory
runhistory with predicted runs
"""
if not isinstance(runhistory, RunHistory) and (self.epm is None
or not reuse_epm):
raise ValueError(
"No runhistory specified for validating with EPM!")
elif not reuse_epm or self.epm is None:
# Create RandomForest
types, bounds = get_types(
self.scen.cs, self.scen.feature_array
) # type: ignore[attr-defined] # noqa F821
epm = RandomForestWithInstances(
configspace=self.scen.
cs, # type: ignore[attr-defined] # noqa F821
types=types,
bounds=bounds,
instance_features=self.scen.feature_array,
seed=self.rng.randint(MAXINT),
ratio_features=1.0,
)
# Use imputor if objective is runtime
imputor = None
impute_state = None
impute_censored_data = False
if self.scen.run_obj == 'runtime':
threshold = self.scen.cutoff * self.scen.par_factor # type: ignore[attr-defined] # noqa F821
imputor = RFRImputator(
rng=self.rng,
cutoff=self.scen.
cutoff, # type: ignore[attr-defined] # noqa F821
threshold=threshold,
model=epm)
impute_censored_data = True
impute_state = [StatusType.CAPPED]
success_states = [
StatusType.SUCCESS,
]
else:
success_states = [
StatusType.SUCCESS, StatusType.CRASHED, StatusType.MEMOUT
]
# Transform training data (from given rh)
rh2epm = RunHistory2EPM4Cost(
num_params=len(self.scen.cs.get_hyperparameters()
), # type: ignore[attr-defined] # noqa F821
scenario=self.scen,
rng=self.rng,
impute_censored_data=impute_censored_data,
imputor=imputor,
impute_state=impute_state,
success_states=success_states)
assert runhistory is not None # please mypy
X, y = rh2epm.transform(runhistory)
self.logger.debug("Training model with data of shape X: %s, y:%s",
str(X.shape), str(y.shape))
# Train random forest
epm.train(X, y)
else:
epm = typing.cast(RandomForestWithInstances, self.epm)
# Predict desired runs
runs, rh_epm = self._get_runs(config_mode, instance_mode, repetitions,
runhistory)
feature_array_size = len(self.scen.cs.get_hyperparameters()
) # type: ignore[attr-defined] # noqa F821
if self.scen.feature_array is not None:
feature_array_size += self.scen.feature_array.shape[1]
X_pred = np.empty((len(runs), feature_array_size))
for idx, run in enumerate(runs):
if self.scen.feature_array is not None and run.inst is not None:
X_pred[idx] = np.hstack([
convert_configurations_to_array([run.config])[0],
self.scen.feature_dict[run.inst]
])
else:
X_pred[idx] = convert_configurations_to_array([run.config])[0]
self.logger.debug("Predicting desired %d runs, data has shape %s",
len(runs), str(X_pred.shape))
y_pred = epm.predict(X_pred)
self.epm = epm
# Add runs to runhistory
for run, pred in zip(runs, y_pred[0]):
rh_epm.add(
config=run.config,
cost=float(pred),
time=float(pred),
status=StatusType.SUCCESS,
instance_id=run.inst,
seed=-1,
additional_info={"additional_info": "ESTIMATED USING EPM!"})
if output_fn:
self._save_results(rh_epm,
output_fn,
backup_fn="validated_runhistory_EPM.json")
return rh_epm
def _get_mean_var_time(self, validator, traj, use_epm, rh):
"""
Parameters
----------
validator: Validator
validator (smac-based)
traj: List[Configuraton]
trajectory to set in validator
use_epm: bool
validated or not (no need to use epm if validated)
rh: RunHistory
??
Returns
-------
mean, var
times: List[float]
times to plot (x-values)
configs
"""
# TODO kinda important: docstrings, what is this function doing?
if validator:
validator.traj = traj # set trajectory
time, configs = [], []
if use_epm and not self.block_epm:
for entry in traj:
time.append(entry["wallclock_time"])
configs.append(entry["incumbent"])
# self.logger.debug('Time: %d Runs: %d', time[-1], len(rh.get_runs_for_config(configs[-1])))
self.logger.debug(
"Using %d samples (%d distinct) from trajectory.", len(time),
len(set(configs)))
# Initialize EPM
if validator.epm: # not log as validator epm is trained on cost, not log cost
epm = validator.epm
else:
self.logger.debug(
"No EPM passed! Training new one from runhistory.")
# Train random forest and transform training data (from given rh)
# Not using validator because we want to plot uncertainties
rh2epm = RunHistory2EPM4Cost(num_params=len(
self.scenario.cs.get_hyperparameters()),
scenario=self.scenario)
X, y = rh2epm.transform(rh)
self.logger.debug(
"Training model with data of shape X: %s, y: %s",
str(X.shape), str(y.shape))
types, bounds = get_types(self.scenario.cs,
self.scenario.feature_array)
epm = RandomForestWithInstances(
self.scenario.cs,
types=types,
bounds=bounds,
seed=self.rng.randint(MAXINT),
instance_features=self.scenario.feature_array,
ratio_features=1.0)
epm.train(X, y)
config_array = convert_configurations_to_array(configs)
mean, var = epm.predict_marginalized_over_instances(config_array)
var = np.zeros(mean.shape)
# We don't want to show the uncertainty of the model but uncertainty over multiple optimizer runs
# This variance is computed in an outer loop.
else:
mean, var = [], []
for entry in traj:
#self.logger.debug(entry)
time.append(entry["wallclock_time"])
configs.append(entry["incumbent"])
costs = _cost(configs[-1], rh,
rh.get_runs_for_config(configs[-1]))
# self.logger.debug(len(costs), time[-1]
if not costs:
time.pop()
else:
mean.append(np.mean(costs))
var.append(0) # No variance over instances
mean, var = np.array(mean).reshape(-1, 1), np.array(var).reshape(
-1, 1)
return mean, var, time, configs
def convert_data_for_epm(scenario: Scenario,
runhistory: RunHistory,
logger=None):
"""
converts data from runhistory into EPM format
Parameters
----------
scenario: Scenario
smac.scenario.scenario.Scenario Object
runhistory: RunHistory
smac.runhistory.runhistory.RunHistory Object with all necessary data
Returns
-------
X: np.array
X matrix with configuartion x features for all observed samples
y: np.array
y matrix with all observations
types: np.array
types of X cols -- necessary to train our RF implementation
"""
types, bounds = get_types(scenario.cs, scenario.feature_array)
model = RandomForestWithInstances(types, bounds)
params = scenario.cs.get_hyperparameters()
num_params = len(params)
run_obj = scenario.run_obj
if run_obj == "runtime":
# if we log the performance data,
# the RFRImputator will already get
# log transform data from the runhistory
cutoff = np.log10(scenario.cutoff)
threshold = np.log10(scenario.cutoff * scenario.par_factor)
imputor = RFRImputator(rng=np.random.RandomState(42),
cutoff=cutoff,
threshold=threshold,
model=model,
change_threshold=0.01,
max_iter=10)
# TODO: Adapt runhistory2EPM object based on scenario
rh2EPM = RunHistory2EPM4LogCost(scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=True,
impute_state=[
StatusType.TIMEOUT,
],
imputor=imputor)
X, Y = rh2EPM.transform(runhistory)
else:
rh2EPM = RunHistory2EPM4Cost(scenario=scenario,
num_params=num_params,
success_states=None,
impute_censored_data=False,
impute_state=None)
X, Y = rh2EPM.transform(runhistory)
return X, Y, types
def run_smbo(self, max_iters=1000):
global evaluator
# == first things first: load the datamanager
self.reset_data_manager()
# == Initialize SMBO stuff
# first create a scenario
seed = self.seed # TODO
num_params = len(self.config_space.get_hyperparameters())
# allocate a run history
run_history = RunHistory()
meta_runhistory = RunHistory()
meta_runs_dataset_indices = {}
num_run = self.start_num_run
instance_id = self.dataset_name + SENTINEL
# == Train on subset
# before doing anything, let us run the default_cfg
# on a subset of the available data to ensure that
# we at least have some models
# we will try three different ratios of decreasing magnitude
# in the hope that at least on the last one we will be able
# to get a model
n_data = self.datamanager.data['X_train'].shape[0]
subset_ratio = 10000. / n_data
if subset_ratio >= 0.5:
subset_ratio = 0.33
subset_ratios = [subset_ratio, subset_ratio * 0.10]
else:
subset_ratios = [subset_ratio, 500. / n_data]
self.logger.info("Training default configurations on a subset of "
"%d/%d data points." %
(int(n_data * subset_ratio), n_data))
# the time limit for these function evaluations is rigorously
# set to only 1/2 of a full function evaluation
subset_time_limit = max(5, int(self.func_eval_time_limit / 2))
# the configs we want to run on the data subset are:
# 1) the default configs
# 2) a set of configs we selected for training on a subset
subset_configs = [self.config_space.get_default_configuration()] \
+ self.collect_additional_subset_defaults()
subset_config_succesful = [False] * len(subset_configs)
for subset_config_id, next_config in enumerate(subset_configs):
for i, ratio in enumerate(subset_ratios):
self.reset_data_manager()
n_data_subsample = int(n_data * ratio)
# run the config, but throw away the result afterwards
# since this cfg was evaluated only on a subset
# and we don't want to confuse SMAC
self.logger.info(
"Starting to evaluate %d on SUBSET "
"with size %d and time limit %ds.", num_run,
n_data_subsample, subset_time_limit)
self.logger.info(next_config)
_info = eval_with_limits(self.datamanager, self.tmp_dir,
next_config, seed, num_run,
self.resampling_strategy,
self.resampling_strategy_args,
self.memory_limit, subset_time_limit,
n_data_subsample)
(duration, result, _, additional_run_info, status) = _info
self.logger.info(
"Finished evaluating %d. configuration on SUBSET. "
"Duration %f; loss %f; status %s; additional run "
"info: %s ", num_run, duration, result, str(status),
additional_run_info)
num_run += 1
if i < len(subset_ratios) - 1:
if status != StatusType.SUCCESS:
# Do not increase num_run here, because we will try
# the same configuration with less data
self.logger.info(
"A CONFIG did not finish "
" for subset ratio %f -> going smaller", ratio)
continue
else:
self.logger.info(
"Finished SUBSET training sucessfully "
"with ratio %f", ratio)
subset_config_succesful[subset_config_id] = True
break
else:
if status != StatusType.SUCCESS:
self.logger.info(
"A CONFIG did not finish "
" for subset ratio %f.", ratio)
continue
else:
self.logger.info(
"Finished SUBSET training sucessfully "
"with ratio %f", ratio)
subset_config_succesful[subset_config_id] = True
break
# Use the first non-failing configuration from the subsets as the new
# default configuration -> this guards us against the random forest
# failing on large, sparse datasets
default_cfg = None
for subset_config_id, next_config in enumerate(subset_configs):
if subset_config_succesful[subset_config_id]:
default_cfg = next_config
break
if default_cfg is None:
default_cfg = self.config_space.get_default_configuration()
# == METALEARNING suggestions
# we start by evaluating the defaults on the full dataset again
# and add the suggestions from metalearning behind it
if self.metadata_directory is None:
metalearning_directory = os.path.dirname(
autosklearn.metalearning.__file__)
# There is no multilabel data in OpenML
if self.task == MULTILABEL_CLASSIFICATION:
meta_task = BINARY_CLASSIFICATION
else:
meta_task = self.task
metadata_directory = os.path.join(
metalearning_directory, 'files', '%s_%s_%s' %
(METRIC_TO_STRING[self.metric],
TASK_TYPES_TO_STRING[meta_task],
'sparse' if self.datamanager.info['is_sparse'] else 'dense'))
self.metadata_directory = metadata_directory
self.logger.info('Metadata directory: %s', self.metadata_directory)
meta_base = MetaBase(self.config_space, self.metadata_directory)
metafeature_calculation_time_limit = int(self.total_walltime_limit / 4)
metafeature_calculation_start_time = time.time()
meta_features = self._calculate_metafeatures_with_limits(
metafeature_calculation_time_limit)
metafeature_calculation_end_time = time.time()
metafeature_calculation_time_limit = \
metafeature_calculation_time_limit - (
metafeature_calculation_end_time -
metafeature_calculation_start_time)
if metafeature_calculation_time_limit < 1:
self.logger.warning(
'Time limit for metafeature calculation less '
'than 1 seconds (%f). Skipping calculation '
'of metafeatures for encoded dataset.',
metafeature_calculation_time_limit)
meta_features_encoded = None
else:
self.datamanager.perform1HotEncoding()
meta_features_encoded = \
self._calculate_metafeatures_encoded_with_limits(
metafeature_calculation_time_limit)
# In case there is a problem calculating the encoded meta-features
if meta_features is None:
if meta_features_encoded is not None:
meta_features = meta_features_encoded
else:
if meta_features_encoded is not None:
meta_features.metafeature_values.update(
meta_features_encoded.metafeature_values)
if meta_features is not None:
meta_base.add_dataset(instance_id, meta_features)
# Do mean imputation of the meta-features - should be done specific
# for each prediction model!
all_metafeatures = meta_base.get_metafeatures(
features=list(meta_features.keys()))
all_metafeatures.fillna(all_metafeatures.mean(), inplace=True)
metalearning_configurations = self.collect_metalearning_suggestions(
meta_base)
if metalearning_configurations is None:
metalearning_configurations = []
self.reset_data_manager()
self.logger.info('%s', meta_features)
# Convert meta-features into a dictionary because the scenario
# expects a dictionary
meta_features_dict = {}
for dataset, series in all_metafeatures.iterrows():
meta_features_dict[dataset] = series.values
meta_features_list = []
for meta_feature_name in all_metafeatures.columns:
meta_features_list.append(
meta_features[meta_feature_name].value)
meta_features_list = np.array(meta_features_list).reshape((1, -1))
self.logger.info(list(meta_features_dict.keys()))
meta_runs = meta_base.get_all_runs(METRIC_TO_STRING[self.metric])
meta_runs_index = 0
try:
meta_durations = meta_base.get_all_runs('runtime')
read_runtime_data = True
except KeyError:
read_runtime_data = False
self.logger.critical('Cannot read runtime data.')
if self.acquisition_function == 'EIPS':
self.logger.critical(
'Reverting to acquisition function EI!')
self.acquisition_function = 'EI'
for meta_dataset in meta_runs.index:
meta_dataset_start_index = meta_runs_index
for meta_configuration in meta_runs.columns:
if np.isfinite(meta_runs.loc[meta_dataset,
meta_configuration]):
try:
config = meta_base.get_configuration_from_algorithm_index(
meta_configuration)
cost = meta_runs.loc[meta_dataset,
meta_configuration]
if read_runtime_data:
runtime = meta_durations.loc[
meta_dataset, meta_configuration]
else:
runtime = 1
# TODO read out other status types!
meta_runhistory.add(config,
cost,
runtime,
StatusType.SUCCESS,
instance_id=meta_dataset)
meta_runs_index += 1
except:
# TODO maybe add warning
pass
meta_runs_dataset_indices[meta_dataset] = (
meta_dataset_start_index, meta_runs_index)
else:
if self.acquisition_function == 'EIPS':
self.logger.critical('Reverting to acquisition function EI!')
self.acquisition_function = 'EI'
meta_features_list = []
meta_features_dict = {}
metalearning_configurations = []
self.scenario = AutoMLScenario(self.config_space,
self.total_walltime_limit,
self.func_eval_time_limit,
meta_features_dict, self.tmp_dir,
self.shared_mode)
types = get_types(self.config_space, self.scenario.feature_array)
if self.acquisition_function == 'EI':
rh2EPM = RunHistory2EPM4Cost(num_params=num_params,
scenario=self.scenario,
success_states=None,
impute_censored_data=False,
impute_state=None)
model = RandomForestWithInstances(
types,
instance_features=meta_features_list,
seed=1,
num_trees=10)
smac = SMBO(self.scenario, model=model, rng=seed)
elif self.acquisition_function == 'EIPS':
rh2EPM = RunHistory2EPM4EIPS(num_params=num_params,
scenario=self.scenario,
success_states=None,
impute_censored_data=False,
impute_state=None)
model = UncorrelatedMultiObjectiveRandomForestWithInstances(
['cost', 'runtime'],
types,
num_trees=10,
instance_features=meta_features_list,
seed=1)
acquisition_function = EIPS(model)
smac = SMBO(self.scenario,
acquisition_function=acquisition_function,
model=model,
runhistory2epm=rh2EPM,
rng=seed)
else:
raise ValueError('Unknown acquisition function value %s!' %
self.acquisition_function)
# Build a runtime model
# runtime_rf = RandomForestWithInstances(types,
# instance_features=meta_features_list,
# seed=1, num_trees=10)
# runtime_rh2EPM = RunHistory2EPM4EIPS(num_params=num_params,
# scenario=self.scenario,
# success_states=None,
# impute_censored_data=False,
# impute_state=None)
# X_runtime, y_runtime = runtime_rh2EPM.transform(meta_runhistory)
# runtime_rf.train(X_runtime, y_runtime[:, 1].flatten())
X_meta, Y_meta = rh2EPM.transform(meta_runhistory)
# Transform Y_meta on a per-dataset base
for meta_dataset in meta_runs_dataset_indices:
start_index, end_index = meta_runs_dataset_indices[meta_dataset]
end_index += 1 # Python indexing
Y_meta[start_index:end_index, 0]\
[Y_meta[start_index:end_index, 0] >2.0] = 2.0
dataset_minimum = np.min(Y_meta[start_index:end_index, 0])
Y_meta[start_index:end_index,
0] = 1 - ((1. - Y_meta[start_index:end_index, 0]) /
(1. - dataset_minimum))
Y_meta[start_index:end_index, 0]\
[Y_meta[start_index:end_index, 0] > 2] = 2
# == first, evaluate all metelearning and default configurations
for i, next_config in enumerate(
([default_cfg] + metalearning_configurations)):
# Do not evaluate default configurations more than once
if i >= len([default_cfg]) and next_config in [default_cfg]:
continue
config_name = 'meta-learning' if i >= len([default_cfg]) \
else 'default'
self.logger.info(
"Starting to evaluate %d. configuration "
"(%s configuration) with time limit %ds.", num_run,
config_name, self.func_eval_time_limit)
self.logger.info(next_config)
self.reset_data_manager()
info = eval_with_limits(self.datamanager, self.tmp_dir,
next_config, seed, num_run,
self.resampling_strategy,
self.resampling_strategy_args,
self.memory_limit,
self.func_eval_time_limit)
(duration, result, _, additional_run_info, status) = info
run_history.add(config=next_config,
cost=result,
time=duration,
status=status,
instance_id=instance_id,
seed=seed)
run_history.update_cost(next_config, result)
self.logger.info(
"Finished evaluating %d. configuration. "
"Duration %f; loss %f; status %s; additional run "
"info: %s ", num_run, duration, result, str(status),
additional_run_info)
num_run += 1
if smac.incumbent is None:
smac.incumbent = next_config
elif result < run_history.get_cost(smac.incumbent):
smac.incumbent = next_config
if self.scenario.shared_model:
pSMAC.write(run_history=run_history,
output_directory=self.scenario.output_dir,
num_run=self.seed)
# == after metalearning run SMAC loop
smac.runhistory = run_history
smac_iter = 0
finished = False
while not finished:
if self.scenario.shared_model:
pSMAC.read(run_history=run_history,
output_directory=self.scenario.output_dir,
configuration_space=self.config_space,
logger=self.logger)
next_configs = []
time_for_choose_next = -1
try:
X_cfg, Y_cfg = rh2EPM.transform(run_history)
if not run_history.empty():
# Update costs by normalization
dataset_minimum = np.min(Y_cfg[:, 0])
Y_cfg[:, 0] = 1 - ((1. - Y_cfg[:, 0]) /
(1. - dataset_minimum))
Y_cfg[:, 0][Y_cfg[:, 0] > 2] = 2
if len(X_meta) > 0 and len(X_cfg) > 0:
pass
#X_cfg = np.concatenate((X_meta, X_cfg))
#Y_cfg = np.concatenate((Y_meta, Y_cfg))
elif len(X_meta) > 0:
X_cfg = X_meta.copy()
Y_cfg = Y_meta.copy()
elif len(X_cfg) > 0:
X_cfg = X_cfg.copy()
Y_cfg = Y_cfg.copy()
else:
raise ValueError(
'No training data for SMAC random forest!')
self.logger.info('Using %d training points for SMAC.' %
X_cfg.shape[0])
choose_next_start_time = time.time()
next_configs_tmp = smac.choose_next(
X_cfg,
Y_cfg,
num_interleaved_random=110,
num_configurations_by_local_search=10,
num_configurations_by_random_search_sorted=100)
time_for_choose_next = time.time() - choose_next_start_time
self.logger.info('Used %g seconds to find next '
'configurations' % (time_for_choose_next))
next_configs.extend(next_configs_tmp)
# TODO put Exception here!
except Exception as e:
self.logger.error(e)
self.logger.error("Error in getting next configurations "
"with SMAC. Using random configuration!")
next_config = self.config_space.sample_configuration()
next_configs.append(next_config)
models_fitted_this_iteration = 0
start_time_this_iteration = time.time()
for next_config in next_configs:
x_runtime = impute_inactive_values(next_config)
x_runtime = impute_inactive_values(x_runtime).get_array()
# predicted_runtime = runtime_rf.predict_marginalized_over_instances(
# x_runtime.reshape((1, -1)))
# predicted_runtime = np.exp(predicted_runtime[0][0][0]) - 1
self.logger.info(
"Starting to evaluate %d. configuration (from "
"SMAC) with time limit %ds.", num_run,
self.func_eval_time_limit)
self.logger.info(next_config)
self.reset_data_manager()
info = eval_with_limits(self.datamanager, self.tmp_dir,
next_config, seed, num_run,
self.resampling_strategy,
self.resampling_strategy_args,
self.memory_limit,
self.func_eval_time_limit)
(duration, result, _, additional_run_info, status) = info
run_history.add(config=next_config,
cost=result,
time=duration,
status=status,
instance_id=instance_id,
seed=seed)
run_history.update_cost(next_config, result)
#self.logger.info('Predicted runtime %g, true runtime %g',
# predicted_runtime, duration)
# TODO add unittest to make sure everything works fine and
# this does not get outdated!
if smac.incumbent is None:
smac.incumbent = next_config
elif result < run_history.get_cost(smac.incumbent):
smac.incumbent = next_config
self.logger.info(
"Finished evaluating %d. configuration. "
"Duration: %f; loss: %f; status %s; additional "
"run info: %s ", num_run, duration, result, str(status),
additional_run_info)
smac_iter += 1
num_run += 1
models_fitted_this_iteration += 1
time_used_this_iteration = time.time(
) - start_time_this_iteration
if models_fitted_this_iteration >= 2 and \
time_for_choose_next > 0 and \
time_used_this_iteration > time_for_choose_next:
break
elif time_for_choose_next <= 0 and \
models_fitted_this_iteration >= 1:
break
elif models_fitted_this_iteration >= 50:
break
if max_iters is not None:
finished = (smac_iter < max_iters)
if self.scenario.shared_model:
pSMAC.write(run_history=run_history,
output_directory=self.scenario.output_dir,
num_run=self.seed)
def run_smbo(self):
self.watcher.start_task('SMBO')
# == first things first: load the datamanager
self.reset_data_manager()
# == Initialize non-SMBO stuff
# first create a scenario
seed = self.seed
self.config_space.seed(seed)
num_params = len(self.config_space.get_hyperparameters())
# allocate a run history
num_run = self.start_num_run
instance_id = self.dataset_name + SENTINEL
# Initialize some SMAC dependencies
runhistory = RunHistory(aggregate_func=average_cost)
# meta_runhistory = RunHistory(aggregate_func=average_cost)
# meta_runs_dataset_indices = {}
# == METALEARNING suggestions
# we start by evaluating the defaults on the full dataset again
# and add the suggestions from metalearning behind it
if self.num_metalearning_cfgs > 0:
if self.metadata_directory is None:
metalearning_directory = os.path.dirname(
autosklearn.metalearning.__file__)
# There is no multilabel data in OpenML
if self.task == MULTILABEL_CLASSIFICATION:
meta_task = BINARY_CLASSIFICATION
else:
meta_task = self.task
metadata_directory = os.path.join(
metalearning_directory, 'files', '%s_%s_%s' %
(METRIC_TO_STRING[self.metric],
TASK_TYPES_TO_STRING[meta_task], 'sparse'
if self.datamanager.info['is_sparse'] else 'dense'))
self.metadata_directory = metadata_directory
self.logger.info('Metadata directory: %s', self.metadata_directory)
meta_base = MetaBase(self.config_space, self.metadata_directory)
metafeature_calculation_time_limit = int(
self.total_walltime_limit / 4)
metafeature_calculation_start_time = time.time()
meta_features = self._calculate_metafeatures_with_limits(
metafeature_calculation_time_limit)
metafeature_calculation_end_time = time.time()
metafeature_calculation_time_limit = \
metafeature_calculation_time_limit - (
metafeature_calculation_end_time -
metafeature_calculation_start_time)
if metafeature_calculation_time_limit < 1:
self.logger.warning(
'Time limit for metafeature calculation less '
'than 1 seconds (%f). Skipping calculation '
'of metafeatures for encoded dataset.',
metafeature_calculation_time_limit)
meta_features_encoded = None
else:
with warnings.catch_warnings():
warnings.showwarning = self._send_warnings_to_log
self.datamanager.perform1HotEncoding()
meta_features_encoded = \
self._calculate_metafeatures_encoded_with_limits(
metafeature_calculation_time_limit)
# In case there is a problem calculating the encoded meta-features
if meta_features is None:
if meta_features_encoded is not None:
meta_features = meta_features_encoded
else:
if meta_features_encoded is not None:
meta_features.metafeature_values.update(
meta_features_encoded.metafeature_values)
if meta_features is not None:
meta_base.add_dataset(instance_id, meta_features)
# Do mean imputation of the meta-features - should be done specific
# for each prediction model!
all_metafeatures = meta_base.get_metafeatures(
features=list(meta_features.keys()))
all_metafeatures.fillna(all_metafeatures.mean(), inplace=True)
with warnings.catch_warnings():
warnings.showwarning = self._send_warnings_to_log
metalearning_configurations = self.collect_metalearning_suggestions(
meta_base)
if metalearning_configurations is None:
metalearning_configurations = []
self.reset_data_manager()
self.logger.info('%s', meta_features)
# Convert meta-features into a dictionary because the scenario
# expects a dictionary
meta_features_dict = {}
for dataset, series in all_metafeatures.iterrows():
meta_features_dict[dataset] = series.values
meta_features_list = []
for meta_feature_name in all_metafeatures.columns:
meta_features_list.append(
meta_features[meta_feature_name].value)
meta_features_list = np.array(meta_features_list).reshape(
(1, -1))
self.logger.info(list(meta_features_dict.keys()))
# meta_runs = meta_base.get_all_runs(METRIC_TO_STRING[self.metric])
# meta_runs_index = 0
# try:
# meta_durations = meta_base.get_all_runs('runtime')
# read_runtime_data = True
# except KeyError:
# read_runtime_data = False
# self.logger.critical('Cannot read runtime data.')
# if self.acquisition_function == 'EIPS':
# self.logger.critical('Reverting to acquisition function EI!')
# self.acquisition_function = 'EI'
# for meta_dataset in meta_runs.index:
# meta_dataset_start_index = meta_runs_index
# for meta_configuration in meta_runs.columns:
# if np.isfinite(meta_runs.loc[meta_dataset, meta_configuration]):
# try:
# config = meta_base.get_configuration_from_algorithm_index(
# meta_configuration)
# cost = meta_runs.loc[meta_dataset, meta_configuration]
# if read_runtime_data:
# runtime = meta_durations.loc[meta_dataset,
# meta_configuration]
# else:
# runtime = 1
# # TODO read out other status types!
# meta_runhistory.add(config, cost, runtime,
# StatusType.SUCCESS,
# instance_id=meta_dataset)
# meta_runs_index += 1
# except:
# # TODO maybe add warning
# pass
#
# meta_runs_dataset_indices[meta_dataset] = (
# meta_dataset_start_index, meta_runs_index)
else:
meta_features = None
if meta_features is None:
if self.acquisition_function == 'EIPS':
self.logger.critical('Reverting to acquisition function EI!')
self.acquisition_function = 'EI'
meta_features_list = []
meta_features_dict = {}
metalearning_configurations = []
if self.resampling_strategy in [
'partial-cv', 'partial-cv-iterative-fit'
]:
num_folds = self.resampling_strategy_args['folds']
instances = [[fold_number] for fold_number in range(num_folds)]
else:
instances = None
startup_time = self.watcher.wall_elapsed(self.dataset_name)
total_walltime_limit = self.total_walltime_limit - startup_time - 5
scenario_dict = {
'cs': self.config_space,
'cutoff-time': self.func_eval_time_limit,
'memory-limit': self.memory_limit,
'wallclock-limit': total_walltime_limit,
# 'instances': [[name] for name in meta_features_dict],
'output-dir': self.backend.temporary_directory,
'shared-model': self.shared_mode,
'run-obj': 'quality',
'deterministic': 'true',
'instances': instances
}
if self.configuration_mode == 'RANDOM':
scenario_dict['minR'] = len(
instances) if instances is not None else 1
scenario_dict['initial_incumbent'] = 'RANDOM'
self.scenario = Scenario(scenario_dict)
# TODO rebuild target algorithm to be it's own target algorithm
# evaluator, which takes into account that a run can be killed prior
# to the model being fully fitted; thus putting intermediate results
# into a queue and querying them once the time is over
exclude = dict()
include = dict()
if self.include_preprocessors is not None and \
self.exclude_preprocessors is not None:
raise ValueError('Cannot specify include_preprocessors and '
'exclude_preprocessors.')
elif self.include_preprocessors is not None:
include['preprocessor'] = self.include_preprocessors
elif self.exclude_preprocessors is not None:
exclude['preprocessor'] = self.exclude_preprocessors
if self.include_estimators is not None and \
self.exclude_preprocessors is not None:
raise ValueError('Cannot specify include_estimators and '
'exclude_estimators.')
elif self.include_estimators is not None:
if self.task in CLASSIFICATION_TASKS:
include['classifier'] = self.include_estimators
elif self.task in REGRESSION_TASKS:
include['regressor'] = self.include_estimators
else:
raise ValueError(self.task)
elif self.exclude_estimators is not None:
if self.task in CLASSIFICATION_TASKS:
exclude['classifier'] = self.exclude_estimators
elif self.task in REGRESSION_TASKS:
exclude['regressor'] = self.exclude_estimators
else:
raise ValueError(self.task)
ta = ExecuteTaFuncWithQueue(
backend=self.backend,
autosklearn_seed=seed,
resampling_strategy=self.resampling_strategy,
initial_num_run=num_run,
logger=self.logger,
include=include,
exclude=exclude,
memory_limit=self.memory_limit,
disable_file_output=self.disable_file_output,
**self.resampling_strategy_args)
types = get_types(self.config_space, self.scenario.feature_array)
# TODO extract generation of SMAC object into it's own function for
# testing
if self.acquisition_function == 'EI':
model = RandomForestWithInstances(
types,
#instance_features=meta_features_list,
seed=1,
num_trees=10)
rh2EPM = RunHistory2EPM4Cost(num_params=num_params,
scenario=self.scenario,
success_states=[
StatusType.SUCCESS,
StatusType.MEMOUT,
StatusType.TIMEOUT
],
impute_censored_data=False,
impute_state=None)
_smac_arguments = dict(scenario=self.scenario,
model=model,
rng=seed,
runhistory2epm=rh2EPM,
tae_runner=ta,
runhistory=runhistory)
elif self.acquisition_function == 'EIPS':
rh2EPM = RunHistory2EPM4EIPS(num_params=num_params,
scenario=self.scenario,
success_states=[
StatusType.SUCCESS,
StatusType.MEMOUT,
StatusType.TIMEOUT
],
impute_censored_data=False,
impute_state=None)
model = UncorrelatedMultiObjectiveRandomForestWithInstances(
['cost', 'runtime'],
types,
num_trees=10,
instance_features=meta_features_list,
seed=1)
acquisition_function = EIPS(model)
_smac_arguments = dict(scenario=self.scenario,
model=model,
rng=seed,
tae_runner=ta,
runhistory2epm=rh2EPM,
runhistory=runhistory,
acquisition_function=acquisition_function)
else:
raise ValueError('Unknown acquisition function value %s!' %
self.acquisition_function)
if self.configuration_mode == 'SMAC':
smac = SMAC(**_smac_arguments)
elif self.configuration_mode in ['ROAR', 'RANDOM']:
for not_in_roar in ['runhistory2epm', 'model']:
if not_in_roar in _smac_arguments:
del _smac_arguments[not_in_roar]
smac = ROAR(**_smac_arguments)
else:
raise ValueError(self.configuration_mode)
# Build a runtime model
# runtime_rf = RandomForestWithInstances(types,
# instance_features=meta_features_list,
# seed=1, num_trees=10)
# runtime_rh2EPM = RunHistory2EPM4EIPS(num_params=num_params,
# scenario=self.scenario,
# success_states=None,
# impute_censored_data=False,
# impute_state=None)
# X_runtime, y_runtime = runtime_rh2EPM.transform(meta_runhistory)
# runtime_rf.train(X_runtime, y_runtime[:, 1].flatten())
# X_meta, Y_meta = rh2EPM.transform(meta_runhistory)
# # Transform Y_meta on a per-dataset base
# for meta_dataset in meta_runs_dataset_indices:
# start_index, end_index = meta_runs_dataset_indices[meta_dataset]
# end_index += 1 # Python indexing
# Y_meta[start_index:end_index, 0]\
# [Y_meta[start_index:end_index, 0] >2.0] = 2.0
# dataset_minimum = np.min(Y_meta[start_index:end_index, 0])
# Y_meta[start_index:end_index, 0] = 1 - (
# (1. - Y_meta[start_index:end_index, 0]) /
# (1. - dataset_minimum))
# Y_meta[start_index:end_index, 0]\
# [Y_meta[start_index:end_index, 0] > 2] = 2
smac.solver.stats.start_timing()
# == first, evaluate all metelearning and default configurations
smac.solver.incumbent = smac.solver.initial_design.run()
for challenger in metalearning_configurations:
smac.solver.incumbent, inc_perf = smac.solver.intensifier.intensify(
challengers=[challenger],
incumbent=smac.solver.incumbent,
run_history=smac.solver.runhistory,
aggregate_func=smac.solver.aggregate_func,
time_bound=self.total_walltime_limit)
if smac.solver.scenario.shared_model:
pSMAC.write(run_history=smac.solver.runhistory,
output_directory=smac.solver.scenario.output_dir,
num_run=self.seed)
if smac.solver.stats.is_budget_exhausted():
break
# == after metalearning run SMAC loop
while True:
if smac.solver.scenario.shared_model:
pSMAC.read(run_history=smac.solver.runhistory,
output_directory=self.scenario.output_dir,
configuration_space=self.config_space,
logger=self.logger)
choose_next_start_time = time.time()
try:
challengers = self.choose_next(smac)
except Exception as e:
self.logger.error(e)
self.logger.error("Error in getting next configurations "
"with SMAC. Using random configuration!")
next_config = self.config_space.sample_configuration()
challengers = [next_config]
time_for_choose_next = time.time() - choose_next_start_time
self.logger.info('Used %g seconds to find next '
'configurations' % (time_for_choose_next))
time_for_choose_next = max(time_for_choose_next, 1.0)
smac.solver.incumbent, inc_perf = smac.solver.intensifier.intensify(
challengers=challengers,
incumbent=smac.solver.incumbent,
run_history=smac.solver.runhistory,
aggregate_func=smac.solver.aggregate_func,
time_bound=time_for_choose_next)
if smac.solver.scenario.shared_model:
pSMAC.write(run_history=smac.solver.runhistory,
output_directory=smac.solver.scenario.output_dir,
num_run=self.seed)
if smac.solver.stats.is_budget_exhausted():
break
self.runhistory = smac.solver.runhistory
self.trajectory = smac.solver.intensifier.traj_logger.trajectory
return self.runhistory, self.trajectory
def __init__(
self,
scenario: Scenario,
tae_runner: Optional[Union[Type[BaseRunner], Callable]] = None,
tae_runner_kwargs: Optional[Dict] = None,
runhistory: Optional[Union[Type[RunHistory], RunHistory]] = None,
runhistory_kwargs: Optional[Dict] = None,
intensifier: Optional[Type[AbstractRacer]] = None,
intensifier_kwargs: Optional[Dict] = None,
acquisition_function: Optional[
Type[AbstractAcquisitionFunction]] = None,
acquisition_function_kwargs: Optional[Dict] = None,
integrate_acquisition_function: bool = False,
acquisition_function_optimizer: Optional[
Type[AcquisitionFunctionMaximizer]] = None,
acquisition_function_optimizer_kwargs: Optional[Dict] = None,
model: Optional[Type[AbstractEPM]] = None,
model_kwargs: Optional[Dict] = None,
runhistory2epm: Optional[Type[AbstractRunHistory2EPM]] = None,
runhistory2epm_kwargs: Optional[Dict] = None,
multi_objective_algorithm: Optional[
Type[AbstractMultiObjectiveAlgorithm]] = None,
multi_objective_kwargs: Optional[Dict] = None,
initial_design: Optional[Type[InitialDesign]] = None,
initial_design_kwargs: Optional[Dict] = None,
initial_configurations: Optional[List[Configuration]] = None,
stats: Optional[Stats] = None,
restore_incumbent: Optional[Configuration] = None,
rng: Optional[Union[np.random.RandomState, int]] = None,
smbo_class: Optional[Type[SMBO]] = None,
run_id: Optional[int] = None,
random_configuration_chooser: Optional[
Type[RandomConfigurationChooser]] = None,
random_configuration_chooser_kwargs: Optional[Dict] = None,
dask_client: Optional[dask.distributed.Client] = None,
n_jobs: Optional[int] = 1,
):
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
self.scenario = scenario
self.output_dir = ""
if not restore_incumbent:
# restore_incumbent is used by the CLI interface which provides a method for restoring a SMAC run given an
# output directory. This is the default path.
# initial random number generator
run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger)
self.output_dir = create_output_directory(scenario, run_id)
elif scenario.output_dir is not None: # type: ignore[attr-defined] # noqa F821
run_id, rng = get_rng(rng=rng, run_id=run_id, logger=self.logger)
# output-directory is created in CLI when restoring from a
# folder. calling the function again in the facade results in two
# folders being created: run_X and run_X.OLD. if we are
# restoring, the output-folder exists already and we omit creating it,
# but set the self-output_dir to the dir.
# necessary because we want to write traj to new output-dir in CLI.
self.output_dir = cast(str, scenario.output_dir_for_this_run
) # type: ignore[attr-defined] # noqa F821
rng = cast(np.random.RandomState, rng)
if (scenario.deterministic is
True # type: ignore[attr-defined] # noqa F821
and getattr(scenario, "tuner_timeout", None) is None
and scenario.run_obj ==
"quality" # type: ignore[attr-defined] # noqa F821
):
self.logger.info(
"Optimizing a deterministic scenario for quality without a tuner timeout - will make "
"SMAC deterministic and only evaluate one configuration per iteration!"
)
scenario.intensification_percentage = 1e-10 # type: ignore[attr-defined] # noqa F821
scenario.min_chall = 1 # type: ignore[attr-defined] # noqa F821
scenario.write()
# initialize stats object
if stats:
self.stats = stats
else:
self.stats = Stats(scenario)
if self.scenario.run_obj == "runtime" and not self.scenario.transform_y == "LOG": # type: ignore[attr-defined] # noqa F821
self.logger.warning(
"Runtime as objective automatically activates log(y) transformation"
)
self.scenario.transform_y = "LOG" # type: ignore[attr-defined] # noqa F821
# initialize empty runhistory
num_obj = len(scenario.multi_objectives
) # type: ignore[attr-defined] # noqa F821
runhistory_def_kwargs = {}
if runhistory_kwargs is not None:
runhistory_def_kwargs.update(runhistory_kwargs)
if runhistory is None:
runhistory = RunHistory(**runhistory_def_kwargs)
elif inspect.isclass(runhistory):
runhistory = runhistory(
**runhistory_def_kwargs) # type: ignore[operator] # noqa F821
elif isinstance(runhistory, RunHistory):
pass
else:
raise ValueError(
"runhistory has to be a class or an object of RunHistory")
rand_conf_chooser_kwargs = {"rng": rng}
if random_configuration_chooser_kwargs is not None:
rand_conf_chooser_kwargs.update(
random_configuration_chooser_kwargs)
if random_configuration_chooser is None:
if "prob" not in rand_conf_chooser_kwargs:
rand_conf_chooser_kwargs[
"prob"] = scenario.rand_prob # type: ignore[attr-defined] # noqa F821
random_configuration_chooser_instance = ChooserProb(
**
rand_conf_chooser_kwargs # type: ignore[arg-type] # noqa F821 # type: RandomConfigurationChooser
)
elif inspect.isclass(random_configuration_chooser):
random_configuration_chooser_instance = random_configuration_chooser( # type: ignore # noqa F821
**
rand_conf_chooser_kwargs # type: ignore[arg-type] # noqa F821
)
elif not isinstance(random_configuration_chooser,
RandomConfigurationChooser):
raise ValueError(
"random_configuration_chooser has to be"
" a class or object of RandomConfigurationChooser")
# reset random number generator in config space to draw different
# random configurations with each seed given to SMAC
scenario.cs.seed(
rng.randint(MAXINT)) # type: ignore[attr-defined] # noqa F821
# initial Trajectory Logger
traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats)
# initial EPM
types, bounds = get_types(
scenario.cs,
scenario.feature_array) # type: ignore[attr-defined] # noqa F821
model_def_kwargs = {
"types": types,
"bounds": bounds,
"instance_features": scenario.feature_array,
"seed": rng.randint(MAXINT),
"pca_components": scenario.PCA_DIM,
}
if model_kwargs is not None:
model_def_kwargs.update(model_kwargs)
if model is None:
for key, value in {
"log_y": scenario.transform_y
in ["LOG",
"LOGS"], # type: ignore[attr-defined] # noqa F821
"num_trees": scenario.
rf_num_trees, # type: ignore[attr-defined] # noqa F821
"do_bootstrapping": scenario.
rf_do_bootstrapping, # type: ignore[attr-defined] # noqa F821
"ratio_features": scenario.
rf_ratio_features, # type: ignore[attr-defined] # noqa F821
"min_samples_split": scenario.
rf_min_samples_split, # type: ignore[attr-defined] # noqa F821
"min_samples_leaf": scenario.
rf_min_samples_leaf, # type: ignore[attr-defined] # noqa F821
"max_depth": scenario.
rf_max_depth, # type: ignore[attr-defined] # noqa F821
}.items():
if key not in model_def_kwargs:
model_def_kwargs[key] = value
model_def_kwargs[
"configspace"] = self.scenario.cs # type: ignore[attr-defined] # noqa F821
model_instance = RandomForestWithInstances(
**
model_def_kwargs # type: ignore[arg-type] # noqa F821 # type: AbstractEPM
)
elif inspect.isclass(model):
model_def_kwargs[
"configspace"] = self.scenario.cs # type: ignore[attr-defined] # noqa F821
model_instance = model(
**model_def_kwargs) # type: ignore # noqa F821
else:
raise TypeError("Model not recognized: %s" % (type(model)))
# initial acquisition function
acq_def_kwargs = {"model": model_instance}
if acquisition_function_kwargs is not None:
acq_def_kwargs.update(acquisition_function_kwargs)
acquisition_function_instance = (
None) # type: Optional[AbstractAcquisitionFunction]
if acquisition_function is None:
if scenario.transform_y in [
"LOG", "LOGS"
]: # type: ignore[attr-defined] # noqa F821
acquisition_function_instance = LogEI(
**acq_def_kwargs # type: ignore[arg-type] # noqa F821
)
else:
acquisition_function_instance = EI(
**acq_def_kwargs # type: ignore[arg-type] # noqa F821
)
elif inspect.isclass(acquisition_function):
acquisition_function_instance = acquisition_function(
**acq_def_kwargs)
else:
raise TypeError(
"Argument acquisition_function must be None or an object implementing the "
"AbstractAcquisitionFunction, not %s." %
type(acquisition_function))
if integrate_acquisition_function:
acquisition_function_instance = IntegratedAcquisitionFunction(
acquisition_function=
acquisition_function_instance, # type: ignore
**acq_def_kwargs,
)
# initialize optimizer on acquisition function
acq_func_opt_kwargs = {
"acquisition_function": acquisition_function_instance,
"config_space":
scenario.cs, # type: ignore[attr-defined] # noqa F821
"rng": rng,
}
if acquisition_function_optimizer_kwargs is not None:
acq_func_opt_kwargs.update(acquisition_function_optimizer_kwargs)
if acquisition_function_optimizer is None:
for key, value in {
"max_steps": scenario.
sls_max_steps, # type: ignore[attr-defined] # noqa F821
"n_steps_plateau_walk": scenario.
sls_n_steps_plateau_walk, # type: ignore[attr-defined] # noqa F821
}.items():
if key not in acq_func_opt_kwargs:
acq_func_opt_kwargs[key] = value
acquisition_function_optimizer_instance = LocalAndSortedRandomSearch(
**acq_func_opt_kwargs # type: ignore
)
elif inspect.isclass(acquisition_function_optimizer):
acquisition_function_optimizer_instance = acquisition_function_optimizer( # type: ignore # noqa F821
**acq_func_opt_kwargs) # type: ignore # noqa F821
else:
raise TypeError(
"Argument acquisition_function_optimizer must be None or an object implementing the "
"AcquisitionFunctionMaximizer, but is '%s'" %
type(acquisition_function_optimizer))
# initialize tae_runner
# First case, if tae_runner is None, the target algorithm is a call
# string in the scenario file
tae_def_kwargs = {
"stats": self.stats,
"run_obj": scenario.run_obj,
"par_factor":
scenario.par_factor, # type: ignore[attr-defined] # noqa F821
"cost_for_crash":
scenario.cost_for_crash, # type: ignore[attr-defined] # noqa F821
"abort_on_first_run_crash": scenario.
abort_on_first_run_crash, # type: ignore[attr-defined] # noqa F821
"multi_objectives": scenario.
multi_objectives, # type: ignore[attr-defined] # noqa F821
}
if tae_runner_kwargs is not None:
tae_def_kwargs.update(tae_runner_kwargs)
if "ta" not in tae_def_kwargs:
tae_def_kwargs[
"ta"] = scenario.ta # type: ignore[attr-defined] # noqa F821
if tae_runner is None:
tae_def_kwargs[
"ta"] = scenario.ta # type: ignore[attr-defined] # noqa F821
tae_runner_instance = ExecuteTARunOld(
**tae_def_kwargs
) # type: ignore[arg-type] # noqa F821 # type: BaseRunner
elif inspect.isclass(tae_runner):
tae_runner_instance = cast(
BaseRunner, tae_runner(**tae_def_kwargs)) # type: ignore
elif callable(tae_runner):
tae_def_kwargs["ta"] = tae_runner
tae_def_kwargs[
"use_pynisher"] = scenario.limit_resources # type: ignore[attr-defined] # noqa F821
tae_def_kwargs[
"memory_limit"] = scenario.memory_limit # type: ignore[attr-defined] # noqa F821
tae_runner_instance = ExecuteTAFuncDict(
**tae_def_kwargs) # type: ignore
else:
raise TypeError(
"Argument 'tae_runner' is %s, but must be "
"either None, a callable or an object implementing "
"BaseRunner. Passing 'None' will result in the "
"creation of target algorithm runner based on the "
"call string in the scenario file." % type(tae_runner))
# In case of a parallel run, wrap the single worker in a parallel
# runner
if n_jobs is None or n_jobs == 1:
_n_jobs = 1
elif n_jobs == -1:
_n_jobs = joblib.cpu_count()
elif n_jobs > 0:
_n_jobs = n_jobs
else:
raise ValueError(
"Number of tasks must be positive, None or -1, but is %s" %
str(n_jobs))
if _n_jobs > 1 or dask_client is not None:
tae_runner_instance = DaskParallelRunner( # type: ignore
tae_runner_instance,
n_workers=_n_jobs,
output_directory=self.output_dir,
dask_client=dask_client,
)
# Check that overall objective and tae objective are the same
# TODO: remove these two ignores once the scenario object knows all its attributes!
if tae_runner_instance.run_obj != scenario.run_obj: # type: ignore[union-attr] # noqa F821
raise ValueError(
"Objective for the target algorithm runner and "
"the scenario must be the same, but are '%s' and "
"'%s'" %
(tae_runner_instance.run_obj,
scenario.run_obj)) # type: ignore[union-attr] # noqa F821
if intensifier is None:
intensifier = Intensifier
if isinstance(intensifier, AbstractRacer):
intensifier_instance = intensifier
elif inspect.isclass(intensifier):
# initialize intensification
intensifier_def_kwargs = {
"stats": self.stats,
"traj_logger": traj_logger,
"rng": rng,
"instances":
scenario.train_insts, # type: ignore[attr-defined] # noqa F821
"cutoff":
scenario.cutoff, # type: ignore[attr-defined] # noqa F821
"deterministic": scenario.
deterministic, # type: ignore[attr-defined] # noqa F821
"run_obj_time": scenario.run_obj ==
"runtime", # type: ignore[attr-defined] # noqa F821
"instance_specifics": scenario.
instance_specific, # type: ignore[attr-defined] # noqa F821
"adaptive_capping_slackfactor": scenario.
intens_adaptive_capping_slackfactor, # type: ignore[attr-defined] # noqa F821
"min_chall": scenario.
intens_min_chall, # type: ignore[attr-defined] # noqa F821
}
if issubclass(intensifier, Intensifier):
intensifier_def_kwargs[
"always_race_against"] = scenario.cs.get_default_configuration(
) # type: ignore[attr-defined] # noqa F821
intensifier_def_kwargs[
"use_ta_time_bound"] = scenario.use_ta_time # type: ignore[attr-defined] # noqa F821
intensifier_def_kwargs[
"minR"] = scenario.minR # type: ignore[attr-defined] # noqa F821
intensifier_def_kwargs[
"maxR"] = scenario.maxR # type: ignore[attr-defined] # noqa F821
if intensifier_kwargs is not None:
intensifier_def_kwargs.update(intensifier_kwargs)
intensifier_instance = intensifier(
**intensifier_def_kwargs) # type: ignore[arg-type] # noqa F821
else:
raise TypeError(
"Argument intensifier must be None or an object implementing the AbstractRacer, but is '%s'"
% type(intensifier))
# initialize multi objective
# the multi_objective_algorithm_instance will be passed to the runhistory2epm object
multi_objective_algorithm_instance = (
None) # type: Optional[AbstractMultiObjectiveAlgorithm]
if scenario.multi_objectives is not None and num_obj > 1: # type: ignore[attr-defined] # noqa F821
# define any defaults here
_multi_objective_kwargs = {"rng": rng, "num_obj": num_obj}
if multi_objective_kwargs is not None:
_multi_objective_kwargs.update(multi_objective_kwargs)
if multi_objective_algorithm is None:
multi_objective_algorithm_instance = MeanAggregationStrategy(
**_multi_objective_kwargs
) # type: ignore[arg-type] # noqa F821
elif inspect.isclass(multi_objective_algorithm):
multi_objective_algorithm_instance = multi_objective_algorithm(
**_multi_objective_kwargs)
else:
raise TypeError(
"Multi-objective algorithm not recognized: %s" %
(type(multi_objective_algorithm)))
# initial design
if initial_design is not None and initial_configurations is not None:
raise ValueError(
"Either use initial_design or initial_configurations; but not both"
)
init_design_def_kwargs = {
"cs": scenario.cs, # type: ignore[attr-defined] # noqa F821
"traj_logger": traj_logger,
"rng": rng,
"ta_run_limit":
scenario.ta_run_limit, # type: ignore[attr-defined] # noqa F821
"configs": initial_configurations,
"n_configs_x_params": 0,
"max_config_fracs": 0.0,
}
if initial_design_kwargs is not None:
init_design_def_kwargs.update(initial_design_kwargs)
if initial_configurations is not None:
initial_design_instance = InitialDesign(**init_design_def_kwargs)
elif initial_design is None:
if scenario.initial_incumbent == "DEFAULT": # type: ignore[attr-defined] # noqa F821
init_design_def_kwargs["max_config_fracs"] = 0.0
initial_design_instance = DefaultConfiguration(
**init_design_def_kwargs)
elif scenario.initial_incumbent == "RANDOM": # type: ignore[attr-defined] # noqa F821
init_design_def_kwargs["max_config_fracs"] = 0.0
initial_design_instance = RandomConfigurations(
**init_design_def_kwargs)
elif scenario.initial_incumbent == "LHD": # type: ignore[attr-defined] # noqa F821
initial_design_instance = LHDesign(**init_design_def_kwargs)
elif scenario.initial_incumbent == "FACTORIAL": # type: ignore[attr-defined] # noqa F821
initial_design_instance = FactorialInitialDesign(
**init_design_def_kwargs)
elif scenario.initial_incumbent == "SOBOL": # type: ignore[attr-defined] # noqa F821
initial_design_instance = SobolDesign(**init_design_def_kwargs)
else:
raise ValueError("Don't know what kind of initial_incumbent "
"'%s' is" %
scenario.initial_incumbent # type: ignore
) # type: ignore[attr-defined] # noqa F821
elif inspect.isclass(initial_design):
initial_design_instance = initial_design(**init_design_def_kwargs)
else:
raise TypeError(
"Argument initial_design must be None or an object implementing the InitialDesign, but is '%s'"
% type(initial_design))
# if we log the performance data,
# the RFRImputator will already get
# log transform data from the runhistory
if scenario.transform_y in [
"LOG", "LOGS"
]: # type: ignore[attr-defined] # noqa F821
cutoff = np.log(np.nanmin([
np.inf, np.float_(scenario.cutoff)
])) # type: ignore[attr-defined] # noqa F821
threshold = cutoff + np.log(
scenario.par_factor) # type: ignore[attr-defined] # noqa F821
else:
cutoff = np.nanmin([np.inf, np.float_(scenario.cutoff)
]) # type: ignore[attr-defined] # noqa F821
threshold = cutoff * scenario.par_factor # type: ignore[attr-defined] # noqa F821
num_params = len(scenario.cs.get_hyperparameters()
) # type: ignore[attr-defined] # noqa F821
imputor = RFRImputator(
rng=rng,
cutoff=cutoff,
threshold=threshold,
model=model_instance,
change_threshold=0.01,
max_iter=2,
)
r2e_def_kwargs = {
"scenario": scenario,
"num_params": num_params,
"success_states": [
StatusType.SUCCESS,
],
"impute_censored_data": True,
"impute_state": [
StatusType.CAPPED,
],
"imputor": imputor,
"scale_perc": 5,
}
# TODO: consider other sorts of multi-objective algorithms
if isinstance(multi_objective_algorithm_instance, AggregationStrategy):
r2e_def_kwargs.update({
"multi_objective_algorithm":
multi_objective_algorithm_instance
})
if scenario.run_obj == "quality":
r2e_def_kwargs.update({
"success_states": [
StatusType.SUCCESS,
StatusType.CRASHED,
StatusType.MEMOUT,
],
"impute_censored_data":
False,
"impute_state":
None,
})
if (isinstance(intensifier_instance, (SuccessiveHalving, Hyperband))
and scenario.run_obj == "quality"):
r2e_def_kwargs.update({
"success_states": [
StatusType.SUCCESS,
StatusType.CRASHED,
StatusType.MEMOUT,
StatusType.DONOTADVANCE,
],
"consider_for_higher_budgets_state": [
StatusType.DONOTADVANCE,
StatusType.TIMEOUT,
StatusType.CRASHED,
StatusType.MEMOUT,
],
})
if runhistory2epm_kwargs is not None:
r2e_def_kwargs.update(runhistory2epm_kwargs)
if runhistory2epm is None:
if scenario.run_obj == "runtime":
rh2epm = RunHistory2EPM4LogCost(
**r2e_def_kwargs # type: ignore
) # type: ignore[arg-type] # noqa F821 # type: AbstractRunHistory2EPM
elif scenario.run_obj == "quality":
if scenario.transform_y == "NONE": # type: ignore[attr-defined] # noqa F821
rh2epm = RunHistory2EPM4Cost(
**r2e_def_kwargs) # type: ignore # noqa F821
elif scenario.transform_y == "LOG": # type: ignore[attr-defined] # noqa F821
rh2epm = RunHistory2EPM4LogCost(
**r2e_def_kwargs) # type: ignore # noqa F821
elif scenario.transform_y == "LOGS": # type: ignore[attr-defined] # noqa F821
rh2epm = RunHistory2EPM4LogScaledCost(
**r2e_def_kwargs) # type: ignore # noqa F821
elif scenario.transform_y == "INVS": # type: ignore[attr-defined] # noqa F821
rh2epm = RunHistory2EPM4InvScaledCost(
**r2e_def_kwargs) # type: ignore # noqa F821
else:
raise ValueError(
"Unknown run objective: %s. Should be either "
"quality or runtime." %
self.scenario.run_obj # type: ignore # noqa F821
)
elif inspect.isclass(runhistory2epm):
rh2epm = runhistory2epm(**
r2e_def_kwargs) # type: ignore # noqa F821
else:
raise TypeError(
"Argument runhistory2epm must be None or an object implementing the RunHistory2EPM, but is '%s'"
% type(runhistory2epm))
smbo_args = {
"scenario": scenario,
"stats": self.stats,
"initial_design": initial_design_instance,
"runhistory": runhistory,
"runhistory2epm": rh2epm,
"intensifier": intensifier_instance,
"num_run": run_id,
"model": model_instance,
"acq_optimizer": acquisition_function_optimizer_instance,
"acquisition_func": acquisition_function_instance,
"rng": rng,
"restore_incumbent": restore_incumbent,
"random_configuration_chooser":
random_configuration_chooser_instance,
"tae_runner": tae_runner_instance,
} # type: Dict[str, Any]
if smbo_class is None:
self.solver = SMBO(**
smbo_args) # type: ignore[arg-type] # noqa F821
else:
self.solver = smbo_class(
**smbo_args) # type: ignore[arg-type] # noqa F821
def create_optimizer(self):
from smac.epm.rf_with_instances import RandomForestWithInstances
from smac.initial_design.default_configuration_design import DefaultConfiguration
from smac.intensification.intensification import Intensifier
from smac.optimizer.smbo import SMBO
from smac.optimizer.acquisition import EI
from smac.optimizer.ei_optimization import InterleavedLocalAndRandomSearch
from smac.optimizer.objective import average_cost
from smac.runhistory.runhistory2epm import RunHistory2EPM4Cost
from smac.tae.execute_ta_run import StatusType
from smac.utils.constants import MAXINT
from smac.utils.util_funcs import get_types
TAE_RUNNER = self._priv_evaluator
runhistory2epm = RunHistory2EPM4Cost(
scenario=self.scenario,
num_params=len(self.param_space),
success_states=[StatusType.SUCCESS, StatusType.CRASHED],
impute_censored_data=False,
impute_state=None)
intensifier = Intensifier(tae_runner=TAE_RUNNER, stats=self.stats, traj_logger=self.traj_logger,
rng=self.rng, instances=self.scenario.train_insts, cutoff=self.scenario.cutoff,
deterministic=self.scenario.deterministic, run_obj_time=self.scenario.run_obj == "runtime",
always_race_against=self.scenario.cs.get_default_configuration() \
if self.scenario.always_race_default else None,
instance_specifics=self.scenario.instance_specific,
minR=self.scenario.minR, maxR=self.scenario.maxR)
types, bounds = get_types(self.scenario.cs,
self.scenario.feature_array)
model = RandomForestWithInstances(
types=types,
bounds=bounds,
seed=self.rng.randint(MAXINT),
instance_features=self.scenario.feature_array,
pca_components=self.scenario.PCA_DIM)
acq_func = EI(model=model)
smbo_args = {
'scenario':
self.scenario,
'stats':
self.stats,
'initial_design':
DefaultConfiguration(tae_runner=TAE_RUNNER,
scenario=self.scenario,
stats=self.stats,
traj_logger=self.traj_logger,
rng=self.rng),
'runhistory':
self.runhistory,
'runhistory2epm':
runhistory2epm,
'intensifier':
intensifier,
'aggregate_func':
average_cost,
'num_run':
self.seed,
'model':
model,
'acq_optimizer':
InterleavedLocalAndRandomSearch(
acq_func, self.scenario.cs,
np.random.RandomState(seed=self.rng.randint(MAXINT))),
'acquisition_func':
acq_func,
'rng':
self.rng,
'restore_incumbent':
None,
}
self.smbo = SMBO(**smbo_args)
