def test_local_search_finds_minimum(self):
class AcquisitionFunction:
model = None
def __call__(self, arrays):
rval = []
for array in arrays:
rval.append([-rosenbrock_4d(array)])
return np.array(rval)
ls = LocalSearch(
acquisition_function=AcquisitionFunction(),
config_space=self.cs,
n_steps_plateau_walk=10,
max_steps=np.inf,
)
runhistory = RunHistory()
self.cs.seed(1)
random_configs = self.cs.sample_configuration(size=100)
costs = [
rosenbrock_4d(random_config) for random_config in random_configs
]
self.assertGreater(np.min(costs), 100)
for random_config, cost in zip(random_configs, costs):
runhistory.add(config=random_config,
cost=cost,
time=0,
status=StatusType.SUCCESS)
minimizer = ls.maximize(runhistory, None, 10)
minima = [-rosenbrock_4d(m) for m in minimizer]
self.assertGreater(minima[0], -0.05)
def test_get_initial_points_moo(self):
class Model:
def predict_marginalized_over_instances(self, X):
return X, X
class AcquisitionFunction:
model = Model()
def __call__(self, X):
return np.array([x.get_array().sum() for x in X]).reshape(
(-1, 1))
ls = LocalSearch(
acquisition_function=AcquisitionFunction(),
config_space=self.cs,
n_steps_plateau_walk=10,
max_steps=np.inf,
)
runhistory = RunHistory()
random_configs = self.cs.sample_configuration(size=100)
costs = np.array(
[rosenbrock_4d(random_config) for random_config in random_configs])
for random_config, cost in zip(random_configs, costs):
runhistory.add(config=random_config,
cost=cost,
time=0,
status=StatusType.SUCCESS)
points = ls._get_initial_points(num_points=5,
runhistory=runhistory,
additional_start_points=None)
self.assertEqual(len(points), 10)
def test_local_search_2(
self,
_calculate_num_points_patch,
_get_initial_points_patch,
):
pcs_file = os.path.join(self.test_files_dir, "test_local_search.pcs")
seed = np.random.randint(1, 100000)
with open(pcs_file) as fh:
config_space = pcs.read(fh.readlines())
config_space.seed(seed)
def acquisition_function(point):
return np.array([np.count_nonzero(point[0].get_array())])
start_point = config_space.get_default_configuration()
_calculate_num_points_patch.return_value = 1
_get_initial_points_patch.return_value = [start_point]
l = LocalSearch(acquisition_function,
config_space,
epsilon=0.01,
max_iterations=100000)
acq_val_incumbent, incumbent = l._maximize(None, None, 10)[0]
np.testing.assert_allclose(
incumbent.get_array(),
np.ones(len(config_space.get_hyperparameters())))
def test_local_search_2(
self,
_get_initial_points_patch,
):
pcs_file = os.path.join(self.test_files_dir, "test_local_search.pcs")
seed = np.random.randint(1, 100000)
runhistory = unittest.mock.Mock()
runhistory.data = [None] * 1000
with open(pcs_file) as fh:
config_space = pcs.read(fh.readlines())
config_space.seed(seed)
def acquisition_function(points):
return np.array([[np.count_nonzero(point.get_array())]
for point in points])
start_point = config_space.get_default_configuration()
_get_initial_points_patch.return_value = [start_point]
ls = LocalSearch(acquisition_function, config_space, max_steps=100000)
# To have some data in a mock runhistory
ls.runhistory = [None] * 1000
acq_val_incumbent, incumbent = ls._maximize(runhistory, None, 1)[0]
np.testing.assert_allclose(
incumbent.get_array(),
np.ones(len(config_space.get_hyperparameters())))
def __init__(
self,
acquisition_function: AbstractAcquisitionFunction,
config_space: ConfigurationSpace,
rng: Union[bool, np.random.RandomState] = None,
):
super().__init__(acquisition_function, config_space, rng)
self.random_search = RandomSearch(acquisition_function, config_space,
rng)
self.local_search = LocalSearch(acquisition_function, config_space,
rng)
self.max_acq_value = sys.float_info.min
def test_local_search(self):
def acquisition_function(point):
point = [p.get_array() for p in point]
opt = np.array([1, 1, 1, 1])
dist = [euclidean(point, opt)]
return np.array([-np.min(dist)])
l = LocalSearch(acquisition_function, self.cs, max_steps=100000)
start_point = self.cs.sample_configuration()
acq_val_start_point = acquisition_function([start_point])
acq_val_incumbent, _ = l._one_iter(start_point)
# Local search needs to find something that is as least as good as the
# start point
self.assertLessEqual(acq_val_start_point, acq_val_incumbent)
def test_get_next_by_local_search(
self,
_get_initial_points_patch,
patch
):
# Without known incumbent
class SideEffect(object):
def __call__(self, *args, **kwargs):
rval = []
for i in range(len(args[0])):
rval.append((i, ConfigurationMock(i)))
return rval
patch.side_effect = SideEffect()
cs = test_helpers.get_branin_config_space()
rand_confs = cs.sample_configuration(size=9)
_get_initial_points_patch.return_value = rand_confs
acq_func = EI(None)
ls = LocalSearch(acq_func, cs)
# To have some data in a mock runhistory
runhistory = unittest.mock.Mock()
runhistory.data = [None] * 1000
rval = ls._maximize(runhistory, None, 9)
self.assertEqual(len(rval), 9)
self.assertEqual(patch.call_count, 1)
for i in range(9):
self.assertIsInstance(rval[i][1], ConfigurationMock)
self.assertEqual(rval[i][1].value, 8 - i)
self.assertEqual(rval[i][0], 8 - i)
self.assertEqual(rval[i][1].origin, 'Local Search')
# Check that the known 'incumbent' is transparently passed through
patch.side_effect = SideEffect()
_get_initial_points_patch.return_value = ['Incumbent'] + rand_confs
rval = ls._maximize(runhistory, None, 10)
self.assertEqual(len(rval), 10)
self.assertEqual(patch.call_count, 2)
# Only the first local search in each iteration starts from the
# incumbent
self.assertEqual(patch.call_args_list[1][0][0][0], 'Incumbent')
for i in range(10):
self.assertEqual(rval[i][1].origin, 'Local Search')
def test_local_search(self):
def acquisition_function(points):
rval = []
for point in points:
opt = np.array([1, 1, 1, 1])
rval.append([-euclidean(point.get_array(), opt)])
return np.array(rval)
ls = LocalSearch(acquisition_function, self.cs, max_steps=100)
start_point = self.cs.sample_configuration()
acq_val_start_point = acquisition_function([start_point])
acq_val_incumbent, _ = ls._do_search(start_point)[0]
# Local search needs to find something that is as least as good as the
# start point
self.assertLessEqual(acq_val_start_point, acq_val_incumbent)
def test_get_next_by_local_search(self, _get_initial_points_patch, patch):
# Without known incumbent
class SideEffect(object):
def __init__(self):
self.call_number = 0
def __call__(self, *args, **kwargs):
rval = 9 - self.call_number
self.call_number += 1
return (rval, ConfigurationMock(rval))
patch.side_effect = SideEffect()
cs = test_helpers.get_branin_config_space()
rand_confs = cs.sample_configuration(size=9)
_get_initial_points_patch.return_value = rand_confs
acq_func = EI(None)
ls = LocalSearch(acq_func, cs)
# To have some data in a mock runhistory
runhistory = unittest.mock.Mock()
runhistory.data = [None] * 1000
rval = ls._maximize(runhistory, None, 9)
self.assertEqual(len(rval), 9)
self.assertEqual(patch.call_count, 9)
for i in range(9):
self.assertIsInstance(rval[i][1], ConfigurationMock)
self.assertEqual(rval[i][1].value, 9 - i)
self.assertEqual(rval[i][0], 9 - i)
self.assertEqual(rval[i][1].origin, 'Local Search')
# With known incumbent
patch.side_effect = SideEffect()
_get_initial_points_patch.return_value = ['Incumbent'] + rand_confs
rval = ls._maximize(runhistory, None, 10)
self.assertEqual(len(rval), 10)
self.assertEqual(patch.call_count, 19)
# Only the first local search in each iteration starts from the
# incumbent
self.assertEqual(patch.call_args_list[9][0][0], 'Incumbent')
for i in range(10):
self.assertEqual(rval[i][1].origin, 'Local Search')
def __init__(
self,
configspace,
random_fraction=1 / 2,
logger=None,
configs=None,
losses=None,
):
self.logger = logger
self.random_fraction = random_fraction
self.configspace = configspace
self.min_points_in_model = len(self.configspace.get_hyperparameters())
rng = np.random.RandomState(random.randint(0, 100))
self.model = _construct_model(configspace, rng)
self.acquisition_func = EI(model=self.model)
self.acq_optimizer = LocalSearch(
acquisition_function=self.acquisition_func,
config_space=configspace,
rng=rng)
self.runhistory = RunHistory()
self.configs = configs or list()
self.losses = losses or list()
if self.has_model:
for config, cost in zip(self.configs, self.losses):
self.runhistory.add(config, cost, 0, StatusType.SUCCESS)
X = convert_configurations_to_array(self.configs)
Y = np.array(self.losses, dtype=np.float64)
self.model.train(X, Y)
self.acquisition_func.update(
model=self.model,
eta=min(self.losses),
)
def __init__(
self,
configspace,
random_fraction=1 / 2,
logger=None,
previous_results=None
):
self.logger = logger
self.random_fraction = random_fraction
self.runhistory = RunHistory()
self.configs = list()
self.losses = list()
rng = np.random.RandomState(random.randint(0, 100))
if previous_results is not None and len(previous_results.batch_results) > 0:
# Assume same-task changing-configspace trajectory for now
results_previous_adjustment = previous_results.batch_results[-1]
configspace_previous = results_previous_adjustment.configspace
# Construct combined config space
configspace_combined = ConfigSpace.ConfigurationSpace()
development_step = CSH.CategoricalHyperparameter("development_step", choices=["old", "new"])
configspace_combined.add_hyperparameter(
development_step
)
configspace_only_old, configspace_both, configspace_only_new = get_configspace_partitioning_cond(configspace, configspace_previous)
configspace_combined.add_hyperparameters(configspace_both.get_hyperparameters())
configspace_combined.add_hyperparameters(configspace_only_old.get_hyperparameters())
configspace_combined.add_hyperparameters(configspace_only_new.get_hyperparameters())
for hyperparameter in configspace_only_old.get_hyperparameters():
configspace_combined.add_condition(
ConfigSpace.EqualsCondition(hyperparameter, development_step, "old")
)
for hyperparameter in configspace_only_new.get_hyperparameters():
configspace_combined.add_condition(
ConfigSpace.EqualsCondition(hyperparameter, development_step, "new")
)
# Read old configs and losses
result_previous = results_previous_adjustment.results[0]
all_runs = result_previous.get_all_runs(only_largest_budget=False)
self.losses_old = [run.loss for run in all_runs]
self.configs_old = [run.config_id for run in all_runs]
id2conf = result_previous.get_id2config_mapping()
self.configs_old = [id2conf[id_]["config"] for id_ in self.configs_old]
# Map old configs to combined space
for config in self.configs_old:
config["development_step"] = "old"
self.configs_old = [ConfigSpace.Configuration(configspace_combined, config) for config in self.configs_old]
for config, cost in zip(self.configs_old, self.losses_old):
self.runhistory.add(config, cost, 0, StatusType.SUCCESS)
# Construct and fit model
self.configspace = configspace_combined
self.model = _construct_model(self.configspace, rng)
self.acquisition_func = EI(model=self.model)
self.acq_optimizer = LocalSearch(acquisition_function=self.acquisition_func,
config_space=self.configspace, rng=rng)
X = convert_configurations_to_array(self.configs_old)
Y = np.array(self.losses_old, dtype=np.float64)
self.model.train(X, Y)
self.acquisition_func.update(
model=self.model,
eta=min(self.losses_old),
)
else:
self.configspace = configspace
self.model = _construct_model(self.configspace, rng)
self.acquisition_func = EI(model=self.model)
self.acq_optimizer = LocalSearch(acquisition_function=self.acquisition_func,
config_space=self.configspace, rng=rng)
self.min_points_in_model = len(self.configspace.get_hyperparameters()) # TODO
class GPCondSampler:
def __init__(
self,
configspace,
random_fraction=1 / 2,
logger=None,
previous_results=None
):
self.logger = logger
self.random_fraction = random_fraction
self.runhistory = RunHistory()
self.configs = list()
self.losses = list()
rng = np.random.RandomState(random.randint(0, 100))
if previous_results is not None and len(previous_results.batch_results) > 0:
# Assume same-task changing-configspace trajectory for now
results_previous_adjustment = previous_results.batch_results[-1]
configspace_previous = results_previous_adjustment.configspace
# Construct combined config space
configspace_combined = ConfigSpace.ConfigurationSpace()
development_step = CSH.CategoricalHyperparameter("development_step", choices=["old", "new"])
configspace_combined.add_hyperparameter(
development_step
)
configspace_only_old, configspace_both, configspace_only_new = get_configspace_partitioning_cond(configspace, configspace_previous)
configspace_combined.add_hyperparameters(configspace_both.get_hyperparameters())
configspace_combined.add_hyperparameters(configspace_only_old.get_hyperparameters())
configspace_combined.add_hyperparameters(configspace_only_new.get_hyperparameters())
for hyperparameter in configspace_only_old.get_hyperparameters():
configspace_combined.add_condition(
ConfigSpace.EqualsCondition(hyperparameter, development_step, "old")
)
for hyperparameter in configspace_only_new.get_hyperparameters():
configspace_combined.add_condition(
ConfigSpace.EqualsCondition(hyperparameter, development_step, "new")
)
# Read old configs and losses
result_previous = results_previous_adjustment.results[0]
all_runs = result_previous.get_all_runs(only_largest_budget=False)
self.losses_old = [run.loss for run in all_runs]
self.configs_old = [run.config_id for run in all_runs]
id2conf = result_previous.get_id2config_mapping()
self.configs_old = [id2conf[id_]["config"] for id_ in self.configs_old]
# Map old configs to combined space
for config in self.configs_old:
config["development_step"] = "old"
self.configs_old = [ConfigSpace.Configuration(configspace_combined, config) for config in self.configs_old]
for config, cost in zip(self.configs_old, self.losses_old):
self.runhistory.add(config, cost, 0, StatusType.SUCCESS)
# Construct and fit model
self.configspace = configspace_combined
self.model = _construct_model(self.configspace, rng)
self.acquisition_func = EI(model=self.model)
self.acq_optimizer = LocalSearch(acquisition_function=self.acquisition_func,
config_space=self.configspace, rng=rng)
X = convert_configurations_to_array(self.configs_old)
Y = np.array(self.losses_old, dtype=np.float64)
self.model.train(X, Y)
self.acquisition_func.update(
model=self.model,
eta=min(self.losses_old),
)
else:
self.configspace = configspace
self.model = _construct_model(self.configspace, rng)
self.acquisition_func = EI(model=self.model)
self.acq_optimizer = LocalSearch(acquisition_function=self.acquisition_func,
config_space=self.configspace, rng=rng)
self.min_points_in_model = len(self.configspace.get_hyperparameters()) # TODO
@property
def has_model(self):
return len(self.configs) >= self.min_points_in_model
def get_config(self, budget): # pylint: disable=unused-argument
self.logger.debug("start sampling a new configuration.")
is_random_fraction = np.random.rand() < self.random_fraction
if is_random_fraction or not self.has_model:
if "development_step" in self.configspace.get_hyperparameter_names():
while True:
sample = self.configspace.sample_configuration()
if sample["development_step"] == "new":
break
else:
sample = self.configspace.sample_configuration()
else:
# Use private _maximize to not return challenger list object
sample = self.acq_optimizer._maximize(
runhistory=self.runhistory,
stats=None,
num_points=1,
)
sample = sample[0][1]
sample = ConfigSpace.util.deactivate_inactive_hyperparameters(sample.get_dictionary(), self.configspace)
sample = sample.get_dictionary()
info = {}
self.logger.debug("done sampling a new configuration.")
return sample, info
def new_result(self, job, config_info): # pylint: disable=unused-argument
if job.exception is not None:
self.logger.warning(f"job {job.id} failed with exception\n{job.exception}")
if job.result is None:
# One could skip crashed results, but we decided to
# assign a +inf loss and count them as bad configurations
loss = np.inf
else:
# same for non numeric losses.
# Note that this means losses of minus infinity will count as bad!
loss = job.result["loss"] if np.isfinite(job.result["loss"]) else np.inf
config = ConfigSpace.Configuration(self.configspace, job.kwargs["config"])
self.configs.append(config)
self.losses.append(loss)
if self.has_model:
# TODO: include old
X = convert_configurations_to_array(self.configs)
Y = np.array(self.losses, dtype=np.float64)
self.model.train(X, Y)
self.acquisition_func.update(
model=self.model,
eta=min(self.losses),
)
def __init__(
self,
scenario: Scenario,
# TODO: once we drop python3.4 add type hint
# typing.Union[ExecuteTARun, callable]
tae_runner=None,
runhistory: RunHistory = None,
intensifier: Intensifier = None,
acquisition_function: AbstractAcquisitionFunction = None,
model: AbstractEPM = None,
runhistory2epm: AbstractRunHistory2EPM = None,
initial_design: InitialDesign = None,
initial_configurations: typing.List[Configuration] = None,
stats: Stats = None,
rng: np.random.RandomState = None,
run_id: int = 1):
"""Constructor"""
self.logger = logging.getLogger(self.__module__ + "." +
self.__class__.__name__)
aggregate_func = average_cost
self.runhistory = None
self.trajectory = None
# initialize stats object
if stats:
self.stats = stats
else:
self.stats = Stats(scenario)
self.output_dir = create_output_directory(scenario, run_id)
scenario.write()
# initialize empty runhistory
if runhistory is None:
runhistory = RunHistory(aggregate_func=aggregate_func)
# inject aggr_func if necessary
if runhistory.aggregate_func is None:
runhistory.aggregate_func = aggregate_func
# initial random number generator
num_run, rng = self._get_rng(rng=rng)
# reset random number generator in config space to draw different
# random configurations with each seed given to SMAC
scenario.cs.seed(rng.randint(MAXINT))
# initial Trajectory Logger
traj_logger = TrajLogger(output_dir=self.output_dir, stats=self.stats)
# initial EPM
types, bounds = get_types(scenario.cs, scenario.feature_array)
if model is None:
model = RandomForestWithInstances(
types=types,
bounds=bounds,
instance_features=scenario.feature_array,
seed=rng.randint(MAXINT),
pca_components=scenario.PCA_DIM,
num_trees=scenario.rf_num_trees,
do_bootstrapping=scenario.rf_do_bootstrapping,
ratio_features=scenario.rf_ratio_features,
min_samples_split=scenario.rf_min_samples_split,
min_samples_leaf=scenario.rf_min_samples_leaf,
max_depth=scenario.rf_max_depth)
# initial acquisition function
if acquisition_function is None:
if scenario.run_obj == "runtime":
acquisition_function = LogEI(model=model)
else:
acquisition_function = EI(model=model)
# inject model if necessary
if acquisition_function.model is None:
acquisition_function.model = model
# initialize optimizer on acquisition function
local_search = LocalSearch(
acquisition_function,
scenario.cs,
max_steps=scenario.sls_max_steps,
n_steps_plateau_walk=scenario.sls_n_steps_plateau_walk)
# initialize tae_runner
# First case, if tae_runner is None, the target algorithm is a call
# string in the scenario file
if tae_runner is None:
tae_runner = ExecuteTARunOld(
ta=scenario.ta,
stats=self.stats,
run_obj=scenario.run_obj,
runhistory=runhistory,
par_factor=scenario.par_factor,
cost_for_crash=scenario.cost_for_crash)
# Second case, the tae_runner is a function to be optimized
elif callable(tae_runner):
tae_runner = ExecuteTAFuncDict(
ta=tae_runner,
stats=self.stats,
run_obj=scenario.run_obj,
memory_limit=scenario.memory_limit,
runhistory=runhistory,
par_factor=scenario.par_factor,
cost_for_crash=scenario.cost_for_crash)
# Third case, if it is an ExecuteTaRun we can simply use the
# instance. Otherwise, the next check raises an exception
elif not isinstance(tae_runner, ExecuteTARun):
raise TypeError("Argument 'tae_runner' is %s, but must be "
"either a callable or an instance of "
"ExecuteTaRun. Passing 'None' will result in the "
"creation of target algorithm runner based on the "
"call string in the scenario file." %
type(tae_runner))
# Check that overall objective and tae objective are the same
if tae_runner.run_obj != scenario.run_obj:
raise ValueError("Objective for the target algorithm runner and "
"the scenario must be the same, but are '%s' and "
"'%s'" % (tae_runner.run_obj, scenario.run_obj))
# inject stats if necessary
if tae_runner.stats is None:
tae_runner.stats = self.stats
# inject runhistory if necessary
if tae_runner.runhistory is None:
tae_runner.runhistory = runhistory
# inject cost_for_crash
if tae_runner.crash_cost != scenario.cost_for_crash:
tae_runner.crash_cost = scenario.cost_for_crash
# initialize intensification
if intensifier is None:
intensifier = Intensifier(
tae_runner=tae_runner,
stats=self.stats,
traj_logger=traj_logger,
rng=rng,
instances=scenario.train_insts,
cutoff=scenario.cutoff,
deterministic=scenario.deterministic,
run_obj_time=scenario.run_obj == "runtime",
always_race_against=scenario.cs.get_default_configuration()
if scenario.always_race_default else None,
instance_specifics=scenario.instance_specific,
minR=scenario.minR,
maxR=scenario.maxR,
adaptive_capping_slackfactor=scenario.
intens_adaptive_capping_slackfactor,
min_chall=scenario.intens_min_chall)
# inject deps if necessary
if intensifier.tae_runner is None:
intensifier.tae_runner = tae_runner
if intensifier.stats is None:
intensifier.stats = self.stats
if intensifier.traj_logger is None:
intensifier.traj_logger = traj_logger
# initial design
if initial_design is not None and initial_configurations is not None:
raise ValueError(
"Either use initial_design or initial_configurations; but not both"
)
if initial_configurations is not None:
initial_design = MultiConfigInitialDesign(
tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
runhistory=runhistory,
rng=rng,
configs=initial_configurations,
intensifier=intensifier,
aggregate_func=aggregate_func)
elif initial_design is None:
if scenario.initial_incumbent == "DEFAULT":
initial_design = DefaultConfiguration(tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
elif scenario.initial_incumbent == "RANDOM":
initial_design = RandomConfiguration(tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
else:
raise ValueError("Don't know what kind of initial_incumbent "
"'%s' is" % scenario.initial_incumbent)
# inject deps if necessary
if initial_design.tae_runner is None:
initial_design.tae_runner = tae_runner
if initial_design.scenario is None:
initial_design.scenario = scenario
if initial_design.stats is None:
initial_design.stats = self.stats
if initial_design.traj_logger is None:
initial_design.traj_logger = traj_logger
# initial conversion of runhistory into EPM data
if runhistory2epm is None:
num_params = len(scenario.cs.get_hyperparameters())
if scenario.run_obj == "runtime":
# if we log the performance data,
# the RFRImputator will already get
# log transform data from the runhistory
cutoff = np.log(scenario.cutoff)
threshold = np.log(scenario.cutoff * scenario.par_factor)
imputor = RFRImputator(rng=rng,
cutoff=cutoff,
threshold=threshold,
model=model,
change_threshold=0.01,
max_iter=2)
runhistory2epm = RunHistory2EPM4LogCost(
scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=True,
impute_state=[
StatusType.CAPPED,
],
imputor=imputor)
elif scenario.run_obj == 'quality':
runhistory2epm = RunHistory2EPM4Cost(
scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=False,
impute_state=None)
else:
raise ValueError('Unknown run objective: %s. Should be either '
'quality or runtime.' % self.scenario.run_obj)
# inject scenario if necessary:
if runhistory2epm.scenario is None:
runhistory2epm.scenario = scenario
self.solver = EPILS_Solver(scenario=scenario,
stats=self.stats,
initial_design=initial_design,
runhistory=runhistory,
runhistory2epm=runhistory2epm,
intensifier=intensifier,
aggregate_func=aggregate_func,
num_run=num_run,
model=model,
acq_optimizer=local_search,
acquisition_func=acquisition_function,
rng=rng)
class GPSampler:
def __init__(
self,
configspace,
random_fraction=1 / 2,
logger=None,
configs=None,
losses=None,
):
self.logger = logger
self.random_fraction = random_fraction
self.configspace = configspace
self.min_points_in_model = len(self.configspace.get_hyperparameters())
rng = np.random.RandomState(random.randint(0, 100))
self.model = _construct_model(configspace, rng)
self.acquisition_func = EI(model=self.model)
self.acq_optimizer = LocalSearch(
acquisition_function=self.acquisition_func,
config_space=configspace,
rng=rng)
self.runhistory = RunHistory()
self.configs = configs or list()
self.losses = losses or list()
if self.has_model:
for config, cost in zip(self.configs, self.losses):
self.runhistory.add(config, cost, 0, StatusType.SUCCESS)
X = convert_configurations_to_array(self.configs)
Y = np.array(self.losses, dtype=np.float64)
self.model.train(X, Y)
self.acquisition_func.update(
model=self.model,
eta=min(self.losses),
)
@property
def has_model(self):
return len(self.configs) >= self.min_points_in_model
def get_config(self, budget): # pylint: disable=unused-argument
self.logger.debug("start sampling a new configuration.")
is_random_fraction = np.random.rand() < self.random_fraction
if is_random_fraction:
sample = self.configspace.sample_configuration()
elif self.has_model:
# Use private _maximize to not return challenger list object
sample = self.acq_optimizer._maximize(
runhistory=self.runhistory,
stats=None,
num_points=1,
)
sample = sample[0][1]
else:
sample = self.configspace.sample_configuration()
sample = sample.get_dictionary()
info = {}
self.logger.debug("done sampling a new configuration.")
return sample, info
def new_result(self, job, config_info): # pylint: disable=unused-argument
if job.exception is not None:
self.logger.warning(
f"job {job.id} failed with exception\n{job.exception}")
if job.result is None:
# One could skip crashed results, but we decided to
# assign a +inf loss and count them as bad configurations
loss = np.inf
else:
# same for non numeric losses.
# Note that this means losses of minus infinity will count as bad!
loss = job.result["loss"] if np.isfinite(
job.result["loss"]) else np.inf
config = ConfigSpace.Configuration(self.configspace,
job.kwargs["config"])
self.configs.append(config)
self.losses.append(loss)
self.runhistory.add(config, loss, 0, StatusType.SUCCESS)
if self.has_model:
X = convert_configurations_to_array(self.configs)
Y = np.array(self.losses, dtype=np.float64)
self.model.train(X, Y)
self.acquisition_func.update(
model=self.model,
eta=min(self.losses),
)
class InterleavedLocalAndRandomSearch(AcquisitionFunctionMaximizer):
"""Implements SMAC's default acquisition function optimization.
This optimizer performs local search from the previous best points
according, to the acquisition function, uses the acquisition function to
sort randomly sampled configurations and interleaves unsorted, randomly
sampled configurations in between.
Parameters
----------
acquisition_function : ~smac.optimizer.acquisition.AbstractAcquisitionFunction
config_space : ~smac.configspace.ConfigurationSpace
rng : np.random.RandomState or int, optional
"""
def __init__(
self,
acquisition_function: AbstractAcquisitionFunction,
config_space: ConfigurationSpace,
rng: Union[bool, np.random.RandomState] = None,
):
super().__init__(acquisition_function, config_space, rng)
self.random_search = RandomSearch(acquisition_function, config_space,
rng)
self.local_search = LocalSearch(acquisition_function, config_space,
rng)
self.max_acq_value = sys.float_info.min
def maximize(self, runhistory: RunHistory, stats: Stats, num_points: int,
*args) -> Iterable[Configuration]:
next_configs_by_local_search = self.local_search._maximize(
runhistory,
stats,
10,
)
# Get configurations sorted by EI
next_configs_by_random_search_sorted = self.random_search._maximize(
runhistory,
stats,
num_points - len(next_configs_by_local_search),
_sorted=True,
)
# Having the configurations from random search, sorted by their
# acquisition function value is important for the first few iterations
# of SMAC. As long as the random forest predicts constant value, we
# want to use only random configurations. Having them at the begging of
# the list ensures this (even after adding the configurations by local
# search, and then sorting them)
next_configs_by_acq_value = (next_configs_by_random_search_sorted +
next_configs_by_local_search)
next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0])
self.logger.debug(
"First 10 acq func (origin) values of selected configurations: %s",
str([[_[0], _[1].origin] for _ in next_configs_by_acq_value[:10]]))
# store the max last expansion (challengers generation)
self.max_acq_value = next_configs_by_acq_value[0][0]
next_configs_by_acq_value = [_[1] for _ in next_configs_by_acq_value]
challengers = ChallengerList(next_configs_by_acq_value,
self.config_space)
return challengers
def _maximize(self, runhistory: RunHistory, stats: Stats,
num_points: int) -> Iterable[Tuple[float, Configuration]]:
raise NotImplementedError()
