def main_cli(self):
'''
main function of SMAC for CLI interface
'''
cmd_reader = CMDReader()
args_, misc_args = cmd_reader.read_cmd()
logging.basicConfig(level=args_.verbose_level)
root_logger = logging.getLogger()
root_logger.setLevel(args_.verbose_level)
scen = Scenario(args_.scenario_file, misc_args)
try:
smbo = SMBO(scenario=scen, rng=np.random.RandomState(args_.seed))
smbo.run(max_iters=args_.max_iterations)
finally:
smbo.stats.print_stats()
self.logger.info("Final Incumbent: %s" % (smbo.incumbent))
smbo.runhistory.save_json(
fn=os.path.join(scen.output_dir, "runhistory.json"))
def test_eips(self):
scenario = Scenario({
'cs': test_helpers.get_branin_config_space(),
'run_obj': 'quality',
'deterministic': True
})
types = get_types(scenario.cs, None)
umrfwi = UncorrelatedMultiObjectiveRandomForestWithInstances(
['cost', 'runtime'], types)
eips = EIPS(umrfwi)
rh2EPM = RunHistory2EPM4EIPS(scenario, 2)
taf = ExecuteTAFunc(test_function)
smbo = SMBO(scenario,
model=umrfwi,
acquisition_function=eips,
runhistory2epm=rh2EPM,
tae_runner=taf)
smbo.run(5)
print(smbo.incumbent)
raise ValueError()
def get_tuned_config(self, scenario: ASlibScenario):
'''
uses SMAC3 to determine a well-performing configuration in the configuration space self.cs on the given scenario
Arguments
---------
scenario: ASlibScenario
ASlib Scenario at hand
Returns
-------
Configuration
best incumbent configuration found by SMAC
'''
taf = ExecuteTAFunc(functools.partial(self.run_cv, scenario=scenario))
ac_scenario = Scenario({"run_obj": "quality", # we optimize quality
# at most 10 function evaluations
"runcount-limit": 10,
"cs": self.cs, # configuration space
"deterministic": "true"
})
# necessary to use stats options related to scenario information
AC_Stats.scenario = ac_scenario
# Optimize
self.logger.info(
">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
self.logger.info("Start Configuration")
self.logger.info(
">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
smbo = SMBO(scenario=ac_scenario, tae_runner=taf,
rng=np.random.RandomState(42))
smbo.run(max_iters=999)
AC_Stats.print_stats()
self.logger.info("Final Incumbent: %s" % (smbo.incumbent))
return smbo.incumbent
"run_obj": "quality", # we optimize quality (alternative runtime)
"runcount-limit": 200, # at most 200 function evaluations
"cs": cs, # configuration space
"deterministic": "true",
#"tuner-timeout" : 1,
#"wallclock_limit": 2
})
stats = Stats(scenario)
# register function to be optimize
taf = ExecuteTAFunc(rosenbrock_4d, stats)
# example call of the function
# it returns: Status, Cost, Runtime, Additional Infos
def_value = taf.run(cs.get_default_configuration())[1]
print("Default Value: %.2f" % (def_value))
# Optimize
smbo = SMBO(scenario=scenario,
tae_runner=taf,
stats=stats,
rng=np.random.RandomState(42))
try:
smbo.run(max_iters=999)
finally:
smbo.stats.print_stats()
print("Final Incumbent: %s" % (smbo.incumbent))
inc_value = taf.run(smbo.incumbent)[1]
print("Optimized Value: %.2f" % (inc_value))
class SMAC(object):
def __init__(
self,
scenario: Scenario,
# TODO: once we drop python3.4 add type hint
# typing.Union[ExecuteTARun, callable]
tae_runner=None,
runhistory: RunHistory = None,
intensifier: Intensifier = None,
acquisition_function: AbstractAcquisitionFunction = None,
model: AbstractEPM = None,
runhistory2epm: AbstractRunHistory2EPM = None,
initial_design: InitialDesign = None,
initial_configurations: typing.List[Configuration] = None,
stats: Stats = None,
rng: np.random.RandomState = None):
'''
Facade to use SMAC default mode
Parameters
----------
scenario: smac.scenario.scenario.Scenario
Scenario object
tae_runner: ExecuteTARun or callable
Callable or implementation of :class:`ExecuteTaRun`. In case a
callable is passed it will be wrapped by tae.ExecuteTaFunc().
If not set, tae_runner will be initialized with
the tae.ExecuteTARunOld()
runhistory: RunHistory
runhistory to store all algorithm runs
intensifier: Intensifier
intensification object to issue a racing to decide the current
incumbent
acquisition_function : AcquisitionFunction
Object that implements the AbstractAcquisitionFunction. Will use
EI if not set.
model : AbstractEPM
Model that implements train() and predict(). Will use a
RandomForest if not set.
runhistory2epm : RunHistory2EMP
Object that implements the AbstractRunHistory2EPM. If None,
will use RunHistory2EPM4Cost if objective is cost or
RunHistory2EPM4LogCost if objective is runtime.
initial_design: InitialDesign
initial sampling design
initial_configurations: typing.List[Configuration]
list of initial configurations for initial design --
cannot be used together with initial_design
stats: Stats
optional stats object
rng: np.random.RandomState
Random number generator
'''
self.logger = logging.getLogger("SMAC")
aggregate_func = average_cost
# initialize stats object
if stats:
self.stats = stats
else:
self.stats = Stats(scenario)
# initialize empty runhistory
if runhistory is None:
runhistory = RunHistory(aggregate_func=aggregate_func)
# initial random number generator
num_run, rng = self._get_rng(rng=rng)
# reset random number generator in config space to draw different
# random configurations with each seed given to SMAC
scenario.cs.seed(rng.randint(MAXINT))
# initial Trajectory Logger
traj_logger = TrajLogger(output_dir=scenario.output_dir,
stats=self.stats)
# initial EPM
types = get_types(scenario.cs, scenario.feature_array)
if model is None:
model = RandomForestWithInstances(
types=types,
instance_features=scenario.feature_array,
seed=rng.randint(MAXINT))
# initial acquisition function
if acquisition_function is None:
acquisition_function = EI(model=model)
# initialize optimizer on acquisition function
local_search = LocalSearch(acquisition_function, scenario.cs)
# initialize tae_runner
# First case, if tae_runner is None, the target algorithm is a call
# string in the scenario file
if tae_runner is None:
tae_runner = ExecuteTARunOld(ta=scenario.ta,
stats=self.stats,
run_obj=scenario.run_obj,
runhistory=runhistory,
par_factor=scenario.par_factor)
# Second case, the tae_runner is a function to be optimized
elif callable(tae_runner):
tae_runner = ExecuteTAFuncDict(ta=tae_runner,
stats=self.stats,
run_obj=scenario.run_obj,
memory_limit=scenario.memory_limit,
runhistory=runhistory,
par_factor=scenario.par_factor)
# Third case, if it is an ExecuteTaRun we can simply use the
# instance. Otherwise, the next check raises an exception
elif not isinstance(tae_runner, ExecuteTARun):
raise TypeError("Argument 'tae_runner' is %s, but must be "
"either a callable or an instance of "
"ExecuteTaRun. Passing 'None' will result in the "
"creation of target algorithm runner based on the "
"call string in the scenario file." %
type(tae_runner))
# Check that overall objective and tae objective are the same
if tae_runner.run_obj != scenario.run_obj:
raise ValueError("Objective for the target algorithm runner and "
"the scenario must be the same, but are '%s' and "
"'%s'" % (tae_runner.run_obj, scenario.run_obj))
# inject stats if necessary
if tae_runner.stats is None:
tae_runner.stats = self.stats
# inject runhistory if necessary
if tae_runner.runhistory is None:
tae_runner.runhistory = runhistory
# initial intensification
if intensifier is None:
intensifier = Intensifier(
tae_runner=tae_runner,
stats=self.stats,
traj_logger=traj_logger,
rng=rng,
instances=scenario.train_insts,
cutoff=scenario.cutoff,
deterministic=scenario.deterministic,
run_obj_time=scenario.run_obj == "runtime",
instance_specifics=scenario.instance_specific,
minR=scenario.minR,
maxR=scenario.maxR)
# initial design
if initial_design is not None and initial_configurations is not None:
raise ValueError(
"Either use initial_design or initial_configurations; but not both"
)
if initial_configurations is not None:
initial_design = MultiConfigInitialDesign(
tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
runhistory=runhistory,
rng=rng,
configs=initial_configurations,
intensifier=intensifier,
aggregate_func=aggregate_func)
elif initial_design is None:
if scenario.initial_incumbent == "DEFAULT":
initial_design = DefaultConfiguration(tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
elif scenario.initial_incumbent == "RANDOM":
initial_design = RandomConfiguration(tae_runner=tae_runner,
scenario=scenario,
stats=self.stats,
traj_logger=traj_logger,
rng=rng)
else:
raise ValueError("Don't know what kind of initial_incumbent "
"'%s' is" % scenario.initial_incumbent)
# initial conversion of runhistory into EPM data
if runhistory2epm is None:
num_params = len(scenario.cs.get_hyperparameters())
if scenario.run_obj == "runtime":
# if we log the performance data,
# the RFRImputator will already get
# log transform data from the runhistory
cutoff = np.log10(scenario.cutoff)
threshold = np.log10(scenario.cutoff * scenario.par_factor)
imputor = RFRImputator(rs=rng,
cutoff=cutoff,
threshold=threshold,
model=model,
change_threshold=0.01,
max_iter=2)
runhistory2epm = RunHistory2EPM4LogCost(
scenario=scenario,
num_params=num_params,
success_states=[
StatusType.SUCCESS,
],
impute_censored_data=True,
impute_state=[
StatusType.TIMEOUT,
],
imputor=imputor)
elif scenario.run_obj == 'quality':
runhistory2epm = RunHistory2EPM4Cost\
(scenario=scenario, num_params=num_params,
success_states=[StatusType.SUCCESS, ],
impute_censored_data=False, impute_state=None)
else:
raise ValueError('Unknown run objective: %s. Should be either '
'quality or runtime.' % self.scenario.run_obj)
self.solver = SMBO(scenario=scenario,
stats=self.stats,
initial_design=initial_design,
runhistory=runhistory,
runhistory2epm=runhistory2epm,
intensifier=intensifier,
aggregate_func=aggregate_func,
num_run=num_run,
model=model,
acq_optimizer=local_search,
acquisition_func=acquisition_function,
rng=rng)
def _get_rng(self, rng):
'''
initial random number generator
Arguments
---------
rng: np.random.RandomState|int|None
Returns
-------
int, np.random.RandomState
'''
# initialize random number generator
if rng is None:
num_run = np.random.randint(1234567980)
rng = np.random.RandomState(seed=num_run)
elif isinstance(rng, int):
num_run = rng
rng = np.random.RandomState(seed=rng)
elif isinstance(rng, np.random.RandomState):
num_run = rng.randint(1234567980)
rng = rng
else:
raise TypeError('Unknown type %s for argument rng. Only accepts '
'None, int or np.random.RandomState' %
str(type(rng)))
return num_run, rng
def optimize(self):
'''
optimize the algorithm provided in scenario (given in constructor)
Arguments
---------
max_iters: int
maximal number of iterations
'''
incumbent = None
try:
incumbent = self.solver.run()
finally:
self.solver.stats.print_stats()
self.logger.info("Final Incumbent: %s" % (self.solver.incumbent))
self.runhistory = self.solver.runhistory
self.trajectory = self.solver.intensifier.traj_logger.trajectory
return incumbent
def get_runhistory(self):
if not hasattr(self, 'runhistory'):
raise ValueError('SMAC was not fitted yet. Call optimize() prior '
'to accessing the runhistory.')
return self.runhistory
def get_trajectory(self):
if not hasattr(self, 'trajectory'):
raise ValueError('SMAC was not fitted yet. Call optimize() prior '
'to accessing the runhistory.')
return self.trajectory