def test_epils(self):
taf = ExecuteTAFuncArray(ta=self.branin)
epils = EPILS(self.scenario, tae_runner=taf)
inc = epils.optimize()
# not enough runs available to change the inc
self.assertEqual(inc["x"], 2.5)
self.assertEqual(inc["y"], 7.5)
def _main_cli(self):
"""Main function of SMAC for CLI interface
Returns
-------
instance
optimizer
"""
self.logger.info("SMAC call: %s" % (" ".join(sys.argv)))
cmd_reader = CMDReader()
args, _ = cmd_reader.read_cmd()
root_logger = logging.getLogger()
root_logger.setLevel(args.verbose_level)
logger_handler = logging.StreamHandler(stream=sys.stdout)
if root_logger.level >= logging.INFO:
formatter = logging.Formatter("%(levelname)s:\t%(message)s")
else:
formatter = logging.Formatter(
"%(asctime)s:%(levelname)s:%(name)s:%(message)s",
"%Y-%m-%d %H:%M:%S")
logger_handler.setFormatter(formatter)
root_logger.addHandler(logger_handler)
# remove default handler
root_logger.removeHandler(root_logger.handlers[0])
# Create defaults
rh = None
initial_configs = None
stats = None
incumbent = None
# Create scenario-object
scen = Scenario(args.scenario_file, [])
if args.mode == "SMAC":
optimizer = SMAC(scenario=scen,
rng=np.random.RandomState(args.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=args.seed)
elif args.mode == "ROAR":
optimizer = ROAR(scenario=scen,
rng=np.random.RandomState(args.seed),
runhistory=rh,
initial_configurations=initial_configs,
run_id=args.seed)
elif args.mode == "EPILS":
optimizer = EPILS(scenario=scen,
rng=np.random.RandomState(args.seed),
runhistory=rh,
initial_configurations=initial_configs,
run_id=args.seed)
else:
optimizer = None
return optimizer
def test_rng(self):
epils = EPILS(self.scenario, rng=None).solver
self.assertIsInstance(epils.rng, np.random.RandomState)
self.assertIsInstance(epils.num_run, int)
epils = EPILS(self.scenario, rng=1).solver
rng = np.random.RandomState(1)
self.assertEqual(epils.num_run, 1)
self.assertIsInstance(epils.rng, np.random.RandomState)
epils = EPILS(self.scenario, rng=rng).solver
self.assertIsInstance(epils.num_run, int)
self.assertIs(epils.rng, rng)
self.assertRaisesRegexp(TypeError,
"Unknown type <(class|type) 'str'> for argument "
'rng. Only accepts None, int or '
'np.random.RandomState',
EPILS, self.scenario, rng='BLA')
def test_init_only_scenario_runtime(self):
self.scenario.run_obj = 'runtime'
self.scenario.cutoff = 300
epils = EPILS(self.scenario).solver
self.assertIsInstance(epils.model, RandomForestWithInstances)
self.assertIsInstance(epils.rh2EPM, RunHistory2EPM4LogCost)
self.assertIsInstance(epils.acquisition_func, LogEI)
def test_abort_on_initial_design(self, patch):
def target(x):
return 5
patch.side_effect = FirstRunCrashedException()
scen = Scenario({'cs': test_helpers.get_branin_config_space(),
'run_obj': 'quality', 'output_dir': '',
'abort_on_first_run_crash': 1})
epils = EPILS(scen, tae_runner=target, rng=1).solver
self.assertRaises(FirstRunCrashedException, epils.run)
def test_init_EIPS_as_arguments(self):
for objective in ['runtime', 'quality']:
self.scenario.run_obj = objective
types, bounds = get_types(self.scenario.cs, None)
umrfwi = UncorrelatedMultiObjectiveRandomForestWithInstances(
['cost', 'runtime'], types, bounds)
eips = EIPS(umrfwi)
rh2EPM = RunHistory2EPM4EIPS(self.scenario, 2)
epils = EPILS(self.scenario, model=umrfwi, acquisition_function=eips,
runhistory2epm=rh2EPM).solver
self.assertIs(umrfwi, epils.model)
self.assertIs(eips, epils.acquisition_func)
self.assertIs(rh2EPM, epils.rh2EPM)
def test_init_only_scenario_quality(self):
epils = EPILS(self.scenario).solver
self.assertIsInstance(epils.model, RandomForestWithInstances)
self.assertIsInstance(epils.rh2EPM, RunHistory2EPM4Cost)
self.assertIsInstance(epils.acquisition_func, EI)
def main_cli(self, commandline_arguments: typing.List[str] = None):
"""Main function of SMAC for CLI interface"""
self.logger.info("SMAC call: %s" % (" ".join(sys.argv)))
cmd_reader = CMDReader()
kwargs = {}
if commandline_arguments:
kwargs['commandline_arguments'] = commandline_arguments
main_args_, smac_args_, scen_args_ = cmd_reader.read_cmd(**kwargs)
root_logger = logging.getLogger()
root_logger.setLevel(main_args_.verbose_level)
logger_handler = logging.StreamHandler(stream=sys.stdout)
if root_logger.level >= logging.INFO:
formatter = logging.Formatter("%(levelname)s:\t%(message)s")
else:
formatter = logging.Formatter(
"%(asctime)s:%(levelname)s:%(name)s:%(message)s",
"%Y-%m-%d %H:%M:%S")
logger_handler.setFormatter(formatter)
root_logger.addHandler(logger_handler)
# remove default handler
if len(root_logger.handlers) > 1:
root_logger.removeHandler(root_logger.handlers[0])
# Create defaults
rh = None
initial_configs = None
stats = None
incumbent = None
# Create scenario-object
scenario = {}
scenario.update(vars(smac_args_))
scenario.update(vars(scen_args_))
scen = Scenario(scenario=scenario)
# Restore state
if main_args_.restore_state:
root_logger.debug("Restoring state from %s...",
main_args_.restore_state)
rh, stats, traj_list_aclib, traj_list_old = self.restore_state(
scen, main_args_)
scen.output_dir_for_this_run = create_output_directory(
scen,
main_args_.seed,
root_logger,
)
scen.write()
incumbent = self.restore_state_after_output_dir(
scen, stats, traj_list_aclib, traj_list_old)
if main_args_.warmstart_runhistory:
aggregate_func = average_cost
rh = RunHistory(aggregate_func=aggregate_func)
scen, rh = merge_foreign_data_from_file(
scenario=scen,
runhistory=rh,
in_scenario_fn_list=main_args_.warmstart_scenario,
in_runhistory_fn_list=main_args_.warmstart_runhistory,
cs=scen.cs,
aggregate_func=aggregate_func)
if main_args_.warmstart_incumbent:
initial_configs = [scen.cs.get_default_configuration()]
for traj_fn in main_args_.warmstart_incumbent:
trajectory = TrajLogger.read_traj_aclib_format(fn=traj_fn,
cs=scen.cs)
initial_configs.append(trajectory[-1]["incumbent"])
if main_args_.mode == "SMAC":
optimizer = SMAC(scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=main_args_.seed)
elif main_args_.mode == "BORF":
optimizer = BORF(scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=main_args_.seed)
elif main_args_.mode == "BOGP":
optimizer = BOGP(scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=main_args_.seed)
elif main_args_.mode == "ROAR":
optimizer = ROAR(scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
run_id=main_args_.seed)
elif main_args_.mode == "EPILS":
optimizer = EPILS(scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
run_id=main_args_.seed)
elif main_args_.mode == "Hydra":
optimizer = Hydra(
scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=main_args_.seed,
random_configuration_chooser=main_args_.
random_configuration_chooser,
n_iterations=main_args_.hydra_iterations,
val_set=main_args_.hydra_validation,
incs_per_round=main_args_.hydra_incumbents_per_round,
n_optimizers=main_args_.hydra_n_optimizers)
elif main_args_.mode == "PSMAC":
optimizer = PSMAC(
scenario=scen,
rng=np.random.RandomState(main_args_.seed),
run_id=main_args_.seed,
shared_model=smac_args_.shared_model,
validate=main_args_.psmac_validate,
n_optimizers=main_args_.hydra_n_optimizers,
n_incs=main_args_.hydra_incumbents_per_round,
)
try:
optimizer.optimize()
except (TAEAbortException, FirstRunCrashedException) as err:
self.logger.error(err)
def main_cli(self):
"""Main function of SMAC for CLI interface"""
self.logger.info("SMAC call: %s" % (" ".join(sys.argv)))
cmd_reader = CMDReader()
args_, misc_args = cmd_reader.read_cmd()
root_logger = logging.getLogger()
root_logger.setLevel(args_.verbose_level)
logger_handler = logging.StreamHandler(
stream=sys.stdout)
if root_logger.level >= logging.INFO:
formatter = logging.Formatter(
"%(levelname)s:\t%(message)s")
else:
formatter = logging.Formatter(
"%(asctime)s:%(levelname)s:%(name)s:%(message)s",
"%Y-%m-%d %H:%M:%S")
logger_handler.setFormatter(formatter)
root_logger.addHandler(logger_handler)
# remove default handler
root_logger.removeHandler(root_logger.handlers[0])
scen = Scenario(args_.scenario_file, misc_args,
run_id=args_.seed)
rh = None
if args_.warmstart_runhistory:
aggregate_func = average_cost
rh = RunHistory(aggregate_func=aggregate_func)
scen, rh = merge_foreign_data_from_file(
scenario=scen,
runhistory=rh,
in_scenario_fn_list=args_.warmstart_scenario,
in_runhistory_fn_list=args_.warmstart_runhistory,
cs=scen.cs,
aggregate_func=aggregate_func)
initial_configs = None
if args_.warmstart_incumbent:
initial_configs = [scen.cs.get_default_configuration()]
for traj_fn in args_.warmstart_incumbent:
trajectory = TrajLogger.read_traj_aclib_format(
fn=traj_fn, cs=scen.cs)
initial_configs.append(trajectory[-1]["incumbent"])
if args_.mode == "SMAC":
optimizer = SMAC(
scenario=scen,
rng=np.random.RandomState(args_.seed),
runhistory=rh,
initial_configurations=initial_configs)
elif args_.mode == "ROAR":
optimizer = ROAR(
scenario=scen,
rng=np.random.RandomState(args_.seed),
runhistory=rh,
initial_configurations=initial_configs)
elif args_.mode == "EPILS":
optimizer = EPILS(
scenario=scen,
rng=np.random.RandomState(args_.seed),
runhistory=rh,
initial_configurations=initial_configs)
try:
optimizer.optimize()
except (TAEAbortException, FirstRunCrashedException) as err:
self.logger.error(err)