def prepare(self, pipeline_elements: list, maximize_metric: bool):
# build space
self.space = ConfigurationSpace()
self.build_smac_space(pipeline_elements)
self.maximize_metric = maximize_metric
self.scenario = Scenario({
"run_obj": self.run_obj,
"cs": self.cspace,
"deterministic": "true",
"wallclock_limit": self.wallclock_limit
})
self.smac = SMAC4BO(scenario=self.scenario,
rng=np.random.RandomState(42),
tae_runner=MyExecuteTARun)
if self.smac_helper:
self.smac_helper["data"] = self.smac
self.optimizer = self.smac.solver
self.optimizer.runhistory.overwrite_existing_runs = True
self.ask = self.ask_generator()
def evaluate(seed, benchmark, rh2epm, rh2epm_name):
benchmark_name = benchmark.get_meta_information()['name']
output_dir = "/home/feurerm/projects/smac3test/%s/%s/%d" % (benchmark_name, rh2epm_name, seed)
if os.path.exists(output_dir):
print('Output directory %s exists - skip' % output_dir)
return
# Scenario object
scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": 50,
# max. number of function evaluations; for this example set to a low number
"cs": benchmark.get_configuration_space(), # configuration space
"deterministic": "true",
'limit_resources': False,
"output_dir": output_dir,
})
import local.treegp
smac = SMAC4BO(scenario=scenario,
rng=seed,
tae_runner=benchmark,
runhistory2epm=rh2epm,
model_type='gp',
initial_design=LHDesign,
)
smac.optimize()
def prepare(self, pipeline_elements: list, maximize_metric: bool):
self.hyperparameter_list = []
# build space
self.space = ConfigurationSpace()
for element in pipeline_elements:
# check if Switch object
if isinstance(element, Switch):
algorithm_options = {
} # mapping algorithm name with their child hyper params
for algo in element.elements:
algo_params = [] # hyper params corresponding to "algo"
for name, value in algo.hyperparameters.items():
smac_param = self._convert_PHOTON_to_smac_space(
value,
(element.name + "__" +
name)) # or element.name__algo.name__name ???
algo_params.append(smac_param)
algorithm_options[(element.name + "__" +
algo.name)] = algo_params
algos = CategoricalHyperparameter(
name=element.name + "__algos",
choices=algorithm_options.keys())
self.space.add_hyperparameter(algos)
for algo, params in algorithm_options.items():
for param in params:
cond = InCondition(child=param,
parent=algos,
values=[algo])
self.space.add_hyperparameter(param)
self.space.add_condition(cond)
else:
for name, value in element.hyperparameters.items():
smac_param = self._convert_PHOTON_to_smac_space(
value, name)
if smac_param is not None:
self.space.add_hyperparameter(smac_param)
self.scenario = Scenario({
"run_obj": self.run_obj,
"cutoff_time": self.cutoff_time,
"runcount_limit": self.runcount_limit,
"tuner-timeout": self.tuner_timeout,
"wallclock_limit": self.wallclock_limit,
"cs": self.space,
"deterministic": "true"
})
self.smac = SMAC4BO(scenario=self.scenario,
rng=np.random.RandomState(42))
self.optimizer = self.smac.solver
self.optimizer.stats.start_timing()
self.optimizer.incumbent = self.get_default_incumbent()
self.flag = False # False: compute performance of challenger, True: compute performance of incumbent
self.ask = self.ask_generator()
def test_facade(self):
config_space = ConfigurationSpace()
n_components = UniformIntegerHyperparameter(
"PCA__n_components", 5, 30)
config_space.add_hyperparameter(n_components)
scenario_dict = {
"run_obj": "quality",
"deterministic": "true",
"cs": config_space,
"wallclock_limit": 60
}
with self.assertRaises(ValueError):
SMACOptimizer(facade="SMAC4BOO", scenario_dict=scenario_dict)
with self.assertRaises(ValueError):
facade = SMAC4BO(scenario=Scenario(scenario_dict))
SMACOptimizer(facade=facade, scenario_dict=scenario_dict)
facades = [
"SMAC4BO", SMAC4BO, "SMAC4AC", SMAC4AC, "SMAC4HPO", SMAC4HPO,
"BOHB4HPO", BOHB4HPO
]
for facade in facades:
SMACOptimizer(facade=facade, scenario_dict=scenario_dict)
def test_exchange_sobol_for_lhd(self):
cs = ConfigurationSpace()
for i in range(40):
cs.add_hyperparameter(
UniformFloatHyperparameter('x%d' % (i + 1), 0, 1))
scenario = Scenario({'cs': cs, 'run_obj': 'quality'})
facade = SMAC4BO(scenario=scenario)
self.assertIsInstance(facade.solver.initial_design, SobolDesign)
cs.add_hyperparameter(UniformFloatHyperparameter('x41', 0, 1))
with self.assertRaisesRegex(
ValueError,
'Sobol sequence" can only handle up to 40 dimensions. Please use a different initial design, such as '
'"the Latin Hypercube design"',
):
SMAC4BO(scenario=scenario)
self.output_dirs.append(scenario.output_dir)
default_value=4)
mutation_prob = UniformFloatHyperparameter("mutation_prob",
0.1,
0.9,
default_value=0.5)
cs.add_hyperparameters(
[num_generations, number_chromosomes, num_parents_mating, mutation_prob])
# Scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit":
10, # max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = gawd(cs.get_default_configuration())
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC4BO(
scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=gawd,
)
smac.optimize()
def test_against_smac(self):
# PHOTON implementation
self.pipe.add(PipelineElement("StandardScaler"))
# then do feature selection using a PCA, specify which values to try in the hyperparameter search
self.pipe += PipelineElement(
"PCA", hyperparameters={"n_components": IntegerRange(5, 30)}
)
# engage and optimize the good old SVM for Classification
self.pipe += PipelineElement(
"SVC",
hyperparameters={
"kernel": Categorical(["linear", "rbf", "poly", "sigmoid"]),
"C": FloatRange(0.5, 200),
},
gamma="auto",
)
self.X, self.y = self.simple_classification()
self.pipe.fit(self.X, self.y)
# AUTO ML direct
# Build Configuration Space which defines all parameters and their ranges
cs = ConfigurationSpace()
# We define a few possible types of SVM-kernels and add them as "kernel" to our cs
n_components = UniformIntegerHyperparameter(
"PCA__n_components", 5, 30
) # , default_value=5)
cs.add_hyperparameter(n_components)
kernel = CategoricalHyperparameter(
"SVC__kernel", ["linear", "rbf", "poly", "sigmoid"]
) # , default_value="linear")
cs.add_hyperparameter(kernel)
c = UniformFloatHyperparameter("SVC__C", 0.5, 200) # , default_value=1)
cs.add_hyperparameter(c)
# Scenario object
scenario = Scenario(
{
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": 800, # maximum function evaluations
"cs": cs, # configuration space
"deterministic": "true",
"shared_model": "false", # !!!!
"wallclock_limit": self.time_limit,
}
)
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC4BO(
scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=self.objective_function,
)
self.traurig = smac
incumbent = smac.optimize()
inc_value = self.objective_function(incumbent)
print(incumbent)
print(inc_value)
runhistory_photon = self.smac_helper["data"].solver.runhistory
runhistory_original = smac.solver.runhistory
x_ax = range(
1,
min(
len(runhistory_original.cost_per_config.keys()),
len(runhistory_photon.cost_per_config.keys()),
)
+ 1,
)
y_ax_original = [runhistory_original.cost_per_config[tmp] for tmp in x_ax]
y_ax_photon = [runhistory_photon.cost_per_config[tmp] for tmp in x_ax]
y_ax_original_inc = [min(y_ax_original[: tmp + 1]) for tmp in x_ax]
y_ax_photon_inc = [min(y_ax_photon[: tmp + 1]) for tmp in x_ax]
plt.figure(figsize=(10, 7))
plt.plot(x_ax, y_ax_original, "g", label="Original")
plt.plot(x_ax, y_ax_photon, "b", label="PHOTON")
plt.plot(x_ax, y_ax_photon_inc, "r", label="PHOTON Incumbent")
plt.plot(x_ax, y_ax_original_inc, "k", label="Original Incumbent")
plt.title("Photon Prove")
plt.xlabel("X")
plt.ylabel("Y")
plt.legend(loc="best")
plt.show()
def neighbours(items, fill=None):
before = itertools.chain([fill], items)
after = itertools.chain(
items, [fill]
) # You could use itertools.zip_longest() later instead.
next(after)
for a, b, c in zip(before, items, after):
yield [value for value in (a, b, c) if value is not fill]
print("---------------")
original_pairing = [
sum(values) / len(values) for values in neighbours(y_ax_original)
]
bias_term = np.mean(
[
abs(y_ax_original_inc[t] - y_ax_photon_inc[t])
for t in range(len(y_ax_photon_inc))
]
)
photon_pairing = [
sum(values) / len(values) - bias_term for values in neighbours(y_ax_photon)
]
counter = 0
for i, x in enumerate(x_ax):
if abs(original_pairing[i] - photon_pairing[i]) > 0.05:
counter += 1
self.assertLessEqual(counter / len(x_ax), 0.15)
# Build Configuration Space which defines all parameters and their ranges
cs = ConfigurationSpace()
x0 = UniformFloatHyperparameter("x0", -5, 10, default_value=-3)
x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=-4)
cs.add_hyperparameters([x0, x1])
# Scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit":
10, # max. number of function evaluations; for this example set to a low number
"cs": cs, # configuration space
"deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = rosenbrock_2d(cs.get_default_configuration())
print("Default Value: %.2f" % def_value)
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC4BO(
scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=rosenbrock_2d,
)
smac.optimize()
def main_cli(self, commandline_arguments: typing.Optional[typing.List[str]] = None) -> None:
"""Main function of SMAC for CLI interface"""
self.logger.info("SMAC call: %s" % (" ".join(sys.argv)))
cmd_reader = CMDReader()
kwargs = {}
if commandline_arguments:
kwargs['commandline_arguments'] = commandline_arguments
main_args_, smac_args_, scen_args_ = cmd_reader.read_cmd(**kwargs)
root_logger = logging.getLogger()
root_logger.setLevel(main_args_.verbose_level)
logger_handler = logging.StreamHandler(stream=sys.stdout)
if root_logger.level >= logging.INFO:
formatter = logging.Formatter(
"%(levelname)s:\t%(message)s")
else:
formatter = logging.Formatter(
"%(asctime)s:%(levelname)s:%(name)s:\t%(message)s",
"%Y-%m-%d %H:%M:%S")
logger_handler.setFormatter(formatter)
root_logger.addHandler(logger_handler)
# remove default handler
if len(root_logger.handlers) > 1:
root_logger.removeHandler(root_logger.handlers[0])
# Create defaults
rh = None
initial_configs = None
stats = None
incumbent = None
# Create scenario-object
scenario = {}
scenario.update(vars(smac_args_))
scenario.update(vars(scen_args_))
scen = Scenario(scenario=scenario)
# Restore state
if main_args_.restore_state:
root_logger.debug("Restoring state from %s...", main_args_.restore_state)
restore_state = main_args_.restore_state
rh, stats, traj_list_aclib, traj_list_old = self.restore_state(scen, restore_state)
scen.output_dir_for_this_run = create_output_directory(
scen, main_args_.seed, root_logger,
)
scen.write()
incumbent = self.restore_state_after_output_dir(scen, stats,
traj_list_aclib, traj_list_old)
if main_args_.warmstart_runhistory:
rh = RunHistory()
scen, rh = merge_foreign_data_from_file(
scenario=scen,
runhistory=rh,
in_scenario_fn_list=main_args_.warmstart_scenario,
in_runhistory_fn_list=main_args_.warmstart_runhistory,
cs=scen.cs, # type: ignore[attr-defined] # noqa F821
)
if main_args_.warmstart_incumbent:
initial_configs = [scen.cs.get_default_configuration()] # type: ignore[attr-defined] # noqa F821
for traj_fn in main_args_.warmstart_incumbent:
trajectory = TrajLogger.read_traj_aclib_format(
fn=traj_fn,
cs=scen.cs, # type: ignore[attr-defined] # noqa F821
)
initial_configs.append(trajectory[-1]["incumbent"])
if main_args_.mode == "SMAC4AC":
optimizer = SMAC4AC(
scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=main_args_.seed)
elif main_args_.mode == "SMAC4HPO":
optimizer = SMAC4HPO(
scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=main_args_.seed)
elif main_args_.mode == "SMAC4BO":
optimizer = SMAC4BO(
scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=main_args_.seed)
elif main_args_.mode == "ROAR":
optimizer = ROAR(
scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
run_id=main_args_.seed)
elif main_args_.mode == "Hydra":
optimizer = Hydra(
scenario=scen,
rng=np.random.RandomState(main_args_.seed),
runhistory=rh,
initial_configurations=initial_configs,
stats=stats,
restore_incumbent=incumbent,
run_id=main_args_.seed,
random_configuration_chooser=main_args_.random_configuration_chooser,
n_iterations=main_args_.hydra_iterations,
val_set=main_args_.hydra_validation,
incs_per_round=main_args_.hydra_incumbents_per_round,
n_optimizers=main_args_.hydra_n_optimizers)
elif main_args_.mode == "PSMAC":
optimizer = PSMAC(
scenario=scen,
rng=np.random.RandomState(main_args_.seed),
run_id=main_args_.seed,
shared_model=smac_args_.shared_model,
validate=main_args_.psmac_validate,
n_optimizers=main_args_.hydra_n_optimizers,
n_incs=main_args_.hydra_incumbents_per_round,
)
try:
optimizer.optimize()
except (TAEAbortException, FirstRunCrashedException) as err:
self.logger.error(err)