def test_example_utils():
from hpobench.util.example_utils import get_travis_settings
res = get_travis_settings('smac')
assert res['runcount-limit'] == 5
res = get_travis_settings('bohb')
assert res['max_budget'] == 2
with pytest.raises(ValueError):
res = get_travis_settings('unknown')
def run_experiment(out_path: str, on_travis: bool = False):
out_path = Path(out_path)
out_path.mkdir(exist_ok=True)
benchmark = Benchmark(rng=1)
scenario_dict = {"run_obj": "quality",
"wallclock-limit": 5 * 60 * 60, # max duration to run the optimization (in seconds)
"cs": benchmark.get_configuration_space(seed=1),
"deterministic": "true",
"runcount-limit": 200,
"limit_resources": True, # Uses pynisher to limit memory and runtime
"cutoff": 1800, # runtime limit for target algorithm
"memory_limit": 10000, # adapt this to reasonable value for your hardware
"output_dir": str(out_path),
}
if on_travis:
scenario_dict.update(get_travis_settings('smac'))
scenario = Scenario(scenario_dict)
# Number of Agents, which are trained to solve the cartpole experiment
max_budget = 9 if not on_travis else 2
def optimization_function_wrapper(cfg, seed, instance, budget):
""" Helper-function: simple wrapper to use the benchmark with smac"""
# Now that we have already downloaded the container,
# we only have to start a new instance. This is a fast operation.
b = Benchmark(rng=seed)
# Old API ---- NO LONGER SUPPORTED ---- This will simply ignore the fidelities
# result_dict = b.objective_function(cfg, budget=int(budget))
# New API ---- Use this
result_dict = b.objective_function(cfg, fidelity={"budget": int(budget)})
return result_dict['function_value']
smac = SMAC4MF(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=optimization_function_wrapper,
intensifier=Hyperband,
intensifier_kwargs={'initial_budget': 1, 'max_budget': max_budget, 'eta': 3}
)
start_time = time()
try:
smac.optimize()
finally:
incumbent = smac.solver.incumbent
end_time = time()
if not on_travis:
inc_value = smac.get_tae_runner().run(config=incumbent, instance='1', budget=max_budget, seed=0)[1]
print(f"Value for optimized configuration: {inc_value:.4f}.\n"
f"Optimization took {end_time-start_time:.0f}s")
def run_experiment(out_path, on_travis):
settings = {
'min_budget': 1,
'max_budget':
9, # number of repetitions; this is the fidelity for this bench
'num_iterations': 10, # Set this to a low number for demonstration
'eta': 3,
'output_dir': Path(out_path)
}
if on_travis:
settings.update(get_travis_settings('bohb'))
b = Benchmark(rng=1)
b.get_configuration_space(seed=1)
settings.get('output_dir').mkdir(exist_ok=True)
cs = b.get_configuration_space()
seed = get_rng(rng=0)
run_id = 'BOHB_on_cartpole'
result_logger = hpres.json_result_logger(directory=str(
settings.get('output_dir')),
overwrite=True)
ns = hpns.NameServer(run_id=run_id,
host='localhost',
working_directory=str(settings.get('output_dir')))
ns_host, ns_port = ns.start()
worker = CustomWorker(seed=seed,
nameserver=ns_host,
nameserver_port=ns_port,
run_id=run_id,
max_budget=settings.get('max_budget'))
worker.run(background=True)
master = BOHB(configspace=cs,
run_id=run_id,
host=ns_host,
nameserver=ns_host,
nameserver_port=ns_port,
eta=settings.get('eta'),
min_budget=settings.get('min_budget'),
max_budget=settings.get('max_budget'),
result_logger=result_logger)
result = master.run(n_iterations=settings.get('num_iterations'))
master.shutdown(shutdown_workers=True)
ns.shutdown()
with open(settings.get('output_dir') / 'results.pkl', 'wb') as f:
pickle.dump(result, f)
id2config = result.get_id2config_mapping()
incumbent = result.get_incumbent_id()
inc_value = result.get_runs_by_id(incumbent)[-1]['loss']
inc_cfg = id2config[incumbent]['config']
logger.info(f'Inc Config:\n{inc_cfg}\n'
f'with Performance: {inc_value:.2f}')
if not on_travis:
benchmark = Benchmark(container_source='library://phmueller/automl')
incumbent_result = benchmark.objective_function_test(
configuration=inc_cfg, fidelity={"budget": settings['max_budget']})
print(incumbent_result)
def run_experiment(out_path: str, on_travis: bool = False):
out_path = Path(out_path)
out_path.mkdir(exist_ok=True)
benchmark = Benchmark(container_source='library://phmueller/automl',
container_name='cartpole',
rng=1)
cs = benchmark.get_configuration_space(seed=1)
scenario_dict = {
"run_obj": "quality", # we optimize quality (alternative to runtime)
"wallclock-limit":
5 * 60 * 60, # max duration to run the optimization (in seconds)
"cs": cs, # configuration space
"deterministic": "true",
"runcount-limit": 200,
"limit_resources": True, # Uses pynisher to limit memory and runtime
"cutoff": 1800, # runtime limit for target algorithm
"memory_limit":
10000, # adapt this to reasonable value for your hardware
"output_dir": str(out_path),
}
if on_travis:
scenario_dict.update(get_travis_settings('smac'))
scenario = Scenario(scenario_dict)
# Number of Agents, which are trained to solve the cartpole experiment
max_budget = 5 if not on_travis else 2
def optimization_function_wrapper(cfg, seed, instance, budget):
""" Helper-function: simple wrapper to use the benchmark with smac """
b = Benchmark(container_source='library://phmueller/automl',
container_name='cartpole',
rng=seed)
# Old API ---- NO LONGER SUPPORTED ---- This will simply ignore the fidelities
# result_dict = b.objective_function(cfg, budget=int(budget))
# New API ---- Use this
result_dict = b.objective_function(cfg,
fidelity={"budget": int(budget)})
return result_dict['function_value']
smac = SMAC4HPO(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=optimization_function_wrapper,
intensifier=SuccessiveHalving,
intensifier_kwargs={
'initial_budget': 1,
'max_budget': max_budget,
'eta': 3
})
start_time = time()
# Example call of the function with default values. It returns: Status, Cost, Runtime, Additional Infos
def_value = smac.get_tae_runner().run(
config=cs.get_default_configuration(), instance='1', budget=1,
seed=0)[1]
print(
f"Value for default configuration: {def_value:.4f}.\nEvaluation took {time() - start_time:.0f}s"
)
# Start optimization
start_time = time()
try:
smac.optimize()
finally:
incumbent = smac.solver.incumbent
end_time = time()
if not on_travis:
inc_value = smac.get_tae_runner().run(config=incumbent,
instance='1',
budget=max_budget,
seed=0)[1]
print(f"Value for optimized configuration: {inc_value:.4f}.\n"
f"Optimization took {end_time-start_time:.0f}s")