def run_experiment(out_path: str, on_travis: bool = False): out_path = Path(out_path) out_path.mkdir(exist_ok=True) benchmark = Benchmark(rng=1) scenario_dict = {"run_obj": "quality", "wallclock-limit": 5 * 60 * 60, # max duration to run the optimization (in seconds) "cs": benchmark.get_configuration_space(seed=1), "deterministic": "true", "runcount-limit": 200, "limit_resources": True, # Uses pynisher to limit memory and runtime "cutoff": 1800, # runtime limit for target algorithm "memory_limit": 10000, # adapt this to reasonable value for your hardware "output_dir": str(out_path), } if on_travis: scenario_dict.update(get_travis_settings('smac')) scenario = Scenario(scenario_dict) # Number of Agents, which are trained to solve the cartpole experiment max_budget = 9 if not on_travis else 2 def optimization_function_wrapper(cfg, seed, instance, budget): """ Helper-function: simple wrapper to use the benchmark with smac""" # Now that we have already downloaded the container, # we only have to start a new instance. This is a fast operation. b = Benchmark(rng=seed) # Old API ---- NO LONGER SUPPORTED ---- This will simply ignore the fidelities # result_dict = b.objective_function(cfg, budget=int(budget)) # New API ---- Use this result_dict = b.objective_function(cfg, fidelity={"budget": int(budget)}) return result_dict['function_value'] smac = SMAC4MF(scenario=scenario, rng=np.random.RandomState(42), tae_runner=optimization_function_wrapper, intensifier=Hyperband, intensifier_kwargs={'initial_budget': 1, 'max_budget': max_budget, 'eta': 3} ) start_time = time() try: smac.optimize() finally: incumbent = smac.solver.incumbent end_time = time() if not on_travis: inc_value = smac.get_tae_runner().run(config=incumbent, instance='1', budget=max_budget, seed=0)[1] print(f"Value for optimized configuration: {inc_value:.4f}.\n" f"Optimization took {end_time-start_time:.0f}s")
def run_experiment(out_path, on_travis): settings = { 'min_budget': 1, 'max_budget': 9, # number of repetitions; this is the fidelity for this bench 'num_iterations': 10, # Set this to a low number for demonstration 'eta': 3, 'output_dir': Path(out_path) } if on_travis: settings.update(get_travis_settings('bohb')) b = Benchmark(rng=1) b.get_configuration_space(seed=1) settings.get('output_dir').mkdir(exist_ok=True) cs = b.get_configuration_space() seed = get_rng(rng=0) run_id = 'BOHB_on_cartpole' result_logger = hpres.json_result_logger(directory=str( settings.get('output_dir')), overwrite=True) ns = hpns.NameServer(run_id=run_id, host='localhost', working_directory=str(settings.get('output_dir'))) ns_host, ns_port = ns.start() worker = CustomWorker(seed=seed, nameserver=ns_host, nameserver_port=ns_port, run_id=run_id, max_budget=settings.get('max_budget')) worker.run(background=True) master = BOHB(configspace=cs, run_id=run_id, host=ns_host, nameserver=ns_host, nameserver_port=ns_port, eta=settings.get('eta'), min_budget=settings.get('min_budget'), max_budget=settings.get('max_budget'), result_logger=result_logger) result = master.run(n_iterations=settings.get('num_iterations')) master.shutdown(shutdown_workers=True) ns.shutdown() with open(settings.get('output_dir') / 'results.pkl', 'wb') as f: pickle.dump(result, f) id2config = result.get_id2config_mapping() incumbent = result.get_incumbent_id() inc_value = result.get_runs_by_id(incumbent)[-1]['loss'] inc_cfg = id2config[incumbent]['config'] logger.info(f'Inc Config:\n{inc_cfg}\n' f'with Performance: {inc_value:.2f}') if not on_travis: benchmark = Benchmark(container_source='library://phmueller/automl') incumbent_result = benchmark.objective_function_test( configuration=inc_cfg, fidelity={"budget": settings['max_budget']}) print(incumbent_result)
def run_experiment(out_path: str, on_travis: bool = False): out_path = Path(out_path) out_path.mkdir(exist_ok=True) benchmark = Benchmark(container_source='library://phmueller/automl', container_name='cartpole', rng=1) cs = benchmark.get_configuration_space(seed=1) scenario_dict = { "run_obj": "quality", # we optimize quality (alternative to runtime) "wallclock-limit": 5 * 60 * 60, # max duration to run the optimization (in seconds) "cs": cs, # configuration space "deterministic": "true", "runcount-limit": 200, "limit_resources": True, # Uses pynisher to limit memory and runtime "cutoff": 1800, # runtime limit for target algorithm "memory_limit": 10000, # adapt this to reasonable value for your hardware "output_dir": str(out_path), } if on_travis: scenario_dict.update(get_travis_settings('smac')) scenario = Scenario(scenario_dict) # Number of Agents, which are trained to solve the cartpole experiment max_budget = 5 if not on_travis else 2 def optimization_function_wrapper(cfg, seed, instance, budget): """ Helper-function: simple wrapper to use the benchmark with smac """ b = Benchmark(container_source='library://phmueller/automl', container_name='cartpole', rng=seed) # Old API ---- NO LONGER SUPPORTED ---- This will simply ignore the fidelities # result_dict = b.objective_function(cfg, budget=int(budget)) # New API ---- Use this result_dict = b.objective_function(cfg, fidelity={"budget": int(budget)}) return result_dict['function_value'] smac = SMAC4HPO(scenario=scenario, rng=np.random.RandomState(42), tae_runner=optimization_function_wrapper, intensifier=SuccessiveHalving, intensifier_kwargs={ 'initial_budget': 1, 'max_budget': max_budget, 'eta': 3 }) start_time = time() # Example call of the function with default values. It returns: Status, Cost, Runtime, Additional Infos def_value = smac.get_tae_runner().run( config=cs.get_default_configuration(), instance='1', budget=1, seed=0)[1] print( f"Value for default configuration: {def_value:.4f}.\nEvaluation took {time() - start_time:.0f}s" ) # Start optimization start_time = time() try: smac.optimize() finally: incumbent = smac.solver.incumbent end_time = time() if not on_travis: inc_value = smac.get_tae_runner().run(config=incumbent, instance='1', budget=max_budget, seed=0)[1] print(f"Value for optimized configuration: {inc_value:.4f}.\n" f"Optimization took {end_time-start_time:.0f}s")