def run_optimization_algorithm(self, pipeline_config, run_id, ns_host, ns_port, nameserver, task_id): config_space = self.pipeline.get_hyperparameter_search_space( **pipeline_config) self.logger.info("[AutoNet] Start " + pipeline_config["algorithm"]) # initialize optimization algorithm loggers = [ json_result_logger(directory=pipeline_config["result_logger_dir"], overwrite=True) ] if pipeline_config['use_tensorboard_logger']: loggers.append(tensorboard_logger()) HB = self.get_optimization_algorithm_instance( config_space=config_space, run_id=run_id, pipeline_config=pipeline_config, ns_host=ns_host, ns_port=ns_port, loggers=loggers) # start algorithm min_num_workers = pipeline_config["min_workers"] if task_id != -1 else 1 reduce_runtime = pipeline_config["max_budget"] if pipeline_config[ "budget_type"] == "time" else 0 HB.run_until(runtime=(pipeline_config["max_runtime"] - reduce_runtime), n_iterations=pipeline_config["num_iterations"], min_n_workers=min_num_workers) HB.shutdown(shutdown_workers=True) nameserver.shutdown()
def runBOHB(cfg): run_id = "0" # assign random port in the 30000-40000 range to avoid using a blocked port because of a previous improper bohb shutdown port = int(30000 + random.random() * 10000) ns = hpns.NameServer(run_id=run_id, host="127.0.0.1", port=port) ns.start() w = BOHBWorker(cfg=cfg, nameserver="127.0.0.1", run_id=run_id, nameserver_port=port) w.run(background=True) result_logger = hpres.json_result_logger(directory=cfg["bohb_log_dir"], overwrite=True) bohb = BOHB( configspace=get_configspace(cfg['model']), run_id=run_id, min_budget=cfg["bohb_min_budget"], max_budget=cfg["bohb_max_budget"], eta=cfg["bohb_eta"], nameserver="127.0.0.1", nameserver_port=port, result_logger=result_logger, ) res = bohb.run(n_iterations=cfg["bohb_iterations"]) bohb.shutdown(shutdown_workers=True) ns.shutdown() return res
def run_master(args): NS = hpns.NameServer(run_id=args.run_id, nic_name=args.nic_name, working_directory=args.bohb_root_path) ns_host, ns_port = NS.start() # Start a background worker for the master node if args.optimize_generalist: w = AggregateWorker(run_id=args.run_id, host=ns_host, nameserver=ns_host, nameserver_port=ns_port, working_directory=args.bohb_root_path, n_repeat=args.n_repeat, has_repeats_as_budget=args.n_repeat is None, time_budget=args.time_budget, time_budget_approx=args.time_budget_approx, performance_matrix=args.performance_matrix) else: w = SingleWorker(run_id=args.run_id, host=ns_host, nameserver=ns_host, nameserver_port=ns_port, working_directory=args.bohb_root_path, n_repeat=args.n_repeat, dataset=args.dataset, time_budget=args.time_budget, time_budget_approx=args.time_budget_approx) w.run(background=True) # Create an optimizer result_logger = hpres.json_result_logger(directory=args.bohb_root_path, overwrite=False) if args.previous_run_dir is not None: previous_result = hpres.logged_results_to_HBS_result( args.previous_run_dir) else: pervious_result = None logger = logging.getLogger(__file__) logging_level = getattr(logging, args.logger_level) logger.setLevel(logging_level) optimizer = BOHB(configspace=get_configspace(), run_id=args.run_id, host=ns_host, nameserver=ns_host, nameserver_port=ns_port, min_budget=args.n_repeat_lower_budget, max_budget=args.n_repeat_upper_budget, result_logger=result_logger, logger=logger, previous_result=previous_result) res = optimizer.run(n_iterations=args.n_iterations) # Shutdown optimizer.shutdown(shutdown_workers=True) NS.shutdown()
def optimize_hyperparameters(model_class, parameters, train_and_validate_fn, num_iterations, min_budget=0.01, working_dir="./bohby_workspace/"): # Make sure the working directory exists os.makedirs(working_dir, exist_ok=True) # Generate a configspace from the given parameters config_space = generate_configspace(parameters) # Start a local nameserver for communication NS = hpns.NameServer(run_id=_runid, nic_name="lo", working_directory=working_dir) ns_host, ns_port = NS.start() # Define the worker worker = WrapWorker(model_class, train_and_validate_fn, working_directory=working_dir, nameserver=ns_host, nameserver_port=ns_port, run_id=_runid) worker.run(background=True) # Enable live logging so a run can be canceled at any time and we can still recover the results result_logger = json_result_logger(directory=working_dir, overwrite=True) # Optimization bohb = BOHB(configspace=config_space, working_directory=working_dir, run_id=_runid, eta=2, min_budget=min_budget, max_budget=1, host=ns_host, nameserver=ns_host, nameserver_port=ns_port, ping_interval=3600, result_logger=result_logger) res = bohb.run(n_iterations=num_iterations) # Clean up bohb.shutdown(shutdown_workers=True) NS.shutdown() # Best found config run_results = hpres.logged_results_to_HB_result(working_dir) id2conf = run_results.get_id2config_mapping() incumbent_id = run_results.get_incumbent_id() incumbent_config = id2conf[incumbent_id]['config'] incumbent_runs = run_results.get_runs_by_id(incumbent_id) val_loss = incumbent_runs[-1].loss return val_loss, incumbent_config
def fit(self, pipeline_config, X_train, Y_train, X_valid, Y_valid, refit=False): autonet_logger = logging.getLogger('autonet') hpbandster_logger = logging.getLogger('hpbandster') level = self.logger_settings[pipeline_config['log_level']] autonet_logger.setLevel(level) hpbandster_logger.setLevel(level) autonet_logger.info("Start autonet with config:\n" + str(pipeline_config)) result_logger = [] if not refit: result_logger = json_result_logger( directory=pipeline_config["result_logger_dir"], overwrite=True) return { 'X_train': X_train, 'Y_train': Y_train, 'X_valid': X_valid, 'Y_valid': Y_valid, 'result_loggers': [result_logger], 'shutdownables': [] }
def run_bohb(exp_name, log_dir='EXP', iterations=20): run_dir = 'bohb-{}-{}'.format(log_dir, exp_name) if not os.path.exists(run_dir): utils.create_exp_dir(run_dir, scripts_to_save=glob.glob('*.py')) # log_format = '%(asctime)s %(message)s' # logging.basicConfig(stream=sys.stdout, level=logging.INFO, # format=log_format, datefmt='%m/%d %I:%M:%S %p') # fh = logging.FileHandler(os.path.join(run_dir, 'log.txt')) # fh.setFormatter(logging.Formatter(log_format)) # logging.getLogger().addHandler(fh) result_logger = hpres.json_result_logger(directory=run_dir, overwrite=True) # Start a nameserver NS = hpns.NameServer(run_id=exp_name, host='127.0.0.1', port=0) ns_host, ns_port = NS.start() # Start a localserver worker = TorchWorker(run_id=exp_name, host='127.0.0.1', nameserver=ns_host, nameserver_port=ns_port, timeout=120, run_dir=run_dir) worker.run(background=True) # Initialise optimiser bohb = BOHB(configspace=worker.get_configspace(), run_id=exp_name, host='127.0.0.1', nameserver=ns_host, nameserver_port=ns_port, result_logger=result_logger, min_budget=2, max_budget=5, ) print('Worker running') res = bohb.run(n_iterations=iterations) # Store the results with open(os.path.join(run_dir, 'result.pkl'), 'wb') as file: pickle.dump(res, file) # Shutdown bohb.shutdown(shutdown_workers=True) NS.shutdown() # get all runs all_runs = res.get_all_runs() # get id to configuration mapping as dictionary id2conf = res.get_id2config_mapping() # get best/incubent run best_run = res.get_incumbent_id() best_config = id2conf[best_run]['config'] print(f"Best run id:{best_run}, \n Config:{best_config}") # Store all run info file = open(os.path.join(run_dir, 'summary.txt'), 'w') file.write(f"{all_runs}") file.close()
def run_bohb_parallel(id, run_id, bohb_workers): # get bohb params bohb_params = get_bohb_parameters() # get suitable interface (eth0 or lo) bohb_interface = get_bohb_interface() # get BOHB log directory working_dir = get_working_dir(run_id) # every process has to lookup the hostname host = hpns.nic_name_to_host(bohb_interface) os.makedirs(working_dir, exist_ok=True) if int(id) > 0: print('START NEW WORKER') time.sleep(10) w = BohbWorker(host=host, run_id=run_id, working_dir=working_dir) w.load_nameserver_credentials(working_directory=working_dir) w.run(background=False) exit(0) print('START NEW MASTER') ns = hpns.NameServer(run_id=run_id, host=host, port=0, working_directory=working_dir) ns_host, ns_port = ns.start() w = BohbWorker(host=host, nameserver=ns_host, nameserver_port=ns_port, run_id=run_id, working_dir=working_dir) w.run(background=True) result_logger = hpres.json_result_logger(directory=working_dir, overwrite=True) bohb = BohbWrapper(configspace=get_configspace(), run_id=run_id, eta=bohb_params['eta'], host=host, nameserver=ns_host, nameserver_port=ns_port, min_budget=bohb_params['min_budget'], max_budget=bohb_params['max_budget'], result_logger=result_logger) res = bohb.run(n_iterations=bohb_params['iterations'], min_n_workers=int(bohb_workers)) bohb.shutdown(shutdown_workers=True) ns.shutdown() return res
def test_optimizer(self): class ResultNode(PipelineNode): def fit(self, X_train, Y_train): return { 'loss': X_train.shape[1], 'info': { 'train_a': X_train.shape[1], 'train_b': Y_train.shape[1] } } def get_hyperparameter_search_space(self, **pipeline_config): cs = CS.ConfigurationSpace() cs.add_hyperparameter( CSH.UniformIntegerHyperparameter('hyper', lower=0, upper=30)) return cs def get_pipeline_config_options(self): return [ ConfigOption("result_logger_dir", default=".", type="directory"), ConfigOption("optimize_metric", default="a", type=str), ] logger = logging.getLogger('hpbandster') logger.setLevel(logging.ERROR) logger = logging.getLogger('autonet') logger.setLevel(logging.ERROR) pipeline = Pipeline([OptimizationAlgorithm([ResultNode()])]) pipeline_config = pipeline.get_pipeline_config(num_iterations=1, budget_type='epochs', result_logger_dir=".") pipeline.fit_pipeline( pipeline_config=pipeline_config, X_train=np.random.rand(15, 10), Y_train=np.random.rand(15, 5), X_valid=None, Y_valid=None, result_loggers=[json_result_logger(directory=".", overwrite=True)], dataset_info=None, shutdownables=[]) result_of_opt_pipeline = pipeline[OptimizationAlgorithm.get_name( )].fit_output['optimized_hyperparameter_config'] print(pipeline[OptimizationAlgorithm.get_name()].fit_output) self.assertIn( result_of_opt_pipeline[ResultNode.get_name() + ConfigWrapper.delimiter + 'hyper'], list(range(0, 31)))
def get_parameters(train_data, kFold, iterations, save=False, filepath = './result/loss_time_bohb.csv'): parser = argparse.ArgumentParser(description='Example 1 - sequential and local execution.') parser.add_argument('--min_budget', type=float, help='Minimum budget used during the optimization.', default=1) parser.add_argument('--max_budget', type=float, help='Maximum budget used during the optimization.', default=1) parser.add_argument('--n_iterations', type=int, help='Number of iterations performed by the optimizer', default=iterations) # max value = 4 # parser.add_argument('--worker', help='Flag to turn this into a worker process', action='store_true') parser.add_argument('--shared_directory', type=str,help='A directory that is accessible for all processes, e.g. a NFS share.', default='./result') # parser.add_argument('--nic_name', type=str, default='lo') args = parser.parse_args() result_logger = hpres.json_result_logger(directory=args.shared_directory, overwrite=True) NS = hpns.NameServer(run_id='BOHB', host='127.0.0.1', port=None) NS.start() w = worker(train_data, kFold, nameserver='127.0.0.1', run_id='BOHB') w.run(background=True) bohb = BOHB(configspace=w.get_configspace(), run_id='BOHB', nameserver='127.0.0.1', min_budget=args.min_budget, max_budget=args.max_budget, result_logger=result_logger ) res = bohb.run(n_iterations=args.n_iterations) bohb.shutdown(shutdown_workers=True) NS.shutdown() id2config = res.get_id2config_mapping() incumbent = res.get_incumbent_id() info = res.get_runs_by_id(incumbent) parameter = id2config[incumbent]['config'] min_error = info[0]['loss'] if save: all_info = res.get_all_runs() timepoint_dic = [] loss_dic = [] for i in all_info: timepoint_dic.append(i['time_stamps']['finished']) loss_dic.append(i['loss']) save_to_csv.save(filepath, timepoint_dic, loss_dic) return parameter, min_error
def generate_bohb_data(): import warnings import hpbandster.core.nameserver as hpns import hpbandster.core.result as hpres from hpbandster.optimizers import BOHB as BOHB run_id = '0' # Every run has to have a unique (at runtime) id. NS = hpns.NameServer(run_id=run_id, host='localhost', port=0) ns_host, ns_port = NS.start() from neural_opt import MyWorker, get_configspace w = MyWorker( nameserver=ns_host, nameserver_port=ns_port, run_id=run_id, # same as nameserver's ) w.run(background=True) # Log the optimization results for later analysis result_logger = hpres.json_result_logger( directory='test/general_example/results/bohb_full_configspace', overwrite=True) bohb = BOHB( configspace=get_configspace(), run_id=run_id, # same as nameserver's eta=2, min_budget=5, max_budget=100, # Hyperband parameters nameserver=ns_host, nameserver_port=ns_port, result_logger=result_logger, ) # Then start the optimizer. The n_iterations parameter specifies # the number of iterations to be performed in this run with warnings.catch_warnings(): warnings.simplefilter("ignore") res = bohb.run(n_iterations=2) # After the run is finished, the services started above need to be shutdown. # This ensures that the worker, the nameserver and the master all properly exit # and no (daemon) threads keep running afterwards. # In particular we shutdown the optimizer (which shuts down all workers) and the nameserver. bohb.shutdown(shutdown_workers=True) NS.shutdown()
def run_bohb_serial(run_id, experiment_wrapper): # get bohb parameters bohb_params = experiment_wrapper.get_bohb_parameters() # get BOHB log directory working_dir = get_working_dir(run_id) # assign random port in the 30000-40000 range to avoid using a blocked port because of a previous improper bohb shutdown port = int(30000 + random.random() * 10000) ns = hpns.NameServer(run_id=run_id, host="127.0.0.1", port=port) ns.start() w = BohbWorker(nameserver="127.0.0.1", id=0, run_id=run_id, nameserver_port=port, working_dir=working_dir, experiment_wrapper = experiment_wrapper) w.run(background=True) result_logger = hpres.json_result_logger(directory=working_dir, overwrite=True) bohb = BohbWrapper( configspace=experiment_wrapper.get_configspace(), run_id=run_id, eta=bohb_params['eta'], min_budget=bohb_params['min_budget'], max_budget=bohb_params['max_budget'], random_fraction=bohb_params['random_fraction'], nameserver="127.0.0.1", nameserver_port=port, result_logger=result_logger) res = bohb.run(n_iterations=bohb_params['iterations']) bohb.shutdown(shutdown_workers=True) ns.shutdown() return res
def test_write_new_config(self): cs = CS.ConfigurationSpace() cs.add_hyperparameter(CS.CategoricalHyperparameter('test', [1])) with tempfile.TemporaryDirectory() as temp_dir: logger = json_result_logger(temp_dir) logger.new_config('1', cs.sample_configuration().get_dictionary(), {'test': 'test'}) self.assertTrue(os.path.exists(temp_dir)) self.assertTrue( os.path.exists(os.path.join(temp_dir, 'configs.json'))) self.assertTrue( os.path.exists(os.path.join(temp_dir, 'results.json'))) self.assertEqual(logger.config_ids, set('1')) with open(os.path.join(temp_dir, 'configs.json')) as fh: data = fh.read() data = data.rstrip() self.assertEqual(data, r'["1", {"test": 1}, {"test": "test"}]')
def model_select_bow_vae(args): dd = datetime.datetime.now() id_str = dd.strftime("%Y-%m-%d_%H-%M-%S") ns_port = get_port() worker, log_dir = get_worker(args, args.budget, id_str, ns_port) worker.search_mode = True result_logger = hpres.json_result_logger(directory=log_dir, overwrite=True) logging.info("Starting nameserver on port {}".format(ns_port)) NS = hpns.NameServer(run_id=id_str, host='127.0.0.1', port=ns_port) NS.start() res = select_model(worker, args.config_space, args.iterations, result_logger, id_str, ns_port) id2config = res.get_id2config_mapping() incumbent = res.get_incumbent_id() logging.info('Best found configuration:', id2config[incumbent]['config']) logging.info( 'Total budget corresponds to %.1f full function evaluations.' % (sum([r.budget for r in res.get_all_runs()]) / 32)) inc_runs = res.get_runs_by_id(incumbent) inc_run = inc_runs[-1] inc_loss = inc_run.loss inc_config = id2config[incumbent]['config'] logging.info("Best configuration loss = {}".format(inc_loss)) logging.info("Best configuration {}".format(inc_config)) with open(os.path.join(log_dir, 'model_selection_results.pkl'), 'wb') as fh: pickle.dump(res, fh) with open(os.path.join(log_dir, 'best.model.config'), 'w') as fp: inc_config['training_epochs'] = args.budget specs = json.dumps(inc_config) fp.write(specs) worker.retrain_best_config(inc_config, inc_run.budget, args.seed, args.num_final_evals) dd_finish = datetime.datetime.now() logging.info("Model selection run FINISHED. Time: {}".format(dd_finish - dd)) NS.shutdown()
def runBohbSerial(run_id): # get BOHB log directory working_dir = get_working_dir(run_id) # assign random port in the 30000-40000 range to avoid using a blocked port because of a previous improper bohb shutdown port = int(30000 + random.random() * 10000) # select whether to process NLP or speech datasets use_nlp = 'NLP' in run_id ns = hpns.NameServer(run_id=run_id, host="127.0.0.1", port=port) ns.start() w = BOHBWorker(nameserver="127.0.0.1", run_id=run_id, nameserver_port=port, working_dir=working_dir, use_nlp=use_nlp) w.run(background=True) result_logger = hpres.json_result_logger(directory=working_dir, overwrite=True) bohb = BohbWrapper(configspace=get_configspace(use_nlp), run_id=run_id, eta=BOHB_ETA, min_budget=BOHB_MIN_BUDGET, max_budget=BOHB_MIN_BUDGET, nameserver="127.0.0.1", nameserver_port=port, result_logger=result_logger) res = bohb.run(n_iterations=BOHB_ITERATIONS) bohb.shutdown(shutdown_workers=True) ns.shutdown() return res
log=True) # cs.add_hyperparameters([ dropout_rate, tfbs_filter_size, tfbs_kernel_length, dilation_level, dilation_kernel_length, compression_filter_size, learning_rate ]) return cs # shared_dir = args.shared_dir # results logger result_logger = hpres.json_result_logger(directory=shared_dir, overwrite=False) # initialize name server run_id = args.run_id NS = hpns.NameServer( run_id=run_id, host='127.0.0.1', working_directory=shared_dir, ) ns_host, ns_port = NS.start() # define worker worker = KerasWorker(dataset=args.dataset, use_bg=False, sequence_length=1000, host='127.0.0.1', run_id=run_id, nameserver=ns_host,
def main(): parser = argparse.ArgumentParser( description= 'Tensorforce hyperparameter tuner, using BOHB optimizer (Bayesian Optimization ' 'and Hyperband)') # Environment arguments (from run.py) parser.add_argument( '-e', '--environment', type=str, help='Environment (name, configuration JSON file, or library module)') parser.add_argument( '-l', '--level', type=str, default=None, help='Level or game id, like `CartPole-v1`, if supported') parser.add_argument('-m', '--max-episode-timesteps', type=int, default=None, help='Maximum number of timesteps per episode') parser.add_argument( '--import-modules', type=str, default=None, help='Import comma-separated modules required for environment') # Runner arguments (from run.py) parser.add_argument('-n', '--episodes', type=int, help='Number of episodes') parser.add_argument( '-p', '--num-parallel', type=int, default=None, help='Number of environment instances to execute in parallel') # Tuner arguments parser.add_argument( '-r', '--runs-per-round', type=str, default='1,2,5,10', help= 'Comma-separated number of runs per optimization round, each with a successively ' 'smaller number of candidates') parser.add_argument( '-s', '--selection-factor', type=int, default=3, help= 'Selection factor n, meaning that one out of n candidates in each round advances to ' 'the next optimization round') parser.add_argument( '-i', '--num-iterations', type=int, default=1, help= 'Number of optimization iterations, each consisting of a series of optimization ' 'rounds with an increasingly reduced candidate pool') parser.add_argument('-d', '--directory', type=str, default='tuner', help='Output directory') parser.add_argument('--restore', type=str, default=None, help='Restore from given directory') parser.add_argument('--id', type=str, default='worker', help='Unique worker id') args = parser.parse_args() if args.import_modules is not None: for module in args.import_modules.split(','): importlib.import_module(name=module) environment = dict(environment=args.environment) if args.level is not None: environment['level'] = args.level if False: host = nic_name_to_host(nic_name=None) port = 123 else: host = 'localhost' port = None runs_per_round = tuple(int(x) for x in args.runs_per_round.split(',')) print('Bayesian Optimization and Hyperband optimization') print( f'{args.num_iterations} iterations of each {len(runs_per_round)} rounds:' ) for n, num_runs in enumerate(runs_per_round, start=1): num_candidates = round( math.pow(args.selection_factor, len(runs_per_round) - n)) print(f'round {n}: {num_candidates} candidates, each {num_runs} runs') print() server = NameServer(run_id=args.id, working_directory=args.directory, host=host, port=port) nameserver, nameserver_port = server.start() worker = TensorforceWorker( environment=environment, max_episode_timesteps=args.max_episode_timesteps, num_episodes=args.episodes, base=args.selection_factor, runs_per_round=runs_per_round, num_parallel=args.num_parallel, run_id=args.id, nameserver=nameserver, nameserver_port=nameserver_port, host=host) worker.run(background=True) if args.restore is None: previous_result = None else: previous_result = logged_results_to_HBS_result(directory=args.restore) result_logger = json_result_logger(directory=args.directory, overwrite=True) optimizer = BOHB(configspace=worker.get_configspace(), eta=args.selection_factor, min_budget=0.9, max_budget=math.pow(args.selection_factor, len(runs_per_round) - 1), run_id=args.id, working_directory=args.directory, nameserver=nameserver, nameserver_port=nameserver_port, host=host, result_logger=result_logger, previous_result=previous_result) # BOHB(configspace=None, eta=3, min_budget=0.01, max_budget=1, min_points_in_model=None, # top_n_percent=15, num_samples=64, random_fraction=1 / 3, bandwidth_factor=3, # min_bandwidth=1e-3, **kwargs) # Master(run_id, config_generator, working_directory='.', ping_interval=60, # nameserver='127.0.0.1', nameserver_port=None, host=None, shutdown_workers=True, # job_queue_sizes=(-1,0), dynamic_queue_size=True, logger=None, result_logger=None, # previous_result = None) # logger: logging.logger like object, the logger to output some (more or less meaningful) # information results = optimizer.run(n_iterations=args.num_iterations) # optimizer.run(n_iterations=1, min_n_workers=1, iteration_kwargs={}) # min_n_workers: int, minimum number of workers before starting the run optimizer.shutdown(shutdown_workers=True) server.shutdown() with open(os.path.join(args.directory, 'results.pkl'), 'wb') as filehandle: pickle.dump(results, filehandle) print('Best found configuration: {}'.format( results.get_id2config_mapping()[results.get_incumbent_id()]['config'])) print('Runs:', results.get_runs_by_id(config_id=results.get_incumbent_id())) print('A total of {} unique configurations where sampled.'.format( len(results.get_id2config_mapping()))) print('A total of {} runs where executed.'.format( len(results.get_all_runs())))
def run_bohb(args): scorer = TrueRougeScorer() run_id = infer_run_id_from_args(args) run_dir = os.path.join(args.hpb_runs_dir, run_id) if args.hpb_worker: w = BeamWorker(args, scorer, nameserver='127.0.0.1', run_id=run_id) w.run(background=False) exit(0) result_logger = hpres.json_result_logger(directory=run_dir, overwrite=getattr( args, 'hpb_overwrite_run', False)) # Step 1: Start a nameserver # Every run needs a nameserver. It could be a 'static' server with a # permanent address, but here it will be started for the local machine with the default port. # The nameserver manages the concurrent running workers across all possible threads or clusternodes. # Note the run_id argument. This uniquely identifies a run of any HpBandSter optimizer. NS = hpns.NameServer(run_id=run_id, host='127.0.0.1', port=None) NS.start() # Step 2: Start a worker # Now we can instantiate a worker, providing the mandatory information # Besides the sleep_interval, we need to define the nameserver information and # the same run_id as above. After that, we can start the worker in the background, # where it will wait for incoming configurations to evaluate. # Step 3: Run an optimizer # Now we can create an optimizer object and start the run. # Here, we run BOHB, but that is not essential. # The run method will return the `Result` that contains all runs performed. bohb = BOHB(configspace=get_conf_space(args), run_id=run_id, nameserver='127.0.0.1', result_logger=result_logger, min_budget=args.hpb_min_budget, max_budget=args.hpb_max_budget) res = bohb.run(n_iterations=args.hpb_n_iterations, min_n_workers=args.hpb_n_workers) # Step 4: Shutdown # After the optimizer run, we must shutdown the master and the nameserver. bohb.shutdown(shutdown_workers=True) NS.shutdown() # Step 5: Analysis # Each optimizer returns a hpbandster.core.result.Result object. # It holds informations about the optimization run like the incumbent (=best) configuration. # For further details about the Result object, see its documentation. # Here we simply print out the best config and some statistics about the performed runs. id2config = res.get_id2config_mapping() incumbent = res.get_incumbent_id() print('Best found configuration:', id2config[incumbent]['config']) print('A total of %i unique configurations where sampled.' % len(id2config.keys())) print('A total of %i runs where executed.' % len(res.get_all_runs())) print('Total budget corresponds to %.1f full function evaluations.' % (sum([r.budget for r in res.get_all_runs()]) / args.hpb_max_budget))
default='.') args = parser.parse_args() # Every process has to lookup the hostname host = hpns.nic_name_to_host(args.nic_name) if args.worker: w = worker('mnist.json', path='../surrogate_data', run_id=args.run_id, host=host) w.load_nameserver_credentials(working_directory=args.shared_directory) w.run(background=False) exit(0) result_logger = hpres.json_result_logger(directory=args.shared_directory, overwrite=True) # Start a nameserver: NS = hpns.NameServer(run_id=args.run_id, host=host, port=0, working_directory=args.shared_directory) ns_host, ns_port = NS.start() # Start local worker w = worker('mnist.json', path='../surrogate_data', run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port)
# BOHB is usually so cheap, that we can # affort to run a worker on the master node, too. worker = worker(min_budget=min_budget, max_budget=max_budget, eta=eta, search_space=args.space, nasbench_data=nasbench, seed=args.seed, nameserver=ns_host, nameserver_port=ns_port, run_id=args.run_id) worker.run(background=True) #instantiate BOHB and run it result_logger = hputil.json_result_logger(directory=args.working_directory, overwrite=True) HPB = BOHB(configspace=worker.get_config_space(), working_directory=args.working_directory, run_id=args.run_id, eta=eta, min_budget=min_budget, max_budget=max_budget, host=ns_host, nameserver=ns_host, nameserver_port=ns_port, ping_interval=3600, result_logger=result_logger) res = HPB.run(n_iterations=args.num_iterations, min_n_workers=args.total_num_workers)
return skopt_space if __name__ == "__main__": NS = hpns.NameServer(run_id=RUN_ID, host=HOST, port=PORT, working_directory=WORKING_DIRECTORY) ns_host, ns_port = NS.start() # Start local worker worker = TrainEvalWorker(run_id=RUN_ID, nameserver=ns_host, nameserver_port=ns_port) worker.run(background=True) result_logger = hpres.json_result_logger( directory="data/hpbandster/{}".format(RUN_ID), overwrite=False) optim = BOHB( configspace=worker.get_configspace(), run_id=RUN_ID, nameserver=ns_host, nameserver_port=ns_port, result_logger=result_logger, eta=HPO_PARAMS["eta"], min_budget=HPO_PARAMS["min_budget"], max_budget=HPO_PARAMS["max_budget"], num_samples=HPO_PARAMS["num_samples"], top_n_percent=HPO_PARAMS["top_n_percent"], min_bandwidth=HPO_PARAMS["min_bandwidth"], bandwidth_factor=HPO_PARAMS["bandwidth_factor"], ) study = optim.run(n_iterations=HPO_PARAMS["n_calls"])
def fmin(func, config_space, func_args=(), eta=2, min_budget=2, max_budget=4, num_iterations=1, num_workers=1, output_dir='.'): """ Starts a local BOHB optimization run for a function over a hyperparameter search space, which is referred to as configuration space. This function's purpose is to give a fast and easy way to run BOHB on a optimization objective on your local machine. The optimized function must satisfy the following conditions: - Contain a parameter ``budget``: This parameter is passed by the optimizer. Its meaning is defined by your interpretation of the budget used by your model. For example it may be the number of epochs for a neural network to train or the number of datapoints, the model receives. The idea is to run many configurations on a small budget and only take the best 1/``eta`` of them to the next round. In the next iteration, the configurations run on the doubled budget. This is repeated until only 2 configurations are left to run on the ``max_budget``. Therefore, bad configurations are rejected fast, and the good ones are explored more. The number of configurations with a minimum budget is calculated similar to the optimization run, just reversed. Having 2 configurations with ``max_budget``, in the iteration before ``eta``-times many configurations with half the budget are sampled, and so on. - Hyperparameter from the configuration space object: The function must implement all hyperparameters defined in the configuration space. The parameter name in the function call must be equal to the name of the hyperparameter. Otherwise, a exception will be thrown. - Function arguments in the right order: Function arguments, which are not hyperparameters and therefore not defined in the configuration space must be passed to the ``fmin`` call in the order of occurrence in the function signature. In the example below, the training data, X and y, is a use case for this kind of function arguments. Example:: import numpy as np from FMin import fmin import ConfigSpace as CS # Create configuration space cs = CS.ConfigurationSpace() cs.add_hyperparameter( CS.UniformFloatHyperparameter('w', lower=-5, upper=5) ) # Create data from function # f(x) = x + :math:`\mathcal{N}(0, 1)` X = np.random.uniform(-5, 5, 100) y = np.random.normal(X, 1) # The function calculates the mean squared error for the first # ``budget`` points compared to their responding true values. # The expected minimum is at w = 1. opt_func = lambda x, y, w, budget: np.mean((y[:int(budget)] - w*x[:int(budget)])**2) inc_best, inc_best_cfg, result = fmin(opt_func, cs, func_args=(X, y), min_budget=3, max_budget=len(X), num_iterations=3, num_worker=1) Args: func (function): function to optimize. Must return a python scalar! See also the section above **The optimized function must satisfy the following conditions** config_space (ConfigSpace.ConfigurationSpace): Definition of the search space containing all hyperparameters and their value ranges. You can find its definition in the `ConfiSpace repository <https://github.com/automl/ConfigSpace/>`_. func_args (tuple): arguments, passed to the function by the user, e.g., the data (X,y). These arguments don't include optimized parameters. Those are defined in the configuration space object and will be passed by the master directly to the function. eta (float): In each iteration, a complete run of sequential halving is executed. In it, after evaluating each configuration on the same subset size, only a fraction of 1/eta of them 'advances' to the next round. Must be greater or equal to 2. min_budget (int, float, optional): Defines the minimum budget to evaluate configurations on it. In combination with the parameter `max_budget` and `eta`, the number of configurations to evaluate is determined. Read more about it in the `Quickstart <https://automl.github.io/HpBandSter/build/html/quickstart.html#id6>`_. By default `min_budget` and `max_budget` is set, so that only a few configurations with budgets from 1 to 4 are evaluated. max_budget (int, float, optional): Defines the maximum budget to evaluate configurations on it. num_iterations (int, optional): number of iterations to be performed in this run. By default, this value is set to 1. num_workers (int, optional): number of parallel workers. By default, just one worker is used. output_dir (str, optional): HpBandSter stores the sampled configurations and the results on these configurations in two .json files. 'configs.json' and 'results.json'. Those files will be stored by default in the current directory (default='.'). Also, we store the configuration space definition for later use to this directory. It may be used for further analysis via `CAVE <https://automl.github.io/CAVE/stable/>`_. Returns: hpbandster.core.result.Run - Best run. Run result with the best loss values of all budgets. It stores information about the - budget - the unique configuration id (tuple) - loss - time stamps: start time and end time for this run. Dict - Best found configuration. Containing the configuration (from the configuration space), which achieved the best results in optimization run hpbandster.core.result.Result - Result object stores all results from all results, which were evaluated. The best run and the best found configuration are extracted from this results-object. """ output_dir = Path(output_dir) output_dir.mkdir(exist_ok=True) # Set up a local nameserver and start it ns = hpns.NameServer(run_id='fmin', nic_name=None, working_directory=output_dir) ns_host, ns_port = ns.start() # Create ``num_workers`` workers and pass the function as well as the # function arguments to each of them. workers = [] for _ in range(num_workers): worker = FMinWorker(func=func, func_args=func_args, nameserver=ns_host, nameserver_port=ns_port, run_id='fmin') worker.run(background=True) workers.append(worker) # The result logger will store the intermediate results and the sampled # configurations in the passed directory. result_logger = hpres.json_result_logger(directory=output_dir, overwrite=True) # For hyperparameter importance analysis via CAVE we store the configuration # space definition to file. with open(output_dir / 'configspace.json', 'w') as f: f.write(json.write(config_space)) # Set up a master, which is book keeping and decides what to run next. opt = BOHB(configspace=config_space, run_id='fmin', min_budget=min_budget, max_budget=max_budget, eta=eta, host=ns_host, nameserver=ns_host, nameserver_port=ns_port, result_logger=result_logger) # The result object stores run information, e.g. the incumbent trajectory. # Force the master to wait until all workers are ready. result = opt.run(n_iterations=num_iterations, min_n_workers=num_workers) # After the run has finished, shut down the master and the workers opt.shutdown(shutdown_workers=True) ns.shutdown() # Save to result object to file. with open(output_dir / 'results.pkl', 'wb') as f: import pickle pickle.dump(result, f) # Return the optimal value and the responding configuration, as well as the # result object. The result object can be used in a second step for further # hyperparameter importance analysis with CAVE. id2config = result.get_id2config_mapping() incumbent = result.get_incumbent_id() inc_value = result.get_runs_by_id(incumbent)[-1]['loss'] inc_cfg = id2config[incumbent]['config'] return inc_value, inc_cfg, result
def main(): parser = argparse.ArgumentParser( parents=[get_train_parser()], description='Parallel execution of hyper-tuning', ) parser.add_argument('--run-id', required=True, help='Name of the run') parser.add_argument('--min-budget', type=float, help='Minimum budget used during the optimization', default=1) parser.add_argument('--max-budget', type=float, help='Maximum budget used during the optimization', default=64) parser.add_argument('--n-iterations', type=int, help='Number of iterations performed by the optimizer', default=3) parser.add_argument('--n-workers', type=int, help='Number of workers to run in parallel', default=3) parser.add_argument('--eta', type=int, help='Parameter of the hyper-tuning algorithm', default=4) parser.add_argument('--worker', help='Flag to turn this into a worker process', action='store_true') parser.add_argument('--hostname', default=None, help='IP of name server.') parser.add_argument('--shared-directory', type=str, help=('A directory that is accessible ' 'for all processes, e.g. a NFS share'), default='output/hypertune') args = parser.parse_args() print(args) MyWorker = WORKERS[args.model_type] if not args.hostname and socket.gethostname().lower().startswith('lenovo'): # If we are on cluster set IP args.hostname = hpns.nic_name_to_host('eno1') elif not args.hostname: args.hostname = '127.0.0.1' logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"), format='%(asctime)s %(message)s', datefmt='%I:%M:%S') args.callbacks = ['learning-rate-scheduler', 'early-stopping'] if args.worker: # Start a worker in listening mode (waiting for jobs from master) w = MyWorker( args, run_id=args.run_id, host=args.hostname, ) w.load_nameserver_credentials(working_directory=args.shared_directory) w.run(background=False) exit(0) result_logger = hpres.json_result_logger( directory=args.shared_directory, overwrite=True, ) # Start a name server name_server = hpns.NameServer( run_id=args.run_id, host=args.hostname, port=0, working_directory=args.shared_directory, ) ns_host, ns_port = name_server.start() # Run and optimizer bohb = HyperBand( configspace=MyWorker.get_configspace(), # model can be an arg here? run_id=args.run_id, result_logger=result_logger, eta=args.eta, host=args.hostname, nameserver=ns_host, nameserver_port=ns_port, min_budget=args.min_budget, max_budget=args.max_budget, ) res = bohb.run(n_iterations=args.n_iterations, min_n_workers=args.n_workers) # After the optimizer run, we must shutdown the master and the nameserver. bohb.shutdown(shutdown_workers=True) name_server.shutdown() id2config = res.get_id2config_mapping() incumbent = res.get_incumbent_id() inc_runs = res.get_runs_by_id(incumbent) inc_run = inc_runs[-1] all_runs = res.get_all_runs() print("Best loss {:6.2f}".format(inc_run.loss)) print('A total of %i unique configurations where sampled.' % len(id2config.keys())) print('A total of %i runs where executed.' % len(all_runs)) print('Total budget corresponds to %.1f full function evaluations.' % (sum([r.budget for r in all_runs]) / args.max_budget)) print('The run took %.1f seconds to complete.' % (all_runs[-1].time_stamps['finished'] - all_runs[0].time_stamps['started']))
def main(): parser = argparse.ArgumentParser(description='Tensorforce hyperparameter tuner') parser.add_argument( 'environment', help='Environment (name, configuration JSON file, or library module)' ) parser.add_argument( '-l', '--level', type=str, default=None, help='Level or game id, like `CartPole-v1`, if supported' ) parser.add_argument( '-m', '--max-repeats', type=int, default=1, help='Maximum number of repetitions' ) parser.add_argument( '-n', '--num-iterations', type=int, default=1, help='Number of BOHB iterations' ) parser.add_argument( '-d', '--directory', type=str, default='tuner', help='Output directory' ) parser.add_argument( '-r', '--restore', type=str, default=None, help='Restore from given directory' ) parser.add_argument('--id', type=str, default='worker', help='Unique worker id') args = parser.parse_args() if args.level is None: environment = Environment.create(environment=args.environment) else: environment = Environment.create(environment=args.environment, level=args.level) if False: host = nic_name_to_host(nic_name=None) port = 123 else: host = 'localhost' port = None server = NameServer(run_id=args.id, working_directory=args.directory, host=host, port=port) nameserver, nameserver_port = server.start() worker = TensorforceWorker( environment=environment, run_id=args.id, nameserver=nameserver, nameserver_port=nameserver_port, host=host ) # TensorforceWorker(run_id, nameserver=None, nameserver_port=None, logger=None, host=None, id=None, timeout=None) # logger: logging.logger instance, logger used for debugging output # id: anything with a __str__method, if multiple workers are started in the same process, you MUST provide a unique id for each one of them using the `id` argument. # timeout: int or float, specifies the timeout a worker will wait for a new after finishing a computation before shutting down. Towards the end of a long run with multiple workers, this helps to shutdown idling workers. We recommend a timeout that is roughly half the time it would take for the second largest budget to finish. The default (None) means that the worker will wait indefinitely and never shutdown on its own. worker.run(background=True) # config = cs.sample_configuration().get_dictionary() # print(config) # res = worker.compute(config=config, budget=1, working_directory='.') # print(res) if args.restore is None: previous_result = None else: previous_result = logged_results_to_HBS_result(directory=args.restore) result_logger = json_result_logger(directory=args.directory, overwrite=True) # ??? optimizer = BOHB( configspace=worker.get_configspace(), min_budget=0.5, max_budget=float(args.max_repeats), run_id=args.id, working_directory=args.directory, nameserver=nameserver, nameserver_port=nameserver_port, host=host, result_logger=result_logger, previous_result=previous_result ) # BOHB(configspace=None, eta=3, min_budget=0.01, max_budget=1, min_points_in_model=None, top_n_percent=15, num_samples=64, random_fraction=1 / 3, bandwidth_factor=3, min_bandwidth=1e-3, **kwargs) # Master(run_id, config_generator, working_directory='.', ping_interval=60, nameserver='127.0.0.1', nameserver_port=None, host=None, shutdown_workers=True, job_queue_sizes=(-1,0), dynamic_queue_size=True, logger=None, result_logger=None, previous_result = None) # logger: logging.logger like object, the logger to output some (more or less meaningful) information results = optimizer.run(n_iterations=args.num_iterations) # optimizer.run(n_iterations=1, min_n_workers=1, iteration_kwargs={}) # min_n_workers: int, minimum number of workers before starting the run optimizer.shutdown(shutdown_workers=True) server.shutdown() environment.close() with open(os.path.join(args.directory, 'results.pkl'), 'wb') as filehandle: pickle.dump(results, filehandle) print('Best found configuration:', results.get_id2config_mapping()[results.get_incumbent_id()]['config']) print('Runs:', results.get_runs_by_id(config_id=results.get_incumbent_id())) print('A total of {} unique configurations where sampled.'.format(len(results.get_id2config_mapping()))) print('A total of {} runs where executed.'.format(len(results.get_all_runs()))) print('Total budget corresponds to {:.1f} full function evaluations.'.format( sum([r.budget for r in results.get_all_runs()]) / args.max_repeats) )
def run_opt(args): # import PyTorchWorker as worker # Every process has to lookup the hostname host = hpns.nic_name_to_host(args.nic_name) result_logger = hpres.json_result_logger(directory=args.shared_directory, overwrite=True) # Start a nameserver: NS = hpns.NameServer(run_id=args.run_id, host=host, port=0, working_directory=args.shared_directory) ns_host, ns_port = NS.start() # Start local worker w = worker(run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port, timeout=120) w.run(background=True) if args.method == "BOHB": print("[RUNNER] method: BOHB") opt = BOHB( configspace=worker.get_configspace(), run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port, result_logger=result_logger, min_budget=args.min_budget, max_budget=args.max_budget, ) elif args.method == "random": print("[RUNNER] method: random") opt = RandomSearch( configspace=worker.get_configspace(), run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port, result_logger=result_logger, min_budget=args.min_budget, max_budget=args.max_budget, ) elif args.method == "BO": print("[RUNNER] method: BO") opt = BO_Search( configspace=worker.get_configspace(), run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port, result_logger=result_logger, min_budget=args.min_budget, max_budget=args.max_budget, ) elif args.method == "HB": opt = HyperBand( configspace=worker.get_configspace(), run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port, result_logger=result_logger, min_budget=args.min_budget, max_budget=args.max_budget, ) res = opt.run(n_iterations=args.n_iterations) # store results with open(os.path.join(args.shared_directory, 'results.pkl'), 'wb') as fh: pickle.dump(res, fh) # shutdown opt.shutdown(shutdown_workers=True) NS.shutdown()
def run_optimization(args): """Runs the optimization process.""" print("Starting name server.") date_time = datetime.datetime.now().strftime('%Y-%m-%d-%H_%M_%S') # First start nameserver NS = hpns.NameServer(run_id=date_time, host='127.0.0.1', port=None) NS.start() print("Preparing result logger and loading previous run, if it exists.") # Also start result logger output_dir = os.path.join( args.output_dir, datetime.datetime.now().strftime('%Y_%m_%d--%H_%M_%S')) result_logger_path = os.path.join(output_dir, 'results_log.json') best_result_path = os.path.join(output_dir, 'best_config.txt') print("Result logger will be written to %s" % result_logger_path) if os.path.exists(result_logger_path): previous_run = hpres.logged_results_to_HBS_result(result_logger_path) else: previous_run = None result_logger = hpres.json_result_logger(directory=output_dir, overwrite=True) print("Starting search worker.\n") # Then start worker w = SearchWorker(args.data_path, iaa, os.path.join(output_dir, "logging"), nameserver='127.0.0.1', run_id=date_time) w.run(background=True) print("Initializing optimizer.") # Run the optimizer bohb = BOHB(configspace=w.get_configspace(), run_id=date_time, nameserver='127.0.0.1', result_logger=result_logger, min_budget=args.min_budget, max_budget=args.max_budget, previous_result=previous_run) print("Initialization complete. Starting optimization run.") res = bohb.run(n_iterations=args.iterations) print("Optimization complete.") output_fp = os.path.join(output_dir, 'results.pkl') id2config = res.get_id2config_mapping() incumbent = res.get_incumbent_id() print("Results will be saved at:\n{}".format(output_fp)) print("Best found configuration: ", id2config[incumbent]['config']) Slacker.send_message( "AutoML Optimization finished with minimum " "budget {}, maximum budget {}, and {} " "iterations.\n" "Output file has been written in {} \n".format( args.min_budget, args.max_budget, args.iterations, output_dir), "AutoML Optimization Finished!") sleep(2) Slacker.send_code("Best found configuration:", "{}".format(id2config[incumbent]['config'])) with open(best_result_path, mode='w') as file: lines = [ "Best results are as follows:\n", "{}".format(id2config[incumbent]['config']) ] file.writelines(lines) with open(output_fp, mode='wb') as file: pickle.dump(res, file) # Shutdown after completion bohb.shutdown(shutdown_workers=True) NS.shutdown()
def main(): # Check quantities of train, validation and test images train_images = np.array(glob("data/train/*/*")) valid_images = np.array(glob("data/valid/*/*")) test_images = np.array(glob("data/test/*/*")) # Check relative percentages of image types train_images_mel = np.array(glob("data/train/melanoma/*")) train_images_nev = np.array(glob("data/train/nevus/*")) train_images_seb = np.array(glob("data/train/seborrheic_keratosis/*")) valid_images_mel = np.array(glob("data/valid/melanoma/*")) valid_images_nev = np.array(glob("data/valid/nevus/*")) valid_images_seb = np.array(glob("data/valid/seborrheic_keratosis/*")) test_images_mel = np.array(glob("data/test/melanoma/*")) test_images_nev = np.array(glob("data/test/nevus/*")) test_images_seb = np.array(glob("data/test/seborrheic_keratosis/*")) print( "There are {} training images, {} validation images and {} test images." .format(len(train_images), len(valid_images), len(test_images))) print( "For the training images, {mel:=.1f}% ({mel2}) are of melanoma, {nev:=.1f}% ({nev2}) are of nevus and {seb:=.1f}% ({seb2}) are for seborrheic keratosis." .format(mel=len(train_images_mel) / len(train_images) * 100, mel2=len(train_images_mel), nev=len(train_images_nev) / len(train_images) * 100, nev2=len(train_images_nev), seb=len(train_images_seb) / len(train_images) * 100, seb2=len(train_images_seb))) print( "For the validation images, {mel:=.1f}% ({mel2}) are of melanoma, {nev:=.1f}% ({nev2}) are of nevus and {seb:=.1f}% ({seb2}) are for seborrheic keratosis." .format(mel=len(valid_images_mel) / len(valid_images) * 100, mel2=len(valid_images_mel), nev=len(valid_images_nev) / len(valid_images) * 100, nev2=len(valid_images_nev), seb=len(valid_images_seb) / len(valid_images) * 100, seb2=len(valid_images_seb))) print( "For the test images, {mel:=.1f}% ({mel2}) are of melanoma, {nev:=.1f}% ({nev2}) are of nevus and {seb:=.1f}% ({seb2}) are for seborrheic keratosis." .format(mel=len(test_images_mel) / len(test_images) * 100, mel2=len(test_images_mel), nev=len(test_images_nev) / len(test_images) * 100, nev2=len(test_images_nev), seb=len(test_images_seb) / len(test_images) * 100, seb2=len(test_images_seb))) # Set HpBandSter logging logging.basicConfig(level=logging.DEBUG) # Define the parser. Note that key parametres are the min_budget, max_budget, shared_directory and n_iterations. parser = argparse.ArgumentParser( description='ISIC2017 - CNN on Derm Dataset') parser.add_argument('--min_budget', type=float, help='Minimum number of epochs for training.', default=1) parser.add_argument('--max_budget', type=float, help='Maximum number of epochs for training.', default=3) parser.add_argument('--n_iterations', type=int, help='Number of iterations performed by the optimizer', default=16) parser.add_argument('--worker', help='Flag to turn this into a worker process', action='store_true') parser.add_argument( '--run_id', type=str, help= 'A unique run id for this optimization run. An easy option is to use the job id of the clusters scheduler.' ) parser.add_argument( '--nic_name', type=str, help='Which network interface to use for communication.', default='lo') parser.add_argument( '--shared_directory', type=str, help= 'A directory that is accessible for all processes, e.g. a NFS share.', default='/home/ubuntu/src/derm-ai/data') parser.add_argument( '--backend', help= 'Toggles which worker is used. Choose between a pytorch and a keras implementation.', choices=['pytorch', 'keras'], default='pytorch') args = parser.parse_args([]) host = hpns.nic_name_to_host(args.nic_name) # This example shows how to log live results. This is most useful # for really long runs, where intermediate results could already be # interesting. The core.result submodule contains the functionality to # read the two generated files (results.json and configs.json) and # create a Result object. result_logger = hpres.json_result_logger(directory=args.shared_directory, overwrite=True) # Start a nameserver: NS = hpns.NameServer(run_id=args.run_id, host=host, port=0, working_directory=args.shared_directory) ns_host, ns_port = NS.start() # Start local worker w = worker(run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port, timeout=120) w.run(background=True) bohb = BOHB( configspace=w.get_configspace(), run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port, result_logger=result_logger, min_budget=args.min_budget, max_budget=args.max_budget, ) # Run an optimizer res = bohb.run(n_iterations=args.n_iterations) bohb.shutdown(shutdown_workers=True) NS.shutdown()
def run_experiment(out_path, on_travis): settings = { 'min_budget': 1, 'max_budget': 9, # number of repetitions; this is the fidelity for this bench 'num_iterations': 10, # Set this to a low number for demonstration 'eta': 3, 'output_dir': Path(out_path) } if on_travis: settings.update(get_travis_settings('bohb')) b = Benchmark(rng=1) b.get_configuration_space(seed=1) settings.get('output_dir').mkdir(exist_ok=True) cs = b.get_configuration_space() seed = get_rng(rng=0) run_id = 'BOHB_on_cartpole' result_logger = hpres.json_result_logger(directory=str( settings.get('output_dir')), overwrite=True) ns = hpns.NameServer(run_id=run_id, host='localhost', working_directory=str(settings.get('output_dir'))) ns_host, ns_port = ns.start() worker = CustomWorker(seed=seed, nameserver=ns_host, nameserver_port=ns_port, run_id=run_id, max_budget=settings.get('max_budget')) worker.run(background=True) master = BOHB(configspace=cs, run_id=run_id, host=ns_host, nameserver=ns_host, nameserver_port=ns_port, eta=settings.get('eta'), min_budget=settings.get('min_budget'), max_budget=settings.get('max_budget'), result_logger=result_logger) result = master.run(n_iterations=settings.get('num_iterations')) master.shutdown(shutdown_workers=True) ns.shutdown() with open(settings.get('output_dir') / 'results.pkl', 'wb') as f: pickle.dump(result, f) id2config = result.get_id2config_mapping() incumbent = result.get_incumbent_id() inc_value = result.get_runs_by_id(incumbent)[-1]['loss'] inc_cfg = id2config[incumbent]['config'] logger.info(f'Inc Config:\n{inc_cfg}\n' f'with Performance: {inc_value:.2f}') if not on_travis: benchmark = Benchmark(container_source='library://phmueller/automl') incumbent_result = benchmark.objective_function_test( configuration=inc_cfg, fidelity={"budget": settings['max_budget']}) print(incumbent_result)
# Import a worker class from MAXWEL_worker import MAXWEL_worker as worker #Build an argument parser parser = argparse.ArgumentParser(description='MAXWEL - sequential execution.') parser.add_argument('--min_budget', type=float, help='Minimum budget used during the optimization.', default=5) parser.add_argument('--max_budget', type=float, help='Maximum budget used during the optimization.', default=30) parser.add_argument('--n_iterations', type=int, help='Number of iterations performed by the optimizer', default=10) parser.add_argument('--n_workers', type=int, help='Number of workers to run in parallel.', default=1) parser.add_argument('--shared_directory',type=str, help='A directory that is accessible for all processes, e.g. a NFS share.', default='.') args=parser.parse_args() #Define a realtime result logger result_logger = hpres.json_result_logger(directory=result_dir, overwrite=True) #Start a nameserver NS = hpns.NameServer(run_id='MAXWEL', host='127.0.0.1', port=None) NS.start() #Start the workers workers=[] for i in range(args.n_workers): w = worker(nameserver='127.0.0.1',run_id='MAXWEL', id=i) w.run(background=True) workers.append(w) #Define and run an optimizer bohb = BOHB(configspace = w.get_configspace(),
def get_parameters(data, target_feature_index): parser = argparse.ArgumentParser( description='Example 1 - sequential and local execution.') parser.add_argument('--min_budget', type=float, help='Minimum budget used during the optimization.', default=9) parser.add_argument('--max_budget', type=float, help='Maximum budget used during the optimization.', default=243) parser.add_argument('--n_iterations', type=int, help='Number of iterations performed by the optimizer', default=4) parser.add_argument('--n_workers', type=int, help='Number of workers to run in parallel.', default=2) parser.add_argument('--worker', help='Flag to turn this into a worker process', action='store_true') parser.add_argument( '--run_id', type=str, help= 'A unique run id for this optimization run. An easy option is to use the job id of the clusters scheduler.' ) parser.add_argument( '--nic_name', type=str, help='Which network interface to use for communication.', default='lo') parser.add_argument( '--shared_directory', type=str, help= 'A directory that is accessible for all processes, e.g. a NFS share.', default='/home/lchen/parameters/result') args = parser.parse_args() host = hpns.nic_name_to_host(args.nic_name) if args.worker: time.sleep( 5 ) # short artificial delay to make sure the nameserver is already running w = worker(0.5, data, target_feature_index, run_id=args.run_id, host=host) w.load_nameserver_credentials(working_directory=args.shared_directory) w.run(background=False) exit(0) result_logger = hpres.json_result_logger(directory=args.shared_directory, overwrite=True) # Step 1: Start a nameserver # Every run needs a nameserver. It could be a 'static' server with a # permanent address, but here it will be started for the local machine with the default port. # The nameserver manages the concurrent running workers across all possible threads or clusternodes. # Note the run_id argument. This uniquely identifies a run of any HpBandSter optimizer. NS = hpns.NameServer(run_id='test1', host=host, port=0, working_directory=args.shared_directory) ns_host, ns_port = NS.start() # Step 2: Start a worker # Now we can instantiate a worker, providing the mandatory information # Besides the sleep_interval, we need to define the nameserver information and # the same run_id as above. After that, we can start the worker in the background, # where it will wait for incoming configurations to evaluate. w = worker(0.5, data, target_feature_index, run_id='test1', host=host, nameserver=ns_host, nameserver_port=ns_port) w.run(background=True) # Step 3: Run an optimizer # Now we can create an optimizer object and start the run. # Here, we run BOHB, but that is not essential. # The run method will return the `Result` that contains all runs performed. bohb = BOHB(configspace=worker.get_configspace(), run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port, result_logger=result_logger, min_budget=args.min_budget, max_budget=args.max_budget) print("daozhele5") res = bohb.run(n_iterations=args.n_iterations) print("daozhele6") bohb.shutdown(shutdown_workers=True) NS.shutdown() # Step 5: Analysis # Each optimizer returns a hpbandster.core.result.Result object. # It holds informations about the optimization run like the incumbent (=best) configuration. # For further details about the Result object, see its documentation. # Here we simply print out the best config and some statistics about the performed runs. id2config = res.get_id2config_mapping() incumbent = res.get_incumbent_id() info = res.get_runs_by_id(incumbent) parameter = id2config[incumbent]['config'] min_error = info[0]['loss'] feature_importance_dict = info[0]['info'] with open(os.path.join(args.shared_directory, 'results.pkl'), 'wb') as fh: pickle.dump(res, fh) return parameter, min_error, feature_importance_dict
def main(args): extra_string = '' if args.dataset == 'flights': if args.n_split == 0: extra_string += '_2M' elif args.n_split == 1: extra_string += '_800k' else: raise Exception( 'Only Valid values for flight splits are 0 (2M) or 1 (800k)') extra_string += '_valprop_' + str(args.valprop) elif args.dataset in [ 'boston', 'concrete', 'energy', 'power', 'wine', 'yacht', 'kin8nm', 'naval', 'protein', 'boston_gap', 'concrete_gap', 'energy_gap', 'power_gap', 'wine_gap', 'yacht_gap', 'kin8nm_gap', 'naval_gap', 'protein_gap' ]: extra_string += '_split_' + str(args.n_split) extra_string += '_valprop_' + str(args.valprop) working_dir = args.result_folder + '/' + args.dataset + extra_string + '/' + args.method +\ ('-' + args.network if args.network != "ResNet" else '') + '/' + str(args.width) + '/' + str(args.batch_size) +\ '/' + args.run_id print("WORKING DIR") print(working_dir) # Create data dir if necessary if not os.path.exists(args.data_folder): mkdir(args.data_folder) # Every process has to lookup the hostname host = hpns.nic_name_to_host(args.nic_name) result_logger = hpres.json_result_logger(directory=working_dir, overwrite=False) # Start a nameserver: NS = hpns.NameServer(run_id=args.run_id, host=host, port=0, working_directory=working_dir) ns_host, ns_port = NS.start() workers = [] for i in range(args.num_workers): print("CREATING WORKER:", i) if args.dataset == 'spirals': worker_class = create_SpiralsWorker(args.method, args.network, args.width, args.batch_size) worker = worker_class(early_stop=args.early_stop, run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port, timeout=600, id=i) elif args.dataset == 'flights': worker_class = create_FlightWorker(args.method, args.network, args.width, args.batch_size) worker = worker_class(base_dir=args.data_folder, prop_val=args.valprop, k800=(args.n_split == 1), early_stop=args.early_stop, run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port, timeout=600, id=i) elif args.dataset in [ 'boston', 'concrete', 'energy', 'power', 'wine', 'yacht', 'kin8nm', 'naval', 'protein', 'boston_gap', 'concrete_gap', 'energy_gap', 'power_gap', 'wine_gap', 'yacht_gap', 'kin8nm_gap', 'naval_gap', 'protein_gap' ]: worker_class = create_UCIWorker(args.method, args.network, args.width, args.batch_size) worker = worker_class(dname=args.dataset, base_dir=args.data_folder, prop_val=args.valprop, n_split=args.n_split, early_stop=args.early_stop, run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port, timeout=600, id=i) else: raise ValueError('Dataset not implemented yet!') worker.run(background=True) workers.append(worker) n_iterations = args.n_iterations previous_run = None if args.previous_result_folder is not None: try: previous_run = hpres.logged_results_to_HBS_result( args.previous_result_folder) except Exception as e: print(e) # Run an optimizer bohb = BOHB( configspace=worker.get_configspace(), run_id=args.run_id, host=host, nameserver=ns_host, nameserver_port=ns_port, result_logger=result_logger, min_budget=args.min_budget, max_budget=args.max_budget, previous_result=previous_run, ) res = bohb.run(n_iterations=n_iterations, min_n_workers=args.num_workers) # store results with open(os.path.join(working_dir, 'results.pkl'), 'wb') as fh: pickle.dump(res, fh) # shutdown bohb.shutdown(shutdown_workers=True) NS.shutdown() id2config = res.get_id2config_mapping() incumbent = res.get_incumbent_id() id2config = res.get_id2config_mapping() incumbent = res.get_incumbent_id() all_runs = res.get_all_runs() print('Best found configuration:', id2config[incumbent]['config']) print('A total of %i unique configurations where sampled.' % len(id2config.keys())) print('A total of %i runs where executed.' % len(res.get_all_runs())) print('Total budget corresponds to %.1f full function evaluations.' % (sum([r.budget for r in all_runs]) / args.max_budget)) print('The run took %.1f seconds to complete.' % (all_runs[-1].time_stamps['finished'] - all_runs[0].time_stamps['started']))