def get_nameserver(self, run_id, task_id, ns_credentials_dir, network_interface_name): if not os.path.isdir(ns_credentials_dir): try: os.mkdir(ns_credentials_dir) except: pass return NameServer(run_id=run_id, nic_name=network_interface_name, working_directory=ns_credentials_dir)
def get_nameserver(self, run_id, task_id, ns_credentials_dir, network_interface_name): """Get the namesever object Arguments: run_id {str} -- The id of the run task_id {int} -- An id for the worker ns_credentials_dir {str} -- Path to ns credentials network_interface_name {str} -- The network interface name Returns: NameServer -- The NameServer object """ if not os.path.isdir(ns_credentials_dir): try: os.mkdir(ns_credentials_dir) except: pass return NameServer(run_id=run_id, nic_name=network_interface_name, working_directory=ns_credentials_dir)
max_budget = args.max_budget eta = args.eta args.working_directory = os.path.join( args.working_directory, "search_space_{}/run{}-seed{}".format(args.space, args.run_id, args.seed)) nasbench = NasbenchWrapper( dataset_file= 'src/nasbench_analysis/nasbench_data/108_e/nasbench_only108.tfrecord') if args.array_id == 1: os.makedirs(args.working_directory, exist_ok=True) NS = NameServer(run_id=args.run_id, nic_name='eth0', working_directory=args.working_directory) ns_host, ns_port = NS.start() # BOHB is usually so cheap, that we can # affort to run a worker on the master node, too. worker = worker(min_budget=min_budget, max_budget=max_budget, eta=eta, search_space=args.space, nasbench_data=nasbench, seed=args.seed, nameserver=ns_host, nameserver_port=ns_port, run_id=args.run_id) worker.run(background=True)
def main(): parser = argparse.ArgumentParser( description= 'Tensorforce hyperparameter tuner, using BOHB optimizer (Bayesian Optimization ' 'and Hyperband)') # Environment arguments (from run.py) parser.add_argument( '-e', '--environment', type=str, help='Environment (name, configuration JSON file, or library module)') parser.add_argument( '-l', '--level', type=str, default=None, help='Level or game id, like `CartPole-v1`, if supported') parser.add_argument('-m', '--max-episode-timesteps', type=int, default=None, help='Maximum number of timesteps per episode') parser.add_argument( '--import-modules', type=str, default=None, help='Import comma-separated modules required for environment') # Runner arguments (from run.py) parser.add_argument('-n', '--episodes', type=int, help='Number of episodes') parser.add_argument( '-p', '--num-parallel', type=int, default=None, help='Number of environment instances to execute in parallel') # Tuner arguments parser.add_argument( '-r', '--runs-per-round', type=str, default='1,2,5,10', help= 'Comma-separated number of runs per optimization round, each with a successively ' 'smaller number of candidates') parser.add_argument( '-s', '--selection-factor', type=int, default=3, help= 'Selection factor n, meaning that one out of n candidates in each round advances to ' 'the next optimization round') parser.add_argument( '-i', '--num-iterations', type=int, default=1, help= 'Number of optimization iterations, each consisting of a series of optimization ' 'rounds with an increasingly reduced candidate pool') parser.add_argument('-d', '--directory', type=str, default='tuner', help='Output directory') parser.add_argument('--restore', type=str, default=None, help='Restore from given directory') parser.add_argument('--id', type=str, default='worker', help='Unique worker id') args = parser.parse_args() if args.import_modules is not None: for module in args.import_modules.split(','): importlib.import_module(name=module) environment = dict(environment=args.environment) if args.level is not None: environment['level'] = args.level if False: host = nic_name_to_host(nic_name=None) port = 123 else: host = 'localhost' port = None runs_per_round = tuple(int(x) for x in args.runs_per_round.split(',')) print('Bayesian Optimization and Hyperband optimization') print( f'{args.num_iterations} iterations of each {len(runs_per_round)} rounds:' ) for n, num_runs in enumerate(runs_per_round, start=1): num_candidates = round( math.pow(args.selection_factor, len(runs_per_round) - n)) print(f'round {n}: {num_candidates} candidates, each {num_runs} runs') print() server = NameServer(run_id=args.id, working_directory=args.directory, host=host, port=port) nameserver, nameserver_port = server.start() worker = TensorforceWorker( environment=environment, max_episode_timesteps=args.max_episode_timesteps, num_episodes=args.episodes, base=args.selection_factor, runs_per_round=runs_per_round, num_parallel=args.num_parallel, run_id=args.id, nameserver=nameserver, nameserver_port=nameserver_port, host=host) worker.run(background=True) if args.restore is None: previous_result = None else: previous_result = logged_results_to_HBS_result(directory=args.restore) result_logger = json_result_logger(directory=args.directory, overwrite=True) optimizer = BOHB(configspace=worker.get_configspace(), eta=args.selection_factor, min_budget=0.9, max_budget=math.pow(args.selection_factor, len(runs_per_round) - 1), run_id=args.id, working_directory=args.directory, nameserver=nameserver, nameserver_port=nameserver_port, host=host, result_logger=result_logger, previous_result=previous_result) # BOHB(configspace=None, eta=3, min_budget=0.01, max_budget=1, min_points_in_model=None, # top_n_percent=15, num_samples=64, random_fraction=1 / 3, bandwidth_factor=3, # min_bandwidth=1e-3, **kwargs) # Master(run_id, config_generator, working_directory='.', ping_interval=60, # nameserver='127.0.0.1', nameserver_port=None, host=None, shutdown_workers=True, # job_queue_sizes=(-1,0), dynamic_queue_size=True, logger=None, result_logger=None, # previous_result = None) # logger: logging.logger like object, the logger to output some (more or less meaningful) # information results = optimizer.run(n_iterations=args.num_iterations) # optimizer.run(n_iterations=1, min_n_workers=1, iteration_kwargs={}) # min_n_workers: int, minimum number of workers before starting the run optimizer.shutdown(shutdown_workers=True) server.shutdown() with open(os.path.join(args.directory, 'results.pkl'), 'wb') as filehandle: pickle.dump(results, filehandle) print('Best found configuration: {}'.format( results.get_id2config_mapping()[results.get_incumbent_id()]['config'])) print('Runs:', results.get_runs_by_id(config_id=results.get_incumbent_id())) print('A total of {} unique configurations where sampled.'.format( len(results.get_id2config_mapping()))) print('A total of {} runs where executed.'.format( len(results.get_all_runs())))
def main(): parser = argparse.ArgumentParser(description='Tensorforce hyperparameter tuner') parser.add_argument( 'environment', help='Environment (name, configuration JSON file, or library module)' ) parser.add_argument( '-l', '--level', type=str, default=None, help='Level or game id, like `CartPole-v1`, if supported' ) parser.add_argument( '-m', '--max-repeats', type=int, default=1, help='Maximum number of repetitions' ) parser.add_argument( '-n', '--num-iterations', type=int, default=1, help='Number of BOHB iterations' ) parser.add_argument( '-d', '--directory', type=str, default='tuner', help='Output directory' ) parser.add_argument( '-r', '--restore', type=str, default=None, help='Restore from given directory' ) parser.add_argument('--id', type=str, default='worker', help='Unique worker id') args = parser.parse_args() if args.level is None: environment = Environment.create(environment=args.environment) else: environment = Environment.create(environment=args.environment, level=args.level) if False: host = nic_name_to_host(nic_name=None) port = 123 else: host = 'localhost' port = None server = NameServer(run_id=args.id, working_directory=args.directory, host=host, port=port) nameserver, nameserver_port = server.start() worker = TensorforceWorker( environment=environment, run_id=args.id, nameserver=nameserver, nameserver_port=nameserver_port, host=host ) # TensorforceWorker(run_id, nameserver=None, nameserver_port=None, logger=None, host=None, id=None, timeout=None) # logger: logging.logger instance, logger used for debugging output # id: anything with a __str__method, if multiple workers are started in the same process, you MUST provide a unique id for each one of them using the `id` argument. # timeout: int or float, specifies the timeout a worker will wait for a new after finishing a computation before shutting down. Towards the end of a long run with multiple workers, this helps to shutdown idling workers. We recommend a timeout that is roughly half the time it would take for the second largest budget to finish. The default (None) means that the worker will wait indefinitely and never shutdown on its own. worker.run(background=True) # config = cs.sample_configuration().get_dictionary() # print(config) # res = worker.compute(config=config, budget=1, working_directory='.') # print(res) if args.restore is None: previous_result = None else: previous_result = logged_results_to_HBS_result(directory=args.restore) result_logger = json_result_logger(directory=args.directory, overwrite=True) # ??? optimizer = BOHB( configspace=worker.get_configspace(), min_budget=0.5, max_budget=float(args.max_repeats), run_id=args.id, working_directory=args.directory, nameserver=nameserver, nameserver_port=nameserver_port, host=host, result_logger=result_logger, previous_result=previous_result ) # BOHB(configspace=None, eta=3, min_budget=0.01, max_budget=1, min_points_in_model=None, top_n_percent=15, num_samples=64, random_fraction=1 / 3, bandwidth_factor=3, min_bandwidth=1e-3, **kwargs) # Master(run_id, config_generator, working_directory='.', ping_interval=60, nameserver='127.0.0.1', nameserver_port=None, host=None, shutdown_workers=True, job_queue_sizes=(-1,0), dynamic_queue_size=True, logger=None, result_logger=None, previous_result = None) # logger: logging.logger like object, the logger to output some (more or less meaningful) information results = optimizer.run(n_iterations=args.num_iterations) # optimizer.run(n_iterations=1, min_n_workers=1, iteration_kwargs={}) # min_n_workers: int, minimum number of workers before starting the run optimizer.shutdown(shutdown_workers=True) server.shutdown() environment.close() with open(os.path.join(args.directory, 'results.pkl'), 'wb') as filehandle: pickle.dump(results, filehandle) print('Best found configuration:', results.get_id2config_mapping()[results.get_incumbent_id()]['config']) print('Runs:', results.get_runs_by_id(config_id=results.get_incumbent_id())) print('A total of {} unique configurations where sampled.'.format(len(results.get_id2config_mapping()))) print('A total of {} runs where executed.'.format(len(results.get_all_runs()))) print('Total budget corresponds to {:.1f} full function evaluations.'.format( sum([r.budget for r in results.get_all_runs()]) / args.max_repeats) )
default='beam_search', help='Ensemble selection algorithm') args = parser.parse_args() np.random.seed(args.seed) torch.manual_seed(args.seed) host = nic_name_to_host(args.nic_name) if args.array_id == 1: os.makedirs(args.working_directory, exist_ok=True) with open(os.path.join(args.working_directory, 'settings.txt'), 'w') as f: f.write(str(args)) NS = NameServer(run_id=args.run_id, host=host, working_directory=args.working_directory) ns_host, ns_port = NS.start() # Regularized Evolution is usually so cheap, that we can afford to run a # worker on the master node as a background process worker = Worker(nameserver=ns_host, nameserver_port=ns_port, host=host, run_id=args.run_id, working_directory=args.working_directory, num_epochs=args.num_epochs, batch_size=args.batch_size, scheme=args.scheme, warmstart_dir=args.warmstart_dir, dataset=args.dataset,