def get_nameserver(self, run_id, task_id, ns_credentials_dir, network_interface_name):
if not os.path.isdir(ns_credentials_dir):
try:
os.mkdir(ns_credentials_dir)
except:
pass
return NameServer(run_id=run_id, nic_name=network_interface_name, working_directory=ns_credentials_dir)
def get_nameserver(self, run_id, task_id, ns_credentials_dir, network_interface_name):
"""Get the namesever object
Arguments:
run_id {str} -- The id of the run
task_id {int} -- An id for the worker
ns_credentials_dir {str} -- Path to ns credentials
network_interface_name {str} -- The network interface name
Returns:
NameServer -- The NameServer object
"""
if not os.path.isdir(ns_credentials_dir):
try:
os.mkdir(ns_credentials_dir)
except:
pass
return NameServer(run_id=run_id, nic_name=network_interface_name, working_directory=ns_credentials_dir)
max_budget = args.max_budget
eta = args.eta
args.working_directory = os.path.join(
args.working_directory,
"search_space_{}/run{}-seed{}".format(args.space, args.run_id, args.seed))
nasbench = NasbenchWrapper(
dataset_file=
'src/nasbench_analysis/nasbench_data/108_e/nasbench_only108.tfrecord')
if args.array_id == 1:
os.makedirs(args.working_directory, exist_ok=True)
NS = NameServer(run_id=args.run_id,
nic_name='eth0',
working_directory=args.working_directory)
ns_host, ns_port = NS.start()
# BOHB is usually so cheap, that we can
# affort to run a worker on the master node, too.
worker = worker(min_budget=min_budget,
max_budget=max_budget,
eta=eta,
search_space=args.space,
nasbench_data=nasbench,
seed=args.seed,
nameserver=ns_host,
nameserver_port=ns_port,
run_id=args.run_id)
worker.run(background=True)
def main():
parser = argparse.ArgumentParser(
description=
'Tensorforce hyperparameter tuner, using BOHB optimizer (Bayesian Optimization '
'and Hyperband)')
# Environment arguments (from run.py)
parser.add_argument(
'-e',
'--environment',
type=str,
help='Environment (name, configuration JSON file, or library module)')
parser.add_argument(
'-l',
'--level',
type=str,
default=None,
help='Level or game id, like `CartPole-v1`, if supported')
parser.add_argument('-m',
'--max-episode-timesteps',
type=int,
default=None,
help='Maximum number of timesteps per episode')
parser.add_argument(
'--import-modules',
type=str,
default=None,
help='Import comma-separated modules required for environment')
# Runner arguments (from run.py)
parser.add_argument('-n',
'--episodes',
type=int,
help='Number of episodes')
parser.add_argument(
'-p',
'--num-parallel',
type=int,
default=None,
help='Number of environment instances to execute in parallel')
# Tuner arguments
parser.add_argument(
'-r',
'--runs-per-round',
type=str,
default='1,2,5,10',
help=
'Comma-separated number of runs per optimization round, each with a successively '
'smaller number of candidates')
parser.add_argument(
'-s',
'--selection-factor',
type=int,
default=3,
help=
'Selection factor n, meaning that one out of n candidates in each round advances to '
'the next optimization round')
parser.add_argument(
'-i',
'--num-iterations',
type=int,
default=1,
help=
'Number of optimization iterations, each consisting of a series of optimization '
'rounds with an increasingly reduced candidate pool')
parser.add_argument('-d',
'--directory',
type=str,
default='tuner',
help='Output directory')
parser.add_argument('--restore',
type=str,
default=None,
help='Restore from given directory')
parser.add_argument('--id',
type=str,
default='worker',
help='Unique worker id')
args = parser.parse_args()
if args.import_modules is not None:
for module in args.import_modules.split(','):
importlib.import_module(name=module)
environment = dict(environment=args.environment)
if args.level is not None:
environment['level'] = args.level
if False:
host = nic_name_to_host(nic_name=None)
port = 123
else:
host = 'localhost'
port = None
runs_per_round = tuple(int(x) for x in args.runs_per_round.split(','))
print('Bayesian Optimization and Hyperband optimization')
print(
f'{args.num_iterations} iterations of each {len(runs_per_round)} rounds:'
)
for n, num_runs in enumerate(runs_per_round, start=1):
num_candidates = round(
math.pow(args.selection_factor,
len(runs_per_round) - n))
print(f'round {n}: {num_candidates} candidates, each {num_runs} runs')
print()
server = NameServer(run_id=args.id,
working_directory=args.directory,
host=host,
port=port)
nameserver, nameserver_port = server.start()
worker = TensorforceWorker(
environment=environment,
max_episode_timesteps=args.max_episode_timesteps,
num_episodes=args.episodes,
base=args.selection_factor,
runs_per_round=runs_per_round,
num_parallel=args.num_parallel,
run_id=args.id,
nameserver=nameserver,
nameserver_port=nameserver_port,
host=host)
worker.run(background=True)
if args.restore is None:
previous_result = None
else:
previous_result = logged_results_to_HBS_result(directory=args.restore)
result_logger = json_result_logger(directory=args.directory,
overwrite=True)
optimizer = BOHB(configspace=worker.get_configspace(),
eta=args.selection_factor,
min_budget=0.9,
max_budget=math.pow(args.selection_factor,
len(runs_per_round) - 1),
run_id=args.id,
working_directory=args.directory,
nameserver=nameserver,
nameserver_port=nameserver_port,
host=host,
result_logger=result_logger,
previous_result=previous_result)
# BOHB(configspace=None, eta=3, min_budget=0.01, max_budget=1, min_points_in_model=None,
# top_n_percent=15, num_samples=64, random_fraction=1 / 3, bandwidth_factor=3,
# min_bandwidth=1e-3, **kwargs)
# Master(run_id, config_generator, working_directory='.', ping_interval=60,
# nameserver='127.0.0.1', nameserver_port=None, host=None, shutdown_workers=True,
# job_queue_sizes=(-1,0), dynamic_queue_size=True, logger=None, result_logger=None,
# previous_result = None)
# logger: logging.logger like object, the logger to output some (more or less meaningful)
# information
results = optimizer.run(n_iterations=args.num_iterations)
# optimizer.run(n_iterations=1, min_n_workers=1, iteration_kwargs={})
# min_n_workers: int, minimum number of workers before starting the run
optimizer.shutdown(shutdown_workers=True)
server.shutdown()
with open(os.path.join(args.directory, 'results.pkl'), 'wb') as filehandle:
pickle.dump(results, filehandle)
print('Best found configuration: {}'.format(
results.get_id2config_mapping()[results.get_incumbent_id()]['config']))
print('Runs:',
results.get_runs_by_id(config_id=results.get_incumbent_id()))
print('A total of {} unique configurations where sampled.'.format(
len(results.get_id2config_mapping())))
print('A total of {} runs where executed.'.format(
len(results.get_all_runs())))
def main():
parser = argparse.ArgumentParser(description='Tensorforce hyperparameter tuner')
parser.add_argument(
'environment', help='Environment (name, configuration JSON file, or library module)'
)
parser.add_argument(
'-l', '--level', type=str, default=None,
help='Level or game id, like `CartPole-v1`, if supported'
)
parser.add_argument(
'-m', '--max-repeats', type=int, default=1, help='Maximum number of repetitions'
)
parser.add_argument(
'-n', '--num-iterations', type=int, default=1, help='Number of BOHB iterations'
)
parser.add_argument(
'-d', '--directory', type=str, default='tuner', help='Output directory'
)
parser.add_argument(
'-r', '--restore', type=str, default=None, help='Restore from given directory'
)
parser.add_argument('--id', type=str, default='worker', help='Unique worker id')
args = parser.parse_args()
if args.level is None:
environment = Environment.create(environment=args.environment)
else:
environment = Environment.create(environment=args.environment, level=args.level)
if False:
host = nic_name_to_host(nic_name=None)
port = 123
else:
host = 'localhost'
port = None
server = NameServer(run_id=args.id, working_directory=args.directory, host=host, port=port)
nameserver, nameserver_port = server.start()
worker = TensorforceWorker(
environment=environment, run_id=args.id, nameserver=nameserver,
nameserver_port=nameserver_port, host=host
)
# TensorforceWorker(run_id, nameserver=None, nameserver_port=None, logger=None, host=None, id=None, timeout=None)
# logger: logging.logger instance, logger used for debugging output
# id: anything with a __str__method, if multiple workers are started in the same process, you MUST provide a unique id for each one of them using the `id` argument.
# timeout: int or float, specifies the timeout a worker will wait for a new after finishing a computation before shutting down. Towards the end of a long run with multiple workers, this helps to shutdown idling workers. We recommend a timeout that is roughly half the time it would take for the second largest budget to finish. The default (None) means that the worker will wait indefinitely and never shutdown on its own.
worker.run(background=True)
# config = cs.sample_configuration().get_dictionary()
# print(config)
# res = worker.compute(config=config, budget=1, working_directory='.')
# print(res)
if args.restore is None:
previous_result = None
else:
previous_result = logged_results_to_HBS_result(directory=args.restore)
result_logger = json_result_logger(directory=args.directory, overwrite=True) # ???
optimizer = BOHB(
configspace=worker.get_configspace(), min_budget=0.5, max_budget=float(args.max_repeats),
run_id=args.id, working_directory=args.directory,
nameserver=nameserver, nameserver_port=nameserver_port, host=host,
result_logger=result_logger, previous_result=previous_result
)
# BOHB(configspace=None, eta=3, min_budget=0.01, max_budget=1, min_points_in_model=None, top_n_percent=15, num_samples=64, random_fraction=1 / 3, bandwidth_factor=3, min_bandwidth=1e-3, **kwargs)
# Master(run_id, config_generator, working_directory='.', ping_interval=60, nameserver='127.0.0.1', nameserver_port=None, host=None, shutdown_workers=True, job_queue_sizes=(-1,0), dynamic_queue_size=True, logger=None, result_logger=None, previous_result = None)
# logger: logging.logger like object, the logger to output some (more or less meaningful) information
results = optimizer.run(n_iterations=args.num_iterations)
# optimizer.run(n_iterations=1, min_n_workers=1, iteration_kwargs={})
# min_n_workers: int, minimum number of workers before starting the run
optimizer.shutdown(shutdown_workers=True)
server.shutdown()
environment.close()
with open(os.path.join(args.directory, 'results.pkl'), 'wb') as filehandle:
pickle.dump(results, filehandle)
print('Best found configuration:', results.get_id2config_mapping()[results.get_incumbent_id()]['config'])
print('Runs:', results.get_runs_by_id(config_id=results.get_incumbent_id()))
print('A total of {} unique configurations where sampled.'.format(len(results.get_id2config_mapping())))
print('A total of {} runs where executed.'.format(len(results.get_all_runs())))
print('Total budget corresponds to {:.1f} full function evaluations.'.format(
sum([r.budget for r in results.get_all_runs()]) / args.max_repeats)
)
default='beam_search',
help='Ensemble selection algorithm')
args = parser.parse_args()
np.random.seed(args.seed)
torch.manual_seed(args.seed)
host = nic_name_to_host(args.nic_name)
if args.array_id == 1:
os.makedirs(args.working_directory, exist_ok=True)
with open(os.path.join(args.working_directory, 'settings.txt'), 'w') as f:
f.write(str(args))
NS = NameServer(run_id=args.run_id,
host=host,
working_directory=args.working_directory)
ns_host, ns_port = NS.start()
# Regularized Evolution is usually so cheap, that we can afford to run a
# worker on the master node as a background process
worker = Worker(nameserver=ns_host,
nameserver_port=ns_port,
host=host,
run_id=args.run_id,
working_directory=args.working_directory,
num_epochs=args.num_epochs,
batch_size=args.batch_size,
scheme=args.scheme,
warmstart_dir=args.warmstart_dir,
dataset=args.dataset,