def runBOHB(cfg):
run_id = "0"
# assign random port in the 30000-40000 range to avoid using a blocked port because of a previous improper bohb shutdown
port = int(30000 + random.random() * 10000)
ns = hpns.NameServer(run_id=run_id, host="127.0.0.1", port=port)
ns.start()
w = BOHBWorker(cfg=cfg,
nameserver="127.0.0.1",
run_id=run_id,
nameserver_port=port)
w.run(background=True)
result_logger = hpres.json_result_logger(directory=cfg["bohb_log_dir"],
overwrite=True)
bohb = BOHB(
configspace=get_configspace(cfg['model']),
run_id=run_id,
min_budget=cfg["bohb_min_budget"],
max_budget=cfg["bohb_max_budget"],
eta=cfg["bohb_eta"],
nameserver="127.0.0.1",
nameserver_port=port,
result_logger=result_logger,
)
res = bohb.run(n_iterations=cfg["bohb_iterations"])
bohb.shutdown(shutdown_workers=True)
ns.shutdown()
return res
def optimize(cfg):
logger = logging.getLogger(__name__)
NS = hpns.NameServer(run_id=cfg.bohb.run_id, host=cfg.bohb.nameserver, port=None)
NS.start()
w = SAC_Worker(cfg.worker, nameserver=cfg.bohb.nameserver, run_id=cfg.bohb.run_id)
w.run(background=True)
bohb = BOHB( configspace = w.get_configspace(),
run_id = cfg.bohb.run_id, nameserver=cfg.bohb.nameserver,
min_budget=cfg.bohb.min_budget, max_budget=cfg.bohb.max_budget )
res = bohb.run(n_iterations=cfg.bohb.n_iterations)
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
id2config = res.get_id2config_mapping()
incumbent = res.get_incumbent_id()
# Store optimization results
if not os.path.exists("optimization_results/"):
os.makedirs("optimization_results/")
with open(os.path.join("optimization_results/", "%s.pkl" % cfg.bohb.run_id), 'wb') as fh:
pickle.dump(res, fh)
logger.info('Best found configuration: %s' % id2config[incumbent]['config'])
logger.info('A total of %i unique configurations where sampled.' % len(id2config.keys()))
logger.info('A total of %i runs where executed.' % len(res.get_all_runs()))
logger.info('Total budget corresponds to %.1f full function evaluations.'%(sum([r.budget for r in res.get_all_runs()])/cfg.bohb.max_budget))
def run_master(args):
NS = hpns.NameServer(run_id=args.run_id,
nic_name=args.nic_name,
working_directory=args.bohb_root_path)
ns_host, ns_port = NS.start()
# Start a background worker for the master node
if args.optimize_generalist:
w = AggregateWorker(run_id=args.run_id,
host=ns_host,
nameserver=ns_host,
nameserver_port=ns_port,
working_directory=args.bohb_root_path,
n_repeat=args.n_repeat,
has_repeats_as_budget=args.n_repeat is None,
time_budget=args.time_budget,
time_budget_approx=args.time_budget_approx,
performance_matrix=args.performance_matrix)
else:
w = SingleWorker(run_id=args.run_id,
host=ns_host,
nameserver=ns_host,
nameserver_port=ns_port,
working_directory=args.bohb_root_path,
n_repeat=args.n_repeat,
dataset=args.dataset,
time_budget=args.time_budget,
time_budget_approx=args.time_budget_approx)
w.run(background=True)
# Create an optimizer
result_logger = hpres.json_result_logger(directory=args.bohb_root_path,
overwrite=False)
if args.previous_run_dir is not None:
previous_result = hpres.logged_results_to_HBS_result(
args.previous_run_dir)
else:
pervious_result = None
logger = logging.getLogger(__file__)
logging_level = getattr(logging, args.logger_level)
logger.setLevel(logging_level)
optimizer = BOHB(configspace=get_configspace(),
run_id=args.run_id,
host=ns_host,
nameserver=ns_host,
nameserver_port=ns_port,
min_budget=args.n_repeat_lower_budget,
max_budget=args.n_repeat_upper_budget,
result_logger=result_logger,
logger=logger,
previous_result=previous_result)
res = optimizer.run(n_iterations=args.n_iterations)
# Shutdown
optimizer.shutdown(shutdown_workers=True)
NS.shutdown()
def run_bohb(exp_name, log_dir='EXP', iterations=20):
run_dir = 'bohb-{}-{}'.format(log_dir, exp_name)
if not os.path.exists(run_dir):
utils.create_exp_dir(run_dir, scripts_to_save=glob.glob('*.py'))
# log_format = '%(asctime)s %(message)s'
# logging.basicConfig(stream=sys.stdout, level=logging.INFO,
# format=log_format, datefmt='%m/%d %I:%M:%S %p')
# fh = logging.FileHandler(os.path.join(run_dir, 'log.txt'))
# fh.setFormatter(logging.Formatter(log_format))
# logging.getLogger().addHandler(fh)
result_logger = hpres.json_result_logger(directory=run_dir, overwrite=True)
# Start a nameserver
NS = hpns.NameServer(run_id=exp_name, host='127.0.0.1', port=0)
ns_host, ns_port = NS.start()
# Start a localserver
worker = TorchWorker(run_id=exp_name, host='127.0.0.1', nameserver=ns_host, nameserver_port=ns_port,
timeout=120, run_dir=run_dir)
worker.run(background=True)
# Initialise optimiser
bohb = BOHB(configspace=worker.get_configspace(),
run_id=exp_name,
host='127.0.0.1',
nameserver=ns_host,
nameserver_port=ns_port,
result_logger=result_logger,
min_budget=2, max_budget=5,
)
print('Worker running')
res = bohb.run(n_iterations=iterations)
# Store the results
with open(os.path.join(run_dir, 'result.pkl'), 'wb') as file:
pickle.dump(res, file)
# Shutdown
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
# get all runs
all_runs = res.get_all_runs()
# get id to configuration mapping as dictionary
id2conf = res.get_id2config_mapping()
# get best/incubent run
best_run = res.get_incumbent_id()
best_config = id2conf[best_run]['config']
print(f"Best run id:{best_run}, \n Config:{best_config}")
# Store all run info
file = open(os.path.join(run_dir, 'summary.txt'), 'w')
file.write(f"{all_runs}")
file.close()
def start(self):
# type: () -> ()
"""
Start the Optimizer controller function loop()
If the calling process is stopped, the controller will stop as well.
.. important::
This function returns only after optimization is completed or :meth:`stop` was called.
"""
# Step 1: Start a NameServer
fake_run_id = 'OptimizerBOHB_{}'.format(time())
# default port is 9090, we must have one, this is how BOHB workers communicate (even locally)
self._namespace = hpns.NameServer(run_id=fake_run_id,
host='127.0.0.1',
port=self._nameserver_port)
self._namespace.start()
# we have to scale the budget to the iterations per job, otherwise numbers might be too high
budget_iteration_scale = self._max_iteration_per_job
# Step 2: Start the workers
workers = []
for i in range(self._num_concurrent_workers):
w = _TrainsBandsterWorker(
optimizer=self,
sleep_interval=int(self.pool_period_minutes * 60),
budget_iteration_scale=budget_iteration_scale,
base_task_id=self._base_task_id,
objective=self._objective_metric,
queue_name=self._execution_queue,
nameserver='127.0.0.1',
nameserver_port=self._nameserver_port,
run_id=fake_run_id,
id=i)
w.run(background=True)
workers.append(w)
# Step 3: Run an optimizer
self._bohb = BOHB(
configspace=self._convert_hyper_parameters_to_cs(),
run_id=fake_run_id,
# num_samples=self.total_max_jobs, # will be set by self._bohb_kwargs
min_budget=float(self._min_iteration_per_job) /
float(self._max_iteration_per_job),
**self._bohb_kwargs)
# scale the budget according to the successive halving iterations
if self.budget.jobs.limit:
self.budget.jobs.limit *= len(self._bohb.budgets)
if self.budget.iterations.limit:
self.budget.iterations.limit *= len(self._bohb.budgets)
# start optimization
self._res = self._bohb.run(n_iterations=self.total_max_jobs,
min_n_workers=self._num_concurrent_workers)
# Step 4: if we get here, Shutdown
self.stop()
def get_parameters(train_data, kFold, iterations, save=False, filepath = './result/loss_time_bohb.csv'):
parser = argparse.ArgumentParser(description='Example 1 - sequential and local execution.')
parser.add_argument('--min_budget', type=float, help='Minimum budget used during the optimization.', default=1)
parser.add_argument('--max_budget', type=float, help='Maximum budget used during the optimization.', default=1)
parser.add_argument('--n_iterations', type=int, help='Number of iterations performed by the optimizer', default=iterations) # max value = 4
# parser.add_argument('--worker', help='Flag to turn this into a worker process', action='store_true')
parser.add_argument('--shared_directory', type=str,help='A directory that is accessible for all processes, e.g. a NFS share.', default='./result')
# parser.add_argument('--nic_name', type=str, default='lo')
args = parser.parse_args()
result_logger = hpres.json_result_logger(directory=args.shared_directory, overwrite=True)
NS = hpns.NameServer(run_id='BOHB', host='127.0.0.1', port=None)
NS.start()
w = worker(train_data, kFold, nameserver='127.0.0.1', run_id='BOHB')
w.run(background=True)
bohb = BOHB(configspace=w.get_configspace(),
run_id='BOHB', nameserver='127.0.0.1',
min_budget=args.min_budget, max_budget=args.max_budget,
result_logger=result_logger
)
res = bohb.run(n_iterations=args.n_iterations)
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
id2config = res.get_id2config_mapping()
incumbent = res.get_incumbent_id()
info = res.get_runs_by_id(incumbent)
parameter = id2config[incumbent]['config']
min_error = info[0]['loss']
if save:
all_info = res.get_all_runs()
timepoint_dic = []
loss_dic = []
for i in all_info:
timepoint_dic.append(i['time_stamps']['finished'])
loss_dic.append(i['loss'])
save_to_csv.save(filepath, timepoint_dic, loss_dic)
return parameter, min_error
def generate_bohb_data():
import warnings
import hpbandster.core.nameserver as hpns
import hpbandster.core.result as hpres
from hpbandster.optimizers import BOHB as BOHB
run_id = '0' # Every run has to have a unique (at runtime) id.
NS = hpns.NameServer(run_id=run_id, host='localhost', port=0)
ns_host, ns_port = NS.start()
from neural_opt import MyWorker, get_configspace
w = MyWorker(
nameserver=ns_host,
nameserver_port=ns_port,
run_id=run_id, # same as nameserver's
)
w.run(background=True)
# Log the optimization results for later analysis
result_logger = hpres.json_result_logger(
directory='test/general_example/results/bohb_full_configspace',
overwrite=True)
bohb = BOHB(
configspace=get_configspace(),
run_id=run_id, # same as nameserver's
eta=2,
min_budget=5,
max_budget=100, # Hyperband parameters
nameserver=ns_host,
nameserver_port=ns_port,
result_logger=result_logger,
)
# Then start the optimizer. The n_iterations parameter specifies
# the number of iterations to be performed in this run
with warnings.catch_warnings():
warnings.simplefilter("ignore")
res = bohb.run(n_iterations=2)
# After the run is finished, the services started above need to be shutdown.
# This ensures that the worker, the nameserver and the master all properly exit
# and no (daemon) threads keep running afterwards.
# In particular we shutdown the optimizer (which shuts down all workers) and the nameserver.
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
def select_model(worker, tmnt_config_space, total_iterations, result_logger,
id_str, ns_port):
tmnt_config = TMNTConfig(tmnt_config_space)
worker.run(background=True)
cs = tmnt_config.get_configspace()
config = cs.sample_configuration().get_dictionary()
logging.info(config)
bohb = BOHB(configspace=cs,
run_id=id_str,
nameserver='127.0.0.1',
result_logger=result_logger,
nameserver_port=ns_port,
min_budget=2,
max_budget=worker.max_budget)
res = bohb.run(n_iterations=total_iterations)
bohb.shutdown(shutdown_workers=True)
return res
def start_hpbandster_process(method,
configspace,
loss,
total_budget,
max_budget_per_config,
eta=3):
"""
Starts a server and a worker object needed for the HpBandSter optimization process
:param method: (str) Specifies if we use BOHB or hyperband
:param configspace: Hyper-parameter search space
:param loss: Loss function to minimize
:param total_budget: Total budget (in number of epochs) allowed for optimization
:param max_budget_per_config: Maximal number of epochs allowed for one config
:param eta: split size between every steps of successful halving
:return: NameServer and HpBandSter optimizer
"""
# Start a nameserver:
NS = hpns.NameServer(run_id=method)
ns_host, ns_port = NS.start()
# Start local worker
w = MyWorker(run_id=method,
nameserver=ns_host,
nameserver_port=ns_port,
timeout=120,
loss_function=loss)
w.run(background=True)
if method == 'BOHB':
optimizer = BOHB(
configspace=configspace,
run_id=method,
nameserver=ns_host,
nameserver_port=ns_port,
min_budget=1,
max_budget=max_budget_per_config,
)
else:
optimizer = HyperBand(
configspace=configspace,
run_id=method,
nameserver=ns_host,
nameserver_port=ns_port,
min_budget=1,
max_budget=max_budget_per_config,
)
# We compute the maximal number of iteration to be exact with the original paper
# (We divide the total budget by the fixed budget per successful halving iteration : (Smax+1)*bmax)
max_iter = total_budget / (
int(-1 * (log(1 / max_budget_per_config)) / log(eta) + 1) *
max_budget_per_config)
return NS, max_iter, optimizer
def optimize(self):
"""
The main optimization procedure
Should return the best arm as and array/ list
:return:
THe instance to be optimized is the self.objective value
"""
try:
if self.algorithm_name == 'BOHB':
algo = BOHB(
configspace=self.worker.get_configspace(),
run_id='example1',
nameserver='127.0.0.1',
min_budget=1,
max_budget=1
# budget is the number of epochs that will run so it is 1 for min and max
)
else:
algo = HyperBand(
configspace=self.worker.get_configspace(),
run_id='example1',
nameserver='127.0.0.1',
min_budget=1,
max_budget=1
# budget is the number of epochs that will run so it is 1 for min and max
)
res = algo.run(n_iterations=self.maxfeval)
#We shutdown the server
algo.shutdown(shutdown_workers=True)
self.NS.shutdown()
id2config = res.get_id2config_mapping()
incumbent = res.get_incumbent_id()
best = id2config[incumbent]['config']
best_arm = convertToArray(best)
success = True
except:
logger.warning('Optimization for ' + str(self.algorithm_name) +
' failed with cost function: ' +
str(self.objective.GetCostFunctionName()))
best_arm = NAN
success = False
return best_arm, success
def run_optimization(args):
"""Runs the optimization process."""
print("Starting name server.")
date_time = datetime.datetime.now().strftime('%Y-%m-%d-%H_%M_%S')
# First start nameserver
NS = hpns.NameServer(run_id=date_time, host='127.0.0.1', port=None)
NS.start()
print("Preparing result logger and loading previous run, if it exists.")
# Also start result logger
output_dir = os.path.join(
args.output_dir,
datetime.datetime.now().strftime('%Y_%m_%d--%H_%M_%S'))
result_logger_path = os.path.join(output_dir, 'results_log.json')
best_result_path = os.path.join(output_dir, 'best_config.txt')
print("Result logger will be written to %s" % result_logger_path)
if os.path.exists(result_logger_path):
previous_run = hpres.logged_results_to_HBS_result(result_logger_path)
else:
previous_run = None
result_logger = hpres.json_result_logger(directory=output_dir,
overwrite=True)
print("Starting search worker.\n")
# Then start worker
w = SearchWorker(args.data_path,
iaa,
os.path.join(output_dir, "logging"),
nameserver='127.0.0.1',
run_id=date_time)
w.run(background=True)
print("Initializing optimizer.")
# Run the optimizer
bohb = BOHB(configspace=w.get_configspace(),
run_id=date_time,
nameserver='127.0.0.1',
result_logger=result_logger,
min_budget=args.min_budget,
max_budget=args.max_budget,
previous_result=previous_run)
print("Initialization complete. Starting optimization run.")
res = bohb.run(n_iterations=args.iterations)
print("Optimization complete.")
output_fp = os.path.join(output_dir, 'results.pkl')
id2config = res.get_id2config_mapping()
incumbent = res.get_incumbent_id()
print("Results will be saved at:\n{}".format(output_fp))
print("Best found configuration: ", id2config[incumbent]['config'])
Slacker.send_message(
"AutoML Optimization finished with minimum "
"budget {}, maximum budget {}, and {} "
"iterations.\n"
"Output file has been written in {} \n".format(
args.min_budget, args.max_budget, args.iterations, output_dir),
"AutoML Optimization Finished!")
sleep(2)
Slacker.send_code("Best found configuration:",
"{}".format(id2config[incumbent]['config']))
with open(best_result_path, mode='w') as file:
lines = [
"Best results are as follows:\n",
"{}".format(id2config[incumbent]['config'])
]
file.writelines(lines)
with open(output_fp, mode='wb') as file:
pickle.dump(res, file)
# Shutdown after completion
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
def main():
parser = argparse.ArgumentParser(
description=
'Tensorforce hyperparameter tuner, using BOHB optimizer (Bayesian Optimization '
'and Hyperband)')
# Environment arguments (from run.py)
parser.add_argument(
'-e',
'--environment',
type=str,
help='Environment (name, configuration JSON file, or library module)')
parser.add_argument(
'-l',
'--level',
type=str,
default=None,
help='Level or game id, like `CartPole-v1`, if supported')
parser.add_argument('-m',
'--max-episode-timesteps',
type=int,
default=None,
help='Maximum number of timesteps per episode')
parser.add_argument(
'--import-modules',
type=str,
default=None,
help='Import comma-separated modules required for environment')
# Runner arguments (from run.py)
parser.add_argument('-n',
'--episodes',
type=int,
help='Number of episodes')
parser.add_argument(
'-p',
'--num-parallel',
type=int,
default=None,
help='Number of environment instances to execute in parallel')
# Tuner arguments
parser.add_argument(
'-r',
'--runs-per-round',
type=str,
default='1,2,5,10',
help=
'Comma-separated number of runs per optimization round, each with a successively '
'smaller number of candidates')
parser.add_argument(
'-s',
'--selection-factor',
type=int,
default=3,
help=
'Selection factor n, meaning that one out of n candidates in each round advances to '
'the next optimization round')
parser.add_argument(
'-i',
'--num-iterations',
type=int,
default=1,
help=
'Number of optimization iterations, each consisting of a series of optimization '
'rounds with an increasingly reduced candidate pool')
parser.add_argument('-d',
'--directory',
type=str,
default='tuner',
help='Output directory')
parser.add_argument('--restore',
type=str,
default=None,
help='Restore from given directory')
parser.add_argument('--id',
type=str,
default='worker',
help='Unique worker id')
args = parser.parse_args()
if args.import_modules is not None:
for module in args.import_modules.split(','):
importlib.import_module(name=module)
environment = dict(environment=args.environment)
if args.level is not None:
environment['level'] = args.level
if False:
host = nic_name_to_host(nic_name=None)
port = 123
else:
host = 'localhost'
port = None
runs_per_round = tuple(int(x) for x in args.runs_per_round.split(','))
print('Bayesian Optimization and Hyperband optimization')
print(
f'{args.num_iterations} iterations of each {len(runs_per_round)} rounds:'
)
for n, num_runs in enumerate(runs_per_round, start=1):
num_candidates = round(
math.pow(args.selection_factor,
len(runs_per_round) - n))
print(f'round {n}: {num_candidates} candidates, each {num_runs} runs')
print()
server = NameServer(run_id=args.id,
working_directory=args.directory,
host=host,
port=port)
nameserver, nameserver_port = server.start()
worker = TensorforceWorker(
environment=environment,
max_episode_timesteps=args.max_episode_timesteps,
num_episodes=args.episodes,
base=args.selection_factor,
runs_per_round=runs_per_round,
num_parallel=args.num_parallel,
run_id=args.id,
nameserver=nameserver,
nameserver_port=nameserver_port,
host=host)
worker.run(background=True)
if args.restore is None:
previous_result = None
else:
previous_result = logged_results_to_HBS_result(directory=args.restore)
result_logger = json_result_logger(directory=args.directory,
overwrite=True)
optimizer = BOHB(configspace=worker.get_configspace(),
eta=args.selection_factor,
min_budget=0.9,
max_budget=math.pow(args.selection_factor,
len(runs_per_round) - 1),
run_id=args.id,
working_directory=args.directory,
nameserver=nameserver,
nameserver_port=nameserver_port,
host=host,
result_logger=result_logger,
previous_result=previous_result)
# BOHB(configspace=None, eta=3, min_budget=0.01, max_budget=1, min_points_in_model=None,
# top_n_percent=15, num_samples=64, random_fraction=1 / 3, bandwidth_factor=3,
# min_bandwidth=1e-3, **kwargs)
# Master(run_id, config_generator, working_directory='.', ping_interval=60,
# nameserver='127.0.0.1', nameserver_port=None, host=None, shutdown_workers=True,
# job_queue_sizes=(-1,0), dynamic_queue_size=True, logger=None, result_logger=None,
# previous_result = None)
# logger: logging.logger like object, the logger to output some (more or less meaningful)
# information
results = optimizer.run(n_iterations=args.num_iterations)
# optimizer.run(n_iterations=1, min_n_workers=1, iteration_kwargs={})
# min_n_workers: int, minimum number of workers before starting the run
optimizer.shutdown(shutdown_workers=True)
server.shutdown()
with open(os.path.join(args.directory, 'results.pkl'), 'wb') as filehandle:
pickle.dump(results, filehandle)
print('Best found configuration: {}'.format(
results.get_id2config_mapping()[results.get_incumbent_id()]['config']))
print('Runs:',
results.get_runs_by_id(config_id=results.get_incumbent_id()))
print('A total of {} unique configurations where sampled.'.format(
len(results.get_id2config_mapping())))
print('A total of {} runs where executed.'.format(
len(results.get_all_runs())))
def a2c_bohb_wrapper(**params):
# Setup directories where live data is logged
logdir = params["logdir"]
a2c_output_dir = os.path.join(logdir, 'a2c_output')
params["logdir"] = a2c_output_dir
bohb_output_dir = os.path.join(logdir, 'bohb_output')
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
logger = logging.getLogger()
logger.propagate = False # no duplicate logging outputs
fh = logging.FileHandler(os.path.join(logdir, 'bohb.log'))
fh.setLevel(logging.INFO)
fh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s:%(name)s: %(message)s'))
logger.addHandler(fh)
# Build configuration space and define all hyperparameters
cs = ConfigurationSpace()
lr = UniformFloatHyperparameter("lr", 1e-4, 1e-2, default_value=1e-3)
units_shared_layer1 = UniformIntegerHyperparameter("units_shared_layer1", 8, 100, default_value=24)
units_shared_layer2 = UniformIntegerHyperparameter("units_shared_layer2", 8, 100, default_value=24)
units_policy_layer = UniformIntegerHyperparameter("units_policy_layer", 8, 100, default_value=24)
vf_coeff = UniformFloatHyperparameter("vf_coeff", 1e-2, 0.5, default_value=0.1)
ent_coeff = UniformFloatHyperparameter("ent_coeff", 5e-6, 1e-4, default_value=1e-5)
gamma = UniformFloatHyperparameter("gamma", 0.6, 1., default_value=0.90)
activ_fcn = CategoricalHyperparameter("activ_fcn", ['relu6', 'elu', 'mixed'], default_value='relu6')
cs.add_hyperparameters([units_shared_layer1, units_shared_layer2, units_policy_layer,
vf_coeff, ent_coeff, gamma, lr, activ_fcn]) # batch_size
logger.info('##############################################')
logger.info('Run Optimization')
logger.info('##############################################')
if params["array_id"] == 1:
# Setup directories where live data is logged
if not os.path.isdir(a2c_output_dir):
os.makedirs(a2c_output_dir)
if not os.path.isdir(bohb_output_dir):
os.makedirs(bohb_output_dir)
# start nameserver
NS = hpns.NameServer(run_id=params["instance_id"], nic_name=params["nic_name"],
working_directory=bohb_output_dir)
ns_host, ns_port = NS.start() # stores information for workers to find in working directory
# BOHB is usually so cheap, that we can affort to run a worker on the master node, too.
worker = A2CWorker(nameserver=ns_host, nameserver_port=ns_port, run_id=params["instance_id"], **params)
worker.run(background=True)
# Create scenario object
logger.info('##############################################')
logger.info('Setup BOHB instance')
logger.info('##############################################')
logger.info('Output_dir: %s' % bohb_output_dir)
HB = BOHB(configspace=cs,
run_id=params["instance_id"],
eta=3,
min_budget=params["min_resource"],
max_budget=params["max_resource"],
host=ns_host,
nameserver=ns_host,
nameserver_port=ns_port,
ping_interval=3600)
res = HB.run(n_iterations=4,
min_n_workers=4) # BOHB can wait until a minimum number of workers is online before starting
# pickle result here for later analysis
with open(os.path.join(bohb_output_dir, 'results.pkl'), 'wb') as f:
pickle.dump(res, f)
id2config = res.get_id2config_mapping()
print('A total of %i unique configurations where sampled.' % len(id2config.keys()))
print('A total of %i runs where executed.' % len(res.get_all_runs()))
# shutdown all workers
HB.shutdown(shutdown_workers=True)
# shutdown nameserver
NS.shutdown()
else:
host = hpns.nic_name_to_host(params["nic_name"])
# workers only instantiate the MyWorker, find the nameserver and start serving
w = A2CWorker(run_id=params["instance_id"], host=host, **params)
w.load_nameserver_credentials(bohb_output_dir)
# run worker in the forground,
w.run(background=False)
def pre_train(self, dataset):
""" Pre-trains the model based on one or more datasets.
"""
run_id = 'first_run'
host = '127.0.0.1'
n_workers = 4
n_iterations = 300
data = DataLoader(dataset, 1, cuda=False)
data = data.data_loader
np.random.shuffle(data)
X_ = []
y_ = []
for point in data:
X_.append(point[0].tolist()[0])
y_.append(np.argmax(point[1].tolist()))
global X
global y
X = np.array(X_)
y = np.array(y_)
# shuffle the data
indices = np.arange(X.shape[0])
np.random.shuffle(indices)
X = X[indices]
y = y[indices]
min_budget = len(X) * 0.01
max_budget = len(X) * 0.25
NS = hpns.NameServer(run_id=run_id, host=host, port=None)
NS.start()
workers = []
print('starting workers:')
for i in range(n_workers):
print(i)
w = RFWorker(nameserver=host, run_id=run_id, id=i)
w.run(background=True)
workers.append(w)
bohb = BOHB(configspace=workers[0].get_configspace(),
run_id=run_id,
min_budget=min_budget,
max_budget=max_budget)
print('running BOHB')
res = bohb.run(n_iterations=n_iterations, min_n_workers=n_workers)
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
id2config = res.get_id2config_mapping()
incumbent = res.get_incumbent_id()
all_runs = res.get_all_runs()
print('Best found configuration:', id2config[incumbent]['config'])
print('A total of %i unique configurations where sampled.' %
len(id2config.keys()))
print('A total of %i runs where executed.' % len(res.get_all_runs()))
print('Total budget corresponds to %.1f full function evaluations.' %
(sum([r.budget for r in all_runs]) / max_budget))
print('The run took %.1f seconds to complete.' %
(all_runs[-1].time_stamps['finished'] -
all_runs[0].time_stamps['started']))
self.max_depth = id2config[incumbent]['config']['max_depth']
self.n_trees = id2config[incumbent]['config']['n_trees']
self.rf_classifier = RandomForestClassifier(n_estimators=self.n_trees,
max_depth=self.max_depth)
self.rf_classifier.fit(X, y)
with open('config.txt', 'a') as f:
print('Best config: n_trees: {}, max_depth: {}'.format(
self.n_trees, self.max_depth),
file=f)
nameserver_port=ns_port,
timeout=120)
w.run(background=True)
# Let us load the old run now to use its results to warmstart a new run with slightly
# different budgets in terms of datapoints and epochs.
# Note that the search space has to be identical though!
previous_run = hpres.logged_results_to_HBS_result(args.previous_run_dir)
# Run an optimizer
bohb = BOHB(
configspace=worker.get_configspace(),
run_id=args.run_id,
host=host,
nameserver=ns_host,
nameserver_port=ns_port,
result_logger=result_logger,
min_budget=args.min_budget,
max_budget=args.max_budget,
previous_result=
previous_run, # this is how you tell any optimizer about previous runs
)
res = bohb.run(n_iterations=args.n_iterations)
# store results
with open(os.path.join(args.shared_directory, 'results.pkl'), 'wb') as fh:
pickle.dump(res, fh)
# shutdown
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
config_space.add_hyperparameters([l3, l4, l5])
latent = CSH.UniformIntegerHyperparameter('latent',
lower=20,
upper=100)
#latent = CSH.CategoricalHyperparameter('latent', [33])
config_space.add_hyperparameters([latent])
return config_space
BUDGET = 10
ITERATIONS = 100
NS = hpns.NameServer(run_id='Search', host='127.0.0.1', port=None)
NS.start()
WK = MyWorker(nameserver='127.0.0.1', run_id='Search')
WK.run(background=True)
BS = BOHB(configspace=WK.get_configspace(),
run_id='Search',
nameserver='127.0.0.1',
min_budget=BUDGET,
max_budget=BUDGET)
BSP = BS.run(n_iterations=ITERATIONS)
BS.shutdown(shutdown_workers=True)
NS.shutdown()
print('Best found configuration:',
setting(BSP.get_id2config_mapping()[BSP.get_incumbent_id()]['config']))
def run_bohb(args):
scorer = TrueRougeScorer()
run_id = infer_run_id_from_args(args)
run_dir = os.path.join(args.hpb_runs_dir, run_id)
if args.hpb_worker:
w = BeamWorker(args, scorer, nameserver='127.0.0.1', run_id=run_id)
w.run(background=False)
exit(0)
result_logger = hpres.json_result_logger(directory=run_dir,
overwrite=getattr(
args, 'hpb_overwrite_run',
False))
# Step 1: Start a nameserver
# Every run needs a nameserver. It could be a 'static' server with a
# permanent address, but here it will be started for the local machine with the default port.
# The nameserver manages the concurrent running workers across all possible threads or clusternodes.
# Note the run_id argument. This uniquely identifies a run of any HpBandSter optimizer.
NS = hpns.NameServer(run_id=run_id, host='127.0.0.1', port=None)
NS.start()
# Step 2: Start a worker
# Now we can instantiate a worker, providing the mandatory information
# Besides the sleep_interval, we need to define the nameserver information and
# the same run_id as above. After that, we can start the worker in the background,
# where it will wait for incoming configurations to evaluate.
# Step 3: Run an optimizer
# Now we can create an optimizer object and start the run.
# Here, we run BOHB, but that is not essential.
# The run method will return the `Result` that contains all runs performed.
bohb = BOHB(configspace=get_conf_space(args),
run_id=run_id,
nameserver='127.0.0.1',
result_logger=result_logger,
min_budget=args.hpb_min_budget,
max_budget=args.hpb_max_budget)
res = bohb.run(n_iterations=args.hpb_n_iterations,
min_n_workers=args.hpb_n_workers)
# Step 4: Shutdown
# After the optimizer run, we must shutdown the master and the nameserver.
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
# Step 5: Analysis
# Each optimizer returns a hpbandster.core.result.Result object.
# It holds informations about the optimization run like the incumbent (=best) configuration.
# For further details about the Result object, see its documentation.
# Here we simply print out the best config and some statistics about the performed runs.
id2config = res.get_id2config_mapping()
incumbent = res.get_incumbent_id()
print('Best found configuration:', id2config[incumbent]['config'])
print('A total of %i unique configurations where sampled.' %
len(id2config.keys()))
print('A total of %i runs where executed.' % len(res.get_all_runs()))
print('Total budget corresponds to %.1f full function evaluations.' %
(sum([r.budget for r in res.get_all_runs()]) / args.hpb_max_budget))
isKeras = False
elif ALGORITHM == 'Keras':
worker = KerasRegressor(X_train, X_val, y_train, y_val, nameserver='127.0.0.1', run_id='example1')
worker.run(background=True)
isKeras = True
else:
isKeras = False
# Step 3: Run an optimizer
result_logger = hpres.json_result_logger(directory='.',
overwrite=True)
if not isKeras:
if OPTIMIZER == 'BOHB':
optimizer = BOHB(configspace=worker.get_warmstart_configspace(), run_id='example1',
nameserver='127.0.0.1', min_budget=10, max_budget=10, eta=3.0,
result_logger=result_logger)
res = optimizer.run(n_iterations=1)
elif OPTIMIZER == 'RandomSearch':
optimizer = RandomSearch(configspace=worker.get_configspace(), run_id='example1',
nameserver='127.0.0.1', min_budget=1, max_budget=9, eta=3.0,
result_logger=result_logger)
res = optimizer.run(n_iterations=10)
else:
if OPTIMIZER == 'BOHB':
optimizer = BOHB(configspace=worker.get_configspace(), run_id='example1',
nameserver='127.0.0.1', min_budget=3, max_budget=100, eta=3.0,
result_logger=result_logger)
res = optimizer.run(n_iterations=5)
elif OPTIMIZER == 'RandomSearch':
parser.add_argument('--shared_directory',type=str, help='A directory that is accessible for all processes, e.g. a NFS share.', default='.')
args=parser.parse_args()
#Define a realtime result logger
result_logger = hpres.json_result_logger(directory=result_dir, overwrite=True)
#Start a nameserver
NS = hpns.NameServer(run_id='MAXWEL', host='127.0.0.1', port=None)
NS.start()
#Start the workers
workers=[]
for i in range(args.n_workers):
w = worker(nameserver='127.0.0.1',run_id='MAXWEL', id=i)
w.run(background=True)
workers.append(w)
#Define and run an optimizer
bohb = BOHB(configspace = w.get_configspace(),
run_id = 'MAXWEL',
result_logger=result_logger,
min_budget=args.min_budget,
max_budget=args.max_budget)
res = bohb.run(n_iterations=args.n_iterations, min_n_workers=args.n_workers)
#Shutdown the nameserver
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
dataset = {'train': train_set, 'val': val_set, 'test': test_set}
name_server = hpns.NameServer(run_id=args.name,
host=args.nameserver,
port=None)
name_server.start()
worker = COVIDWorker(sleep_interval=0,
nameserver=args.nameserver,
run_id=args.name)
worker.set_dataset(args, dataset)
worker.run(background=True)
bohb = BOHB(configspace=worker.get_configspace(),
run_id=args.name,
nameserver=args.nameserver,
min_budget=args.min_budget,
max_budget=args.max_budget)
result = bohb.run(n_iterations=args.n_iterations)
bohb.shutdown(shutdown_workers=True)
name_server.shutdown()
id2config = result.get_id2config_mapping()
incumbent = result.get_incumbent_id()
print('Best found configurations:', id2config[incumbent]['config'])
print('A total of %i unique configurations where sampled.' %
len(id2config.keys()))
print('A total of %i runs where excuted.' % len(result.get_all_runs()))
# start nameserver
NS = hpns.NameServer(run_id=args.run_id, nic_name=args.nic_name,
working_directory=args.working_dir)
ns_host, ns_port = NS.start() # stores information for workers to find in working_directory
# BOHB is usually so cheap, that we can affort to run a worker on the master node, too.
worker = MyWorker(nameserver=ns_host, nameserver_port=ns_port, run_id=args.run_id)
worker.run(background=True)
HB = BOHB( configspace = config_space,
run_id = args.run_id,
eta=3,min_budget=27, max_budget=243,
host=ns_host,
nameserver=ns_host,
nameserver_port = ns_port,
ping_interval=3600,
)
res = HB.run( n_iterations = 4,
min_n_workers = 4 # BOHB can wait until a minimum number of workers is online before starting
)
# pickle result here for later analysis
with open(os.path.join(args.working_dir, 'results.pkl'), 'wb') as fh:
pickle.dump(res, fh)
# shutdown all workers
HB.shutdown(shutdown_workers=True)
w = MyWorker(sleep_interval=0.05,
nameserver='127.0.0.1',
run_id='example2',
id=i,
n_jobs=args.n_jobs)
w.run(background=True)
workers.append(w)
# Step 3: Run an optimizer
# Now we can create an optimizer object and start the run.
# We add the min_n_workers argument to the run methods to make the optimizer wait
# for all workers to start. This is not mandatory, and workers can be added
# at any time, but if the timing of the run is essential, this can be used to
# synchronize all workers right at the start.
bohb = BOHB(configspace=w.get_configspace(),
run_id='example2',
min_budget=args.min_budget,
max_budget=args.max_budget)
res = bohb.run(n_iterations=args.n_iterations, min_n_workers=args.n_workers)
# Step 4: Shutdown
# After the optimizer run, we must shutdown the master and the nameserver.
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
# Step 5: Analysis
# Each optimizer returns a hpbandster.core.result.Result object.
# It holds informations about the optimization run like the incumbent (=best) configuration.
# For further details about the Result object, see its documentation.
# Here we simply print out the best config and some statistics about the performed runs.
id2config = res.get_id2config_mapping()
incumbent = res.get_incumbent_id()
ns_host, ns_port = NS.start()
# Start local worker
worker = TrainEvalWorker(run_id=RUN_ID,
nameserver=ns_host,
nameserver_port=ns_port)
worker.run(background=True)
result_logger = hpres.json_result_logger(
directory="data/hpbandster/{}".format(RUN_ID), overwrite=False)
optim = BOHB(
configspace=worker.get_configspace(),
run_id=RUN_ID,
nameserver=ns_host,
nameserver_port=ns_port,
result_logger=result_logger,
eta=HPO_PARAMS["eta"],
min_budget=HPO_PARAMS["min_budget"],
max_budget=HPO_PARAMS["max_budget"],
num_samples=HPO_PARAMS["num_samples"],
top_n_percent=HPO_PARAMS["top_n_percent"],
min_bandwidth=HPO_PARAMS["min_bandwidth"],
bandwidth_factor=HPO_PARAMS["bandwidth_factor"],
)
study = optim.run(n_iterations=HPO_PARAMS["n_calls"])
results = hpbandster2skopt(study)
best_auc = -1.0 * results.fun
best_params = results.x
print(results)
# log metrics
print("Best Validation AUC: {}".format(best_auc))
def get_parameters(data, target_feature_index):
parser = argparse.ArgumentParser(
description='Example 1 - sequential and local execution.')
parser.add_argument('--min_budget',
type=float,
help='Minimum budget used during the optimization.',
default=9)
parser.add_argument('--max_budget',
type=float,
help='Maximum budget used during the optimization.',
default=243)
parser.add_argument('--n_iterations',
type=int,
help='Number of iterations performed by the optimizer',
default=4)
parser.add_argument('--n_workers',
type=int,
help='Number of workers to run in parallel.',
default=2)
parser.add_argument('--worker',
help='Flag to turn this into a worker process',
action='store_true')
parser.add_argument(
'--run_id',
type=str,
help=
'A unique run id for this optimization run. An easy option is to use the job id of the clusters scheduler.'
)
parser.add_argument(
'--nic_name',
type=str,
help='Which network interface to use for communication.',
default='lo')
parser.add_argument(
'--shared_directory',
type=str,
help=
'A directory that is accessible for all processes, e.g. a NFS share.',
default='/home/lchen/parameters/result')
args = parser.parse_args()
host = hpns.nic_name_to_host(args.nic_name)
if args.worker:
time.sleep(
5
) # short artificial delay to make sure the nameserver is already running
w = worker(0.5,
data,
target_feature_index,
run_id=args.run_id,
host=host)
w.load_nameserver_credentials(working_directory=args.shared_directory)
w.run(background=False)
exit(0)
result_logger = hpres.json_result_logger(directory=args.shared_directory,
overwrite=True)
# Step 1: Start a nameserver
# Every run needs a nameserver. It could be a 'static' server with a
# permanent address, but here it will be started for the local machine with the default port.
# The nameserver manages the concurrent running workers across all possible threads or clusternodes.
# Note the run_id argument. This uniquely identifies a run of any HpBandSter optimizer.
NS = hpns.NameServer(run_id='test1',
host=host,
port=0,
working_directory=args.shared_directory)
ns_host, ns_port = NS.start()
# Step 2: Start a worker
# Now we can instantiate a worker, providing the mandatory information
# Besides the sleep_interval, we need to define the nameserver information and
# the same run_id as above. After that, we can start the worker in the background,
# where it will wait for incoming configurations to evaluate.
w = worker(0.5,
data,
target_feature_index,
run_id='test1',
host=host,
nameserver=ns_host,
nameserver_port=ns_port)
w.run(background=True)
# Step 3: Run an optimizer
# Now we can create an optimizer object and start the run.
# Here, we run BOHB, but that is not essential.
# The run method will return the `Result` that contains all runs performed.
bohb = BOHB(configspace=worker.get_configspace(),
run_id=args.run_id,
host=host,
nameserver=ns_host,
nameserver_port=ns_port,
result_logger=result_logger,
min_budget=args.min_budget,
max_budget=args.max_budget)
print("daozhele5")
res = bohb.run(n_iterations=args.n_iterations)
print("daozhele6")
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
# Step 5: Analysis
# Each optimizer returns a hpbandster.core.result.Result object.
# It holds informations about the optimization run like the incumbent (=best) configuration.
# For further details about the Result object, see its documentation.
# Here we simply print out the best config and some statistics about the performed runs.
id2config = res.get_id2config_mapping()
incumbent = res.get_incumbent_id()
info = res.get_runs_by_id(incumbent)
parameter = id2config[incumbent]['config']
min_error = info[0]['loss']
feature_importance_dict = info[0]['info']
with open(os.path.join(args.shared_directory, 'results.pkl'), 'wb') as fh:
pickle.dump(res, fh)
return parameter, min_error, feature_importance_dict
def run_experiment(out_path, on_travis):
settings = {
'min_budget': 1,
'max_budget':
9, # number of repetitions; this is the fidelity for this bench
'num_iterations': 10, # Set this to a low number for demonstration
'eta': 3,
'output_dir': Path(out_path)
}
if on_travis:
settings.update(get_travis_settings('bohb'))
b = Benchmark(rng=1)
b.get_configuration_space(seed=1)
settings.get('output_dir').mkdir(exist_ok=True)
cs = b.get_configuration_space()
seed = get_rng(rng=0)
run_id = 'BOHB_on_cartpole'
result_logger = hpres.json_result_logger(directory=str(
settings.get('output_dir')),
overwrite=True)
ns = hpns.NameServer(run_id=run_id,
host='localhost',
working_directory=str(settings.get('output_dir')))
ns_host, ns_port = ns.start()
worker = CustomWorker(seed=seed,
nameserver=ns_host,
nameserver_port=ns_port,
run_id=run_id,
max_budget=settings.get('max_budget'))
worker.run(background=True)
master = BOHB(configspace=cs,
run_id=run_id,
host=ns_host,
nameserver=ns_host,
nameserver_port=ns_port,
eta=settings.get('eta'),
min_budget=settings.get('min_budget'),
max_budget=settings.get('max_budget'),
result_logger=result_logger)
result = master.run(n_iterations=settings.get('num_iterations'))
master.shutdown(shutdown_workers=True)
ns.shutdown()
with open(settings.get('output_dir') / 'results.pkl', 'wb') as f:
pickle.dump(result, f)
id2config = result.get_id2config_mapping()
incumbent = result.get_incumbent_id()
inc_value = result.get_runs_by_id(incumbent)[-1]['loss']
inc_cfg = id2config[incumbent]['config']
logger.info(f'Inc Config:\n{inc_cfg}\n'
f'with Performance: {inc_value:.2f}')
if not on_travis:
benchmark = Benchmark(container_source='library://phmueller/automl')
incumbent_result = benchmark.objective_function_test(
configuration=inc_cfg, fidelity={"budget": settings['max_budget']})
print(incumbent_result)
def MAXWEL_bandster_station(raw_name, width):
name = raw_name.split('.')
if name[-1] != "csv":
return
name = name[0]
df = pd.read_csv('Train_Data/Width_' + width + '/' + name +
'.csv').drop(columns=['time'])
try:
result_dir = 'Results/' + name
os.mkdir(result_dir)
except:
result_dir = 'Results/' + name
assert df.isna().sum().sum() == 0, name + " has NANs"
data_info = {}
data_info['primary'] = name
data_info['num_features'] = df.shape[1]
data_info['train_time'] = 12
data_info['predict_time'] = 4
data_info['num_samples'] = df.shape[0]
X = np.zeros(
(data_info['num_samples'], data_info['train_time'], df.shape[1]))
Y = np.zeros((data_info['num_samples'], data_info['predict_time']))
for i in range(data_info['num_samples'] -
(data_info['train_time'] + data_info['predict_time'] + 1)):
X[i, :, :] = df.iloc[i:i + data_info['train_time']].values
Y[i, :] = np.asarray(
df[name + '_scaled_demand'][i + data_info['train_time']:i +
data_info['train_time'] +
data_info['predict_time']].values)
idx = sample(range(len(X)), 9000)
train_idx = idx[:6000]
valid_idx = idx[6000:8000]
test_idx = idx[8000:]
data_info['num_train_samples'] = len(train_idx)
data_info['train_idx'] = train_idx
data_info['num_valid_samples'] = len(valid_idx)
data_info['valid_idx'] = valid_idx
data_info['num_test_samples'] = len(test_idx)
data_info['test_idx'] = test_idx
data_info['fit_cnt'] = 1
# data_info['station_cnt']=station_cnt
# data_info['num_stations']=len(names)
dill.dump(data_info, open("Temp_Data/data_info.pkl", 'wb'))
X_train = np.zeros((data_info['num_train_samples'],
data_info['train_time'], data_info['num_features']))
Y_train = np.zeros(
(data_info['num_train_samples'], data_info['predict_time']))
X_valid = np.zeros((data_info['num_valid_samples'],
data_info['train_time'], data_info['num_features']))
Y_valid = np.zeros(
(data_info['num_valid_samples'], data_info['predict_time']))
X_test = np.zeros((data_info['num_test_samples'], data_info['train_time'],
data_info['num_features']))
Y_test = np.zeros(
(data_info['num_test_samples'], data_info['predict_time']))
for i, ind in enumerate(data_info['train_idx']):
X_train[i, :, :] = X[ind, :, :]
Y_train[i, :] = Y[ind, :]
for i, ind in enumerate(data_info['valid_idx']):
X_valid[i, :, :] = X[ind, :, :]
Y_valid[i, :] = Y[ind, :]
for i, ind in enumerate(data_info['test_idx']):
X_test[i, :, :] = X[ind, :, :]
Y_test[i, :] = Y[ind, :]
dill.dump(X_train, open("Temp_Data/X_train.pkl", 'wb'))
dill.dump(Y_train, open("Temp_Data/Y_train.pkl", 'wb'))
dill.dump(X_valid, open("Temp_Data/X_valid.pkl", 'wb'))
dill.dump(Y_valid, open("Temp_Data/Y_valid.pkl", 'wb'))
dill.dump(X_test, open("Temp_Data/X_test.pkl", 'wb'))
dill.dump(Y_test, open("Temp_Data/Y_test.pkl", 'wb'))
# Import a worker class
from MAXWEL_worker import MAXWEL_worker as worker
#Build an argument parser
parser = argparse.ArgumentParser(
description='MAXWEL - sequential execution.')
parser.add_argument('--min_budget',
type=float,
help='Minimum budget used during the optimization.',
default=5)
parser.add_argument('--max_budget',
type=float,
help='Maximum budget used during the optimization.',
default=20)
parser.add_argument('--n_iterations',
type=int,
help='Number of iterations performed by the optimizer',
default=10)
parser.add_argument('--n_workers',
type=int,
help='Number of workers to run in parallel.',
default=1)
parser.add_argument(
'--shared_directory',
type=str,
help=
'A directory that is accessible for all processes, e.g. a NFS share.',
default='.')
args = parser.parse_args()
#Define a realtime result logger
result_logger = hpres.json_result_logger(directory=result_dir,
overwrite=True)
#Start a nameserver
NS = hpns.NameServer(run_id='MAXWEL', host='127.0.0.1', port=None)
NS.start()
#Start the workers
workers = []
for i in range(args.n_workers):
w = worker(nameserver='127.0.0.1', run_id='MAXWEL', id=i)
w.run(background=True)
workers.append(w)
#Define and run an optimizer
bohb = BOHB(configspace=w.get_configspace(),
run_id='MAXWEL',
result_logger=result_logger,
min_budget=args.min_budget,
max_budget=args.max_budget)
res = bohb.run(n_iterations=args.n_iterations,
min_n_workers=args.n_workers)
#Shutdown the nameserver
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
dataID = args.openml_dataid)
w.run(background=True)
#if True:
# time.sleep(5) # short artificial delay to make sure the nameserver is already running
# w = MyWorker(sleep_interval = 0.5,run_id=args.run_id, host=host, dataID = args.openml_dataid)
# w.load_nameserver_credentials(working_directory=args.shared_directory)
# w.run(background=False)
# exit(0)
# Run an optimizer
# We now have to specify the host, and the nameserver information
bohb = BOHB(configspace=MyWorker.get_configspace(),
run_id=args.run_id,
host=host,
nameserver=ns_host,
nameserver_port=ns_port,
min_budget=args.min_budget,
max_budget=args.max_budget)
res = bohb.run(n_iterations=args.n_iterations, min_n_workers=args.n_workers)
d1 = res.get_pandas_dataframe()[0]
loss = res.get_pandas_dataframe()[1]
d1['loss'] = loss
d1.to_csv(os.path.join(args.shared_directory, outputName + ".csv"))
# In a cluster environment, you usually want to store the results for later analysis.
# One option is to simply pickle the Result object
def main():
parser = argparse.ArgumentParser(description='Tensorforce hyperparameter tuner')
parser.add_argument(
'environment', help='Environment (name, configuration JSON file, or library module)'
)
parser.add_argument(
'-l', '--level', type=str, default=None,
help='Level or game id, like `CartPole-v1`, if supported'
)
parser.add_argument(
'-m', '--max-repeats', type=int, default=1, help='Maximum number of repetitions'
)
parser.add_argument(
'-n', '--num-iterations', type=int, default=1, help='Number of BOHB iterations'
)
parser.add_argument(
'-d', '--directory', type=str, default='tuner', help='Output directory'
)
parser.add_argument(
'-r', '--restore', type=str, default=None, help='Restore from given directory'
)
parser.add_argument('--id', type=str, default='worker', help='Unique worker id')
args = parser.parse_args()
if args.level is None:
environment = Environment.create(environment=args.environment)
else:
environment = Environment.create(environment=args.environment, level=args.level)
if False:
host = nic_name_to_host(nic_name=None)
port = 123
else:
host = 'localhost'
port = None
server = NameServer(run_id=args.id, working_directory=args.directory, host=host, port=port)
nameserver, nameserver_port = server.start()
worker = TensorforceWorker(
environment=environment, run_id=args.id, nameserver=nameserver,
nameserver_port=nameserver_port, host=host
)
# TensorforceWorker(run_id, nameserver=None, nameserver_port=None, logger=None, host=None, id=None, timeout=None)
# logger: logging.logger instance, logger used for debugging output
# id: anything with a __str__method, if multiple workers are started in the same process, you MUST provide a unique id for each one of them using the `id` argument.
# timeout: int or float, specifies the timeout a worker will wait for a new after finishing a computation before shutting down. Towards the end of a long run with multiple workers, this helps to shutdown idling workers. We recommend a timeout that is roughly half the time it would take for the second largest budget to finish. The default (None) means that the worker will wait indefinitely and never shutdown on its own.
worker.run(background=True)
# config = cs.sample_configuration().get_dictionary()
# print(config)
# res = worker.compute(config=config, budget=1, working_directory='.')
# print(res)
if args.restore is None:
previous_result = None
else:
previous_result = logged_results_to_HBS_result(directory=args.restore)
result_logger = json_result_logger(directory=args.directory, overwrite=True) # ???
optimizer = BOHB(
configspace=worker.get_configspace(), min_budget=0.5, max_budget=float(args.max_repeats),
run_id=args.id, working_directory=args.directory,
nameserver=nameserver, nameserver_port=nameserver_port, host=host,
result_logger=result_logger, previous_result=previous_result
)
# BOHB(configspace=None, eta=3, min_budget=0.01, max_budget=1, min_points_in_model=None, top_n_percent=15, num_samples=64, random_fraction=1 / 3, bandwidth_factor=3, min_bandwidth=1e-3, **kwargs)
# Master(run_id, config_generator, working_directory='.', ping_interval=60, nameserver='127.0.0.1', nameserver_port=None, host=None, shutdown_workers=True, job_queue_sizes=(-1,0), dynamic_queue_size=True, logger=None, result_logger=None, previous_result = None)
# logger: logging.logger like object, the logger to output some (more or less meaningful) information
results = optimizer.run(n_iterations=args.num_iterations)
# optimizer.run(n_iterations=1, min_n_workers=1, iteration_kwargs={})
# min_n_workers: int, minimum number of workers before starting the run
optimizer.shutdown(shutdown_workers=True)
server.shutdown()
environment.close()
with open(os.path.join(args.directory, 'results.pkl'), 'wb') as filehandle:
pickle.dump(results, filehandle)
print('Best found configuration:', results.get_id2config_mapping()[results.get_incumbent_id()]['config'])
print('Runs:', results.get_runs_by_id(config_id=results.get_incumbent_id()))
print('A total of {} unique configurations where sampled.'.format(len(results.get_id2config_mapping())))
print('A total of {} runs where executed.'.format(len(results.get_all_runs())))
print('Total budget corresponds to {:.1f} full function evaluations.'.format(
sum([r.budget for r in results.get_all_runs()]) / args.max_repeats)
)
class OptimizerBOHB(SearchStrategy, RandomSeed):
def __init__(
self,
base_task_id, # type: str
hyper_parameters, # type: Sequence[Parameter]
objective_metric, # type: Objective
execution_queue, # type: str
num_concurrent_workers, # type: int
min_iteration_per_job, # type: Optional[int]
max_iteration_per_job, # type: Optional[int]
total_max_jobs, # type: Optional[int]
pool_period_min=2., # type: float
time_limit_per_job=None, # type: Optional[float]
local_port=9090, # type: int
**bohb_kwargs, # type: Any
):
# type: (...) -> None
"""
Initialize a BOHB search strategy optimizer
BOHB performs robust and efficient hyperparameter optimization at scale by combining
the speed of Hyperband searches with the guidance and guarantees of convergence of Bayesian
Optimization. Instead of sampling new configurations at random,
BOHB uses kernel density estimators to select promising candidates.
For reference: ::
@InProceedings{falkner-icml-18,
title = {{BOHB}: Robust and Efficient Hyperparameter Optimization at Scale},
author = {Falkner, Stefan and Klein, Aaron and Hutter, Frank},
booktitle = {Proceedings of the 35th International Conference on Machine Learning},
pages = {1436--1445},
year = {2018},
}
:param str base_task_id: Task ID (str)
:param list hyper_parameters: list of Parameter objects to optimize over
:param Objective objective_metric: Objective metric to maximize / minimize
:param str execution_queue: execution queue to use for launching Tasks (experiments).
:param int num_concurrent_workers: Limit number of concurrent running Tasks (machines)
:param int min_iteration_per_job: minimum number of iterations for a job to run.
'iterations' are the reported iterations for the specified objective,
not the maximum reported iteration of the Task.
:param int max_iteration_per_job: number of iteration per job
'iterations' are the reported iterations for the specified objective,
not the maximum reported iteration of the Task.
:param int total_max_jobs: total maximum job for the optimization process.
Must be provided in order to calculate the total budget for the optimization process.
The total budget is measured by "iterations" (see above)
and will be set to `max_iteration_per_job * total_max_jobs`
This means more than total_max_jobs could be created, as long as the cumulative iterations
(summed over all created jobs) will not exceed `max_iteration_per_job * total_max_jobs`
:param float pool_period_min: time in minutes between two consecutive pools
:param float time_limit_per_job: Optional, maximum execution time per single job in minutes,
when time limit is exceeded job is aborted
:param int local_port: default port 9090 tcp, this is a must for the BOHB workers to communicate, even locally.
:param bohb_kwargs: arguments passed directly yo the BOHB object
"""
if not max_iteration_per_job or not min_iteration_per_job or not total_max_jobs:
raise ValueError(
"OptimizerBOHB is missing a defined budget.\n"
"The following arguments must be defined: "
"max_iteration_per_job, min_iteration_per_job, total_max_jobs.\n"
"Maximum optimization budget is: max_iteration_per_job * total_max_jobs\n"
)
super(OptimizerBOHB,
self).__init__(base_task_id=base_task_id,
hyper_parameters=hyper_parameters,
objective_metric=objective_metric,
execution_queue=execution_queue,
num_concurrent_workers=num_concurrent_workers,
pool_period_min=pool_period_min,
time_limit_per_job=time_limit_per_job,
max_iteration_per_job=max_iteration_per_job,
total_max_jobs=total_max_jobs)
self._max_iteration_per_job = max_iteration_per_job
self._min_iteration_per_job = min_iteration_per_job
self._bohb_kwargs = bohb_kwargs or {}
self._param_iterator = None
self._namespace = None
self._bohb = None
self._res = None
self._nameserver_port = local_port
def set_optimization_args(
self,
eta=3, # type: float
min_budget=None, # type: Optional[float]
max_budget=None, # type: Optional[float]
min_points_in_model=None, # type: Optional[int]
top_n_percent=15, # type: Optional[int]
num_samples=None, # type: Optional[int]
random_fraction=1 / 3., # type: Optional[float]
bandwidth_factor=3, # type: Optional[float]
min_bandwidth=1e-3, # type: Optional[float]
):
# type: (...) -> ()
"""
Defaults copied from BOHB constructor, see details in BOHB.__init__
BOHB performs robust and efficient hyperparameter optimization
at scale by combining the speed of Hyperband searches with the
guidance and guarantees of convergence of Bayesian
Optimization. Instead of sampling new configurations at random,
BOHB uses kernel density estimators to select promising candidates.
.. highlight:: none
For reference: ::
@InProceedings{falkner-icml-18,
title = {{BOHB}: Robust and Efficient Hyperparameter Optimization at Scale},
author = {Falkner, Stefan and Klein, Aaron and Hutter, Frank},
booktitle = {Proceedings of the 35th International Conference on Machine Learning},
pages = {1436--1445},
year = {2018},
}
Parameters
----------
eta : float (3)
In each iteration, a complete run of sequential halving is executed. In it,
after evaluating each configuration on the same subset size, only a fraction of
1/eta of them 'advances' to the next round.
Must be greater or equal to 2.
min_budget : float (0.01)
The smallest budget to consider. Needs to be positive!
max_budget : float (1)
The largest budget to consider. Needs to be larger than min_budget!
The budgets will be geometrically distributed
:math:`a^2 + b^2 = c^2 /sim /eta^k` for :math:`k/in [0, 1, ... , num/_subsets - 1]`.
min_points_in_model: int (None)
number of observations to start building a KDE. Default 'None' means
dim+1, the bare minimum.
top_n_percent: int (15)
percentage ( between 1 and 99, default 15) of the observations that are considered good.
num_samples: int (64)
number of samples to optimize EI (default 64)
random_fraction: float (1/3.)
fraction of purely random configurations that are sampled from the
prior without the model.
bandwidth_factor: float (3.)
to encourage diversity, the points proposed to optimize EI, are sampled
from a 'widened' KDE where the bandwidth is multiplied by this factor (default: 3)
min_bandwidth: float (1e-3)
to keep diversity, even when all (good) samples have the same value for one of the parameters,
a minimum bandwidth (Default: 1e-3) is used instead of zero.
"""
if min_budget:
self._bohb_kwargs['min_budget'] = min_budget
if max_budget:
self._bohb_kwargs['max_budget'] = max_budget
if num_samples:
self._bohb_kwargs['num_samples'] = num_samples
self._bohb_kwargs['eta'] = eta
self._bohb_kwargs['min_points_in_model'] = min_points_in_model
self._bohb_kwargs['top_n_percent'] = top_n_percent
self._bohb_kwargs['random_fraction'] = random_fraction
self._bohb_kwargs['bandwidth_factor'] = bandwidth_factor
self._bohb_kwargs['min_bandwidth'] = min_bandwidth
def start(self):
# type: () -> ()
"""
Start the Optimizer controller function loop()
If the calling process is stopped, the controller will stop as well.
Notice: This function returns only after optimization is completed! or stop() was called.
"""
# Step 1: Start a NameServer
fake_run_id = 'OptimizerBOHB_{}'.format(time())
# default port is 9090, we must have one, this is how BOHB workers communicate (even locally)
self._namespace = hpns.NameServer(run_id=fake_run_id,
host='127.0.0.1',
port=self._nameserver_port)
self._namespace.start()
# we have to scale the budget to the iterations per job, otherwise numbers might be too high
budget_iteration_scale = self._max_iteration_per_job
# Step 2: Start the workers
workers = []
for i in range(self._num_concurrent_workers):
w = _TrainsBandsterWorker(
optimizer=self,
sleep_interval=int(self.pool_period_minutes * 60),
budget_iteration_scale=budget_iteration_scale,
base_task_id=self._base_task_id,
objective=self._objective_metric,
queue_name=self._execution_queue,
nameserver='127.0.0.1',
nameserver_port=self._nameserver_port,
run_id=fake_run_id,
id=i)
w.run(background=True)
workers.append(w)
# Step 3: Run an optimizer
self._bohb = BOHB(configspace=self._convert_hyper_parameters_to_cs(),
run_id=fake_run_id,
num_samples=self.total_max_jobs,
min_budget=float(self._min_iteration_per_job) /
float(self._max_iteration_per_job),
**self._bohb_kwargs)
self._res = self._bohb.run(n_iterations=self.total_max_jobs,
min_n_workers=self._num_concurrent_workers)
# Step 4: if we get here, Shutdown
self.stop()
def stop(self):
# type: () -> ()
"""
Stop the current running optimization loop,
Called from a different thread than the start()
"""
# After the optimizer run, we must shutdown the master and the nameserver.
self._bohb.shutdown(shutdown_workers=True)
self._namespace.shutdown()
if not self._res:
return
# Step 5: Analysis
id2config = self._res.get_id2config_mapping()
incumbent = self._res.get_incumbent_id()
all_runs = self._res.get_all_runs()
# Step 6: Print Analysis
print('Best found configuration:', id2config[incumbent]['config'])
print('A total of {} unique configurations where sampled.'.format(
len(id2config.keys())))
print('A total of {} runs where executed.'.format(
len(self._res.get_all_runs())))
print('Total budget corresponds to {:.1f} full function evaluations.'.
format(
sum([r.budget for r in all_runs]) /
self._bohb_kwargs.get('max_budget', 1.0)))
print('Total budget corresponds to {:.1f} full function evaluations.'.
format(
sum([r.budget for r in all_runs]) /
self._bohb_kwargs.get('max_budget', 1.0)))
print('The run took {:.1f} seconds to complete.'.format(
all_runs[-1].time_stamps['finished'] -
all_runs[0].time_stamps['started']))
def _convert_hyper_parameters_to_cs(self):
# type: () -> CS.ConfigurationSpace
cs = CS.ConfigurationSpace(seed=self._seed)
for p in self._hyper_parameters:
if isinstance(p, UniformParameterRange):
hp = CSH.UniformFloatHyperparameter(p.name,
lower=p.min_value,
upper=p.max_value,
log=False,
q=p.step_size)
elif isinstance(p, UniformIntegerParameterRange):
hp = CSH.UniformIntegerHyperparameter(p.name,
lower=p.min_value,
upper=p.max_value,
log=False,
q=p.step_size)
elif isinstance(p, DiscreteParameterRange):
hp = CSH.CategoricalHyperparameter(p.name, choices=p.values)
else:
raise ValueError(
"HyperParameter type {} not supported yet with OptimizerBOHB"
.format(type(p)))
cs.add_hyperparameter(hp)
return cs
def main():
# Check quantities of train, validation and test images
train_images = np.array(glob("data/train/*/*"))
valid_images = np.array(glob("data/valid/*/*"))
test_images = np.array(glob("data/test/*/*"))
# Check relative percentages of image types
train_images_mel = np.array(glob("data/train/melanoma/*"))
train_images_nev = np.array(glob("data/train/nevus/*"))
train_images_seb = np.array(glob("data/train/seborrheic_keratosis/*"))
valid_images_mel = np.array(glob("data/valid/melanoma/*"))
valid_images_nev = np.array(glob("data/valid/nevus/*"))
valid_images_seb = np.array(glob("data/valid/seborrheic_keratosis/*"))
test_images_mel = np.array(glob("data/test/melanoma/*"))
test_images_nev = np.array(glob("data/test/nevus/*"))
test_images_seb = np.array(glob("data/test/seborrheic_keratosis/*"))
print(
"There are {} training images, {} validation images and {} test images."
.format(len(train_images), len(valid_images), len(test_images)))
print(
"For the training images, {mel:=.1f}% ({mel2}) are of melanoma, {nev:=.1f}% ({nev2}) are of nevus and {seb:=.1f}% ({seb2}) are for seborrheic keratosis."
.format(mel=len(train_images_mel) / len(train_images) * 100,
mel2=len(train_images_mel),
nev=len(train_images_nev) / len(train_images) * 100,
nev2=len(train_images_nev),
seb=len(train_images_seb) / len(train_images) * 100,
seb2=len(train_images_seb)))
print(
"For the validation images, {mel:=.1f}% ({mel2}) are of melanoma, {nev:=.1f}% ({nev2}) are of nevus and {seb:=.1f}% ({seb2}) are for seborrheic keratosis."
.format(mel=len(valid_images_mel) / len(valid_images) * 100,
mel2=len(valid_images_mel),
nev=len(valid_images_nev) / len(valid_images) * 100,
nev2=len(valid_images_nev),
seb=len(valid_images_seb) / len(valid_images) * 100,
seb2=len(valid_images_seb)))
print(
"For the test images, {mel:=.1f}% ({mel2}) are of melanoma, {nev:=.1f}% ({nev2}) are of nevus and {seb:=.1f}% ({seb2}) are for seborrheic keratosis."
.format(mel=len(test_images_mel) / len(test_images) * 100,
mel2=len(test_images_mel),
nev=len(test_images_nev) / len(test_images) * 100,
nev2=len(test_images_nev),
seb=len(test_images_seb) / len(test_images) * 100,
seb2=len(test_images_seb)))
# Set HpBandSter logging
logging.basicConfig(level=logging.DEBUG)
# Define the parser. Note that key parametres are the min_budget, max_budget, shared_directory and n_iterations.
parser = argparse.ArgumentParser(
description='ISIC2017 - CNN on Derm Dataset')
parser.add_argument('--min_budget',
type=float,
help='Minimum number of epochs for training.',
default=1)
parser.add_argument('--max_budget',
type=float,
help='Maximum number of epochs for training.',
default=3)
parser.add_argument('--n_iterations',
type=int,
help='Number of iterations performed by the optimizer',
default=16)
parser.add_argument('--worker',
help='Flag to turn this into a worker process',
action='store_true')
parser.add_argument(
'--run_id',
type=str,
help=
'A unique run id for this optimization run. An easy option is to use the job id of the clusters scheduler.'
)
parser.add_argument(
'--nic_name',
type=str,
help='Which network interface to use for communication.',
default='lo')
parser.add_argument(
'--shared_directory',
type=str,
help=
'A directory that is accessible for all processes, e.g. a NFS share.',
default='/home/ubuntu/src/derm-ai/data')
parser.add_argument(
'--backend',
help=
'Toggles which worker is used. Choose between a pytorch and a keras implementation.',
choices=['pytorch', 'keras'],
default='pytorch')
args = parser.parse_args([])
host = hpns.nic_name_to_host(args.nic_name)
# This example shows how to log live results. This is most useful
# for really long runs, where intermediate results could already be
# interesting. The core.result submodule contains the functionality to
# read the two generated files (results.json and configs.json) and
# create a Result object.
result_logger = hpres.json_result_logger(directory=args.shared_directory,
overwrite=True)
# Start a nameserver:
NS = hpns.NameServer(run_id=args.run_id,
host=host,
port=0,
working_directory=args.shared_directory)
ns_host, ns_port = NS.start()
# Start local worker
w = worker(run_id=args.run_id,
host=host,
nameserver=ns_host,
nameserver_port=ns_port,
timeout=120)
w.run(background=True)
bohb = BOHB(
configspace=w.get_configspace(),
run_id=args.run_id,
host=host,
nameserver=ns_host,
nameserver_port=ns_port,
result_logger=result_logger,
min_budget=args.min_budget,
max_budget=args.max_budget,
)
# Run an optimizer
res = bohb.run(n_iterations=args.n_iterations)
bohb.shutdown(shutdown_workers=True)
NS.shutdown()
