def compute(self, x, budget, config, **kwargs):
"""
Get model with hyperparameters from config generated by get_configspace()
"""
config = get_config_dictionary(x, config)
print("config", config)
if (len(config.keys()) < len(x)):
return 100
print('Retraining...')
nn_model = self.model_class(
n_classes=self.n_classes,
n_cells=20,
channels_init=self.channels_init,
affine=True,
ops_server=self.ops_server_class(affine=True),
trained_theta=self.trained_theta)
nn_model.fix_arc()
print('n_params: %fM' % (nn_model.get_params_mle() / 1e6))
with open(os.path.join(self.run_dir, 'description.txt'), 'a') as o:
o.write('dim_theta: %d\n' % nn_model.p_model.C.sum())
o.write('channels_init: %d\n' % self.channels_init)
o.write('n_params: %fM\n' % (nn_model.get_params_mle() / 1e6))
if self.gpu_id >= 0:
nn_model = nn_model.cuda()
optimizer = settings.opti_dict[config['optimizer']](
nn_model.parameters(),
lr=config['initial_lr'],
weight_decay=config['weight_decay'])
if config['lr_schedular'] == 'Cosine':
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, int(budget))
elif config['lr_schedular'] == 'Exponential':
lr_schedular = torch.optim.lr_schedular.ExponnetialLR(optimizer,
gamma=0.1)
elif config['lr_schedular'] == 'Plateau':
lr_schedular = torch.optim.lr_schedular.ReduceLROnPlateau(
optimizer)
nn_model, test_loss = train(
nn_model=nn_model,
loss_func=torch.nn.CrossEntropyLoss,
optimizer=optimizer,
lr_scheduler=lr_scheduler,
n_epochs=int(budget),
batch_size=config['batch_size_retrain'],
max_droppath_rate=config['max_droppath_rate'],
clip_value=config['grad_clip_value'],
weight_auxiliary=config['weight_auxiliary'],
train_data=self.train_dataset,
test_data=self.test_dataset,
gpu_id=self.gpu_id,
log_file=os.path.join(self.run_dir, 'train_log.csv'))
return test_loss # Hyperband always minimizes, so we want to minimise the error, error = 1-accuracy
def run_bohamiann(self, iterations=20):
cs = self.__class__.get_configspace()
lower, upper = get_upper_lower(cs)
results = bohamiann(self.compute, lower, upper, num_iterations=iterations, cs=cs)
x_best = results["x_opt"]
config = get_config_dictionary(x_best, cs)
print(config)
def run_es(self, iterations=20):
cs = self.__class__.get_configspace()
lower, upper = get_upper_lower(cs)
results = entropy_search(self.compute,
lower,
upper,
num_iterations=iterations,
cs=cs,
min_budget=2,
max_budget=5)
x_best = results["x_opt"]
config = get_config_dictionary(x_best, cs)
print(config)
def compute(self, x, budget, config, **kwargs):
"""
Get model with hyperparameters from config generated by get_configspace()
"""
config = get_config_dictionary(x, config)
print("config", config)
if (len(config.keys())<len(x)):
return 100
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
gpu = 'cuda:0'
np.random.seed(self.seed)
torch.cuda.set_device(gpu)
cudnn.benchmark = True
torch.manual_seed(self.seed)
cudnn.enabled=True
torch.cuda.manual_seed(self.seed)
logging.info('gpu device = %s' % gpu)
logging.info("config = %s", config)
genotype = eval("genotypes.%s" % 'PCDARTS')
model = Network(self.init_channels, self.n_classes, config['n_conv_layers'], genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
if config['optimizer'] == 'sgd':
optimizer = torch.optim.SGD(model.parameters(),
lr=config['initial_lr'],
momentum=0.9,
weight_decay=config['weight_decay'],
nesterov=True)
else:
optimizer = settings.opti_dict[config['optimizer']](model.parameters(), lr=config['initial_lr'])
if config['lr_scheduler'] == 'Cosine':
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, int(budget))
elif config['lr_scheduler'] == 'Exponential':
lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1)
indices = list(range(int(self.split*len(self.train_dataset))))
valid_indices = list(range(int(self.split*len(self.train_dataset)), len(self.train_dataset)))
print("Training size=", len(indices))
training_sampler = SubsetRandomSampler(indices)
valid_sampler = SubsetRandomSampler(valid_indices)
train_queue = torch.utils.data.DataLoader(dataset=self.train_dataset,
batch_size=self.batch_size,
sampler=training_sampler)
valid_queue = torch.utils.data.DataLoader(dataset=self.train_dataset,
batch_size=self.batch_size,
sampler=valid_sampler)
for epoch in range(int(budget)):
lr_scheduler.step()
logging.info('epoch %d lr %e', epoch, lr_scheduler.get_lr()[0])
model.drop_path_prob = config['drop_path_prob'] * epoch / int(budget)
train_acc, train_obj = train(train_queue, model, criterion, optimizer, grad_clip=config['grad_clip_value'])
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
return valid_obj # Hyperband always minimizes, so we want to minimise the error, error = 1-acc
def set_plugin_config(core: IECore, device: str, config: str = None):
core.set_config(get_config_dictionary(config_file=config), device_name=device)
def entropy_search(objective_function,
lower,
upper,
cs,
num_iterations=30,
maximizer="direct",
model="gp_mcmc",
min_budget=0.1,
max_budget=1,
n_init=3,
output_path=None,
rng=None):
"""
Entropy search for global black box optimization problems. This is a reimplemenation of the entropy search
algorithm by Henning and Schuler[1].
[1] Entropy search for information-efficient global optimization.
P. Hennig and C. Schuler.
JMLR, (1), 2012.
Parameters
----------
objective_function: function
The objective function that is minimized. This function gets a numpy array (D,) as input and returns
the function value (scalar)
lower: np.ndarray (D,)
The lower bound of the search space
upper: np.ndarray (D,)
The upper bound of the search space
cs: ConfigSpace
The configspace for the hyperparameters to be optimised.
num_iterations: int
The number of iterations (initial design + BO)
maximizer: {"direct", "cmaes", "differential_evolution"}
Defines how the acquisition function is maximized. NOTE: "cmaes" only works in D > 1 dimensions
model: {"gp", "gp_mcmc"}
The model for the objective function.
n_init: int
Number of points for the initial design. Make sure that it is <= num_iterations.
output_path: string
Specifies the path where the intermediate output after each iteration will be saved.
If None no output will be saved to disk.
rng: numpy.random.RandomState
Random number generator
Returns
-------
dict with all results
"""
assert upper.shape[0] == lower.shape[0], "Dimension miss match"
assert np.all(lower < upper), "Lower bound >= upper bound"
assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations"
if rng is None:
rng = np.random.RandomState(np.random.randint(0, 10000))
cov_amp = 2
n_dims = lower.shape[0]
initial_ls = np.ones([n_dims])
exp_kernel = george.kernels.Matern52Kernel(initial_ls, ndim=n_dims)
kernel = cov_amp * exp_kernel
prior = DefaultPrior(len(kernel) + 1)
n_hypers = 3 * len(kernel)
if n_hypers % 2 == 1:
n_hypers += 1
if model == "gp":
gp = GaussianProcess(kernel,
prior=prior,
rng=rng,
normalize_output=False,
normalize_input=True,
lower=lower,
upper=upper)
elif model == "gp_mcmc":
gp = GaussianProcessMCMC(kernel,
prior=prior,
n_hypers=n_hypers,
chain_length=200,
burnin_steps=100,
normalize_input=True,
normalize_output=False,
rng=rng,
lower=lower,
upper=upper)
else:
print("ERROR: %s is not a valid model!" % model)
return
a = InformationGain(gp, lower=lower, upper=upper, sampling_acquisition=EI)
if model == "gp":
acquisition_func = a
elif model == "gp_mcmc":
acquisition_func = MarginalizationGPMCMC(a)
if maximizer == "cmaes":
max_func = CMAES(acquisition_func,
lower,
upper,
verbose=False,
rng=rng)
elif maximizer == "direct":
max_func = Direct(acquisition_func, lower, upper)
elif maximizer == "differential_evolution":
max_func = DifferentialEvolution(acquisition_func,
lower,
upper,
rng=rng)
else:
print(
"ERROR: %s is not a valid function to maximize the acquisition function!"
% maximizer)
return
bo = BayesianOptimization(objective_function,
lower,
upper,
acquisition_func,
gp,
max_func,
initial_design=init_latin_hypercube_sampling,
initial_points=n_init,
rng=rng,
output_path=output_path,
min_budget=min_budget,
max_budget=max_budget,
cs=cs)
x_best, f_min = bo.run(num_iterations)
results = dict()
results["x_opt"] = get_config_dictionary(x_best, cs)
results["f_opt"] = f_min
results["incumbents"] = [
get_config_dictionary(inc, cs) for inc in bo.incumbents
]
results["incumbent_values"] = [val for val in bo.incumbents_values]
results["runtime"] = bo.runtime
results["overhead"] = bo.time_overhead
results["X"] = [get_config_dictionary(x.tolist(), cs) for x in bo.X]
results["y"] = [y for y in bo.y]
return results
def set_plugin_config(plugin: IEPlugin, config: str = None):
plugin.set_config(get_config_dictionary(config_file=config))
return plugin
def bohamiann(objective_function,
lower,
upper,
cs,
num_iterations=30,
maximizer="direct",
acquisition_func="log_ei",
n_init=3,
output_path=None,
rng=None):
"""
Bohamiann uses Bayesian neural networks to model the objective function [1] inside Bayesian optimization.
Bayesian neural networks usually scale better with the number of function evaluations and the number of dimensions
than Gaussian processes.
[1] Bayesian optimization with robust Bayesian neural networks
J. T. Springenberg and A. Klein and S. Falkner and F. Hutter
Advances in Neural Information Processing Systems 29
Parameters
----------
objective_function: function
The objective function that is minimized. This function gets a numpy array (D,) as input and returns
the function value (scalar)
lower: np.ndarray (D,)
The lower bound of the search space
upper: np.ndarray (D,)
The upper bound of the search space
num_iterations: int
The number of iterations (initial design + BO)
acquisition_func: {"ei", "log_ei", "lcb", "pi"}
The acquisition function
maximizer: {"direct", "cmaes", "random", "scipy", "differential_evolution"}
The optimizer for the acquisition function. NOTE: "cmaes" only works in D > 1 dimensions
n_init: int
Number of points for the initial design. Make sure that it is <= num_iterations.
output_path: string
Specifies the path where the intermediate output after each iteration will be saved.
If None no output will be saved to disk.
rng: numpy.random.RandomState
Random number generator
Returns
-------
dict with all results
"""
assert upper.shape[0] == lower.shape[0]
assert n_init <= num_iterations, "Number of initial design point has to be <= than the number of iterations"
if rng is None:
rng = np.random.RandomState(np.random.randint(0, 10000))
model = DG(num_epochs=20000,
learning_rate=0.01,
momentum=0.9,
adapt_epoch=5000,
prior=None,
H=50,
D=10,
alpha=1.0,
beta=1000)
if acquisition_func == "ei":
a = EI(model)
elif acquisition_func == "log_ei":
a = LogEI(model)
elif acquisition_func == "pi":
a = PI(model)
elif acquisition_func == "lcb":
a = LCB(model)
else:
print("ERROR: %s is not a valid acquisition function!" %
acquisition_func)
return
if maximizer == "cmaes":
max_func = CMAES(a, lower, upper, verbose=True, rng=rng)
elif maximizer == "direct":
max_func = Direct(a, lower, upper, verbose=True)
elif maximizer == "random":
max_func = RandomSampling(a, lower, upper, rng=rng)
elif maximizer == "scipy":
max_func = SciPyOptimizer(a, lower, upper, rng=rng)
elif maximizer == "differential_evolution":
max_func = DifferentialEvolution(a, lower, upper, rng=rng)
bo = BayesianOptimization(objective_function,
lower,
upper,
a,
model,
max_func,
cs,
initial_points=n_init,
output_path=output_path,
rng=rng,
min_budget=2,
max_budget=5)
x_best, f_min = bo.run(num_iterations)
results = dict()
results["x_opt"] = get_config_dictionary(x_best, cs)
results["f_opt"] = f_min
results["incumbents"] = [
get_config_dictionary(inc, cs) for inc in bo.incumbents
]
results["incumbent_values"] = [val for val in bo.incumbents_values]
results["runtime"] = bo.runtime
results["overhead"] = bo.time_overhead
results["X"] = [get_config_dictionary(x.tolist(), cs) for x in bo.X]
results["y"] = [y for y in bo.y]
return results