def main(benchmark_name, dataset_name, dimensions, method_name, num_runs,
run_start, num_iterations, eta, min_budget, max_budget, input_dir,
output_dir):
benchmark = make_benchmark(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name,
input_dir=input_dir)
name = make_name(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name)
output_path = Path(output_dir).joinpath(name, method_name)
output_path.mkdir(parents=True, exist_ok=True)
options = dict(eta=eta, min_budget=min_budget, max_budget=max_budget)
with output_path.joinpath("options.yaml").open('w') as f:
yaml.dump(options, f)
for run_id in range(run_start, num_runs):
NS = hpns.NameServer(run_id=run_id, host='localhost', port=0)
ns_host, ns_port = NS.start()
num_workers = 1
workers = []
for worker_id in range(num_workers):
w = BenchmarkWorker(benchmark=benchmark,
nameserver=ns_host,
nameserver_port=ns_port,
run_id=run_id,
id=worker_id)
w.run(background=True)
workers.append(w)
rs = RandomSearch(configspace=benchmark.get_config_space(),
run_id=run_id,
nameserver=ns_host,
nameserver_port=ns_port,
ping_interval=10,
**options)
results = rs.run(num_iterations, min_n_workers=num_workers)
rs.shutdown(shutdown_workers=True)
NS.shutdown()
data = HpBandSterLogs(results).to_frame()
data.to_csv(output_path.joinpath(f"{run_id:03d}.csv"))
return 0
def main(benchmark_name, dataset_name, dimensions, method_name, num_runs,
run_start, num_iterations, acquisition_name,
acquisition_optimizer_name, num_random_init, use_ard,
use_input_warping, input_dir, output_dir):
benchmark = make_benchmark(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name,
input_dir=input_dir)
name = make_name(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name)
output_path = Path(output_dir).joinpath(name, method_name)
output_path.mkdir(parents=True, exist_ok=True)
options = dict(acquisition_name=acquisition_name,
acquisition_optimizer_name=acquisition_optimizer_name,
use_ard=use_ard,
use_input_warping=use_input_warping)
with output_path.joinpath("options.yaml").open('w') as f:
yaml.dump(options, f)
space = benchmark.get_domain()
config_space = benchmark.get_config_space()
def func(array, *args, **kwargs):
kws = dict_from_array(array, cs=config_space)
return benchmark(kws).value
model_type = "input_warped_GP" if use_input_warping else "GP"
for run_id in trange(run_start, num_runs):
BO = GPyOpt.methods.BayesianOptimization(
f=func,
domain=space,
initial_design_numdata=num_random_init,
model_type=model_type,
ARD=use_ard,
normalize_Y=True,
exact_feval=False,
acquisition_type=acquisition_name,
acquisition_optimizer_type=acquisition_optimizer_name)
BO.run_optimization(num_iterations)
data = pd.DataFrame(data=BO.X, columns=[d["name"] for d in space]) \
.assign(loss=BO.Y)
data.to_csv(output_path.joinpath(f"{run_id:03d}.csv"))
return 0
def main(benchmark_name, dataset_name, dimensions, method_name, num_runs,
run_start, num_iterations, input_dir, output_dir):
benchmark = make_benchmark(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name,
input_dir=input_dir)
name = make_name(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name)
output_path = Path(output_dir).joinpath(name, method_name)
output_path.mkdir(parents=True, exist_ok=True)
options = dict()
with output_path.joinpath("options.yaml").open('w') as f:
yaml.dump(options, f)
def objective(config, seed):
return benchmark.evaluate(config).value
for run_id in range(run_start, num_runs):
random_state = np.random.RandomState(run_id)
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": num_iterations,
"cs": benchmark.get_config_space(),
"deterministic": "true",
"output_dir": "foo/"
})
run_history = RunHistory()
smac = SMAC4HPO(scenario=scenario,
tae_runner=objective,
runhistory=run_history,
rng=random_state)
smac.optimize()
data = SMACLogs(run_history).to_frame()
data.to_csv(output_path.joinpath(f"{run_id:03d}.csv"))
return 0
def main(benchmark_name, dataset_name, dimensions, method_name, num_runs,
run_start, num_iterations, input_dir, output_dir):
benchmark = make_benchmark(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name,
input_dir=input_dir)
space = benchmark.get_search_space()
name = make_name(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name)
output_path = Path(output_dir).joinpath(name, method_name)
output_path.mkdir(parents=True, exist_ok=True)
options = dict()
with output_path.joinpath("options.yaml").open('w') as f:
yaml.dump(options, f)
def objective(kws):
evaluation = benchmark.evaluate(kws)
return dict(loss=evaluation.value,
status=STATUS_OK,
info=evaluation.duration)
for run_id in range(run_start, num_runs):
trials = Trials()
best = fmin(objective,
space=space,
algo=tpe.suggest,
max_evals=num_iterations,
trials=trials)
data = HyperOptLogs(trials).to_frame()
data.to_csv(output_path.joinpath(f"{run_id:03d}.csv"))
return 0
def main(
benchmark_name,
dataset_name,
dimensions,
method_name,
num_runs,
run_start,
num_iterations,
acquisition_name,
# acquisition_optimizer_name,
gamma,
num_random_init,
mc_samples,
batch_size,
num_fantasies,
num_restarts,
raw_samples,
noise_variance_init,
# use_ard,
# use_input_warping,
standardize_targets,
input_dir,
output_dir):
# TODO(LT): Turn into options
# device = "cpu"
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
dtype = torch.double
benchmark = make_benchmark(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name,
input_dir=input_dir)
name = make_name(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name)
output_path = Path(output_dir).joinpath(name, method_name)
output_path.mkdir(parents=True, exist_ok=True)
options = dict(gamma=gamma,
num_random_init=num_random_init,
acquisition_name=acquisition_name,
mc_samples=mc_samples,
batch_size=batch_size,
num_restarts=num_restarts,
raw_samples=raw_samples,
num_fantasies=num_fantasies,
noise_variance_init=noise_variance_init,
standardize_targets=standardize_targets)
with output_path.joinpath("options.yaml").open('w') as f:
yaml.dump(options, f)
config_space = DenseConfigurationSpace(benchmark.get_config_space())
bounds = create_bounds(config_space.get_bounds(),
device=device,
dtype=dtype)
input_dim = config_space.get_dimensions()
def func(tensor, *args, **kwargs):
"""
Wrapper that receives and returns torch.Tensor
"""
config = dict_from_tensor(tensor, cs=config_space)
# turn into maximization problem
res = -benchmark.evaluate(config).value
return torch.tensor(res, device=device, dtype=dtype)
for run_id in trange(run_start, num_runs, unit="run"):
run_begin_t = batch_end_t_adj = batch_end_t = datetime.now()
frames = []
features = []
targets = []
noise_variance = torch.tensor(noise_variance_init,
device=device,
dtype=dtype)
state_dict = None
with trange(num_iterations) as iterations:
for batch in iterations:
if len(targets) < num_random_init:
# click.echo(f"Completed {i}/{num_random_init} initial runs. "
# "Suggesting random candidate...")
# TODO(LT): support random seed
X_batch = torch.rand(size=(batch_size, input_dim),
device=device,
dtype=dtype)
else:
# construct dataset
X = torch.vstack(features)
y = torch.hstack(targets).unsqueeze(axis=-1)
y = standardize(y) if standardize_targets else y
# construct model
# model = FixedNoiseGP(X, standardize(y), noise_variance.expand_as(y),
model = FixedNoiseGP(X,
y,
noise_variance.expand_as(y),
input_transform=None).to(X)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
if state_dict is not None:
model.load_state_dict(state_dict)
# update model
fit_gpytorch_model(mll)
# construct acquisition function
tau = torch.quantile(y, q=1 - gamma)
iterations.set_postfix(tau=tau.item())
if acquisition_name == "q-KG":
assert num_fantasies is not None and num_fantasies > 0
acq = qKnowledgeGradient(model,
num_fantasies=num_fantasies)
elif acquisition_name == "q-EI":
assert mc_samples is not None and mc_samples > 0
qmc_sampler = SobolQMCNormalSampler(
num_samples=mc_samples)
acq = qExpectedImprovement(model=model,
best_f=tau,
sampler=qmc_sampler)
# optimize acquisition function
X_batch, b = optimize_acqf(acq_function=acq,
bounds=bounds,
q=batch_size,
num_restarts=num_restarts,
raw_samples=raw_samples,
options=dict(batch_limit=5,
maxiter=200))
state_dict = model.state_dict()
# begin batch evaluation
batch_begin_t = datetime.now()
decision_duration = batch_begin_t - batch_end_t
batch_begin_t_adj = batch_end_t_adj + decision_duration
eval_end_times = []
# TODO(LT): Deliberately not doing broadcasting for now since
# batch sizes are so small anyway. Can revisit later if there
# is a compelling reason to do it.
rows = []
for j, x_next in enumerate(X_batch):
# eval begin time
eval_begin_t = datetime.now()
# evaluate blackbox objective
y_next = func(x_next)
# eval end time
eval_end_t = datetime.now()
# eval duration
eval_duration = eval_end_t - eval_begin_t
# adjusted eval end time is the duration added to the
# time at which batch eval was started
eval_end_t_adj = batch_begin_t_adj + eval_duration
eval_end_times.append(eval_end_t_adj)
elapsed = eval_end_t_adj - run_begin_t
# update dataset
features.append(x_next)
targets.append(y_next)
row = dict_from_tensor(x_next, cs=config_space)
row["loss"] = -y_next.item()
row["cost_eval"] = eval_duration.total_seconds()
row["finished"] = elapsed.total_seconds()
rows.append(row)
batch_end_t = datetime.now()
batch_end_t_adj = max(eval_end_times)
frame = pd.DataFrame(data=rows) \
.assign(batch=batch,
cost_decision=decision_duration.total_seconds())
frames.append(frame)
data = pd.concat(frames, axis="index", ignore_index=True)
data.to_csv(output_path.joinpath(f"{run_id:03d}.csv"))
return 0
def main(benchmark_name, dataset_name, dimensions, method_name, num_runs,
run_start, num_iterations, eta, min_budget, max_budget, gamma,
num_random_init, init_method, batch_size,
batch_size_training, num_steps_per_iter, num_epochs, optimizer,
num_layers, num_units, activation, transform, max_iter,
step_size, length_scale, tau, lambd, input_dir, output_dir):
benchmark = make_benchmark(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name,
input_dir=input_dir)
name = make_name(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name)
output_path = Path(output_dir).joinpath(name, method_name)
output_path.mkdir(parents=True, exist_ok=True)
options = dict(eta=eta, min_budget=min_budget, max_budget=max_budget,
gamma=gamma, num_random_init=num_random_init,
batch_size=batch_size,
batch_size_training=batch_size_training,
num_steps_per_iter=num_steps_per_iter,
num_epochs=num_epochs, optimizer=optimizer,
num_layers=num_layers, num_units=num_units,
activation=activation, transform=transform,
max_iter=max_iter, step_size=step_size, tau=tau,
lambd=lambd, length_scale=length_scale,
init_method=init_method)
with output_path.joinpath("options.yaml").open('w') as f:
yaml.dump(options, f)
n_init_batches = ceil_divide(num_random_init, batch_size) * batch_size
for run_id in trange(run_start, num_runs, unit="run"):
run_begin_t = batch_end_t_adj = batch_end_t = datetime.now()
frames = []
random_state = np.random.RandomState(seed=run_id)
config_space = DenseConfigurationSpace(benchmark.get_config_space(),
seed=run_id)
input_dim = config_space.get_dimensions(sparse=False)
bounds = config_space.get_bounds()
model = create_model(input_dim, num_units, num_layers, activation,
optimizer)
record = Record()
if init_method == "latin_hypercube":
sampler = LatinHypercube(d=input_dim, seed=run_id)
X_init = (bounds.ub - bounds.lb) * sampler.random(n_init_batches) + bounds.lb
else:
X_init = random_state.uniform(low=bounds.lb,
high=bounds.ub,
size=(n_init_batches, input_dim))
with trange(num_iterations) as iterations:
for batch in iterations:
if record.size() < num_random_init:
# config_batch = config_space.sample_configuration(size=batch_size)
# X_batch = [config.to_array() for config in config_batch]
a = batch * batch_size
b = a + batch_size
X_batch = X_init[a:b]
else:
# construct binary classification problem
X, z = record.load_classification_data(gamma)
if num_epochs is None:
dataset_size = record.size()
num_steps = steps_per_epoch(batch_size_training, dataset_size)
num_epochs = num_steps_per_iter // num_steps
# update classifier
model.fit(X, z, epochs=num_epochs,
batch_size=batch_size_training, verbose=False)
X_batch = model.argmax_batch(batch_size=batch_size,
n_iter=max_iter,
length_scale=length_scale,
step_size=step_size,
bounds=bounds,
tau=tau, lambd=lambd,
# print_fn=click.echo,
random_state=random_state)
# begin batch evaluation
batch_begin_t = datetime.now()
decision_duration = batch_begin_t - batch_end_t
batch_begin_t_adj = batch_end_t_adj + decision_duration
eval_end_times = []
# TODO(LT): Deliberately not doing broadcasting for now since
# batch sizes are so small anyway. Can revisit later if there
# is a compelling reason to do it.
rows = []
for j, x_next in enumerate(X_batch):
config = dict_from_array(config_space, x_next)
# eval begin time
eval_begin_t = datetime.now()
# evaluate blackbox objective
y_next = benchmark.evaluate(config).value
# eval end time
eval_end_t = datetime.now()
# eval duration
eval_duration = eval_end_t - eval_begin_t
# adjusted eval end time is the duration added to the
# time at which batch eval was started
eval_end_t_adj = batch_begin_t_adj + eval_duration
eval_end_times.append(eval_end_t_adj)
elapsed = eval_end_t_adj - run_begin_t
# update dataset
record.append(x=x_next, y=y_next)
row = dict(config)
row["loss"] = y_next
row["cost_eval"] = eval_duration.total_seconds()
row["finished"] = elapsed.total_seconds()
rows.append(row)
batch_end_t = datetime.now()
batch_end_t_adj = max(eval_end_times)
frame = pd.DataFrame(data=rows) \
.assign(batch=batch,
cost_decision=decision_duration.total_seconds())
frames.append(frame)
data = pd.concat(frames, axis="index", ignore_index=True)
data.to_csv(output_path.joinpath(f"{run_id:03d}.csv"))
return 0
def main(benchmark_name, dataset_name, dimensions, method_name, num_runs,
run_start, num_iterations, eta, min_budget, max_budget, gamma,
num_random_init, random_rate, retrain, num_starts, num_samples,
batch_size, num_steps_per_iter, num_epochs_per_iter, optimizer,
num_layers, num_units, activation, l2_factor, transform, method,
max_iter, ftol, distortion, input_dir, output_dir):
benchmark = make_benchmark(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name,
input_dir=input_dir)
name = make_name(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name)
output_path = Path(output_dir).joinpath(name, method_name)
output_path.mkdir(parents=True, exist_ok=True)
options = dict(eta=eta,
min_budget=min_budget,
max_budget=max_budget,
gamma=gamma,
num_random_init=num_random_init,
random_rate=random_rate,
retrain=retrain,
num_starts=num_starts,
num_samples=num_samples,
batch_size=batch_size,
num_steps_per_iter=num_steps_per_iter,
num_epochs_per_iter=num_epochs_per_iter,
optimizer=optimizer,
num_layers=num_layers,
num_units=num_units,
activation=activation,
l2_factor=l2_factor,
transform=transform,
method=method,
max_iter=max_iter,
ftol=ftol,
distortion=distortion)
with output_path.joinpath("options.yaml").open('w') as f:
yaml.dump(options, f)
for run_id in range(run_start, num_runs):
NS = hpns.NameServer(run_id=run_id, host='localhost', port=0)
ns_host, ns_port = NS.start()
num_workers = 1
workers = []
for worker_id in range(num_workers):
w = BenchmarkWorker(benchmark=benchmark,
nameserver=ns_host,
nameserver_port=ns_port,
run_id=run_id,
id=worker_id)
w.run(background=True)
workers.append(w)
rs = BORE(config_space=benchmark.get_config_space(),
run_id=run_id,
nameserver=ns_host,
nameserver_port=ns_port,
ping_interval=10,
seed=run_id,
**options)
results = rs.run(num_iterations, min_n_workers=num_workers)
rs.shutdown(shutdown_workers=True)
NS.shutdown()
data = HpBandSterLogs(results).to_frame()
data.to_csv(output_path.joinpath(f"{run_id:03d}.csv"))
return 0
def main(benchmark_name, method_name, num_runs, x_key, y_key, x_num, y_num,
log_error_lim, num_error_levels, col_wrap, transparent, context,
style, palette, width, height, aspect, dpi, extension,
input_dir, output_dir):
# preamble
if height is None:
height = width / aspect
# height *= num_iterations
# figsize = size(width, aspect)
figsize = (width, height)
suffix = f"{width*dpi:.0f}x{height*dpi:.0f}"
rc = {
"figure.figsize": figsize,
"font.serif": ["Times New Roman"],
"text.usetex": True,
}
sns.set(context=context, style=style, palette=palette, font="serif", rc=rc)
input_path = Path(input_dir).joinpath(benchmark_name)
output_path = Path(output_dir).joinpath(benchmark_name, method_name)
output_path.mkdir(parents=True, exist_ok=True)
benchmark = make_benchmark(benchmark_name)
bounds = benchmark.get_bounds()
(low, high), input_dim = from_bounds(bounds)
(x1_min, x2_min), (x1_max, x2_max) = low, high
x2, x1 = np.ogrid[x2_min:x2_max:y_num*1j,
x1_min:x1_max:x_num*1j]
X1, X2 = np.broadcast_arrays(x1, x2)
Y = benchmark(X1, X2) - benchmark.get_minimum()
frames = []
for run in trange(num_runs):
path = input_path.joinpath(method_name, f"{run:03d}.csv")
frame = load_frame(path, run, loss_min=benchmark.get_minimum())
frames.append(frame.assign(method=method_name))
data = pd.concat(frames, axis="index", ignore_index=True, sort=True)
# TODO: height should actually be `height_in / row_wrap`, but we don't
# know what `row_wrap` is.
g = sns.relplot(x=x_key, y=y_key, hue="batch", # size="error",
col="run", col_wrap=col_wrap,
palette="mako_r", alpha=0.8,
height=height, aspect=aspect,
kind="scatter", data=data, rasterized=True)
# facet_kws=dict(subplot_kws=dict(rasterized=True)))
g.map(contour, X=X1, Y=X2, Z=Y,
levels=np.logspace(*log_error_lim, num_error_levels),
norm=LogNorm(), alpha=0.4, cmap="turbo", zorder=-1)
for ext in extension:
g.savefig(output_path.joinpath(f"scatter_{context}_{suffix}.{ext}"))
return 0
def main(benchmark_name, dataset_name, dimensions, method_name, num_runs,
run_start, num_iterations,
# acquisition_name,
# acquisition_optimizer_name,
gamma, num_random_init,
num_restarts, raw_samples, noise_variance_init,
# use_ard,
# use_input_warping,
standardize_targets,
input_dir, output_dir):
# TODO(LT): Turn into options
# device = "cpu"
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
dtype = torch.double
benchmark = make_benchmark(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name,
input_dir=input_dir)
name = make_name(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name)
output_path = Path(output_dir).joinpath(name, method_name)
output_path.mkdir(parents=True, exist_ok=True)
options = dict(gamma=gamma, num_random_init=num_random_init,
num_restarts=num_restarts, raw_samples=raw_samples,
noise_variance_init=noise_variance_init,
standardize_targets=standardize_targets)
with output_path.joinpath("options.yaml").open('w') as f:
yaml.dump(options, f)
config_space = DenseConfigurationSpace(benchmark.get_config_space())
bounds = create_bounds(config_space.get_bounds(), device=device, dtype=dtype)
input_dim = config_space.get_dimensions()
def func(tensor, *args, **kwargs):
"""
Wrapper that receives and returns torch.Tensor
"""
config = dict_from_tensor(tensor, cs=config_space)
# turn into maximization problem
res = - benchmark.evaluate(config).value
return torch.tensor(res, device=device, dtype=dtype)
for run_id in trange(run_start, num_runs, unit="run"):
t_start = datetime.now()
rows = []
features = []
targets = []
noise_variance = torch.tensor(noise_variance_init, device=device, dtype=dtype)
state_dict = None
with trange(num_iterations) as iterations:
for i in iterations:
if len(targets) < num_random_init:
# click.echo(f"Completed {i}/{num_random_init} initial runs. "
# "Suggesting random candidate...")
# TODO(LT): support random seed
x_new = torch.rand(size=(input_dim,), device=device, dtype=dtype)
else:
# construct dataset
X = torch.vstack(features)
y = torch.hstack(targets).unsqueeze(axis=-1)
y = standardize(y) if standardize_targets else y
# construct model
# model = FixedNoiseGP(X, standardize(y), noise_variance.expand_as(y),
model = FixedNoiseGP(X, y, noise_variance.expand_as(y),
input_transform=None).to(X)
mll = ExactMarginalLogLikelihood(model.likelihood, model)
if state_dict is not None:
model.load_state_dict(state_dict)
# update model
fit_gpytorch_model(mll)
# construct acquisition function
tau = torch.quantile(y, q=1-gamma)
iterations.set_postfix(tau=tau.item())
ei = ExpectedImprovement(model=model, best_f=tau)
# optimize acquisition function
X_batch, b = optimize_acqf(acq_function=ei, bounds=bounds,
q=1,
num_restarts=num_restarts,
raw_samples=raw_samples,
options=dict(batch_limit=5,
maxiter=200))
x_new = X_batch.squeeze(axis=0)
state_dict = model.state_dict()
# evaluate blackbox objective
# t0 = datetime.now()
y_new = func(x_new)
t1 = datetime.now()
delta = t1 - t_start
# update dataset
features.append(x_new)
targets.append(y_new)
row = dict_from_tensor(x_new, cs=config_space)
row["loss"] = - y_new.item()
row["finished"] = delta.total_seconds()
rows.append(row)
data = pd.DataFrame(data=rows)
data.to_csv(output_path.joinpath(f"{run_id:03d}.csv"))
return 0
def main(name, benchmark_name, output_dir, n_iterations, gamma, batch_size,
num_epochs, optimizer, num_layers, num_units, activation, transparent,
context, style, palette, width, height, aspect, dpi, extension, seed):
next_loc_color = "tab:green"
n_index_points_x = n_index_points_y = 512
n_random_init = 5
n_starts = 5
noise_scale = 0.05
random_state = np.random.RandomState(seed)
# preamble
if height is None:
height = width / aspect
# height *= num_iterations
# figsize = size(width, aspect)
figsize = (width, height)
suffix = f"{width*dpi:.0f}x{height*dpi:.0f}"
rc = {
"figure.figsize": figsize,
"font.serif": ["Times New Roman"],
"text.usetex": True,
}
sns.set(context=context, style=style, palette=palette, font="serif", rc=rc)
output_path = Path(output_dir).joinpath(name)
output_path.mkdir(parents=True, exist_ok=True)
# / preamble
benchmark = make_benchmark(benchmark_name)
def func(x, y):
benchmark(x, y) # - benchmark.get_minimum()
bounds = benchmark.get_bounds()
(low, high), input_dim = from_bounds(bounds)
(x1_min, x2_min), (x1_max, x2_max) = low, high
x2, x1 = np.ogrid[x2_min:x2_max:n_index_points_y * 1j,
x1_min:x1_max:n_index_points_x * 1j]
X1, X2 = np.broadcast_arrays(x1, x2)
Y = benchmark(X1, X2) - benchmark.get_minimum()
fig, ax = plt.subplots()
ax.contour(X1,
X2,
Y,
levels=np.logspace(-2, 2, 13),
norm=LogNorm(),
alpha=0.6,
cmap="turbo")
plt.tight_layout()
for ext in extension:
fig.savefig(output_path.joinpath(f"foo_{context}_{suffix}.{ext}"),
dpi=dpi,
transparent=transparent)
plt.clf()
model = BatchMaximizableSequential()
model.add(Dense(num_units, input_dim=input_dim, activation=activation))
model.add(Dense(num_units, activation=activation))
model.add(Dense(num_units, activation=activation))
model.add(Dense(1))
# model.add(Dense(1, activation="sigmoid"))
model.compile(optimizer=optimizer,
loss=BinaryCrossentropy(
from_logits=True)) # loss="binary_crossentropy")
model.summary(print_fn=click.echo)
# random initial design
features = []
targets = []
for i in range(n_random_init):
# propose new point
x = random_state.uniform(low=low, high=high, size=(input_dim, ))
# initial_designs = [0.025, 0.15, 0.825, 0.975]
# for x in initial_designs:
# evaluate
y = benchmark(x[0], x[1]) + random_state.normal(scale=noise_scale)
features.append(x)
targets.append(y)
# fig, axes = plt.subplots(nrows=num_iterations, sharex=True)
# for i, ax in enumerate(axes):
prev = None
for i in range(n_iterations):
# construct binary classification problem
X = np.vstack(features)
y = np.hstack(targets)
tau = np.quantile(y, q=gamma, interpolation="lower")
z = np.less_equal(y, tau)
# update classifier
model.fit(X,
z,
epochs=num_epochs,
batch_size=batch_size,
verbose=False)
# suggest new candidate
res = model.argmax(method="L-BFGS-B",
num_starts=n_starts,
bounds=bounds,
print_fn=click.echo)
X_batch = model.argmax_batch(batch_size=5,
n_iter=2000,
length_scale=None,
step_size=1e-3,
bounds=bounds)
x_next = res.x
# print(x_next)
# evaluate blackbox
y_next = benchmark(x_next[0],
x_next[1]) + random_state.normal(scale=noise_scale)
# update dataset
features.append(x_next)
targets.append(y_next)
fig, (ax1, ax2) = plt.subplots(ncols=2, sharex=True, sharey=True)
ax1.contour(X1,
X2,
Y,
levels=np.logspace(-2, 2, 13),
norm=LogNorm(),
alpha=0.6,
cmap="turbo")
ax1.scatter(*X[z].T, marker='x', alpha=0.9)
ax1.scatter(*X[~z].T, marker='x', alpha=0.9)
# ax1.scatter(*X.T, c=z, marker='x', alpha=0.9)
ax2.contour(
X1,
X2,
model.predict(np.dstack((X1, X2))).squeeze(axis=-1),
# levels=np.logspace(-2, 2, 13), norm=LogNorm(),
alpha=0.6,
cmap="coolwarm")
ax2.scatter(x_next[0], x_next[1], marker='x', alpha=0.9)
ax2.scatter(*X_batch.T, marker='x', alpha=0.9)
plt.tight_layout()
for ext in extension:
fig.savefig(output_path.joinpath(
f"frame_{i:02d}_{context}_{suffix}.{ext}"),
dpi=dpi,
transparent=transparent)
plt.clf()
# fig, ax = plt.subplots()
# # ax.set_title(f"Iteration {i+1:d}")
# ax.plot(X_grid, y_grid, color="tab:gray", label="latent function",
# zorder=-1)
# ax.scatter(X[z], y[z], marker='x', alpha=0.9,
# label=r'observations $y < \tau$', zorder=2)
# ax.scatter(X[~z], y[~z], marker='x', alpha=0.9,
# label=r'observations $y \geq \tau$', zorder=2)
# ax.axhline(tau, xmin=0.0, xmax=1.0, color='k',
# linewidth=1.0, linestyle='dashed', zorder=5)
# ax.annotate(rf"$\tau={{{tau:.2f}}}$", xy=(X_grid.max(), tau),
# xycoords='data', xytext=(10, 2), textcoords='offset points',
# # arrowprops=dict(facecolor='black', shrink=0.05),
# # bbox=dict(boxstyle="round"),
# fontsize="xx-small", horizontalalignment='right', verticalalignment='bottom')
# if i < num_iterations - 1:
# ax.axvline(x_next, ymin=0.0, ymax=1.0, color=next_loc_color,
# alpha=0.8, linewidth=1.0, label="next location",
# zorder=10)
# ax.set_ylabel(r"$y$")
# if not i:
# ax.legend(loc="upper left")
# divider = make_axes_locatable(ax)
# ax_top = divider.append_axes("bottom", size=0.6, pad=0.1, sharex=ax)
# ax_top.scatter(X[z], np.ones_like(X[z]), marker='s',
# edgecolors="none", alpha=0.7, zorder=2)
# ax_top.scatter(X[~z], np.zeros_like(X[~z]), marker='s',
# edgecolors="none", alpha=0.7, zorder=2)
# ax_top.plot(X_grid, tf.sigmoid(model.predict(X_grid)), c='tab:gray',
# label=r"$\pi_{\theta}(x)$", zorder=-1)
# if i < num_iterations - 1:
# ax_top.axvline(x_next, ymin=0.0, ymax=1.0, color=next_loc_color,
# alpha=0.8, linewidth=1.0, zorder=10)
# ax_top.legend(loc="upper left")
# ax_top.set_ylabel(r"$z$")
# ax_top.set_xlabel(r"$x$")
# if prev is not None:
# ax.scatter(*prev, marker='X', s=100, alpha=0.3,
# color="tab:gray", edgecolors="none", zorder=1)
# # TODO(LT): plot border for point in ax_top as well.
# ax_right = divider.append_axes("right", size=0.6, pad=0.1, sharey=ax)
# sns.ecdfplot(y=y, c='tab:gray', ax=ax_right, zorder=-1)
# ax_right.scatter(gamma, tau, c='k', marker='.', zorder=5)
# ax_right.hlines(tau, xmin=0, xmax=gamma, colors='k', linestyles='dashed', linewidth=1.0, zorder=5)
# ax_right.vlines(gamma, ymin=1.1*y_grid.min(), ymax=tau, colors='k', linestyles='dashed', linewidth=1.0, zorder=5)
# ax_right.annotate(rf"$\gamma={{{gamma:.2f}}}$",
# xy=(gamma, y_grid.min()),
# xycoords='data', xytext=(-2, -5),
# textcoords='offset points',
# fontsize="xx-small",
# # arrowprops=dict(facecolor='black', shrink=0.05),
# # bbox=dict(boxstyle="round", fc="none"),
# horizontalalignment='left', verticalalignment='top')
# ax_right.set_xlabel(r'$\Phi(y)$')
# ax_right.yaxis.set_tick_params(labelleft=False)
# ax_right.set_ylim(1.1*y_grid.min(), 1.1*y_grid.max())
# plt.tight_layout()
# for ext in extension:
# fig.savefig(output_path.joinpath(f"frame_{i:02d}_{context}_{suffix}.{ext}"),
# dpi=dpi, transparent=transparent)
# plt.show()
# prev = x_next, y_next
return 0
def main(benchmark_name, dataset_name, dimensions, method_name, num_runs,
run_start, num_iterations, acquisition_name,
acquisition_optimizer_name, num_random_init, batch_size, use_ard,
use_input_warping, standardize_targets, input_dir, output_dir):
benchmark = make_benchmark(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name,
input_dir=input_dir)
name = make_name(benchmark_name,
dimensions=dimensions,
dataset_name=dataset_name)
output_path = Path(output_dir).joinpath(name, method_name)
output_path.mkdir(parents=True, exist_ok=True)
options = dict(batch_size=batch_size,
acquisition_name=acquisition_name,
acquisition_optimizer_name=acquisition_optimizer_name,
use_ard=use_ard,
use_input_warping=use_input_warping,
standardize_targets=standardize_targets)
with output_path.joinpath("options.yaml").open('w') as f:
yaml.dump(options, f)
space = benchmark.get_domain()
config_space = benchmark.get_config_space()
def func(array, *args, **kwargs):
kws = dict_from_array(array, cs=config_space)
return benchmark.evaluate(kws).value
model_type = "input_warped_GP" if use_input_warping else "GP"
initial_design_numdata = ceil_divide(num_random_init,
batch_size) * batch_size
for run_id in trange(run_start, num_runs, unit="run"):
BO = GPyOpt.methods.BayesianOptimization(
f=func,
domain=space,
initial_design_numdata=initial_design_numdata,
model_type=model_type,
ARD=use_ard,
normalize_Y=standardize_targets,
exact_feval=False,
acquisition_type=acquisition_name,
acquisition_optimizer_type=acquisition_optimizer_name,
batch_size=batch_size,
evaluator_type="local_penalization")
BO.run_optimization(
num_iterations,
# evaluations_file="bar/evaluations.csv",
# models_file="bar/models.csv",
verbosity=True)
cost_decision_arr = np.hstack(BO.cost_decision)
cost_eval_arr = np.hstack(BO.cost_eval)
data = pd.DataFrame(data=BO.X, columns=[d["name"] for d in space]) \
.assign(loss=BO.Y, batch=lambda row: row.index // batch_size,
cost_decision=lambda row: cost_decision_arr[row.batch],
cost_eval=cost_eval_arr)
data.to_csv(output_path.joinpath(f"{run_id:03d}.csv"))
return 0