def sherpaopt():
train, targ, test = data_prep()
sigb, sigw, layers = 0.35204672, 2.1220488, 87
gp = DNNGP(train,targ[1][:-targ[2]],test,sigb,sigw,layers)
t0 = time.time()
parameters = [sherpa.Discrete(name='layers',range=[2,100]),
sherpa.Continuous(name='bias',range=[0,5]),
sherpa.Continuous(name='weight',range=[.1,2.09])]
bayesopt = sherpa.algorithms.PopulationBasedTraining(4)
stop = sherpa.algorithms.MedianStoppingRule(0,1)
study = sherpa.Study(parameters=parameters,
algorithm = bayesopt,
stopping_rule=stop,
lower_is_better=True,
disable_dashboard=True)
train = study.get_suggestion()
for trial in study:
print('still going: ',trial.id)
for iteration in range(1):
error = GPerror(gp,targ,trial.parameters["bias"],
trial.parameters["weight"],
trial.parameters["layers"])
study.add_observation(trial=trial,
iteration=iteration,
objective=error)
study.finalize(trial)
if(trial.id == 100): #set to around ~200
break
print(study.get_best_result())
print("Time Optimizing: ", (time.time() - t0)/60, " minutes")
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [sherpa.Continuous('learning_rate', [1e-5, 5e-1], 'log'),
sherpa.Continuous('decay', [1e-8, 1e-2], 'log'),
sherpa.Continuous('momentum', [0., 0.99]),
sherpa.Continuous('dropout', [0.0001, 0.7]),
sherpa.Ordinal('batch_size', [32, 64, 128, 256])]
algorithm = bayesian_optimization.GPyOpt(max_concurrent=FLAGS.concurrent,
model_type='GP',
acquisition_type='EI',
max_num_trials=100)
# Run on local machine.
scheduler = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=algorithm,
dashboard_port=FLAGS.port,
lower_is_better=False,
command='python fashion_mlp.py',
scheduler=scheduler,
verbose=0,
max_concurrent=FLAGS.concurrent,
output_dir='./output_gpyopt_{}'.format(
time.strftime("%Y-%m-%d--%H-%M-%S")))
def test_branin():
def branin(x1, x2):
# Global minimum 0.397887 at (-pi, 12.275), (pi, 2.275),
# and (9.42478, 2.475)
a = 1
b = 5.1/(4*math.pi**2)
c = 5/math.pi
r = 6
s = 10
t = 1/(8*math.pi)
return a*(x2 - b*x1**2 + c*x1 - r)**2 + s*(1-t)*math.cos(x1)+s
parameters = [sherpa.Continuous('x1', [-5., 10.]),
sherpa.Continuous('x2', [0., 15.])]
bayesian_optimization = BayesianOptimization(num_grid_points=2, max_num_trials=50, fine_tune=True)
study = sherpa.Study(algorithm=bayesian_optimization,
parameters=parameters,
lower_is_better=True,
disable_dashboard=True)
for trial in study:
print("Trial {}:\t{}".format(trial.id, trial.parameters))
fval = branin(trial.parameters['x1'], trial.parameters['x2'])
print("Branin-Hoo: {}".format(fval))
study.add_observation(trial=trial,
iteration=1,
objective=fval)
study.finalize(trial, status='COMPLETED')
rval = study.get_best_result()
print(study.get_best_result())
assert np.isclose(rval['Objective'], 0.397887, rtol=1e-3)
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [
sherpa.Continuous('learning_rate', [1e-5, 5e-1], 'log'),
sherpa.Continuous('decay', [1e-8, 1e-2], 'log'),
sherpa.Continuous('momentum', [0., 0.99]),
sherpa.Continuous('dropout', [0.0001, 0.7]),
sherpa.Ordinal('batch_size', [32, 64, 128, 256])
]
algorithm = alg = sherpa.algorithms.PopulationBasedTraining(
num_generations=26,
population_size=100,
parameter_range={
'learning_rate': [1e-10, 9e-1],
'decay': [1e-10, 9e-1]
},
perturbation_factors=(0.8, 1.2))
# Run on local machine.
scheduler = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=algorithm,
dashboard_port=FLAGS.port,
lower_is_better=False,
command='python fashion_mlp.py',
scheduler=scheduler,
verbose=0,
max_concurrent=FLAGS.concurrent,
output_dir='./output_pbt_{}'.format(
time.strftime("%Y-%m-%d--%H-%M-%S")))
def test_convex():
def convex(x1, x2):
# Global minimum is at x1=3., x2=5.
return (x1 - 3.)**2 + (x2 - 5.)**2 + 0.1
parameters = [
sherpa.Continuous('x1', [-5., 10.]),
sherpa.Continuous('x2', [0., 15.])
]
bayesian_optimization = BayesianOptimization(num_grid_points=2,
max_num_trials=50,
fine_tune=True)
study = sherpa.Study(algorithm=bayesian_optimization,
parameters=parameters,
lower_is_better=True,
disable_dashboard=True)
for trial in study:
print("Trial {}:\t{}".format(trial.id, trial.parameters))
fval = convex(trial.parameters['x1'], trial.parameters['x2'])
print("Function Value: {}".format(fval))
study.add_observation(trial=trial, iteration=1, objective=fval)
study.finalize(trial, status='COMPLETED')
rval = study.get_best_result()
print(rval)
assert np.isclose(rval['Objective'], 0.1, rtol=1e-3)
def parameters():
parameters = [
sherpa.Continuous('dropout', [0., 0.5]),
sherpa.Continuous('lr', [1e-7, 1e-1], 'log'),
sherpa.Choice('activation', ['relu', 'tanh', 'sigmoid']),
sherpa.Discrete('num_hidden', [100, 300])
]
return parameters
def build_sherpa_augmentations_space():
params = [
sherpa.Continuous(name='width_shift', range=[0.0, 0.2]),
sherpa.Continuous(name='height_shift', range=[0.0, 0.2]),
sherpa.Continuous(name='zoom', range=[0.0, 0.3]),
sherpa.Choice(name='horizontal_flip', range=[False, True]),
sherpa.Discrete(name='rotation', range=[0, 30])
]
return params
def build_sherpa_parameter_space():
params = [
sherpa.Ordinal(name='depth', range=[2,3]),
sherpa.Discrete(name='dense_neurons', range=[100, 164]),
sherpa.Discrete(name='init_filters', range=[8,32]),
sherpa.Choice(name='use_batchnorm', range=[False, True]),
sherpa.Continuous(name='dropout', range=[0.35, 0.55]),
sherpa.Ordinal(name='batch_size', range=[512, 1024]),
sherpa.Continuous(name='learning_rate', range=[0.005, 0.01]),
sherpa.Continuous(name='beta1', range=[0.45, 0.55]),
sherpa.Continuous(name='beta2', range=[0.95, 1.0])
]
return params
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [
sherpa.Continuous('lrinit', [0.001, 0.1], 'log'),
sherpa.Continuous('momentum', [0., 0.99]),
sherpa.Continuous('lrdecay', [1e-7, 1e-2], 'log'),
sherpa.Continuous('dropout', [0., 0.5])
]
if FLAGS.algorithm == 'BayesianOptimization':
print('Running GPyOpt')
alg = bayesian_optimization.GPyOpt(max_concurrent=FLAGS.max_concurrent,
model_type='GP_MCMC',
acquisition_type='EI_MCMC',
max_num_trials=150)
elif FLAGS.algorithm == 'LocalSearch':
print('Running Local Search')
alg = sherpa.algorithms.LocalSearch(seed_configuration={
'lrinit': 0.038,
'momentum': 0.92,
'lrdecay': 0.0001,
'dropout': 0.
},
perturbation_factors=(0.9, 1.1))
else:
print('Running Random Search')
alg = sherpa.algorithms.RandomSearch(max_num_trials=150)
if FLAGS.sge:
assert FLAGS.env, "For SGE use, you need to set an environment path."
# Submit to SGE queue.
env = FLAGS.env # Script specifying environment variables.
opt = '-N MNISTExample -P {} -q {} -l {}'.format(
FLAGS.P, FLAGS.q, FLAGS.l)
sched = SGEScheduler(environment=env, submit_options=opt)
else:
# Run on local machine.
sched = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=alg,
lower_is_better=True,
filename='trial.py',
output_dir='output_{}'.format(FLAGS.studyname),
scheduler=sched,
max_concurrent=FLAGS.max_concurrent)
print('Best results:')
print(rval)
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [
sherpa.Continuous('log10_learning_rate', [-5, 0]),
sherpa.Continuous('log2_batch_size', [5, 8]),
sherpa.Continuous('log2_n_steps', [4, 11]),
sherpa.Continuous('x_n_opt_epochs', [0, 7]),
sherpa.Continuous('log10_entcoeff', [-8, -1]),
sherpa.Continuous('x_gamma', [-4, -1]),
sherpa.Continuous('cliprange', [0.1, 0.4]),
sherpa.Continuous('lam', [0.8, 1.0])
]
alg = sherpa.algorithms.RandomSearch(max_num_trials=300)
alg = sherpa.algorithms.Repeat(alg, 25, agg=True)
# Run on local machine.
sched = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=alg,
lower_is_better=False,
filename='trial.py',
scheduler=sched,
verbose=0,
max_concurrent=FLAGS.max_concurrent,
disable_dashboard=True,
output_dir='./output_{}'.format(
time.strftime("%Y-%m-%d--%H-%M-%S")))
print('Best results:')
print(rval)
def test_transformers():
parameter = sherpa.Choice('choice', ['a', 'b', 'c', 'd'])
transformer = BayesianOptimization.ChoiceTransformer(parameter)
assert np.all(
transformer.transform(['d', 'c', 'b', 'a']) == np.flip(np.eye(4),
axis=0))
assert all(
transformer.reverse(transformer.transform(['d', 'c', 'b', 'a'])) ==
np.array(['d', 'c', 'b', 'a']))
parameter = sherpa.Continuous('continuous', [0., 0.4])
transformer = BayesianOptimization.ContinuousTransformer(parameter)
assert np.all(
transformer.transform([0.2, 0.4, 0.]) == np.array([0.5, 1.0, 0.0]))
assert np.all(
transformer.reverse(transformer.transform([0.2, 0.4, 0.])) == np.array(
[0.2, 0.4, 0.]))
parameter = sherpa.Continuous('continuous-log', [0.00001, 0.1], 'log')
transformer = BayesianOptimization.ContinuousTransformer(parameter)
print(transformer.transform([0.01]))
assert np.all(
transformer.transform([0.0001, 0.001, 0.01]) == np.array(
[0.25, 0.5, 0.75]))
print(transformer.reverse(transformer.transform([0.0001, 0.001, 0.01])))
assert np.all(
transformer.reverse(transformer.transform([0.0001, 0.001, 0.01])) ==
np.array([0.0001, 0.001, 0.01]))
parameter = sherpa.Discrete('discrete', [0, 12])
transformer = BayesianOptimization.DiscreteTransformer(parameter)
assert np.all(
transformer.transform([3, 6, 9]) == np.array([0.25, 0.5, 0.75]))
assert np.all(
transformer.reverse(transformer.transform([3, 6, 9])) == np.array(
[3, 6, 9]))
assert np.all(transformer.reverse([0.2, 0.3, 0.4]) == np.array([2, 4, 5]))
parameter = sherpa.Discrete('discrete-log', [10, 100000], 'log')
transformer = BayesianOptimization.DiscreteTransformer(parameter)
assert np.all(
transformer.transform([10, 100, 1000, 10000, 100000]) == np.array(
[0., 0.25, 0.5, 0.75, 1.]))
assert np.all(
transformer.reverse(
transformer.transform([10, 100, 1000, 10000, 100000])) == np.array(
[10, 100, 1000, 10000, 100000]))
def test_random_search():
parameters = [
sherpa.Continuous('a', [0, 1]),
sherpa.Choice('b', ['x', 'y', 'z'])
]
rs = sherpa.algorithms.RandomSearch(max_num_trials=10, repeat=10)
config_repeat = {}
for i in range(10):
config = rs.get_suggestion(parameters=parameters)
assert config != config_repeat
for j in range(9):
config_repeat = rs.get_suggestion(parameters=parameters)
assert config == config_repeat
assert rs.get_suggestion(parameters=parameters) is None
rs = sherpa.algorithms.RandomSearch(max_num_trials=10, repeat=1)
last_config = {}
for i in range(10):
config = rs.get_suggestion(parameters=parameters)
assert config != last_config
last_config = config
assert rs.get_suggestion(parameters=parameters) is None
rs = sherpa.algorithms.RandomSearch()
last_config = {}
for _ in range(1000):
config = rs.get_suggestion(parameters=parameters)
assert config != last_config
last_config = config
def test_user_code_fails(test_dir):
tempdir = test_dir
parameters = [
sherpa.Choice(name="param_a", range=[1, 2, 3]),
sherpa.Continuous(name="param_b", range=[0, 1])
]
algorithm = sherpa.algorithms.RandomSearch(max_num_trials=3)
db_port = 27000
scheduler = sherpa.schedulers.LocalScheduler()
filename = os.path.join(tempdir, "test.py")
with open(filename, 'w') as f:
f.write(testscript2)
with pytest.warns(RuntimeWarning):
results = sherpa.optimize(filename=filename,
lower_is_better=True,
algorithm=algorithm,
parameters=parameters,
output_dir=tempdir,
scheduler=scheduler,
max_concurrent=1,
db_port=db_port)
def test_1d():
def obj_func(x):
# Global maximum of 4 is at x=4
return 4. * np.exp(-(x - 4.) ** 2 / 10.) * np.cos(1.5 * (x - 4.)) ** 2
parameters = [sherpa.Continuous('x1', [0., 7.])]
bayesian_optimization = BayesianOptimization(num_grid_points=5,
max_num_trials=50,
fine_tune=False)
study = sherpa.Study(algorithm=bayesian_optimization,
parameters=parameters,
lower_is_better=False,
disable_dashboard=True)
for trial in study:
print("Trial {}:\t{}".format(trial.id, trial.parameters))
fval = obj_func(trial.parameters['x1'])
print("Function Value: {}".format(fval))
study.add_observation(trial=trial,
iteration=1,
objective=fval)
study.finalize(trial, status='COMPLETED')
rval = study.get_best_result()
assert np.isclose(rval['Objective'], 4.)
def test_local_search():
parameters = [
sherpa.Continuous('cont', [0, 1]),
sherpa.Ordinal('ord', [1, 2, 3])
]
seed = {'cont': 0.5, 'ord': 2}
alg = sherpa.algorithms.LocalSearch(seed_configuration=seed)
study = sherpa.Study(parameters=parameters,
algorithm=alg,
lower_is_better=True,
disable_dashboard=True)
def mock_objective(p):
return p['cont'] / p['ord']
# Initial suggestion.
t = study.get_suggestion()
tlist = [t]
tbest = t
assert t.parameters == seed
study.add_observation(t,
objective=mock_objective(t.parameters),
iteration=1)
study.finalize(t)
# Perform a suggestion.
t = study.get_suggestion()
tlist.append(t)
if mock_objective(t.parameters) < mock_objective(tbest.parameters):
tbest = t
study.add_observation(t,
objective=mock_objective(t.parameters),
iteration=1)
study.finalize(t)
if t.parameters['ord'] == 2:
assert t.parameters['cont'] != 0.5
assert abs(t.parameters['cont'] - 0.5) < 0.2
else:
assert t.parameters['cont'] == 0.5
t.parameters['ord'] in [1, 3]
# Do more iterations.
for i in range(50):
t = study.get_suggestion()
#print(t.parameters)
assert t.parameters['ord'] in [1, 2, 3]
assert t.parameters['cont'] >= 0.0
assert t.parameters['cont'] <= 1.0
# All new suggestions should be based on tbest.
assert t.parameters['ord'] == tbest.parameters['ord'] \
or t.parameters['cont'] == tbest.parameters['cont']
tlist.append(t)
if mock_objective(t.parameters) < mock_objective(tbest.parameters):
tbest = t
study.add_observation(t,
objective=mock_objective(t.parameters),
iteration=1)
study.finalize(t)
def test_results_aggregation():
parameters = [sherpa.Continuous('myparam', [0, 1])]
class MyAlg(Algorithm):
def get_suggestion(self, parameters, results, lower_is_better):
if results is not None and len(results) > 0:
print(results)
assert 'ObjectiveStdErr' in results.columns
assert (results.loc[:, 'Objective'] == 0.).all()
exp_std_err = numpy.sqrt(numpy.var([-1,0,1],ddof=1)/(3-1))
assert (numpy.isclose(results.loc[:, 'ObjectiveStdErr'], exp_std_err)).all()
return {'myparam': numpy.random.random()}
alg = MyAlg()
gs = SequentialTesting(algorithm=alg,
K=10,
n=(3, 6, 9),
P=0.5)
study = sherpa.Study(algorithm=gs,
parameters=parameters,
lower_is_better=True,
disable_dashboard=True)
for trial in study:
study.add_observation(trial,
iteration=1,
objective=trial.id%3-1)
study.finalize(trial)
print(study.results)
def test_bayes_opt():
def f(x, sd=1):
y = (x - 3) ** 2 + 10.
if sd == 0:
return y
else:
return y + numpy.random.normal(loc=0., scale=sd,
size=numpy.array(x).shape)
parameters = [sherpa.Continuous('x', [1, 6])]
alg = GPyOpt(max_num_trials=10)
gs = SequentialTesting(algorithm=alg,
K=10,
n=(3, 6, 9),
P=0.5)
study = sherpa.Study(algorithm=gs,
parameters=parameters,
lower_is_better=True,
disable_dashboard=True)
for trial in study:
study.add_observation(trial,
iteration=1,
objective=f(trial.parameters['x']))
study.finalize(trial)
print(study.results)
def test_wait():
parameters = [sherpa.Continuous('myparam', [0, 1])]
rs = sherpa.algorithms.RandomSearch()
gs = SequentialTesting(algorithm=rs,
K=10,
n=(3, 6, 9),
P=0.5)
study = sherpa.Study(algorithm=gs,
parameters=parameters,
lower_is_better=True,
disable_dashboard=True)
for _ in range(10*3 - 1):
trial = study.get_suggestion()
print(trial.id, trial.parameters, "{}/{}".format(gs.k, gs.K[gs.t]),
"{}/{}".format(gs.t, gs.T))
study.add_observation(trial,
iteration=1,
objective=trial.parameters['myparam'] + numpy.random.normal(
scale=0.01))
study.finalize(trial)
trial = study.get_suggestion()
assert trial.parameters['stage'] == 1
waittrial = study.get_suggestion()
assert waittrial == 'WAIT'
study.add_observation(trial,
iteration=1,
objective=trial.parameters['myparam'] + numpy.random.normal(
scale=0.01))
study.finalize(trial)
trial = study.get_suggestion()
assert trial.parameters['stage'] == 2
def test_parallel():
tempdir = tempfile.mkdtemp(dir=".")
parameters = [sherpa.Continuous('myparam', [0, 1])]
rs = sherpa.algorithms.RandomSearch()
gs = SequentialTesting(algorithm=rs,
K=3,
n=(3, 6, 9),
P=0.5,
verbose=True)
scheduler = sherpa.schedulers.LocalScheduler()
filename = os.path.join(tempdir, "test.py")
with open(filename, 'w') as f:
f.write(testscript)
try:
results = sherpa.optimize(parameters=parameters,
algorithm=gs,
lower_is_better=True,
command="python {}".format(filename),
output_dir=tempdir,
scheduler=scheduler,
max_concurrent=2,
verbose=1,
disable_dashboard=True)
finally:
shutil.rmtree(tempdir)
def get_study():
parameters = [
sherpa.Ordinal("conv3d_num_filters", [16, 32, 64, 128]),
sherpa.Ordinal("conv3d_kernel_size", [(3, 5, 5), (5, 5, 5), (5, 7, 7)]),
sherpa.Discrete("encoder_rnn_num_layers", [1, 3]),
sherpa.Continuous("encoder_rnn_dropout", [0.0, 0.3]),
sherpa.Continuous("lr", [2e-4, 4e-3], scale="log"),
]
algorithm = sherpa.algorithms.RandomSearch(max_num_trials=16)
stopping_rule = sherpa.algorithms.MedianStoppingRule(min_iterations=8, min_trials=4)
return sherpa.Study(
parameters=parameters,
algorithm=algorithm,
lower_is_better=True,
stopping_rule=stopping_rule,
)
def test_1d():
def obj_func(x):
# Global maximum of 4 is at x=4
return 4. * numpy.exp(-(x - 4.)**2 / 10.) * numpy.cos(1.5 *
(x - 4.))**2
parameters = [sherpa.Continuous('x1', [0., 7.])]
bayesian_optimization = GPyOpt(max_concurrent=1,
max_num_trials=50,
model_type='GP',
acquisition_type='EI')
study = sherpa.Study(algorithm=bayesian_optimization,
parameters=parameters,
lower_is_better=False,
disable_dashboard=True)
for trial in study:
print("Trial {}:\t{}".format(trial.id, trial.parameters))
fval = obj_func(trial.parameters['x1'])
print("Function Value: {}".format(fval))
study.add_observation(trial=trial, iteration=1, objective=fval)
study.finalize(trial, status='COMPLETED')
rval = study.get_best_result()
print(rval)
assert numpy.isclose(rval['Objective'], 4., atol=0.2)
def test_optimize_mix():
# Test a mixture of these
parameters = [
sherpa.Continuous('continuous', [
0.,
1,
]),
sherpa.Choice('choice', [1, 2, 3, 4]),
sherpa.Choice('choice2', [1, 2, 3]),
sherpa.Discrete('discrete', [0, 100])
]
# Function should have maximum: [0.5, 0, 0, 0, 1, 0, 0, 1, 0.5]
# Maximum should be 7
def fun(x):
cont = -1. * (x[0] - 0.5)**2
ch = np.dot(x[1:5], np.array([1, 2, 3, 4]))
ch2 = np.dot(x[5:8], np.array([1, 2, 3]))
discr = -1. * (x[-1] - 0.5)**2
return cont + ch + ch2 + discr
bayesian_optimization = BayesianOptimization()
bayesian_optimization.num_candidates = 100
candidates = bayesian_optimization._generate_candidates(parameters)
X = bayesian_optimization._to_design(candidates, parameters)
Xoptimized, fun_values = bayesian_optimization._maximize(X, fun)
# print(Xoptimized)
# print(fun_values)
print(Xoptimized[fun_values.argmax()])
print(fun_values.max())
assert np.all(
np.isclose(Xoptimized[fun_values.argmax()],
np.array([0.5, 0., 0., 0., 1., 0., 0., 1., 0.5])))
def test_repeat_grid_search():
parameters = [
sherpa.Choice('a', [1, 2]),
sherpa.Choice('b', ['a', 'b']),
sherpa.Continuous('c', [1, 4])
]
alg = sherpa.algorithms.GridSearch()
alg = sherpa.algorithms.Repeat(algorithm=alg, num_times=3)
suggestion = alg.get_suggestion(parameters)
seen = list()
while suggestion:
seen.append((suggestion['a'], suggestion['b'], suggestion['c']))
suggestion = alg.get_suggestion(parameters)
expected_params = [(1, 'a', 2.0), (1, 'a', 3.0), (1, 'b', 2.0),
(1, 'b', 3.0), (2, 'a', 2.0), (2, 'a', 3.0),
(2, 'b', 2.0), (2, 'b', 3.0)]
expected = list(
itertools.chain.from_iterable(
itertools.repeat(x, 3) for x in expected_params))
print(sorted(expected))
print(sorted(seen))
assert sorted(expected) == sorted(seen)
def test_noisy_parabola():
def f(x, sd=1):
y = (x - 3)**2 + 10.
if sd == 0:
return y
else:
return y + numpy.random.normal(
loc=0., scale=sd, size=numpy.array(x).shape)
parameters = [sherpa.Continuous('x1', [0., 7.])]
bayesian_optimization = GPyOpt(max_concurrent=1,
max_num_trials=5,
model_type='GP',
acquisition_type='EI')
rep = Repeat(algorithm=bayesian_optimization, num_times=3, agg=True)
study = sherpa.Study(algorithm=rep,
parameters=parameters,
lower_is_better=True,
disable_dashboard=True)
for trial in study:
# print("Trial {}:\t{}".format(trial.id, trial.parameters))
fval = f(trial.parameters['x1'], sd=1)
# print("Function Value: {}".format(fval))
study.add_observation(trial=trial, iteration=1, objective=fval)
study.finalize(trial, status='COMPLETED')
# rval = study.get_best_result()
# print(rval)
print(study.results.query("Status=='COMPLETED'"))
def test_optimize():
# Test Continuous
parameters = [sherpa.Continuous('continuous', [
0.,
1,
])]
bayesian_optimization = BayesianOptimization()
bayesian_optimization.num_candidates = 100
candidates = bayesian_optimization._generate_candidates(parameters)
X = bayesian_optimization._to_design(candidates, parameters)
fun = lambda x: -1 * (x - 0.5)**2
Xoptimized, fun_values = bayesian_optimization._maximize(X, fun)
assert np.isclose(Xoptimized[fun_values.argmax()][0], 0.5)
# Test Discrete
parameters = [sherpa.Discrete('discrete', [0, 100])]
bayesian_optimization = BayesianOptimization()
bayesian_optimization.num_candidates = 100
candidates = bayesian_optimization._generate_candidates(parameters)
X = bayesian_optimization._to_design(candidates, parameters)
fun = lambda x: -1. * (x - 0.5)**2
Xoptimized, fun_values = bayesian_optimization._maximize(X, fun)
assert np.isclose(Xoptimized[fun_values.argmax()][0], 0.5)
def test_overall_larger_is_better():
parameters = [sherpa.Continuous('myparam', [0, 10]),
sherpa.Discrete('myparam2', [0, 10])]
rs = sherpa.algorithms.RandomSearch()
gs = SequentialTesting(algorithm=rs,
K=10,
n=(3, 6, 9),
P=0.5)
study = sherpa.Study(algorithm=gs,
parameters=parameters,
lower_is_better=False,
disable_dashboard=True)
for trial in study:
print(trial.id, trial.parameters, "{}/{}".format(gs.k, gs.K[gs.t]),
"{}/{}".format(gs.t, gs.T))
study.add_observation(trial,
iteration=1,
objective=trial.parameters[
'myparam'] + numpy.random.normal(
scale=1.))
study.finalize(trial)
completed = study.results.query("Status == 'COMPLETED'")
assert completed.myparam.max() in completed[completed.stage == 2].myparam.unique()
def test_design():
parameters = [
sherpa.Choice('choice', ['a', 'b', 'c', 'd']),
sherpa.Continuous('continuous', [0., 0.4]),
sherpa.Discrete('discrete', [0, 12])
]
bayesian_optimization = BayesianOptimization()
bayesian_optimization.num_candidates = 100
candidates = bayesian_optimization._generate_candidates(parameters)
assert len(candidates) == bayesian_optimization.num_candidates
assert len(candidates.columns) == len(parameters)
X = bayesian_optimization._to_design(candidates, parameters)
assert X.shape == (bayesian_optimization.num_candidates, 6)
for j in range(X.shape[1]):
assert (X[:, j] >= 0.).all() and (X[:, j] <= 1.).all()
df = bayesian_optimization._from_design(X)
pd.testing.assert_frame_equal(df, candidates)
row = bayesian_optimization._from_design(X[0])
row_dict = row.iloc[0].to_dict()
candidates_dict = candidates.iloc[0].to_dict()
assert row_dict['choice'] == candidates_dict['choice']
assert row_dict['discrete'] == candidates_dict['discrete']
assert np.isclose(row_dict['continuous'], candidates_dict['continuous'])
def test_repeat_results_aggregation():
parameters = [sherpa.Continuous('myparam', [0, 1])]
class MyAlg(sherpa.algorithms.Algorithm):
allows_repetition = True
def get_suggestion(self, parameters, results, lower_is_better):
if results is not None and len(results) > 0:
print(results)
assert 'ObjectiveStdErr' in results.columns
assert 'ObjectiveVar' in results.columns
assert (results.loc[:, 'Objective'] == 0.).all()
exp_std_err = numpy.sqrt(numpy.var([-1,0,1],ddof=1)/(3-1))
assert (numpy.isclose(results.loc[:, 'ObjectiveStdErr'], exp_std_err)).all()
return {'myparam': numpy.random.random()}
alg = MyAlg()
gs = sherpa.algorithms.Repeat(algorithm=alg,
num_times=3,
agg=True)
study = sherpa.Study(algorithm=gs,
parameters=parameters,
lower_is_better=True,
disable_dashboard=True)
for trial in study:
study.add_observation(trial,
iteration=1,
objective=trial.id%3-1) # 1->-1, 2->0, 3->1, 4->-1, ...
study.finalize(trial)
print(study.results)
if trial.id > 10:
break
class TestLocalSearch:
@pytest.mark.parametrize("parameter,seed,expected",
[(sherpa.Ordinal('p', [0, 1, 2, 3, 4]), {'p': 2}, [1, 3]),
(sherpa.Continuous('p', [0, 1]), {'p': 0.5}, [0.5*0.8, 0.5*1.2]),
(sherpa.Discrete('p', [0, 10]), {'p': 5}, [4, 6]),
(sherpa.Choice('p', [0, 1, 2, 3, 4]), {'p': 2}, [0, 1, 3, 4])])
def test_seed_and_first_suggestion(self, parameter, seed, expected):
study = get_local_search_study_lower_is_better([parameter],
seed)
trial = study.get_suggestion()
assert trial.parameters['p'] == seed['p']
study.add_observation(trial, objective=trial.parameters['p'], iteration=1)
study.finalize(trial)
trial = study.get_suggestion()
assert trial.parameters['p'] in expected
@pytest.mark.parametrize("parameter,seed,expected",
[(sherpa.Ordinal('p', [0, 1, 2, 3, 4]), {'p': 2}, [0, 1]),
(sherpa.Continuous('p', [0, 1]), {'p': 0.5}, [0.5*(0.8), 0.5*(0.8)**2]),
(sherpa.Discrete('p', [0, 10]), {'p': 5}, [int(5*(0.8)), int(5*(0.8)**2)]),
(sherpa.Choice('p', [0, 1, 2]), {'p': 2}, [0])])
def test_expected_value_after_three_iterations(self, parameter, seed, expected):
study = get_local_search_study_lower_is_better([parameter],
seed)
for trial in study:
study.add_observation(trial, objective=trial.parameters['p'], iteration=1)
study.finalize(trial)
if trial.id == 3:
break
assert study.get_best_result()['Objective'] in expected
@pytest.mark.parametrize("param1,seed1,param2,seed2", [(sherpa.Ordinal('p1', [0, 1, 2, 3, 4]), {'p1': 2},
sherpa.Continuous('p2', [0, 1]), {'p2': 0.5})])
def test_only_one_parameter_is_perturbed_at_a_time(self, param1, seed1, param2, seed2):
seed = dict(seed1, **seed2)
study = get_local_search_study_lower_is_better([param1, param2],
seed=seed)
trial = study.get_suggestion()
study.add_observation(trial, objective=1, iteration=1)
study.finalize(trial)
trial = study.get_suggestion()
assert not all(
param_value != seed[param_name] for param_name, param_value in
trial.parameters.items())
def test_1d_minimize():
def obj_func(x):
# Global maximum of 4 is at x=4
return -4. * numpy.exp(-(x - 4.)**2 / 10.) * numpy.cos(1.5 *
(x - 4.))**2
parameters = [sherpa.Continuous('x1', [0., 7.])]
bayesian_optimization = GPyOpt(max_concurrent=1,
max_num_trials=12,
model_type='GP',
acquisition_type='EI',
initial_data_points=[{
'x1': 2
}, {
'x1': 5
}],
num_initial_data_points=2)
study = sherpa.Study(algorithm=bayesian_optimization,
parameters=parameters,
lower_is_better=True,
disable_dashboard=True)
for trial in study:
print("Trial {}:\t{}".format(trial.id, trial.parameters))
fval = obj_func(trial.parameters['x1'])
print("Function Value: {}".format(fval))
study.add_observation(trial=trial, iteration=1, objective=fval)
study.finalize(trial, status='COMPLETED')
rval = study.get_best_result()
print(rval)
# bounds = [{'name': 'x', 'type': 'continuous', 'domain': (0, 7)}]
# Xinit = numpy.array([2, 5]).reshape(-1, 1)
# yinit = obj_func(Xinit)
# myBopt = gpyopt_package.methods.BayesianOptimization(f=obj_func,
# # function to optimize
# domain=bounds,
# # box-constraints of the problem
# acquisition_type='EI',
# X=Xinit,
# y=yinit,
# initial_design_numdata=0,
# initial_design_type='random',
# evaluator_type='local_penalization',
# batch_size=1,
# maximize=True,
# exact_feval=False)
# # Run the optimization
# max_iter = 10 # evaluation budget
# max_time = 60 # time budget
# eps = 10e-6 # Minimum allows distance between the las two observations
#
# myBopt.run_optimization(max_iter, max_time, eps)
# print(myBopt.get_evaluations())
assert numpy.isclose(rval['x1'], 4., atol=0.1)
