def __init__(self, pop, log_dir):
parameters = [
# Auto Encoder Parameters
sherpa.Discrete('hidden_units_AE', range=[2, 8]),
sherpa.Discrete('lstm_units_AE', range=[2, 16]),
sherpa.Continuous('dropout_AE', range=[0, 0.5]),
sherpa.Continuous('lr_AE', range=[0.000001, 0.1], scale='log'),
# Detector Parameters
sherpa.Discrete('hidden_units_DET', range=[2, 8]),
sherpa.Discrete('lstm_units_DET', range=[2, 8]),
sherpa.Continuous('dropout_DET', range=[0, 0.5]),
sherpa.Continuous('leaky_alpha_DET', range=[0.01, 0.4]),
sherpa.Continuous('lr_DET', range=[0.000001, 0.1], scale='log'),
# GAN parameters
sherpa.Continuous('lr_GAN', range=[0.000001, 0.1], scale='log')
]
# Set an evolutionary algorithm for parameter search, enforce early stopping
algorithm = sherpa.algorithms.PopulationBasedTraining(
population_size=pop)
rule = sherpa.algorithms.MedianStoppingRule(min_iterations=5,
min_trials=1)
self.study = sherpa.Study(parameters,
algorithm,
lower_is_better=True,
stopping_rule=rule,
dashboard_port=9800)
self.logs_dir = log_dir
def parameters():
parameters = [
sherpa.Continuous('dropout', [0., 0.5]),
sherpa.Continuous('lr', [1e-7, 1e-1], 'log'),
sherpa.Choice('activation', ['relu', 'tanh', 'sigmoid']),
sherpa.Discrete('num_hidden', [100, 300]),
sherpa.Discrete('batch_size', [1, 1000], 'log')
]
return parameters
def build_sherpa_parameter_space():
params = [
sherpa.Ordinal(name='depth', range=[2,3]),
sherpa.Discrete(name='dense_neurons', range=[100, 164]),
sherpa.Discrete(name='init_filters', range=[8,32]),
sherpa.Choice(name='use_batchnorm', range=[False, True]),
sherpa.Continuous(name='dropout', range=[0.35, 0.55]),
sherpa.Ordinal(name='batch_size', range=[512, 1024]),
sherpa.Continuous(name='learning_rate', range=[0.005, 0.01]),
sherpa.Continuous(name='beta1', range=[0.45, 0.55]),
sherpa.Continuous(name='beta2', range=[0.95, 1.0])
]
return params
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
# max_features = trial.parameters('max_features', 20000)
# batch_size = trial.parameters('batch_size', 32)
# hidden_dim = trial.parameters('hidden_dim', 128)
# dropout = trial.parameters('dropout', 0.2)
# recurrent_dropout = trial.parameters('recurrent_dropout', 0.2)
# optimizer = trial.parameters('optimizer', 'adam')
parameters = []
parameters += [sherpa.Discrete('max_features', [15000, 40000])]
parameters += [sherpa.Discrete('batch_size', [10, 150], 'log')]
parameters += [sherpa.Discrete('hidden_dim', [100, 500], 'log')]
parameters += [sherpa.Continuous('dropout_embedding', [0.0001, 0.5])]
parameters += [sherpa.Continuous('dropout_lstm', [0.0001, 0.5])]
parameters += [sherpa.Continuous('embedding_regularizer', [1e-12, 1e-6], 'log')]
parameters += [sherpa.Continuous('kernel_regularizer', [1e-8, 1e-0], 'log')]
parameters += [sherpa.Continuous('recurrent_regularizer', [1e-8, 1e-0], 'log')]
parameters += [sherpa.Continuous('lr', [5e-4, 5e-3], scale='log')]
parameters += [sherpa.Continuous('decay', [1e-5, 1e-10], scale='log')]
parameters += [sherpa.Continuous('rho', [0.5, 0.99])]
print('Running Random Search')
alg = sherpa.algorithms.RandomSearch(max_num_trials=200, repeat=25)
if FLAGS.sge:
assert FLAGS.env, "For SGE use, you need to set an environment path."
# Submit to SGE queue.
env = FLAGS.env # Script specifying environment variables.
opt = '-N LSTMSearch -P {} -q {} -l {} -l gpu=1'.format(FLAGS.P, FLAGS.q, FLAGS.l)
sched = SGEScheduler(environment=env, submit_options=opt)
else:
# Run on local machine.
resources = [int(x) for x in FLAGS.gpus.split(',')]
sched = LocalScheduler(resources=resources)
rval = sherpa.optimize(parameters=parameters,
algorithm=alg,
lower_is_better=False,
filename='trials.py',
scheduler=sched,
verbose=0,
max_concurrent=FLAGS.max_concurrent,
output_dir='./output_imdb_{}_{}_gpu-{}'.format(time.strftime("%Y-%m-%d--%H-%M-%S"), FLAGS.name, FLAGS.gpus.replace(',', '-')))
print('Best results:')
print(rval)
def test_transformers():
parameter = sherpa.Choice('choice', ['a', 'b', 'c', 'd'])
transformer = BayesianOptimization.ChoiceTransformer(parameter)
assert np.all(
transformer.transform(['d', 'c', 'b', 'a']) == np.flip(np.eye(4),
axis=0))
assert all(
transformer.reverse(transformer.transform(['d', 'c', 'b', 'a'])) ==
np.array(['d', 'c', 'b', 'a']))
parameter = sherpa.Continuous('continuous', [0., 0.4])
transformer = BayesianOptimization.ContinuousTransformer(parameter)
assert np.all(
transformer.transform([0.2, 0.4, 0.]) == np.array([0.5, 1.0, 0.0]))
assert np.all(
transformer.reverse(transformer.transform([0.2, 0.4, 0.])) == np.array(
[0.2, 0.4, 0.]))
parameter = sherpa.Continuous('continuous-log', [0.00001, 0.1], 'log')
transformer = BayesianOptimization.ContinuousTransformer(parameter)
print(transformer.transform([0.01]))
assert np.all(
transformer.transform([0.0001, 0.001, 0.01]) == np.array(
[0.25, 0.5, 0.75]))
print(transformer.reverse(transformer.transform([0.0001, 0.001, 0.01])))
assert np.all(
transformer.reverse(transformer.transform([0.0001, 0.001, 0.01])) ==
np.array([0.0001, 0.001, 0.01]))
parameter = sherpa.Discrete('discrete', [0, 12])
transformer = BayesianOptimization.DiscreteTransformer(parameter)
assert np.all(
transformer.transform([3, 6, 9]) == np.array([0.25, 0.5, 0.75]))
assert np.all(
transformer.reverse(transformer.transform([3, 6, 9])) == np.array(
[3, 6, 9]))
assert np.all(transformer.reverse([0.2, 0.3, 0.4]) == np.array([2, 4, 5]))
parameter = sherpa.Discrete('discrete-log', [10, 100000], 'log')
transformer = BayesianOptimization.DiscreteTransformer(parameter)
assert np.all(
transformer.transform([10, 100, 1000, 10000, 100000]) == np.array(
[0., 0.25, 0.5, 0.75, 1.]))
assert np.all(
transformer.reverse(
transformer.transform([10, 100, 1000, 10000, 100000])) == np.array(
[10, 100, 1000, 10000, 100000]))
def test_overall_larger_is_better():
parameters = [sherpa.Continuous('myparam', [0, 10]),
sherpa.Discrete('myparam2', [0, 10])]
rs = sherpa.algorithms.RandomSearch()
gs = SequentialTesting(algorithm=rs,
K=10,
n=(3, 6, 9),
P=0.5)
study = sherpa.Study(algorithm=gs,
parameters=parameters,
lower_is_better=False,
disable_dashboard=True)
for trial in study:
print(trial.id, trial.parameters, "{}/{}".format(gs.k, gs.K[gs.t]),
"{}/{}".format(gs.t, gs.T))
study.add_observation(trial,
iteration=1,
objective=trial.parameters[
'myparam'] + numpy.random.normal(
scale=1.))
study.finalize(trial)
completed = study.results.query("Status == 'COMPLETED'")
assert completed.myparam.max() in completed[completed.stage == 2].myparam.unique()
def test_optimize():
# Test Continuous
parameters = [sherpa.Continuous('continuous', [
0.,
1,
])]
bayesian_optimization = BayesianOptimization()
bayesian_optimization.num_candidates = 100
candidates = bayesian_optimization._generate_candidates(parameters)
X = bayesian_optimization._to_design(candidates, parameters)
fun = lambda x: -1 * (x - 0.5)**2
Xoptimized, fun_values = bayesian_optimization._maximize(X, fun)
assert np.isclose(Xoptimized[fun_values.argmax()][0], 0.5)
# Test Discrete
parameters = [sherpa.Discrete('discrete', [0, 100])]
bayesian_optimization = BayesianOptimization()
bayesian_optimization.num_candidates = 100
candidates = bayesian_optimization._generate_candidates(parameters)
X = bayesian_optimization._to_design(candidates, parameters)
fun = lambda x: -1. * (x - 0.5)**2
Xoptimized, fun_values = bayesian_optimization._maximize(X, fun)
assert np.isclose(Xoptimized[fun_values.argmax()][0], 0.5)
def sherpaopt():
train, targ, test = data_prep()
sigb, sigw, layers = 0.35204672, 2.1220488, 87
gp = DNNGP(train,targ[1][:-targ[2]],test,sigb,sigw,layers)
t0 = time.time()
parameters = [sherpa.Discrete(name='layers',range=[2,100]),
sherpa.Continuous(name='bias',range=[0,5]),
sherpa.Continuous(name='weight',range=[.1,2.09])]
bayesopt = sherpa.algorithms.PopulationBasedTraining(4)
stop = sherpa.algorithms.MedianStoppingRule(0,1)
study = sherpa.Study(parameters=parameters,
algorithm = bayesopt,
stopping_rule=stop,
lower_is_better=True,
disable_dashboard=True)
train = study.get_suggestion()
for trial in study:
print('still going: ',trial.id)
for iteration in range(1):
error = GPerror(gp,targ,trial.parameters["bias"],
trial.parameters["weight"],
trial.parameters["layers"])
study.add_observation(trial=trial,
iteration=iteration,
objective=error)
study.finalize(trial)
if(trial.id == 100): #set to around ~200
break
print(study.get_best_result())
print("Time Optimizing: ", (time.time() - t0)/60, " minutes")
def test_optimize_mix():
# Test a mixture of these
parameters = [
sherpa.Continuous('continuous', [
0.,
1,
]),
sherpa.Choice('choice', [1, 2, 3, 4]),
sherpa.Choice('choice2', [1, 2, 3]),
sherpa.Discrete('discrete', [0, 100])
]
# Function should have maximum: [0.5, 0, 0, 0, 1, 0, 0, 1, 0.5]
# Maximum should be 7
def fun(x):
cont = -1. * (x[0] - 0.5)**2
ch = np.dot(x[1:5], np.array([1, 2, 3, 4]))
ch2 = np.dot(x[5:8], np.array([1, 2, 3]))
discr = -1. * (x[-1] - 0.5)**2
return cont + ch + ch2 + discr
bayesian_optimization = BayesianOptimization()
bayesian_optimization.num_candidates = 100
candidates = bayesian_optimization._generate_candidates(parameters)
X = bayesian_optimization._to_design(candidates, parameters)
Xoptimized, fun_values = bayesian_optimization._maximize(X, fun)
# print(Xoptimized)
# print(fun_values)
print(Xoptimized[fun_values.argmax()])
print(fun_values.max())
assert np.all(
np.isclose(Xoptimized[fun_values.argmax()],
np.array([0.5, 0., 0., 0., 1., 0., 0., 1., 0.5])))
def test_design():
parameters = [
sherpa.Choice('choice', ['a', 'b', 'c', 'd']),
sherpa.Continuous('continuous', [0., 0.4]),
sherpa.Discrete('discrete', [0, 12])
]
bayesian_optimization = BayesianOptimization()
bayesian_optimization.num_candidates = 100
candidates = bayesian_optimization._generate_candidates(parameters)
assert len(candidates) == bayesian_optimization.num_candidates
assert len(candidates.columns) == len(parameters)
X = bayesian_optimization._to_design(candidates, parameters)
assert X.shape == (bayesian_optimization.num_candidates, 6)
for j in range(X.shape[1]):
assert (X[:, j] >= 0.).all() and (X[:, j] <= 1.).all()
df = bayesian_optimization._from_design(X)
pd.testing.assert_frame_equal(df, candidates)
row = bayesian_optimization._from_design(X[0])
row_dict = row.iloc[0].to_dict()
candidates_dict = candidates.iloc[0].to_dict()
assert row_dict['choice'] == candidates_dict['choice']
assert row_dict['discrete'] == candidates_dict['discrete']
assert np.isclose(row_dict['continuous'], candidates_dict['continuous'])
class TestLocalSearch:
@pytest.mark.parametrize("parameter,seed,expected",
[(sherpa.Ordinal('p', [0, 1, 2, 3, 4]), {'p': 2}, [1, 3]),
(sherpa.Continuous('p', [0, 1]), {'p': 0.5}, [0.5*0.8, 0.5*1.2]),
(sherpa.Discrete('p', [0, 10]), {'p': 5}, [4, 6]),
(sherpa.Choice('p', [0, 1, 2, 3, 4]), {'p': 2}, [0, 1, 3, 4])])
def test_seed_and_first_suggestion(self, parameter, seed, expected):
study = get_local_search_study_lower_is_better([parameter],
seed)
trial = study.get_suggestion()
assert trial.parameters['p'] == seed['p']
study.add_observation(trial, objective=trial.parameters['p'], iteration=1)
study.finalize(trial)
trial = study.get_suggestion()
assert trial.parameters['p'] in expected
@pytest.mark.parametrize("parameter,seed,expected",
[(sherpa.Ordinal('p', [0, 1, 2, 3, 4]), {'p': 2}, [0, 1]),
(sherpa.Continuous('p', [0, 1]), {'p': 0.5}, [0.5*(0.8), 0.5*(0.8)**2]),
(sherpa.Discrete('p', [0, 10]), {'p': 5}, [int(5*(0.8)), int(5*(0.8)**2)]),
(sherpa.Choice('p', [0, 1, 2]), {'p': 2}, [0])])
def test_expected_value_after_three_iterations(self, parameter, seed, expected):
study = get_local_search_study_lower_is_better([parameter],
seed)
for trial in study:
study.add_observation(trial, objective=trial.parameters['p'], iteration=1)
study.finalize(trial)
if trial.id == 3:
break
assert study.get_best_result()['Objective'] in expected
@pytest.mark.parametrize("param1,seed1,param2,seed2", [(sherpa.Ordinal('p1', [0, 1, 2, 3, 4]), {'p1': 2},
sherpa.Continuous('p2', [0, 1]), {'p2': 0.5})])
def test_only_one_parameter_is_perturbed_at_a_time(self, param1, seed1, param2, seed2):
seed = dict(seed1, **seed2)
study = get_local_search_study_lower_is_better([param1, param2],
seed=seed)
trial = study.get_suggestion()
study.add_observation(trial, objective=1, iteration=1)
study.finalize(trial)
trial = study.get_suggestion()
assert not all(
param_value != seed[param_name] for param_name, param_value in
trial.parameters.items())
def build_sherpa_augmentations_space():
params = [
sherpa.Continuous(name='width_shift', range=[0.0, 0.2]),
sherpa.Continuous(name='height_shift', range=[0.0, 0.2]),
sherpa.Continuous(name='zoom', range=[0.0, 0.3]),
sherpa.Choice(name='horizontal_flip', range=[False, True]),
sherpa.Discrete(name='rotation', range=[0, 30])
]
return params
def runner():
parameters = [
sherpa.Discrete('num_units', [64, 512]),
sherpa.Discrete('num_layers', [2, 15]),
sherpa.Ordinal('batch_size', [4096]),
sherpa.Ordinal('learning_rate',
[0.0001, 0.0003162278, 0.001, 0.003162278, 0.001]),
sherpa.Continuous('dropout_rate', [0, 0.6]),
sherpa.Continuous('leaky_relu_alpha', [0, 1]),
sherpa.Ordinal('batch_norm', [0, 1]),
]
#alg = sherpa.algorithms.BayesianOptimization(max_num_trials=50)
#alg = sherpa.algorithms.GPyOpt(max_num_trials=50)
alg = sherpa.algorithms.RandomSearch(max_num_trials=40)
#resources = ['/device:GPU:0', '/device:GPU:1', '/device:GPU:2', '/device:GPU:3']
resources = [0, 1, 2, 3]
scheduler = sherpa.schedulers.LocalScheduler(resources=resources)
#study = sherpa.Study(parameters=parameters,
# algorithm=alg,
# lower_is_better=True)
script = '/work/00157/walling/projects/cloud_emulator/walling-CBRAIN-CAM/notebooks/tbeucler_devlog/sherpa/sherpa-trial-phase2-study1-pos.py'
tempdir = '/work/00157/walling/projects/cloud_emulator/walling-CBRAIN-CAM/notebooks/tbeucler_devlog/sherpa/sherpa-temp-phase2-study1-pos'
#import time
#time.sleep(1000)
results = sherpa.optimize(parameters=parameters,
algorithm=alg,
lower_is_better=True,
filename=script,
output_dir=tempdir,
scheduler=scheduler,
max_concurrent=4,
verbose=1,
mongodb_args={'bind_ip':
'0.0.0.0'}) #, 'port':47001})
def get_mock_study():
mock_algorithm = mock.MagicMock()
mock_algorithm.get_suggestion.return_value = {'a': 1, 'b': 2}
mock_stopping_rule = mock.MagicMock()
return sherpa.Study(parameters=[
sherpa.Discrete('a', [1, 2]),
sherpa.Choice('b', [2, 5, 7])
],
algorithm=mock_algorithm,
stopping_rule=mock_stopping_rule,
lower_is_better=True,
disable_dashboard=True)
def get_sherpa_parameter(self, name, good_values_only=False):
if good_values_only:
return sherpa.Choice(name=name, range=self.good_values)
elif self.is_categorical:
return sherpa.Choice(name=name, range=self.categories)
elif self.type == int:
return sherpa.Discrete(name=name,
range=self.range,
scale='log' if self.log_scale else 'linear')
else:
return sherpa.Continuous(
name=name,
range=self.range,
scale='log' if self.log_scale else 'linear')
def get_study():
parameters = [
sherpa.Ordinal("conv3d_num_filters", [16, 32, 64, 128]),
sherpa.Ordinal("conv3d_kernel_size", [(3, 5, 5), (5, 5, 5), (5, 7, 7)]),
sherpa.Discrete("encoder_rnn_num_layers", [1, 3]),
sherpa.Continuous("encoder_rnn_dropout", [0.0, 0.3]),
sherpa.Continuous("lr", [2e-4, 4e-3], scale="log"),
]
algorithm = sherpa.algorithms.RandomSearch(max_num_trials=16)
stopping_rule = sherpa.algorithms.MedianStoppingRule(min_iterations=8, min_trials=4)
return sherpa.Study(
parameters=parameters,
algorithm=algorithm,
lower_is_better=True,
stopping_rule=stopping_rule,
)
def categorical_embbedings_setting(data, categorical_features, max_sz=300):
'''
Sets size of
:param data:
:param categorical_features:
:param max_sz:
:return:
'''
embeddings_sherpa = {}
for feature in categorical_features:
cardinality = data[feature].nunique()
max_emb_sz = min(max_sz, round(1.6 * cardinality**0.56))
min_emb_sz = 1
type_sufix = parameter_class_sufix_mapper('embbeding_size')
embeddings_sherpa[feature] = (sherpa.Discrete(
feature + type_sufix, [min_emb_sz, max_emb_sz]))
return embeddings_sherpa
def hyper_param(self, epochs):
parameters = [
sherpa.Continuous('learning_rate', [1e-4, 1e-2]),
sherpa.Discrete('num_units', [32, 128]),
sherpa.Choice('activation', ['relu', 'adam', 'sigmoid'])
]
algorithm = bayesian_optimization.GPyOpt(max_num_trials=50)
study = sherpa.Study(parameters=parameters,
algorithm=algorithm,
lower_is_better=False)
x_test = self._label / 300
y_test = self._label
# Create model
model = models.Sequential()
model.add(
layers.Embedding(self.totalwords,
64,
input_length=maxSequenceLen - 1))
model.add(layers.LSTM(128))
model.add(layers.Dense(self.totalwords, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
epochs = epochs
for trial in study:
lr = trial.parameters['learning_rate']
num_units = trial.parameters['num_units']
act = trial.parameters['activation']
for i in range(epochs):
model.fit(self.predictors, self._label, batch_size=self.batch)
loss, accuracy = model.evaluate(x_test, y_test)
study.add_observation(trial=trial,
iteration=i,
objective=accuracy,
context={'loss': loss})
if study.should_trial_stop(trial):
break
study.finalize(trial=trial)
print(study.get_best_result())
def test_strip_add_choice():
parameters = [sherpa.Choice('choice', ['a', 'b', 'c', 'd']),
sherpa.Continuous('continuous', [0., 0.4]),
sherpa.Choice('choice2', [1, 2, 3]),
sherpa.Discrete('discrete', [0, 12])]
bayesian_optimization = BayesianOptimization()
bayesian_optimization.num_candidates = 5
candidates = bayesian_optimization._generate_candidates(parameters)
X = bayesian_optimization._to_design(candidates, parameters)
for i, row in enumerate(X):
print(row)
x, args = bayesian_optimization._strip_choice(row)
# print("x: ", x)
# print("args: ", args)
new_row = bayesian_optimization._add_choice(x, *args)
print(new_row)
assert np.all(row == new_row)
def test_get_suggestion():
for _ in range(10):
parameters = [sherpa.Continuous('myparam', [0, 1]),
sherpa.Discrete('myparam2', [0, 10])]
rs = sherpa.algorithms.RandomSearch()
gs = SequentialTesting(algorithm=rs,
K=10,
n=(3, 6, 9),
P=0.5)
study = sherpa.Study(algorithm=gs,
parameters=parameters,
lower_is_better=True,
disable_dashboard=True)
seen_configs = []
last_config = {}
config_count = 3
for trial in study:
print(trial.id, trial.parameters, "{}/{}".format(gs.k, gs.K[gs.t]),
"{}/{}".format(gs.t, gs.T))
if trial.parameters == last_config:
config_count += 1
assert config_count <= 3
elif trial.parameters == "DONE":
assert gs.K[gs.t] == 1 or gs.t == 3
break
else:
assert config_count == 3
config_count = 1
last_config = trial.parameters
if trial.id <= 30:
seen_configs.append(trial.parameters['myparam'])
else:
assert trial.parameters['myparam'] in seen_configs
study.add_observation(trial,
iteration=1,
objective=trial.parameters[
'myparam'] + numpy.random.normal(
scale=0.01))
study.finalize(trial)
def test_3d():
def obj_func(x, y, z):
assert isinstance(x, float)
assert isinstance(y, str)
assert isinstance(z, int)
# Global maximum of 4 is at x=4
return -4. * numpy.exp(-(x - 4.)**2 / 10.) * numpy.cos(
1.5 * (x - 4.))**2 - int(y) * z
parameters = [
sherpa.Continuous('x', [0., 7.]),
sherpa.Choice('y', ["-1", "0", "1"]),
sherpa.Discrete('z', [1, 5])
]
bayesian_optimization = GPyOpt(max_concurrent=1,
max_num_trials=100,
model_type='GP',
acquisition_type='EI')
study = sherpa.Study(algorithm=bayesian_optimization,
parameters=parameters,
lower_is_better=True,
disable_dashboard=True)
for trial in study:
print("Trial {}:\t{}".format(trial.id, trial.parameters))
fval = obj_func(**trial.parameters)
print("Function Value: {}".format(fval))
study.add_observation(trial=trial, iteration=1, objective=fval)
study.finalize(trial, status='COMPLETED')
rval = study.get_best_result()
print(rval)
assert numpy.isclose(rval['x'], 4., atol=0.1)
assert rval['y'] == 1
assert rval['z'] == 5
register_gresnet_args(parser)
parser.add_argument('--study-dir', help='file to write study results to')
parser.add_argument('--n-trials',
type=int,
default=100,
help='number of trials to run')
args = parser.parse_args()
print('Parsed args:', args)
with open(os.path.join(args.study_dir, 'args.json'), 'w') as out:
json.dump(vars(args), out)
parent_out_dir = args.output_dir
parameters = [
sherpa.Continuous(name='lr', range=[5e-4, 5e-2], scale='log'),
sherpa.Continuous(name='dropout', range=[0.01, 0.6]),
sherpa.Discrete(name='num_hidden_units', range=[15, 45], scale='log'),
sherpa.Discrete(name='num_hidden_layers', range=[1, 6])
]
algorithm = sherpa.algorithms.RandomSearch(max_num_trials=args.n_trials)
study = sherpa.Study(parameters=parameters,
algorithm=algorithm,
lower_is_better=False,
disable_dashboard=True,
output_dir=args.study_dir)
data = load_graph_data.load(args)
for trial in study:
print('Starting trial {} with params {}'.format(
trial.id, trial.parameters))
args.output_dir = os.path.join(parent_out_dir, str(trial.id))
os.mkdir(args.output_dir)
if __name__ == "__main__":
logger.info("AUTOML ON ACCURACY")
logger.info("MOVING LNCS2")
config["LANG"].remove("latin")
config["LANG"].remove("english")
port = 1
for lang in config["LANG"]:
i = 1
parameters = [
sherpa.Continuous("threshold", [0.5, 0.85]),
sherpa.Discrete("topn", [10, 50]),
sherpa.Continuous("t", [0.0, 1.0]),
]
algorithm = bayesian_optimization.GPyOpt(max_num_trials=40)
study = sherpa.Study(
parameters=parameters,
algorithm=algorithm,
lower_is_better=False,
dashboard_port=(9999 - port + 1),
)
model1 = Word2Vec.load(
CURRENT_EXP_DIR.split("_")[0] + "_0" + "/model/" + lang +
"/corpus1.model")
model2 = Word2Vec.load(
CURRENT_EXP_DIR.split("_")[0] + "_0" + "/model/" + lang +
"/corpus2.model")
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(train_dir,
target_size=(width,
length),
batch_size=batch_size,
class_mode='categorical')
validation_generator = test_datagen.flow_from_directory(
test_dir,
target_size=(width, length),
batch_size=batch_size,
class_mode='categorical')
parameters = [sherpa.Discrete('num_units', [50, 200])]
alg = sherpa.algorithms.BayesianOptimization(max_num_trials=50)
study = sherpa.Study(parameters=parameters,
algorithm=alg,
lower_is_better=True)
for trial in study:
#model
base_model = ResNet50(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=(width, length, 3),
pooling=None,
classes=classes)
#model = Model(inputs = main_input, outputs = out)
#model = VGG19(include_top=False, weights='imagenet', input_tensor=None, input_shape=(width,length,3), pooling=None, classes=classes)
def test_3d():
def obj_func(x, y, z):
# Global maximum of 4 is at x=4
return -4. * numpy.exp(-(x - 4.)**2 / 10.) * numpy.cos(
1.5 * (x - 4.))**2 - y * z
parameters = [
sherpa.Continuous('x', [0., 7.]),
sherpa.Choice('y', [-1, 0, 1]),
sherpa.Discrete('z', [1, 5])
]
bayesian_optimization = GPyOpt(max_concurrent=1,
max_num_trials=100,
model_type='GP',
acquisition_type='EI')
study = sherpa.Study(algorithm=bayesian_optimization,
parameters=parameters,
lower_is_better=True,
disable_dashboard=True)
for trial in study:
print("Trial {}:\t{}".format(trial.id, trial.parameters))
fval = obj_func(**trial.parameters)
print("Function Value: {}".format(fval))
study.add_observation(trial=trial, iteration=1, objective=fval)
study.finalize(trial, status='COMPLETED')
rval = study.get_best_result()
print(rval)
assert numpy.isclose(rval['x'], 4., atol=0.1)
assert rval['y'] == 1
assert rval['z'] == 5
# def test_noisy_parabola():
# def f(x, sd=1):
# y = (x - 3) ** 2 + 10.
# if sd == 0:
# return y
# else:
# return y + numpy.random.normal(loc=0., scale=sd,
# size=numpy.array(x).shape)
#
# parameters = [sherpa.Continuous('x1', [0., 7.])]
#
# bayesian_optimization = GPyOpt(max_concurrent=1,
# max_num_trials=20,
# model_type='GP',
# acquisition_type='EI')
# rep = Repeat(algorithm=bayesian_optimization,
# num_times=5)
# study = sherpa.Study(algorithm=rep,
# parameters=parameters,
# lower_is_better=True,
# disable_dashboard=True)
#
# for trial in study:
# print("Trial {}:\t{}".format(trial.id, trial.parameters))
#
# fval = f(trial.parameters['x1'], sd=1)
# print("Function Value: {}".format(fval))
# study.add_observation(trial=trial,
# iteration=1,
# objective=fval)
# study.finalize(trial, status='COMPLETED')
# rval = study.get_best_result()
# print(rval)
# # assert numpy.sqrt((rval['Objective'] - 3.)**2) < 0.2
#
# if __name__ == '__main__':
# test_noisy_parabola()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='GCN hparam search')
register_general_args(parser)
register_gat_args(parser)
parser.add_argument('--study-dir', help='file to write study results to')
parser.add_argument('--n-trials', type=int, default=100,
help='number of trials to run')
args = parser.parse_args()
print('Parsed args:', args)
with open(os.path.join(args.study_dir, 'args.json'), 'w') as out:
json.dump(vars(args), out)
parameters = [
sherpa.Continuous(name='lr', range=[1e-3, 1e-1], scale='log'),
sherpa.Continuous(name='dropout', range=[0.2, 0.8]),
sherpa.Discrete(name='num_hidden_units', range=[12, 20], scale='log'),
sherpa.Choice(name='residual', range=[True, False]),
sherpa.Discrete(name='num_heads', range=[4,8], scale='log'),
sherpa.Discrete(name='num_layers', range=[1,2])
]
algorithm = sherpa.algorithms.GPyOpt(max_num_trials=args.n_trials)
study = sherpa.Study(parameters=parameters,
algorithm=algorithm,
lower_is_better=False,
disable_dashboard=True,
output_dir=args.study_dir)
data = load_graph_data.load(args)
for trial in study:
print('Starting trial {} with params {}'.format(
trial.id, trial.parameters))
def test_transformation_to_gpyopt_domain_log_discrete():
parameters = [sherpa.Discrete('a', [1, 100], 'log')]
with pytest.warns(UserWarning, match='does not support log-scale'):
GPyOpt._initialize_domain(parameters)
sherpa.Continuous('a', [0.1, 1], 'log'),
sherpa.Continuous('b', [10., 100], 'log')
])])
def test_transformation_to_gpyopt_domain_log_continuous(parameters):
domain = GPyOpt._initialize_domain(parameters)
for p, d in zip(parameters, domain):
assert d['name'] == p.name
assert d['type'] == 'continuous'
assert d['domain'] == tuple(
[numpy.log10(p.range[0]),
numpy.log10(p.range[1])])
@pytest.mark.parametrize(
"parameters",
[([sherpa.Discrete('a', [0, 100])]),
([sherpa.Discrete('a', [1, 100]),
sherpa.Discrete('b', [0, 100])])])
def test_transformation_to_gpyopt_domain_discrete(parameters):
domain = GPyOpt._initialize_domain(parameters)
for p, d in zip(parameters, domain):
assert d['name'] == p.name
assert d['type'] == 'discrete'
assert d['domain'] == tuple(range(p.range[0], p.range[1] + 1))
def test_transformation_to_gpyopt_domain_log_discrete():
parameters = [sherpa.Discrete('a', [1, 100], 'log')]
with pytest.warns(UserWarning, match='does not support log-scale'):
GPyOpt._initialize_domain(parameters)
from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
import time
import sherpa
parameters = [
sherpa.Discrete('n_estimators', [2, 50]),
sherpa.Choice('criterion', ['gini', 'entropy']),
sherpa.Continuous('max_features', [0.1, 0.9])
]
algorithm = sherpa.algorithms.BayesianOptimization(max_num_trials=100,
num_grid_points=2)
X, y = load_breast_cancer(return_X_y=True)
study = sherpa.Study(parameters=parameters,
algorithm=algorithm,
lower_is_better=False)
for trial in study:
clf = RandomForestClassifier(criterion=trial.parameters['criterion'],
max_features=trial.parameters['max_features'],
n_estimators=trial.parameters['n_estimators'],
random_state=0)
scores = cross_val_score(clf, X, y, cv=5)
study.add_observation(trial, iteration=1, objective=scores.mean())
study.finalize(trial)
time.sleep(60)
bkg_test = test[test_y == 0].copy()
bkg_test_x_nJet = bkg_test['nJets30Clean']
bkg_test_x = bkg_test.drop(columns=['target', 'sampleName', 'nJets30Clean'])
test_x = test.drop(columns=['target', 'sampleName', 'nJets30Clean'])
train_x_ttdilep_nJet = train_x_ttdilep['nJets30Clean']
train_x_ttdilep = train_x_ttdilep.drop(
columns=['target', 'sampleName', 'nJets30Clean'])
### Hyperparameter search ###
parameters = [
sherpa.Continuous('DRf_learning_rate', [0.001, 0.1], scale='log'),
sherpa.Continuous('DfR_learning_rate', [0.001, 0.1], scale='log'),
sherpa.Continuous('DRf_momentum', [0.5, 0.9]),
sherpa.Continuous('DfR_momentum', [0.5, 0.9]),
sherpa.Discrete('additional_hidden_layers_classifier', [0, 3]),
sherpa.Discrete('additional_hidden_layers_adversarial', [0, 3]),
sherpa.Ordinal('num_hidden_units_classifier', [50, 100, 150]),
sherpa.Ordinal('num_hidden_units_adversarial', [50, 100, 150]),
sherpa.Choice('first_layer_activation', ['tanh', 'no_tanh']),
sherpa.Continuous('dropout_rate', [0.2, 0.5]),
sherpa.Ordinal('adversarial_batch_size', [128, 256, 512, 1024]),
sherpa.Continuous('lambda', [2, 12])
]
random_search = sherpa.algorithms.RandomSearch(max_num_trials=num_of_trials)
study = sherpa.Study(parameters=parameters,
algorithm=random_search,
lower_is_better=True,
disable_dashboard=True)
