def test_user_code_fails(test_dir):
tempdir = test_dir
parameters = [
sherpa.Choice(name="param_a", range=[1, 2, 3]),
sherpa.Continuous(name="param_b", range=[0, 1])
]
algorithm = sherpa.algorithms.RandomSearch(max_num_trials=3)
db_port = 27000
scheduler = sherpa.schedulers.LocalScheduler()
filename = os.path.join(tempdir, "test.py")
with open(filename, 'w') as f:
f.write(testscript2)
with pytest.warns(RuntimeWarning):
results = sherpa.optimize(filename=filename,
lower_is_better=True,
algorithm=algorithm,
parameters=parameters,
output_dir=tempdir,
scheduler=scheduler,
max_concurrent=1,
db_port=db_port)
def test_parallel():
tempdir = tempfile.mkdtemp(dir=".")
parameters = [sherpa.Continuous('myparam', [0, 1])]
rs = sherpa.algorithms.RandomSearch()
gs = SequentialTesting(algorithm=rs,
K=3,
n=(3, 6, 9),
P=0.5,
verbose=True)
scheduler = sherpa.schedulers.LocalScheduler()
filename = os.path.join(tempdir, "test.py")
with open(filename, 'w') as f:
f.write(testscript)
try:
results = sherpa.optimize(parameters=parameters,
algorithm=gs,
lower_is_better=True,
command="python {}".format(filename),
output_dir=tempdir,
scheduler=scheduler,
max_concurrent=2,
verbose=1,
disable_dashboard=True)
finally:
shutil.rmtree(tempdir)
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [
sherpa.Continuous('log10_learning_rate', [-5, 0]),
sherpa.Continuous('log2_batch_size', [5, 8]),
sherpa.Continuous('log2_n_steps', [4, 11]),
sherpa.Continuous('x_n_opt_epochs', [0, 7]),
sherpa.Continuous('log10_entcoeff', [-8, -1]),
sherpa.Continuous('x_gamma', [-4, -1]),
sherpa.Continuous('cliprange', [0.1, 0.4]),
sherpa.Continuous('lam', [0.8, 1.0])
]
alg = sherpa.algorithms.RandomSearch(max_num_trials=300)
alg = sherpa.algorithms.Repeat(alg, 25, agg=True)
# Run on local machine.
sched = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=alg,
lower_is_better=False,
filename='trial.py',
scheduler=sched,
verbose=0,
max_concurrent=FLAGS.max_concurrent,
disable_dashboard=True,
output_dir='./output_{}'.format(
time.strftime("%Y-%m-%d--%H-%M-%S")))
print('Best results:')
print(rval)
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [
sherpa.Continuous('learning_rate', [1e-5, 5e-1], 'log'),
sherpa.Continuous('decay', [1e-8, 1e-2], 'log'),
sherpa.Continuous('momentum', [0., 0.99]),
sherpa.Continuous('dropout', [0.0001, 0.7]),
sherpa.Ordinal('batch_size', [32, 64, 128, 256])
]
algorithm = alg = sherpa.algorithms.PopulationBasedTraining(
num_generations=26,
population_size=100,
parameter_range={
'learning_rate': [1e-10, 9e-1],
'decay': [1e-10, 9e-1]
},
perturbation_factors=(0.8, 1.2))
# Run on local machine.
scheduler = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=algorithm,
dashboard_port=FLAGS.port,
lower_is_better=False,
command='python fashion_mlp.py',
scheduler=scheduler,
verbose=0,
max_concurrent=FLAGS.concurrent,
output_dir='./output_pbt_{}'.format(
time.strftime("%Y-%m-%d--%H-%M-%S")))
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [sherpa.Continuous('learning_rate', [1e-5, 5e-1], 'log'),
sherpa.Continuous('decay', [1e-8, 1e-2], 'log'),
sherpa.Continuous('momentum', [0., 0.99]),
sherpa.Continuous('dropout', [0.0001, 0.7]),
sherpa.Ordinal('batch_size', [32, 64, 128, 256])]
algorithm = bayesian_optimization.GPyOpt(max_concurrent=FLAGS.concurrent,
model_type='GP',
acquisition_type='EI',
max_num_trials=100)
# Run on local machine.
scheduler = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=algorithm,
dashboard_port=FLAGS.port,
lower_is_better=False,
command='python fashion_mlp.py',
scheduler=scheduler,
verbose=0,
max_concurrent=FLAGS.concurrent,
output_dir='./output_gpyopt_{}'.format(
time.strftime("%Y-%m-%d--%H-%M-%S")))
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [
sherpa.Continuous('lrinit', [0.001, 0.1], 'log'),
sherpa.Continuous('momentum', [0., 0.99]),
sherpa.Continuous('lrdecay', [1e-7, 1e-2], 'log'),
sherpa.Continuous('dropout', [0., 0.5])
]
if FLAGS.algorithm == 'BayesianOptimization':
print('Running GPyOpt')
alg = bayesian_optimization.GPyOpt(max_concurrent=FLAGS.max_concurrent,
model_type='GP_MCMC',
acquisition_type='EI_MCMC',
max_num_trials=150)
elif FLAGS.algorithm == 'LocalSearch':
print('Running Local Search')
alg = sherpa.algorithms.LocalSearch(seed_configuration={
'lrinit': 0.038,
'momentum': 0.92,
'lrdecay': 0.0001,
'dropout': 0.
},
perturbation_factors=(0.9, 1.1))
else:
print('Running Random Search')
alg = sherpa.algorithms.RandomSearch(max_num_trials=150)
if FLAGS.sge:
assert FLAGS.env, "For SGE use, you need to set an environment path."
# Submit to SGE queue.
env = FLAGS.env # Script specifying environment variables.
opt = '-N MNISTExample -P {} -q {} -l {}'.format(
FLAGS.P, FLAGS.q, FLAGS.l)
sched = SGEScheduler(environment=env, submit_options=opt)
else:
# Run on local machine.
sched = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=alg,
lower_is_better=True,
filename='trial.py',
output_dir='output_{}'.format(FLAGS.studyname),
scheduler=sched,
max_concurrent=FLAGS.max_concurrent)
print('Best results:')
print(rval)
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
# max_features = trial.parameters('max_features', 20000)
# batch_size = trial.parameters('batch_size', 32)
# hidden_dim = trial.parameters('hidden_dim', 128)
# dropout = trial.parameters('dropout', 0.2)
# recurrent_dropout = trial.parameters('recurrent_dropout', 0.2)
# optimizer = trial.parameters('optimizer', 'adam')
parameters = []
parameters += [sherpa.Discrete('max_features', [15000, 40000])]
parameters += [sherpa.Discrete('batch_size', [10, 150], 'log')]
parameters += [sherpa.Discrete('hidden_dim', [100, 500], 'log')]
parameters += [sherpa.Continuous('dropout_embedding', [0.0001, 0.5])]
parameters += [sherpa.Continuous('dropout_lstm', [0.0001, 0.5])]
parameters += [sherpa.Continuous('embedding_regularizer', [1e-12, 1e-6], 'log')]
parameters += [sherpa.Continuous('kernel_regularizer', [1e-8, 1e-0], 'log')]
parameters += [sherpa.Continuous('recurrent_regularizer', [1e-8, 1e-0], 'log')]
parameters += [sherpa.Continuous('lr', [5e-4, 5e-3], scale='log')]
parameters += [sherpa.Continuous('decay', [1e-5, 1e-10], scale='log')]
parameters += [sherpa.Continuous('rho', [0.5, 0.99])]
print('Running Random Search')
alg = sherpa.algorithms.RandomSearch(max_num_trials=200, repeat=25)
if FLAGS.sge:
assert FLAGS.env, "For SGE use, you need to set an environment path."
# Submit to SGE queue.
env = FLAGS.env # Script specifying environment variables.
opt = '-N LSTMSearch -P {} -q {} -l {} -l gpu=1'.format(FLAGS.P, FLAGS.q, FLAGS.l)
sched = SGEScheduler(environment=env, submit_options=opt)
else:
# Run on local machine.
resources = [int(x) for x in FLAGS.gpus.split(',')]
sched = LocalScheduler(resources=resources)
rval = sherpa.optimize(parameters=parameters,
algorithm=alg,
lower_is_better=False,
filename='trials.py',
scheduler=sched,
verbose=0,
max_concurrent=FLAGS.max_concurrent,
output_dir='./output_imdb_{}_{}_gpu-{}'.format(time.strftime("%Y-%m-%d--%H-%M-%S"), FLAGS.name, FLAGS.gpus.replace(',', '-')))
print('Best results:')
print(rval)
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
# Iterate algorithm accepts dictionary containing lists of possible values.
hp_space = {
'act': ['tanh', 'relu'],
'lrinit': [0.1, 0.01],
'momentum': [0.0],
'lrdecay': [0.0],
'arch': [[20, 5], [20, 10], [10, 10, 10]],
'epochs': [20],
}
parameters = sherpa.Parameter.grid(hp_space)
alg = sherpa.algorithms.GridSearch()
stopping_rule = sherpa.algorithms.MedianStoppingRule(min_iterations=10,
min_trials=5)
f = './bianchini.py' # Python script to run.
dir = './output' # All files written to here.
if not FLAGS.local:
# Submit to SGE queue.
# env = '/home/pjsadows/profiles/auto.profile' # Script specifying environment variables.
env = FLAGS.env
opt = '-N example -P {} -q {} -l {}'.format(FLAGS.P, FLAGS.q, FLAGS.l)
sched = SGEScheduler(environment=env,
submit_options=opt,
output_dir=dir)
else:
# Run on local machine.
sched = LocalScheduler() # Run on local machine without SGE.
rval = sherpa.optimize(parameters=parameters,
algorithm=alg,
stopping_rule=stopping_rule,
output_dir=dir,
lower_is_better=True,
filename=f,
scheduler=sched,
max_concurrent=FLAGS.max_concurrent)
print()
print('Best results:')
print(rval)
def runner():
parameters = [
sherpa.Discrete('num_units', [64, 512]),
sherpa.Discrete('num_layers', [2, 15]),
sherpa.Ordinal('batch_size', [4096]),
sherpa.Ordinal('learning_rate',
[0.0001, 0.0003162278, 0.001, 0.003162278, 0.001]),
sherpa.Continuous('dropout_rate', [0, 0.6]),
sherpa.Continuous('leaky_relu_alpha', [0, 1]),
sherpa.Ordinal('batch_norm', [0, 1]),
]
#alg = sherpa.algorithms.BayesianOptimization(max_num_trials=50)
#alg = sherpa.algorithms.GPyOpt(max_num_trials=50)
alg = sherpa.algorithms.RandomSearch(max_num_trials=40)
#resources = ['/device:GPU:0', '/device:GPU:1', '/device:GPU:2', '/device:GPU:3']
resources = [0, 1, 2, 3]
scheduler = sherpa.schedulers.LocalScheduler(resources=resources)
#study = sherpa.Study(parameters=parameters,
# algorithm=alg,
# lower_is_better=True)
script = '/work/00157/walling/projects/cloud_emulator/walling-CBRAIN-CAM/notebooks/tbeucler_devlog/sherpa/sherpa-trial-phase2-study1-pos.py'
tempdir = '/work/00157/walling/projects/cloud_emulator/walling-CBRAIN-CAM/notebooks/tbeucler_devlog/sherpa/sherpa-temp-phase2-study1-pos'
#import time
#time.sleep(1000)
results = sherpa.optimize(parameters=parameters,
algorithm=alg,
lower_is_better=True,
filename=script,
output_dir=tempdir,
scheduler=scheduler,
max_concurrent=4,
verbose=1,
mongodb_args={'bind_ip':
'0.0.0.0'}) #, 'port':47001})
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [
sherpa.Continuous('learning_rate', [1e-5, 5e-1], 'log'),
sherpa.Continuous('decay', [1e-8, 1e-2], 'log'),
sherpa.Continuous('momentum', [0., 0.99]),
sherpa.Continuous('dropout', [0.0001, 0.7]),
sherpa.Ordinal('batch_size', [32, 64, 128, 256])
]
# 27 epochs to produce one finished 13 epoch model
# => 54 epochs to produce one finished 26 epoch model (resource unit = 2epochs)
# RS uses 100 models x 26 epochs = 2600 epochs.models
# ASHA can produce 2600epochs.models//54epochs = 48 models
algorithm = sherpa.algorithms.SuccessiveHalving(r=1,
R=9,
eta=3,
s=0,
max_finished_configs=48)
# Run on local machine.
scheduler = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=algorithm,
dashboard_port=FLAGS.port,
lower_is_better=False,
command='python fashion_mlp.py',
scheduler=scheduler,
verbose=0,
max_concurrent=FLAGS.concurrent,
output_dir='./output_successive_halving_{}'.format(
time.strftime("%Y-%m-%d--%H-%M-%S")))
'lr': [0.0000001, 1.],
'batch_size': [16, 32, 64, 128, 256, 512]
})
parameters.append(sherpa.Choice(name='epochs', range=[3]))
stoppingrule = None
else:
parameters.append(
sherpa.Continuous(name='lr_decay', range=[1e-4, 1e-7], scale='log'))
parameters.append(sherpa.Choice(name='epochs', range=[25]))
algorithm = sherpa.algorithms.BayesianOptimization(num_grid_points=2)
# stoppingrule = sherpa.algorithms.MedianStoppingRule(min_trials=10,
# min_iterations=8)
# The scheduler
if not FLAGS.local:
env = FLAGS.env
opt = '-N MNISTPBT -P {} -q {} -l {} -l gpu=1'.format(
FLAGS.P, FLAGS.q, FLAGS.l)
scheduler = sherpa.schedulers.SGEScheduler(environment=env,
submit_options=opt)
else:
scheduler = sherpa.schedulers.LocalScheduler()
# Running it all
sherpa.optimize(algorithm=algorithm,
scheduler=scheduler,
parameters=parameters,
lower_is_better=True,
filename="mnist_cnn.py",
max_concurrent=FLAGS.max_concurrent,
output_dir='./output')
sherpa.Choice('optimizer', ['adam', 'sgd', 'rmsprop']),
# SETTINGS
sherpa.Choice('epochs', [30]),
sherpa.Choice('batch_size', [16384]),
sherpa.Choice('patience', [10]),
sherpa.Choice('data', [FLAGS.data]),
sherpa.Choice('data_dir', ['/oasis/projects/nsf/sio134/jott1/']),
sherpa.Choice('model_dir', [output_dir + 'Models/']),
]
# sched = sherpa.schedulers.LocalScheduler(resources=resources)
sched = sherpa.schedulers.SLURMScheduler(
submit_options='run.csh',
environment='/home/jott1/.bashrc',
username='******'
)
alg = sherpa.algorithms.RandomSearch(max_num_trials=10000)
sherpa.optimize(
parameters=parameters,
algorithm=alg,
lower_is_better=True,
command='python main.py',
scheduler=sched,
num_distributors=FLAGS.num_distributors,
num_tasks_per_distributor=FLAGS.num_tasks_per_distributor,
output_dir=output_dir
)
import time
client = sherpa.Client()
trial = client.get_trial()
# Simulate model training
num_iterations = 10
for i in range(num_iterations):
pseudo_objective = trial.parameters['param_a'] / float(i + 1) * trial.parameters['param_b']
time.sleep(1)
client.send_metrics(trial=trial, iteration=i+1,
objective=pseudo_objective)
# print("Trial {} Iteration {}.".format(trial.id, i+1))
# print("Trial {} finished.".format(trial.id))
"""
filename = os.path.join(tempdir, "test.py")
with open(filename, 'w') as f:
f.write(testscript)
results = sherpa.optimize(parameters=parameters,
algorithm=algorithm,
lower_is_better=True,
filename=filename,
output_dir=tempdir,
scheduler=scheduler,
max_concurrent=4,
verbose=1)
print(results)
sherpa.Continuous(name='top_dropout', range=[0.0001, 0.7]),
sherpa.Continuous(name='lr_decay', range=[1e-4, 1e-9], scale='log')
]
# parameters = [sherpa.Choice(name='lr', range=[0.09]),
# sherpa.Choice(name='dropout', range=[0.2]),
# sherpa.Choice(name='lr_decay', range=[1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9])]
algorithm = sherpa.algorithms.RandomSearch(max_num_trials=100, repeat=25)
# algorithm = sherpa.algorithms.GridSearch()
# The scheduler
# if not FLAGS.local:
# env = FLAGS.env
# opt = '-N MNIST-H1 -P {} -q {} -l {} -l gpu=1'.format(FLAGS.P, FLAGS.q, FLAGS.l)
# scheduler = sherpa.schedulers.SGEScheduler(environment=env, submit_options=opt)
# else:
resources = [int(x) for x in FLAGS.gpus.split(',')] * 3
# resources = [0,0,0,1,1,1,2,2,2,3,3,3]
scheduler = sherpa.schedulers.LocalScheduler(resources=resources)
# Running it all
sherpa.optimize(algorithm=algorithm,
scheduler=scheduler,
parameters=parameters,
lower_is_better=True,
filename="mnist_cnn.py",
max_concurrent=FLAGS.max_concurrent,
output_dir='./output_mnistcnn_unshuffled_{}_{}_gpu-{}'.format(
time.strftime("%Y-%m-%d--%H-%M-%S"), FLAGS.name,
FLAGS.gpus.replace(',', '-')))
sherpa.Choice('lr', [1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6]),
# sherpa.Choice('activation', ['relu', 'prelu', 'elu', 'leaky_relu', 'sigmoid']),
# sherpa.Choice('kernel_initializer', ['glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform']),
]
for k,v in vars(FLAGS).iteritems():
parameters.append(
sherpa.Choice(k, [v])
)
# Run on local machine.
gpus = [int(x) for x in FLAGS.gpus.split(',')]
processes_per_gpu = FLAGS.max_concurrent//len(gpus)
assert FLAGS.max_concurrent%len(gpus) == 0
resources = list(itertools.chain.from_iterable(itertools.repeat(x, processes_per_gpu) for x in gpus))
sched = sherpa.schedulers.LocalScheduler(resources=resources)
alg = sherpa.algorithms.RandomSearch(max_num_trials=200)
build_directory('SherpaResults/' + FLAGS.dataset + '/Models')
sherpa.optimize(
parameters=parameters,
algorithm=alg,
lower_is_better=True,
command='python hp_main.py',
scheduler=sched,
max_concurrent=FLAGS.max_concurrent,
output_dir='SherpaResults/' + FLAGS.dataset +'/'
)
# USING KERAS DATA GENERATOR FOR AUG
sherpa.Choice('data_aug', [1]),
sherpa.Choice('normalize', [1]),
# LOSS PENALTY FOR CLASS IMBALANCE
sherpa.Choice('class_weight', [0, 1]),
]
for k, v in vars(FLAGS).items():
parameters.append(sherpa.Choice(k, [v]))
# Run on local machine.
gpus = [int(x) for x in FLAGS.gpus.split(',')]
processes_per_gpu = FLAGS.max_concurrent // len(gpus)
assert FLAGS.max_concurrent % len(gpus) == 0
resources = list(
itertools.chain.from_iterable(
itertools.repeat(x, processes_per_gpu) for x in gpus))
sched = sherpa.schedulers.LocalScheduler(resources=resources)
alg = sherpa.algorithms.RandomSearch(max_num_trials=1000)
# alg = sherpa.algorithms.GPyOpt(max_num_trials=1000)
sherpa.optimize(parameters=parameters,
algorithm=alg,
lower_is_better=False,
command='/pkg/python/3.6.1-centos7/bin/python3.6 main.py',
scheduler=sched,
max_concurrent=FLAGS.max_concurrent,
output_dir='SherpaResults')
