def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [
sherpa.Continuous('log10_learning_rate', [-5, 0]),
sherpa.Continuous('log2_batch_size', [5, 8]),
sherpa.Continuous('log2_n_steps', [4, 11]),
sherpa.Continuous('x_n_opt_epochs', [0, 7]),
sherpa.Continuous('log10_entcoeff', [-8, -1]),
sherpa.Continuous('x_gamma', [-4, -1]),
sherpa.Continuous('cliprange', [0.1, 0.4]),
sherpa.Continuous('lam', [0.8, 1.0])
]
alg = sherpa.algorithms.RandomSearch(max_num_trials=300)
alg = sherpa.algorithms.Repeat(alg, 25, agg=True)
# Run on local machine.
sched = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=alg,
lower_is_better=False,
filename='trial.py',
scheduler=sched,
verbose=0,
max_concurrent=FLAGS.max_concurrent,
disable_dashboard=True,
output_dir='./output_{}'.format(
time.strftime("%Y-%m-%d--%H-%M-%S")))
print('Best results:')
print(rval)
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [sherpa.Continuous('learning_rate', [1e-5, 5e-1], 'log'),
sherpa.Continuous('decay', [1e-8, 1e-2], 'log'),
sherpa.Continuous('momentum', [0., 0.99]),
sherpa.Continuous('dropout', [0.0001, 0.7]),
sherpa.Ordinal('batch_size', [32, 64, 128, 256])]
algorithm = bayesian_optimization.GPyOpt(max_concurrent=FLAGS.concurrent,
model_type='GP',
acquisition_type='EI',
max_num_trials=100)
# Run on local machine.
scheduler = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=algorithm,
dashboard_port=FLAGS.port,
lower_is_better=False,
command='python fashion_mlp.py',
scheduler=scheduler,
verbose=0,
max_concurrent=FLAGS.concurrent,
output_dir='./output_gpyopt_{}'.format(
time.strftime("%Y-%m-%d--%H-%M-%S")))
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [
sherpa.Continuous('learning_rate', [1e-5, 5e-1], 'log'),
sherpa.Continuous('decay', [1e-8, 1e-2], 'log'),
sherpa.Continuous('momentum', [0., 0.99]),
sherpa.Continuous('dropout', [0.0001, 0.7]),
sherpa.Ordinal('batch_size', [32, 64, 128, 256])
]
algorithm = alg = sherpa.algorithms.PopulationBasedTraining(
num_generations=26,
population_size=100,
parameter_range={
'learning_rate': [1e-10, 9e-1],
'decay': [1e-10, 9e-1]
},
perturbation_factors=(0.8, 1.2))
# Run on local machine.
scheduler = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=algorithm,
dashboard_port=FLAGS.port,
lower_is_better=False,
command='python fashion_mlp.py',
scheduler=scheduler,
verbose=0,
max_concurrent=FLAGS.concurrent,
output_dir='./output_pbt_{}'.format(
time.strftime("%Y-%m-%d--%H-%M-%S")))
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [
sherpa.Continuous('lrinit', [0.001, 0.1], 'log'),
sherpa.Continuous('momentum', [0., 0.99]),
sherpa.Continuous('lrdecay', [1e-7, 1e-2], 'log'),
sherpa.Continuous('dropout', [0., 0.5])
]
if FLAGS.algorithm == 'BayesianOptimization':
print('Running GPyOpt')
alg = bayesian_optimization.GPyOpt(max_concurrent=FLAGS.max_concurrent,
model_type='GP_MCMC',
acquisition_type='EI_MCMC',
max_num_trials=150)
elif FLAGS.algorithm == 'LocalSearch':
print('Running Local Search')
alg = sherpa.algorithms.LocalSearch(seed_configuration={
'lrinit': 0.038,
'momentum': 0.92,
'lrdecay': 0.0001,
'dropout': 0.
},
perturbation_factors=(0.9, 1.1))
else:
print('Running Random Search')
alg = sherpa.algorithms.RandomSearch(max_num_trials=150)
if FLAGS.sge:
assert FLAGS.env, "For SGE use, you need to set an environment path."
# Submit to SGE queue.
env = FLAGS.env # Script specifying environment variables.
opt = '-N MNISTExample -P {} -q {} -l {}'.format(
FLAGS.P, FLAGS.q, FLAGS.l)
sched = SGEScheduler(environment=env, submit_options=opt)
else:
# Run on local machine.
sched = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=alg,
lower_is_better=True,
filename='trial.py',
output_dir='output_{}'.format(FLAGS.studyname),
scheduler=sched,
max_concurrent=FLAGS.max_concurrent)
print('Best results:')
print(rval)
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
# max_features = trial.parameters('max_features', 20000)
# batch_size = trial.parameters('batch_size', 32)
# hidden_dim = trial.parameters('hidden_dim', 128)
# dropout = trial.parameters('dropout', 0.2)
# recurrent_dropout = trial.parameters('recurrent_dropout', 0.2)
# optimizer = trial.parameters('optimizer', 'adam')
parameters = []
parameters += [sherpa.Discrete('max_features', [15000, 40000])]
parameters += [sherpa.Discrete('batch_size', [10, 150], 'log')]
parameters += [sherpa.Discrete('hidden_dim', [100, 500], 'log')]
parameters += [sherpa.Continuous('dropout_embedding', [0.0001, 0.5])]
parameters += [sherpa.Continuous('dropout_lstm', [0.0001, 0.5])]
parameters += [sherpa.Continuous('embedding_regularizer', [1e-12, 1e-6], 'log')]
parameters += [sherpa.Continuous('kernel_regularizer', [1e-8, 1e-0], 'log')]
parameters += [sherpa.Continuous('recurrent_regularizer', [1e-8, 1e-0], 'log')]
parameters += [sherpa.Continuous('lr', [5e-4, 5e-3], scale='log')]
parameters += [sherpa.Continuous('decay', [1e-5, 1e-10], scale='log')]
parameters += [sherpa.Continuous('rho', [0.5, 0.99])]
print('Running Random Search')
alg = sherpa.algorithms.RandomSearch(max_num_trials=200, repeat=25)
if FLAGS.sge:
assert FLAGS.env, "For SGE use, you need to set an environment path."
# Submit to SGE queue.
env = FLAGS.env # Script specifying environment variables.
opt = '-N LSTMSearch -P {} -q {} -l {} -l gpu=1'.format(FLAGS.P, FLAGS.q, FLAGS.l)
sched = SGEScheduler(environment=env, submit_options=opt)
else:
# Run on local machine.
resources = [int(x) for x in FLAGS.gpus.split(',')]
sched = LocalScheduler(resources=resources)
rval = sherpa.optimize(parameters=parameters,
algorithm=alg,
lower_is_better=False,
filename='trials.py',
scheduler=sched,
verbose=0,
max_concurrent=FLAGS.max_concurrent,
output_dir='./output_imdb_{}_{}_gpu-{}'.format(time.strftime("%Y-%m-%d--%H-%M-%S"), FLAGS.name, FLAGS.gpus.replace(',', '-')))
print('Best results:')
print(rval)
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
# Iterate algorithm accepts dictionary containing lists of possible values.
hp_space = {
'act': ['tanh', 'relu'],
'lrinit': [0.1, 0.01],
'momentum': [0.0],
'lrdecay': [0.0],
'arch': [[20, 5], [20, 10], [10, 10, 10]],
'epochs': [20],
}
parameters = sherpa.Parameter.grid(hp_space)
alg = sherpa.algorithms.GridSearch()
stopping_rule = sherpa.algorithms.MedianStoppingRule(min_iterations=10,
min_trials=5)
f = './bianchini.py' # Python script to run.
dir = './output' # All files written to here.
if not FLAGS.local:
# Submit to SGE queue.
# env = '/home/pjsadows/profiles/auto.profile' # Script specifying environment variables.
env = FLAGS.env
opt = '-N example -P {} -q {} -l {}'.format(FLAGS.P, FLAGS.q, FLAGS.l)
sched = SGEScheduler(environment=env,
submit_options=opt,
output_dir=dir)
else:
# Run on local machine.
sched = LocalScheduler() # Run on local machine without SGE.
rval = sherpa.optimize(parameters=parameters,
algorithm=alg,
stopping_rule=stopping_rule,
output_dir=dir,
lower_is_better=True,
filename=f,
scheduler=sched,
max_concurrent=FLAGS.max_concurrent)
print()
print('Best results:')
print(rval)
def run_example(FLAGS):
"""
Run parallel Sherpa optimization over a set of discrete hp combinations.
"""
parameters = [
sherpa.Continuous('learning_rate', [1e-5, 5e-1], 'log'),
sherpa.Continuous('decay', [1e-8, 1e-2], 'log'),
sherpa.Continuous('momentum', [0., 0.99]),
sherpa.Continuous('dropout', [0.0001, 0.7]),
sherpa.Ordinal('batch_size', [32, 64, 128, 256])
]
# 27 epochs to produce one finished 13 epoch model
# => 54 epochs to produce one finished 26 epoch model (resource unit = 2epochs)
# RS uses 100 models x 26 epochs = 2600 epochs.models
# ASHA can produce 2600epochs.models//54epochs = 48 models
algorithm = sherpa.algorithms.SuccessiveHalving(r=1,
R=9,
eta=3,
s=0,
max_finished_configs=48)
# Run on local machine.
scheduler = LocalScheduler()
rval = sherpa.optimize(parameters=parameters,
algorithm=algorithm,
dashboard_port=FLAGS.port,
lower_is_better=False,
command='python fashion_mlp.py',
scheduler=scheduler,
verbose=0,
max_concurrent=FLAGS.concurrent,
output_dir='./output_successive_halving_{}'.format(
time.strftime("%Y-%m-%d--%H-%M-%S")))