def test_user_code_fails(test_dir): tempdir = test_dir parameters = [ sherpa.Choice(name="param_a", range=[1, 2, 3]), sherpa.Continuous(name="param_b", range=[0, 1]) ] algorithm = sherpa.algorithms.RandomSearch(max_num_trials=3) db_port = 27000 scheduler = sherpa.schedulers.LocalScheduler() filename = os.path.join(tempdir, "test.py") with open(filename, 'w') as f: f.write(testscript2) with pytest.warns(RuntimeWarning): results = sherpa.optimize(filename=filename, lower_is_better=True, algorithm=algorithm, parameters=parameters, output_dir=tempdir, scheduler=scheduler, max_concurrent=1, db_port=db_port)
def test_parallel(): tempdir = tempfile.mkdtemp(dir=".") parameters = [sherpa.Continuous('myparam', [0, 1])] rs = sherpa.algorithms.RandomSearch() gs = SequentialTesting(algorithm=rs, K=3, n=(3, 6, 9), P=0.5, verbose=True) scheduler = sherpa.schedulers.LocalScheduler() filename = os.path.join(tempdir, "test.py") with open(filename, 'w') as f: f.write(testscript) try: results = sherpa.optimize(parameters=parameters, algorithm=gs, lower_is_better=True, command="python {}".format(filename), output_dir=tempdir, scheduler=scheduler, max_concurrent=2, verbose=1, disable_dashboard=True) finally: shutil.rmtree(tempdir)
def run_example(FLAGS): """ Run parallel Sherpa optimization over a set of discrete hp combinations. """ parameters = [ sherpa.Continuous('log10_learning_rate', [-5, 0]), sherpa.Continuous('log2_batch_size', [5, 8]), sherpa.Continuous('log2_n_steps', [4, 11]), sherpa.Continuous('x_n_opt_epochs', [0, 7]), sherpa.Continuous('log10_entcoeff', [-8, -1]), sherpa.Continuous('x_gamma', [-4, -1]), sherpa.Continuous('cliprange', [0.1, 0.4]), sherpa.Continuous('lam', [0.8, 1.0]) ] alg = sherpa.algorithms.RandomSearch(max_num_trials=300) alg = sherpa.algorithms.Repeat(alg, 25, agg=True) # Run on local machine. sched = LocalScheduler() rval = sherpa.optimize(parameters=parameters, algorithm=alg, lower_is_better=False, filename='trial.py', scheduler=sched, verbose=0, max_concurrent=FLAGS.max_concurrent, disable_dashboard=True, output_dir='./output_{}'.format( time.strftime("%Y-%m-%d--%H-%M-%S"))) print('Best results:') print(rval)
def run_example(FLAGS): """ Run parallel Sherpa optimization over a set of discrete hp combinations. """ parameters = [ sherpa.Continuous('learning_rate', [1e-5, 5e-1], 'log'), sherpa.Continuous('decay', [1e-8, 1e-2], 'log'), sherpa.Continuous('momentum', [0., 0.99]), sherpa.Continuous('dropout', [0.0001, 0.7]), sherpa.Ordinal('batch_size', [32, 64, 128, 256]) ] algorithm = alg = sherpa.algorithms.PopulationBasedTraining( num_generations=26, population_size=100, parameter_range={ 'learning_rate': [1e-10, 9e-1], 'decay': [1e-10, 9e-1] }, perturbation_factors=(0.8, 1.2)) # Run on local machine. scheduler = LocalScheduler() rval = sherpa.optimize(parameters=parameters, algorithm=algorithm, dashboard_port=FLAGS.port, lower_is_better=False, command='python fashion_mlp.py', scheduler=scheduler, verbose=0, max_concurrent=FLAGS.concurrent, output_dir='./output_pbt_{}'.format( time.strftime("%Y-%m-%d--%H-%M-%S")))
def run_example(FLAGS): """ Run parallel Sherpa optimization over a set of discrete hp combinations. """ parameters = [sherpa.Continuous('learning_rate', [1e-5, 5e-1], 'log'), sherpa.Continuous('decay', [1e-8, 1e-2], 'log'), sherpa.Continuous('momentum', [0., 0.99]), sherpa.Continuous('dropout', [0.0001, 0.7]), sherpa.Ordinal('batch_size', [32, 64, 128, 256])] algorithm = bayesian_optimization.GPyOpt(max_concurrent=FLAGS.concurrent, model_type='GP', acquisition_type='EI', max_num_trials=100) # Run on local machine. scheduler = LocalScheduler() rval = sherpa.optimize(parameters=parameters, algorithm=algorithm, dashboard_port=FLAGS.port, lower_is_better=False, command='python fashion_mlp.py', scheduler=scheduler, verbose=0, max_concurrent=FLAGS.concurrent, output_dir='./output_gpyopt_{}'.format( time.strftime("%Y-%m-%d--%H-%M-%S")))
def run_example(FLAGS): """ Run parallel Sherpa optimization over a set of discrete hp combinations. """ parameters = [ sherpa.Continuous('lrinit', [0.001, 0.1], 'log'), sherpa.Continuous('momentum', [0., 0.99]), sherpa.Continuous('lrdecay', [1e-7, 1e-2], 'log'), sherpa.Continuous('dropout', [0., 0.5]) ] if FLAGS.algorithm == 'BayesianOptimization': print('Running GPyOpt') alg = bayesian_optimization.GPyOpt(max_concurrent=FLAGS.max_concurrent, model_type='GP_MCMC', acquisition_type='EI_MCMC', max_num_trials=150) elif FLAGS.algorithm == 'LocalSearch': print('Running Local Search') alg = sherpa.algorithms.LocalSearch(seed_configuration={ 'lrinit': 0.038, 'momentum': 0.92, 'lrdecay': 0.0001, 'dropout': 0. }, perturbation_factors=(0.9, 1.1)) else: print('Running Random Search') alg = sherpa.algorithms.RandomSearch(max_num_trials=150) if FLAGS.sge: assert FLAGS.env, "For SGE use, you need to set an environment path." # Submit to SGE queue. env = FLAGS.env # Script specifying environment variables. opt = '-N MNISTExample -P {} -q {} -l {}'.format( FLAGS.P, FLAGS.q, FLAGS.l) sched = SGEScheduler(environment=env, submit_options=opt) else: # Run on local machine. sched = LocalScheduler() rval = sherpa.optimize(parameters=parameters, algorithm=alg, lower_is_better=True, filename='trial.py', output_dir='output_{}'.format(FLAGS.studyname), scheduler=sched, max_concurrent=FLAGS.max_concurrent) print('Best results:') print(rval)
def run_example(FLAGS): """ Run parallel Sherpa optimization over a set of discrete hp combinations. """ # max_features = trial.parameters('max_features', 20000) # batch_size = trial.parameters('batch_size', 32) # hidden_dim = trial.parameters('hidden_dim', 128) # dropout = trial.parameters('dropout', 0.2) # recurrent_dropout = trial.parameters('recurrent_dropout', 0.2) # optimizer = trial.parameters('optimizer', 'adam') parameters = [] parameters += [sherpa.Discrete('max_features', [15000, 40000])] parameters += [sherpa.Discrete('batch_size', [10, 150], 'log')] parameters += [sherpa.Discrete('hidden_dim', [100, 500], 'log')] parameters += [sherpa.Continuous('dropout_embedding', [0.0001, 0.5])] parameters += [sherpa.Continuous('dropout_lstm', [0.0001, 0.5])] parameters += [sherpa.Continuous('embedding_regularizer', [1e-12, 1e-6], 'log')] parameters += [sherpa.Continuous('kernel_regularizer', [1e-8, 1e-0], 'log')] parameters += [sherpa.Continuous('recurrent_regularizer', [1e-8, 1e-0], 'log')] parameters += [sherpa.Continuous('lr', [5e-4, 5e-3], scale='log')] parameters += [sherpa.Continuous('decay', [1e-5, 1e-10], scale='log')] parameters += [sherpa.Continuous('rho', [0.5, 0.99])] print('Running Random Search') alg = sherpa.algorithms.RandomSearch(max_num_trials=200, repeat=25) if FLAGS.sge: assert FLAGS.env, "For SGE use, you need to set an environment path." # Submit to SGE queue. env = FLAGS.env # Script specifying environment variables. opt = '-N LSTMSearch -P {} -q {} -l {} -l gpu=1'.format(FLAGS.P, FLAGS.q, FLAGS.l) sched = SGEScheduler(environment=env, submit_options=opt) else: # Run on local machine. resources = [int(x) for x in FLAGS.gpus.split(',')] sched = LocalScheduler(resources=resources) rval = sherpa.optimize(parameters=parameters, algorithm=alg, lower_is_better=False, filename='trials.py', scheduler=sched, verbose=0, max_concurrent=FLAGS.max_concurrent, output_dir='./output_imdb_{}_{}_gpu-{}'.format(time.strftime("%Y-%m-%d--%H-%M-%S"), FLAGS.name, FLAGS.gpus.replace(',', '-'))) print('Best results:') print(rval)
def run_example(FLAGS): """ Run parallel Sherpa optimization over a set of discrete hp combinations. """ # Iterate algorithm accepts dictionary containing lists of possible values. hp_space = { 'act': ['tanh', 'relu'], 'lrinit': [0.1, 0.01], 'momentum': [0.0], 'lrdecay': [0.0], 'arch': [[20, 5], [20, 10], [10, 10, 10]], 'epochs': [20], } parameters = sherpa.Parameter.grid(hp_space) alg = sherpa.algorithms.GridSearch() stopping_rule = sherpa.algorithms.MedianStoppingRule(min_iterations=10, min_trials=5) f = './bianchini.py' # Python script to run. dir = './output' # All files written to here. if not FLAGS.local: # Submit to SGE queue. # env = '/home/pjsadows/profiles/auto.profile' # Script specifying environment variables. env = FLAGS.env opt = '-N example -P {} -q {} -l {}'.format(FLAGS.P, FLAGS.q, FLAGS.l) sched = SGEScheduler(environment=env, submit_options=opt, output_dir=dir) else: # Run on local machine. sched = LocalScheduler() # Run on local machine without SGE. rval = sherpa.optimize(parameters=parameters, algorithm=alg, stopping_rule=stopping_rule, output_dir=dir, lower_is_better=True, filename=f, scheduler=sched, max_concurrent=FLAGS.max_concurrent) print() print('Best results:') print(rval)
def runner(): parameters = [ sherpa.Discrete('num_units', [64, 512]), sherpa.Discrete('num_layers', [2, 15]), sherpa.Ordinal('batch_size', [4096]), sherpa.Ordinal('learning_rate', [0.0001, 0.0003162278, 0.001, 0.003162278, 0.001]), sherpa.Continuous('dropout_rate', [0, 0.6]), sherpa.Continuous('leaky_relu_alpha', [0, 1]), sherpa.Ordinal('batch_norm', [0, 1]), ] #alg = sherpa.algorithms.BayesianOptimization(max_num_trials=50) #alg = sherpa.algorithms.GPyOpt(max_num_trials=50) alg = sherpa.algorithms.RandomSearch(max_num_trials=40) #resources = ['/device:GPU:0', '/device:GPU:1', '/device:GPU:2', '/device:GPU:3'] resources = [0, 1, 2, 3] scheduler = sherpa.schedulers.LocalScheduler(resources=resources) #study = sherpa.Study(parameters=parameters, # algorithm=alg, # lower_is_better=True) script = '/work/00157/walling/projects/cloud_emulator/walling-CBRAIN-CAM/notebooks/tbeucler_devlog/sherpa/sherpa-trial-phase2-study1-pos.py' tempdir = '/work/00157/walling/projects/cloud_emulator/walling-CBRAIN-CAM/notebooks/tbeucler_devlog/sherpa/sherpa-temp-phase2-study1-pos' #import time #time.sleep(1000) results = sherpa.optimize(parameters=parameters, algorithm=alg, lower_is_better=True, filename=script, output_dir=tempdir, scheduler=scheduler, max_concurrent=4, verbose=1, mongodb_args={'bind_ip': '0.0.0.0'}) #, 'port':47001})
def run_example(FLAGS): """ Run parallel Sherpa optimization over a set of discrete hp combinations. """ parameters = [ sherpa.Continuous('learning_rate', [1e-5, 5e-1], 'log'), sherpa.Continuous('decay', [1e-8, 1e-2], 'log'), sherpa.Continuous('momentum', [0., 0.99]), sherpa.Continuous('dropout', [0.0001, 0.7]), sherpa.Ordinal('batch_size', [32, 64, 128, 256]) ] # 27 epochs to produce one finished 13 epoch model # => 54 epochs to produce one finished 26 epoch model (resource unit = 2epochs) # RS uses 100 models x 26 epochs = 2600 epochs.models # ASHA can produce 2600epochs.models//54epochs = 48 models algorithm = sherpa.algorithms.SuccessiveHalving(r=1, R=9, eta=3, s=0, max_finished_configs=48) # Run on local machine. scheduler = LocalScheduler() rval = sherpa.optimize(parameters=parameters, algorithm=algorithm, dashboard_port=FLAGS.port, lower_is_better=False, command='python fashion_mlp.py', scheduler=scheduler, verbose=0, max_concurrent=FLAGS.concurrent, output_dir='./output_successive_halving_{}'.format( time.strftime("%Y-%m-%d--%H-%M-%S")))
'lr': [0.0000001, 1.], 'batch_size': [16, 32, 64, 128, 256, 512] }) parameters.append(sherpa.Choice(name='epochs', range=[3])) stoppingrule = None else: parameters.append( sherpa.Continuous(name='lr_decay', range=[1e-4, 1e-7], scale='log')) parameters.append(sherpa.Choice(name='epochs', range=[25])) algorithm = sherpa.algorithms.BayesianOptimization(num_grid_points=2) # stoppingrule = sherpa.algorithms.MedianStoppingRule(min_trials=10, # min_iterations=8) # The scheduler if not FLAGS.local: env = FLAGS.env opt = '-N MNISTPBT -P {} -q {} -l {} -l gpu=1'.format( FLAGS.P, FLAGS.q, FLAGS.l) scheduler = sherpa.schedulers.SGEScheduler(environment=env, submit_options=opt) else: scheduler = sherpa.schedulers.LocalScheduler() # Running it all sherpa.optimize(algorithm=algorithm, scheduler=scheduler, parameters=parameters, lower_is_better=True, filename="mnist_cnn.py", max_concurrent=FLAGS.max_concurrent, output_dir='./output')
sherpa.Choice('optimizer', ['adam', 'sgd', 'rmsprop']), # SETTINGS sherpa.Choice('epochs', [30]), sherpa.Choice('batch_size', [16384]), sherpa.Choice('patience', [10]), sherpa.Choice('data', [FLAGS.data]), sherpa.Choice('data_dir', ['/oasis/projects/nsf/sio134/jott1/']), sherpa.Choice('model_dir', [output_dir + 'Models/']), ] # sched = sherpa.schedulers.LocalScheduler(resources=resources) sched = sherpa.schedulers.SLURMScheduler( submit_options='run.csh', environment='/home/jott1/.bashrc', username='******' ) alg = sherpa.algorithms.RandomSearch(max_num_trials=10000) sherpa.optimize( parameters=parameters, algorithm=alg, lower_is_better=True, command='python main.py', scheduler=sched, num_distributors=FLAGS.num_distributors, num_tasks_per_distributor=FLAGS.num_tasks_per_distributor, output_dir=output_dir )
import time client = sherpa.Client() trial = client.get_trial() # Simulate model training num_iterations = 10 for i in range(num_iterations): pseudo_objective = trial.parameters['param_a'] / float(i + 1) * trial.parameters['param_b'] time.sleep(1) client.send_metrics(trial=trial, iteration=i+1, objective=pseudo_objective) # print("Trial {} Iteration {}.".format(trial.id, i+1)) # print("Trial {} finished.".format(trial.id)) """ filename = os.path.join(tempdir, "test.py") with open(filename, 'w') as f: f.write(testscript) results = sherpa.optimize(parameters=parameters, algorithm=algorithm, lower_is_better=True, filename=filename, output_dir=tempdir, scheduler=scheduler, max_concurrent=4, verbose=1) print(results)
sherpa.Continuous(name='top_dropout', range=[0.0001, 0.7]), sherpa.Continuous(name='lr_decay', range=[1e-4, 1e-9], scale='log') ] # parameters = [sherpa.Choice(name='lr', range=[0.09]), # sherpa.Choice(name='dropout', range=[0.2]), # sherpa.Choice(name='lr_decay', range=[1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9])] algorithm = sherpa.algorithms.RandomSearch(max_num_trials=100, repeat=25) # algorithm = sherpa.algorithms.GridSearch() # The scheduler # if not FLAGS.local: # env = FLAGS.env # opt = '-N MNIST-H1 -P {} -q {} -l {} -l gpu=1'.format(FLAGS.P, FLAGS.q, FLAGS.l) # scheduler = sherpa.schedulers.SGEScheduler(environment=env, submit_options=opt) # else: resources = [int(x) for x in FLAGS.gpus.split(',')] * 3 # resources = [0,0,0,1,1,1,2,2,2,3,3,3] scheduler = sherpa.schedulers.LocalScheduler(resources=resources) # Running it all sherpa.optimize(algorithm=algorithm, scheduler=scheduler, parameters=parameters, lower_is_better=True, filename="mnist_cnn.py", max_concurrent=FLAGS.max_concurrent, output_dir='./output_mnistcnn_unshuffled_{}_{}_gpu-{}'.format( time.strftime("%Y-%m-%d--%H-%M-%S"), FLAGS.name, FLAGS.gpus.replace(',', '-')))
sherpa.Choice('lr', [1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6]), # sherpa.Choice('activation', ['relu', 'prelu', 'elu', 'leaky_relu', 'sigmoid']), # sherpa.Choice('kernel_initializer', ['glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform']), ] for k,v in vars(FLAGS).iteritems(): parameters.append( sherpa.Choice(k, [v]) ) # Run on local machine. gpus = [int(x) for x in FLAGS.gpus.split(',')] processes_per_gpu = FLAGS.max_concurrent//len(gpus) assert FLAGS.max_concurrent%len(gpus) == 0 resources = list(itertools.chain.from_iterable(itertools.repeat(x, processes_per_gpu) for x in gpus)) sched = sherpa.schedulers.LocalScheduler(resources=resources) alg = sherpa.algorithms.RandomSearch(max_num_trials=200) build_directory('SherpaResults/' + FLAGS.dataset + '/Models') sherpa.optimize( parameters=parameters, algorithm=alg, lower_is_better=True, command='python hp_main.py', scheduler=sched, max_concurrent=FLAGS.max_concurrent, output_dir='SherpaResults/' + FLAGS.dataset +'/' )
# USING KERAS DATA GENERATOR FOR AUG sherpa.Choice('data_aug', [1]), sherpa.Choice('normalize', [1]), # LOSS PENALTY FOR CLASS IMBALANCE sherpa.Choice('class_weight', [0, 1]), ] for k, v in vars(FLAGS).items(): parameters.append(sherpa.Choice(k, [v])) # Run on local machine. gpus = [int(x) for x in FLAGS.gpus.split(',')] processes_per_gpu = FLAGS.max_concurrent // len(gpus) assert FLAGS.max_concurrent % len(gpus) == 0 resources = list( itertools.chain.from_iterable( itertools.repeat(x, processes_per_gpu) for x in gpus)) sched = sherpa.schedulers.LocalScheduler(resources=resources) alg = sherpa.algorithms.RandomSearch(max_num_trials=1000) # alg = sherpa.algorithms.GPyOpt(max_num_trials=1000) sherpa.optimize(parameters=parameters, algorithm=alg, lower_is_better=False, command='/pkg/python/3.6.1-centos7/bin/python3.6 main.py', scheduler=sched, max_concurrent=FLAGS.max_concurrent, output_dir='SherpaResults')