class TPESampler(Sampler):
def __init__(self, optimize_mode='minimize'):
self.tpe_tuner = HyperoptTuner('tpe', optimize_mode)
self.cur_sample = None
self.index = None
self.total_parameters = {}
def update_sample_space(self, sample_space):
search_space = {}
for i, each in enumerate(sample_space):
search_space[str(i)] = {'_type': 'choice', '_value': each}
self.tpe_tuner.update_search_space(search_space)
def generate_samples(self, model_id):
self.cur_sample = self.tpe_tuner.generate_parameters(model_id)
self.total_parameters[model_id] = self.cur_sample
self.index = 0
def receive_result(self, model_id, result):
self.tpe_tuner.receive_trial_result(model_id,
self.total_parameters[model_id],
result)
def choice(self, candidates, mutator, model, index):
chosen = self.cur_sample[str(self.index)]
self.index += 1
return chosen
class TPESampler(Sampler):
def __init__(self, optimize_mode='minimize'):
# Move import here to eliminate some warning messages about dill.
from nni.algorithms.hpo.hyperopt_tuner import HyperoptTuner
self.tpe_tuner = HyperoptTuner('tpe', optimize_mode)
self.cur_sample: Optional[dict] = None
self.index: Optional[int] = None
self.total_parameters = {}
def update_sample_space(self, sample_space):
search_space = {}
for i, each in enumerate(sample_space):
search_space[str(i)] = {'_type': 'choice', '_value': each}
self.tpe_tuner.update_search_space(search_space)
def generate_samples(self, model_id):
self.cur_sample = self.tpe_tuner.generate_parameters(model_id)
self.total_parameters[model_id] = self.cur_sample
self.index = 0
def receive_result(self, model_id, result):
self.tpe_tuner.receive_trial_result(model_id,
self.total_parameters[model_id],
result)
def choice(self, candidates, mutator, model, index):
assert isinstance(self.index, int) and isinstance(
self.cur_sample, dict)
chosen = self.cur_sample[str(self.index)]
self.index += 1
return chosen
def __init__(self, optimize_mode='minimize'):
# Move import here to eliminate some warning messages about dill.
from nni.algorithms.hpo.hyperopt_tuner import HyperoptTuner
self.tpe_tuner = HyperoptTuner('tpe', optimize_mode)
self.cur_sample: Optional[dict] = None
self.index: Optional[int] = None
self.total_parameters = {}
def test_tpe(self):
tuner_fn = lambda: HyperoptTuner("tpe")
self.search_space_test_all(tuner_fn,
ignore_types=[
"uniform_equal", "qloguniform_equal",
"loguniform_equal", "quniform_clip_2"
])
# NOTE: types are ignored because `tpe.py line 465, in adaptive_parzen_normal assert prior_sigma > 0`
self.import_data_test(tuner_fn)
def get_tuner(config: TaskConfig):
# Users may add their customized Tuners here
if config.framework_params['tuner_type'] == 'tpe':
return HyperoptTuner('tpe'), 'TPE Tuner'
elif config.framework_params['tuner_type'] == 'random_search':
return HyperoptTuner('random_search'), 'Random Search Tuner'
elif config.framework_params['tuner_type'] == 'anneal':
return HyperoptTuner('anneal'), 'Annealing Tuner'
elif config.framework_params['tuner_type'] == 'evolution':
return EvolutionTuner(), 'Evolution Tuner'
elif config.framework_params['tuner_type'] == 'smac':
return SMACTuner(), 'SMAC Tuner'
elif config.framework_params['tuner_type'] == 'gp':
return GPTuner(), 'GP Tuner'
elif config.framework_params['tuner_type'] == 'metis':
return MetisTuner(), 'Metis Tuner'
elif config.framework_params['tuner_type'] == 'hyperband':
if 'max_resource' in config.framework_params:
tuner = Hyperband(R=config.framework_params['max_resource'])
else:
tuner = Hyperband()
return tuner, 'Hyperband Advisor'
elif config.framework_params['tuner_type'] == 'bohb':
if 'max_resource' in config.framework_params:
tuner = BOHB(max_budget=config.framework_params['max_resource'])
else:
tuner = BOHB(max_budget=60)
return tuner, 'BOHB Advisor'
else:
raise RuntimeError('The requested tuner type in framework.yaml is unavailable.')
def test_tuner_generate(self):
for algorithm in ["tpe", "random_search", "anneal"]:
tuner = HyperoptTuner(algorithm)
choice_list = ["a", "b", 1, 2]
tuner.update_search_space({
"a": {
"_type": "randint",
"_value": [1, 3]
},
"b": {
"_type": "choice",
"_value": choice_list
}
})
for k in range(30):
# sample multiple times
param = tuner.generate_parameters(k)
print(param)
self.assertIsInstance(param["a"], int)
self.assertGreaterEqual(param["a"], 1)
self.assertLessEqual(param["a"], 2)
self.assertIn(param["b"], choice_list)
# FIXME: For demonstration only. It should not be here
from pathlib import Path
from nni.experiment import Experiment
from nni.algorithms.hpo.hyperopt_tuner import HyperoptTuner
tuner = HyperoptTuner('tpe')
search_space = {
"dropout_rate": { "_type": "uniform", "_value": [0.5, 0.9] },
"conv_size": { "_type": "choice", "_value": [2, 3, 5, 7] },
"hidden_size": { "_type": "choice", "_value": [124, 512, 1024] },
"batch_size": { "_type": "choice", "_value": [16, 32] },
"learning_rate": { "_type": "choice", "_value": [0.0001, 0.001, 0.01, 0.1] }
}
experiment = Experiment(tuner, 'local')
experiment.config.experiment_name = 'test'
experiment.config.trial_concurrency = 2
experiment.config.max_trial_number = 5
experiment.config.search_space = search_space
experiment.config.trial_command = 'python3 mnist.py'
experiment.config.trial_code_directory = Path(__file__).parent
experiment.config.training_service.use_active_gpu = True
experiment.run(8081)
def test_anneal(self):
tuner_fn = lambda: HyperoptTuner("anneal")
self.search_space_test_all(tuner_fn)
self.import_data_test(tuner_fn)
def test_random_search(self):
tuner_fn = lambda: HyperoptTuner("random_search")
self.search_space_test_all(tuner_fn)
self.import_data_test(tuner_fn)
def __init__(self, optimize_mode='minimize'):
self.tpe_tuner = HyperoptTuner('tpe', optimize_mode)
self.cur_sample = None
self.index = None
self.total_parameters = {}
from pathlib import Path
from nni.experiment import Experiment, RemoteMachineConfig
from nni.algorithms.hpo.hyperopt_tuner import HyperoptTuner
tuner = HyperoptTuner('tpe', optimize_mode='maximize')
search_space = {
"learning_rate": {
"_type": "loguniform",
"_value": [1e-2, 20]
},
"output_dropout": {
"_type": "uniform",
"_value": [0.3, 0.6]
},
"hidden_dropout": {
"_type": "uniform",
"_value": [0.4, 0.6]
},
"input_dropout": {
"_type": "uniform",
"_value": [0.5, 0.7]
},
"embedding_dropout": {
"_type": "uniform",
"_value": [0, 0.5]
},
"weight_dropout": {
"_type": "uniform",
"_value": [0.5, 0.8]