def get_tuner(config: TaskConfig):
# Users may add their customized Tuners here
if config.framework_params['tuner_type'] == 'tpe':
return HyperoptTuner('tpe'), 'TPE Tuner'
elif config.framework_params['tuner_type'] == 'random_search':
return HyperoptTuner('random_search'), 'Random Search Tuner'
elif config.framework_params['tuner_type'] == 'anneal':
return HyperoptTuner('anneal'), 'Annealing Tuner'
elif config.framework_params['tuner_type'] == 'evolution':
return EvolutionTuner(), 'Evolution Tuner'
elif config.framework_params['tuner_type'] == 'smac':
return SMACTuner(), 'SMAC Tuner'
elif config.framework_params['tuner_type'] == 'gp':
return GPTuner(), 'GP Tuner'
elif config.framework_params['tuner_type'] == 'metis':
return MetisTuner(), 'Metis Tuner'
elif config.framework_params['tuner_type'] == 'hyperband':
if 'max_resource' in config.framework_params:
tuner = Hyperband(R=config.framework_params['max_resource'])
else:
tuner = Hyperband()
return tuner, 'Hyperband Advisor'
elif config.framework_params['tuner_type'] == 'bohb':
if 'max_resource' in config.framework_params:
tuner = BOHB(max_budget=config.framework_params['max_resource'])
else:
tuner = BOHB(max_budget=60)
return tuner, 'BOHB Advisor'
else:
raise RuntimeError('The requested tuner type in framework.yaml is unavailable.')
def test_evolution(self):
# Needs enough population size, otherwise it will throw a runtime error
tuner_fn = lambda: EvolutionTuner(population_size=100)
self.search_space_test_all(tuner_fn)
self.import_data_test(tuner_fn)
def run(dataset: Dataset, config: TaskConfig):
log.info(
"\n**** Random Forest (sklearn) Tuned with NNI EvolutionTuner ****\n")
is_classification = config.type == 'classification'
X_train, X_test = impute(dataset.train.X, dataset.test.X)
y_train, y_test = dataset.train.y, dataset.test.y
estimator = RandomForestClassifier if is_classification else RandomForestRegressor
# model = estimator(random_state=config.seed, **config.framework_params)
best_score, best_params, best_model = None, None, None
score_higher_better = True
log.info(
"Tuning hyperparameters with NNI EvolutionTuner with a maximum time of {}s\n"
.format(config.max_runtime_seconds))
tuner = EvolutionTuner()
tuner.update_search_space(SEARCH_SPACE)
start_time = time.time()
param_idx = 0
while True:
try:
cur_params = tuner.generate_parameters(param_idx)
cur_model = estimator(random_state=config.seed,
**cur_params,
**config.framework_params)
# Here score is the output of score() from the estimator
cur_score = cross_val_score(cur_model, X_train, y_train)
cur_score = sum(cur_score) / float(len(cur_score))
if best_score is None or (score_higher_better
and cur_score > best_score) or (
not score_higher_better
and cur_score < best_score):
best_score, best_params, best_model = cur_score, cur_params, cur_model
log.info("Trial {}: \n{}\nScore: {}\n".format(
param_idx, cur_params, cur_score))
tuner.receive_trial_result(param_idx, cur_params, cur_score)
param_idx += 1
current_time = time.time()
elapsed_time = current_time - start_time
if elapsed_time > config.max_runtime_seconds:
break
except:
break
log.info("Tuning done, the best parameters are:\n{}\n".format(best_params))
# retrain on the whole dataset
with Timer() as training:
best_model.fit(X_train, y_train)
predictions = best_model.predict(X_test)
probabilities = best_model.predict_proba(
X_test) if is_classification else None
save_predictions_to_file(dataset=dataset,
output_file=config.output_predictions_file,
probabilities=probabilities,
predictions=predictions,
truth=y_test)
return dict(models_count=1, training_duration=training.duration)