def test_hyperparams_repository_should_load_all_trials(tmpdir):
tmpdir = os.path.join(tmpdir, "__json__")
os.mkdir(tmpdir)
hyperparams_json_repository = HyperparamsJSONRepository(tmpdir)
n_trials = 3
for i in range(n_trials):
hyperparams = HyperparameterSamples({'learning_rate': 0.01 + i * 0.01})
hyperparams_json_repository.save_score_for_success_trial(hyperparams, i)
trials = hyperparams_json_repository.load_all_trials()
assert len(trials) == n_trials
for i in range(n_trials):
assert trials[i].hyperparams == HyperparameterSamples(
{'learning_rate': 0.01 + i * 0.01}).to_flat_as_dict_primitive(), (i, str(trials))
def test_hyperparams_repository_should_load_all_trials(tmpdir):
hyperparams_json_repository = HyperparamsJSONRepository(tmpdir)
for i in range(2):
hyperparams = HyperparameterSamples({'learning_rate': 0.01 + i * 0.01})
hyperparams_json_repository.save_score_for_success_trial(
hyperparams, i)
trials = hyperparams_json_repository.load_all_trials()
assert len(trials) == 2
assert trials[0].hyperparams == HyperparameterSamples({
'learning_rate':
0.01 + 0 * 0.01
}).to_flat_as_dict_primitive()
assert trials[1].hyperparams == HyperparameterSamples({
'learning_rate':
0.01 + 1 * 0.01
}).to_flat_as_dict_primitive()
def test_automl_savebestmodel_callback(tmpdir):
# Given
hp_repository = HyperparamsJSONRepository(cache_folder=str('caching'))
validation_splitter = ValidationSplitter(0.20)
auto_ml = AutoML(
pipeline=Pipeline([
MultiplyByN(2).set_hyperparams_space(HyperparameterSpace({
'multiply_by': FixedHyperparameter(2)
})),
NumpyReshape(new_shape=(-1, 1)),
linear_model.LinearRegression()
]),
validation_splitter=validation_splitter,
hyperparams_optimizer=RandomSearchHyperparameterSelectionStrategy(),
scoring_callback=ScoringCallback(mean_squared_error, higher_score_is_better=False),
callbacks=[
BestModelCheckpoint()
],
n_trials=1,
epochs=10,
refit_trial=False,
print_func=print,
hyperparams_repository=hp_repository,
continue_loop_on_error=False
)
data_inputs = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
expected_outputs = data_inputs * 4
# When
auto_ml.fit(data_inputs=data_inputs, expected_outputs=expected_outputs)
#Then
trials: Trials = hp_repository.load_all_trials()
best_trial = trials.get_best_trial()
best_trial_score = best_trial.get_validation_score()
best_trial.cache_folder = hp_repository.cache_folder
best_model = best_trial.get_model('best')
_, _, valid_inputs, valid_outputs = ValidationSplitter(0.20).split(data_inputs, expected_outputs)
predicted_output = best_model.predict(valid_inputs)
score = mean_squared_error(valid_outputs, predicted_output)
assert best_trial_score == score