def test_optimizer_add_observations_calls_space_add_observations(self):
optimizer = BOOptimizer(hptuning_config=self.manager1.hptuning_config)
with patch.object(SearchSpace,
'add_observations') as add_observations_mock:
optimizer.add_observations(configs=[], metrics=[])
assert add_observations_mock.call_count == 1
def get_suggestions(self, iteration_config=None):
if not iteration_config:
return get_random_suggestions(matrix=self.hptuning_config.matrix,
n_suggestions=self.n_initial_trials,
seed=self.hptuning_config.seed)
# Use the iteration_config to construct observed point and metrics
experiments_configs = dict(
iteration_config.combined_experiments_configs)
experiments_metrics = dict(
iteration_config.combined_experiments_metrics)
configs = []
metrics = []
for key in experiments_metrics.keys():
configs.append(experiments_configs[key])
metrics.append(experiments_metrics[key])
optimizer = BOOptimizer(hptuning_config=self.hptuning_config)
optimizer.add_observations(configs=configs, metrics=metrics)
suggestion = optimizer.get_suggestion()
return [suggestion] if suggestion else None
def test_concrete_example(self):
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'bo': {
'n_iterations': 5,
'n_initial_trials': 10,
'metric': {
'name': 'loss',
'optimization': 'minimize'
},
'utility_function': {
'acquisition_function': 'ucb',
'kappa': 2.576,
'gaussian_process': {
'kernel': 'matern',
'length_scale': 1.0,
'nu': 1.9,
'n_restarts_optimizer': 0
},
'n_warmup': 1,
'n_iter': 1
}
},
'matrix': {
'learning_rate': {'uniform': [0.001, 0.01]},
'dropout': {'values': [0.25, 0.3]},
'activation': {'values': ['relu', 'sigmoid']}
}
})
optimizer = BOOptimizer(hptuning_config=hptuning_config)
configs = [
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.004544653508229265,
"activation": "sigmoid",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.005615296199690899,
"activation": "sigmoid",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.008784330869587902,
"activation": "sigmoid",
"dropout": 0.25
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.0058591075447430065,
"activation": "sigmoid",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.007464080062927171,
"activation": "sigmoid",
"dropout": 0.25
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.0024763129571936738,
"activation": "relu",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.0074881581817925705,
"activation": "sigmoid",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.003360405779075163,
"activation": "relu",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.009916904455792564,
"activation": "sigmoid",
"dropout": 0.25
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.000881723263162717,
"activation": "sigmoid",
"dropout": 0.3
}
]
metrics = [
2.3018131256103516,
2.302884340286255,
2.3071441650390625,
2.3034636974334717,
2.301487922668457,
0.05087224021553993,
2.3032383918762207,
0.06383182853460312,
2.3120572566986084,
0.7617478370666504
]
optimizer.add_observations(configs=configs, metrics=metrics)
suggestion = optimizer.get_suggestion()
assert 0.001 <= suggestion['learning_rate'] <= 0.01
assert suggestion['dropout'] in [0.25, 0.3]
assert suggestion['activation'] in ['relu', 'sigmoid']
def test_optimizer_get_suggestion(self):
# Manager 1
optimizer1 = BOOptimizer(hptuning_config=self.manager1.hptuning_config)
optimizer1.N_ITER = 1
optimizer1.N_WARMUP = 1
configs = [
{'feature1': 1, 'feature2': 1, 'feature3': 1},
{'feature1': 1, 'feature2': 1.2, 'feature3': 1},
{'feature1': 1, 'feature2': 1, 'feature3': 1},
{'feature1': 1, 'feature2': 1.11, 'feature3': 1},
{'feature1': 1, 'feature2': 1.1, 'feature3': 1},
{'feature1': 1, 'feature2': 1.21, 'feature3': 1},
{'feature1': 2, 'feature2': 2, 'feature3': 2},
{'feature1': 3, 'feature2': 2, 'feature3': 2},
{'feature1': 2, 'feature2': 1.8, 'feature3': 3},
{'feature1': 3, 'feature2': 2, 'feature3': 3},
{'feature1': 2, 'feature2': 2, 'feature3': 2},
{'feature1': 3, 'feature2': 2, 'feature3': 2},
{'feature1': 2, 'feature2': 1.8, 'feature3': 3},
{'feature1': 3, 'feature2': 2, 'feature3': 3}
]
metrics = [0, 1.1, 0.1, 0.1, 1.09, 0.4, 100, 200, 200, 300, 110, 210, 210, 310]
optimizer1.add_observations(configs=configs, metrics=metrics)
suggestion = optimizer1.get_suggestion()
assert 1 <= suggestion['feature1'] <= 3
assert 1 <= suggestion['feature2'] <= 2
assert 1 <= suggestion['feature3'] <= 5
# Manager 2
optimizer2 = BOOptimizer(hptuning_config=self.manager2.hptuning_config)
optimizer2.N_ITER = 1
optimizer2.N_WARMUP = 1
configs = [
{'feature1': 1, 'feature2': 1, 'feature3': 1, 'feature4': 1, 'feature5': 'a'},
{'feature1': 2, 'feature2': 1.2, 'feature3': 2, 'feature4': 4, 'feature5': 'b'},
{'feature1': 3, 'feature2': 1.3, 'feature3': 3, 'feature4': 3, 'feature5': 'a'}
]
metrics = [1, 2, 3]
optimizer2.add_observations(configs=configs, metrics=metrics)
suggestion = optimizer2.get_suggestion()
assert 1 <= suggestion['feature1'] <= 5
assert 1 <= suggestion['feature2'] <= 5
assert 1 <= suggestion['feature3'] <= 6
assert 1 <= suggestion['feature4'] <= 5
assert suggestion['feature5'] in ['a', 'b', 'c']
def test_concrete_example(self):
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'bo': {
'n_iterations': 5,
'n_initial_trials': 10,
'metric': {
'name': 'loss',
'optimization': 'minimize'
},
'utility_function': {
'acquisition_function': 'ucb',
'kappa': 2.576,
'gaussian_process': {
'kernel': 'matern',
'length_scale': 1.0,
'nu': 1.9,
'n_restarts_optimizer': 0
},
'n_warmup': 1,
'n_iter': 1
}
},
'matrix': {
'learning_rate': {'uniform': [0.001, 0.01]},
'dropout': {'values': [0.25, 0.3]},
'activation': {'values': ['relu', 'sigmoid']}
}
})
optimizer = BOOptimizer(hptuning_config=hptuning_config)
configs = [
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.004544653508229265,
"activation": "sigmoid",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.005615296199690899,
"activation": "sigmoid",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.008784330869587902,
"activation": "sigmoid",
"dropout": 0.25
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.0058591075447430065,
"activation": "sigmoid",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.007464080062927171,
"activation": "sigmoid",
"dropout": 0.25
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.0024763129571936738,
"activation": "relu",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.0074881581817925705,
"activation": "sigmoid",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.003360405779075163,
"activation": "relu",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.009916904455792564,
"activation": "sigmoid",
"dropout": 0.25
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.000881723263162717,
"activation": "sigmoid",
"dropout": 0.3
}
]
metrics = [
2.3018131256103516,
2.302884340286255,
2.3071441650390625,
2.3034636974334717,
2.301487922668457,
0.05087224021553993,
2.3032383918762207,
0.06383182853460312,
2.3120572566986084,
0.7617478370666504
]
optimizer.add_observations(configs=configs, metrics=metrics)
suggestion = optimizer.get_suggestion()
assert 0.001 <= suggestion['learning_rate'] <= 0.01
assert suggestion['dropout'] in [0.25, 0.3]
assert suggestion['activation'] in ['relu', 'sigmoid']
def test_optimizer_get_suggestion(self):
# Manager 1
optimizer1 = BOOptimizer(hptuning_config=self.manager1.hptuning_config)
optimizer1.N_ITER = 1
optimizer1.N_WARMUP = 1
configs = [
{'feature1': 1, 'feature2': 1, 'feature3': 1},
{'feature1': 1, 'feature2': 1.2, 'feature3': 1},
{'feature1': 1, 'feature2': 1, 'feature3': 1},
{'feature1': 1, 'feature2': 1.11, 'feature3': 1},
{'feature1': 1, 'feature2': 1.1, 'feature3': 1},
{'feature1': 1, 'feature2': 1.21, 'feature3': 1},
{'feature1': 2, 'feature2': 2, 'feature3': 2},
{'feature1': 3, 'feature2': 2, 'feature3': 2},
{'feature1': 2, 'feature2': 1.8, 'feature3': 3},
{'feature1': 3, 'feature2': 2, 'feature3': 3},
{'feature1': 2, 'feature2': 2, 'feature3': 2},
{'feature1': 3, 'feature2': 2, 'feature3': 2},
{'feature1': 2, 'feature2': 1.8, 'feature3': 3},
{'feature1': 3, 'feature2': 2, 'feature3': 3}
]
metrics = [0, 1.1, 0.1, 0.1, 1.09, 0.4, 100, 200, 200, 300, 110, 210, 210, 310]
optimizer1.add_observations(configs=configs, metrics=metrics)
suggestion = optimizer1.get_suggestion()
assert 1 <= suggestion['feature1'] <= 3
assert 1 <= suggestion['feature2'] <= 2
assert 1 <= suggestion['feature3'] <= 5
# Manager 2
optimizer2 = BOOptimizer(hptuning_config=self.manager2.hptuning_config)
optimizer2.N_ITER = 1
optimizer2.N_WARMUP = 1
configs = [
{'feature1': 1, 'feature2': 1, 'feature3': 1, 'feature4': 1, 'feature5': 'a'},
{'feature1': 2, 'feature2': 1.2, 'feature3': 2, 'feature4': 4, 'feature5': 'b'},
{'feature1': 3, 'feature2': 1.3, 'feature3': 3, 'feature4': 3, 'feature5': 'a'}
]
metrics = [1, 2, 3]
optimizer2.add_observations(configs=configs, metrics=metrics)
suggestion = optimizer2.get_suggestion()
assert 1 <= suggestion['feature1'] <= 5
assert 1 <= suggestion['feature2'] <= 5
assert 1 <= suggestion['feature3'] <= 6
assert 1 <= suggestion['feature4'] <= 5
assert suggestion['feature5'] in ['a', 'b', 'c']
def test_optimizer_add_observations_calls_space_add_observations(self):
optimizer = BOOptimizer(hptuning_config=self.manager1.hptuning_config)
with patch.object(SearchSpace, 'add_observations') as add_observations_mock:
optimizer.add_observations(configs=[], metrics=[])
assert add_observations_mock.call_count == 1