def test_get_suggestions_calls_to_numpy(self):
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'grid_search': {'n_experiments': 10},
'matrix': {'feature': {'values': [1, 2, 3]}}
})
manager = GridSearchManager(hptuning_config=hptuning_config)
with patch.object(MatrixConfig, 'to_numpy') as to_numpy_mock:
manager.get_suggestions()
assert to_numpy_mock.call_count == 1
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'grid_search': {'n_experiments': 10},
'matrix': {
'feature1': {'values': [1, 2, 3]},
'feature2': {'logspace': '0.01:0.1:5'}
}
})
manager = GridSearchManager(hptuning_config=hptuning_config)
with patch.object(MatrixConfig, 'to_numpy') as to_numpy_mock:
manager.get_suggestions()
assert to_numpy_mock.call_count == 2
def test_get_suggestions_calls_sample(self):
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'random_search': {'n_experiments': 1},
'matrix': {
'feature1': {'values': [1, 2, 3]},
'feature2': {'linspace': [1, 2, 5]},
'feature3': {'range': [1, 5, 1]}
}
})
manager = RandomSearchManager(hptuning_config=hptuning_config)
with patch.object(MatrixConfig, 'sample') as sample_mock:
manager.get_suggestions()
assert sample_mock.call_count == 3
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'random_search': {'n_experiments': 1},
'matrix': {
'feature1': {'pvalues': [(1, 0.3), (2, 0.3), (3, 0.3)]},
'feature2': {'uniform': [0, 1]},
'feature3': {'qlognormal': [0, 0.5, 0.51]},
'feature4': {'range': [1, 5, 1]}
}
})
manager = RandomSearchManager(hptuning_config=hptuning_config)
with patch.object(MatrixConfig, 'sample') as sample_mock:
manager.get_suggestions()
assert sample_mock.call_count == 4
def test_get_suggestions(self):
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'grid_search': {
'n_experiments': 10
},
'matrix': {
'feature': {
'values': [1, 2, 3]
}
}
})
manager = GridSearchManager(hptuning_config=hptuning_config)
assert len(manager.get_suggestions()) == 3
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'grid_search': {
'n_experiments': 10
},
'matrix': {
'feature1': {
'values': [1, 2, 3]
},
'feature2': {
'linspace': [1, 2, 5]
},
'feature3': {
'range': [1, 5, 1]
}
}
})
manager = GridSearchManager(hptuning_config=hptuning_config)
assert len(manager.get_suggestions()) == 10
def setUp(self):
super().setUp()
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'bo': {
'n_iterations': 5,
'n_initial_trials': 5,
'metric': {
'name': 'loss',
'optimization': 'minimize'
},
'utility_function': {
'acquisition_function': 'ucb',
'kappa': 1.2,
'gaussian_process': {
'kernel': 'matern',
'length_scale': 1.0,
'nu': 1.9,
'n_restarts_optimizer': 0
}
}
},
'matrix': {
'feature1': {'values': [1, 2, 3]},
'feature2': {'linspace': [1, 2, 5]},
'feature3': {'range': [1, 5, 1]}
}
})
self.manager1 = BOSearchManager(hptuning_config=hptuning_config)
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'bo': {
'n_iterations': 4,
'n_initial_trials': 4,
'metric': {
'name': 'accuracy',
'optimization': 'maximize'
},
'utility_function': {
'acquisition_function': 'ei',
'eps': 1.2,
'gaussian_process': {
'kernel': 'matern',
'length_scale': 1.0,
'nu': 1.9,
'n_restarts_optimizer': 0
}
}
},
'matrix': {
'feature1': {'values': [1, 2, 3, 4, 5]},
'feature2': {'linspace': [1, 5, 5]},
'feature3': {'range': [1, 6, 1]},
'feature4': {'uniform': [1, 5]},
'feature5': {'values': ['a', 'b', 'c']},
}
})
self.manager2 = BOSearchManager(hptuning_config=hptuning_config)
def validate_group_hptuning_config(config, raise_for_rest: bool = False):
try:
HPTuningConfig.from_dict(config)
except MarshmallowValidationError as e:
if raise_for_rest:
raise ValidationError(e)
else:
raise DjangoValidationError(e)
def test_get_suggestions(self):
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'random_search': {
'n_experiments': 10
},
'matrix': {
'feature1': {
'values': [1, 2]
},
'feature3': {
'range': [1, 3, 1]
}
}
})
manager = RandomSearchManager(hptuning_config=hptuning_config)
assert len(manager.get_suggestions()) == 4
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'random_search': {
'n_experiments': 10
},
'matrix': {
'feature1': {
'values': [1, 2, 3]
},
'feature2': {
'linspace': [1, 2, 5]
},
'feature3': {
'range': [1, 5, 1]
}
}
})
manager = RandomSearchManager(hptuning_config=hptuning_config)
assert len(manager.get_suggestions()) == 10
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'random_search': {
'n_experiments': 10
},
'matrix': {
'feature1': {
'pvalues': [(1, 0.3), (2, 0.3), (3, 0.3)]
},
'feature2': {
'uniform': [0, 1]
},
'feature3': {
'qlognormal': [0, 0.5, 0.51]
}
}
})
manager = RandomSearchManager(hptuning_config=hptuning_config)
assert len(manager.get_suggestions()) == 10
def setUp(self):
super().setUp()
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'hyperband': {
'max_iter': 10,
'eta': 3,
'resource': {'name': 'steps', 'type': 'float'},
'resume': False,
'metric': {'name': 'loss', 'optimization': 'minimize'}
},
'matrix': {
'feature1': {'values': [1, 2, 3]},
'feature2': {'linspace': [1, 2, 5]},
'feature3': {'range': [1, 5, 1]}
}
})
self.manager1 = HyperbandSearchManager(hptuning_config=hptuning_config)
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'hyperband': {
'max_iter': 81,
'eta': 3,
'resource': {'name': 'size', 'type': 'int'},
'resume': False,
'metric': {'name': 'loss', 'optimization': 'minimize'}
},
'matrix': {
'feature1': {'values': [1, 2, 3]},
'feature2': {'linspace': [1, 2, 5]},
'feature3': {'range': [1, 5, 1]},
'feature4': {'range': [1, 5, 1]}
}
})
self.manager2 = HyperbandSearchManager(hptuning_config=hptuning_config)
def hptuning_config(self) -> Optional['HPTuningConfig']:
return HPTuningConfig.from_dict(
self.hptuning) if self.hptuning else None
def test_concrete_example(self):
hptuning_config = HPTuningConfig.from_dict({
'concurrency': 2,
'bo': {
'n_iterations': 5,
'n_initial_trials': 10,
'metric': {
'name': 'loss',
'optimization': 'minimize'
},
'utility_function': {
'acquisition_function': 'ucb',
'kappa': 2.576,
'gaussian_process': {
'kernel': 'matern',
'length_scale': 1.0,
'nu': 1.9,
'n_restarts_optimizer': 0
},
'n_warmup': 1,
'n_iter': 1
}
},
'matrix': {
'learning_rate': {'uniform': [0.001, 0.01]},
'dropout': {'values': [0.25, 0.3]},
'activation': {'values': ['relu', 'sigmoid']}
}
})
optimizer = BOOptimizer(hptuning_config=hptuning_config)
configs = [
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.004544653508229265,
"activation": "sigmoid",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.005615296199690899,
"activation": "sigmoid",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.008784330869587902,
"activation": "sigmoid",
"dropout": 0.25
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.0058591075447430065,
"activation": "sigmoid",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.007464080062927171,
"activation": "sigmoid",
"dropout": 0.25
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.0024763129571936738,
"activation": "relu",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.0074881581817925705,
"activation": "sigmoid",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.003360405779075163,
"activation": "relu",
"dropout": 0.3
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.009916904455792564,
"activation": "sigmoid",
"dropout": 0.25
},
{
"num_epochs": 1,
"num_steps": 300,
"batch_size": 128,
"learning_rate": 0.000881723263162717,
"activation": "sigmoid",
"dropout": 0.3
}
]
metrics = [
2.3018131256103516,
2.302884340286255,
2.3071441650390625,
2.3034636974334717,
2.301487922668457,
0.05087224021553993,
2.3032383918762207,
0.06383182853460312,
2.3120572566986084,
0.7617478370666504
]
optimizer.add_observations(configs=configs, metrics=metrics)
suggestion = optimizer.get_suggestion()
assert 0.001 <= suggestion['learning_rate'] <= 0.01
assert suggestion['dropout'] in [0.25, 0.3]
assert suggestion['activation'] in ['relu', 'sigmoid']
def hptuning_config(self):
return HPTuningConfig.from_dict(
self.hptuning) if self.hptuning else None
