def test_from_dict(self, mock_bool, mock_cat, mock_float, mock_int):
# setup
mock_bool.return_value.dimensions = 1
mock_cat.return_value .dimensions = 1
mock_float.return_value.dimensions = 1
mock_int.return_value.dimensions = 1
mock_bool.return_value.cardinality = 1
mock_cat.return_value .cardinality = 1
mock_float.return_value.cardinality = 1
mock_int.return_value.cardinality = 1
# run
hyperparameters = {
'bhp': {
'type': 'bool',
'default': False
},
'chp': {
'type': 'str',
'default': 'cat',
'range': ['a', 'b', 'cat']
},
'fhp': {
'type': 'float',
'default': None,
'range': [0.1, 1.0]
},
'ihp': {
'type': 'int',
'default': 5,
'range': [1, 10]
}
}
result = Tunable.from_dict(hyperparameters)
# assert
mock_bool.assert_called_once_with(default=False)
mock_cat.assert_called_once_with(choices=['a', 'b', 'cat'], default='cat')
mock_float.assert_called_once_with(min=0.1, max=1.0, default=None)
mock_int.assert_called_once_with(min=1, max=10, default=5)
expected_tunable_hp = {
'bhp': mock_bool.return_value,
'chp': mock_cat.return_value,
'fhp': mock_float.return_value,
'ihp': mock_int.return_value
}
assert result.hyperparams == expected_tunable_hp
assert result.dimensions == 4
assert result.cardinality == 1
def _tuning_function(tuner_class, scoring_function, tunable_hyperparameters,
iterations):
tunable = Tunable.from_dict(tunable_hyperparameters)
tuner = tuner_class(tunable)
best_score = -np.inf
for _ in range(iterations):
proposal = tuner.propose()
score = scoring_function(**proposal)
tuner.record(proposal, score)
best_score = max(score, best_score)
return best_score
def propose(self):
"""Propose a new configuration to score.
Every time ``propose`` is called, a new tunable will be selected and a new
hyperparameter proposal will be generated for it.
At the begining, the default hyperparameters of each one of the tunables
will be returned sequencially in the same order as they were passed to
the ``BTBSession``.
After that, once each tunable has been scored at least once, the tunable
used to generate the new proposals will be selected optimally each time
by the selector.
If a tunable runs out of proposals, it will be discarded from the list and will
not be proposed again.
Finally, when all the tunables have ran out of proposals, a ``StopTuning`` exception
will be raised.
Returns:
tuple (str, dict):
* Name of the tunable to try next.
* Hyperparameters proposal.
Raises:
StopTuning:
If the ``BTBSession`` has run out of proposals to generate.
"""
if not self._tunables:
raise StopTuning('There are no tunables left to try.')
if len(self._tuners) < len(self._tunable_names):
tunable_name = self._tunable_names[len(self._tuners)]
tunable = self._tunables[tunable_name]
if isinstance(tunable, dict):
LOGGER.info('Creating Tunable instance from dict.')
tunable = Tunable.from_dict(tunable)
if not isinstance(tunable, Tunable):
raise TypeError(
'Tunable can only be an instance of btb.tuning.Tunable or dict'
)
LOGGER.info('Obtaining default configuration for %s', tunable_name)
config = tunable.get_defaults()
if tunable.cardinality == 1:
LOGGER.warn(
'Skipping tuner creation for Tunable %s with cardinality 1',
tunable_name)
tuner = None
else:
tuner = self._tuner_class(tunable)
self._tuners[tunable_name] = tuner
else:
tunable_name = self._get_next_tunable_name()
tuner = self._tuners[tunable_name]
try:
if tuner is None:
raise StopTuning(
'Tunable %s has no tunable hyperparameters',
tunable_name)
LOGGER.info('Generating new proposal configuration for %s',
tunable_name)
config = tuner.propose(1)
except StopTuning:
LOGGER.info('%s has no more configs to propose.', tunable_name)
self._remove_tunable(tunable_name)
tunable_name, config = self.propose()
proposal_id = self._make_id(tunable_name, config)
self.proposals[proposal_id] = {
'id': proposal_id,
'name': tunable_name,
'config': config
}
return tunable_name, config
def tuning_function(scoring_function, tunable_hyperparameters, iterations):
tunable = Tunable.from_dict(tunable_hyperparameters)
tuner = tuner_class(tunable, **tuner_kwargs)
return tune(tuner, scoring_function, iterations)
def propose(self):
"""Propose a new configuration for a tunable.
``BTBSession``, ensures that every tunable has been scored atleast once. The
following proposals use the ``self.selector`` in order to select the ``tunable``
from which a proposal is generated.
If the ``tuner`` can not propose more configurations it will return
``None`` and will remove the ``tunable`` from the list.
Returns:
tuple (str, dict):
Returns a tuple with the name of the tunable and the proposal as a dictionary.
None:
``None`` is being returned When the ``tunable`` has no more combinations to be
evaluated.
Raises:
ValueError:
A ``ValueErorr`` is being raised if ``self.tunables`` is empty.
"""
if not self.tunables:
raise ValueError('All the tunables failed.')
if len(self._normalized_scores) < len(self._tunable_names):
tunable_name = self._tunable_names[len(self._normalized_scores)]
tunable = self.tunables[tunable_name]
if isinstance(tunable, dict):
LOGGER.info('Creating Tunable instance from dict.')
tunable = Tunable.from_dict(tunable)
if not isinstance(tunable, Tunable):
raise TypeError(
'Tunable can only be an instance of btb.tuning.Tunable or dict'
)
LOGGER.info('Obtaining default configuration for %s', tunable_name)
config = tunable.get_defaults()
self._tuners[tunable_name] = self.tuner(tunable)
else:
tunable_name = self.selector.select(self._normalized_scores)
tuner = self._tuners[tunable_name]
try:
LOGGER.info('Generating new proposal configuration for %s',
tunable_name)
config = tuner.propose(1)
except StopTuning:
LOGGER.info('%s has no more configs to propose.' %
tunable_name)
self._normalized_scores.pop(tunable_name, None)
self._tunable_names.remove(tunable_name)
tunable_name, config = self.propose()
proposal_id = self._make_id(tunable_name, config)
self.proposals[proposal_id] = {
'id': proposal_id,
'name': tunable_name,
'config': config
}
return tunable_name, config
def test_from_dict_not_a_dict(self):
# run
with self.assertRaises(TypeError):
Tunable.from_dict(1)