def test___eq__not_implemented(self):
an_hyperparam = HyperParameter(ParamTypes.INT, [1, 5])
not_an_hyperparam = 3
self.assertEqual(an_hyperparam.__eq__(not_an_hyperparam),
NotImplemented)
self.assertNotEqual(an_hyperparam, not_an_hyperparam)
def test_int_exp(self):
hyp = HyperParameter(ParamTypes.INT_EXP, [10, 10000])
self.assertEqual(hyp.range, [1, 4])
transformed = hyp.fit_transform(np.array([100]), np.array([0.5]))
np.testing.assert_array_equal(transformed, np.array([2]))
inverse_transform = hyp.inverse_transform([3])
np.testing.assert_array_equal(inverse_transform, np.array([1000]))
def test_fit_gt_r_min(self, gpr_mock):
"""If the length of X is greater than r_minimum, gpr is fit."""
# Set-up
tunables = (
('a_float_param', HyperParameter(ParamTypes.FLOAT, [1., 2.])),
('an_int_param', HyperParameter(ParamTypes.INT, [1, 5])),
)
tuner = GP(tunables, r_minimum=2)
gp_mock = Mock()
gpr_mock.return_value = gp_mock
# Run
X = np.array([[1., 1], [1.2, 2], [1.4, 4]])
y = np.array([0.5, 0.6, 0.7])
tuner.fit(X, y)
# assert
np.testing.assert_array_equal(tuner.X, X)
np.testing.assert_array_equal(tuner.y, y)
gpr_mock.assert_called_once_with(normalize_y=True)
# compare arrays for equality, ignoring differences from reshaping
(X_actual, y_actual), _ = gp_mock.fit.call_args
np.testing.assert_array_equal(X_actual.ravel(), X.ravel())
np.testing.assert_array_equal(y_actual.ravel(), y.ravel())
def test_int_exp(self):
hyp = HyperParameter(ParamTypes.INT_EXP, [10, 10000])
self.assertEqual(hyp.range, [1, 4])
transformed = hyp.fit_transform(np.array([100]), np.array([0.5]))
np.testing.assert_array_equal(transformed, np.array([2]))
inverse_transform = hyp.inverse_transform([3])
np.testing.assert_array_equal(inverse_transform, np.array([1000]))
def test_predict_x_gt_r_min(self, uniform_mock):
"""If the length of self.X is greater than r_minimum, self.gp is used."""
# Set-up
tunables = (
('a_float_param', HyperParameter(ParamTypes.FLOAT, [1., 2.])),
('an_int_param', HyperParameter(ParamTypes.INT, [1, 5])),
)
tuner = GP(tunables, r_minimum=2)
tuner.X = np.array([[1., 1], [1.2, 2], [1.4, 4]])
tuner.gp = Mock()
tuner.gp.predict.return_value = (np.array([0.8,
0.9]), np.array([0.1, 0.3]))
# Run
X = np.array([
[1.6, 4],
[1.8, 5],
])
predicted = tuner.predict(X)
# assert
expected = np.array([[0.8, 0.1], [0.9, 0.3]])
np.testing.assert_array_equal(predicted, expected)
tuner.gp.predict.assert_called_once_with(X, return_std=True)
def test_fit_predict_uniform(self, predict_mock, random_mock,
uniform_mock):
"""If random is lower than POU, Uniform is used."""
# Set-up
tunables = (
('a_float_param', HyperParameter(ParamTypes.FLOAT, [1., 2.])),
('an_int_param', HyperParameter(ParamTypes.INT, [1, 5])),
)
tuner = GPEiVelocity(tunables, r_minimum=2)
tuner.POU = 0.05
random_mock.random.return_value = 0.01
uniform_instance_mock = Mock()
uniform_mock.return_value = uniform_instance_mock
# Run
X = np.array([[1., 1], [1.2, 2], [1.4, 3], [1.6, 4], [1.8, 5]])
tuner.predict(X)
# assert
uniform_mock.assert_called_once_with(tunables)
uniform_instance_mock.predict.assert_called_once_with(X)
predict_mock.assert_not_called()
def test__create_candidates_no_grid(self, np_random_mock):
"""self.grid is False"""
# Set-up
np_random_mock.rand.return_value = np.array([.0, .2, .4, .6, .8])
np_random_mock.randint.return_value = np.array([1, 2, 3, 4, 5])
tunables = (
('a_float_param', HyperParameter(ParamTypes.FLOAT, [1., 2.])),
('an_int_param', HyperParameter(ParamTypes.INT, [1, 5])),
('a_string_param',
HyperParameter(ParamTypes.STRING, ['a', 'b', 'c'])),
)
tuner = BaseTuner(tunables)
# Run
candidates = tuner._create_candidates(5)
# Assert
expected_candidates = np.array([[1.0, 1, 0.], [1.2, 2, 0.2],
[1.4, 3, 0.4], [1.6, 4, 0.6],
[1.8, 5, 0.8]])
np.testing.assert_array_equal(candidates, expected_candidates)
expected_calls = [
call(5),
call(5),
]
np_random_mock.rand.assert_has_calls(expected_calls)
np_random_mock.randint.assert_called_once_with(1, 6, size=5)
def test___init__(self):
# Set-up
tunables = (
('a_float_param', HyperParameter(ParamTypes.FLOAT, [1., 2.])),
('an_int_param', HyperParameter(ParamTypes.INT, [1, 5])),
)
# Run
tuner = BaseTuner(tunables, gridding=5)
# assert
assert tuner.tunables == tunables
assert tuner.grid is True
assert tuner._best_score == -np.inf
assert tuner._best_hyperparams is None
assert tuner.grid_width == 5
assert tuner.X_raw is None
assert tuner.y_raw == []
assert tuner.X.tolist() == []
assert tuner.y.tolist() == []
expected_grid_axes = [
np.array([1., 1.25, 1.5, 1.75, 2.]),
np.array([1., 2., 3., 4., 5.])
]
np.testing.assert_array_equal(tuner._grid_axes, expected_grid_axes)
def test__create_candidates_gt_n_remaining(self, np_random_mock):
"""more than n points remaining"""
# Set-up
mock_context = dict()
def shuffle(array):
np_shuffle(array)
# Store a copy of the array for the assert
mock_context['shuffled_array'] = array.copy()
np_random_mock.shuffle.side_effect = shuffle
tunables = (
('a_float_param', HyperParameter(ParamTypes.FLOAT, [1., 2.])),
('an_int_param', HyperParameter(ParamTypes.INT, [1, 5])),
)
tuner = BaseTuner(tunables, gridding=3)
# Insert 5 used vectors into X (4 remaining)
tuner.X = np.array([
[1., 1.],
[1., 3.],
[1., 5.],
[1.5, 1.],
[1.5, 5.],
])
# Run
candidates = tuner._create_candidates(2)
# Assert
expected_candidates = mock_context['shuffled_array'][0:2]
np.testing.assert_array_equal(candidates, expected_candidates)
def test_predict_x_lt_r_min(self, uniform_mock):
"""If the length of self.X is smaller than r_minimum, Uniform is used."""
# Set-up
tunables = (
('a_float_param', HyperParameter(ParamTypes.FLOAT, [1., 2.])),
('an_int_param', HyperParameter(ParamTypes.INT, [1, 5])),
)
tuner = GP(tunables, r_minimum=5)
tuner.X = np.array([[1., 1], [1.2, 2], [1.4, 4]])
tuner.gp = Mock()
predict_mock = Mock()
predict_mock.return_value = np.array([0.8, 0.9])
uniform_instance_mock = Mock()
uniform_instance_mock.predict = predict_mock
uniform_mock.return_value = uniform_instance_mock
# Run
X = np.array([
[1.6, 4],
[1.8, 5],
])
predicted = tuner.predict(X)
# assert
expected = np.array([0.8, 0.9])
np.testing.assert_array_equal(predicted, expected)
uniform_mock.assert_called_once_with(tunables)
predict_mock.assert_called_once_with(X)
tuner.gp.predict.assert_not_called()
def test___init___with_unicode_param_type(self):
param_type_str = 'int'
param_type_unicode = u'int'
param_range = [0, 10]
self.assertEqual(HyperParameter(param_type_unicode, param_range),
HyperParameter(param_type_str, param_range))
def test_predict(self, np_random_mock):
# Set-up
np_random_mock.rand.return_value = np.array([.4, .6, .8])
tunables = (
('a_float_param', HyperParameter(ParamTypes.FLOAT, [1., 2.])),
('an_int_param', HyperParameter(ParamTypes.INT, [1, 5])),
)
tuner = Uniform(tunables)
# Run
x = np.array([
[1., 1],
[1.2, 2],
[1.4, 3],
])
predicted = tuner.predict(x)
# Assert
expected = np.array([.4, .6, .8])
np.testing.assert_array_equal(predicted, expected)
np_random_mock.rand.assert_called_once_with(3, 1)
def test_int_cat(self):
hyp = HyperParameter(ParamTypes.INT_CAT, [10, 10000])
transformed = hyp.fit_transform(np.array([10, 10000]), np.array([1,
2]))
np.testing.assert_array_equal(transformed, np.array([1, 2]))
inverse_transform = hyp.inverse_transform([3, 0, 1, 2])
np.testing.assert_array_equal(inverse_transform,
np.array([10000, 10, 10, 10000]))
def test_int_cat(self):
hyp = HyperParameter(ParamTypes.INT_CAT, [None, 10, 10000])
transformed = hyp.fit_transform(np.array([None, 10, 10000]),
np.array([0.1, 0.5, 0.9]))
np.testing.assert_array_equal(transformed, np.array([0.1, 0.5, 0.9]))
inverse_transform = hyp.inverse_transform([1.0, 0.0, 0.4, 0.8])
np.testing.assert_array_equal(inverse_transform,
np.array([10000, None, 10, 10000]))
def test_string(self):
hyp = HyperParameter(ParamTypes.STRING, ['a', 'b', 'c'])
transformed = hyp.fit_transform(np.array(['a', 'b', 'c']),
np.array([2, 1, 3]))
self.assertEqual(hyp.range, [1.0, 3.0])
np.testing.assert_array_equal(transformed, np.array([2.0, 1.0, 3.0]))
inverse_transform = hyp.inverse_transform([3])
np.testing.assert_array_equal(inverse_transform, np.array(['c']))
def test_bool(self):
hyp = HyperParameter(ParamTypes.BOOL, [True, False])
transformed = hyp.fit_transform(np.array([True, False]),
np.array([0.5, 0.7]))
np.testing.assert_array_equal(transformed, np.array([0.5, 0.7]))
self.assertEqual(hyp.range, [0.5, 0.7])
inverse_transform = hyp.inverse_transform([0.7, 0.6, 0.5])
np.testing.assert_array_equal(inverse_transform,
np.array([False, False, True]))
def test_float(self):
hyp = HyperParameter(ParamTypes.FLOAT, [1.5, 3.2])
self.assertEqual(hyp.range, [1.5, 3.2])
transformed = hyp.fit_transform(
np.array([2, 2.4, 3.1]),
np.array([0.5, 0.6, 0.1]),
)
np.testing.assert_array_equal(transformed, np.array([2, 2.4, 3.1]))
inverse_transform = hyp.inverse_transform([1.7])
np.testing.assert_array_equal(inverse_transform, np.array([1.7]))
def test_float(self):
hyp = HyperParameter(ParamTypes.FLOAT, [1.5, 3.2])
self.assertEqual(hyp.range, [1.5, 3.2])
transformed = hyp.fit_transform(
np.array([2, 2.4, 3.1]),
np.array([0.5, 0.6, 0.1]),
)
np.testing.assert_array_equal(transformed, np.array([2, 2.4, 3.1]))
inverse_transform = hyp.inverse_transform([1.7])
np.testing.assert_array_equal(inverse_transform, np.array([1.7]))
def test_float_exp(self):
hyp = HyperParameter(ParamTypes.FLOAT_EXP, [0.001, 100])
self.assertEqual(hyp.range, [-3.0, 2.0])
transformed = hyp.fit_transform(np.array([0.01, 1, 10]),
np.array([-2.0, 0.0, 1.0]))
np.testing.assert_array_equal(transformed, np.array([-2.0, 0.0, 1.0]))
inverse_transform = hyp.inverse_transform([-1.0])
np.testing.assert_array_equal(inverse_transform, np.array([0.1]))
inverse_transform = hyp.inverse_transform([1.0])
np.testing.assert_array_equal(inverse_transform, np.array([10.0]))
def test_float_cat(self):
hyp = HyperParameter(ParamTypes.FLOAT_CAT, [0.1, 0.6, 0.5])
transformed = hyp.fit_transform(np.array([0.1, 0.6, 0.1, 0.6]),
np.array([1, 2, 3, 4]))
np.testing.assert_array_equal(transformed,
np.array([2.0, 3.0, 2.0, 3.0]))
self.assertEqual(hyp.range, [0.0, 3.0])
inverse_transform = hyp.inverse_transform([3, 0, 1, 5, 2.5])
np.testing.assert_array_equal(inverse_transform,
np.array([0.6, 0.5, 0.1, 0.6, 0.6]))
def test_float_cat(self):
hyp = HyperParameter(ParamTypes.FLOAT_CAT, [None, 0.1, 0.6, 0.5])
transformed = hyp.fit_transform(np.array([None, 0.1, 0.6, 0.1, 0.6]),
np.array([1.0, 0.1, 0.2, 0.3, 0.4]))
np.testing.assert_allclose(transformed,
np.array([1.0, 0.2, 0.3, 0.2, 0.3]))
self.assertEqual(hyp.range, [0.0, 1.0])
inverse_transform = hyp.inverse_transform(
[0.3, 0.0, 0.1, 0.5, 0.27, 0.9])
np.testing.assert_array_equal(
inverse_transform, np.array([0.6, 0.5, 0.1, 0.6, 0.6, None]))
def test_int(self):
hyp = HyperParameter(ParamTypes.INT, [1, 3])
self.assertEqual(hyp.range, [1, 3])
transformed = hyp.fit_transform(
np.array([1, 2, 3]),
np.array([0.5, 0.6, 0.1])
)
np.testing.assert_array_equal(transformed, np.array([1, 2, 3]))
inverse_transform = hyp.inverse_transform(np.array([0.5]))
np.testing.assert_array_equal(inverse_transform, np.array([0]))
hyp = HyperParameter(ParamTypes.INT, [1.0, 3.0])
self.assertEqual(hyp.range, [1, 3])
def test_int_cat(self):
hyp = HyperParameter(ParamTypes.INT_CAT, [10, 10000])
transformed = hyp.fit_transform(
np.array([10, 10000]),
np.array([1, 2])
)
np.testing.assert_array_equal(transformed, np.array([1, 2]))
inverse_transform = hyp.inverse_transform([3, 0, 1, 2])
np.testing.assert_array_equal(
inverse_transform,
np.array([10000, 10, 10, 10000])
)
def test_bool(self):
hyp = HyperParameter(ParamTypes.BOOL, [True, False])
transformed = hyp.fit_transform(
np.array([True, False]),
np.array([0.5, 0.7])
)
np.testing.assert_array_equal(transformed, np.array([0.5, 0.7]))
self.assertEqual(hyp.range, [0.5, 0.7])
inverse_transform = hyp.inverse_transform([0.7, 0.6, 0.5])
np.testing.assert_array_equal(
inverse_transform,
np.array([False, False, True])
)
def test_string(self):
hyp = HyperParameter(ParamTypes.STRING, ['a', 'b', 'c'])
transformed = hyp.fit_transform(
np.array(['a', 'b', 'c']),
np.array([2, 1, 3])
)
self.assertEqual(hyp.range, [1.0, 3.0])
np.testing.assert_array_equal(
transformed,
np.array([2.0, 1.0, 3.0])
)
inverse_transform = hyp.inverse_transform([3])
np.testing.assert_array_equal(inverse_transform, np.array(['c']))
def test_cast_not_implemented(self, subclasses_mock):
# create a fake HyperParameter subclass
class FakeHyperParameter(HyperParameter):
param_type = ParamTypes.INT
# Make FakeHyperParameter the only subclass for this test
subclasses_mock.return_value = [FakeHyperParameter]
fake = HyperParameter(ParamTypes.INT, [None])
with pytest.raises(NotImplementedError):
fake.cast(1)
def test_cast_not_implemented(self, subclasses_mock):
# create a fake HyperParameter subclass
class FakeHyperParameter(HyperParameter):
param_type = ParamTypes.INT
# Make FakeHyperParameter the only subclass for this test
subclasses_mock.return_value = [FakeHyperParameter]
fake = HyperParameter(ParamTypes.INT, [None])
with pytest.raises(NotImplementedError):
fake.cast(1)
def test___init___with_string_param_type_valid(self):
r = random.Random()
r.seed(1)
def random_case(s):
return ''.join(r.choice([str.upper, str.lower])(c) for c in s)
# allowed string param types
for type_, range_ in self.parameter_constructions:
for recase in [str.upper, str.lower, random_case]:
str_type = recase(type_.name)
self.assertEqual(HyperParameter(str_type, range_),
HyperParameter(type_, range_))
def test_string(self):
hyp = HyperParameter(ParamTypes.STRING, [None, 'a', 'b', 'c'])
transformed = hyp.fit_transform(np.array([None, 'a', 'b', 'c']),
np.array([0.1, 0.3, 0.6, 0.9]))
self.assertEqual(hyp.range, [0.1, 0.9])
np.testing.assert_array_equal(transformed,
np.array([0.1, 0.3, 0.6, 0.9]))
inverse_transform = hyp.inverse_transform(
np.array([0.0, 0.4, 0.5, 0.8]))
np.testing.assert_array_equal(inverse_transform,
np.array([None, 'a', 'b', 'c']))
inverse_transform = hyp.inverse_transform([3])
np.testing.assert_array_equal(inverse_transform, np.array(['c']))
def test_float_exp(self):
hyp = HyperParameter(ParamTypes.FLOAT_EXP, [0.001, 100])
self.assertEqual(hyp.range, [-3.0, 2.0])
transformed = hyp.fit_transform(
np.array([0.01, 1, 10]),
np.array([-2.0, 0.0, 1.0])
)
np.testing.assert_array_equal(
transformed,
np.array([-2.0, 0.0, 1.0])
)
inverse_transform = hyp.inverse_transform([-1.0])
np.testing.assert_array_equal(inverse_transform, np.array([0.1]))
inverse_transform = hyp.inverse_transform([1.0])
np.testing.assert_array_equal(inverse_transform, np.array([10.0]))
def test_float_cat(self):
hyp = HyperParameter(ParamTypes.FLOAT_CAT, [0.1, 0.6, 0.5])
transformed = hyp.fit_transform(
np.array([0.1, 0.6, 0.1, 0.6]),
np.array([1, 2, 3, 4])
)
np.testing.assert_array_equal(
transformed,
np.array([2.0, 3.0, 2.0, 3.0])
)
self.assertEqual(hyp.range, [0.0, 3.0])
inverse_transform = hyp.inverse_transform([3, 0, 1, 5, 2.5])
np.testing.assert_array_equal(
inverse_transform,
np.array([0.6, 0.5, 0.1, 0.6, 0.6])
)
def test_can_pickle(self):
for protocol in range(0, pickle.HIGHEST_PROTOCOL + 1):
for param_type, param_range in self.parameter_constructions:
param = HyperParameter(param_type, param_range)
pickled = pickle.dumps(param, protocol)
unpickled = pickle.loads(pickled)
self.assertEqual(param, unpickled)
def test_add_single_tunable(self):
# See HDI-Project/BTB#84
tunables = [
('a_float_param', HyperParameter(ParamTypes.FLOAT_EXP, [1., 2.])),
]
tuner = GP(tunables)
tuner.add({'a_float_param': 1.}, 1)
tuner.add({'a_float_param': 1.}, 1)
def test__create_candidates_every_point_used(self):
"""every point has been used"""
# Set-up
tunables = (
('a_float_param', HyperParameter(ParamTypes.FLOAT, [1., 2.])),
('an_int_param', HyperParameter(ParamTypes.INT, [1, 5])),
)
tuner = BaseTuner(tunables, gridding=5)
# Insert 25 part_vecs into X
tuner.X = np.array(list(product(*tuner._grid_axes)))
# Run
candidates = tuner._create_candidates(5)
# Assert
assert candidates is None
def test_propse_done(self, predict_mock, create_candidates_mock):
"""gridding is done"""
# Set-up
tunables = (
('a_float_param', HyperParameter(ParamTypes.FLOAT, [1., 2.])),
('an_int_param', HyperParameter(ParamTypes.INT, [1, 5])),
)
tuner = BaseTuner(tunables)
create_candidates_mock.return_value = None
# Run
params = tuner.propose(1)
# Assert
expected_params = None
assert params == expected_params
def test_fit_lt_r_min(self, gpr_mock):
"""If the length of X is smaller than r_minimum, nothing is done."""
# Set-up
tunables = (
('a_float_param', HyperParameter(ParamTypes.FLOAT, [1., 2.])),
('an_int_param', HyperParameter(ParamTypes.INT, [1, 5])),
)
tuner = GP(tunables, r_minimum=5)
# Run
X = np.array([[1., 1], [1.2, 2], [1.4, 4]])
y = np.array([0.5, 0.6, 0.7])
tuner.fit(X, y)
# assert
np.testing.assert_array_equal(tuner.X, X)
np.testing.assert_array_equal(tuner.y, y)
gpr_mock.assert_not_called()
def test_propse(self, create_candidates_mock):
# Set-up
tunables = (
('a_float_param', HyperParameter(ParamTypes.FLOAT, [1., 2.])),
('an_int_param', HyperParameter(ParamTypes.INT, [1, 5])),
)
tuner = CustomTuner(tunables)
create_candidates_mock.return_value = np.array([[1.0, 1]])
# Run
params = tuner.propose()
# Assert
expected_params = np.array([1.0, 1])
np.testing.assert_array_equal(params, expected_params)
create_candidates_mock.assert_called_once_with(1)
def test_copy(self):
for typ, rang in self.parameter_constructions:
hyp = HyperParameter(typ, rang)
hyp_copy = copy.copy(hyp)
self.assertIsNot(hyp, hyp_copy)
self.assertIs(type(hyp), type(hyp_copy))
self.assertEqual(hyp.range, hyp_copy.range)
# shallow copy should just have copied references
self.assertIs(hyp.range, hyp_copy.range)
def test_deepcopy(self):
for typ, rang in self.parameter_constructions:
hyp = HyperParameter(typ, rang)
hyp_copy = copy.deepcopy(hyp)
self.assertIsNot(hyp, hyp_copy)
self.assertIs(type(hyp), type(hyp_copy))
self.assertEqual(hyp.range, hyp_copy.range)
# deep copy should have new attributes
self.assertIsNot(hyp.range, hyp_copy.range)