def test_sampling(self):
"""Make sure Fidelity simply returns `high`"""
dim = Fidelity("epoch", 1, 2)
assert dim.sample() == [2]
dim = Fidelity("epoch", 1, 5)
assert dim.sample() == [5]
dim = Fidelity("epoch", 1, 5)
assert dim.sample(4) == [5] * 4
def test_min_max_resources(self):
"""Test that an error is raised if min is larger than max"""
with pytest.raises(AttributeError) as exc:
Fidelity("epoch", 3, 2)
assert "Minimum resources must be smaller than maximum resources." == str(
exc.value
)
def test_unsupported_space(self):
"""Test tpe only work for supported search space"""
space = Space()
dim1 = Real('yolo1', 'uniform', -10, 10)
space.register(dim1)
dim2 = Real('yolo2', 'reciprocal', 10, 20)
space.register(dim2)
categories = ['a', 0.1, 2, 'c']
dim3 = Categorical('yolo3', categories)
space.register(dim3)
dim4 = Fidelity('epoch', 1, 9, 3)
space.register(dim4)
TPE(space)
space = Space()
dim = Real('yolo1', 'norm', 0.9)
space.register(dim)
with pytest.raises(ValueError) as ex:
TPE(space)
assert 'TPE now only supports uniform, loguniform, uniform discrete and choices' \
in str(ex.value)
space = Space()
dim = Real('yolo1', 'uniform', 0.9, shape=(2, 1))
space.register(dim)
with pytest.raises(ValueError) as ex:
TPE(space)
assert 'TPE now only supports 1D shape' in str(ex.value)
def test_contains(self):
"""Make sure fidelity.__contains__ always returns True"""
dim = Fidelity('epoch')
assert None in dim
assert 0 in dim
assert object() in dim
def space1():
"""Create a Space with two real dimensions and a fidelity value."""
space = Space()
space.register(Real("lr", "uniform", 0, 1))
space.register(Real("weight_decay", "uniform", 0, 1))
space.register(Fidelity("epoch", 1, 8, 2))
return space
def test_get_id_multidim(self):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity("epoch", 1, 9, 3))
space.register(Real("lr", "uniform", 0, 1, shape=2))
hyperband = Hyperband(space)
assert hyperband.get_id(["whatever", [1, 1]]) == hyperband.get_id(
["is here", [1, 1]]
)
assert hyperband.get_id(["whatever", [1, 1]]) != hyperband.get_id(
["is here", [2, 2]]
)
assert hyperband.get_id(
["whatever", [1, 1]], ignore_fidelity=False
) != hyperband.get_id(["is here", [1, 1]], ignore_fidelity=False)
assert hyperband.get_id(
["whatever", [1, 1]], ignore_fidelity=False
) != hyperband.get_id(["is here", [2, 2]], ignore_fidelity=False)
assert hyperband.get_id(
["same", [1, 1]], ignore_fidelity=False
) == hyperband.get_id(["same", [1, 1]], ignore_fidelity=False)
assert hyperband.get_id(
["same", [1, 1]], ignore_fidelity=False
) != hyperband.get_id(["same", [1, 1]])
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer("yolo1", "uniform", 0, 5))
space.register(Fidelity("epoch", 1, 9, 3))
asha = ASHA(space)
for i in range(6):
force_observe(
asha,
create_trial(
(1, i),
names=("epoch", "yolo1"),
types=("fidelity", "integer"),
results={"objective": i},
),
)
for i in range(2):
force_observe(
asha,
create_trial(
(3, i),
names=("epoch", "yolo1"),
types=("fidelity", "integer"),
results={"objective": i},
),
)
assert asha.suggest(1) == []
def space():
"""Create a Space with a real dimension and a fidelity value."""
space = Space()
space.register(Real("lr", "uniform", 0, 1))
# NOTE: Slightly different value than HyperBand (which has (1, 9, 3))
space.register(Fidelity("epoch", 1, 9, 1))
return space
def test_unsupported_space(self):
"""Test tpe only work for supported search space"""
space = Space()
dim1 = Real("yolo1", "uniform", -10, 10)
space.register(dim1)
dim2 = Real("yolo2", "reciprocal", 10, 20)
space.register(dim2)
categories = ["a", 0.1, 2, "c"]
dim3 = Categorical("yolo3", categories)
space.register(dim3)
dim4 = Fidelity("epoch", 1, 9, 3)
space.register(dim4)
TPE(space)
space = Space()
dim = Real("yolo1", "norm", 0.9)
space.register(dim)
with pytest.raises(ValueError) as ex:
tpe = TPE(space)
tpe.space = build_required_space(
space, shape_requirement=TPE.requires_shape
)
assert (
"TPE now only supports uniform, loguniform, uniform discrete and choices"
in str(ex.value)
)
def test_simple_instance(self):
"""Test Fidelity.__init__."""
dim = Fidelity('epoch')
assert str(dim) == "Fidelity(name=epoch)"
assert dim.name == 'epoch'
assert dim.type == 'fidelity'
assert dim.shape is None
def test_contains(self):
"""Make sure fidelity.__contains__ tests based on (min, max)"""
dim = Fidelity('epoch', 1, 10)
assert 0 not in dim
assert 1 in dim
assert 5 in dim
assert 10 in dim
assert 20 not in dim
def test_get_id_multidim(self):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity('epoch', 1, 9, 3))
space.register(Real('lr', 'uniform', 0, 1, shape=2))
hyperband = Hyperband(space)
assert hyperband.get_id(['whatever', [1, 1]]) == hyperband.get_id(['is here', [1, 1]])
assert hyperband.get_id(['whatever', [1, 1]]) != hyperband.get_id(['is here', [2, 2]])
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer("yolo1", "uniform", 0, 5))
space.register(Fidelity("epoch", 1, 9, 3))
hyperband = Hyperband(space, repetitions=1)
for i in range(6):
force_observe(hyperband, (1, i), {"objective": i})
assert hyperband.suggest() is None
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer('yolo1', 'uniform', 0, 6))
space.register(Fidelity('epoch', 1, 9, 3))
hyperband = Hyperband(space, repetitions=1)
for i in range(6):
hyperband.observe([(1, i)], [{'objective': i}])
assert hyperband.suggest() is None
def test_simple_instance(self):
"""Test Fidelity.__init__."""
dim = Fidelity('epoch', 1, 2)
assert str(dim) == "Fidelity(name=epoch, low=1, high=2, base=2)"
assert dim.low == 1
assert dim.high == 2
assert dim.base == 2
assert dim.name == 'epoch'
assert dim.type == 'fidelity'
assert dim.shape is None
def space():
"""Return an optimization space"""
space = Space()
dim1 = Integer('yolo1', 'uniform', -3, 6)
space.register(dim1)
dim2 = Real('yolo2', 'uniform', 0, 1)
space.register(dim2)
dim3 = Fidelity('yolo3', 1, 4, 2)
space.register(dim3)
return space
def test_get_id_multidim(self, b_config):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity("epoch", 1, 9, 3))
space.register(Real("lr", "uniform", 0, 1, shape=2))
asha = ASHA(space, num_brackets=3)
assert asha.get_id(["whatever",
[1, 1]]) == asha.get_id(["is here", [1, 1]])
assert asha.get_id(["whatever", [1, 1]]) != asha.get_id(
["is here", [2, 2]])
def test_get_id_multidim(self, b_config):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity('epoch', 1, 9, 3))
space.register(Real('lr', 'uniform', 0, 1, shape=2))
asha = ASHA(space, num_brackets=3)
assert asha.get_id(['whatever',
[1, 1]]) == asha.get_id(['is here', [1, 1]])
assert asha.get_id(['whatever', [1, 1]]) != asha.get_id(
['is here', [2, 2]])
def cardinality(self):
"""Wrap original :class:`orion.algo.space.Dimension` capacity"""
if self.type == "real":
return Real.get_cardinality(self.shape, self.interval())
elif self.type == "integer":
return Integer.get_cardinality(self.shape, self.interval())
elif self.type == "categorical":
return Categorical.get_cardinality(self.shape, self.interval())
elif self.type == "fidelity":
return Fidelity.get_cardinality(self.shape, self.interval())
else:
raise RuntimeError(
f"No cardinality can be computed for type `{self.type}`")
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer('yolo1', 'uniform', 0, 6))
space.register(Fidelity('epoch', 1, 9, 3))
asha = ASHA(space)
for i in range(6):
asha.observe([(1, i)], [{'objective': i}])
for i in range(2):
asha.observe([(3, i)], [{'objective': i}])
assert asha.suggest() is None
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer("yolo1", "uniform", 0, 5))
space.register(Fidelity("epoch", 1, 9, 3))
asha = ASHA(space)
for i in range(6):
force_observe(asha, (1, i), {"objective": i})
for i in range(2):
force_observe(asha, (3, i), {"objective": i})
assert asha.suggest() is None
def test_get_id_multidim(self, b_config):
"""Test valid id for points with dim of shape > 1"""
space = Space()
space.register(Fidelity('epoch'))
space.register(Real('lr', 'uniform', 0, 1, shape=2))
asha = ASHA(space,
max_resources=b_config['R'],
grace_period=b_config['r'],
reduction_factor=b_config['eta'],
num_brackets=3)
assert asha.get_id(['whatever',
[1, 1]]) == asha.get_id(['is here', [1, 1]])
assert asha.get_id(['whatever', [1, 1]]) != asha.get_id(
['is here', [2, 2]])
def test_suggest_in_finite_cardinality(self):
"""Test that suggest None when search space is empty"""
space = Space()
space.register(Integer("yolo1", "uniform", 0, 5))
space.register(Fidelity("epoch", 1, 9, 3))
hyperband = Hyperband(space, repetitions=1)
for i in range(6):
force_observe(
hyperband,
create_trial(
(1, i),
names=("epoch", "yolo1"),
types=("fidelity", "integer"),
results={"objective": i},
),
)
assert hyperband.suggest(100) == []
def test_cardinality(self):
"""Check whether space capacity is correct"""
space = Space()
probs = (0.1, 0.2, 0.3, 0.4)
categories = ("asdfa", 2, 3, 4)
dim = Categorical("yolo", OrderedDict(zip(categories, probs)), shape=2)
space.register(dim)
dim = Integer("yolo2", "uniform", -3, 6)
space.register(dim)
dim = Fidelity("epoch", 1, 9, 3)
space.register(dim)
assert space.cardinality == (4 ** 2) * (6 + 1) * 1
dim = Integer("yolo3", "uniform", -3, 2, shape=(3, 2))
space.register(dim)
assert space.cardinality == (4 ** 2) * (6 + 1) * 1 * ((2 + 1) ** (3 * 2))
dim = Real("yolo4", "norm", 0.9)
space.register(dim)
assert np.inf == space.cardinality
def test_cardinality(self):
"""Check whether space capacity is correct"""
space = Space()
probs = (0.1, 0.2, 0.3, 0.4)
categories = ('asdfa', 2, 3, 4)
dim = Categorical('yolo', OrderedDict(zip(categories, probs)), shape=2)
space.register(dim)
dim = Integer('yolo2', 'uniform', -3, 6)
space.register(dim)
dim = Fidelity('epoch', 1, 9, 3)
space.register(dim)
assert (4 * 2) * 6 * 1 == space.cardinality
dim = Integer('yolo3', 'uniform', -3, 2, shape=(3, 1))
space.register(dim)
assert (4 * 2) * 6 * 1 * (2 * 3 * 1) == space.cardinality
dim = Real('yolo4', 'norm', 0.9)
space.register(dim)
assert np.inf == space.cardinality
def fidelity(self, *args, **kwargs):
"""Create a `Fidelity` dimension."""
name = self.name
return Fidelity(name, *args, **kwargs)
def space():
"""Create a Space with a real dimension and a fidelity value."""
space = Space()
space.register(Real("lr", "uniform", 0, 1))
space.register(Fidelity("epoch", 1, 9, 3))
return space
def test_cast(self):
"""Check that error is being raised."""
dim = Fidelity('epoch', 1, 10)
with pytest.raises(NotImplementedError):
dim.cast()
def test_interval(self):
"""Check that interval() is (min, max)."""
dim = Fidelity('epoch', 1, 10)
dim.interval() == (1, 10)
def test_default_value(self):
"""Make sure Fidelity simply returns `high`"""
dim = Fidelity('epoch', 1, 2)
assert dim.default_value == 2
dim = Fidelity('epoch', 1, 5)
assert dim.default_value == 5
