def test_sampling(self): """Make sure Fidelity simply returns `high`""" dim = Fidelity("epoch", 1, 2) assert dim.sample() == [2] dim = Fidelity("epoch", 1, 5) assert dim.sample() == [5] dim = Fidelity("epoch", 1, 5) assert dim.sample(4) == [5] * 4
def test_min_max_resources(self): """Test that an error is raised if min is larger than max""" with pytest.raises(AttributeError) as exc: Fidelity("epoch", 3, 2) assert "Minimum resources must be smaller than maximum resources." == str( exc.value )
def test_unsupported_space(self): """Test tpe only work for supported search space""" space = Space() dim1 = Real('yolo1', 'uniform', -10, 10) space.register(dim1) dim2 = Real('yolo2', 'reciprocal', 10, 20) space.register(dim2) categories = ['a', 0.1, 2, 'c'] dim3 = Categorical('yolo3', categories) space.register(dim3) dim4 = Fidelity('epoch', 1, 9, 3) space.register(dim4) TPE(space) space = Space() dim = Real('yolo1', 'norm', 0.9) space.register(dim) with pytest.raises(ValueError) as ex: TPE(space) assert 'TPE now only supports uniform, loguniform, uniform discrete and choices' \ in str(ex.value) space = Space() dim = Real('yolo1', 'uniform', 0.9, shape=(2, 1)) space.register(dim) with pytest.raises(ValueError) as ex: TPE(space) assert 'TPE now only supports 1D shape' in str(ex.value)
def test_contains(self): """Make sure fidelity.__contains__ always returns True""" dim = Fidelity('epoch') assert None in dim assert 0 in dim assert object() in dim
def space1(): """Create a Space with two real dimensions and a fidelity value.""" space = Space() space.register(Real("lr", "uniform", 0, 1)) space.register(Real("weight_decay", "uniform", 0, 1)) space.register(Fidelity("epoch", 1, 8, 2)) return space
def test_get_id_multidim(self): """Test valid id for points with dim of shape > 1""" space = Space() space.register(Fidelity("epoch", 1, 9, 3)) space.register(Real("lr", "uniform", 0, 1, shape=2)) hyperband = Hyperband(space) assert hyperband.get_id(["whatever", [1, 1]]) == hyperband.get_id( ["is here", [1, 1]] ) assert hyperband.get_id(["whatever", [1, 1]]) != hyperband.get_id( ["is here", [2, 2]] ) assert hyperband.get_id( ["whatever", [1, 1]], ignore_fidelity=False ) != hyperband.get_id(["is here", [1, 1]], ignore_fidelity=False) assert hyperband.get_id( ["whatever", [1, 1]], ignore_fidelity=False ) != hyperband.get_id(["is here", [2, 2]], ignore_fidelity=False) assert hyperband.get_id( ["same", [1, 1]], ignore_fidelity=False ) == hyperband.get_id(["same", [1, 1]], ignore_fidelity=False) assert hyperband.get_id( ["same", [1, 1]], ignore_fidelity=False ) != hyperband.get_id(["same", [1, 1]])
def test_suggest_in_finite_cardinality(self): """Test that suggest None when search space is empty""" space = Space() space.register(Integer("yolo1", "uniform", 0, 5)) space.register(Fidelity("epoch", 1, 9, 3)) asha = ASHA(space) for i in range(6): force_observe( asha, create_trial( (1, i), names=("epoch", "yolo1"), types=("fidelity", "integer"), results={"objective": i}, ), ) for i in range(2): force_observe( asha, create_trial( (3, i), names=("epoch", "yolo1"), types=("fidelity", "integer"), results={"objective": i}, ), ) assert asha.suggest(1) == []
def space(): """Create a Space with a real dimension and a fidelity value.""" space = Space() space.register(Real("lr", "uniform", 0, 1)) # NOTE: Slightly different value than HyperBand (which has (1, 9, 3)) space.register(Fidelity("epoch", 1, 9, 1)) return space
def test_unsupported_space(self): """Test tpe only work for supported search space""" space = Space() dim1 = Real("yolo1", "uniform", -10, 10) space.register(dim1) dim2 = Real("yolo2", "reciprocal", 10, 20) space.register(dim2) categories = ["a", 0.1, 2, "c"] dim3 = Categorical("yolo3", categories) space.register(dim3) dim4 = Fidelity("epoch", 1, 9, 3) space.register(dim4) TPE(space) space = Space() dim = Real("yolo1", "norm", 0.9) space.register(dim) with pytest.raises(ValueError) as ex: tpe = TPE(space) tpe.space = build_required_space( space, shape_requirement=TPE.requires_shape ) assert ( "TPE now only supports uniform, loguniform, uniform discrete and choices" in str(ex.value) )
def test_simple_instance(self): """Test Fidelity.__init__.""" dim = Fidelity('epoch') assert str(dim) == "Fidelity(name=epoch)" assert dim.name == 'epoch' assert dim.type == 'fidelity' assert dim.shape is None
def test_contains(self): """Make sure fidelity.__contains__ tests based on (min, max)""" dim = Fidelity('epoch', 1, 10) assert 0 not in dim assert 1 in dim assert 5 in dim assert 10 in dim assert 20 not in dim
def test_get_id_multidim(self): """Test valid id for points with dim of shape > 1""" space = Space() space.register(Fidelity('epoch', 1, 9, 3)) space.register(Real('lr', 'uniform', 0, 1, shape=2)) hyperband = Hyperband(space) assert hyperband.get_id(['whatever', [1, 1]]) == hyperband.get_id(['is here', [1, 1]]) assert hyperband.get_id(['whatever', [1, 1]]) != hyperband.get_id(['is here', [2, 2]])
def test_suggest_in_finite_cardinality(self): """Test that suggest None when search space is empty""" space = Space() space.register(Integer("yolo1", "uniform", 0, 5)) space.register(Fidelity("epoch", 1, 9, 3)) hyperband = Hyperband(space, repetitions=1) for i in range(6): force_observe(hyperband, (1, i), {"objective": i}) assert hyperband.suggest() is None
def test_suggest_in_finite_cardinality(self): """Test that suggest None when search space is empty""" space = Space() space.register(Integer('yolo1', 'uniform', 0, 6)) space.register(Fidelity('epoch', 1, 9, 3)) hyperband = Hyperband(space, repetitions=1) for i in range(6): hyperband.observe([(1, i)], [{'objective': i}]) assert hyperband.suggest() is None
def test_simple_instance(self): """Test Fidelity.__init__.""" dim = Fidelity('epoch', 1, 2) assert str(dim) == "Fidelity(name=epoch, low=1, high=2, base=2)" assert dim.low == 1 assert dim.high == 2 assert dim.base == 2 assert dim.name == 'epoch' assert dim.type == 'fidelity' assert dim.shape is None
def space(): """Return an optimization space""" space = Space() dim1 = Integer('yolo1', 'uniform', -3, 6) space.register(dim1) dim2 = Real('yolo2', 'uniform', 0, 1) space.register(dim2) dim3 = Fidelity('yolo3', 1, 4, 2) space.register(dim3) return space
def test_get_id_multidim(self, b_config): """Test valid id for points with dim of shape > 1""" space = Space() space.register(Fidelity("epoch", 1, 9, 3)) space.register(Real("lr", "uniform", 0, 1, shape=2)) asha = ASHA(space, num_brackets=3) assert asha.get_id(["whatever", [1, 1]]) == asha.get_id(["is here", [1, 1]]) assert asha.get_id(["whatever", [1, 1]]) != asha.get_id( ["is here", [2, 2]])
def test_get_id_multidim(self, b_config): """Test valid id for points with dim of shape > 1""" space = Space() space.register(Fidelity('epoch', 1, 9, 3)) space.register(Real('lr', 'uniform', 0, 1, shape=2)) asha = ASHA(space, num_brackets=3) assert asha.get_id(['whatever', [1, 1]]) == asha.get_id(['is here', [1, 1]]) assert asha.get_id(['whatever', [1, 1]]) != asha.get_id( ['is here', [2, 2]])
def cardinality(self): """Wrap original :class:`orion.algo.space.Dimension` capacity""" if self.type == "real": return Real.get_cardinality(self.shape, self.interval()) elif self.type == "integer": return Integer.get_cardinality(self.shape, self.interval()) elif self.type == "categorical": return Categorical.get_cardinality(self.shape, self.interval()) elif self.type == "fidelity": return Fidelity.get_cardinality(self.shape, self.interval()) else: raise RuntimeError( f"No cardinality can be computed for type `{self.type}`")
def test_suggest_in_finite_cardinality(self): """Test that suggest None when search space is empty""" space = Space() space.register(Integer('yolo1', 'uniform', 0, 6)) space.register(Fidelity('epoch', 1, 9, 3)) asha = ASHA(space) for i in range(6): asha.observe([(1, i)], [{'objective': i}]) for i in range(2): asha.observe([(3, i)], [{'objective': i}]) assert asha.suggest() is None
def test_suggest_in_finite_cardinality(self): """Test that suggest None when search space is empty""" space = Space() space.register(Integer("yolo1", "uniform", 0, 5)) space.register(Fidelity("epoch", 1, 9, 3)) asha = ASHA(space) for i in range(6): force_observe(asha, (1, i), {"objective": i}) for i in range(2): force_observe(asha, (3, i), {"objective": i}) assert asha.suggest() is None
def test_get_id_multidim(self, b_config): """Test valid id for points with dim of shape > 1""" space = Space() space.register(Fidelity('epoch')) space.register(Real('lr', 'uniform', 0, 1, shape=2)) asha = ASHA(space, max_resources=b_config['R'], grace_period=b_config['r'], reduction_factor=b_config['eta'], num_brackets=3) assert asha.get_id(['whatever', [1, 1]]) == asha.get_id(['is here', [1, 1]]) assert asha.get_id(['whatever', [1, 1]]) != asha.get_id( ['is here', [2, 2]])
def test_suggest_in_finite_cardinality(self): """Test that suggest None when search space is empty""" space = Space() space.register(Integer("yolo1", "uniform", 0, 5)) space.register(Fidelity("epoch", 1, 9, 3)) hyperband = Hyperband(space, repetitions=1) for i in range(6): force_observe( hyperband, create_trial( (1, i), names=("epoch", "yolo1"), types=("fidelity", "integer"), results={"objective": i}, ), ) assert hyperband.suggest(100) == []
def test_cardinality(self): """Check whether space capacity is correct""" space = Space() probs = (0.1, 0.2, 0.3, 0.4) categories = ("asdfa", 2, 3, 4) dim = Categorical("yolo", OrderedDict(zip(categories, probs)), shape=2) space.register(dim) dim = Integer("yolo2", "uniform", -3, 6) space.register(dim) dim = Fidelity("epoch", 1, 9, 3) space.register(dim) assert space.cardinality == (4 ** 2) * (6 + 1) * 1 dim = Integer("yolo3", "uniform", -3, 2, shape=(3, 2)) space.register(dim) assert space.cardinality == (4 ** 2) * (6 + 1) * 1 * ((2 + 1) ** (3 * 2)) dim = Real("yolo4", "norm", 0.9) space.register(dim) assert np.inf == space.cardinality
def test_cardinality(self): """Check whether space capacity is correct""" space = Space() probs = (0.1, 0.2, 0.3, 0.4) categories = ('asdfa', 2, 3, 4) dim = Categorical('yolo', OrderedDict(zip(categories, probs)), shape=2) space.register(dim) dim = Integer('yolo2', 'uniform', -3, 6) space.register(dim) dim = Fidelity('epoch', 1, 9, 3) space.register(dim) assert (4 * 2) * 6 * 1 == space.cardinality dim = Integer('yolo3', 'uniform', -3, 2, shape=(3, 1)) space.register(dim) assert (4 * 2) * 6 * 1 * (2 * 3 * 1) == space.cardinality dim = Real('yolo4', 'norm', 0.9) space.register(dim) assert np.inf == space.cardinality
def fidelity(self, *args, **kwargs): """Create a `Fidelity` dimension.""" name = self.name return Fidelity(name, *args, **kwargs)
def space(): """Create a Space with a real dimension and a fidelity value.""" space = Space() space.register(Real("lr", "uniform", 0, 1)) space.register(Fidelity("epoch", 1, 9, 3)) return space
def test_cast(self): """Check that error is being raised.""" dim = Fidelity('epoch', 1, 10) with pytest.raises(NotImplementedError): dim.cast()
def test_interval(self): """Check that interval() is (min, max).""" dim = Fidelity('epoch', 1, 10) dim.interval() == (1, 10)
def test_default_value(self): """Make sure Fidelity simply returns `high`""" dim = Fidelity('epoch', 1, 2) assert dim.default_value == 2 dim = Fidelity('epoch', 1, 5) assert dim.default_value == 5