def test_cast_list(self):
"""Make sure list are cast to categories and returned as list"""
categories = {"asdfa": 0.1, 2: 0.2, 3.0: 0.3, "lalala": 0.4}
dim = Categorical("yolo", categories)
assert dim.cast(["asdfa"]) == ["asdfa"]
assert dim.cast(["2"]) == [2]
assert dim.cast(["3.0"]) == [3.0]
def test_cast_list(self):
"""Make sure list are cast to categories and returned as list"""
categories = {'asdfa': 0.1, 2: 0.2, 3.0: 0.3, 'lalala': 0.4}
dim = Categorical('yolo', categories)
assert dim.cast(['asdfa']) == ['asdfa']
assert dim.cast(['2']) == [2]
assert dim.cast(['3.0']) == [3.0]
def test_cast_array_multidim(self):
"""Make sure array are cast to int and returned as array of values"""
categories = list(range(10))
categories[0] = "asdfa"
categories[2] = "lalala"
dim = Categorical("yolo", categories, shape=2)
sample = np.array(["asdfa", "1"], dtype=np.object)
assert np.all(dim.cast(sample) == np.array(["asdfa", 1], dtype=np.object))
def test_cast_bad_category(self):
"""Make sure array are cast to int and returned as array of values"""
categories = list(range(10))
dim = Categorical('yolo', categories, shape=2)
sample = np.array(['asdfa', '1'], dtype=np.object)
with pytest.raises(ValueError) as exc:
dim.cast(sample)
assert "Invalid category: asdfa" in str(exc.value)
def test_cast_list_multidim(self):
"""Make sure array are cast to int and returned as array of values"""
categories = list(range(10))
categories[0] = 'asdfa'
categories[2] = 'lalala'
dim = Categorical('yolo', categories, shape=2)
sample = ['asdfa', '1'] # np.array(['asdfa', '1'], dtype=np.object)
assert dim.cast(sample) == ['asdfa', 1]
def cat_space(space):
"""Return a search space with a categorical dimension"""
dim3 = Categorical("cat2d", ["hello", "kitty"])
space.register(dim3)
dim4 = Categorical("cat3d", ["hello", "kitty", "cat"])
space.register(dim4)
return space
def test_interval_is_banned(self):
"""Check that calling `Categorical.interval` raises `RuntimeError`."""
categories = {'asdfa': 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical('yolo', categories, shape=2)
with pytest.raises(RuntimeError) as exc:
dim.interval()
assert 'not ordered' in str(exc.value)
def test_get_prior_string_dict(self):
"""Test that prior string can be rebuilt with dict of choices."""
categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, "lalala": 0.4}
dim = Categorical(
"yolo", categories, shape=2, default_value=["asdfa", "lalala"]
)
assert dim.get_prior_string() == (
"choices({'asdfa': 0.10, 2: 0.20, 3: 0.30, 'lalala': 0.40}, "
"shape=2, default_value=['asdfa', 'lalala'])"
)
def choices(self, *args, **kwargs):
"""Create a `Categorical` dimension."""
name = self.name
try:
if isinstance(args[0], (dict, list)):
return Categorical(name, *args, **kwargs)
except IndexError as exc:
raise TypeError(
"Parameter '{}': "
"Expected argument with categories.".format(name)) from exc
return Categorical(name, args, **kwargs)
def test_get_prior_string_list(self):
"""Test that prior string can be rebuilt with list of choices."""
categories = list(range(10))
categories[0] = "asdfa"
categories[2] = "lalala"
dim = Categorical(
"yolo", categories, shape=2, default_value=["asdfa", "lalala"]
)
assert dim.get_prior_string() == (
"choices(['asdfa', 1, 'lalala', 3, 4, 5, 6, 7, 8, 9], "
"shape=2, default_value=['asdfa', 'lalala'])"
)
def test_probabilities_are_ok(self, seed):
"""Test that the probabilities given are legit using law of big numbers."""
bins = defaultdict(int)
probs = (0.1, 0.2, 0.3, 0.4)
categories = ('asdfa', '2', '3', '4')
categories = OrderedDict(zip(categories, probs))
dim = Categorical('yolo', categories)
for _ in range(500):
sample = dim.sample(seed=seed)[0]
bins[sample] += 1
for keys in bins.keys():
bins[keys] /= float(500)
for key, value in categories.items():
assert abs(bins[key] - value) < 0.01
def test_sample_categorical_dimension(self):
"""Test sample values for a categorical dimension"""
space = Space()
categories = ["a", "b", 11, 15, 17, 18, 19, 20, 25, "c"]
dim1 = Categorical("yolo1", categories)
space.register(dim1)
dim2 = Categorical("yolo2", categories, shape=(2))
space.register(dim2)
tpe = TPE(space)
obs_points = numpy.random.randint(0, 10, 100)
obs_points = [categories[point] for point in obs_points]
below_points = [obs_points[:25]]
above_points = [obs_points[25:]]
points = tpe.sample_one_dimension(
dim1, 1, below_points, above_points, tpe._sample_categorical_point
)
assert len(points) == 1
assert points[0] in categories
obs_points_below = numpy.random.randint(0, 3, 25)
obs_points_above = numpy.random.randint(3, 10, 75)
below_points = [[categories[point] for point in obs_points_below]]
above_points = [[categories[point] for point in obs_points_above]]
points = tpe.sample_one_dimension(
dim1, 1, below_points, above_points, tpe._sample_categorical_point
)
assert len(points) == 1
assert points[0] in categories[:3]
obs_points = numpy.random.randint(0, 10, 100)
obs_points = [categories[point] for point in obs_points]
below_points = [obs_points[:25], obs_points[25:50]]
above_points = [obs_points[50:75], obs_points[75:]]
points = tpe.sample_one_dimension(
dim2, 2, below_points, above_points, tpe._sample_categorical_point
)
assert len(points) == 2
assert points[0] in categories
assert points[1] in categories
tpe.n_ei_candidates = 0
points = tpe.sample_one_dimension(
dim2, 2, below_points, above_points, tpe._sample_categorical_point
)
assert len(points) == 0
def test_unsupported_space(self):
"""Test tpe only work for supported search space"""
space = Space()
dim1 = Real('yolo1', 'uniform', -10, 10)
space.register(dim1)
dim2 = Real('yolo2', 'reciprocal', 10, 20)
space.register(dim2)
categories = ['a', 0.1, 2, 'c']
dim3 = Categorical('yolo3', categories)
space.register(dim3)
dim4 = Fidelity('epoch', 1, 9, 3)
space.register(dim4)
TPE(space)
space = Space()
dim = Real('yolo1', 'norm', 0.9)
space.register(dim)
with pytest.raises(ValueError) as ex:
TPE(space)
assert 'TPE now only supports uniform, loguniform, uniform discrete and choices' \
in str(ex.value)
space = Space()
dim = Real('yolo1', 'uniform', 0.9, shape=(2, 1))
space.register(dim)
with pytest.raises(ValueError) as ex:
TPE(space)
assert 'TPE now only supports 1D shape' in str(ex.value)
def test_capacity(self, space_each_type):
"""Check transformer space capacity"""
tspace = build_required_space(space_each_type, type_requirement="real")
assert tspace.cardinality == numpy.inf
space = Space()
probs = (0.1, 0.2, 0.3, 0.4)
categories = ("asdfa", 2, 3, 4)
dim = Categorical("yolo0",
OrderedDict(zip(categories, probs)),
shape=2)
space.register(dim)
dim = Integer("yolo2", "uniform", -3, 6)
space.register(dim)
tspace = build_required_space(space, type_requirement="integer")
assert tspace.cardinality == (4**2) * (6 + 1)
dim = Integer("yolo3", "uniform", -3, 6, shape=(2, 1))
space.register(dim)
tspace = build_required_space(space, type_requirement="integer")
assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
tspace = build_required_space(space,
type_requirement="integer",
shape_requirement="flattened")
assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
tspace = build_required_space(space,
type_requirement="integer",
dist_requirement="linear")
assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
def dim2():
"""Create a second example of `Dimension`."""
probs = (0.1, 0.2, 0.3, 0.4)
categories = ("asdfa", "2", "3", "4")
categories = OrderedDict(zip(categories, probs))
dim2 = Categorical("yolo2", categories)
return dim2
def test_unsupported_space(self):
"""Test tpe only work for supported search space"""
space = Space()
dim1 = Real("yolo1", "uniform", -10, 10)
space.register(dim1)
dim2 = Real("yolo2", "reciprocal", 10, 20)
space.register(dim2)
categories = ["a", 0.1, 2, "c"]
dim3 = Categorical("yolo3", categories)
space.register(dim3)
dim4 = Fidelity("epoch", 1, 9, 3)
space.register(dim4)
TPE(space)
space = Space()
dim = Real("yolo1", "norm", 0.9)
space.register(dim)
with pytest.raises(ValueError) as ex:
tpe = TPE(space)
tpe.space = build_required_space(
space, shape_requirement=TPE.requires_shape
)
assert (
"TPE now only supports uniform, loguniform, uniform discrete and choices"
in str(ex.value)
)
def test_that_objects_types_are_ok(self):
"""Check that output samples are of the correct type.
Don't let numpy mess with their automatic type inference.
"""
categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, "lalala": 0.4}
dim = Categorical("yolo", categories)
assert "2" not in dim
assert 2 in dim
assert "asdfa" in dim
dim = Categorical("yolo", categories, shape=(2,))
assert ["2", "asdfa"] not in dim
assert [2, "asdfa"] in dim
def test_hierarchical_register_and_contain(self):
"""Register hierarchical dimensions and check if points/name are in space."""
space = Space()
categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical("yolo.nested", categories, shape=2)
space.register(dim)
dim = Integer("yolo2.nested", "uniform", -3, 6)
space.register(dim)
dim = Real("yolo3", "norm", 0.9)
space.register(dim)
trial = Trial(
params=[
{"name": "yolo.nested", "value": ["asdfa", 2], "type": "categorical"},
{"name": "yolo2.nested", "value": 1, "type": "integer"},
{"name": "yolo3", "value": 0.5, "type": "real"},
]
)
assert "yolo" in trial.params
assert "nested" in trial.params["yolo"]
assert "yolo2" in trial.params
assert "nested" in trial.params["yolo2"]
assert "yolo3" in trial.params
assert trial in space
def dim2():
"""Create a second example of `Dimension`."""
probs = (0.1, 0.2, 0.3, 0.4)
categories = ('asdfa', '2', '3', '4')
categories = OrderedDict(zip(categories, probs))
dim2 = Categorical('yolo2', categories)
return dim2
def test_that_objects_types_are_ok(self):
"""Check that output samples are of the correct type.
Don't let numpy mess with their automatic type inference.
"""
categories = {'asdfa': 0.1, 2: 0.2, 3: 0.3, 'lalala': 0.4}
dim = Categorical('yolo', categories)
assert '2' not in dim
assert 2 in dim
assert 'asdfa' in dim
dim = Categorical('yolo', categories, shape=(2, ))
assert ['2', 'asdfa'] not in dim
assert [2, 'asdfa'] in dim
def _(dim: Categorical):
if dim.shape:
raise NotImplementedError("Array of Categorical cannot be converted.")
if len(set(dim.original_dimension.prior.pk)) != 1:
raise NotImplementedError(
"All categories in Categorical must have the same probability.")
return ng.p.Choice(dim.interval())
def test_contains_wrong_shape(self):
"""Check correct category but wrongly shaped array."""
categories = {'asdfa': 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical('yolo', categories, shape=2)
assert 3 not in dim
assert ('asdfa', 2) in dim
def test_perturb_cat(self):
explore = PerturbExplore()
rng = RNGStub()
rng.randint = lambda low, high, size: [1]
rng.choice = lambda choices: choices[0]
dim = Categorical("name", ["one", "two", 3, 4.0])
assert explore.perturb_cat(rng, "whatever", dim) in dim
def categorical_grid(dim: Categorical, num: int):
"""Build categorical grid, that is, all categories"""
categories = dim.interval()
if len(categories) != num:
log.warning(
f"Categorical dimension {dim.name} does not have {num} choices: {categories}. "
"Will use {len(categories)} choices instead.")
return categories
def test_repr_too_many_cats(self):
"""Check ellipsis on str/repr of too many categories."""
categories = tuple(range(10))
dim = Categorical('yolo', categories, shape=2)
assert str(dim) == "Categorical(name=yolo, " \
"prior={0: 0.10, 1: 0.10, ..., 8: 0.10, 9: 0.10}, " \
"shape=(2,), default value=None)"
def test_build_grid_cannot_limit_size(caplog):
"""Test that when choices are too large GridSearch raises ValueError"""
dim1 = Real("dim1", "uniform", 0, 1)
dim2 = Integer("dim2", "uniform", 0, 10)
dim3 = Categorical("dim3", "abcde")
dim4 = Categorical("dim4", "abcde")
space = Space()
space.register(dim1)
space.register(dim2)
space.register(dim3)
space.register(dim4)
with pytest.raises(ValueError) as exc:
GridSearch.build_grid(space, {k: 5 for k in space}, 10)
assert exc.match(
"Cannot build a grid smaller than 10. "
"Try reducing the number of choices in categorical dimensions.")
def test_with_dict(self, seed):
"""Test Categorical.__init__ with a dictionary."""
probs = (0.1, 0.2, 0.3, 0.4)
categories = ('asdfa', 2, 3, 4)
dim = Categorical('yolo', OrderedDict(zip(categories, probs)))
samples = dim.sample(seed=seed)
assert len(samples) == 1
assert samples[0] == 2
assert dim._probs == probs
assert categories == dim.categories
assert 2 in dim
assert 0 not in dim
assert dim.name == 'yolo'
assert dim.type == 'categorical'
assert dim.shape == ()
def test_sample(self):
"""Check whether sampling works correctly."""
seed = 5
space = Space()
probs = (0.1, 0.2, 0.3, 0.4)
categories = ("asdfa", 2, 3, 4)
dim1 = Categorical("yolo", OrderedDict(zip(categories, probs)), shape=(2, 2))
space.register(dim1)
dim2 = Integer("yolo2", "uniform", -3, 6)
space.register(dim2)
dim3 = Real("yolo3", "norm", 0.9)
space.register(dim3)
point = space.sample(seed=seed)
rng = check_random_state(seed)
test_point = [
dict(
yolo=dim1.sample(seed=rng)[0],
yolo2=dim2.sample(seed=rng)[0],
yolo3=dim3.sample(seed=rng)[0],
)
]
assert len(point) == len(test_point) == 1
assert len(point[0].params) == len(test_point[0]) == 3
assert np.all(point[0].params["yolo"] == test_point[0]["yolo"])
assert point[0].params["yolo2"] == test_point[0]["yolo2"]
assert point[0].params["yolo3"] == test_point[0]["yolo3"]
points = space.sample(2, seed=seed)
rng = check_random_state(seed)
points1 = dim1.sample(2, seed=rng)
points2 = dim2.sample(2, seed=rng)
points3 = dim3.sample(2, seed=rng)
test_points = [
dict(yolo=points1[0], yolo2=points2[0], yolo3=points3[0]),
dict(yolo=points1[1], yolo2=points2[1], yolo3=points3[1]),
]
assert len(points) == len(test_points) == 2
for i in range(2):
assert len(points[i].params) == len(test_points[i]) == 3
assert np.all(points[i].params["yolo"] == test_points[i]["yolo"])
assert points[i].params["yolo2"] == test_points[i]["yolo2"]
assert points[i].params["yolo3"] == test_points[i]["yolo3"]
def space():
"""Construct a simple space with every possible kind of Dimension."""
space = Space()
categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical("yolo", categories, shape=2)
space.register(dim)
dim = Integer("yolo2", "uniform", -3, 6)
space.register(dim)
dim = Real("yolo3", "alpha", 0.9)
space.register(dim)
return space
def test_with_tuple(self, seed):
"""Test Categorical.__init__ with a tuple."""
categories = ('asdfa', 2)
dim = Categorical('yolo', categories)
samples = dim.sample(seed=seed)
assert len(samples) == 1
assert samples[0] == 'asdfa'
assert dim._probs == (0.5, 0.5)
assert categories == dim.categories
assert 2 in dim
assert 3 not in dim
assert str(dim) == "Categorical(name=yolo, prior={asdfa: 0.50, 2: 0.50}, "\
"shape=(), default value=None)"
assert dim.name == 'yolo'
assert dim.type == 'categorical'
assert dim.shape == ()
