def test_cast_list(self): """Make sure list are cast to categories and returned as list""" categories = {"asdfa": 0.1, 2: 0.2, 3.0: 0.3, "lalala": 0.4} dim = Categorical("yolo", categories) assert dim.cast(["asdfa"]) == ["asdfa"] assert dim.cast(["2"]) == [2] assert dim.cast(["3.0"]) == [3.0]
def test_cast_list(self): """Make sure list are cast to categories and returned as list""" categories = {'asdfa': 0.1, 2: 0.2, 3.0: 0.3, 'lalala': 0.4} dim = Categorical('yolo', categories) assert dim.cast(['asdfa']) == ['asdfa'] assert dim.cast(['2']) == [2] assert dim.cast(['3.0']) == [3.0]
def test_cast_array_multidim(self): """Make sure array are cast to int and returned as array of values""" categories = list(range(10)) categories[0] = "asdfa" categories[2] = "lalala" dim = Categorical("yolo", categories, shape=2) sample = np.array(["asdfa", "1"], dtype=np.object) assert np.all(dim.cast(sample) == np.array(["asdfa", 1], dtype=np.object))
def test_cast_bad_category(self): """Make sure array are cast to int and returned as array of values""" categories = list(range(10)) dim = Categorical('yolo', categories, shape=2) sample = np.array(['asdfa', '1'], dtype=np.object) with pytest.raises(ValueError) as exc: dim.cast(sample) assert "Invalid category: asdfa" in str(exc.value)
def test_cast_list_multidim(self): """Make sure array are cast to int and returned as array of values""" categories = list(range(10)) categories[0] = 'asdfa' categories[2] = 'lalala' dim = Categorical('yolo', categories, shape=2) sample = ['asdfa', '1'] # np.array(['asdfa', '1'], dtype=np.object) assert dim.cast(sample) == ['asdfa', 1]
def cat_space(space): """Return a search space with a categorical dimension""" dim3 = Categorical("cat2d", ["hello", "kitty"]) space.register(dim3) dim4 = Categorical("cat3d", ["hello", "kitty", "cat"]) space.register(dim4) return space
def test_interval_is_banned(self): """Check that calling `Categorical.interval` raises `RuntimeError`.""" categories = {'asdfa': 0.1, 2: 0.2, 3: 0.3, 4: 0.4} dim = Categorical('yolo', categories, shape=2) with pytest.raises(RuntimeError) as exc: dim.interval() assert 'not ordered' in str(exc.value)
def test_get_prior_string_dict(self): """Test that prior string can be rebuilt with dict of choices.""" categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, "lalala": 0.4} dim = Categorical( "yolo", categories, shape=2, default_value=["asdfa", "lalala"] ) assert dim.get_prior_string() == ( "choices({'asdfa': 0.10, 2: 0.20, 3: 0.30, 'lalala': 0.40}, " "shape=2, default_value=['asdfa', 'lalala'])" )
def choices(self, *args, **kwargs): """Create a `Categorical` dimension.""" name = self.name try: if isinstance(args[0], (dict, list)): return Categorical(name, *args, **kwargs) except IndexError as exc: raise TypeError( "Parameter '{}': " "Expected argument with categories.".format(name)) from exc return Categorical(name, args, **kwargs)
def test_get_prior_string_list(self): """Test that prior string can be rebuilt with list of choices.""" categories = list(range(10)) categories[0] = "asdfa" categories[2] = "lalala" dim = Categorical( "yolo", categories, shape=2, default_value=["asdfa", "lalala"] ) assert dim.get_prior_string() == ( "choices(['asdfa', 1, 'lalala', 3, 4, 5, 6, 7, 8, 9], " "shape=2, default_value=['asdfa', 'lalala'])" )
def test_probabilities_are_ok(self, seed): """Test that the probabilities given are legit using law of big numbers.""" bins = defaultdict(int) probs = (0.1, 0.2, 0.3, 0.4) categories = ('asdfa', '2', '3', '4') categories = OrderedDict(zip(categories, probs)) dim = Categorical('yolo', categories) for _ in range(500): sample = dim.sample(seed=seed)[0] bins[sample] += 1 for keys in bins.keys(): bins[keys] /= float(500) for key, value in categories.items(): assert abs(bins[key] - value) < 0.01
def test_sample_categorical_dimension(self): """Test sample values for a categorical dimension""" space = Space() categories = ["a", "b", 11, 15, 17, 18, 19, 20, 25, "c"] dim1 = Categorical("yolo1", categories) space.register(dim1) dim2 = Categorical("yolo2", categories, shape=(2)) space.register(dim2) tpe = TPE(space) obs_points = numpy.random.randint(0, 10, 100) obs_points = [categories[point] for point in obs_points] below_points = [obs_points[:25]] above_points = [obs_points[25:]] points = tpe.sample_one_dimension( dim1, 1, below_points, above_points, tpe._sample_categorical_point ) assert len(points) == 1 assert points[0] in categories obs_points_below = numpy.random.randint(0, 3, 25) obs_points_above = numpy.random.randint(3, 10, 75) below_points = [[categories[point] for point in obs_points_below]] above_points = [[categories[point] for point in obs_points_above]] points = tpe.sample_one_dimension( dim1, 1, below_points, above_points, tpe._sample_categorical_point ) assert len(points) == 1 assert points[0] in categories[:3] obs_points = numpy.random.randint(0, 10, 100) obs_points = [categories[point] for point in obs_points] below_points = [obs_points[:25], obs_points[25:50]] above_points = [obs_points[50:75], obs_points[75:]] points = tpe.sample_one_dimension( dim2, 2, below_points, above_points, tpe._sample_categorical_point ) assert len(points) == 2 assert points[0] in categories assert points[1] in categories tpe.n_ei_candidates = 0 points = tpe.sample_one_dimension( dim2, 2, below_points, above_points, tpe._sample_categorical_point ) assert len(points) == 0
def test_unsupported_space(self): """Test tpe only work for supported search space""" space = Space() dim1 = Real('yolo1', 'uniform', -10, 10) space.register(dim1) dim2 = Real('yolo2', 'reciprocal', 10, 20) space.register(dim2) categories = ['a', 0.1, 2, 'c'] dim3 = Categorical('yolo3', categories) space.register(dim3) dim4 = Fidelity('epoch', 1, 9, 3) space.register(dim4) TPE(space) space = Space() dim = Real('yolo1', 'norm', 0.9) space.register(dim) with pytest.raises(ValueError) as ex: TPE(space) assert 'TPE now only supports uniform, loguniform, uniform discrete and choices' \ in str(ex.value) space = Space() dim = Real('yolo1', 'uniform', 0.9, shape=(2, 1)) space.register(dim) with pytest.raises(ValueError) as ex: TPE(space) assert 'TPE now only supports 1D shape' in str(ex.value)
def test_capacity(self, space_each_type): """Check transformer space capacity""" tspace = build_required_space(space_each_type, type_requirement="real") assert tspace.cardinality == numpy.inf space = Space() probs = (0.1, 0.2, 0.3, 0.4) categories = ("asdfa", 2, 3, 4) dim = Categorical("yolo0", OrderedDict(zip(categories, probs)), shape=2) space.register(dim) dim = Integer("yolo2", "uniform", -3, 6) space.register(dim) tspace = build_required_space(space, type_requirement="integer") assert tspace.cardinality == (4**2) * (6 + 1) dim = Integer("yolo3", "uniform", -3, 6, shape=(2, 1)) space.register(dim) tspace = build_required_space(space, type_requirement="integer") assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1)) tspace = build_required_space(space, type_requirement="integer", shape_requirement="flattened") assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1)) tspace = build_required_space(space, type_requirement="integer", dist_requirement="linear") assert tspace.cardinality == (4**2) * (6 + 1) * ((6 + 1)**(2 * 1))
def dim2(): """Create a second example of `Dimension`.""" probs = (0.1, 0.2, 0.3, 0.4) categories = ("asdfa", "2", "3", "4") categories = OrderedDict(zip(categories, probs)) dim2 = Categorical("yolo2", categories) return dim2
def test_unsupported_space(self): """Test tpe only work for supported search space""" space = Space() dim1 = Real("yolo1", "uniform", -10, 10) space.register(dim1) dim2 = Real("yolo2", "reciprocal", 10, 20) space.register(dim2) categories = ["a", 0.1, 2, "c"] dim3 = Categorical("yolo3", categories) space.register(dim3) dim4 = Fidelity("epoch", 1, 9, 3) space.register(dim4) TPE(space) space = Space() dim = Real("yolo1", "norm", 0.9) space.register(dim) with pytest.raises(ValueError) as ex: tpe = TPE(space) tpe.space = build_required_space( space, shape_requirement=TPE.requires_shape ) assert ( "TPE now only supports uniform, loguniform, uniform discrete and choices" in str(ex.value) )
def test_that_objects_types_are_ok(self): """Check that output samples are of the correct type. Don't let numpy mess with their automatic type inference. """ categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, "lalala": 0.4} dim = Categorical("yolo", categories) assert "2" not in dim assert 2 in dim assert "asdfa" in dim dim = Categorical("yolo", categories, shape=(2,)) assert ["2", "asdfa"] not in dim assert [2, "asdfa"] in dim
def test_hierarchical_register_and_contain(self): """Register hierarchical dimensions and check if points/name are in space.""" space = Space() categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4} dim = Categorical("yolo.nested", categories, shape=2) space.register(dim) dim = Integer("yolo2.nested", "uniform", -3, 6) space.register(dim) dim = Real("yolo3", "norm", 0.9) space.register(dim) trial = Trial( params=[ {"name": "yolo.nested", "value": ["asdfa", 2], "type": "categorical"}, {"name": "yolo2.nested", "value": 1, "type": "integer"}, {"name": "yolo3", "value": 0.5, "type": "real"}, ] ) assert "yolo" in trial.params assert "nested" in trial.params["yolo"] assert "yolo2" in trial.params assert "nested" in trial.params["yolo2"] assert "yolo3" in trial.params assert trial in space
def dim2(): """Create a second example of `Dimension`.""" probs = (0.1, 0.2, 0.3, 0.4) categories = ('asdfa', '2', '3', '4') categories = OrderedDict(zip(categories, probs)) dim2 = Categorical('yolo2', categories) return dim2
def test_that_objects_types_are_ok(self): """Check that output samples are of the correct type. Don't let numpy mess with their automatic type inference. """ categories = {'asdfa': 0.1, 2: 0.2, 3: 0.3, 'lalala': 0.4} dim = Categorical('yolo', categories) assert '2' not in dim assert 2 in dim assert 'asdfa' in dim dim = Categorical('yolo', categories, shape=(2, )) assert ['2', 'asdfa'] not in dim assert [2, 'asdfa'] in dim
def _(dim: Categorical): if dim.shape: raise NotImplementedError("Array of Categorical cannot be converted.") if len(set(dim.original_dimension.prior.pk)) != 1: raise NotImplementedError( "All categories in Categorical must have the same probability.") return ng.p.Choice(dim.interval())
def test_contains_wrong_shape(self): """Check correct category but wrongly shaped array.""" categories = {'asdfa': 0.1, 2: 0.2, 3: 0.3, 4: 0.4} dim = Categorical('yolo', categories, shape=2) assert 3 not in dim assert ('asdfa', 2) in dim
def test_perturb_cat(self): explore = PerturbExplore() rng = RNGStub() rng.randint = lambda low, high, size: [1] rng.choice = lambda choices: choices[0] dim = Categorical("name", ["one", "two", 3, 4.0]) assert explore.perturb_cat(rng, "whatever", dim) in dim
def categorical_grid(dim: Categorical, num: int): """Build categorical grid, that is, all categories""" categories = dim.interval() if len(categories) != num: log.warning( f"Categorical dimension {dim.name} does not have {num} choices: {categories}. " "Will use {len(categories)} choices instead.") return categories
def test_repr_too_many_cats(self): """Check ellipsis on str/repr of too many categories.""" categories = tuple(range(10)) dim = Categorical('yolo', categories, shape=2) assert str(dim) == "Categorical(name=yolo, " \ "prior={0: 0.10, 1: 0.10, ..., 8: 0.10, 9: 0.10}, " \ "shape=(2,), default value=None)"
def test_build_grid_cannot_limit_size(caplog): """Test that when choices are too large GridSearch raises ValueError""" dim1 = Real("dim1", "uniform", 0, 1) dim2 = Integer("dim2", "uniform", 0, 10) dim3 = Categorical("dim3", "abcde") dim4 = Categorical("dim4", "abcde") space = Space() space.register(dim1) space.register(dim2) space.register(dim3) space.register(dim4) with pytest.raises(ValueError) as exc: GridSearch.build_grid(space, {k: 5 for k in space}, 10) assert exc.match( "Cannot build a grid smaller than 10. " "Try reducing the number of choices in categorical dimensions.")
def test_with_dict(self, seed): """Test Categorical.__init__ with a dictionary.""" probs = (0.1, 0.2, 0.3, 0.4) categories = ('asdfa', 2, 3, 4) dim = Categorical('yolo', OrderedDict(zip(categories, probs))) samples = dim.sample(seed=seed) assert len(samples) == 1 assert samples[0] == 2 assert dim._probs == probs assert categories == dim.categories assert 2 in dim assert 0 not in dim assert dim.name == 'yolo' assert dim.type == 'categorical' assert dim.shape == ()
def test_sample(self): """Check whether sampling works correctly.""" seed = 5 space = Space() probs = (0.1, 0.2, 0.3, 0.4) categories = ("asdfa", 2, 3, 4) dim1 = Categorical("yolo", OrderedDict(zip(categories, probs)), shape=(2, 2)) space.register(dim1) dim2 = Integer("yolo2", "uniform", -3, 6) space.register(dim2) dim3 = Real("yolo3", "norm", 0.9) space.register(dim3) point = space.sample(seed=seed) rng = check_random_state(seed) test_point = [ dict( yolo=dim1.sample(seed=rng)[0], yolo2=dim2.sample(seed=rng)[0], yolo3=dim3.sample(seed=rng)[0], ) ] assert len(point) == len(test_point) == 1 assert len(point[0].params) == len(test_point[0]) == 3 assert np.all(point[0].params["yolo"] == test_point[0]["yolo"]) assert point[0].params["yolo2"] == test_point[0]["yolo2"] assert point[0].params["yolo3"] == test_point[0]["yolo3"] points = space.sample(2, seed=seed) rng = check_random_state(seed) points1 = dim1.sample(2, seed=rng) points2 = dim2.sample(2, seed=rng) points3 = dim3.sample(2, seed=rng) test_points = [ dict(yolo=points1[0], yolo2=points2[0], yolo3=points3[0]), dict(yolo=points1[1], yolo2=points2[1], yolo3=points3[1]), ] assert len(points) == len(test_points) == 2 for i in range(2): assert len(points[i].params) == len(test_points[i]) == 3 assert np.all(points[i].params["yolo"] == test_points[i]["yolo"]) assert points[i].params["yolo2"] == test_points[i]["yolo2"] assert points[i].params["yolo3"] == test_points[i]["yolo3"]
def space(): """Construct a simple space with every possible kind of Dimension.""" space = Space() categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4} dim = Categorical("yolo", categories, shape=2) space.register(dim) dim = Integer("yolo2", "uniform", -3, 6) space.register(dim) dim = Real("yolo3", "alpha", 0.9) space.register(dim) return space
def test_with_tuple(self, seed): """Test Categorical.__init__ with a tuple.""" categories = ('asdfa', 2) dim = Categorical('yolo', categories) samples = dim.sample(seed=seed) assert len(samples) == 1 assert samples[0] == 'asdfa' assert dim._probs == (0.5, 0.5) assert categories == dim.categories assert 2 in dim assert 3 not in dim assert str(dim) == "Categorical(name=yolo, prior={asdfa: 0.50, 2: 0.50}, "\ "shape=(), default value=None)" assert dim.name == 'yolo' assert dim.type == 'categorical' assert dim.shape == ()