def test_register(self, space: Space, transformed_space: TransformedSpace):
"""Tests for the `register` method of the `RegistryMapping` class."""
original_reg = Registry()
transformed_reg = Registry()
mapping = RegistryMapping(original_registry=original_reg,
transformed_registry=transformed_reg)
original_trial = space.sample(1)[0]
transformed_trial = transformed_space.transform(original_trial)
mapping.register(original_trial, transformed_trial)
# NOTE: register doesn't actually register the trial, it just adds it to the mapping.
assert len(mapping) == 1
assert original_trial in mapping
# NOTE: Here since we assume that the trials are supposed to be registered in the registries
# externally, we can't yet iterate over the mapping (e.g. with keys(), values() or items()).
# Now we actually register the trials in the individual registries.
assert original_trial not in original_reg
original_stored_id = original_reg.register(original_trial)
assert transformed_trial not in transformed_reg
transformed_stored_id = transformed_reg.register(transformed_trial)
assert mapping._mapping == {
original_stored_id: {transformed_stored_id}
}
assert list(mapping.keys()) == [original_trial]
assert list(mapping.values()) == [[transformed_trial]]
assert mapping[original_trial] == [transformed_trial]
def test_register(self, space: Space):
"""Tests that appending a trial to a registry works as expected."""
registry = Registry()
trial = space.sample(1)[0]
registered_id = registry.register(trial)
assert len(registry) == 1
assert list(registry) == [trial]
assert registry[registered_id] == trial
def test_sample(self):
"""Check whether sampling works correctly."""
seed = 5
space = Space()
probs = (0.1, 0.2, 0.3, 0.4)
categories = ("asdfa", 2, 3, 4)
dim1 = Categorical("yolo", OrderedDict(zip(categories, probs)), shape=(2, 2))
space.register(dim1)
dim2 = Integer("yolo2", "uniform", -3, 6)
space.register(dim2)
dim3 = Real("yolo3", "norm", 0.9)
space.register(dim3)
point = space.sample(seed=seed)
rng = check_random_state(seed)
test_point = [
dict(
yolo=dim1.sample(seed=rng)[0],
yolo2=dim2.sample(seed=rng)[0],
yolo3=dim3.sample(seed=rng)[0],
)
]
assert len(point) == len(test_point) == 1
assert len(point[0].params) == len(test_point[0]) == 3
assert np.all(point[0].params["yolo"] == test_point[0]["yolo"])
assert point[0].params["yolo2"] == test_point[0]["yolo2"]
assert point[0].params["yolo3"] == test_point[0]["yolo3"]
points = space.sample(2, seed=seed)
rng = check_random_state(seed)
points1 = dim1.sample(2, seed=rng)
points2 = dim2.sample(2, seed=rng)
points3 = dim3.sample(2, seed=rng)
test_points = [
dict(yolo=points1[0], yolo2=points2[0], yolo3=points3[0]),
dict(yolo=points1[1], yolo2=points2[1], yolo3=points3[1]),
]
assert len(points) == len(test_points) == 2
for i in range(2):
assert len(points[i].params) == len(test_points[i]) == 3
assert np.all(points[i].params["yolo"] == test_points[i]["yolo"])
assert points[i].params["yolo2"] == test_points[i]["yolo2"]
assert points[i].params["yolo3"] == test_points[i]["yolo3"]
def test_sample(self, seed):
"""Check whether sampling works correctly."""
space = Space()
probs = (0.1, 0.2, 0.3, 0.4)
categories = ("asdfa", 2, 3, 4)
dim1 = Categorical("yolo",
OrderedDict(zip(categories, probs)),
shape=(2, 2))
space.register(dim1)
dim2 = Integer("yolo2", "uniform", -3, 6)
space.register(dim2)
dim3 = Real("yolo3", "norm", 0.9)
space.register(dim3)
point = space.sample(seed=seed)
test_point = [
(dim1.sample()[0], dim2.sample()[0], dim3.sample()[0]),
]
assert len(point) == len(test_point) == 1
assert len(point[0]) == len(test_point[0]) == 3
assert np.all(point[0][0] == test_point[0][0])
assert point[0][1] == test_point[0][1]
assert point[0][2] == test_point[0][2]
points = space.sample(2, seed=seed)
points1 = dim1.sample(2)
points2 = dim2.sample(2)
points3 = dim3.sample(2)
test_points = [
(points1[0], points2[0], points3[0]),
(points1[1], points2[1], points3[1]),
]
assert len(points) == len(test_points) == 2
for i in range(2):
assert len(points[i]) == len(test_points[i]) == 3
assert np.all(points[i][0] == test_points[i][0])
assert points[i][1] == test_points[i][1]
assert points[i][2] == test_points[i][2]
def test_register_overwrite_with_experiment(self, space: Space):
"""Tests that registering a trial with the same params overwrites the existing trial."""
registry = Registry()
trial = space.sample(1)[0]
registered_id = registry.register(trial)
assert len(registry) == 1
assert list(registry) == [trial]
assert registry[registered_id] == trial
experiment = "BLABLABOB" # TODO: Use an experiment fixture of some sort.
same_but_with_experiment = copy.deepcopy(trial)
same_but_with_experiment.experiment = experiment
same_id = registry.register(same_but_with_experiment)
assert same_id == registered_id
assert len(registry) == 1
assert list(registry) == [same_but_with_experiment]
def test_register_overwrite_with_status(self, space: Space, status: str):
"""Tests that registering a trial with the same params overwrites the existing trial."""
registry = Registry()
trial = space.sample(1)[0]
registered_id = registry.register(trial)
assert len(registry) == 1
assert list(registry) == [trial]
assert registry[registered_id] == trial
same_but_with_status = copy.deepcopy(trial)
same_but_with_status._status = status
same_id = registry.register(same_but_with_status)
assert same_id == registered_id
assert len(registry) == 1
assert list(registry) == [same_but_with_status]
def test_register_overwrite_with_results(self, space: Space):
"""Tests that registering a trial with the same params overwrites the existing trial."""
registry = Registry()
trial = space.sample(1)[0]
registered_id = registry.register(trial)
assert len(registry) == 1
assert list(registry) == [trial]
assert registry[registered_id] == trial
same_but_with_results = copy.deepcopy(trial)
same_but_with_results._results.append(
Trial.Result(name="objective", type="objective", value=1))
same_id = registry.register(same_but_with_results)
assert same_id == registered_id
assert len(registry) == 1
assert list(registry) == [same_but_with_results]
def test_extenal_register_doesnt_increase_len(
self, space: Space, transformed_space: TransformedSpace):
"""Test that externally registering trials in the original or transformed registries does
not affect the length of the mapping.
"""
original = Registry()
transformed = Registry()
mapping = RegistryMapping(original_registry=original,
transformed_registry=transformed)
assert not mapping
assert len(mapping) == 0
original_trial = space.sample(1)[0]
original.register(original_trial)
assert not mapping
transformed_trial = transformed_space.sample(1)[0]
transformed.register(transformed_trial)
assert not mapping
