def test_register(self, space: Space, transformed_space: TransformedSpace): """Tests for the `register` method of the `RegistryMapping` class.""" original_reg = Registry() transformed_reg = Registry() mapping = RegistryMapping(original_registry=original_reg, transformed_registry=transformed_reg) original_trial = space.sample(1)[0] transformed_trial = transformed_space.transform(original_trial) mapping.register(original_trial, transformed_trial) # NOTE: register doesn't actually register the trial, it just adds it to the mapping. assert len(mapping) == 1 assert original_trial in mapping # NOTE: Here since we assume that the trials are supposed to be registered in the registries # externally, we can't yet iterate over the mapping (e.g. with keys(), values() or items()). # Now we actually register the trials in the individual registries. assert original_trial not in original_reg original_stored_id = original_reg.register(original_trial) assert transformed_trial not in transformed_reg transformed_stored_id = transformed_reg.register(transformed_trial) assert mapping._mapping == { original_stored_id: {transformed_stored_id} } assert list(mapping.keys()) == [original_trial] assert list(mapping.values()) == [[transformed_trial]] assert mapping[original_trial] == [transformed_trial]
def test_register(self, space: Space): """Tests that appending a trial to a registry works as expected.""" registry = Registry() trial = space.sample(1)[0] registered_id = registry.register(trial) assert len(registry) == 1 assert list(registry) == [trial] assert registry[registered_id] == trial
def test_sample(self): """Check whether sampling works correctly.""" seed = 5 space = Space() probs = (0.1, 0.2, 0.3, 0.4) categories = ("asdfa", 2, 3, 4) dim1 = Categorical("yolo", OrderedDict(zip(categories, probs)), shape=(2, 2)) space.register(dim1) dim2 = Integer("yolo2", "uniform", -3, 6) space.register(dim2) dim3 = Real("yolo3", "norm", 0.9) space.register(dim3) point = space.sample(seed=seed) rng = check_random_state(seed) test_point = [ dict( yolo=dim1.sample(seed=rng)[0], yolo2=dim2.sample(seed=rng)[0], yolo3=dim3.sample(seed=rng)[0], ) ] assert len(point) == len(test_point) == 1 assert len(point[0].params) == len(test_point[0]) == 3 assert np.all(point[0].params["yolo"] == test_point[0]["yolo"]) assert point[0].params["yolo2"] == test_point[0]["yolo2"] assert point[0].params["yolo3"] == test_point[0]["yolo3"] points = space.sample(2, seed=seed) rng = check_random_state(seed) points1 = dim1.sample(2, seed=rng) points2 = dim2.sample(2, seed=rng) points3 = dim3.sample(2, seed=rng) test_points = [ dict(yolo=points1[0], yolo2=points2[0], yolo3=points3[0]), dict(yolo=points1[1], yolo2=points2[1], yolo3=points3[1]), ] assert len(points) == len(test_points) == 2 for i in range(2): assert len(points[i].params) == len(test_points[i]) == 3 assert np.all(points[i].params["yolo"] == test_points[i]["yolo"]) assert points[i].params["yolo2"] == test_points[i]["yolo2"] assert points[i].params["yolo3"] == test_points[i]["yolo3"]
def test_sample(self, seed): """Check whether sampling works correctly.""" space = Space() probs = (0.1, 0.2, 0.3, 0.4) categories = ("asdfa", 2, 3, 4) dim1 = Categorical("yolo", OrderedDict(zip(categories, probs)), shape=(2, 2)) space.register(dim1) dim2 = Integer("yolo2", "uniform", -3, 6) space.register(dim2) dim3 = Real("yolo3", "norm", 0.9) space.register(dim3) point = space.sample(seed=seed) test_point = [ (dim1.sample()[0], dim2.sample()[0], dim3.sample()[0]), ] assert len(point) == len(test_point) == 1 assert len(point[0]) == len(test_point[0]) == 3 assert np.all(point[0][0] == test_point[0][0]) assert point[0][1] == test_point[0][1] assert point[0][2] == test_point[0][2] points = space.sample(2, seed=seed) points1 = dim1.sample(2) points2 = dim2.sample(2) points3 = dim3.sample(2) test_points = [ (points1[0], points2[0], points3[0]), (points1[1], points2[1], points3[1]), ] assert len(points) == len(test_points) == 2 for i in range(2): assert len(points[i]) == len(test_points[i]) == 3 assert np.all(points[i][0] == test_points[i][0]) assert points[i][1] == test_points[i][1] assert points[i][2] == test_points[i][2]
def test_register_overwrite_with_experiment(self, space: Space): """Tests that registering a trial with the same params overwrites the existing trial.""" registry = Registry() trial = space.sample(1)[0] registered_id = registry.register(trial) assert len(registry) == 1 assert list(registry) == [trial] assert registry[registered_id] == trial experiment = "BLABLABOB" # TODO: Use an experiment fixture of some sort. same_but_with_experiment = copy.deepcopy(trial) same_but_with_experiment.experiment = experiment same_id = registry.register(same_but_with_experiment) assert same_id == registered_id assert len(registry) == 1 assert list(registry) == [same_but_with_experiment]
def test_register_overwrite_with_status(self, space: Space, status: str): """Tests that registering a trial with the same params overwrites the existing trial.""" registry = Registry() trial = space.sample(1)[0] registered_id = registry.register(trial) assert len(registry) == 1 assert list(registry) == [trial] assert registry[registered_id] == trial same_but_with_status = copy.deepcopy(trial) same_but_with_status._status = status same_id = registry.register(same_but_with_status) assert same_id == registered_id assert len(registry) == 1 assert list(registry) == [same_but_with_status]
def test_register_overwrite_with_results(self, space: Space): """Tests that registering a trial with the same params overwrites the existing trial.""" registry = Registry() trial = space.sample(1)[0] registered_id = registry.register(trial) assert len(registry) == 1 assert list(registry) == [trial] assert registry[registered_id] == trial same_but_with_results = copy.deepcopy(trial) same_but_with_results._results.append( Trial.Result(name="objective", type="objective", value=1)) same_id = registry.register(same_but_with_results) assert same_id == registered_id assert len(registry) == 1 assert list(registry) == [same_but_with_results]
def test_extenal_register_doesnt_increase_len( self, space: Space, transformed_space: TransformedSpace): """Test that externally registering trials in the original or transformed registries does not affect the length of the mapping. """ original = Registry() transformed = Registry() mapping = RegistryMapping(original_registry=original, transformed_registry=transformed) assert not mapping assert len(mapping) == 0 original_trial = space.sample(1)[0] original.register(original_trial) assert not mapping transformed_trial = transformed_space.sample(1)[0] transformed.register(transformed_trial) assert not mapping