def _test_repeater(self, repeat): ray.init(num_cpus=4) class TestSuggestion(SuggestionAlgorithm): count = 0 def suggest(self, trial_id): return {"test_variable": 5} def on_trial_complete(self, *args, **kwargs): self.count += 1 alg = TestSuggestion(metric="episode_reward_mean") repeat_alg = Repeater(alg, repeat=repeat, set_index=False) experiment_spec = { "run": "__fake", "num_samples": 1, "stop": { "training_iteration": 1 } } repeat_alg.add_configurations({"test": experiment_spec}) runner = TrialRunner(search_alg=repeat_alg) for i in range(repeat * 2): runner.step() trials = runner.get_trials() self.assertEquals(len(trials), repeat)
def _test_repeater(self, num_samples, repeat): class TestSuggestion(Searcher): index = 0 def suggest(self, trial_id): self.index += 1 return {"test_variable": 5 + self.index} def on_trial_complete(self, *args, **kwargs): return searcher = TestSuggestion(metric="episode_reward_mean") repeat_searcher = Repeater(searcher, repeat=repeat, set_index=False) alg = SearchGenerator(repeat_searcher) experiment_spec = { "run": "__fake", "num_samples": num_samples, "stop": { "training_iteration": 1 } } alg.add_configurations({"test": experiment_spec}) runner = TrialRunner(search_alg=alg) while not runner.is_finished(): runner.step() return runner.get_trials()
def create_searcher(): class TestSuggestion(Searcher): def __init__(self, index): self.index = index self.returned_result = [] super().__init__(metric="episode_reward_mean", mode="max") def suggest(self, trial_id): self.index += 1 return {"test_variable": self.index} def on_trial_complete(self, trial_id, result=None, **kwargs): self.returned_result.append(result) def save(self, checkpoint_path): with open(checkpoint_path, "wb") as f: pickle.dump(self.__dict__, f) def restore(self, checkpoint_path): with open(checkpoint_path, "rb") as f: self.__dict__.update(pickle.load(f)) searcher = TestSuggestion(0) searcher = ConcurrencyLimiter(searcher, max_concurrent=2) searcher = Repeater(searcher, repeat=3, set_index=False) search_alg = SearchGenerator(searcher) experiment_spec = { "run": "__fake", "num_samples": 20, "stop": {"training_iteration": 2}, } experiments = [Experiment.from_json("test", experiment_spec)] search_alg.add_configurations(experiments) return search_alg
def testSetGetRepeater(self): class TestSuggestion(Searcher): def __init__(self, index): self.index = index self.returned_result = [] super().__init__(metric="result", mode="max") def suggest(self, trial_id): self.index += 1 return {"score": self.index} def on_trial_complete(self, trial_id, result=None, **kwargs): self.returned_result.append(result) searcher = TestSuggestion(0) repeater1 = Repeater(searcher, repeat=3, set_index=False) for i in range(3): assert repeater1.suggest(f"test_{i}")["score"] == 1 for i in range(2): # An incomplete set of results assert repeater1.suggest(f"test_{i}_2")["score"] == 2 # Restore a new one state = repeater1.get_state() del repeater1 new_repeater = Repeater(searcher, repeat=1, set_index=True) new_repeater.set_state(state) assert new_repeater.repeat == 3 assert new_repeater.suggest("test_2_2")["score"] == 2 assert new_repeater.suggest("test_x")["score"] == 3 # Report results for i in range(3): new_repeater.on_trial_complete(f"test_{i}", {"result": 2}) for i in range(3): new_repeater.on_trial_complete(f"test_{i}_2", {"result": -i * 10}) assert len(new_repeater.searcher.returned_result) == 2 assert new_repeater.searcher.returned_result[-1] == {"result": -10} # Finish the rest of the last trial group new_repeater.on_trial_complete("test_x", {"result": 3}) assert new_repeater.suggest("test_y")["score"] == 3 new_repeater.on_trial_complete("test_y", {"result": 3}) assert len(new_repeater.searcher.returned_result) == 2 assert new_repeater.suggest("test_z")["score"] == 3 new_repeater.on_trial_complete("test_z", {"result": 3}) assert len(new_repeater.searcher.returned_result) == 3 assert new_repeater.searcher.returned_result[-1] == {"result": 3}