def _test_repeater(self, repeat):
ray.init(num_cpus=4)
class TestSuggestion(SuggestionAlgorithm):
count = 0
def suggest(self, trial_id):
return {"test_variable": 5}
def on_trial_complete(self, *args, **kwargs):
self.count += 1
alg = TestSuggestion(metric="episode_reward_mean")
repeat_alg = Repeater(alg, repeat=repeat, set_index=False)
experiment_spec = {
"run": "__fake",
"num_samples": 1,
"stop": {
"training_iteration": 1
}
}
repeat_alg.add_configurations({"test": experiment_spec})
runner = TrialRunner(search_alg=repeat_alg)
for i in range(repeat * 2):
runner.step()
trials = runner.get_trials()
self.assertEquals(len(trials), repeat)
def _test_repeater(self, num_samples, repeat):
class TestSuggestion(Searcher):
index = 0
def suggest(self, trial_id):
self.index += 1
return {"test_variable": 5 + self.index}
def on_trial_complete(self, *args, **kwargs):
return
searcher = TestSuggestion(metric="episode_reward_mean")
repeat_searcher = Repeater(searcher, repeat=repeat, set_index=False)
alg = SearchGenerator(repeat_searcher)
experiment_spec = {
"run": "__fake",
"num_samples": num_samples,
"stop": {
"training_iteration": 1
}
}
alg.add_configurations({"test": experiment_spec})
runner = TrialRunner(search_alg=alg)
while not runner.is_finished():
runner.step()
return runner.get_trials()
def create_searcher():
class TestSuggestion(Searcher):
def __init__(self, index):
self.index = index
self.returned_result = []
super().__init__(metric="episode_reward_mean", mode="max")
def suggest(self, trial_id):
self.index += 1
return {"test_variable": self.index}
def on_trial_complete(self, trial_id, result=None, **kwargs):
self.returned_result.append(result)
def save(self, checkpoint_path):
with open(checkpoint_path, "wb") as f:
pickle.dump(self.__dict__, f)
def restore(self, checkpoint_path):
with open(checkpoint_path, "rb") as f:
self.__dict__.update(pickle.load(f))
searcher = TestSuggestion(0)
searcher = ConcurrencyLimiter(searcher, max_concurrent=2)
searcher = Repeater(searcher, repeat=3, set_index=False)
search_alg = SearchGenerator(searcher)
experiment_spec = {
"run": "__fake",
"num_samples": 20,
"stop": {"training_iteration": 2},
}
experiments = [Experiment.from_json("test", experiment_spec)]
search_alg.add_configurations(experiments)
return search_alg
def testSetGetRepeater(self):
class TestSuggestion(Searcher):
def __init__(self, index):
self.index = index
self.returned_result = []
super().__init__(metric="result", mode="max")
def suggest(self, trial_id):
self.index += 1
return {"score": self.index}
def on_trial_complete(self, trial_id, result=None, **kwargs):
self.returned_result.append(result)
searcher = TestSuggestion(0)
repeater1 = Repeater(searcher, repeat=3, set_index=False)
for i in range(3):
assert repeater1.suggest(f"test_{i}")["score"] == 1
for i in range(2): # An incomplete set of results
assert repeater1.suggest(f"test_{i}_2")["score"] == 2
# Restore a new one
state = repeater1.get_state()
del repeater1
new_repeater = Repeater(searcher, repeat=1, set_index=True)
new_repeater.set_state(state)
assert new_repeater.repeat == 3
assert new_repeater.suggest("test_2_2")["score"] == 2
assert new_repeater.suggest("test_x")["score"] == 3
# Report results
for i in range(3):
new_repeater.on_trial_complete(f"test_{i}", {"result": 2})
for i in range(3):
new_repeater.on_trial_complete(f"test_{i}_2", {"result": -i * 10})
assert len(new_repeater.searcher.returned_result) == 2
assert new_repeater.searcher.returned_result[-1] == {"result": -10}
# Finish the rest of the last trial group
new_repeater.on_trial_complete("test_x", {"result": 3})
assert new_repeater.suggest("test_y")["score"] == 3
new_repeater.on_trial_complete("test_y", {"result": 3})
assert len(new_repeater.searcher.returned_result) == 2
assert new_repeater.suggest("test_z")["score"] == 3
new_repeater.on_trial_complete("test_z", {"result": 3})
assert len(new_repeater.searcher.returned_result) == 3
assert new_repeater.searcher.returned_result[-1] == {"result": 3}
