def testSetGetLimiter(self):
class TestSuggestion(Searcher):
def __init__(self, index):
self.index = index
self.returned_result = []
super().__init__(metric="result", mode="max")
def suggest(self, trial_id):
self.index += 1
return {"score": self.index}
def on_trial_complete(self, trial_id, result=None, **kwargs):
self.returned_result.append(result)
searcher = TestSuggestion(0)
limiter = ConcurrencyLimiter(searcher, max_concurrent=2)
assert limiter.suggest("test_1")["score"] == 1
assert limiter.suggest("test_2")["score"] == 2
assert limiter.suggest("test_3") is None
state = limiter.get_state()
del limiter
limiter2 = ConcurrencyLimiter(searcher, max_concurrent=3)
limiter2.set_state(state)
assert limiter2.suggest("test_4") is None
assert limiter2.suggest("test_5") is None
limiter2.on_trial_complete("test_1", {"result": 3})
limiter2.on_trial_complete("test_2", {"result": 3})
assert limiter2.suggest("test_3")["score"] == 3
def testBatchLimiter(self):
ray.init(num_cpus=4)
class TestSuggestion(Searcher):
def __init__(self, index):
self.index = index
self.returned_result = []
super().__init__(metric="result", mode="max")
def suggest(self, trial_id):
self.index += 1
return {"score": self.index}
def on_trial_complete(self, trial_id, result=None, **kwargs):
self.returned_result.append(result)
searcher = TestSuggestion(0)
limiter = ConcurrencyLimiter(searcher, max_concurrent=2, batch=True)
assert limiter.suggest("test_1")["score"] == 1
assert limiter.suggest("test_2")["score"] == 2
assert limiter.suggest("test_3") is None
limiter.on_trial_complete("test_1", {"result": 3})
assert limiter.suggest("test_3") is None
limiter.on_trial_complete("test_2", {"result": 3})
assert limiter.suggest("test_3") is not None
def testSetMaxConcurrency(self):
"""Test whether ``set_max_concurrency`` is called by the
``ConcurrencyLimiter`` and works correctly.
"""
class TestSuggestion(Searcher):
def __init__(self, index):
self.index = index
self.returned_result = []
self._max_concurrent = 1
super().__init__(metric="result", mode="max")
def suggest(self, trial_id):
self.index += 1
return {"score": self.index}
def on_trial_complete(self, trial_id, result=None, **kwargs):
self.returned_result.append(result)
def set_max_concurrency(self, max_concurrent: int) -> bool:
self._max_concurrent = max_concurrent
return True
searcher = TestSuggestion(0)
limiter_max_concurrent = 2
limiter = ConcurrencyLimiter(searcher,
max_concurrent=limiter_max_concurrent,
batch=True)
assert limiter.searcher._max_concurrent == limiter_max_concurrent
# Since set_max_concurrency returns True, ConcurrencyLimiter should not
# be limiting concurrency itself
assert not limiter._limit_concurrency
assert limiter.suggest("test_1")["score"] == 1
assert limiter.suggest("test_2")["score"] == 2
assert limiter.suggest("test_3")["score"] == 3
def testBatchLimiterInfiniteLoop(self):
"""Check whether an infinite loop when less than max_concurrent trials
are suggested with batch mode is avoided.
"""
class TestSuggestion(Searcher):
def __init__(self, index, max_suggestions=10):
self.index = index
self.max_suggestions = max_suggestions
self.returned_result = []
super().__init__(metric="result", mode="max")
def suggest(self, trial_id):
self.index += 1
if self.index > self.max_suggestions:
return None
return {"score": self.index}
def on_trial_complete(self, trial_id, result=None, **kwargs):
self.returned_result.append(result)
self.index = 0
searcher = TestSuggestion(0, 2)
limiter = ConcurrencyLimiter(searcher, max_concurrent=5, batch=True)
limiter.suggest("test_1")
limiter.suggest("test_2")
limiter.suggest("test_3") # TestSuggestion return None
limiter.on_trial_complete("test_1", {"result": 3})
limiter.on_trial_complete("test_2", {"result": 3})
assert limiter.searcher.returned_result
searcher = TestSuggestion(0, 10)
limiter = ConcurrencyLimiter(searcher, max_concurrent=5, batch=True)
limiter.suggest("test_1")
limiter.suggest("test_2")
limiter.suggest("test_3")
limiter.on_trial_complete("test_1", {"result": 3})
limiter.on_trial_complete("test_2", {"result": 3})
assert not limiter.searcher.returned_result